From bd66813f978bdf4741e4830a00f6225f4ea4f700 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 00:28:54 -0700 Subject: [PATCH 01/20] Emit --interval_padding only when it is set `BaseDriver.interval_padding` now defaults to `None` instead of `0`, so driver commands carry `--interval_padding` only when a caller passes a value. The dnascope pipeline's attribute follows the argparse option, which has no default. The hybrid pipeline's DNAModelApply no longer passes `--interval`, matching every other model-apply invocation. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 2 +- sentieon_cli/dnascope_hybrid.py | 1 - sentieon_cli/driver.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 2a3d837..2885492 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -235,7 +235,7 @@ def __init__(self) -> None: self.model_bundle: Optional[pathlib.Path] = None self.dbsnp: Optional[pathlib.Path] = None self.bed: Optional[pathlib.Path] = None - self.interval_padding = 0 + self.interval_padding: Optional[int] = None self.pcr_free = False self.gvcf = False self.duplicate_marking = "markdup" diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 45a8cbc..dfb20e6 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -1227,7 +1227,6 @@ def call_variants( driver = Driver( reference=self.reference, thread_count=self.cores, - interval=self.bed, ) driver.add_algo( DNAModelApply( diff --git a/sentieon_cli/driver.py b/sentieon_cli/driver.py index 4574df8..e442a89 100644 --- a/sentieon_cli/driver.py +++ b/sentieon_cli/driver.py @@ -575,7 +575,7 @@ def __init__( reference: Optional[pathlib.Path] = None, thread_count: Optional[int] = None, interval: Optional[Union[pathlib.Path, str]] = None, - interval_padding: int = 0, + interval_padding: Optional[int] = None, read_filter: Optional[List[str]] = None, replace_rg: Optional[List[List[str]]] = None, input: Optional[List[pathlib.Path]] = None, From 6f7d77764653adc13a1cf326d788efd1b0becde5 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 00:35:47 -0700 Subject: [PATCH 02/20] Add the stage building blocks Introduce `sentieon_cli/stages/`, a layer of reusable pipeline stages that build their jobs and insert them into the DAG with the right edges. `stages/base.py` defines `StageContext` (run-wide settings), `StageResult` (the inserted jobs and the terminal jobs downstream stages depend on), the `Stage` base class with its `add_to` contract, and the `driver_job` / `rm_job` helpers for the two most common job shapes. `BasePipeline` gains `stage_context()` and `required()`, and `util` gains `require_versions()` / `versions_available()` for the version check idioms. Nothing uses them yet; the pipelines migrate stage by stage. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/pipeline.py | 46 +++- sentieon_cli/stages/__init__.py | 1 + sentieon_cli/stages/base.py | 127 +++++++++++ sentieon_cli/util.py | 42 +++- tests/unit/test_stage_base.py | 364 ++++++++++++++++++++++++++++++++ 5 files changed, 578 insertions(+), 2 deletions(-) create mode 100644 sentieon_cli/stages/__init__.py create mode 100644 sentieon_cli/stages/base.py create mode 100644 tests/unit/test_stage_base.py diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index f6d3cd6..a2732a4 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -11,7 +11,7 @@ import shutil import sys import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, TypeVar import packaging.version @@ -30,6 +30,7 @@ detect_reference_build, par_bed_for_build, ) +from .stages.base import StageContext from .util import ( SampleSex, __version__, @@ -45,6 +46,8 @@ BWA_INDEX_SUFFIXES = (".amb", ".ann", ".bwt", ".pac", ".sa") +T = TypeVar("T") + class BasePipeline(ABC): """A pipeline base class""" @@ -246,6 +249,47 @@ def check_execution( f"Ready jobs: {dag.ready_jobs}\n" ) + def required(self, value: Optional[T], name: str) -> T: + """Return `value`, ending the run when it was not supplied. + + Narrows an optional pipeline attribute to its value, so callers do + not have to repeat the "missing argument" check. + """ + if value is None: + self.logger.error("%s is required", name) + sys.exit(2) + return value + + def stage_context(self) -> StageContext: + """The run-wide settings this pipeline's stages need. + + Built fresh on every call, so it always reflects the pipeline's + current attributes. Only available once `main` has created the + run's temporary directory. + """ + if self.reference is None: + self.logger.error("reference is required") + sys.exit(2) + if self.output_vcf is None: + self.logger.error("output_vcf is required") + sys.exit(2) + tmp_dir: Optional[pathlib.Path] = getattr(self, "tmp_dir", None) + if tmp_dir is None: + self.logger.error( + "The temporary directory has not been created yet; " + "`stage_context` is only available after `main` sets " + "`tmp_dir`" + ) + sys.exit(2) + return StageContext( + reference=self.reference, + output_vcf=self.output_vcf, + tmp_dir=tmp_dir, + cores=self.cores, + dry_run=self.dry_run, + skip_version_check=self.skip_version_check, + ) + @abstractmethod def validate(self) -> None: pass diff --git a/sentieon_cli/stages/__init__.py b/sentieon_cli/stages/__init__.py new file mode 100644 index 0000000..e74d5da --- /dev/null +++ b/sentieon_cli/stages/__init__.py @@ -0,0 +1 @@ +"""Reusable pipeline stages""" diff --git a/sentieon_cli/stages/base.py b/sentieon_cli/stages/base.py new file mode 100644 index 0000000..ecee86f --- /dev/null +++ b/sentieon_cli/stages/base.py @@ -0,0 +1,127 @@ +""" +Building blocks for pipeline stages + +A *stage* is a reusable piece of a pipeline: it knows how to build the jobs +for one operation (dedup, metrics, small-variant calling, ...) and how to +wire them into a :class:`~sentieon_cli.dag.DAG`. Stages are configured with +plain dataclass fields plus a :class:`StageContext` of run-wide settings. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional, Sequence, Set, Union + +from ..dag import DAG +from ..driver import BaseAlgo, Driver +from ..job import Job +from ..shell_pipeline import Command, Pipeline + + +@dataclass(frozen=True) +class StageContext: + """Run-wide settings every stage needs.""" + + reference: pathlib.Path + output_vcf: pathlib.Path + tmp_dir: pathlib.Path + cores: int + dry_run: bool + skip_version_check: bool + + +@dataclass(kw_only=True) +class StageResult: + """The jobs a stage added to the DAG. + + * ``jobs`` -- every job the stage inserted, in insertion order. It + exists for the occasional edge that has to attach to a job in the + middle of a stage rather than to its output. + * ``terminal`` -- the stage's sink job(s), the ones that carry its + declared outputs. Downstream stages depend on these. + + Subclasses add named fields for the files the stage produces. + """ + + jobs: List[Job] + terminal: Set[Job] + + +@dataclass(kw_only=True) +class Stage(ABC): + """A reusable group of jobs that inserts itself into a DAG. + + Subclasses are dataclasses (``@dataclass(kw_only=True)``) that add + their own configuration fields and return a :class:`StageResult` + subclass naming the outputs they produce. + + The :meth:`add_to` contract: + + * ``upstream`` is any iterable of jobs already in the DAG; it is + consumed once into a set. + * Every *entry* job -- an in-stage job with no in-stage dependency -- + is inserted with that set as its dependencies. Edges between the + stage's own jobs are added on top of that. + * Insertion is topological (a job is added only after the jobs it + depends on) and deterministic: the same stage and DAG produce the + same jobs, in the same order, every time. + * The jobs are created inside ``add_to``; constructing a stage has no + side effects and assigns no job ids. + * ``add_to`` may be called only once per stage instance. Job identity + is the shell pipeline, so a second call raises in + :meth:`~sentieon_cli.dag.DAG.add_job`. + """ + + ctx: StageContext + + @abstractmethod + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> StageResult: + """Add this stage's jobs to ``dag``, downstream of ``upstream``""" + + +def driver_job( + ctx: StageContext, + algos: Sequence[BaseAlgo], + *, + name: str, + task_name: str, + threads: Optional[int] = None, + inputs: Optional[List[pathlib.Path]] = None, + interval: Optional[Union[pathlib.Path, str]] = None, + interval_padding: Optional[int] = None, + read_filter: Optional[List[str]] = None, + replace_rg: Optional[List[List[str]]] = None, +) -> Job: + """A job running `sentieon driver` with ``algos``. + + ``threads`` defaults to the run's core count; the driver arguments are + passed through untouched so the command matches what the pipelines + build by hand. + """ + driver = Driver( + reference=ctx.reference, + thread_count=ctx.cores, + interval=interval, + interval_padding=interval_padding, + read_filter=read_filter, + replace_rg=replace_rg, + input=inputs, + ) + for algo in algos: + driver.add_algo(algo) + return Job( + Pipeline(Command(*driver.build_cmd())), + name, + ctx.cores if threads is None else threads, + task_name=task_name, + ) + + +def rm_job(paths: Iterable[Union[pathlib.Path, str]], name: str) -> Job: + """A job removing intermediate files, tolerating missing ones""" + return Job( + Pipeline(Command("rm", *[str(p) for p in paths], fail_ok=True)), + name, + 0, + task_name="cleanup", + ) diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 2499def..67fa1c4 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -12,8 +12,9 @@ import shlex import shutil import subprocess as sp +import sys import tempfile -from typing import Callable, Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Mapping, Optional, Tuple import packaging.version @@ -137,6 +138,45 @@ def check_version( return True +def require_versions( + min_versions: Mapping[str, Optional[packaging.version.Version]], + *, + skip: bool = False, +) -> None: + """Exit unless every executable meets its minimum version. + + `check_version` has already logged the reason, so the exit is silent. + Pass `skip=True` (the pipelines' `--skip_version_check`) to do nothing. + + A `Mapping` rather than a `Dict`, so the pipelines' module-level + `*_MIN_VERSIONS` constants -- some of which mypy infers as + `Dict[str, Version]`, with no `None` entry -- are accepted. + """ + if skip: + return + for cmd, min_version in min_versions.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + +def versions_available( + min_versions: Mapping[str, Optional[packaging.version.Version]], + *, + skip: bool = False, +) -> bool: + """Whether every executable meets its minimum version. + + For optional tools, where a missing or outdated executable skips a step + rather than ending the run. `skip=True` reports them as available. + """ + if skip: + return True + return all( + check_version(cmd, min_version) + for cmd, min_version in min_versions.items() + ) + + def path_arg( exists: Optional[bool] = None, is_dir: Optional[bool] = None, diff --git a/tests/unit/test_stage_base.py b/tests/unit/test_stage_base.py new file mode 100644 index 0000000..46468ed --- /dev/null +++ b/tests/unit/test_stage_base.py @@ -0,0 +1,364 @@ +""" +Unit tests for the pipeline stage building blocks +""" + +import argparse +import dataclasses +import pathlib +from typing import Iterable +from unittest.mock import patch + +import packaging.version +import pytest + +from sentieon_cli import util +from sentieon_cli.dag import DAG +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.driver import LocusCollector +from sentieon_cli.job import Job +from sentieon_cli.stages.base import ( + Stage, + StageContext, + StageResult, + driver_job, + rm_job, +) +from sentieon_cli.util import require_versions, versions_available + + +def create_mock_args(loglevel: str = "WARNING") -> argparse.Namespace: + """The argparse namespace `setup_logging` needs""" + return argparse.Namespace(loglevel=loglevel) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +@dataclasses.dataclass(kw_only=True) +class ToyStage(Stage): + """Two jobs: `b` depends on `a`, `a` is the only entry job""" + + tag: str = "toy" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> StageResult: + deps = set(upstream) + job_a = rm_job( + [self.ctx.tmp_dir / f"{self.tag}-a"], f"{self.tag}-first" + ) + job_b = rm_job( + [self.ctx.tmp_dir / f"{self.tag}-b"], f"{self.tag}-second" + ) + dag.add_job(job_a, deps) + dag.add_job(job_b, {job_a}) + return StageResult(jobs=[job_a, job_b], terminal={job_b}) + + +class TestStageContext: + """The run-wide settings object""" + + def test_fields(self, tmp_path): + ctx = make_ctx(tmp_path) + assert ctx.reference == tmp_path / "ref.fa" + assert ctx.output_vcf == tmp_path / "output.vcf.gz" + assert ctx.tmp_dir == tmp_path + assert ctx.cores == 4 + assert ctx.dry_run is True + assert ctx.skip_version_check is True + + def test_frozen(self, tmp_path): + ctx = make_ctx(tmp_path) + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.cores = 8 # type: ignore[misc] + + +class TestStage: + """The `add_to` contract""" + + def test_upstream_lands_on_entry_jobs_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag, [upstream]) + + job_a, job_b = result.jobs + assert result.jobs == [job_a, job_b] + assert result.terminal == {job_b} + # The entry job gets the upstream dependencies... + assert dag.waiting_jobs[job_a] == {upstream} + # ...and the internal edge is added on top of them. + assert dag.waiting_jobs[job_b] == {job_a} + + def test_no_upstream_makes_the_entry_job_ready(self, tmp_path): + dag = DAG() + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag) + + job_a, job_b = result.jobs + assert job_a in dag.ready_jobs + assert dag.waiting_jobs[job_b] == {job_a} + + def test_upstream_iterable_consumed_once(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag, iter([upstream])) + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + + def test_add_to_twice_raises(self, tmp_path): + dag = DAG() + stage = ToyStage(ctx=make_ctx(tmp_path)) + stage.add_to(dag) + # Job identity is the shell pipeline, so the same jobs collide + with pytest.raises(ValueError): + stage.add_to(dag) + + def test_constructing_a_stage_adds_nothing(self, tmp_path): + dag = DAG() + ToyStage(ctx=make_ctx(tmp_path)) + assert not dag.waiting_jobs + assert not dag.ready_jobs + + +class TestDriverJob: + """The `sentieon driver` job helper""" + + def test_basic_command(self, tmp_path): + ctx = make_ctx(tmp_path, cores=8) + job = driver_job( + ctx, + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + inputs=[tmp_path / "sample.bam"], + ) + expected = ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 8 " + f"--algo LocusCollector {tmp_path}/score.txt.gz" + ) + assert str(job.shell) == expected + assert job.name == "locuscollector" + assert job.task_name == "dedup" + assert job.threads == 8 + + def test_no_interval_padding_by_default(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + ) + assert "--interval_padding" not in str(job.shell) + assert "--interval" not in str(job.shell) + assert "--read_filter" not in str(job.shell) + assert "--input" not in str(job.shell) + + def test_interval_and_padding(self, tmp_path): + bed = tmp_path / "regions.bed" + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + interval=bed, + interval_padding=5, + ) + assert f"--interval {bed} --interval_padding 5" in str(job.shell) + + def test_interval_as_a_string(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="readwriter", + task_name="extract", + interval="chr6:28510020-33480577", + ) + assert "--interval chr6:28510020-33480577" in str(job.shell) + + def test_threads_zero(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + threads=0, + ) + # `threads` is the scheduling budget only; the driver still gets + # the run's core count. + assert job.threads == 0 + assert "--thread_count 4" in str(job.shell) + + def test_read_filter_repeats_the_flag(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + read_filter=["QualCalFilter,table=t", "IndelLeftAlign,rgid=rg"], + ) + assert ( + "--read_filter QualCalFilter,table=t " + "--read_filter IndelLeftAlign,rgid=rg" in str(job.shell) + ) + + def test_replace_rg_interleaves_with_input(self, tmp_path): + bam1 = tmp_path / "one.bam" + bam2 = tmp_path / "two.bam" + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + inputs=[bam1, bam2], + replace_rg=[["rg1=@RG\\tID:rg1"], ["rg2=@RG\\tID:rg2"]], + ) + shell = str(job.shell) + assert ( + f"--replace_rg 'rg1=@RG\\tID:rg1' --input {bam1} " + f"--replace_rg 'rg2=@RG\\tID:rg2' --input {bam2}" in shell + ) + + def test_algos_are_added_in_order(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [ + LocusCollector(tmp_path / "first.txt.gz"), + LocusCollector(tmp_path / "second.txt.gz"), + ], + name="locuscollector", + task_name="dedup", + ) + shell = str(job.shell) + assert shell.index("first.txt.gz") < shell.index("second.txt.gz") + + +class TestRmJob: + """The cleanup job helper""" + + def test_command_and_metadata(self, tmp_path): + vcf = tmp_path / "tmp.vcf.gz" + job = rm_job([vcf, str(vcf) + ".tbi"], "rm-tmp-vcf") + assert str(job.shell) == f"rm {vcf} {vcf}.tbi" + assert job.name == "rm-tmp-vcf" + assert job.task_name == "cleanup" + assert job.threads == 0 + assert job.shell.nodes[0].fail_ok is True + + +class TestRequireVersions: + """`require_versions` and `versions_available`""" + + min_versions = {"sentieon driver": packaging.version.Version("202308")} + + def test_require_skips(self): + with patch.object(util, "check_version") as check: + require_versions(self.min_versions, skip=True) + check.assert_not_called() + + def test_require_passes(self): + with patch.object(util, "check_version", return_value=True) as check: + require_versions(self.min_versions) + check.assert_called_once_with( + "sentieon driver", packaging.version.Version("202308") + ) + + def test_require_exits_on_failure(self): + with patch.object(util, "check_version", return_value=False): + with pytest.raises(SystemExit) as excinfo: + require_versions(self.min_versions) + assert excinfo.value.code == 2 + + def test_require_exits_on_the_first_failure(self): + versions = { + "samtools": packaging.version.Version("1.16"), + "bcftools": packaging.version.Version("1.10"), + } + with patch.object(util, "check_version", return_value=False) as check: + with pytest.raises(SystemExit): + require_versions(versions) + assert check.call_count == 1 + + def test_available_skips(self): + with patch.object(util, "check_version") as check: + assert versions_available(self.min_versions, skip=True) is True + check.assert_not_called() + + def test_available_true(self): + with patch.object(util, "check_version", return_value=True): + assert versions_available(self.min_versions) is True + + def test_available_false(self): + with patch.object(util, "check_version", return_value=False): + assert versions_available(self.min_versions) is False + + +class TestPipelineStageContext: + """`BasePipeline.stage_context` and `BasePipeline.required`""" + + def make_pipeline(self, tmp_path) -> DNAscopePipeline: + pipeline = DNAscopePipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.reference = tmp_path / "ref.fa" + pipeline.output_vcf = tmp_path / "output.vcf.gz" + pipeline.cores = 3 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = tmp_path + return pipeline + + def test_stage_context(self, tmp_path): + ctx = self.make_pipeline(tmp_path).stage_context() + assert ctx == StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=3, + dry_run=True, + skip_version_check=True, + ) + + def test_missing_reference(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + pipeline.reference = None + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_missing_output_vcf(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + pipeline.output_vcf = None + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_missing_tmp_dir(self, tmp_path): + pipeline = DNAscopePipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.reference = tmp_path / "ref.fa" + pipeline.output_vcf = tmp_path / "output.vcf.gz" + assert not hasattr(pipeline, "tmp_dir") + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_required_returns_the_value(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + assert pipeline.required(pipeline.output_vcf, "output_vcf") == ( + tmp_path / "output.vcf.gz" + ) + + def test_required_exits_when_missing(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + with pytest.raises(SystemExit) as excinfo: + pipeline.required(None, "model_bundle") + assert excinfo.value.code == 2 From 54913fd174ab6d2fea09f617198446c3f1839b49 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 00:45:39 -0700 Subject: [PATCH 03/20] Move CNV calling and ploidy estimation into stages `CNVscopeStage` (CNVscope followed by CNVModelApply) replaces the module-level `call_cnvs` used by dnascope-hybrid and the two private builders in dnascope-pangenome; `PloidyStage` replaces `BasePipeline.build_ploidy_job` and derives the `_ploidy.json` path itself. `CNV_MIN_VERSIONS` moves to `stages/cnv.py`. The duplicate version check that ran inside `call_cnvs` is gone; `validate_cnv` already performs it before the first DAG runs. The hybrid pipeline's CNV jobs are now named `cnvscope` and `cnv-model-apply`, matching the pangenome pipeline. Commands and dependency edges are unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 80 +------ sentieon_cli/dnascope_hybrid.py | 46 ++-- sentieon_cli/pipeline.py | 30 --- sentieon_cli/sentieon_pangenome.py | 102 ++------- sentieon_cli/stages/cnv.py | 95 ++++++++ sentieon_cli/stages/ploidy.py | 67 ++++++ tests/integration/test_pipeline_execution.py | 2 +- tests/unit/test_cnv_sex.py | 36 +-- tests/unit/test_dnascope_hybrid_cnv.py | 18 +- tests/unit/test_stage_cnv.py | 217 +++++++++++++++++++ tests/unit/test_stage_ploidy.py | 130 +++++++++++ 11 files changed, 589 insertions(+), 234 deletions(-) create mode 100644 sentieon_cli/stages/cnv.py create mode 100644 sentieon_cli/stages/ploidy.py create mode 100644 tests/unit/test_stage_cnv.py create mode 100644 tests/unit/test_stage_ploidy.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 2885492..f006666 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -5,12 +5,11 @@ import argparse import copy import itertools -import multiprocessing as mp import os import pathlib import shutil import sys -from typing import Any, List, Optional, Set, Tuple +from typing import List, Optional, Set, Tuple import packaging.version @@ -21,8 +20,6 @@ from .driver import ( AlignmentStat, BaseDistributionByCycle, - CNVscope, - CNVModelApply, CoverageMetrics, Dedup, DNAModelApply, @@ -42,9 +39,7 @@ from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline from .util import ( - SampleSex, check_version, - cnvscope_sex_args, library_preloaded, parse_rg_line, path_arg, @@ -65,11 +60,6 @@ "sentieon driver": packaging.version.Version("202308"), } -CNV_MIN_VERSIONS = { - # 202503.04 adds the CNVscope `--sex` and `--par` arguments - "sentieon driver": packaging.version.Version("202503.04"), -} - class DNAscopePipeline(BasePipeline): """The DNAscope pipeline""" @@ -986,71 +976,3 @@ def sr_call_variants( svsolver_job, sv_rm_job, ) - - -def call_cnvs( - tmp_dir: pathlib.Path, - output_vcf: pathlib.Path, - reference: pathlib.Path, - sample_input: List[pathlib.Path], - model_bundle: pathlib.Path, - bed: Optional[pathlib.Path] = None, - cores: int = mp.cpu_count(), - skip_version_check: bool = False, - replace_rg: Optional[List[List[str]]] = None, - sample_sex: Optional[SampleSex] = None, - par_bed: Optional[pathlib.Path] = None, - **_kwargs: Any, -) -> Tuple[Job, Job]: - """ - Call CNVs using CNVscope - """ - if not skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - sex, par = cnvscope_sex_args(sample_sex, par_bed) - cnvscope_vcf = tmp_dir.joinpath("cnvscope.vcf.gz") - cnv_vcf = pathlib.Path(str(output_vcf).replace(".vcf.gz", ".cnv.vcf.gz")) - driver = Driver( - reference=reference, - thread_count=cores, - replace_rg=replace_rg, - input=sample_input, - interval=bed, - ) - driver.add_algo( - CNVscope( - cnvscope_vcf, - model_bundle.joinpath("cnv.model"), - sex=sex, - par=par, - ) - ) - cnvscope_job = Job( - Pipeline(Command(*driver.build_cmd())), - "CNVscope", - cores, - task_name="cnv", - ) - - driver = Driver( - reference=reference, - thread_count=cores, - ) - driver.add_algo( - CNVModelApply( - cnv_vcf, - model_bundle.joinpath("cnv.model"), - vcf=cnvscope_vcf, - ) - ) - cnvmodelapply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "CNVModelApply", - cores, - task_name="cnv", - ) - - return (cnvscope_job, cnvmodelapply_job) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index dfb20e6..c29101e 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -26,11 +26,13 @@ HybridStage2, HybridStage3, ) -from .dnascope import call_cnvs, CNV_MIN_VERSIONS, DNAscopePipeline +from .dnascope import DNAscopePipeline from .dnascope_longread import DNAscopeLRPipeline from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline +from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.ploidy import PloidyStage from .util import ( __version__, check_version, @@ -44,7 +46,6 @@ from .shard import ( determine_shards_from_fai, parse_fai, - ploidy_contigs_for_build, vcf_contigs, ) from .transfer import build_transfer_jobs @@ -579,6 +580,7 @@ def build_dag(self) -> DAG: if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) + ctx = self.stage_context() rg_info = RgInfo( self.lr_aln_readgroups, @@ -662,15 +664,12 @@ def build_dag(self) -> DAG: # Estimate the sample ploidy and sex. The JSON output is always # written; `--sample_sex` takes precedence for the sex used by # sex-aware CNV calling. - self.ploidy_json = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_ploidy.json") - ) - ploidy_job = self.build_ploidy_job( - self.ploidy_json, - sr_aln, - ploidy_contigs_for_build(self.reference_build), - ) - dag.add_job(ploidy_job, sr_preprocessing_jobs) + ploidy_result = PloidyStage( + ctx=ctx, + inputs=sr_aln, + reference_build=self.reference_build, + ).add_to(dag, sr_preprocessing_jobs) + self.ploidy_json = ploidy_result.ploidy_json if not self.skip_cnv: # CNV calling is sex-aware and runs in the second DAG @@ -758,26 +757,25 @@ def build_second_dag(self) -> Optional[DAG]: self.logger.error("model_bundle is required") sys.exit(2) assert self.ploidy_json is not None + ctx = self.stage_context() self.get_sex(self.ploidy_json) self.logger.info("Building the DNAscope Hybrid CNV DAG") dag = DAG() - cnvscope_job, cnvmodelapply_job = call_cnvs( - self.tmp_dir, - self.output_vcf, - self.reference, - self.cnv_sr_aln, - self.model_bundle, - self.bed, - self.cores, - self.skip_version_check, - replace_rg=self.cnv_replace_rg, + CNVscopeStage( + ctx=ctx, + inputs=self.cnv_sr_aln, + model=self.model_bundle.joinpath("cnv.model"), + cnvscope_vcf=ctx.tmp_dir.joinpath("cnvscope.vcf.gz"), + cnv_vcf=pathlib.Path( + str(ctx.output_vcf).replace(".vcf.gz", ".cnv.vcf.gz") + ), sample_sex=self.sample_sex, par_bed=self.cnv_par_bed, - ) - dag.add_job(cnvscope_job) - dag.add_job(cnvmodelapply_job, {cnvscope_job}) + interval=self.bed, + replace_rg=self.cnv_replace_rg, + ).add_to(dag) return dag def call_variants( diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index a2732a4..3638390 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -15,8 +15,6 @@ import packaging.version -from importlib.resources import files - from . import command_strings as cmds from .dag import DAG from .exceptions import DagExecutionError @@ -26,7 +24,6 @@ from .run_logs import RunLogs from .scheduler import ThreadScheduler from .shard import ( - PloidyContigs, detect_reference_build, par_bed_for_build, ) @@ -356,33 +353,6 @@ def build_second_dag(self) -> Optional[DAG]: """ return None - def build_ploidy_job( - self, - ploidy_json: pathlib.Path, - deduped_bam: List[pathlib.Path], - ploidy_contigs: Optional[PloidyContigs] = None, - ) -> Job: - """Estimate sample ploidy and sex""" - estimate_ploidy = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) - ).resolve() - ploidy_contigs = ploidy_contigs or PloidyContigs() - ploidy_job = Job( - cmds.cmd_estimate_ploidy( - ploidy_json, - deduped_bam, - estimate_ploidy, - contigs=ploidy_contigs.contigs, - autosomes=ploidy_contigs.autosomes, - x_contig=ploidy_contigs.x_contig, - y_contig=ploidy_contigs.y_contig, - ), - "estimate-ploidy", - 0, - task_name="ploidy", - ) - return ploidy_job - def get_sex(self, ploidy_json: pathlib.Path) -> None: """Retrieve the sample sex""" if self.sample_sex is not None: diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 8099897..2002bc3 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -19,10 +19,7 @@ from .archive import ar_load from .base_pangenome import BasePangenome from .dag import DAG -from .dnascope import CNV_MIN_VERSIONS from .driver import ( - CNVModelApply, - CNVscope, DNAModelApply, DNAscope, Driver, @@ -33,6 +30,8 @@ from .job import Job from .logging import get_logger from .shell_pipeline import Command, Pipeline +from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.ploidy import PloidyStage from .util import ( SampleSex, __version__, @@ -49,7 +48,6 @@ GRCH38_CONTIGS, determine_shards_from_fai, parse_fai, - ploidy_contigs_for_build, vcf_contigs, ) from .transfer import build_transfer_jobs @@ -679,6 +677,8 @@ def build_first_dag(self) -> DAG: self.logger.error("output_vcf is required") sys.exit(2) + ctx = self.stage_context() + self.logger.info("Building the Sentieon pangenome DAG") dag = DAG() @@ -690,9 +690,6 @@ def build_first_dag(self) -> DAG: out_mm2_aln = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") ) - self.ploidy_json = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_ploidy.json") - ) out_gvcf = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) @@ -853,12 +850,12 @@ def build_first_dag(self) -> DAG: # Estimate the sample ploidy and sex. The JSON output is always # written; `--sample_sex` takes precedence for the sex used by # the sex-aware callers. - ploidy_job = self.build_ploidy_job( - self.ploidy_json, - [cnv_input_bams[0]], - ploidy_contigs_for_build(self.reference_build), - ) - dag.add_job(ploidy_job, cnvscope_deps) + ploidy_result = PloidyStage( + ctx=ctx, + inputs=[cnv_input_bams[0]], + reference_build=self.reference_build, + ).add_to(dag, cnvscope_deps) + self.ploidy_json = ploidy_result.ploidy_json # T1K HLA/KIR calling if self.t1k_hla_seq and self.t1k_hla_coord: @@ -1431,6 +1428,7 @@ def _add_cnv_jobs( self.logger.error("output_vcf is required") sys.exit(2) + ctx = self.stage_context() cnvscope_vcf = self.tmp_dir.joinpath("sample-cnvscope.vcf.gz") cnv_apply_vcf = self.tmp_dir.joinpath("sample-cnv_model_apply.vcf.gz") indel2cnv_vcf = self.tmp_dir.joinpath("sample-sv_cnv.vcf.gz") @@ -1438,16 +1436,18 @@ def _add_cnv_jobs( str(self.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") ) - # CNVscope on BWA deduped BAM. The alignment and the PangenomeSV - # output were both written by the first DAG. - cnvscope_job = self._build_cnvscope_job(cnvscope_vcf, cnv_input_bams) - dag.add_job(cnvscope_job) - - # CNVModelApply on CNVscope output - cnv_model_apply_job = self._build_cnv_model_apply_job( - cnv_apply_vcf, cnvscope_vcf - ) - dag.add_job(cnv_model_apply_job, {cnvscope_job}) + # CNVscope on BWA deduped BAM, then CNVModelApply. The alignment + # and the PangenomeSV output were both written by the first DAG. + cnv_result = CNVscopeStage( + ctx=ctx, + inputs=cnv_input_bams, + model=self.model_bundle.joinpath("cnv.model"), + cnvscope_vcf=cnvscope_vcf, + cnv_vcf=cnv_apply_vcf, + sample_sex=self.sample_sex, + par_bed=self.cnv_par_bed, + interval=self.bed, + ).add_to(dag) # Convert PangenomeSV output to CNVs indel2cnv_job = self._build_indel2cnv_job(indel2cnv_vcf, sv_vcf) @@ -1457,61 +1457,7 @@ def _add_cnv_jobs( combine_job = self._build_combine_cnv_job( cnv_vcf, cnv_apply_vcf, indel2cnv_vcf ) - dag.add_job(combine_job, {cnv_model_apply_job, indel2cnv_job}) - - def _build_cnvscope_job( - self, - output_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - ) -> Job: - """Build a CNVscope job""" - assert self.model_bundle is not None - sex, par = self.cnv_sex_args() - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - ) - driver.add_algo( - CNVscope( - output=output_vcf, - model=self.model_bundle.joinpath("cnv.model"), - sex=sex, - par=par, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "cnvscope", - self.cores, - task_name="cnv", - ) - - def _build_cnv_model_apply_job( - self, - output_vcf: pathlib.Path, - input_vcf: pathlib.Path, - ) -> Job: - """Build a CNVModelApply job""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - CNVModelApply( - output=output_vcf, - model=self.model_bundle.joinpath("cnv.model"), - vcf=input_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "cnv-model-apply", - self.cores, - task_name="cnv", - ) + dag.add_job(combine_job, {cnv_result.apply_job, indel2cnv_job}) def _build_indel2cnv_job( self, diff --git a/sentieon_cli/stages/cnv.py b/sentieon_cli/stages/cnv.py new file mode 100644 index 0000000..91904ad --- /dev/null +++ b/sentieon_cli/stages/cnv.py @@ -0,0 +1,95 @@ +""" +CNV calling with CNVscope +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +import packaging.version + +from ..dag import DAG +from ..driver import CNVModelApply, CNVscope +from ..job import Job +from ..util import SampleSex, cnvscope_sex_args +from .base import Stage, StageResult, driver_job + +CNV_MIN_VERSIONS = { + # 202503.04 adds the CNVscope `--sex` and `--par` arguments + "sentieon driver": packaging.version.Version("202503.04"), +} + + +@dataclass(kw_only=True) +class CNVResult(StageResult): + """The jobs and output of `CNVscopeStage`""" + + cnvscope_job: Job + apply_job: Job + cnv_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class CNVscopeStage(Stage): + """Call CNVs with CNVscope, then filter them with CNVModelApply. + + The sample sex has to be known before the jobs are built, so callers + run this stage after ploidy estimation. Version checks stay with the + pipelines, which run them during validation. + """ + + inputs: List[pathlib.Path] + model: pathlib.Path + cnvscope_vcf: pathlib.Path + cnv_vcf: pathlib.Path + sample_sex: Optional[SampleSex] = None + par_bed: Optional[pathlib.Path] = None + interval: Optional[pathlib.Path] = None + replace_rg: Optional[List[List[str]]] = None + name: str = "cnvscope" + apply_name: str = "cnv-model-apply" + task_name: str = "cnv" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> CNVResult: + deps = set(upstream) + sex, par = cnvscope_sex_args(self.sample_sex, self.par_bed) + + cnvscope = driver_job( + self.ctx, + [ + CNVscope( + self.cnvscope_vcf, + self.model, + sex=sex, + par=par, + ) + ], + inputs=self.inputs, + interval=self.interval, + replace_rg=self.replace_rg, + name=self.name, + task_name=self.task_name, + ) + apply_job = driver_job( + self.ctx, + [ + CNVModelApply( + self.cnv_vcf, + self.model, + vcf=self.cnvscope_vcf, + ) + ], + name=self.apply_name, + task_name=self.task_name, + ) + + dag.add_job(cnvscope, deps) + dag.add_job(apply_job, {cnvscope}) + + return CNVResult( + jobs=[cnvscope, apply_job], + terminal={apply_job}, + cnvscope_job=cnvscope, + apply_job=apply_job, + cnv_vcf=self.cnv_vcf, + ) diff --git a/sentieon_cli/stages/ploidy.py b/sentieon_cli/stages/ploidy.py new file mode 100644 index 0000000..49e87da --- /dev/null +++ b/sentieon_cli/stages/ploidy.py @@ -0,0 +1,67 @@ +""" +Sample ploidy and sex estimation +""" + +from dataclasses import dataclass +from importlib.resources import files +import pathlib +from typing import Iterable, List, Optional + +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from ..shard import ploidy_contigs_for_build +from .base import Stage, StageResult + + +@dataclass(kw_only=True) +class PloidyResult(StageResult): + """The job and output of `PloidyStage`""" + + job: Job + ploidy_json: pathlib.Path + + +@dataclass(kw_only=True) +class PloidyStage(Stage): + """Estimate the sample ploidy and sex from an alignment. + + The JSON report is written next to the output VCF and is read after + the DAG has run, by the pipelines that call variants with a + sex-aware caller. + """ + + inputs: List[pathlib.Path] + reference_build: Optional[str] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> PloidyResult: + deps = set(upstream) + ploidy_json = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", "_ploidy.json") + ) + estimate_ploidy = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) + ).resolve() + ploidy_contigs = ploidy_contigs_for_build(self.reference_build) + job = Job( + cmds.cmd_estimate_ploidy( + ploidy_json, + self.inputs, + estimate_ploidy, + contigs=ploidy_contigs.contigs, + autosomes=ploidy_contigs.autosomes, + x_contig=ploidy_contigs.x_contig, + y_contig=ploidy_contigs.y_contig, + ), + "estimate-ploidy", + 0, + task_name="ploidy", + ) + dag.add_job(job, deps) + + return PloidyResult( + jobs=[job], + terminal={job}, + job=job, + ploidy_json=ploidy_json, + ) diff --git a/tests/integration/test_pipeline_execution.py b/tests/integration/test_pipeline_execution.py index eba087e..e7a25c1 100644 --- a/tests/integration/test_pipeline_execution.py +++ b/tests/integration/test_pipeline_execution.py @@ -160,7 +160,7 @@ def test_hybrid_dry_run_execution_rgsm(self): second_jobs = list(second_dag.waiting_jobs.keys()) + list(second_dag.ready_jobs.keys()) # should use replace_rg in CNV calling - cnv_job = [job for job in second_jobs if job.name == "CNVscope"][0] + cnv_job = [job for job in second_jobs if job.name == "cnvscope"][0] assert "--replace_rg" in cnv_job.shell.nodes[0].args # should use replace_rg in SV calling diff --git a/tests/unit/test_cnv_sex.py b/tests/unit/test_cnv_sex.py index fa256b0..914d5a2 100644 --- a/tests/unit/test_cnv_sex.py +++ b/tests/unit/test_cnv_sex.py @@ -12,10 +12,11 @@ import pytest from sentieon_cli import shard -from sentieon_cli.dnascope import call_cnvs from sentieon_cli.driver import CNVscope from sentieon_cli.pipeline import BasePipeline from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext +from sentieon_cli.stages.cnv import CNVscopeStage from sentieon_cli.shard import ( BUILD_SIGNATURES, detect_reference_build, @@ -335,22 +336,31 @@ def test_resolve_cnv_par_bed_skips_the_lookup_without_cnv_calling(messages): assert not any("PAR" in msg for msg in messages) -# `call_cnvs` passes the sample sex through to CNVscope +# `CNVscopeStage` passes the sample sex through to CNVscope def _cnvscope_cmd(**kwargs) -> str: - cnvscope_job, _ = call_cnvs( - pathlib.Path("/tmp/cnv"), - pathlib.Path("/out/sample.vcf.gz"), - pathlib.Path("/ref/reference.fa"), - [pathlib.Path("/in/sample.cram")], - pathlib.Path("/bundle/model.bundle"), - None, - mp.cpu_count(), - True, # skip_version_check - **kwargs, + tmp_dir = pathlib.Path("/tmp/cnv") + output_vcf = pathlib.Path("/out/sample.vcf.gz") + ctx = StageContext( + reference=pathlib.Path("/ref/reference.fa"), + output_vcf=output_vcf, + tmp_dir=tmp_dir, + cores=mp.cpu_count(), + dry_run=True, + skip_version_check=True, ) - return str(cnvscope_job.shell) + result = CNVscopeStage( + ctx=ctx, + inputs=[pathlib.Path("/in/sample.cram")], + model=pathlib.Path("/bundle/model.bundle/cnv.model"), + cnvscope_vcf=tmp_dir.joinpath("cnvscope.vcf.gz"), + cnv_vcf=pathlib.Path( + str(output_vcf).replace(".vcf.gz", ".cnv.vcf.gz") + ), + **kwargs, + ).add_to(DAG()) + return str(result.cnvscope_job.shell) def test_call_cnvs_passes_the_sex_of_a_male_sample(): diff --git a/tests/unit/test_dnascope_hybrid_cnv.py b/tests/unit/test_dnascope_hybrid_cnv.py index 20c4597..924a712 100644 --- a/tests/unit/test_dnascope_hybrid_cnv.py +++ b/tests/unit/test_dnascope_hybrid_cnv.py @@ -105,14 +105,14 @@ def test_cnv_jobs_run_in_the_second_dag(self): pipeline.sample_sex = SampleSex.FEMALE _dag, jobs = self.build_dag(pipeline) - assert "CNVscope" not in jobs - assert "CNVModelApply" not in jobs + assert "cnvscope" not in jobs + assert "cnv-model-apply" not in jobs second_dag, second_jobs = self.build_second_dag(pipeline) - assert "CNVscope" in second_jobs - assert "CNVModelApply" in second_jobs - assert second_dag.waiting_jobs[second_jobs["CNVModelApply"]] == { - second_jobs["CNVscope"] + assert "cnvscope" in second_jobs + assert "cnv-model-apply" in second_jobs + assert second_dag.waiting_jobs[second_jobs["cnv-model-apply"]] == { + second_jobs["cnvscope"] } def test_the_first_dag_estimates_the_sample_ploidy(self): @@ -164,7 +164,7 @@ def test_a_female_sample_is_called_without_a_par_bed(self): self.build_dag(pipeline) _dag, jobs = self.build_second_dag(pipeline) - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex F" in cmd_str assert "--par" not in cmd_str @@ -175,7 +175,7 @@ def test_a_male_sample_is_called_with_the_par_bed(self): self.build_dag(pipeline) _dag, jobs = self.build_second_dag(pipeline) - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex M" in cmd_str assert f"--par {self.mock_par_bed}" in cmd_str @@ -187,7 +187,7 @@ def test_a_dry_run_assumes_a_male_sample(self): _dag, jobs = self.build_second_dag(pipeline) assert pipeline.sample_sex is SampleSex.MALE - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex M" in cmd_str assert f"--par {self.mock_par_bed}" in cmd_str diff --git a/tests/unit/test_stage_cnv.py b/tests/unit/test_stage_cnv.py new file mode 100644 index 0000000..3fbf962 --- /dev/null +++ b/tests/unit/test_stage_cnv.py @@ -0,0 +1,217 @@ +""" +Unit tests for the CNVscope stage +""" + +import logging +import pathlib +from typing import List + +import packaging.version +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from sentieon_cli.util import SampleSex + +PACKAGE_LOGGER = "sentieon_cli" + + +class _RecordingHandler(logging.Handler): + """Collects formatted messages from the package logger.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages: List[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def messages(): + """`caplog` does not see the package logger, which does not propagate""" + logger = logging.getLogger(PACKAGE_LOGGER) + handler = _RecordingHandler() + logger.addHandler(handler) + try: + yield handler.messages + finally: + logger.removeHandler(handler) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> CNVscopeStage: + """A CNVscopeStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + model=tmp_path / "bundle" / "cnv.model", + cnvscope_vcf=tmp_path / "cnvscope.vcf.gz", + cnv_vcf=tmp_path / "output.cnv.vcf.gz", + ) + defaults.update(kwargs) + return CNVscopeStage(**defaults) # type: ignore[arg-type] + + +class TestMinVersions: + """The version constant moved here with the stage""" + + def test_cnvscope_sex_arguments_need_202503_04(self): + assert CNV_MIN_VERSIONS["sentieon driver"] == ( + packaging.version.Version("202503.04") + ) + + +class TestCommands: + """The `sentieon driver` commands the stage builds""" + + def test_cnvscope_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.cnvscope_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo CNVscope --model {tmp_path}/bundle/cnv.model " + f"{tmp_path}/cnvscope.vcf.gz" + ) + + def test_cnv_model_apply_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + # CNVModelApply reads the CNVscope VCF; it takes no alignment + assert str(result.apply_job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo CNVModelApply " + f"--model {tmp_path}/bundle/cnv.model " + f"--vcf {tmp_path}/cnvscope.vcf.gz " + f"{tmp_path}/output.cnv.vcf.gz" + ) + + def test_interval_is_passed_to_cnvscope_only(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.cnvscope_job.shell) + assert "--interval" not in str(result.apply_job.shell) + + def test_replace_rg_is_passed_to_cnvscope_only(self, tmp_path): + result = make_stage( + tmp_path, + replace_rg=[["rg1=ID:rg1\\tSM:sample"]], + ).add_to(DAG()) + + assert "--replace_rg 'rg1=ID:rg1\\tSM:sample'" in str( + result.cnvscope_job.shell + ) + assert "--replace_rg" not in str(result.apply_job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + +class TestSexArguments: + """`--sex` and `--par` follow the sample sex""" + + def test_male_sample(self, tmp_path): + par_bed = tmp_path / "par.bed" + result = make_stage( + tmp_path, sample_sex=SampleSex.MALE, par_bed=par_bed + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex M" in shell + assert f"--par {par_bed}" in shell + + def test_female_sample(self, tmp_path): + # A female sample does not need the PAR regions + result = make_stage( + tmp_path, + sample_sex=SampleSex.FEMALE, + par_bed=tmp_path / "par.bed", + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex F" in shell + assert "--par" not in shell + + @pytest.mark.parametrize("sample_sex", [None, SampleSex.UNKNOWN]) + def test_without_a_known_sex(self, tmp_path, sample_sex, messages): + result = make_stage(tmp_path, sample_sex=sample_sex).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex" not in shell + assert "--par" not in shell + assert any("diploid" in msg for msg in messages) + + +class TestJobMetadata: + """Job names, task name and thread counts""" + + def test_default_names(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.cnvscope_job.name == "cnvscope" + assert result.apply_job.name == "cnv-model-apply" + + def test_overridden_names(self, tmp_path): + result = make_stage( + tmp_path, name="CNVscope", apply_name="CNVModelApply" + ).add_to(DAG()) + + assert result.cnvscope_job.name == "CNVscope" + assert result.apply_job.name == "CNVModelApply" + + def test_task_name_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "cnv" + assert job.threads == 4 + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_cnvscope_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.cnvscope_job] == {upstream} + assert dag.waiting_jobs[result.apply_job] == {result.cnvscope_job} + + def test_cnvscope_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.cnvscope_job in dag.ready_jobs + assert dag.waiting_jobs[result.apply_job] == {result.cnvscope_job} + + def test_result_fields(self, tmp_path): + cnv_vcf = tmp_path / "output.cnv.vcf.gz" + result = make_stage(tmp_path, cnv_vcf=cnv_vcf).add_to(DAG()) + + assert result.jobs == [result.cnvscope_job, result.apply_job] + assert result.terminal == {result.apply_job} + assert result.cnv_vcf == cnv_vcf diff --git a/tests/unit/test_stage_ploidy.py b/tests/unit/test_stage_ploidy.py new file mode 100644 index 0000000..d696903 --- /dev/null +++ b/tests/unit/test_stage_ploidy.py @@ -0,0 +1,130 @@ +""" +Unit tests for the ploidy estimation stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.ploidy import PloidyStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> PloidyStage: + """A PloidyStage over one alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + ) + defaults.update(kwargs) + return PloidyStage(**defaults) # type: ignore[arg-type] + + +class TestOutput: + """The ploidy JSON sits next to the output VCF""" + + def test_ploidy_json_derivation(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.ploidy_json == tmp_path / "output_ploidy.json" + assert str(result.ploidy_json) in str(result.job.shell) + + +class TestCommand: + """The `estimate_ploidy.py` command line""" + + def test_script_and_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + cmd = str(result.job.shell) + assert cmd.startswith("python3 ") + assert "estimate_ploidy.py" in cmd + assert f"-i {tmp_path}/one.bam {tmp_path}/two.bam" in cmd + assert f"--outfile {tmp_path}/output_ploidy.json" in cmd + + def test_the_script_defaults_apply_without_a_build(self, tmp_path): + # The `estimate_ploidy.py` defaults are the chr-prefixed names + cmd = str(make_stage(tmp_path).add_to(DAG()).job.shell) + + assert "--contigs" not in cmd + assert "--autosomes" not in cmd + assert "--x_contig" not in cmd + assert "--y_contig" not in cmd + + def test_an_unrecognized_build_uses_the_defaults(self, tmp_path): + cmd = str( + make_stage(tmp_path, reference_build="hg38") + .add_to(DAG()) + .job.shell + ) + + assert "--contigs" not in cmd + assert "--x_contig" not in cmd + + def test_b37_contig_names(self, tmp_path): + cmd = str( + make_stage(tmp_path, reference_build="b37").add_to(DAG()).job.shell + ) + + assert "--contigs 1 2 " in cmd + assert "--autosomes 1 2 " in cmd + # The sex chromosomes are in `--contigs` but not in `--autosomes` + assert ( + "--contigs " + + " ".join([str(i) for i in range(1, 23)] + ["X", "Y"]) + in cmd + ) + assert "--autosomes " + " ".join([str(i) for i in range(1, 23)]) in cmd + assert "--x_contig X" in cmd + assert "--y_contig Y" in cmd + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_metadata(self, tmp_path): + job = make_stage(tmp_path).add_to(DAG()).job + + assert job.name == "estimate-ploidy" + assert job.task_name == "ploidy" + # The script is single-threaded and does not reserve a core + assert job.threads == 0 + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_edge(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} From c64e9419dc6de7f207812df472fc0324fe2fd9c1 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 00:58:16 -0700 Subject: [PATCH 04/20] Move duplicate marking and metrics into stages `DedupStage` (LocusCollector followed by Dedup) replaces the short-read `dedup_and_metrics` builder and the pangenome `build_dedup_job`; `MetricsStage` (one metrics pass plus the optional WGS rehead job) replaces the post-dedup metrics code in both places. `MetricsPaths` derives the metrics directory and every metrics file name once, instead of at five call sites. `DNAscopePipeline.add_sr_preprocessing` composes the two stages and inserts the jobs itself, returning the deduplicated alignment, the Dedup job, and the QC jobs MultiQC waits on; the hybrid pipeline inherits it. Renames: the rehead job is `rehead-metrics`, and with `--duplicate_marking none` the metrics-only pass is now the `metrics` job under the `metrics` task rather than `locuscollector` under `dedup`. Commands and dependency edges are unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/base_pangenome.py | 150 ++----------- sentieon_cli/dnascope.py | 324 +++++++++++----------------- sentieon_cli/dnascope_hybrid.py | 35 +-- sentieon_cli/hybrid_pangenome.py | 59 ++--- sentieon_cli/pipeline.py | 7 +- sentieon_cli/sentieon_pangenome.py | 64 +++--- sentieon_cli/stages/dedup.py | 94 ++++++++ sentieon_cli/stages/metrics.py | 181 ++++++++++++++++ tests/unit/test_hybrid_pangenome.py | 6 +- tests/unit/test_stage_dedup.py | 200 +++++++++++++++++ tests/unit/test_stage_metrics.py | 244 +++++++++++++++++++++ 11 files changed, 939 insertions(+), 425 deletions(-) create mode 100644 sentieon_cli/stages/dedup.py create mode 100644 sentieon_cli/stages/metrics.py create mode 100644 tests/unit/test_stage_dedup.py create mode 100644 tests/unit/test_stage_metrics.py diff --git a/sentieon_cli/base_pangenome.py b/sentieon_cli/base_pangenome.py index c617c48..f21e9a6 100644 --- a/sentieon_cli/base_pangenome.py +++ b/sentieon_cli/base_pangenome.py @@ -4,28 +4,23 @@ import copy import pathlib -import sys -from typing import List, Optional, Tuple - -from importlib.resources import files +from typing import List, Optional from . import command_strings as cmds from .driver import ( AlignmentStat, + BaseAlgo, BaseDistributionByCycle, CoverageMetrics, - Dedup, - Driver, GCBias, InsertSizeMetricAlgo, - LocusCollector, MeanQualityByCycle, QualDistribution, WgsMetricsAlgo, ) from .job import Job from .pipeline import BasePipeline -from .shell_pipeline import Command, Pipeline +from .stages.metrics import MetricsPaths from .util import path_arg @@ -141,130 +136,15 @@ def build_kmc_job( return kmc_job - def build_dedup_job( - self, - output_bam, - input_bam: List[pathlib.Path], - tag: str, - left_align_rgid: Optional[str] = None, - metrics: Optional[pathlib.Path] = None, - ) -> Tuple[Job, Job]: - """Build deduplication job""" - score_file = self.tmp_dir.joinpath(f"sample-{tag}-score.txt.gz") - - read_filters = [] - if left_align_rgid: - read_filters.append( - f"IndelLeftAlignReadTransform,rgid={left_align_rgid}" - ) - - # LocusCollector + Dedup - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bam, - read_filter=read_filters, - ) - driver.add_algo(LocusCollector(score_file)) - - lc_job = Job( - Pipeline(Command(*driver.build_cmd())), - f"locuscollector-{tag}", - self.cores, - task_name="dedup", - ) - - driver2 = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bam, - read_filter=read_filters, - ) - driver2.add_algo(Dedup(output_bam, score_file, metrics=metrics)) - - dedup_job = Job( - Pipeline(Command(*driver2.build_cmd())), - f"dedup-{tag}", - self.cores, - task_name="dedup", - ) - - return lc_job, dedup_job - - def build_metrics_job( - self, - sample_input: List[pathlib.Path], - ) -> Tuple[Job, Job]: - """Build a metrics job""" - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - - # Create the metrics directory - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - metric_base = sample_name + ".txt" - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - - is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") - mqbc_metrics = metrics_dir.joinpath( - metric_base + ".mean_qual_by_cycle.txt" - ) - bdbc_metrics = metrics_dir.joinpath( - metric_base + ".base_distribution_by_cycle.txt" - ) - qualdist_metrics = metrics_dir.joinpath( - metric_base + ".qual_distribution.txt" - ) - as_metrics = metrics_dir.joinpath(metric_base + ".alignment_stat.txt") - coverage_metrics = metrics_dir.joinpath("coverage") - - # WGS metrics - wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") - gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - driver.add_algo(MeanQualityByCycle(mqbc_metrics)) - driver.add_algo(BaseDistributionByCycle(bdbc_metrics)) - driver.add_algo(QualDistribution(qualdist_metrics)) - driver.add_algo(AlignmentStat(as_metrics)) - driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) - driver.add_algo(WgsMetricsAlgo(wgs_metrics, include_unpaired="true")) - driver.add_algo(CoverageMetrics(coverage_metrics)) - - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) - - rehead_script = pathlib.Path( - str( - files("sentieon_cli.scripts").joinpath("rehead_wgs_metrics.py") - ) - ) - rehead_job = Job( - Pipeline( - Command( - sys.executable, - str(rehead_script), - "--metrics_file", - str(wgs_metrics), - ) - ), - "Rehead metrics", - 0, - task_name="metrics", - ) - return (metrics_job, rehead_job) + def pangenome_metrics_algos(self, paths: MetricsPaths) -> List[BaseAlgo]: + """The metrics collected from a deduplicated pangenome alignment""" + return [ + InsertSizeMetricAlgo(paths.insert_size), + MeanQualityByCycle(paths.mean_qual_by_cycle), + BaseDistributionByCycle(paths.base_distribution_by_cycle), + QualDistribution(paths.qual_distribution), + AlignmentStat(paths.alignment_stat), + GCBias(paths.gc_bias, summary=paths.gc_bias_summary), + WgsMetricsAlgo(paths.wgs, include_unpaired="true"), + CoverageMetrics(paths.coverage), + ] diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index f006666..4ffb8fd 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -4,6 +4,7 @@ import argparse import copy +from dataclasses import dataclass, field import itertools import os import pathlib @@ -13,15 +14,13 @@ import packaging.version -from importlib.resources import files - from . import command_strings as cmds from .dag import DAG from .driver import ( AlignmentStat, + BaseAlgo, BaseDistributionByCycle, CoverageMetrics, - Dedup, DNAModelApply, DNAscope, Driver, @@ -30,7 +29,6 @@ HsMetricAlgo, InsertSizeMetricAlgo, MeanQualityByCycle, - LocusCollector, QualDistribution, SVSolver, WgsMetricsAlgo, @@ -38,6 +36,9 @@ from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline +from .stages.base import StageContext +from .stages.dedup import DedupStage +from .stages.metrics import MetricsPaths, MetricsStage from .util import ( check_version, library_preloaded, @@ -61,6 +62,21 @@ } +@dataclass +class SrPreprocessing: + """The short-read dedup and metrics jobs added to a DAG. + + * ``deduped`` -- the alignment to call variants from; the input files + unchanged when duplicate marking is off. + * ``dedup_job`` -- the Dedup job, absent when duplicate marking is off. + * ``qc_jobs`` -- every metrics-producing job, for MultiQC to wait on. + """ + + deduped: List[pathlib.Path] + dedup_job: Optional[Job] = None + qc_jobs: List[Job] = field(default_factory=list) + + class DNAscopePipeline(BasePipeline): """The DNAscope pipeline""" @@ -384,6 +400,7 @@ def build_dag(self) -> DAG: """Build the DAG for the pipeline""" self.logger.info("Building the DAG") dag = DAG() + ctx = self.stage_context() # Alignment align_jobs: Set[Job] = set() @@ -399,25 +416,16 @@ def build_dag(self) -> DAG: sample_input += aligned_fastq # Dedup and metrics - ( - deduped, - lc_job, - dedup_job, - metrics_job, - rehead_job, - ) = self.dedup_and_metrics(sample_input) - if lc_job: - dag.add_job(lc_job, align_jobs.union(align_fastq_jobs)) - if dedup_job: - dag.add_job(dedup_job, {lc_job}) - if metrics_job: - dag.add_job(metrics_job, {dedup_job}) - if rehead_job: - dag.add_job(rehead_job, {metrics_job}) - if bam_rm_job: - dag.add_job(bam_rm_job, {dedup_job}) - if fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + preprocessing = self.add_sr_preprocessing( + dag, ctx, sample_input, align_jobs.union(align_fastq_jobs) + ) + deduped = preprocessing.deduped + dedup_job = preprocessing.dedup_job + if dedup_job: + if bam_rm_job: + dag.add_job(bam_rm_job, {dedup_job}) + if fq_rm_job: + dag.add_job(fq_rm_job, {dedup_job}) # Small variants if not self.skip_small_variants: @@ -447,16 +455,8 @@ def build_dag(self) -> DAG: # Multiqc if not self.skip_multiqc: multiqc_job = self.multiqc() - multiqc_dependencies: Set[Job] = set() - if lc_job: - multiqc_dependencies.add(lc_job) - if metrics_job: - multiqc_dependencies.add(metrics_job) - if rehead_job: - multiqc_dependencies.add(rehead_job) - if multiqc_job: - dag.add_job(multiqc_job, multiqc_dependencies) + dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) return dag def sr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job], Job]: @@ -629,188 +629,116 @@ def sr_align_fastq( return (res, jobs, rm_job) - def dedup_and_metrics( + def sr_metrics_algos(self, paths: MetricsPaths) -> List[BaseAlgo]: + """The metrics that do not need duplicate-marked reads""" + algos: List[BaseAlgo] = [ + MeanQualityByCycle(paths.mean_qual_by_cycle), + BaseDistributionByCycle(paths.base_distribution_by_cycle), + QualDistribution(paths.qual_distribution), + AlignmentStat(paths.alignment_stat), + ] + if self.assay == "WGS": + algos.append(GCBias(paths.gc_bias, summary=paths.gc_bias_summary)) + return algos + + def add_sr_preprocessing( self, - sample_input, - ) -> Tuple[ - List[pathlib.Path], - Optional[Job], - Optional[Job], - Optional[Job], - Optional[Job], - ]: - """Perform dedup and metrics collection""" + dag: DAG, + ctx: StageContext, + sample_input: List[pathlib.Path], + upstream: Set[Job], + ) -> SrPreprocessing: + """Mark duplicates and collect metrics from the short reads""" if not self.output_vcf: self.logger.error("output_vcf is required") sys.exit(2) suffix = "bam" if self.bam_format else "cram" - # Create the metrics directory - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - metric_base = sample_name + ".txt" - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - - # LocusCollector and Metrics - out_score = metrics_dir.joinpath(metric_base + ".score.txt.gz") - is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") - mqbc_metrics = metrics_dir.joinpath( - metric_base + ".mean_qual_by_cycle.txt" - ) - bdbc_metrics = metrics_dir.joinpath( - metric_base + ".base_distribution_by_cycle.txt" - ) - qualdist_metrics = metrics_dir.joinpath( - metric_base + ".qual_distribution.txt" - ) - as_metrics = metrics_dir.joinpath(metric_base + ".alignment_stat.txt") - coverage_metrics = metrics_dir.joinpath("coverage") + paths = MetricsPaths.from_output_vcf(self.output_vcf) + paths.ensure_dir(self.dry_run) - # WES metrics - hs_metrics = metrics_dir.joinpath( - metric_base + ".hybrid-selection.txt" - ) - - # WGS metrics - wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") - gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") - - lc_job = None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - if self.sr_duplicate_marking != "none": - driver.add_algo( - LocusCollector( - out_score, - consensus=self.consensus, + if self.sr_duplicate_marking == "none": + # Without duplicate marking there is nothing to collect after + # the fact, so every metric comes from one pass over the input + if self.skip_metrics: + return SrPreprocessing( + deduped=sample_input, dedup_job=None, qc_jobs=[] ) + metrics_result = MetricsStage( + ctx=ctx, + inputs=sample_input, + algos=[ + InsertSizeMetricAlgo(paths.insert_size), + *self.sr_metrics_algos(paths), + ], + name="metrics", + task_name="metrics", + threads=ctx.cores, + ).add_to(dag, upstream) + return SrPreprocessing( + deduped=sample_input, + dedup_job=None, + qc_jobs=list(metrics_result.jobs), ) - # Prefer to run InsertSizeMetricAlgo after duplicate marking - if not self.skip_metrics and ( - (self.assay == "WES" and not self.bed) - or self.sr_duplicate_marking == "none" - ): - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - + # Metrics that ride along on the LocusCollector pass + lc_extra_algos: List[BaseAlgo] = [] if not self.skip_metrics: - driver.add_algo(MeanQualityByCycle(mqbc_metrics)) - driver.add_algo(BaseDistributionByCycle(bdbc_metrics)) - driver.add_algo(QualDistribution(qualdist_metrics)) - driver.add_algo(AlignmentStat(as_metrics)) - if self.assay == "WGS": - driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) - - if not (self.sr_duplicate_marking == "none" and self.skip_metrics): - lc_job = Job( - Pipeline(Command(*driver.build_cmd())), - "locuscollector", - self.cores, - task_name="dedup", - ) + # Prefer to run InsertSizeMetricAlgo after duplicate marking + if self.assay == "WES" and not self.bed: + lc_extra_algos.append(InsertSizeMetricAlgo(paths.insert_size)) + lc_extra_algos.extend(self.sr_metrics_algos(paths)) - if self.sr_duplicate_marking == "none": - return (sample_input, lc_job, None, None, None) - - # Dedup deduped = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") ) - dedup_metrics = metrics_dir.joinpath( - metric_base + ".dedup_metrics.txt" - ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - driver.add_algo( - Dedup( - deduped, - out_score, - cram_write_options="version=3.0,compressor=rans", - metrics=dedup_metrics, - rmdup=(self.sr_duplicate_marking == "rmdup"), - ) - ) - dedup_job = Job( - Pipeline(Command(*driver.build_cmd())), - "dedup", - self.cores, - task_name="dedup", - ) - - if self.skip_metrics: - return ([deduped], lc_job, dedup_job, None, None) - - # Run HsMetricAlgo after duplicate marking to account for - # duplicate reads - metrics_job = None - rehead_job = None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=( - [deduped] - if self.sr_duplicate_marking != "none" - else sample_input - ), - interval=self.bed, + dedup_result = DedupStage( + ctx=ctx, + inputs=sample_input, + output=deduped, + score_file=paths.score, + consensus=self.consensus, + rmdup=(self.sr_duplicate_marking == "rmdup"), + cram_write_options="version=3.0,compressor=rans", + dedup_metrics=paths.dedup_metrics, + lc_extra_algos=lc_extra_algos, + ).add_to(dag, upstream) + qc_jobs: List[Job] = [dedup_result.lc_job] + + # HsMetricAlgo and WgsMetricsAlgo run after duplicate marking to + # account for duplicate reads + post_algos: List[BaseAlgo] = [] + rehead_metrics: Optional[pathlib.Path] = None + if not self.skip_metrics: + if self.assay == "WES" and self.bed: + post_algos = [ + HsMetricAlgo(paths.hybrid_selection, self.bed, self.bed), + InsertSizeMetricAlgo(paths.insert_size), + ] + elif self.assay == "WGS": + post_algos = [ + InsertSizeMetricAlgo(paths.insert_size), + WgsMetricsAlgo(paths.wgs, include_unpaired="true"), + CoverageMetrics(paths.coverage), + ] + # Rehead WGS metrics so they are recognized by MultiQC + rehead_metrics = paths.wgs + if post_algos: + metrics_result = MetricsStage( + ctx=ctx, + inputs=[deduped], + algos=post_algos, + interval=self.bed, + rehead_metrics=rehead_metrics, + threads=0, # Run metrics in the background + ).add_to(dag, {dedup_result.dedup_job}) + qc_jobs.extend(metrics_result.jobs) + + return SrPreprocessing( + deduped=[deduped], + dedup_job=dedup_result.dedup_job, + qc_jobs=qc_jobs, ) - if self.assay == "WES" and self.bed: - driver.add_algo(HsMetricAlgo(hs_metrics, self.bed, self.bed)) - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) # Run metrics in the background - - # Run WgsMetricsAlgo after duplicate marking to account for - # duplicate reads - if self.assay == "WGS": - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - driver.add_algo( - WgsMetricsAlgo(wgs_metrics, include_unpaired="true") - ) - driver.add_algo(CoverageMetrics(coverage_metrics)) - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) # Run metrics in the background - - # Rehead WGS metrics so they are recognized by MultiQC - rehead_script = pathlib.Path( - str( - files("sentieon_cli.scripts").joinpath( - "rehead_wgs_metrics.py" - ) - ) - ) - rehead_job = Job( - Pipeline( - Command( - sys.executable, - str(rehead_script), - "--metrics_file", - str(wgs_metrics), - ) - ), - "Rehead metrics", - 0, - task_name="metrics", - ) - return ([deduped], lc_job, dedup_job, metrics_job, rehead_job) def sr_call_variants( self, diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index c29101e..4036279 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -604,26 +604,13 @@ def build_dag(self) -> DAG: sr_aln.extend(aligned_fastq) # Short-read dedup - ( - deduped, - lc_job, - dedup_job, - metrics_job, - rehead_job, - ) = self.dedup_and_metrics( - sample_input=sr_aln, + preprocessing = self.add_sr_preprocessing( + dag, ctx, sr_aln, set(align_fastq_jobs) ) - sr_aln = deduped - if lc_job: - dag.add_job(lc_job, align_fastq_jobs) - if dedup_job: - dag.add_job(dedup_job, {lc_job}) - if metrics_job and not self.skip_metrics: - dag.add_job(metrics_job, {dedup_job}) - if rehead_job: - dag.add_job(rehead_job, {metrics_job}) - if fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + sr_aln = preprocessing.deduped + dedup_job = preprocessing.dedup_job + if dedup_job and fq_rm_job: + dag.add_job(fq_rm_job, {dedup_job}) # Long-read alignment realign_jobs: Set[Job] = set() @@ -640,16 +627,8 @@ def build_dag(self) -> DAG: if not self.skip_multiqc: multiqc_job = self.multiqc() - multiqc_dependencies: Set[Job] = set() - if lc_job: - multiqc_dependencies.add(lc_job) - if metrics_job: - multiqc_dependencies.add(metrics_job) - if rehead_job: - multiqc_dependencies.add(rehead_job) - if multiqc_job: - dag.add_job(multiqc_job, multiqc_dependencies) + dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) if not self.skip_svs: longreadsv_job = self.call_svs( diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index 0f33b8e..bf8f748 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -36,6 +36,8 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline +from .stages.dedup import DedupStage +from .stages.metrics import MetricsPaths, MetricsStage from .transfer import build_transfer_jobs from .util import ( __version__, @@ -617,6 +619,7 @@ def build_dag(self) -> DAG: self.logger.info("Building the hybrid-pangenome DAG") dag = DAG() + ctx = self.stage_context() ref_fai = pathlib.Path(str(self.reference) + ".fai") @@ -762,42 +765,42 @@ def build_dag(self) -> DAG: # is assumed to be deduplicated already. # Emit Dedup metrics for the primary (bwa) short-read alignment # so they land in the metrics directory scanned by MultiQC. + paths = MetricsPaths.from_output_vcf(self.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - dedup_metrics = metrics_dir.joinpath( - sample_name + ".txt.dedup_metrics.txt" - ) - - bwa_lc_job, bwa_dedup_job = self.build_dedup_job( - out_bwa_aln, [bwa_bam], "bwa", metrics=dedup_metrics - ) - lift_lc_job, lift_dedup_job = self.build_dedup_job( - out_lift_aln, - [lift_bam], - "lift", - ) - dag.add_job(bwa_lc_job, {bwa_job}) - dag.add_job(bwa_dedup_job, {bwa_lc_job}) - dag.add_job(lift_lc_job, {mm2_job}) - dag.add_job(lift_dedup_job, {lift_lc_job}) + paths.ensure_dir(self.dry_run) + dedup_metrics = paths.dedup_metrics + + bwa_dedup = DedupStage( + ctx=ctx, + tag="bwa", + inputs=[bwa_bam], + output=out_bwa_aln, + score_file=self.tmp_dir.joinpath("sample-bwa-score.txt.gz"), + dedup_metrics=dedup_metrics, + ).add_to(dag, {bwa_job}) + bwa_dedup_job = bwa_dedup.dedup_job + lift_dedup = DedupStage( + ctx=ctx, + tag="lift", + inputs=[lift_bam], + output=out_lift_aln, + score_file=self.tmp_dir.joinpath("sample-lift-score.txt.gz"), + ).add_to(dag, {mm2_job}) + lift_dedup_job = lift_dedup.dedup_job # Alignment metrics from the deduplicated short reads if not self.skip_metrics: - metrics_job, rehead_job = self.build_metrics_job( - [out_bwa_aln, out_lift_aln], - ) - dag.add_job(metrics_job, {bwa_dedup_job, lift_dedup_job}) - dag.add_job(rehead_job, {metrics_job}) + metrics_result = MetricsStage( + ctx=ctx, + inputs=[out_bwa_aln, out_lift_aln], + algos=self.pangenome_metrics_algos(paths), + rehead_metrics=paths.wgs, + ).add_to(dag, {bwa_dedup_job, lift_dedup_job}) if not self.skip_multiqc: multiqc_job = self.multiqc() if multiqc_job: - dag.add_job(multiqc_job, {rehead_job}) + dag.add_job(multiqc_job, metrics_result.terminal) calling_bams = [out_bwa_aln, out_lift_aln] + calling_lr calling_dependencies = {bwa_dedup_job, lift_dedup_job} diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 3638390..428f3af 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -28,6 +28,7 @@ par_bed_for_build, ) from .stages.base import StageContext +from .stages.metrics import MetricsPaths from .util import ( SampleSex, __version__, @@ -428,9 +429,9 @@ def multiqc(self) -> Optional[Job]: ) return None - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) + metrics_dir = MetricsPaths.from_output_vcf( + pathlib.Path(str(self.output_vcf)) + ).metrics_dir multiqc_job = Job( cmds.cmd_multiqc( metrics_dir, diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 2002bc3..f2f8d60 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -31,6 +31,8 @@ from .logging import get_logger from .shell_pipeline import Command, Pipeline from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.dedup import DedupStage +from .stages.metrics import MetricsPaths, MetricsStage from .stages.ploidy import PloidyStage from .util import ( SampleSex, @@ -798,47 +800,49 @@ def build_first_dag(self) -> DAG: # Emit Dedup metrics for the primary (bwa) short-read alignment so # they land in the metrics directory scanned by MultiQC. + paths = MetricsPaths.from_output_vcf(self.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - dedup_metrics = metrics_dir.joinpath( - sample_name + ".txt.dedup_metrics.txt" - ) - - bwa_lc_job, bwa_dedup_job = self.build_dedup_job( - out_bwa_aln, [bwa_bam], "bwa", metrics=dedup_metrics - ) - mm2_lc_job, mm2_dedup_job = self.build_dedup_job( - out_mm2_aln, - [mm2_bam], - "mm2", - left_align_rgid=f"{self.fastq_readgroup['ID']}-mm2", - ) - dag.add_job(bwa_lc_job, bwa_lc_dependencies) - dag.add_job(bwa_dedup_job, {bwa_lc_job}) + paths.ensure_dir(self.dry_run) + dedup_metrics = paths.dedup_metrics + + bwa_dedup = DedupStage( + ctx=ctx, + tag="bwa", + inputs=[bwa_bam], + output=out_bwa_aln, + score_file=self.tmp_dir.joinpath("sample-bwa-score.txt.gz"), + dedup_metrics=dedup_metrics, + ).add_to(dag, bwa_lc_dependencies) + bwa_dedup_job = bwa_dedup.dedup_job dnascope_dependencies.add(bwa_dedup_job) - dag.add_job(mm2_lc_job, {mm2_job}) - dag.add_job(mm2_dedup_job, {mm2_lc_job}) - dnascope_dependencies.add(mm2_dedup_job) + mm2_dedup = DedupStage( + ctx=ctx, + tag="mm2", + inputs=[mm2_bam], + output=out_mm2_aln, + score_file=self.tmp_dir.joinpath("sample-mm2-score.txt.gz"), + read_filters=[ + "IndelLeftAlignReadTransform,rgid=" + f"{self.fastq_readgroup['ID']}-mm2" + ], + ).add_to(dag, {mm2_job}) + dnascope_dependencies.add(mm2_dedup.dedup_job) cnv_input_bams = [out_bwa_aln] cnvscope_deps = {bwa_dedup_job} if not self.skip_metrics: - metrics_job, rehead_job = self.build_metrics_job( - [out_bwa_aln], - ) - dag.add_job(metrics_job, {bwa_dedup_job}) - dag.add_job(rehead_job, {metrics_job}) + metrics_result = MetricsStage( + ctx=ctx, + inputs=[out_bwa_aln], + algos=self.pangenome_metrics_algos(paths), + rehead_metrics=paths.wgs, + ).add_to(dag, {bwa_dedup_job}) if not self.skip_multiqc: multiqc_job = self.multiqc() if multiqc_job: - dag.add_job(multiqc_job, {rehead_job}) + dag.add_job(multiqc_job, metrics_result.terminal) else: dnascope_bams.append(mm2_bam) cnv_input_bams = list(self.sample_input) diff --git a/sentieon_cli/stages/dedup.py b/sentieon_cli/stages/dedup.py new file mode 100644 index 0000000..534dd19 --- /dev/null +++ b/sentieon_cli/stages/dedup.py @@ -0,0 +1,94 @@ +""" +Duplicate marking with LocusCollector and Dedup +""" + +from dataclasses import dataclass, field +import pathlib +from typing import Iterable, List, Optional + +from ..dag import DAG +from ..driver import BaseAlgo, Dedup, LocusCollector +from ..job import Job +from .base import Stage, StageResult, driver_job + + +@dataclass(kw_only=True) +class DedupResult(StageResult): + """The jobs and output of `DedupStage`""" + + lc_job: Job + dedup_job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class DedupStage(Stage): + """Mark duplicates: LocusCollector collects the scores, Dedup applies + them. + + Both passes read the same input files with the same read filters, so + the two jobs only differ in their algo. Metrics that are cheap to + collect before duplicate marking can ride along on the LocusCollector + pass through ``lc_extra_algos``. + """ + + inputs: List[pathlib.Path] + output: pathlib.Path + score_file: pathlib.Path + tag: Optional[str] = None + consensus: bool = False + rmdup: bool = False + cram_write_options: Optional[str] = None + dedup_metrics: Optional[pathlib.Path] = None + read_filters: List[str] = field(default_factory=list) + lc_extra_algos: List[BaseAlgo] = field(default_factory=list) + threads: Optional[int] = None + task_name: str = "dedup" + + def _name(self, base: str) -> str: + return base if self.tag is None else f"{base}-{self.tag}" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> DedupResult: + deps = set(upstream) + read_filter = self.read_filters or None + + lc_job = driver_job( + self.ctx, + [ + LocusCollector(self.score_file, consensus=self.consensus), + *self.lc_extra_algos, + ], + inputs=self.inputs, + read_filter=read_filter, + name=self._name("locuscollector"), + task_name=self.task_name, + threads=self.threads, + ) + dedup_job = driver_job( + self.ctx, + [ + Dedup( + self.output, + self.score_file, + cram_write_options=self.cram_write_options, + metrics=self.dedup_metrics, + rmdup=self.rmdup, + ) + ], + inputs=self.inputs, + read_filter=read_filter, + name=self._name("dedup"), + task_name=self.task_name, + threads=self.threads, + ) + + dag.add_job(lc_job, deps) + dag.add_job(dedup_job, {lc_job}) + + return DedupResult( + jobs=[lc_job, dedup_job], + terminal={dedup_job}, + lc_job=lc_job, + dedup_job=dedup_job, + output=self.output, + ) diff --git a/sentieon_cli/stages/metrics.py b/sentieon_cli/stages/metrics.py new file mode 100644 index 0000000..ee9d82b --- /dev/null +++ b/sentieon_cli/stages/metrics.py @@ -0,0 +1,181 @@ +""" +Alignment metrics collection +""" + +from dataclasses import dataclass +from importlib.resources import files +import pathlib +import sys +from typing import Iterable, List, Optional + +from ..dag import DAG +from ..driver import BaseAlgo +from ..job import Job +from ..shell_pipeline import Command, Pipeline +from .base import Stage, StageResult, driver_job + + +@dataclass(frozen=True) +class MetricsPaths: + """The metrics directory and the files the pipelines write into it. + + Every metrics file is named after the output VCF, so the whole set is + derived once, from :meth:`from_output_vcf`, and passed around instead + of being rebuilt at each call site. + """ + + metrics_dir: pathlib.Path + metric_base: str + + @classmethod + def from_output_vcf(cls, output_vcf: pathlib.Path) -> "MetricsPaths": + """The metrics paths belonging to an output VCF""" + return cls( + metrics_dir=pathlib.Path( + str(output_vcf).replace(".vcf.gz", "_metrics") + ), + metric_base=output_vcf.name.replace(".vcf.gz", "") + ".txt", + ) + + def ensure_dir(self, dry_run: bool) -> None: + """Create the metrics directory, unless this is a dry run""" + if not dry_run: + self.metrics_dir.mkdir(exist_ok=True) + + def _metric(self, suffix: str) -> pathlib.Path: + return self.metrics_dir.joinpath(self.metric_base + suffix) + + @property + def score(self) -> pathlib.Path: + """The LocusCollector score file""" + return self._metric(".score.txt.gz") + + @property + def insert_size(self) -> pathlib.Path: + """InsertSizeMetricAlgo output""" + return self._metric(".insert_size.txt") + + @property + def mean_qual_by_cycle(self) -> pathlib.Path: + """MeanQualityByCycle output""" + return self._metric(".mean_qual_by_cycle.txt") + + @property + def base_distribution_by_cycle(self) -> pathlib.Path: + """BaseDistributionByCycle output""" + return self._metric(".base_distribution_by_cycle.txt") + + @property + def qual_distribution(self) -> pathlib.Path: + """QualDistribution output""" + return self._metric(".qual_distribution.txt") + + @property + def alignment_stat(self) -> pathlib.Path: + """AlignmentStat output""" + return self._metric(".alignment_stat.txt") + + @property + def coverage(self) -> pathlib.Path: + """CoverageMetrics output prefix (not named after the VCF)""" + return self.metrics_dir.joinpath("coverage") + + @property + def hybrid_selection(self) -> pathlib.Path: + """HsMetricAlgo output, for WES runs""" + return self._metric(".hybrid-selection.txt") + + @property + def wgs(self) -> pathlib.Path: + """WgsMetricsAlgo output, for WGS runs""" + return self._metric(".wgs.txt") + + @property + def gc_bias(self) -> pathlib.Path: + """GCBias output""" + return self._metric(".gc_bias.txt") + + @property + def gc_bias_summary(self) -> pathlib.Path: + """The GCBias summary output""" + return self._metric(".gc_bias_summary.txt") + + @property + def dedup_metrics(self) -> pathlib.Path: + """The Dedup metrics output""" + return self._metric(".dedup_metrics.txt") + + +@dataclass(kw_only=True) +class MetricsResult(StageResult): + """The jobs of `MetricsStage`""" + + metrics_job: Job + rehead_job: Optional[Job] + + +@dataclass(kw_only=True) +class MetricsStage(Stage): + """Collect alignment metrics with a single `sentieon driver` pass. + + The caller supplies the ordered algo list, since which metrics are + worth collecting depends on the assay and on whether the alignment + has been deduplicated yet. When ``rehead_metrics`` names a WGS + metrics file, a second job rewrites its header so MultiQC recognizes + it. + """ + + inputs: List[pathlib.Path] + algos: List[BaseAlgo] + interval: Optional[pathlib.Path] = None + rehead_metrics: Optional[pathlib.Path] = None + name: str = "metrics" + task_name: str = "metrics" + threads: int = 0 + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> MetricsResult: + deps = set(upstream) + + metrics = driver_job( + self.ctx, + self.algos, + inputs=self.inputs, + interval=self.interval, + name=self.name, + task_name=self.task_name, + threads=self.threads, + ) + dag.add_job(metrics, deps) + jobs = [metrics] + + rehead: Optional[Job] = None + if self.rehead_metrics is not None: + rehead_script = pathlib.Path( + str( + files("sentieon_cli.scripts").joinpath( + "rehead_wgs_metrics.py" + ) + ) + ) + rehead = Job( + Pipeline( + Command( + sys.executable, + str(rehead_script), + "--metrics_file", + str(self.rehead_metrics), + ) + ), + "rehead-metrics", + 0, + task_name="metrics", + ) + dag.add_job(rehead, {metrics}) + jobs.append(rehead) + + return MetricsResult( + jobs=jobs, + terminal={rehead} if rehead else {metrics}, + metrics_job=metrics, + rehead_job=rehead, + ) diff --git a/tests/unit/test_hybrid_pangenome.py b/tests/unit/test_hybrid_pangenome.py index 42c836c..9239e63 100644 --- a/tests/unit/test_hybrid_pangenome.py +++ b/tests/unit/test_hybrid_pangenome.py @@ -273,7 +273,7 @@ def test_metrics_job(self): ) assert "--algo GCBias" in cmd_str assert "--algo WgsMetricsAlgo" in cmd_str - assert "Rehead metrics" in job_names + assert "rehead-metrics" in job_names def test_skip_metrics(self): """skip_metrics removes the metrics jobs and the Dedup metrics""" @@ -283,7 +283,7 @@ def test_skip_metrics(self): job_names, all_jobs = self._get_all_job_names(dag) assert "metrics" not in job_names - assert "Rehead metrics" not in job_names + assert "rehead-metrics" not in job_names assert "multiqc" not in job_names # Deduplication still runs assert "dedup-bwa" in job_names @@ -561,7 +561,7 @@ def test_aligned_dag_jobs(self): "locuscollector-lift", "dedup-lift", "metrics", - "Rehead metrics", + "rehead-metrics", "multiqc", ): assert name not in job_names, f"unexpected job: {name}" diff --git a/tests/unit/test_stage_dedup.py b/tests/unit/test_stage_dedup.py new file mode 100644 index 0000000..b48081e --- /dev/null +++ b/tests/unit/test_stage_dedup.py @@ -0,0 +1,200 @@ +""" +Unit tests for the duplicate-marking stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import AlignmentStat, GCBias +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.dedup import DedupStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> DedupStage: + """A DedupStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + output=tmp_path / "deduped.cram", + score_file=tmp_path / "score.txt.gz", + ) + defaults.update(kwargs) + return DedupStage(**defaults) # type: ignore[arg-type] + + +class TestCommands: + """The `sentieon driver` commands the stage builds""" + + def test_locuscollector_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.lc_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LocusCollector {tmp_path}/score.txt.gz" + ) + + def test_dedup_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + # The defaults produce the shortest possible Dedup command + assert str(result.dedup_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo Dedup --score_info {tmp_path}/score.txt.gz " + f"{tmp_path}/deduped.cram" + ) + + def test_consensus_reads(self, tmp_path): + result = make_stage(tmp_path, consensus=True).add_to(DAG()) + + assert "--algo LocusCollector --consensus" in str(result.lc_job.shell) + assert "--consensus" not in str(result.dedup_job.shell) + + def test_rmdup_and_cram_write_options(self, tmp_path): + result = make_stage( + tmp_path, + rmdup=True, + cram_write_options="version=3.0,compressor=rans", + ).add_to(DAG()) + + shell = str(result.dedup_job.shell) + assert "--cram_write_options version=3.0,compressor=rans" in shell + assert "--rmdup" in shell + # The options belong to Dedup alone + assert "--rmdup" not in str(result.lc_job.shell) + + def test_dedup_metrics(self, tmp_path): + metrics = tmp_path / "sample.dedup_metrics.txt" + result = make_stage(tmp_path, dedup_metrics=metrics).add_to(DAG()) + + assert f"--metrics {metrics}" in str(result.dedup_job.shell) + + def test_without_optional_dedup_arguments(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + shell = str(result.dedup_job.shell) + for flag in ("--cram_write_options", "--metrics", "--rmdup"): + assert flag not in shell + + def test_read_filters_reach_both_passes(self, tmp_path): + read_filter = "IndelLeftAlignReadTransform,rgid=rg1-mm2" + result = make_stage(tmp_path, read_filters=[read_filter]).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + assert f"--read_filter {read_filter}" in str(job.shell) + + def test_no_read_filters_emits_nothing(self, tmp_path): + result = make_stage(tmp_path, read_filters=[]).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + assert "--read_filter" not in str(job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + shell = str(job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + +class TestExtraAlgos: + """Metrics that ride along on the LocusCollector pass""" + + def test_extra_algos_follow_locuscollector(self, tmp_path): + result = make_stage( + tmp_path, + lc_extra_algos=[ + AlignmentStat(tmp_path / "as.txt"), + GCBias(tmp_path / "gc.txt", summary=tmp_path / "sum.txt"), + ], + ).add_to(DAG()) + + shell = str(result.lc_job.shell) + assert shell.index("--algo LocusCollector") < shell.index( + "--algo AlignmentStat" + ) + assert shell.index("--algo AlignmentStat") < shell.index( + "--algo GCBias" + ) + # They stay off the Dedup pass + assert "--algo AlignmentStat" not in str(result.dedup_job.shell) + + +class TestJobMetadata: + """Job names, task name and thread counts""" + + def test_default_names(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.lc_job.name == "locuscollector" + assert result.dedup_job.name == "dedup" + + def test_tagged_names(self, tmp_path): + result = make_stage(tmp_path, tag="bwa").add_to(DAG()) + + assert result.lc_job.name == "locuscollector-bwa" + assert result.dedup_job.name == "dedup-bwa" + + def test_threads_default_to_the_core_count(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.threads == 4 + assert job.task_name == "dedup" + + def test_overridden_threads_and_task_name(self, tmp_path): + result = make_stage( + tmp_path, threads=1, task_name="preprocess" + ).add_to(DAG()) + + for job in result.jobs: + assert job.threads == 1 + assert job.task_name == "preprocess" + # The driver still uses every core + assert "--thread_count 4" in str(result.lc_job.shell) + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_locuscollector_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.lc_job] == {upstream} + assert dag.waiting_jobs[result.dedup_job] == {result.lc_job} + + def test_locuscollector_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.lc_job in dag.ready_jobs + assert dag.waiting_jobs[result.dedup_job] == {result.lc_job} + + def test_result_fields(self, tmp_path): + output = tmp_path / "deduped.cram" + result = make_stage(tmp_path, output=output).add_to(DAG()) + + assert result.jobs == [result.lc_job, result.dedup_job] + assert result.terminal == {result.dedup_job} + assert result.output == output diff --git a/tests/unit/test_stage_metrics.py b/tests/unit/test_stage_metrics.py new file mode 100644 index 0000000..1792560 --- /dev/null +++ b/tests/unit/test_stage_metrics.py @@ -0,0 +1,244 @@ +""" +Unit tests for the metrics stage and the metrics paths +""" + +import pathlib +import sys + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import ( + AlignmentStat, + InsertSizeMetricAlgo, + WgsMetricsAlgo, +) +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.metrics import MetricsPaths, MetricsStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> MetricsStage: + """A MetricsStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "deduped.cram"], + algos=[AlignmentStat(tmp_path / "as.txt")], + ) + defaults.update(kwargs) + return MetricsStage(**defaults) # type: ignore[arg-type] + + +class TestMetricsPaths: + """Every metrics file is derived from the output VCF""" + + def test_directory_and_base(self): + paths = MetricsPaths.from_output_vcf( + pathlib.Path("/data/run/sample.vcf.gz") + ) + + assert paths.metrics_dir == pathlib.Path("/data/run/sample_metrics") + assert paths.metric_base == "sample.txt" + + def test_metric_files(self): + paths = MetricsPaths.from_output_vcf(pathlib.Path("/data/out.vcf.gz")) + base = pathlib.Path("/data/out_metrics") + + assert paths.score == base / "out.txt.score.txt.gz" + assert paths.insert_size == base / "out.txt.insert_size.txt" + assert ( + paths.mean_qual_by_cycle == base / "out.txt.mean_qual_by_cycle.txt" + ) + assert paths.base_distribution_by_cycle == ( + base / "out.txt.base_distribution_by_cycle.txt" + ) + assert paths.qual_distribution == ( + base / "out.txt.qual_distribution.txt" + ) + assert paths.alignment_stat == base / "out.txt.alignment_stat.txt" + assert paths.hybrid_selection == base / "out.txt.hybrid-selection.txt" + assert paths.wgs == base / "out.txt.wgs.txt" + assert paths.gc_bias == base / "out.txt.gc_bias.txt" + assert paths.gc_bias_summary == base / "out.txt.gc_bias_summary.txt" + assert paths.dedup_metrics == base / "out.txt.dedup_metrics.txt" + + def test_coverage_is_not_named_after_the_vcf(self): + paths = MetricsPaths.from_output_vcf(pathlib.Path("/data/out.vcf.gz")) + + assert paths.coverage == pathlib.Path("/data/out_metrics/coverage") + + def test_ensure_dir_creates_the_directory(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.ensure_dir(dry_run=False) + + assert paths.metrics_dir.is_dir() + + def test_ensure_dir_creates_nothing_on_a_dry_run(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.ensure_dir(dry_run=True) + + assert not paths.metrics_dir.exists() + + def test_ensure_dir_tolerates_an_existing_directory(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.metrics_dir.mkdir() + paths.ensure_dir(dry_run=False) + + assert paths.metrics_dir.is_dir() + + +class TestCommands: + """The commands the stage builds""" + + def test_metrics_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.metrics_job.shell) == ( + f"sentieon driver --input {tmp_path}/deduped.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo AlignmentStat --adapter_seq '' {tmp_path}/as.txt" + ) + + def test_algo_order_is_the_callers(self, tmp_path): + result = make_stage( + tmp_path, + algos=[ + InsertSizeMetricAlgo(tmp_path / "is.txt"), + WgsMetricsAlgo(tmp_path / "wgs.txt", include_unpaired="true"), + AlignmentStat(tmp_path / "as.txt"), + ], + ).add_to(DAG()) + + shell = str(result.metrics_job.shell) + assert shell.index("--algo InsertSizeMetricAlgo") < shell.index( + "--algo WgsMetricsAlgo" + ) + assert shell.index("--algo WgsMetricsAlgo") < shell.index( + "--algo AlignmentStat" + ) + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.metrics_job.shell) + + def test_without_an_interval(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--interval" not in str(result.metrics_job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + ).add_to(DAG()) + + shell = str(result.metrics_job.shell) + assert f"--input {tmp_path}/one.cram" in shell + assert f"--input {tmp_path}/two.cram" in shell + + +class TestRehead: + """The optional WGS-metrics rehead job""" + + def test_rehead_command(self, tmp_path): + wgs = tmp_path / "output.txt.wgs.txt" + result = make_stage(tmp_path, rehead_metrics=wgs).add_to(DAG()) + + assert result.rehead_job is not None + shell = str(result.rehead_job.shell) + assert shell.startswith(sys.executable) + assert "rehead_wgs_metrics.py" in shell + assert f"--metrics_file {wgs}" in shell + + def test_rehead_job_metadata(self, tmp_path): + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(DAG()) + + assert result.rehead_job is not None + assert result.rehead_job.name == "rehead-metrics" + assert result.rehead_job.threads == 0 + assert result.rehead_job.task_name == "metrics" + + def test_no_rehead_job_by_default(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.rehead_job is None + assert result.jobs == [result.metrics_job] + assert result.terminal == {result.metrics_job} + + def test_terminal_is_the_rehead_job(self, tmp_path): + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(DAG()) + + assert result.jobs == [result.metrics_job, result.rehead_job] + assert result.terminal == {result.rehead_job} + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_defaults(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.metrics_job.name == "metrics" + assert result.metrics_job.task_name == "metrics" + # Metrics run in the background by default + assert result.metrics_job.threads == 0 + + def test_custom_name_task_and_threads(self, tmp_path): + result = make_stage( + tmp_path, + name="sr-metrics", + task_name="qc", + threads=4, + ).add_to(DAG()) + + assert result.metrics_job.name == "sr-metrics" + assert result.metrics_job.task_name == "qc" + assert result.metrics_job.threads == 4 + + def test_rehead_keeps_the_metrics_task(self, tmp_path): + result = make_stage( + tmp_path, + name="sr-metrics", + task_name="qc", + rehead_metrics=tmp_path / "wgs.txt", + ).add_to(DAG()) + + assert result.rehead_job is not None + assert result.rehead_job.task_name == "metrics" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_the_metrics_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.metrics_job] == {upstream} + assert dag.waiting_jobs[result.rehead_job] == {result.metrics_job} + + def test_metrics_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.metrics_job in dag.ready_jobs From b0b3a7334321daed74d324e3d0b4733c7ec1bb2b Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 01:08:24 -0700 Subject: [PATCH 05/20] Move small-variant calling into stages Add the small-variant stages: `DNAscopeStage` (any DNAscope-family pass), `ModelApplyStage`, `GVCFtyperStage`, `TransferStage` (the sharded annotation transfer, wrapping `build_transfer_jobs`, which gains a `tag` for pipelines that transfer more than once), and the `TransferApplyStage` composite for the common "transfer, then apply the model" tail, whose DNAModelApply waits on the raw-VCF producer as well as the transfer. The dnascope pipeline's `sr_call_variants` (a 6-tuple) becomes `add_small_variant_calling`, and the pangenome pipelines drop their `build_dnascope_job` / `build_dnamodelapply_job` / `build_gvcftyper_job` builders. The DNAscope job is now named `dnascope` everywhere (was `variant-calling` and `dnascope-raw`). Commands and dependency edges are unchanged, except that a dnascope-pangenome run without a population VCF now feeds DNAModelApply the raw VCF rather than the transfer output that no job would have written; validation requires `--pop_vcf`, so real runs are unaffected. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 163 +++----- sentieon_cli/hybrid_pangenome.py | 125 +++--- sentieon_cli/sentieon_pangenome.py | 246 +++++------- sentieon_cli/stages/small_variants.py | 235 +++++++++++ sentieon_cli/stages/transfer.py | 80 ++++ sentieon_cli/transfer.py | 17 +- tests/integration/test_pipeline_execution.py | 4 +- tests/unit/test_dag_construction.py | 2 +- tests/unit/test_hybrid_pangenome.py | 36 +- tests/unit/test_sentieon_pangenome.py | 24 +- tests/unit/test_stage_small_variants.py | 385 +++++++++++++++++++ tests/unit/test_stage_transfer.py | 133 +++++++ 12 files changed, 1057 insertions(+), 393 deletions(-) create mode 100644 sentieon_cli/stages/small_variants.py create mode 100644 sentieon_cli/stages/transfer.py create mode 100644 tests/unit/test_stage_small_variants.py create mode 100644 tests/unit/test_stage_transfer.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 4ffb8fd..2467fbe 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -21,11 +21,8 @@ BaseAlgo, BaseDistributionByCycle, CoverageMetrics, - DNAModelApply, DNAscope, - Driver, GCBias, - GVCFtyper, HsMetricAlgo, InsertSizeMetricAlgo, MeanQualityByCycle, @@ -36,9 +33,15 @@ from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline -from .stages.base import StageContext +from .stages.base import StageContext, driver_job, rm_job from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + GVCFtyperStage, + TransferApplyStage, +) from .util import ( check_version, library_preloaded, @@ -429,28 +432,15 @@ def build_dag(self) -> DAG: # Small variants if not self.skip_small_variants: - ( - call_job, - apply_job, - rm_job, - gvcftyper_job, - svsolver_job, - sv_rm_job, - ) = self.sr_call_variants(deduped) call_dependencies: Set[Job] = set() if dedup_job: call_dependencies.add(dedup_job) else: call_dependencies.update(align_jobs) call_dependencies.update(align_fastq_jobs) - dag.add_job(call_job, call_dependencies) - dag.add_job(apply_job, {call_job}) - dag.add_job(rm_job, {apply_job}) - if gvcftyper_job: - dag.add_job(gvcftyper_job, {apply_job}) - if svsolver_job and sv_rm_job: - dag.add_job(svsolver_job, {call_job}) - dag.add_job(sv_rm_job, {svsolver_job}) + self.add_small_variant_calling( + dag, ctx, deduped, call_dependencies + ) # Multiqc if not self.skip_multiqc: @@ -740,10 +730,13 @@ def add_sr_preprocessing( qc_jobs=qc_jobs, ) - def sr_call_variants( + def add_small_variant_calling( self, + dag: DAG, + ctx: StageContext, deduped: List[pathlib.Path], - ) -> Tuple[Job, Job, Job, Optional[Job], Optional[Job], Optional[Job]]: + upstream: Set[Job], + ) -> None: """Call SNVs, indels, and SVs using DNAscope""" if not self.model_bundle: self.logger.error("model_bundle is required") @@ -766,7 +759,6 @@ def sr_call_variants( out_svs = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", "_svs.vcf.gz") ) - emit_mode = "gvcf" if self.gvcf else "variant" pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" model = self.model_bundle.joinpath("dnascope.model") @@ -782,14 +774,8 @@ def sr_call_variants( tmp_vcf = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", "_tmp.g.vcf.gz") ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=deduped, - interval=self.bed, - interval_padding=self.interval_padding, - ) - driver.add_algo( + + algos: List[BaseAlgo] = [ DNAscope( tmp_vcf, dbsnp=self.dbsnp, @@ -797,110 +783,57 @@ def sr_call_variants( pcr_indel_model=pcr_indel_model, model=model, ) - ) + ] if not self.skip_svs: - driver.add_algo( + algos.append( DNAscope( out_svs_tmp, dbsnp=self.dbsnp, var_type="BND", ) ) - call_job = Job( - Pipeline(Command(*driver.build_cmd())), - "variant-calling", - self.cores, - task_name="variant-calling", - ) + call = DNAscopeStage( + ctx=ctx, + algos=algos, + inputs=deduped, + interval=self.bed, + interval_padding=self.interval_padding, + ).add_to(dag, upstream) # Genotyping and filtering with DNAModelApply - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model, - tmp_vcf, - ds_out, - ) - ) - apply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=tmp_vcf, + apply=ApplySpec(model=model, output=ds_out), + ).add_to(dag, call.terminal) # Remove the tmp_vcf - rm_cmd = ["rm", str(tmp_vcf), str(tmp_vcf) + ".tbi"] - rm_job = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), - "rm-tmp-vcf", - 0, - task_name="cleanup", + dag.add_job( + rm_job([tmp_vcf, str(tmp_vcf) + ".tbi"], "rm-tmp-vcf"), + transfer_apply.terminal, ) # Genotype gVCFs - gvcftyper_job = None if self.gvcf: - driver = Driver( - reference=self.reference, - thread_count=self.cores, + GVCFtyperStage( + ctx=ctx, + gvcf=out_gvcf, + output=self.output_vcf, interval=self.bed, - ) - driver.add_algo( - GVCFtyper( - output=self.output_vcf, - vcf=out_gvcf, - ) - ) - gvcftyper_job = Job( - Pipeline(Command(*driver.build_cmd())), - "gvcftyper", - self.cores, - task_name="gvcftyper", - ) + ).add_to(dag, transfer_apply.terminal) # Call SVs - svsolver_job = None - sv_rm_job = None if not self.skip_svs: - driver = Driver( - reference=self.reference, - thread_count=self.cores, + svsolver_job = driver_job( + ctx, + [SVSolver(output=out_svs, vcf=out_svs_tmp)], interval=self.bed, - ) - driver.add_algo( - SVSolver( - output=out_svs, - vcf=out_svs_tmp, - ) - ) - svsolver_job = Job( - Pipeline(Command(*driver.build_cmd())), - "svsolver", + threads=1, + name="svsolver", task_name="sv-calling", ) - sv_rm_job = Job( - Pipeline( - Command( - "rm", - str(out_svs_tmp), - str(out_svs_tmp) + ".tbi", - fail_ok=True, - ) - ), - "rm-tmp-sv", - 0, - task_name="cleanup", + dag.add_job(svsolver_job, call.terminal) + dag.add_job( + rm_job([out_svs_tmp, str(out_svs_tmp) + ".tbi"], "rm-tmp-sv"), + {svsolver_job}, ) - - return ( - call_job, - apply_job, - rm_job, - gvcftyper_job, - svsolver_job, - sv_rm_job, - ) diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index bf8f748..cc0e2f1 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -20,7 +20,6 @@ from .base_pangenome import BasePangenome from .dag import DAG from .driver import ( - DNAModelApply, DNAscope, Driver, LongReadSV, @@ -38,7 +37,8 @@ from .shell_pipeline import Command, Pipeline from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage -from .transfer import build_transfer_jobs +from .stages.small_variants import DNAscopeStage, ModelApplyStage +from .stages.transfer import TransferConfig, TransferStage from .util import ( __version__, check_kmc_patch, @@ -825,40 +825,57 @@ def build_dag(self) -> DAG: if self.skip_small_variants: return dag - dnascope_job = self.build_dnascope_job( - raw_vcf, - calling_bams, - replace_rg, - ) - dag.add_job(dnascope_job, calling_dependencies) + model = self.model_bundle.joinpath("dnascope.model") + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" + call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + raw_vcf, + model=model, + pcr_indel_model=pcr_indel_model, + dbsnp=self.dbsnp, + ) + ], + inputs=calling_bams, + interval=self.bed, + replace_rg=replace_rg, + ).add_to(dag, calling_dependencies) # transfer annotations from the pop_vcf transfer_target = ( transfer_vcf if not self.skip_model_apply else self.output_vcf ) - transfer_jobs, concat_job = build_transfer_jobs( - transfer_target, - self.pop_vcf, - raw_vcf, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, - ) - for job in transfer_jobs: - dag.add_job(job, {dnascope_job}) - dag.add_job(concat_job, set(transfer_jobs)) + transfer = TransferStage( + ctx=ctx, + config=self.transfer_config(), + raw_vcf=raw_vcf, + out_vcf=transfer_target, + ).add_to(dag, call.terminal) if not self.skip_model_apply: - apply_job = self.build_dnamodelapply_job( - transfer_vcf, self.output_vcf - ) - dag.add_job(apply_job, {concat_job}) + ModelApplyStage( + ctx=ctx, + model=model, + vcf=transfer_vcf, + output=self.output_vcf, + ).add_to(dag, transfer.terminal) return dag + def transfer_config(self) -> TransferConfig: + """The inputs the annotation-transfer stage needs. + + `build_dag` requires `--pop_vcf` before it calls this. + """ + assert self.pop_vcf is not None + return TransferConfig( + pop_vcf=self.pop_vcf, + shards=self.shards, + pop_vcf_contigs=self.pop_vcf_contigs, + fai_data=self.fai_data, + ) + def lr_kmc_pairs(self) -> List[Tuple[pathlib.Path, pathlib.Path]]: """`(alignment, decode reference)` pairs for the long-read k-mer counting. @@ -1206,59 +1223,3 @@ def build_pangenomesv_job( self.cores, task_name="sv-calling", ) - - def build_dnascope_job( - self, - out_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - replace_rg: List[List[str]], - ) -> Job: - """Call small variants with the original, lifted, and long reads""" - assert self.model_bundle is not None - pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - replace_rg=replace_rg, - ) - driver.add_algo( - DNAscope( - out_vcf, - model=self.model_bundle.joinpath("dnascope.model"), - pcr_indel_model=pcr_indel_model, - dbsnp=self.dbsnp, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "dnascope-raw", - self.cores, - task_name="variant-calling", - ) - - def build_dnamodelapply_job( - self, - in_vcf: pathlib.Path, - out_vcf: pathlib.Path, - ) -> Job: - """Apply the DNAscope model""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("dnascope.model"), - vcf=in_vcf, - output=out_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index f2f8d60..ccd4877 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -20,10 +20,9 @@ from .base_pangenome import BasePangenome from .dag import DAG from .driver import ( - DNAModelApply, + BaseAlgo, DNAscope, Driver, - GVCFtyper, PangenomeSV, ReadWriter, ) @@ -34,6 +33,14 @@ from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage from .stages.ploidy import PloidyStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + GVCFtyperStage, + TransferApplyStage, + TransferSpec, +) +from .stages.transfer import TransferConfig from .util import ( SampleSex, __version__, @@ -52,7 +59,6 @@ parse_fai, vcf_contigs, ) -from .transfer import build_transfer_jobs SENT_PANGENOME_MIN_VERSIONS = { "kmc": None, @@ -903,27 +909,48 @@ def build_first_dag(self) -> DAG: if self.skip_small_variants and not self.call_svs: return dag - if self.skip_small_variants and self.call_svs and sv_vcf: - # SV calling only - dnascope_job = self.build_dnascope_job( - raw_vcf, - dnascope_bams, - sv_vcf=sv_vcf, - gfa_file=sample_gfa, + read_filters: List[str] = [] + if self.tech.upper() == "ULTIMA": + read_filters.append("UltimaReadFilter") + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" + model = self.model_bundle.joinpath("dnascope.model") + gfa_file = sample_gfa if self.call_svs else None + + algos: List[BaseAlgo] = [] + if not self.skip_small_variants: + algos.append( + DNAscope( + raw_vcf, + model=model, + pcr_indel_model=pcr_indel_model, + dbsnp=self.dbsnp, + emit_mode="gvcf" if self.gvcf else "variant", + ) + ) + if sv_vcf and gfa_file: + algos.append( + PangenomeSV( + sv_vcf, + gfa_file=gfa_file, + prefix=self.pangenome_contig_prefix, + ) ) - dag.add_job(dnascope_job, dnascope_dependencies) - # CNV calling is sex-aware and runs in the second DAG + call = DNAscopeStage( + ctx=ctx, + algos=algos, + inputs=dnascope_bams, + interval=self.bed, + read_filter=read_filters, + ).add_to(dag, dnascope_dependencies) + + if self.skip_small_variants: + # SV calling only. CNV calling is sex-aware and runs in the + # second DAG return dag - apply_dependencies = set() - dnascope_job = self.build_dnascope_job( - raw_vcf, - dnascope_bams, - sv_vcf=sv_vcf, - gfa_file=sample_gfa if self.call_svs else None, - ) - dag.add_job(dnascope_job, dnascope_dependencies) - apply_dependencies.add(dnascope_job) + if self.skip_model_apply and not self.pop_vcf: + # Nothing post-processes the raw VCF + return dag # When --gvcf is set, the model-apply / transfer outputs are # gVCFs; GVCFtyper produces the final VCF at self.output_vcf. @@ -934,56 +961,60 @@ def build_first_dag(self) -> DAG: else "sample-snv_apply.vcf.gz" ) - # transfer annotations from the pop_vcf + # Transfer annotations from the pop_vcf, then apply the model + transfer: Optional[TransferSpec] = None if self.pop_vcf: - transfer_jobs, concat_job = build_transfer_jobs( - ( + transfer = TransferSpec( + config=self.transfer_config(), + out_vcf=( transfer_vcf if not self.skip_model_apply - else (snv_apply_vcf) + else snv_apply_vcf ), - self.pop_vcf, - raw_vcf, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, ) - for job in transfer_jobs: - dag.add_job(job, {dnascope_job}) - dag.add_job(concat_job, set(transfer_jobs)) - apply_dependencies.add(concat_job) - - small_variants_last_job: Optional[Job] = None + apply_spec: Optional[ApplySpec] = None if not self.skip_model_apply: - # DNAModelApply - apply_job = self.build_dnamodelapply_job( - transfer_vcf, snv_apply_vcf - ) - dag.add_job(apply_job, apply_dependencies) - small_variants_last_job = apply_job - elif self.pop_vcf: - small_variants_last_job = concat_job + apply_spec = ApplySpec(model=model, output=snv_apply_vcf) + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=raw_vcf, + transfer=transfer, + apply=apply_spec, + ).add_to(dag, call.terminal) # Update the overestimated AD/DP of the joint pileup - if small_variants_last_job is not None: - ad_update_job = self.build_count_ad_update_job( - small_variants_out, snv_apply_vcf - ) - dag.add_job(ad_update_job, {small_variants_last_job}) - small_variants_last_job = ad_update_job + ad_update_job = self.build_count_ad_update_job( + small_variants_out, snv_apply_vcf + ) + dag.add_job(ad_update_job, transfer_apply.terminal) # Genotype the gVCF to also produce a regular VCF at output_vcf - if self.gvcf and small_variants_last_job is not None: - gvcftyper_job = self.build_gvcftyper_job(self.output_vcf, out_gvcf) - dag.add_job(gvcftyper_job, {small_variants_last_job}) + if self.gvcf: + GVCFtyperStage( + ctx=ctx, + gvcf=out_gvcf, + output=self.output_vcf, + interval=self.bed, + ).add_to(dag, {ad_update_job}) # CNV calling is sex-aware and runs in the second DAG return dag + def transfer_config(self) -> TransferConfig: + """The inputs the annotation-transfer stage needs. + + Only valid once `--pop_vcf` is known to be set; `validate_bundle` + requires it. + """ + assert self.pop_vcf is not None + return TransferConfig( + pop_vcf=self.pop_vcf, + shards=self.shards, + pop_vcf_contigs=self.pop_vcf_contigs, + fai_data=self.fai_data, + ) + def build_alignment_job( self, sample_bam: pathlib.Path, @@ -1134,109 +1165,6 @@ def build_minimap2_lift_job( ) return mm2_job - def build_dnascope_job( - self, - out_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - sv_vcf: Optional[pathlib.Path] = None, - gfa_file: Optional[pathlib.Path] = None, - ) -> Job: - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - read_filters = [] - if self.tech.upper() == "ULTIMA": - read_filters.append("UltimaReadFilter") - - pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - read_filter=read_filters, - ) - if not self.skip_small_variants: - driver.add_algo( - DNAscope( - out_vcf, - model=self.model_bundle.joinpath("dnascope.model"), - pcr_indel_model=pcr_indel_model, - dbsnp=self.dbsnp, - emit_mode="gvcf" if self.gvcf else "variant", - ) - ) - if sv_vcf and gfa_file: - driver.add_algo( - PangenomeSV( - sv_vcf, - gfa_file=gfa_file, - prefix=self.pangenome_contig_prefix, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "dnascope-raw", - self.cores, - task_name="variant-calling", - ) - - def build_dnamodelapply_job( - self, - in_vcf: pathlib.Path, - out_vcf: pathlib.Path, - ) -> Job: - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("dnascope.model"), - vcf=in_vcf, - output=out_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) - - def build_gvcftyper_job( - self, - out_vcf: pathlib.Path, - in_gvcf: pathlib.Path, - ) -> Job: - """Genotype a gVCF to produce a VCF""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - interval=self.bed, - ) - driver.add_algo( - GVCFtyper( - output=out_vcf, - vcf=in_gvcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "gvcftyper", - self.cores, - task_name="gvcftyper", - ) - def build_count_ad_update_job( self, out_vcf: pathlib.Path, diff --git a/sentieon_cli/stages/small_variants.py b/sentieon_cli/stages/small_variants.py new file mode 100644 index 0000000..9757781 --- /dev/null +++ b/sentieon_cli/stages/small_variants.py @@ -0,0 +1,235 @@ +""" +Small-variant calling, annotation transfer, and genotyping +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional, Set, Union + +from ..dag import DAG +from ..driver import BaseAlgo, DNAModelApply, GVCFtyper +from ..job import Job +from .base import Stage, StageResult, driver_job +from .transfer import TransferConfig, TransferResult, TransferStage + + +@dataclass(kw_only=True) +class DriverResult(StageResult): + """The job of a single-job `sentieon driver` stage""" + + job: Job + + +@dataclass(kw_only=True) +class DriverStage(Stage): + """A single `sentieon driver` invocation, wired into the DAG. + + The driver arguments are passed through untouched; a stage that needs + more structure than "run these algos" subclasses this one and fixes + the defaults it cares about. + """ + + algos: List[BaseAlgo] + name: str + task_name: str + threads: Optional[int] = None + inputs: Optional[List[pathlib.Path]] = None + interval: Optional[Union[pathlib.Path, str]] = None + interval_padding: Optional[int] = None + read_filter: Optional[List[str]] = None + replace_rg: Optional[List[List[str]]] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> DriverResult: + job = driver_job( + self.ctx, + self.algos, + name=self.name, + task_name=self.task_name, + threads=self.threads, + inputs=self.inputs, + interval=self.interval, + interval_padding=self.interval_padding, + read_filter=self.read_filter, + replace_rg=self.replace_rg, + ) + dag.add_job(job, set(upstream)) + return DriverResult(jobs=[job], terminal={job}, job=job) + + +@dataclass(kw_only=True) +class DNAscopeStage(DriverStage): + """The DNAscope-family calling pass. + + The caller supplies the algos in the order the driver should see + them -- SNV/indel calling first, then a BND or PangenomeSV pass in + the same pileup. + """ + + name: str = "dnascope" + task_name: str = "variant-calling" + + +@dataclass(kw_only=True) +class ModelApplyResult(StageResult): + """The job and output of `ModelApplyStage`""" + + job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class ModelApplyStage(Stage): + """Genotype and filter a raw VCF with DNAModelApply""" + + model: pathlib.Path + vcf: pathlib.Path + output: pathlib.Path + name: str = "model-apply" + task_name: str = "model-apply" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> ModelApplyResult: + job = driver_job( + self.ctx, + [DNAModelApply(self.model, self.vcf, self.output)], + name=self.name, + task_name=self.task_name, + ) + dag.add_job(job, set(upstream)) + return ModelApplyResult( + jobs=[job], + terminal={job}, + job=job, + output=self.output, + ) + + +@dataclass(kw_only=True) +class GVCFtyperResult(StageResult): + """The job and output of `GVCFtyperStage`""" + + job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class GVCFtyperStage(Stage): + """Genotype a gVCF to produce a VCF""" + + gvcf: pathlib.Path + output: pathlib.Path + interval: Optional[pathlib.Path] = None + name: str = "gvcftyper" + task_name: str = "gvcftyper" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> GVCFtyperResult: + job = driver_job( + self.ctx, + [GVCFtyper(output=self.output, vcf=self.gvcf)], + interval=self.interval, + name=self.name, + task_name=self.task_name, + ) + dag.add_job(job, set(upstream)) + return GVCFtyperResult( + jobs=[job], + terminal={job}, + job=job, + output=self.output, + ) + + +@dataclass(frozen=True) +class TransferSpec: + """Ask `TransferApplyStage` for an annotation transfer""" + + config: TransferConfig + out_vcf: pathlib.Path + tag: Optional[str] = None + + +@dataclass(frozen=True) +class ApplySpec: + """Ask `TransferApplyStage` for a DNAModelApply pass""" + + model: pathlib.Path + output: pathlib.Path + name: str = "model-apply" + + +@dataclass(kw_only=True) +class TransferApplyResult(StageResult): + """The jobs and output of `TransferApplyStage`""" + + transfer: Optional[TransferResult] + apply_job: Optional[Job] + output: pathlib.Path + + +@dataclass(kw_only=True) +class TransferApplyStage(Stage): + """Post-process a raw VCF: transfer annotations, then apply the model. + + Either half may be omitted. When both run, DNAModelApply reads the + transfer output and waits on the transfer *and* on ``upstream`` -- + the raw-VCF producer stays a direct dependency, matching the DAGs the + pipelines built by hand. + """ + + raw_vcf: pathlib.Path + transfer: Optional[TransferSpec] = None + apply: Optional[ApplySpec] = None + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> TransferApplyResult: + deps = set(upstream) + if self.transfer is None and self.apply is None: + raise ValueError( + "TransferApplyStage needs a transfer, an apply, or both" + ) + + jobs: List[Job] = [] + transfer_result: Optional[TransferResult] = None + apply_in = self.raw_vcf + apply_deps = deps + if self.transfer is not None: + transfer_result = TransferStage( + ctx=self.ctx, + config=self.transfer.config, + raw_vcf=self.raw_vcf, + out_vcf=self.transfer.out_vcf, + tag=self.transfer.tag, + ).add_to(dag, deps) + jobs.extend(transfer_result.jobs) + apply_in = self.transfer.out_vcf + apply_deps = deps | transfer_result.terminal + + apply_job: Optional[Job] = None + if self.apply is not None: + apply_result = ModelApplyStage( + ctx=self.ctx, + model=self.apply.model, + vcf=apply_in, + output=self.apply.output, + name=self.apply.name, + ).add_to(dag, apply_deps) + apply_job = apply_result.job + jobs.append(apply_job) + terminal: Set[Job] = {apply_job} + output = self.apply.output + else: + assert transfer_result is not None + terminal = transfer_result.terminal + output = transfer_result.out_vcf + + return TransferApplyResult( + jobs=jobs, + terminal=terminal, + transfer=transfer_result, + apply_job=apply_job, + output=output, + ) diff --git a/sentieon_cli/stages/transfer.py b/sentieon_cli/stages/transfer.py new file mode 100644 index 0000000..7ba294b --- /dev/null +++ b/sentieon_cli/stages/transfer.py @@ -0,0 +1,80 @@ +""" +Annotation transfer from a population VCF +""" + +from dataclasses import dataclass +import pathlib +from typing import Dict, Iterable, List, Optional + +from ..dag import DAG +from ..job import Job +from ..shard import Shard +from ..transfer import build_transfer_jobs +from .base import Stage, StageResult + + +@dataclass(frozen=True) +class TransferConfig: + """The run-wide inputs the annotation transfer needs. + + Every pipeline that transfers annotations builds the same fan-out from + the same four values, so they travel together instead of being threaded + through each call site. + """ + + pop_vcf: pathlib.Path + shards: List[Shard] + pop_vcf_contigs: Dict[str, Optional[int]] + fai_data: Dict[str, Dict[str, int]] + + +@dataclass(kw_only=True) +class TransferResult(StageResult): + """The jobs and output of `TransferStage`""" + + shard_jobs: List[Job] + concat_job: Job + out_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class TransferStage(Stage): + """Transfer annotations from the population VCF onto a raw VCF. + + One job per genomic shard merges the shard's annotations, then a + single concat job assembles ``out_vcf``. ``tag`` disambiguates the + job names when a pipeline transfers more than once in a run. + """ + + config: TransferConfig + raw_vcf: pathlib.Path + out_vcf: pathlib.Path + tag: Optional[str] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> TransferResult: + deps = set(upstream) + + shard_jobs, concat_job = build_transfer_jobs( + self.out_vcf, + self.config.pop_vcf, + self.raw_vcf, + self.ctx.tmp_dir, + self.config.shards, + self.config.pop_vcf_contigs, + self.config.fai_data, + self.ctx.dry_run, + self.ctx.cores, + tag=self.tag, + ) + + for job in shard_jobs: + dag.add_job(job, deps) + dag.add_job(concat_job, set(shard_jobs)) + + return TransferResult( + jobs=[*shard_jobs, concat_job], + terminal={concat_job}, + shard_jobs=shard_jobs, + concat_job=concat_job, + out_vcf=self.out_vcf, + ) diff --git a/sentieon_cli/transfer.py b/sentieon_cli/transfer.py index ed082fb..24f9c11 100644 --- a/sentieon_cli/transfer.py +++ b/sentieon_cli/transfer.py @@ -28,8 +28,17 @@ def build_transfer_jobs( fai_data: Dict[str, Dict[str, int]], dry_run: bool = False, cores: int = 1, + *, + tag: Optional[str] = None, ) -> Tuple[List[Job], Job]: - """Transfer annotations from the pop_vcf to the raw_vcf""" + """Transfer annotations from the pop_vcf to the raw_vcf. + + ``tag`` is inserted into the job names (``merge-trim-{tag}-...``) so a + pipeline that transfers annotations more than once in a run can tell + the two fan-outs apart. + """ + + name_base = "merge-trim" if tag is None else f"merge-trim-{tag}" # Get a unique tmpdir tmp_dir_str = tempfile.mkdtemp(dir=base_tmp_dir) @@ -87,7 +96,7 @@ def build_transfer_jobs( raw_vcf, regions_file=subset_bed, ), - "merge-trim-extra", + f"{name_base}-extra", 1, task_name="annotation-transfer", ) @@ -122,7 +131,7 @@ def build_transfer_jobs( ], view_xargs=["--no-version"], ), - f"merge-trim-{i}", + f"{name_base}-{i}", 1, task_name="annotation-transfer", ) @@ -136,7 +145,7 @@ def build_transfer_jobs( sharded_vcfs, xargs=["--no-version", "--threads", str(cores)], ), - "merge-trim-concat", + f"{name_base}-concat", cores, task_name="annotation-transfer", ) diff --git a/tests/integration/test_pipeline_execution.py b/tests/integration/test_pipeline_execution.py index e7a25c1..3f89de3 100644 --- a/tests/integration/test_pipeline_execution.py +++ b/tests/integration/test_pipeline_execution.py @@ -192,7 +192,7 @@ def test_dnascope_command_parameters(self): dag = pipeline.build_dag() # Find variant calling job - variant_job = DAGAnalyzer.get_job_by_name(dag, "variant-calling") + variant_job = DAGAnalyzer.get_job_by_name(dag, "dnascope") assert variant_job is not None, "Should have variant calling job" # Check that command includes expected parameters @@ -364,7 +364,7 @@ def test_variant_calling_dependencies(self): dag = pipeline.build_dag() # Variant calling should depend on either dedup or input processing - variant_jobs = DAGAnalyzer.get_jobs_by_type(dag, "variant") + variant_jobs = DAGAnalyzer.get_jobs_by_type(dag, "dnascope") assert len(variant_jobs) > 0, "Should have variant calling jobs" diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index c0df2cf..1749f70 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -166,7 +166,7 @@ def test_skip_small_variants_dag(self, mock_lib_preloaded): job_names = [job.name for job in all_jobs] # Should not have variant calling jobs - variant_job_keywords = ["variant-calling", "model-apply", "gvcftyper"] + variant_job_keywords = ["dnascope", "model-apply", "gvcftyper"] for keyword in variant_job_keywords: assert not any(keyword in name for name in job_names) diff --git a/tests/unit/test_hybrid_pangenome.py b/tests/unit/test_hybrid_pangenome.py index 9239e63..f3f25e1 100644 --- a/tests/unit/test_hybrid_pangenome.py +++ b/tests/unit/test_hybrid_pangenome.py @@ -159,7 +159,7 @@ def test_dag_jobs_present(self): "dedup-lift", "metrics", "pangenome-sv", - "dnascope-raw", + "dnascope", "model-apply", ): assert name in job_names, f"missing job: {name}" @@ -366,7 +366,7 @@ def test_calling_inputs_and_replace_rg(self): lift_aln = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.cram") replace_arg = r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" - for job_name in ("pangenome-sv", "dnascope-raw"): + for job_name in ("pangenome-sv", "dnascope"): cmd_str = str(self._get_job(all_jobs, job_name).shell) assert bwa_aln in cmd_str assert lift_aln in cmd_str @@ -421,7 +421,7 @@ def test_dnascope_command(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert "--algo DNAscope" in cmd_str assert "dnascope.model" in cmd_str assert "--pcr_indel_model CONSERVATIVE" in cmd_str @@ -435,7 +435,7 @@ def test_pcr_free(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert "--pcr_indel_model NONE" in cmd_str def test_bam_format(self): @@ -451,7 +451,7 @@ def test_bam_format(self): assert bwa_bam in str(self._get_job(all_jobs, "dedup-bwa").shell) assert lift_bam in str(self._get_job(all_jobs, "dedup-lift").shell) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert bwa_bam in cmd_str assert lift_bam in cmd_str @@ -484,7 +484,7 @@ def test_skip_svs(self): job_names, _ = self._get_all_job_names(dag) assert "pangenome-sv" not in job_names - assert "dnascope-raw" in job_names + assert "dnascope" in job_names assert "graph-update" in job_names def test_skip_small_variants(self): @@ -494,7 +494,7 @@ def test_skip_small_variants(self): dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) - assert "dnascope-raw" not in job_names + assert "dnascope" not in job_names assert "model-apply" not in job_names assert "merge-trim-concat" not in job_names assert "pangenome-sv" in job_names @@ -513,7 +513,7 @@ def test_multiple_lr_inputs(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" in cmd_str assert r"lr-rg2=ID:lr-rg2\tSM:sample1\tLR:1" in cmd_str assert str(lr_bam2) in cmd_str @@ -526,7 +526,7 @@ def test_rgsm_overrides_sm(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:override_sm\tLR:1" in cmd_str bwa_cmd = str(self._get_job(all_jobs, "bwa-extract").shell) @@ -548,7 +548,7 @@ def test_aligned_dag_jobs(self): "graph-update", "mm2-lift", "pangenome-sv", - "dnascope-raw", + "dnascope", "model-apply", ): assert name in job_names, f"missing job: {name}" @@ -619,7 +619,7 @@ def test_aligned_dag_dependencies(self): "extract-kmc" } assert "extract-kmc" in self._get_dep_names(dag, all_jobs, "mm2-lift") - assert self._get_dep_names(dag, all_jobs, "dnascope-raw") == { + assert self._get_dep_names(dag, all_jobs, "dnascope") == { "mm2-lift" } @@ -646,7 +646,7 @@ def test_aligned_replace_rg(self): sr_arg = r"sr-rg1=ID:sr-rg1\tSM:sample1\tLR:0" lr_arg = r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert cmd_str.count("--replace_rg") == 2 # Each row precedes the input file it applies to assert cmd_str.index(sr_arg) < cmd_str.index(str(self.mock_sr_bam)) @@ -669,7 +669,7 @@ def test_aligned_bam_format(self): lift_bam = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.bam") assert lift_bam in str(self._get_job(all_jobs, "mm2-lift").shell) - assert lift_bam in str(self._get_job(all_jobs, "dnascope-raw").shell) + assert lift_bam in str(self._get_job(all_jobs, "dnascope").shell) # Short-read input validation @@ -753,19 +753,19 @@ def test_lr_realign_downstream(self): "graph-update-raw", "graph-update", "pangenome-sv", - "dnascope-raw", + "dnascope", ): cmd_str = str(self._get_job(all_jobs, job_name).shell) assert realigned in cmd_str, job_name assert str(self.mock_lr_bam) not in cmd_str, job_name - for job_name in ("longreadsv", "graph-update-raw", "dnascope-raw"): + for job_name in ("longreadsv", "graph-update-raw", "dnascope"): assert "bam-realign-0" in self._get_dep_names( dag, all_jobs, job_name ), job_name # The readgroups of the realigned input are unchanged - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" in cmd_str def test_lr_realign_kmc_reads_original_input(self): @@ -803,13 +803,13 @@ def test_aligned_sr_with_lr_realign(self): realigned = str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") lift_cram = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.cram") - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert ( cmd_str.index(str(self.mock_sr_bam)) < cmd_str.index(lift_cram) < cmd_str.index(realigned) ) - assert self._get_dep_names(dag, all_jobs, "dnascope-raw") == { + assert self._get_dep_names(dag, all_jobs, "dnascope") == { "mm2-lift", "bam-realign-0", } diff --git a/tests/unit/test_sentieon_pangenome.py b/tests/unit/test_sentieon_pangenome.py index f1d459e..5d17bb3 100644 --- a/tests/unit/test_sentieon_pangenome.py +++ b/tests/unit/test_sentieon_pangenome.py @@ -236,10 +236,10 @@ def test_gvcf_adds_gvcftyper_and_routes_outputs(self): assert apply_gvcf.endswith("sample-snv_apply.g.vcf.gz") # DNAscope emits gVCF; model-apply writes gVCF - for name in ("dnascope-raw", "model-apply"): + for name in ("dnascope", "model-apply"): job = next(j for j in all_jobs if j.name == name) cmd_str = str(job.shell) - if name == "dnascope-raw": + if name == "dnascope": assert "--emit_mode gvcf" in cmd_str else: assert apply_gvcf in cmd_str @@ -346,18 +346,18 @@ def test_call_svs(self): assert isinstance(dag, DAG) - # The dnascope-raw job should exist + # The dnascope job should exist all_jobs = list(dag.waiting_jobs.keys()) + list( dag.ready_jobs.keys() ) job_names = [job.name for job in all_jobs] - assert "dnascope-raw" in job_names + assert "dnascope" in job_names - # Find the dnascope-raw job and check its command includes + # Find the dnascope job and check its command includes # both DNAscope and PangenomeSV dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -382,7 +382,7 @@ def test_call_svs_disabled_by_default(self): ) dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -402,7 +402,7 @@ def test_skip_small_variants(self): dag.ready_jobs.keys() ) job_names = [job.name for job in all_jobs] - assert "dnascope-raw" not in job_names + assert "dnascope" not in job_names assert "model-apply" not in job_names assert "merge-trim-concat" not in job_names @@ -421,7 +421,7 @@ def test_skip_small_variants_with_call_svs(self): job_names = [job.name for job in all_jobs] # The driver job should still run for SV calling - assert "dnascope-raw" in job_names + assert "dnascope" in job_names # Transfer and model-apply should be skipped assert "model-apply" not in job_names @@ -430,7 +430,7 @@ def test_skip_small_variants_with_call_svs(self): # The driver command should have PangenomeSV but NOT DNAscope dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -612,8 +612,8 @@ def test_call_svs_without_cnv_model(self): job_names, all_jobs = self._get_all_job_names(dag) # PangenomeSV still runs - assert "dnascope-raw" in job_names - dnascope_job = next(j for j in all_jobs if j.name == "dnascope-raw") + assert "dnascope" in job_names + dnascope_job = next(j for j in all_jobs if j.name == "dnascope") cmd_str = str(dnascope_job.shell) assert "--algo PangenomeSV" in cmd_str diff --git a/tests/unit/test_stage_small_variants.py b/tests/unit/test_stage_small_variants.py new file mode 100644 index 0000000..2a561ec --- /dev/null +++ b/tests/unit/test_stage_small_variants.py @@ -0,0 +1,385 @@ +""" +Unit tests for the small-variant stages +""" + +import pathlib + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import DNAscope, SVSolver +from sentieon_cli.shard import Shard +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.small_variants import ( + ApplySpec, + DNAscopeStage, + DriverStage, + GVCFtyperStage, + ModelApplyStage, + TransferApplyStage, + TransferSpec, +) +from sentieon_cli.stages.transfer import TransferConfig + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + # `build_transfer_jobs` shells out to bcftools unless this is set + dry_run=True, + skip_version_check=True, + ) + + +def make_call_stage(tmp_path: pathlib.Path, **kwargs) -> DNAscopeStage: + """A DNAscopeStage calling SNVs and indels""" + defaults = dict( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + inputs=[tmp_path / "sample.cram"], + ) + defaults.update(kwargs) + return DNAscopeStage(**defaults) # type: ignore[arg-type] + + +def make_transfer_config(tmp_path: pathlib.Path) -> TransferConfig: + """One shard the pop VCF has, one it lacks""" + return TransferConfig( + pop_vcf=tmp_path / "pop.vcf.gz", + shards=[Shard("chr1", 0, 1000), Shard("chrUn_extra", 0, 500)], + pop_vcf_contigs={"chr1": 1000}, + fai_data={ + "chr1": {"length": 1000}, + "chrUn_extra": {"length": 500}, + }, + ) + + +class TestDriverStage: + """The generic single-job driver stage""" + + def test_minimal_command(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[ + SVSolver( + output=tmp_path / "sv.vcf.gz", + vcf=tmp_path / "sv_tmp.vcf.gz", + ) + ], + name="svsolver", + task_name="sv-calling", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo SVSolver " + f"--vcf {tmp_path}/sv_tmp.vcf.gz {tmp_path}/sv.vcf.gz" + ) + + def test_name_task_name_and_default_threads(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + name="a-name", + task_name="a-task", + ).add_to(DAG()) + + assert result.job.name == "a-name" + assert result.job.task_name == "a-task" + assert result.job.threads == 4 + + def test_threads_override_leaves_the_driver_alone(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + name="a-name", + task_name="a-task", + threads=1, + ).add_to(DAG()) + + assert result.job.threads == 1 + assert "--thread_count 4" in str(result.job.shell) + + def test_result_fields(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + + +class TestDNAscopeStage: + """The DNAscope calling pass""" + + def test_default_name_and_task_name(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "dnascope" + assert result.job.task_name == "variant-calling" + + def test_inputs_and_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_call_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + interval=bed, + ).add_to(DAG()) + + shell = str(result.job.shell) + assert f"--input {tmp_path}/one.cram" in shell + assert f"--input {tmp_path}/two.cram" in shell + assert f"--interval {bed}" in shell + + def test_interval_padding_is_absent_unless_set(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert "--interval_padding" not in str(result.job.shell) + + def test_interval_padding(self, tmp_path): + result = make_call_stage(tmp_path, interval_padding=100).add_to(DAG()) + + assert "--interval_padding 100" in str(result.job.shell) + + def test_read_filter(self, tmp_path): + result = make_call_stage( + tmp_path, read_filter=["UltimaReadFilter"] + ).add_to(DAG()) + + assert "--read_filter UltimaReadFilter" in str(result.job.shell) + + def test_empty_read_filter_emits_nothing(self, tmp_path): + result = make_call_stage(tmp_path, read_filter=[]).add_to(DAG()) + + assert "--read_filter" not in str(result.job.shell) + + def test_replace_rg_precedes_its_input(self, tmp_path): + result = make_call_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + replace_rg=[["rg1=rg1-lr"], ["rg2=rg2-lr"]], + ).add_to(DAG()) + + assert str(result.job.shell).startswith( + f"sentieon driver --replace_rg rg1=rg1-lr " + f"--input {tmp_path}/one.cram " + f"--replace_rg rg2=rg2-lr --input {tmp_path}/two.cram" + ) + + def test_algos_keep_their_order(self, tmp_path): + result = make_call_stage( + tmp_path, + algos=[ + DNAscope(tmp_path / "raw.vcf.gz"), + DNAscope(tmp_path / "sv_tmp.vcf.gz", var_type="BND"), + ], + ).add_to(DAG()) + + shell = str(result.job.shell) + assert shell.index("raw.vcf.gz") < shell.index("--var_type BND") + + def test_upstream_becomes_the_dependency(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_call_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + +class TestModelApplyStage: + """DNAModelApply""" + + def test_command(self, tmp_path): + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "bundle" / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=tmp_path / "applied.vcf.gz", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo DNAModelApply " + f"--model {tmp_path}/bundle/dnascope.model " + f"--vcf {tmp_path}/raw.vcf.gz " + f"{tmp_path}/applied.vcf.gz" + ) + + def test_defaults_and_result(self, tmp_path): + output = tmp_path / "applied.vcf.gz" + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=output, + ).add_to(DAG()) + + assert result.job.name == "model-apply" + assert result.job.task_name == "model-apply" + assert result.job.threads == 4 + assert result.output == output + assert result.terminal == {result.job} + + def test_renamed_job(self, tmp_path): + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=tmp_path / "applied.vcf.gz", + name="snv-apply", + ).add_to(DAG()) + + assert result.job.name == "snv-apply" + + +class TestGVCFtyperStage: + """Genotyping a gVCF""" + + def test_command_without_an_interval(self, tmp_path): + result = GVCFtyperStage( + ctx=make_ctx(tmp_path), + gvcf=tmp_path / "sample.g.vcf.gz", + output=tmp_path / "sample.vcf.gz", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo GVCFtyper " + f"--vcf {tmp_path}/sample.g.vcf.gz " + f"{tmp_path}/sample.vcf.gz" + ) + assert result.job.name == "gvcftyper" + assert result.job.task_name == "gvcftyper" + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = GVCFtyperStage( + ctx=make_ctx(tmp_path), + gvcf=tmp_path / "sample.g.vcf.gz", + output=tmp_path / "sample.vcf.gz", + interval=bed, + ).add_to(DAG()) + + assert f"--interval {bed}" in str(result.job.shell) + assert result.output == tmp_path / "sample.vcf.gz" + + +class TestTransferApplyStage: + """Transfer, apply, or both""" + + def test_neither_half_is_an_error(self, tmp_path): + with pytest.raises(ValueError): + TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + ).add_to(DAG()) + + def test_apply_only_reads_the_raw_vcf(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.transfer is None + assert result.apply_job is not None + assert f"--vcf {tmp_path}/raw.vcf.gz" in str(result.apply_job.shell) + assert dag.waiting_jobs[result.apply_job] == {upstream} + assert result.terminal == {result.apply_job} + assert result.output == tmp_path / "applied.vcf.gz" + + def test_transfer_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.apply_job is None + assert result.transfer is not None + assert result.terminal == {result.transfer.concat_job} + assert result.output == tmp_path / "transfer.vcf.gz" + for job in result.transfer.shard_jobs: + assert dag.waiting_jobs[job] == {upstream} + + def test_both_halves(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + ), + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.transfer is not None + assert result.apply_job is not None + # DNAModelApply reads the transfer output + assert f"--vcf {tmp_path}/transfer.vcf.gz" in str( + result.apply_job.shell + ) + # ... and waits on both the transfer and the raw-VCF producer + assert dag.waiting_jobs[result.apply_job] == { + upstream, + result.transfer.concat_job, + } + assert result.terminal == {result.apply_job} + assert result.jobs == [ + *result.transfer.jobs, + result.apply_job, + ] + + def test_transfer_tag_reaches_the_job_names(self, tmp_path): + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + tag="hp", + ), + ).add_to(DAG()) + + assert result.transfer is not None + assert result.transfer.concat_job.name == "merge-trim-hp-concat" + + def test_apply_name_override(self, tmp_path): + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + name="snv-apply", + ), + ).add_to(DAG()) + + assert result.apply_job is not None + assert result.apply_job.name == "snv-apply" diff --git a/tests/unit/test_stage_transfer.py b/tests/unit/test_stage_transfer.py new file mode 100644 index 0000000..ce3d807 --- /dev/null +++ b/tests/unit/test_stage_transfer.py @@ -0,0 +1,133 @@ +""" +Unit tests for the annotation-transfer stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.shard import Shard +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.transfer import TransferConfig, TransferStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + # `build_transfer_jobs` shells out to bcftools unless this is set + dry_run=True, + skip_version_check=True, + ) + + +def make_config(tmp_path: pathlib.Path) -> TransferConfig: + """Two shards on a contig the pop VCF has, one on a contig it lacks""" + return TransferConfig( + pop_vcf=tmp_path / "pop.vcf.gz", + shards=[ + Shard("chr1", 0, 1000), + Shard("chr1", 1000, 2000), + Shard("chrUn_extra", 0, 500), + ], + pop_vcf_contigs={"chr1": 2000}, + fai_data={ + "chr1": {"length": 2000}, + "chrUn_extra": {"length": 500}, + }, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> TransferStage: + """A TransferStage over `make_config`""" + defaults = dict( + ctx=make_ctx(tmp_path), + config=make_config(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + out_vcf=tmp_path / "transfer.vcf.gz", + ) + defaults.update(kwargs) + return TransferStage(**defaults) # type: ignore[arg-type] + + +class TestJobs: + """The jobs the fan-out produces""" + + def test_one_job_per_shard_plus_the_extra_contig(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert [job.name for job in result.shard_jobs] == [ + "merge-trim-0", + "merge-trim-1", + "merge-trim-extra", + ] + assert result.concat_job.name == "merge-trim-concat" + + def test_task_name_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "annotation-transfer" + for job in result.shard_jobs: + assert job.threads == 1 + # The concat runs with every core + assert result.concat_job.threads == 4 + + def test_shard_jobs_read_the_raw_vcf(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.shard_jobs: + assert str(tmp_path / "raw.vcf.gz") in str(job.shell) + + def test_concat_writes_the_output(self, tmp_path): + out_vcf = tmp_path / "annotated.vcf.gz" + result = make_stage(tmp_path, out_vcf=out_vcf).add_to(DAG()) + + assert str(out_vcf) in str(result.concat_job.shell) + assert result.out_vcf == out_vcf + + +class TestTag: + """`tag` disambiguates two fan-outs in one run""" + + def test_tagged_names(self, tmp_path): + result = make_stage(tmp_path, tag="hp").add_to(DAG()) + + assert [job.name for job in result.shard_jobs] == [ + "merge-trim-hp-0", + "merge-trim-hp-1", + "merge-trim-hp-extra", + ] + assert result.concat_job.name == "merge-trim-hp-concat" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_shards_depend_on_upstream_and_concat_on_the_shards( + self, tmp_path + ): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + for job in result.shard_jobs: + assert dag.waiting_jobs[job] == {upstream} + assert dag.waiting_jobs[result.concat_job] == set(result.shard_jobs) + + def test_shards_are_roots_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + for job in result.shard_jobs: + assert job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [*result.shard_jobs, result.concat_job] + assert result.terminal == {result.concat_job} From 11bf13c37b4864468ab2a6238d1bf5d795efe878 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 01:16:44 -0700 Subject: [PATCH 06/20] Insert the hybrid calling jobs directly into the DAG `DNAscopeHybridPipeline.call_variants` returned its 23 jobs as a tuple for `build_dag` to wire by hand. `add_variant_calling` now builds each job and inserts it with its dependencies as it goes, using `DNAscopeStage`, `ModelApplyStage`, `TransferStage` and the `driver_job` / `rm_job` helpers where they fit. Cleanup jobs are simply not created under `--retain_tmpdir`. The two DNAscope passes are now named `dnascope-1` and `dnascope-2` (were `calling-1` and `call2`). Commands and dependency edges are unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope_hybrid.py | 446 +++++++------------ tests/integration/test_pipeline_execution.py | 2 +- tests/unit/test_dnascope_hybrid_stage1.py | 22 + 3 files changed, 186 insertions(+), 284 deletions(-) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 4036279..4e35685 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -7,7 +7,7 @@ import json import pathlib import sys -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple import packaging.version @@ -18,10 +18,8 @@ from . import command_strings as cmds from .dag import DAG from .driver import ( - BaseDriver, Driver, DNAscope, - DNAModelApply, HybridStage1, HybridStage2, HybridStage3, @@ -31,8 +29,11 @@ from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline +from .stages.base import StageContext, driver_job, rm_job from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage from .stages.ploidy import PloidyStage +from .stages.small_variants import DNAscopeStage, ModelApplyStage +from .stages.transfer import TransferConfig, TransferStage from .util import ( __version__, check_version, @@ -48,7 +49,6 @@ parse_fai, vcf_contigs, ) -from .transfer import build_transfer_jobs logger = get_logger(__name__) @@ -659,66 +659,14 @@ def build_dag(self) -> DAG: else None ) - ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat, - apply_job, - norm_job, - ) = self.call_variants(sr_aln, lr_aln, rg_info) - dag.add_job(call_job, realign_jobs | sr_preprocessing_jobs) - dag.add_job(select_job, {call_job}) - dag.add_job(mapq0_job, realign_jobs | sr_preprocessing_jobs) - dag.add_job(mapq0_slop_job, {mapq0_job}) - dag.add_job(cat_merge_job, {mapq0_slop_job, select_job}) - # The haplotype job writes the fifo read by the first-stage job - dag.add_job(stage1_fifo_job) - dag.add_job(stage1_hap_job, {stage1_fifo_job, cat_merge_job}) - dag.add_job(stage1_job, {stage1_fifo_job, cat_merge_job}) - dag.add_job(second_stage_job, {stage1_job, stage1_hap_job}) - dag.add_job(third_stage_job, {second_stage_job}) - dag.add_job(call2_job, {third_stage_job}) - dag.add_job(subset_job, {second_stage_job}) - dag.add_job(concat_job, {subset_job, call2_job}) - dag.add_job(anno_job, {concat_job}) - - apply_dependencies = {anno_job} - if transfer_jobs and transfer_concat: - for job in transfer_jobs: - dag.add_job(job, {anno_job}) - dag.add_job(transfer_concat, set(transfer_jobs)) - apply_dependencies = {transfer_concat} - - if apply_job: - dag.add_job(apply_job, apply_dependencies) - if apply_job and norm_job: - dag.add_job(norm_job, {apply_job}) - - # Remove intermediate files during processing - if not self.retain_tmpdir: - dag.add_job(rm_job1, {cat_merge_job}) - dag.add_job(rm_job2, {stage1_job, stage1_hap_job}) - dag.add_job(rm_job3, {second_stage_job}) - dag.add_job(rm_job4, {third_stage_job}) - dag.add_job(rm_job5, {concat_job}) + self.add_variant_calling( + dag, + ctx, + sr_aln, + lr_aln, + rg_info, + realign_jobs | sr_preprocessing_jobs, + ) return dag @@ -757,39 +705,52 @@ def build_second_dag(self) -> Optional[DAG]: ).add_to(dag) return dag - def call_variants( + def transfer_config(self) -> TransferConfig: + """The inputs the annotation-transfer stage needs. + + Only valid once `--pop_vcf` is known to be set; the calling code + checks it first. + """ + assert self.pop_vcf is not None + return TransferConfig( + pop_vcf=self.pop_vcf, + shards=self.shards, + pop_vcf_contigs=self.pop_vcf_contigs, + fai_data=self.fai_data, + ) + + def add_cleanup( + self, + dag: DAG, + paths: List[pathlib.Path], + name: str, + upstream: Set[Job], + ) -> None: + """Remove intermediate files once `upstream` has finished. + + A no-op under `--retain_tmpdir`, which keeps everything the run + wrote in the temporary directory. + """ + if self.retain_tmpdir: + return + dag.add_job(rm_job(paths, name), upstream) + + def add_variant_calling( self, + dag: DAG, + ctx: StageContext, sr_aln: List[pathlib.Path], lr_aln: List[pathlib.Path], rg_info: RgInfo, - **_kwargs: Any, - ) -> Tuple[ - Job, # call_job - Job, # select_job - Job, # mapq0_job - Job, # mapq0_slop_job - Job, # cat_merge_job - Job, # rm_job1 - Job, # stage1_fifo_job - Job, # stage1_hap_job - Job, # stage1_job - Job, # rm_job2 - Job, # second_stage_job - Job, # rm_job3 - Job, # third_stage_job - Job, # rm_job4 - Job, # call2_job - Job, # subset_job - Job, # concat_job - Job, # rm_job5 - Job, # anno_job - Optional[List[Job]], # transfer_jobs - Optional[Job], # transfer_concat_job - Optional[Job], # apply_job - Optional[Job], # norm_job - ]: + upstream: Set[Job], + ) -> None: """ Call SNVs and indels using the DNAscope hybrid pipeline + + A first calling pass over the input alignments picks the regions + that need realignment; the three hybrid-realignment stages realign + the short reads there; a second pass recalls those regions, and the + two call sets are merged, annotated and filtered. """ if not self.output_vcf: self.logger.error("output_vcf is required") @@ -808,35 +769,31 @@ def call_variants( if not check_version(cmd, min_version): sys.exit(2) + hybrid_model = self.model_bundle.joinpath("hybrid.model") + # First pass - combined variant calling vcf_suffix = ".g.vcf.gz" if self.gvcf else ".vcf.gz" combined_vcf = self.tmp_dir.joinpath("initial" + vcf_suffix) emit_mode = "gvcf" if self.gvcf else None - driver: BaseDriver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], - input=lr_aln + sr_aln, + call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + combined_vcf, + dbsnp=self.dbsnp, + model=hybrid_model, + pcr_indel_model="none", + emit_mode=emit_mode, + ) + ], + inputs=lr_aln + sr_aln, interval=self.bed, read_filter=rg_info.ultima_read_filter + rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, - ) - driver.add_algo( - DNAscope( - combined_vcf, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("hybrid.model"), - pcr_indel_model="none", - emit_mode=emit_mode, - ) - ) - call_job = Job( - Pipeline(Command(*driver.build_cmd())), - "calling-1", - self.cores, - task_name="variant-calling", - ) + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], + name="dnascope-1", + ).add_to(dag, upstream) # Region selection hybrid_select = pathlib.Path( @@ -855,27 +812,26 @@ def call_variants( 0, task_name="region-selection", ) + dag.add_job(select_job, call.terminal) mapq0_bed = self.tmp_dir.joinpath("hybrid_mapq0.bed") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], - input=lr_aln + sr_aln, - read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, - ) - driver.add_algo( - HybridStage2( - model=self.model_bundle.joinpath("HybridStage2_region.model"), - all_bed=mapq0_bed, - ) - ) - mapq0_job = Job( - Pipeline(Command(*driver.build_cmd())), - "mapq0-bed", - self.cores, + mapq0_job = driver_job( + ctx, + [ + HybridStage2( + model=self.model_bundle.joinpath( + "HybridStage2_region.model" + ), + all_bed=mapq0_bed, + ) + ], + name="mapq0-bed", task_name="region-selection", + inputs=lr_aln + sr_aln, + read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], ) + dag.add_job(mapq0_job, upstream) mapq0_slop_bed = self.tmp_dir.joinpath("hybrid_mapq0.ex1000.bed") mapq0_slop_job = Job( @@ -889,6 +845,7 @@ def call_variants( 0, task_name="region-selection", ) + dag.add_job(mapq0_slop_job, {mapq0_job}) diff_bed = self.tmp_dir.joinpath("merged_diff.bed") cat_merge_job = Job( @@ -901,12 +858,12 @@ def call_variants( 0, task_name="region-selection", ) - rm_cmd = ["rm", str(selected_bed), str(mapq0_slop_bed)] - rm_job1 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(cat_merge_job, {mapq0_slop_job, select_job}) + self.add_cleanup( + dag, + [selected_bed, mapq0_slop_bed], "rm-tmp1", - 0, - task_name="cleanup", + {cat_merge_job}, ) stage1_ins_fa = self.tmp_dir.joinpath("stage1_ins.fa") @@ -940,6 +897,8 @@ def call_variants( 1, task_name="hybrid-realignment", ) + # The haplotype job writes the fifo read by the first-stage job + dag.add_job(stage1_fifo_job) stage1_driver = Driver( reference=self.reference, @@ -973,6 +932,7 @@ def call_variants( 0, task_name="hybrid-realignment", ) + dag.add_job(stage1_hap_job, {stage1_fifo_job, cat_merge_job}) stage1_bam = self.tmp_dir.joinpath("hybrid_stage1.bam") stage1_job = Job( @@ -989,56 +949,45 @@ def call_variants( self.cores, task_name="hybrid-realignment", ) - rm_cmd = [ - "rm", - str(stage1_ins_fa), - str(stage1_ins_bed), - str(stage1_hap_vcf), - ] - rm_job2 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(stage1_job, {stage1_fifo_job, cat_merge_job}) + self.add_cleanup( + dag, + [stage1_ins_fa, stage1_ins_bed, stage1_hap_vcf], "rm-tmp2", - 0, - task_name="cleanup", + {stage1_job, stage1_hap_job}, ) stage2_bed = self.tmp_dir.joinpath("hybrid_stage2.bed") stage2_unmap_bam = self.tmp_dir.joinpath("hybrid_stage2_unmap.bam") stage2_alt_bam = self.tmp_dir.joinpath("hybrid_stage2_alt.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=[stage1_bam, stage1_hap_bam], - ) - driver.add_algo( - HybridStage2( - model=self.model_bundle.joinpath("HybridStage2.model"), - hap_bed=stage1_hap_bed, - unmap_bam=stage2_unmap_bam, - alt_bam=stage2_alt_bam, - all_bed=stage2_bed, - ) - ) - second_stage_job = Job( - Pipeline(Command(*driver.build_cmd())), - "second-stage", - self.cores, + second_stage_job = driver_job( + ctx, + [ + HybridStage2( + model=self.model_bundle.joinpath("HybridStage2.model"), + hap_bed=stage1_hap_bed, + unmap_bam=stage2_unmap_bam, + alt_bam=stage2_alt_bam, + all_bed=stage2_bed, + ) + ], + name="second-stage", task_name="hybrid-realignment", + inputs=[stage1_bam, stage1_hap_bam], ) - - rm_cmd = ["rm", str(stage1_bam), str(stage1_hap_bam)] - rm_job3 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(second_stage_job, {stage1_job, stage1_hap_job}) + self.add_cleanup( + dag, + [stage1_bam, stage1_hap_bam], "rm-tmp3", - 0, - task_name="cleanup", + {second_stage_job}, ) suffix = "bam" if self.bam_format else "cram" stage3_aln = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", f"_sr_realigned.{suffix}") ) - driver = Driver( + stage3_driver = Driver( reference=self.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], @@ -1046,7 +995,7 @@ def call_variants( interval=stage2_bed, read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, ) - driver.add_algo( + stage3_driver.add_algo( HybridStage3( "-", model=self.model_bundle.joinpath("HybridStage3.model"), @@ -1056,46 +1005,40 @@ def call_variants( cmds.hybrid_stage3( stage3_aln, reference=self.reference, - driver=driver, + driver=stage3_driver, cores=self.cores, ), "third-stage", self.cores, task_name="hybrid-realignment", ) - rm_cmd = ["rm", str(stage2_unmap_bam), str(stage2_alt_bam)] - rm_job4 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(third_stage_job, {second_stage_job}) + self.add_cleanup( + dag, + [stage2_unmap_bam, stage2_alt_bam], "rm-tmp4", - 0, - task_name="cleanup", + {third_stage_job}, ) # pass 2 of variant calling pass2_vcf = self.tmp_dir.joinpath("hybrid_pass2.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0], - input=lr_aln + [stage3_aln], + call2 = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + pass2_vcf, + dbsnp=self.dbsnp, + model=hybrid_model, + pcr_indel_model="none", + emit_mode=emit_mode, + ) + ], + inputs=lr_aln + [stage3_aln], interval=stage2_bed, read_filter=rg_info.ultima_read_filter + rg_info.lr_rg_read_filter, - ) - driver.add_algo( - DNAscope( - pass2_vcf, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("hybrid.model"), - pcr_indel_model="none", - emit_mode=emit_mode, - ) - ) - call2_job = Job( - Pipeline(Command(*driver.build_cmd())), - "call2", - self.cores, - task_name="variant-calling", - ) + replace_rg=rg_info.replace_rg_args[0], + name="dnascope-2", + ).add_to(dag, {third_stage_job}) # Merge and normalize the VCFs subset_vcf = self.tmp_dir.joinpath("mix_subset.vcf.gz") @@ -1110,6 +1053,7 @@ def call_variants( 0, task_name="vcf-merge", ) + dag.add_job(subset_job, {second_stage_job}) concat_job = Job( cmds.bcftools_concat( combined_tmp_vcf, @@ -1119,12 +1063,12 @@ def call_variants( 0, task_name="vcf-merge", ) - rm_cmd = ["rm", str(combined_vcf), str(subset_vcf), str(pass2_vcf)] - rm_job5 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(concat_job, {subset_job} | call2.terminal) + self.add_cleanup( + dag, + [combined_vcf, subset_vcf, pass2_vcf], "rm-tmp5", - 0, - task_name="cleanup", + {concat_job}, ) # Annotate the output VCF @@ -1147,11 +1091,10 @@ def call_variants( 0, task_name="annotation", ) + dag.add_job(anno_job, {concat_job}) - transfer_jobs: Optional[List[Job]] = None - transfer_concat_job: Optional[Job] = None input_to_apply = anno_target - + apply_dependencies: Set[Job] = {anno_job} if self.pop_vcf: transfer_target = self.tmp_dir.joinpath( "combined_tmp_transfer.vcf.gz" @@ -1159,65 +1102,26 @@ def call_variants( if self.skip_model_apply: transfer_target = self.output_vcf - transfer_jobs, transfer_concat_job = build_transfer_jobs( - transfer_target, - self.pop_vcf, - anno_target, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, - ) + transfer = TransferStage( + ctx=ctx, + config=self.transfer_config(), + raw_vcf=anno_target, + out_vcf=transfer_target, + ).add_to(dag, {anno_job}) input_to_apply = transfer_target + apply_dependencies = transfer.terminal if self.skip_model_apply: - return ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat_job, - None, - None, - ) + return # Model Apply apply_vcf = self.tmp_dir.joinpath("combined_apply.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("hybrid.model"), - vcf=input_to_apply, - output=apply_vcf, - ) - ) - apply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) + apply = ModelApplyStage( + ctx=ctx, + model=hybrid_model, + vcf=input_to_apply, + output=apply_vcf, + ).add_to(dag, apply_dependencies) # Final normalize norm_job = Job( @@ -1231,28 +1135,4 @@ def call_variants( 0, task_name="vcf-norm", ) - return ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat_job, - apply_job, - norm_job, - ) + dag.add_job(norm_job, apply.terminal) diff --git a/tests/integration/test_pipeline_execution.py b/tests/integration/test_pipeline_execution.py index 3f89de3..9ed9e6a 100644 --- a/tests/integration/test_pipeline_execution.py +++ b/tests/integration/test_pipeline_execution.py @@ -152,7 +152,7 @@ def test_hybrid_dry_run_execution_rgsm(self): all_jobs = [job for job in list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys())] # with --rgsm should use replace_rg during first pass - first_pass_job = [job for job in all_jobs if job.name == "calling-1"][0] + first_pass_job = [job for job in all_jobs if job.name == "dnascope-1"][0] assert "--replace_rg" in first_pass_job.shell.nodes[0].args # CNV calling is sex-aware and runs in the second DAG diff --git a/tests/unit/test_dnascope_hybrid_stage1.py b/tests/unit/test_dnascope_hybrid_stage1.py index 6b72aeb..231fe8c 100644 --- a/tests/unit/test_dnascope_hybrid_stage1.py +++ b/tests/unit/test_dnascope_hybrid_stage1.py @@ -170,6 +170,28 @@ def test_stage1_dependencies(self): "first-stage-hap", } + def test_cleanup_jobs(self): + """Each cleanup job waits for the jobs writing what it removes""" + pipeline = self.create_pipeline() + dag, jobs = self.build_dag(pipeline) + + assert self.dep_names(dag, jobs["rm-tmp1"]) == {"concat-merge-bed"} + assert self.dep_names(dag, jobs["rm-tmp2"]) == { + "first-stage", + "first-stage-hap", + } + assert self.dep_names(dag, jobs["rm-tmp3"]) == {"second-stage"} + assert self.dep_names(dag, jobs["rm-tmp4"]) == {"third-stage"} + assert self.dep_names(dag, jobs["rm-tmp5"]) == {"concat-calls"} + + def test_retain_tmpdir_skips_cleanup(self): + """`--retain_tmpdir` keeps the intermediate files""" + pipeline = self.create_pipeline() + pipeline.retain_tmpdir = True + _dag, jobs = self.build_dag(pipeline) + + assert [name for name in jobs if name.startswith("rm-tmp")] == [] + def test_second_stage_inputs(self): """The second stage consumes the sorted haplotype BAM""" pipeline = self.create_pipeline() From a82a97321718efd1c0f1d1a4d07f699506edb0d0 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 01:27:26 -0700 Subject: [PATCH 07/20] Insert the long-read calling jobs directly into the DAG `DNAscopeLRPipeline.lr_call_variants` returned a 27-field dataclass of jobs that `build_dag` wired by hand. `add_small_variant_calling` now composes the small-variant stages and inserts each job with its dependencies as it goes: `DNAscopeStage` for the five calling passes, `TransferApplyStage` for the diploid and unphased tails, and `TransferStage` plus `ModelApplyStage` for the two haplotypes, whose model-apply jobs wait on both haplotype transfers. The passes are now named after the stage: `dnascope-diploid`, `dnascope-hap1`/`hap2`, `dnascope-unphased`, `dnascope-haploid`, with matching `model-apply-*` jobs, and the four annotation transfers carry `diploid`/`hap1`/`hap2`/`unphased` tags in their job names. Commands and dependency edges are unchanged; a new unit test pins the non-obvious edges. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope_longread.py | 791 +++++++------------ tests/unit/test_dnascope_longread_dag.py | 218 +++++ tests/unit/test_dnascope_longread_pop_vcf.py | 10 +- 3 files changed, 522 insertions(+), 497 deletions(-) create mode 100644 tests/unit/test_dnascope_longread_dag.py diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index ca4aeb0..c0ddf13 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -4,7 +4,6 @@ import argparse import copy -from dataclasses import dataclass import json import pathlib import shutil @@ -19,10 +18,10 @@ from .archive import ar_load from .dag import DAG from .driver import ( + BaseAlgo, Driver, DNAscope, DNAscopeHP, - DNAModelApply, LongReadSV, ReadWriter, RepeatModel, @@ -36,7 +35,15 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline -from .transfer import build_transfer_jobs +from .stages.base import StageContext, driver_job +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + ModelApplyStage, + TransferApplyStage, + TransferSpec, +) +from .stages.transfer import TransferConfig, TransferStage from .util import ( __version__, check_version, @@ -82,39 +89,6 @@ } -@dataclass -class LRCallVariantsResult: - """Results from lr_call_variants""" - - first_calling_job: Job - diploid_transfer_jobs: List[Job] - diploid_concat_job: Optional[Job] - first_modelapply_job: Job - phaser_job: Job - bcftools_subset_phased_job: Optional[Job] - fai_to_bed_job: Optional[Job] - bcftools_subtract_job: Job - repeatmodel_job: Optional[Job] - bcftools_subset_unphased_job: Job - second_calling_job: Set[Job] - haploid_patch_job: Job - patch_transfer_jobs: List[List[Job]] - patch_concat_jobs: List[Job] - second_modelapply_job: Set[Job] - calling_unphased_job: Job - diploid_patch_job: Job - unphased_transfer_jobs: List[Job] - unphased_concat_job: Optional[Job] - modelapply_unphased_job: Job - merge_job: Job - gvcf_combine_job: Optional[Job] - haploid_calling_job: Optional[Job] - haploid_patch2_job: Optional[Job] - haploid_concat_job: Optional[Job] - haploid_gvcf_combine_job: Optional[Job] - haploid_gvcf_concat_job: Optional[Job] - - class DNAscopeLRPipeline(BasePipeline): """The DNAscope LongRead pipeline""" @@ -468,6 +442,7 @@ def configure(self) -> None: def build_dag(self) -> DAG: self.logger.info("Building the DAG") dag = DAG() + ctx = self.stage_context() sample_input = self.lr_aln realign_jobs: Set[Job] = set() @@ -500,122 +475,10 @@ def build_dag(self) -> DAG: dag.add_job(hificnv_job, {merge_job}) if not self.skip_small_variants: - cv = self.lr_call_variants(sample_input) - dag.add_job(cv.first_calling_job, realign_jobs.union(align_jobs)) - - first_ma_deps = {cv.first_calling_job} - if cv.diploid_transfer_jobs and cv.diploid_concat_job: - for job in cv.diploid_transfer_jobs: - dag.add_job(job, {cv.first_calling_job}) - dag.add_job( - cv.diploid_concat_job, set(cv.diploid_transfer_jobs) - ) - first_ma_deps.add(cv.diploid_concat_job) - - dag.add_job(cv.first_modelapply_job, first_ma_deps) - dag.add_job(cv.phaser_job, {cv.first_modelapply_job}) - - haploid_patch_deps: Set[Job] = set() - if cv.bcftools_subset_phased_job: - dag.add_job(cv.bcftools_subset_phased_job, {cv.phaser_job}) - haploid_patch_deps.add(cv.bcftools_subset_phased_job) - - subtract_deps: Set[Job] = {cv.phaser_job} - if cv.fai_to_bed_job: - dag.add_job(cv.fai_to_bed_job) - subtract_deps.add(cv.fai_to_bed_job) - dag.add_job(cv.bcftools_subtract_job, subtract_deps) - dag.add_job( - cv.bcftools_subset_unphased_job, - {cv.bcftools_subtract_job}, + self.add_small_variant_calling( + dag, ctx, sample_input, realign_jobs.union(align_jobs) ) - second_pass_deps: Set[Job] = {cv.phaser_job} - calling_unphased_deps: Set[Job] = {cv.bcftools_subtract_job} - haploid_calling_deps: Set[Job] = set() - if cv.repeatmodel_job: - dag.add_job(cv.repeatmodel_job, {cv.phaser_job}) - second_pass_deps.add(cv.repeatmodel_job) - calling_unphased_deps.add(cv.repeatmodel_job) - haploid_calling_deps.add(cv.repeatmodel_job) - - merge_deps: Set[Job] = set() - for job in cv.second_calling_job: - dag.add_job(job, second_pass_deps) - haploid_patch_deps.add(job) - dag.add_job(cv.haploid_patch_job, haploid_patch_deps) - - second_ma_deps = {cv.haploid_patch_job} - if cv.patch_transfer_jobs: # pop_vcf - for patch_transfer_hap, patch_concat in zip( - cv.patch_transfer_jobs, cv.patch_concat_jobs - ): - for job in patch_transfer_hap: - dag.add_job(job, {cv.haploid_patch_job}) - dag.add_job(patch_concat, set(patch_transfer_hap)) - second_ma_deps.add(patch_concat) - - for job in cv.second_modelapply_job: - dag.add_job(job, second_ma_deps) - merge_deps.add(job) - - dag.add_job(cv.calling_unphased_job, calling_unphased_deps) - dag.add_job( - cv.diploid_patch_job, - { - cv.bcftools_subset_unphased_job, - cv.calling_unphased_job, - }, - ) - - unphased_ma_deps = {cv.diploid_patch_job} - if cv.unphased_transfer_jobs and cv.unphased_concat_job: # pop_vcf - for job in cv.unphased_transfer_jobs: - dag.add_job(job, {cv.diploid_patch_job}) - dag.add_job( - cv.unphased_concat_job, set(cv.unphased_transfer_jobs) - ) - unphased_ma_deps.add(cv.unphased_concat_job) - - dag.add_job(cv.modelapply_unphased_job, unphased_ma_deps) - merge_deps.add(cv.modelapply_unphased_job) - dag.add_job(cv.merge_job, merge_deps) - - if cv.gvcf_combine_job: - dag.add_job(cv.gvcf_combine_job, {cv.merge_job}) - - if ( - cv.haploid_calling_job - and cv.haploid_patch2_job - and cv.haploid_concat_job - ): - dag.add_job(cv.haploid_calling_job, haploid_calling_deps) - dag.add_job( - cv.haploid_patch2_job, - {cv.haploid_calling_job}, - ) - dag.add_job( - cv.haploid_concat_job, - {cv.haploid_patch2_job, cv.merge_job}, - ) - - if ( - cv.haploid_gvcf_combine_job - and cv.haploid_gvcf_concat_job - and cv.gvcf_combine_job - ): - dag.add_job( - cv.haploid_gvcf_combine_job, - {cv.haploid_patch2_job}, - ) - dag.add_job( - cv.haploid_gvcf_concat_job, - { - cv.haploid_gvcf_combine_job, - cv.gvcf_combine_job, - }, - ) - if not self.skip_svs: longreadsv_job = self.call_svs(sample_input) dag.add_job(longreadsv_job, realign_jobs.union(align_jobs)) @@ -888,12 +751,34 @@ def hificnv( ) return hificnv_job - def lr_call_variants( + def transfer_config(self) -> TransferConfig: + """The inputs the annotation-transfer stage needs. + + Only valid once `--pop_vcf` is known to be set; the calling code + checks it first. + """ + assert self.pop_vcf is not None + return TransferConfig( + pop_vcf=self.pop_vcf, + shards=self.shards, + pop_vcf_contigs=self.pop_vcf_contigs, + fai_data=self.fai_data, + ) + + def add_small_variant_calling( self, + dag: DAG, + ctx: StageContext, sample_input: List[pathlib.Path], - ) -> LRCallVariantsResult: + upstream: Set[Job], + ) -> None: """ Call SNVs and indels using the DNAscope LongRead pipeline + + A diploid first pass phases the reads; the phased regions are then + called once per haplotype and the unphased regions are called with + the diploid HP model. The three call sets are patched, filtered and + merged into the output VCF, with optional haploid calling on top. """ if not self.model_bundle: self.logger.error("model_bundle is required") @@ -909,26 +794,13 @@ def lr_call_variants( if not check_version(check_cmd, min_version): sys.exit(2) - transfer_xargs = ( - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, - ) - # First pass - diploid calling diploid_gvcf_fn = self.tmp_dir.joinpath("out_diploid.g.vcf.gz") diploid_tmp_vcf = self.tmp_dir.joinpath("out_diploid_tmp.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.bed, - ) + diploid_model = self.model_bundle.joinpath("diploid_model") + diploid_algos: List[BaseAlgo] = [] if self.gvcf: - driver.add_algo( + diploid_algos.append( DNAscope( diploid_gvcf_fn, dbsnp=self.dbsnp, @@ -936,54 +808,40 @@ def lr_call_variants( model=self.model_bundle.joinpath("gvcf_model"), ) ) - driver.add_algo( + diploid_algos.append( DNAscope( diploid_tmp_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("diploid_model"), + model=diploid_model, ) ) - first_calling_job = Job( - Pipeline(Command(*driver.build_cmd())), - "first-pass", - self.cores, - task_name="variant-calling", - ) - - # Transfer annotations to the tmp vcf - diploid_transfer_jobs: List[Job] = [] - diploid_concat_job: Optional[Job] = None - if self.pop_vcf: - diploid_unanno_tmp_vcf = diploid_tmp_vcf - diploid_tmp_vcf = self.tmp_dir.joinpath( - "out_diploid_tmp_anno.vcf.gz" - ) - diploid_transfer_jobs, diploid_concat_job = build_transfer_jobs( - diploid_tmp_vcf, - self.pop_vcf, - diploid_unanno_tmp_vcf, - *transfer_xargs, - ) + first_call = DNAscopeStage( + ctx=ctx, + algos=diploid_algos, + inputs=sample_input, + interval=self.bed, + name="dnascope-diploid", + ).add_to(dag, upstream) - # DNAModelApply on the diploid VCF + # Transfer annotations to the tmp vcf, then apply the diploid model diploid_vcf = self.tmp_dir.joinpath("out_diploid.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("diploid_model"), - diploid_tmp_vcf, - diploid_vcf, + diploid_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + diploid_transfer = TransferSpec( + config=self.transfer_config(), + out_vcf=self.tmp_dir.joinpath("out_diploid_tmp_anno.vcf.gz"), + tag="diploid", ) - ) - first_modelapply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "first-modelapply", - self.cores, - task_name="model-apply", - ) + first_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=diploid_tmp_vcf, + transfer=diploid_transfer, + apply=ApplySpec( + model=diploid_model, + output=diploid_vcf, + name="model-apply-diploid", + ), + ).add_to(dag, first_call.terminal) # Phasing and RepeatModel phased_bed = self.tmp_dir.joinpath("out_diploid_phased.bed") @@ -996,28 +854,24 @@ def lr_call_variants( phased_phased = self.tmp_dir.joinpath( "out_diploid_phased_phased.vcf.gz" ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.bed, - ) - driver.add_algo( - VariantPhaser( - diploid_vcf, - phased_vcf, - out_bed=phased_bed, - out_ext=phased_ext, - ) - ) - phaser_job = Job( - Pipeline(Command(*driver.build_cmd())), - "variantphaser", - self.cores, + phaser_job = driver_job( + ctx, + [ + VariantPhaser( + diploid_vcf, + phased_vcf, + out_bed=phased_bed, + out_ext=phased_ext, + ) + ], + name="variantphaser", task_name="phasing", + inputs=sample_input, + interval=self.bed, ) + dag.add_job(phaser_job, first_apply.terminal) - bcftools_subset_phased_job = None + haploid_patch_deps: Set[Job] = set() if self.tech.upper() == "ONT": bcftools_subset_phased_job = Job( Pipeline( @@ -1040,8 +894,10 @@ def lr_call_variants( 0, task_name="phasing", ) + dag.add_job(bcftools_subset_phased_job, {phaser_job}) + haploid_patch_deps.add(bcftools_subset_phased_job) - fai_to_bed_job = None + subtract_deps: Set[Job] = {phaser_job} if self.bed: bed = self.bed else: @@ -1055,6 +911,8 @@ def lr_call_variants( 0, task_name="phasing", ) + dag.add_job(fai_to_bed_job) + subtract_deps.add(fai_to_bed_job) bcftools_subtract_job = Job( cmds.cmd_bedtools_subtract(bed, phased_bed, unphased_bed), @@ -1062,33 +920,7 @@ def lr_call_variants( 0, task_name="phasing", ) - - repeatmodel_job = None - if not self.repeat_model: - self.repeat_model = self.tmp_dir.joinpath("out_repeat.model") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=phased_bed, - read_filter=[ - f"PhasedReadFilter,phased_vcf={phased_ext}," - "phase_select=tag" - ], # noqa: E501 - ) - driver.add_algo( - RepeatModel( - self.repeat_model, - phased=True, - read_flag_mask="drop=supplementary", - ) - ) - repeatmodel_job = Job( - Pipeline(Command(*driver.build_cmd())), - "repeatmodel", - self.cores, - task_name="repeat-model", - ) + dag.add_job(bcftools_subtract_job, subtract_deps) bcftools_subset_unphased_job = Job( Pipeline( @@ -1107,50 +939,70 @@ def lr_call_variants( 0, task_name="phasing", ) + dag.add_job(bcftools_subset_unphased_job, {bcftools_subtract_job}) + + repeat_model_deps: Set[Job] = set() + if not self.repeat_model: + self.repeat_model = self.tmp_dir.joinpath("out_repeat.model") + repeatmodel_job = driver_job( + ctx, + [ + RepeatModel( + self.repeat_model, + phased=True, + read_flag_mask="drop=supplementary", + ) + ], + name="repeatmodel", + task_name="repeat-model", + inputs=sample_input, + interval=phased_bed, + read_filter=[ + f"PhasedReadFilter,phased_vcf={phased_ext}," + "phase_select=tag" + ], # noqa: E501 + ) + dag.add_job(repeatmodel_job, {phaser_job}) + repeat_model_deps.add(repeatmodel_job) # Second pass - phased variants - second_calling_job = set() + haploid_model = self.model_bundle.joinpath("haploid_model") + haploid_hp_model = self.model_bundle.joinpath("haploid_hp_model") for phase in (1, 2): hp_std_vcf = self.tmp_dir.joinpath( f"out_hap{phase}_nohp_tmp.vcf.gz" ) hp_vcf = self.tmp_dir.joinpath(f"out_hap{phase}_tmp.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=phased_bed, - read_filter=[ - f"PhasedReadFilter,phased_vcf={phased_ext}" - f",phase_select={phase}" - ], - ) - + phase_algos: List[BaseAlgo] = [] if self.tech.upper() == "HIFI": # ONT doesn't do DNAscope in 2nd pass. - driver.add_algo( + phase_algos.append( DNAscope( hp_std_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_model"), + model=haploid_model, ) ) - driver.add_algo( + phase_algos.append( DNAscopeHP( hp_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_hp_model"), + model=haploid_hp_model, pcr_indel_model=self.repeat_model, ) ) - second_calling_job.add( - Job( - Pipeline(Command(*driver.build_cmd())), - "second-pass", - self.cores, - task_name="variant-calling", - ) - ) + hap_call = DNAscopeStage( + ctx=ctx, + algos=phase_algos, + inputs=sample_input, + interval=phased_bed, + read_filter=[ + f"PhasedReadFilter,phased_vcf={phased_ext}" + f",phase_select={phase}" + ], + name=f"dnascope-hap{phase}", + ).add_to(dag, {phaser_job} | repeat_model_deps) + haploid_patch_deps.update(hap_call.terminal) kwargs: Dict[str, str] = dict() kwargs["gvcf_combine_py"] = str( @@ -1177,128 +1029,108 @@ def lr_call_variants( self.cores, task_name="variant-patch", ) + dag.add_job(haploid_patch_job, haploid_patch_deps) - # Transfer annotations to the patched VCFs - patch_transfer_jobs = [] - patch_concat_jobs = [] + # Transfer annotations to the patched VCFs. Both model-apply jobs + # wait on both transfers, as they did when the DAG was wired by hand. + second_ma_deps: Set[Job] = {haploid_patch_job} if self.pop_vcf: patch_unanno_vcfs = patch_vcfs.copy() patch_vcfs = [ self.tmp_dir.joinpath(f"out_hap{i}_patch_anno.vcf.gz") for i in (1, 2) ] - for patch_unanno_vcf, patch_vcf in zip( - patch_unanno_vcfs, patch_vcfs + for i, (patch_unanno_vcf, patch_vcf) in enumerate( + zip(patch_unanno_vcfs, patch_vcfs), start=1 ): - transfer_jobs, concat_job = build_transfer_jobs( - patch_vcf, - self.pop_vcf, - patch_unanno_vcf, - *transfer_xargs, - ) - patch_transfer_jobs.append(transfer_jobs) - patch_concat_jobs.append(concat_job) + patch_transfer = TransferStage( + ctx=ctx, + config=self.transfer_config(), + raw_vcf=patch_unanno_vcf, + out_vcf=patch_vcf, + tag=f"hap{i}", + ).add_to(dag, {haploid_patch_job}) + second_ma_deps.update(patch_transfer.terminal) # apply trained model to the patched vcfs. hap_vcfs = [ self.tmp_dir.joinpath(f"out_hap{i}.vcf.gz") for i in (1, 2) ] - second_modelapply_job = set() - for patch_vcf, hap_vcf in zip(patch_vcfs, hap_vcfs): - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("haploid_model"), - patch_vcf, - hap_vcf, - ) - ) - second_modelapply_job.add( - Job( - Pipeline(Command(*driver.build_cmd())), - "second-modelapply", - self.cores, - task_name="model-apply", - ) - ) + merge_deps: Set[Job] = set() + for i, (patch_vcf, hap_vcf) in enumerate( + zip(patch_vcfs, hap_vcfs), start=1 + ): + hap_apply = ModelApplyStage( + ctx=ctx, + model=haploid_model, + vcf=patch_vcf, + output=hap_vcf, + name=f"model-apply-hap{i}", + ).add_to(dag, second_ma_deps) + merge_deps.update(hap_apply.terminal) # Second pass - unphased regions diploid_unphased_hp = self.tmp_dir.joinpath( "out_diploid_phased_unphased_hp.vcf.gz" ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, + unphased_call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscopeHP( + diploid_unphased_hp, + dbsnp=self.dbsnp, + model=self.model_bundle.joinpath("diploid_hp_model"), + pcr_indel_model=self.repeat_model, + ) + ], + inputs=sample_input, interval=unphased_bed, - ) - driver.add_algo( - DNAscopeHP( - diploid_unphased_hp, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("diploid_hp_model"), - pcr_indel_model=self.repeat_model, - ) - ) - calling_unphased_job = Job( - Pipeline(Command(*driver.build_cmd())), - "calling-unphased", - self.cores, - task_name="variant-calling", - ) + name="dnascope-unphased", + ).add_to(dag, {bcftools_subtract_job} | repeat_model_deps) # Patch DNA and DNAHP variants diploid_unphased_patch = self.tmp_dir.joinpath( "out_diploid_unphased_patch.vcf.gz" ) - cmd = cmds.cmd_pyexec_vcf_mod_patch( - str(diploid_unphased_patch), - str(phased_unphased), - str(diploid_unphased_hp), + diploid_patch_job = Job( + cmds.cmd_pyexec_vcf_mod_patch( + str(diploid_unphased_patch), + str(phased_unphased), + str(diploid_unphased_hp), + self.cores, + kwargs, + ), + "diploid-patch", self.cores, - kwargs, + task_name="variant-patch", ) - diploid_patch_job = Job( - cmd, "diploid-patch", self.cores, task_name="variant-patch" + dag.add_job( + diploid_patch_job, + {bcftools_subset_unphased_job} | unphased_call.terminal, ) - # Transfer annotations to the diploid VCF - unphased_transfer_jobs: List[Job] = [] - unphased_concat_job: Optional[Job] = None - if self.pop_vcf: - diploid_unphased_unanno = diploid_unphased_patch - diploid_unphased_patch = self.tmp_dir.joinpath( - "out_diploid_unphased_patch_anno.vcf.gz" - ) - unphased_transfer_jobs, unphased_concat_job = build_transfer_jobs( - diploid_unphased_patch, - self.pop_vcf, - diploid_unphased_unanno, - *transfer_xargs, - ) - - # DNAModelApply on the unphased VCF + # Transfer annotations to the diploid VCF, then apply the model diploid_unphased = self.tmp_dir.joinpath("out_diploid_unphased.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("diploid_model_unphased"), - diploid_unphased_patch, - diploid_unphased, + unphased_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + unphased_transfer = TransferSpec( + config=self.transfer_config(), + out_vcf=self.tmp_dir.joinpath( + "out_diploid_unphased_patch_anno.vcf.gz" + ), + tag="unphased", ) - ) - modelapply_unphased_job = Job( - Pipeline(Command(*driver.build_cmd())), - "modelapply-unphased", - self.cores, - task_name="model-apply", - ) + unphased_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=diploid_unphased_patch, + transfer=unphased_transfer, + apply=ApplySpec( + model=self.model_bundle.joinpath("diploid_model_unphased"), + output=diploid_unphased, + name="model-apply-unphased", + ), + ).add_to(dag, {diploid_patch_job}) + merge_deps.update(unphased_apply.terminal) # merge calls to create the output diploid_merged_vcf = self.tmp_dir.joinpath("out_diploid_merged.vcf.gz") @@ -1320,6 +1152,7 @@ def lr_call_variants( self.cores, task_name="vcf-merge", ) + dag.add_job(merge_job, merge_deps) gvcf_combine_job = None if self.gvcf: @@ -1335,137 +1168,111 @@ def lr_call_variants( 0, task_name="gvcf", ) + dag.add_job(gvcf_combine_job, {merge_job}) - haploid_calling_job = None - haploid_patch2_job = None - haploid_concat_job = None - haploid_gvcf_combine_job = None - haploid_gvcf_concat_job = None - if self.haploid_bed: - # Haploid variant calling - haploid_fn = self.tmp_dir.joinpath("haploid.vcf.gz") - haploid_gvcf_fn = self.tmp_dir.joinpath("haploid.g.vcf.gz") - haploid_hp_fn = self.tmp_dir.joinpath("haploid_hp.vcf.gz") - haploid_out_fn = self.tmp_dir.joinpath("haploid_patched.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.haploid_bed, - ) - driver.add_algo( + if not self.haploid_bed: + return + + # Haploid variant calling. This job depends only on the repeat + # model, not on the alignment jobs. + haploid_fn = self.tmp_dir.joinpath("haploid.vcf.gz") + haploid_gvcf_fn = self.tmp_dir.joinpath("haploid.g.vcf.gz") + haploid_hp_fn = self.tmp_dir.joinpath("haploid_hp.vcf.gz") + haploid_out_fn = self.tmp_dir.joinpath("haploid_patched.vcf.gz") + haploid_algos: List[BaseAlgo] = [ + DNAscope( + haploid_fn, + dbsnp=self.dbsnp, + model=haploid_model, + ), + DNAscopeHP( + haploid_hp_fn, + dbsnp=self.dbsnp, + model=haploid_hp_model, + pcr_indel_model=self.repeat_model, + ), + ] + if self.gvcf: + haploid_algos.append( DNAscope( - haploid_fn, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_model"), - ) - ) - driver.add_algo( - DNAscopeHP( - haploid_hp_fn, + haploid_gvcf_fn, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_hp_model"), - pcr_indel_model=self.repeat_model, + emit_mode="gvcf", + ploidy=1, + model=self.model_bundle.joinpath("gvcf_model"), ) ) - if self.gvcf: - driver.add_algo( - DNAscope( - haploid_gvcf_fn, - dbsnp=self.dbsnp, - emit_mode="gvcf", - ploidy=1, - model=self.model_bundle.joinpath("gvcf_model"), - ) - ) - haploid_calling_job = Job( - Pipeline(Command(*driver.build_cmd())), - "haploid-calling", + haploid_call = DNAscopeStage( + ctx=ctx, + algos=haploid_algos, + inputs=sample_input, + interval=self.haploid_bed, + name="dnascope-haploid", + ).add_to(dag, repeat_model_deps) + + haploid_patch2_job = Job( + cmds.cmd_pyexec_vcf_mod_haploid_patch2( + str(haploid_out_fn), + str(haploid_fn), + str(haploid_hp_fn), self.cores, - task_name="variant-calling", - ) + kwargs, + ), + "haploid-patch2", + self.cores, + task_name="variant-patch", + ) + dag.add_job(haploid_patch2_job, haploid_call.terminal) - haploid_patch2_job = Job( - cmds.cmd_pyexec_vcf_mod_haploid_patch2( - str(haploid_out_fn), - str(haploid_fn), - str(haploid_hp_fn), - self.cores, - kwargs, - ), - "haploid-patch2", + haploid_concat_job = Job( + cmds.bcftools_concat( + self.output_vcf, + [diploid_merged_vcf, haploid_out_fn], + ), + "haploid-diploid-concat", + 0, + task_name="vcf-merge", + ) + dag.add_job(haploid_concat_job, {haploid_patch2_job, merge_job}) + + if not self.gvcf or gvcf_combine_job is None: + return + + output_gvcf = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + ) + diploid_gvcf = pathlib.Path( + str(diploid_merged_vcf).replace(".vcf.gz", ".g.vcf.gz") + ) + haploid_gvcf = pathlib.Path( + str(haploid_out_fn).replace(".vcf.gz", ".g.vcf.gz") + ) + haploid_gvcf_combine_job = Job( + cmds.cmd_pyexec_gvcf_combine( + self.reference, + str(haploid_gvcf_fn), + str(haploid_out_fn), self.cores, - task_name="variant-patch", - ) - haploid_concat_job = Job( - cmds.bcftools_concat( - self.output_vcf, - [diploid_merged_vcf, haploid_out_fn], - ), - "haploid-diploid-concat", - 0, - task_name="vcf-merge", - ) + kwargs, + ), + "haploid-gvcf-combine", + 0, + task_name="gvcf", + ) + dag.add_job(haploid_gvcf_combine_job, {haploid_patch2_job}) - if self.gvcf: - output_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") - ) - diploid_gvcf = pathlib.Path( - str(diploid_merged_vcf).replace(".vcf.gz", ".g.vcf.gz") - ) - haploid_gvcf = pathlib.Path( - str(haploid_out_fn).replace(".vcf.gz", ".g.vcf.gz") - ) - haploid_gvcf_combine_job = Job( - cmds.cmd_pyexec_gvcf_combine( - self.reference, - str(haploid_gvcf_fn), - str(haploid_out_fn), - self.cores, - kwargs, - ), - "haploid-gvcf-combine", - 0, - task_name="gvcf", - ) - haploid_gvcf_concat_job = Job( - cmds.bcftools_concat( - output_gvcf, - [diploid_gvcf, haploid_gvcf], - ), - "haploid-gvcf-concat", - 0, - task_name="gvcf", - ) - return LRCallVariantsResult( - first_calling_job, - diploid_transfer_jobs, - diploid_concat_job, - first_modelapply_job, - phaser_job, - bcftools_subset_phased_job, - fai_to_bed_job, - bcftools_subtract_job, - repeatmodel_job, - bcftools_subset_unphased_job, - second_calling_job, - haploid_patch_job, - patch_transfer_jobs, - patch_concat_jobs, - second_modelapply_job, - calling_unphased_job, - diploid_patch_job, - unphased_transfer_jobs, - unphased_concat_job, - modelapply_unphased_job, - merge_job, - gvcf_combine_job, - haploid_calling_job, - haploid_patch2_job, - haploid_concat_job, - haploid_gvcf_combine_job, + haploid_gvcf_concat_job = Job( + cmds.bcftools_concat( + output_gvcf, + [diploid_gvcf, haploid_gvcf], + ), + "haploid-gvcf-concat", + 0, + task_name="gvcf", + ) + dag.add_job( haploid_gvcf_concat_job, + {haploid_gvcf_combine_job, gvcf_combine_job}, ) def call_svs( diff --git a/tests/unit/test_dnascope_longread_dag.py b/tests/unit/test_dnascope_longread_dag.py new file mode 100644 index 0000000..c48bcca --- /dev/null +++ b/tests/unit/test_dnascope_longread_dag.py @@ -0,0 +1,218 @@ +""" +The non-obvious edges of the DNAscope LongRead small-variant DAG +""" + +import json +import pathlib +from typing import Dict, List, Set +from unittest.mock import patch + +from sentieon_cli.dag import DAG +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.job import Job + +from tests.utils.test_helpers import create_mock_args + +BUNDLE_MEMBERS = [ + "diploid_hp_model", + "diploid_model", + "diploid_model_unphased", + "gvcf_model", + "haploid_hp_model", + "haploid_model", + "longreadsv.model", + "minimap2.model", +] + +BUNDLE_INFO = { + "platform": "HiFi", + "minScriptVersion": "1.5.2", + "pipeline": "DNAscope LongRead", +} + + +def build_dag(tmp_path: pathlib.Path, **kwargs) -> DAG: + """Build the first DAG of a HiFi long-read run""" + reference = tmp_path / "reference.fa" + reference.touch() + (tmp_path / "reference.fa.fai").write_text( + "chr1\t1000\t10\t60\t61\nchr2\t2000\t1030\t60\t61\n" + ) + bam = tmp_path / "longread.bam" + bam.touch() + bundle = tmp_path / "model.bundle" + bundle.touch() + bed = tmp_path / "regions.bed" + bed.touch() + + pipeline = DNAscopeLRPipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.output_vcf = tmp_path / "output.vcf.gz" + pipeline.reference = reference + pipeline.model_bundle = bundle + pipeline.sample_input = [bam] + pipeline.fastq = [] + pipeline.readgroups = [] + pipeline.bed = bed + pipeline.cores = 2 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = tmp_path + # Only the small-variant jobs are under test + pipeline.skip_mosdepth = True + pipeline.skip_cnv = True + pipeline.skip_svs = True + for key, value in kwargs.items(): + setattr(pipeline, key, value) + + bundle_info = dict(BUNDLE_INFO) + if pipeline.pop_vcf: + bundle_info["SentieonVcfID"] = "test-pop-vcf" + pipeline.skip_pop_vcf_id_check = True + + with patch("sentieon_cli.dnascope_longread.ar_load") as mock_ar_load: + mock_ar_load.side_effect = [ + BUNDLE_MEMBERS, + json.dumps(bundle_info).encode(), + ] + pipeline.validate() + pipeline.configure() + return pipeline.build_dag() + + +def deps_of(dag: DAG, name: str) -> List[str]: + """The sorted dependency names of the single job called `name`""" + return sorted(job.name for job in dag.waiting_jobs[job_named(dag, name)]) + + +def job_named(dag: DAG, name: str) -> Job: + """The one job called `name`""" + matches = [job for job in all_jobs(dag) if job.name == name] + assert len(matches) == 1, f"{name}: {len(matches)} jobs" + return matches[0] + + +def all_jobs(dag: DAG) -> List[Job]: + return list(dag.waiting_jobs) + list(dag.ready_jobs) + + +def roots(dag: DAG) -> Set[str]: + """The names of the jobs with no dependencies""" + names = {job.name for job in dag.ready_jobs} + names.update( + job.name for job, deps in dag.waiting_jobs.items() if not deps + ) + return names + + +class TestSmallVariantEdges: + """The edges `add_small_variant_calling` wires by hand""" + + def test_the_calling_passes_follow_the_phaser(self, tmp_path): + dag = build_dag(tmp_path) + + assert deps_of(dag, "variantphaser") == ["model-apply-diploid"] + for hap in ("dnascope-hap1", "dnascope-hap2"): + assert deps_of(dag, hap) == ["repeatmodel", "variantphaser"] + assert deps_of(dag, "dnascope-unphased") == [ + "bedtools-subtract", + "repeatmodel", + ] + assert deps_of(dag, "patch") == ["dnascope-hap1", "dnascope-hap2"] + + def test_merge_waits_on_the_three_model_apply_jobs(self, tmp_path): + dag = build_dag(tmp_path) + + assert deps_of(dag, "merge") == [ + "model-apply-hap1", + "model-apply-hap2", + "model-apply-unphased", + ] + + def test_both_haplotype_applies_wait_on_both_transfers(self, tmp_path): + pop_vcf = tmp_path / "population.vcf.gz" + pop_vcf.touch() + dag = build_dag(tmp_path, pop_vcf=pop_vcf) + + # Each model-apply reads only its own haplotype's transfer, but + # both wait on both concat jobs, as the hand-wired DAG did + for hap in ("model-apply-hap1", "model-apply-hap2"): + assert deps_of(dag, hap) == [ + "merge-trim-hap1-concat", + "merge-trim-hap2-concat", + "patch", + ] + + def test_each_transfer_is_tagged(self, tmp_path): + pop_vcf = tmp_path / "population.vcf.gz" + pop_vcf.touch() + dag = build_dag(tmp_path, pop_vcf=pop_vcf) + + concats = { + job.name for job in all_jobs(dag) if job.name.endswith("-concat") + } + assert concats == { + "merge-trim-diploid-concat", + "merge-trim-hap1-concat", + "merge-trim-hap2-concat", + "merge-trim-unphased-concat", + } + assert deps_of(dag, "model-apply-diploid") == [ + "dnascope-diploid", + "merge-trim-diploid-concat", + ] + assert deps_of(dag, "model-apply-unphased") == [ + "diploid-patch", + "merge-trim-unphased-concat", + ] + + def test_fai_to_bed_is_a_root_without_a_bed(self, tmp_path): + dag = build_dag(tmp_path, bed=None) + + assert "fai-to-bed" in roots(dag) + assert deps_of(dag, "bedtools-subtract") == [ + "fai-to-bed", + "variantphaser", + ] + + def test_haploid_calling_only_follows_the_repeat_model(self, tmp_path): + haploid_bed = tmp_path / "haploid.bed" + haploid_bed.touch() + dag = build_dag(tmp_path, haploid_bed=haploid_bed) + + # It does not wait on the alignment jobs, only on the repeat model + assert deps_of(dag, "dnascope-haploid") == ["repeatmodel"] + assert deps_of(dag, "haploid-patch2") == ["dnascope-haploid"] + assert deps_of(dag, "haploid-diploid-concat") == [ + "haploid-patch2", + "merge", + ] + + def test_haploid_calling_is_a_root_with_a_repeat_model(self, tmp_path): + haploid_bed = tmp_path / "haploid.bed" + haploid_bed.touch() + repeat_model = tmp_path / "given_repeat.model" + repeat_model.touch() + dag = build_dag( + tmp_path, haploid_bed=haploid_bed, repeat_model=repeat_model + ) + + assert "repeatmodel" not in {job.name for job in all_jobs(dag)} + assert "dnascope-haploid" in roots(dag) + + +class TestJobIdentity: + """Job names and task names of the migrated stages""" + + def test_task_names(self, tmp_path): + dag = build_dag(tmp_path) + + tasks: Dict[str, str] = { + job.name: job.task_name for job in all_jobs(dag) + } + assert tasks["dnascope-diploid"] == "variant-calling" + assert tasks["dnascope-hap1"] == "variant-calling" + assert tasks["dnascope-unphased"] == "variant-calling" + assert tasks["model-apply-diploid"] == "model-apply" + assert tasks["model-apply-hap2"] == "model-apply" + assert tasks["repeatmodel"] == "repeat-model" diff --git a/tests/unit/test_dnascope_longread_pop_vcf.py b/tests/unit/test_dnascope_longread_pop_vcf.py index bdcfd6c..0b65130 100644 --- a/tests/unit/test_dnascope_longread_pop_vcf.py +++ b/tests/unit/test_dnascope_longread_pop_vcf.py @@ -170,7 +170,7 @@ def test_transfer_jobs_creation(self): # Expect merge-trim jobs assert any("merge-trim" in name for name in job_names) - assert "merge-trim-concat" in job_names + assert "merge-trim-diploid-concat" in job_names def test_model_apply_dependency(self): """Test that model apply uses the transfer output when pop_vcf is present""" @@ -188,16 +188,16 @@ def test_model_apply_dependency(self): dag = pipeline.build_dag() - # Find first-modelapply job + # Find the diploid model-apply job apply_job = None for job in dag.waiting_jobs: - if job.name == "first-modelapply": + if job.name == "model-apply-diploid": apply_job = job break assert apply_job is not None - # Check dependency: apply_job should depend on merge-trim-concat (transfer_vcf_job) + # Check dependency: apply_job should depend on the transfer concat job deps = dag.waiting_jobs[apply_job] dep_names = [j.name for j in deps] - assert "merge-trim-concat" in dep_names + assert "merge-trim-diploid-concat" in dep_names From 6f0c225baaacf2437a6c4047ff78fc98fe56615b Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 01:42:14 -0700 Subject: [PATCH 08/20] Move alignment and read extraction into stages `stages/alignment.py` holds the bwa and minimap2 alignment stages (from BAM/CRAM input and from fastq), the pangenome bwa-plus-extract stage, and a ReadWriter stage, together with the shared `aln_suffix` and `find_unzip` helpers. Alignment jobs are always DAG roots, so these stages expose `build()` for callers that insert the jobs themselves as well as the usual `add_to()`. The hybrid-pangenome pipeline's copy of the minimap2 realignment and both pangenomes' bwa-extract builders are gone; the dnascope pipeline builds its alignment cleanup jobs from the stage's `cleanup_paths`. The long-read pipeline keeps its `lr_align_inputs`, `lr_align_fastq` and `merge_input_files` methods as thin wrappers. Commands, job names and dependency edges are unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 212 ++++--------- sentieon_cli/dnascope_hybrid.py | 16 +- sentieon_cli/dnascope_longread.py | 133 +++------ sentieon_cli/hybrid_pangenome.py | 121 +++----- sentieon_cli/sentieon_pangenome.py | 84 +++--- sentieon_cli/stages/alignment.py | 443 +++++++++++++++++++++++++++ tests/unit/test_stage_alignment.py | 464 +++++++++++++++++++++++++++++ 7 files changed, 1091 insertions(+), 382 deletions(-) create mode 100644 sentieon_cli/stages/alignment.py create mode 100644 tests/unit/test_stage_alignment.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 2467fbe..83397f7 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -10,11 +10,10 @@ import pathlib import shutil import sys -from typing import List, Optional, Set, Tuple +from typing import List, Optional, Set import packaging.version -from . import command_strings as cmds from .dag import DAG from .driver import ( AlignmentStat, @@ -32,7 +31,12 @@ ) from .job import Job from .pipeline import BasePipeline -from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + AlignResult, + BwaFastqStage, + BwaRealignStage, + find_unzip, +) from .stages.base import StageContext, driver_job, rm_job from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage @@ -410,13 +414,18 @@ def build_dag(self) -> DAG: sample_input = copy.deepcopy(self.sample_input) bam_rm_job = None if self.align or self.collate_align: - sample_input, align_jobs, bam_rm_job = self.sr_align_inputs() - for job in align_jobs: - dag.add_job(job) - aligned_fastq, align_fastq_jobs, fq_rm_job = self.sr_align_fastq() - for job in align_fastq_jobs: - dag.add_job(job) - sample_input += aligned_fastq + aln_result = self.add_sr_input_alignment(dag, ctx) + sample_input = aln_result.outputs + align_jobs = set(aln_result.jobs) + bam_rm_job = rm_job(aln_result.cleanup_paths, "rm-bam-aln") + fq_result = self.add_sr_fastq_alignment(dag, ctx) + align_fastq_jobs = set(fq_result.jobs) + fq_rm_job = ( + rm_job(fq_result.cleanup_paths, "rm-fq-aln") + if fq_result.cleanup_paths + else None + ) + sample_input += fq_result.outputs # Dedup and metrics preprocessing = self.add_sr_preprocessing( @@ -449,11 +458,10 @@ def build_dag(self) -> DAG: dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) return dag - def sr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job], Job]: + def add_sr_input_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: """Align input BAM/CRAM/uBAM/uCRAM files with bwa""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) @@ -463,161 +471,51 @@ def sr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job], Job]: if not check_version(cmd, min_version): sys.exit(2) - res: List[pathlib.Path] = [] - suffix = "bam" if self.bam_format else "cram" - util_sort_args = self.util_sort_args - if self.sr_duplicate_marking != "none": - suffix = "bam" - util_sort_args += " --bam_compression 1 " - jobs = set() - align_outputs: List[pathlib.Path] = [] - for i, input_aln in enumerate(self.sample_input): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_bwa_sorted_{i}.{suffix}" - ) - ) - if self.sr_duplicate_marking != "none": - out_aln = self.tmp_dir.joinpath(f"bwa_sorted_{i}.{suffix}") - rg_lines = cmds.get_rg_lines(input_aln, self.dry_run) - rg_header = self.tmp_dir.joinpath(f"input_{i}.hdr") - with open(rg_header, "w", encoding="utf-8") as rg_fh: - for line in rg_lines: - print(line, file=rg_fh) - job = Job( - cmds.cmd_samtools_fastq_bwa( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_header, - self.input_ref, - collate=self.collate_align, - bwa_args=self.bwa_args, - bwa_k=str(self.bwa_k), - util_sort_args=util_sort_args, - ), - f"bam-align-{i}", - self.cores, - task_name="alignment", - ) - res.append(out_aln) - jobs.add(job) - align_outputs.append(out_aln) - align_outputs.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - align_outputs.append(pathlib.Path(str(out_aln) + ".crai")) - - # Create an unscheduled job to remove the aligned inputs - rm_job = Job( - Pipeline( - Command("rm", *[str(x) for x in align_outputs], fail_ok=True) - ), - "rm-bam-aln", - 0, - task_name="cleanup", - ) - - return (res, jobs, rm_job) - - def sr_align_fastq( - self, - ) -> Tuple[List[pathlib.Path], Set[Job], Optional[Job]]: + return BwaRealignStage( + ctx=ctx, + inputs=self.sample_input, + model_bundle=self.model_bundle, + bam_format=self.bam_format, + duplicate_marking=self.sr_duplicate_marking, + input_ref=self.input_ref, + collate=self.collate_align, + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_sr_fastq_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: """Align fastq files to the reference genome using bwa""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) - res: List[pathlib.Path] = [] - jobs: Set[Job] = set() if not self.sr_r1_fastq and not self.sr_readgroups: - return (res, jobs, None) + return AlignResult( + jobs=[], terminal=set(), outputs=[], cleanup_paths=[] + ) if not self.skip_version_check: for cmd, min_version in FQ_MIN_VERSIONS.items(): if not check_version(cmd, min_version): sys.exit(2) - unzip = "igzip" - if not shutil.which(unzip): - self.logger.warning( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - - n_alignment_jobs = max(1, len(self.numa_nodes)) - suffix = "bam" if self.bam_format else "cram" - util_sort_args = self.util_sort_args - if self.sr_duplicate_marking != "none": - suffix = "bam" - util_sort_args += " --bam_compression 1 " - align_outputs: List[pathlib.Path] = [] - for i, (r1, r2, rg) in enumerate( - itertools.zip_longest( - self.sr_r1_fastq, self.sr_r2_fastq, self.sr_readgroups - ) - ): - for j in range(n_alignment_jobs): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_bwa_sorted_fq_{i}_{j}.{suffix}" - ) - ) - if self.sr_duplicate_marking != "none": - out_aln = self.tmp_dir.joinpath( - f"bwa_sorted_fq_{i}_{j}.{suffix}" - ) - numa = self.numa_nodes[j] if len(self.numa_nodes) > 1 else None - split = ( - f"{j}/{n_alignment_jobs}" - if len(self.numa_nodes) > 1 - else None - ) - split_cores = max(1, int(self.cores / n_alignment_jobs)) - job = Job( - cmds.cmd_fastq_bwa( - out_aln, - r1, - r2, - rg, - self.reference, - self.model_bundle, - split_cores, - unzip, - self.bwa_args, - str(self.bwa_k), - util_sort_args, - numa, - split, - ), - f"bam-align-{i}-{j}", - split_cores, - resources={f"node{j}": 1}, - task_name="alignment", - ) - res.append(out_aln) - jobs.add(job) - align_outputs.append(out_aln) - align_outputs.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - align_outputs.append(pathlib.Path(str(out_aln) + ".crai")) - - # Create an unscheduled job to remove the aligned inputs - rm_job = Job( - Pipeline( - Command("rm", *[str(x) for x in align_outputs], fail_ok=True) - ), - "rm-fq-aln", - 0, - task_name="cleanup", - ) - - return (res, jobs, rm_job) + return BwaFastqStage( + ctx=ctx, + r1_fastq=self.sr_r1_fastq, + r2_fastq=self.sr_r2_fastq, + readgroups=self.sr_readgroups, + model_bundle=self.model_bundle, + numa_nodes=self.numa_nodes, + bam_format=self.bam_format, + duplicate_marking=self.sr_duplicate_marking, + unzip=find_unzip(self.logger), + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) def sr_metrics_algos(self, paths: MetricsPaths) -> List[BaseAlgo]: """The metrics that do not need duplicate-marked reads""" diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 4e35685..503f322 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -594,14 +594,14 @@ def build_dag(self) -> DAG: # Short-read alignment sr_aln = self.sr_aln - ( - aligned_fastq, - align_fastq_jobs, - fq_rm_job, - ) = self.sr_align_fastq() - for job in align_fastq_jobs: - dag.add_job(job) - sr_aln.extend(aligned_fastq) + fq_result = self.add_sr_fastq_alignment(dag, ctx) + align_fastq_jobs = set(fq_result.jobs) + fq_rm_job = ( + rm_job(fq_result.cleanup_paths, "rm-fq-aln") + if fq_result.cleanup_paths + else None + ) + sr_aln.extend(fq_result.outputs) # Short-read dedup preprocessing = self.add_sr_preprocessing( diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index c0ddf13..e3f8da5 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -6,7 +6,6 @@ import copy import json import pathlib -import shutil import sys from typing import Dict, List, Optional, Set, Tuple @@ -23,7 +22,6 @@ DNAscope, DNAscopeHP, LongReadSV, - ReadWriter, RepeatModel, VariantPhaser, ) @@ -35,6 +33,12 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + Minimap2FastqStage, + Minimap2RealignStage, + ReadWriterStage, + find_unzip, +) from .stages.base import StageContext, driver_job from .stages.small_variants import ( ApplySpec, @@ -492,9 +496,6 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: if not self.output_vcf: self.logger.error("output_vcf is required") sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) @@ -503,99 +504,45 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: if not check_version(cmd, min_version): sys.exit(2) - res: List[pathlib.Path] = [] - suffix = "bam" if self.bam_format else "cram" - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - realign_jobs = set() - for i, input_aln in enumerate(self.lr_aln): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" - ) - ) - rg_lines = cmds.get_rg_lines( - input_aln, - self.dry_run, - ) - - realign_jobs.add( - Job( - cmds.cmd_samtools_fastq_minimap2( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_lines, - sample_name, - self.lr_input_ref, - self.fastq_taglist, - self.minimap2_args, - self.util_sort_args, - ), - f"bam-realign-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, realign_jobs) + result = Minimap2RealignStage( + ctx=self.stage_context(), + inputs=self.lr_aln, + model_bundle=self.model_bundle, + sample_name=self.output_vcf.name.replace(".vcf.gz", ""), + bam_format=self.bam_format, + input_ref=self.lr_input_ref, + fastq_taglist=self.fastq_taglist, + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).build() + return (result.outputs, set(result.jobs)) def lr_align_fastq(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ Align fastq to the reference genome using minimap2 """ - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) - res: List[pathlib.Path] = [] if self.fastq is None and self.readgroups is None: - return (res, set()) + return ([], set()) if not self.skip_version_check: for cmd, min_version in FQ_MIN_VERSIONS.items(): if not check_version(cmd, min_version): sys.exit(2) - unzip = "igzip" - if not shutil.which(unzip): - self.logger.warning( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - - suffix = "bam" if self.bam_format else "cram" - align_jobs = set() - for i, (fq, rg) in enumerate(zip(self.fastq, self.readgroups)): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_fq_{i}.{suffix}" - ) - ) - align_jobs.add( - Job( - cmds.cmd_fastq_minimap2( - out_aln, - fq, - rg, - self.reference, - self.model_bundle, - self.cores, - unzip, - self.minimap2_args, - self.util_sort_args, - ), - f"align-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, align_jobs) + result = Minimap2FastqStage( + ctx=self.stage_context(), + fastq=self.fastq, + readgroups=self.readgroups, + model_bundle=self.model_bundle, + bam_format=self.bam_format, + unzip=find_unzip(self.logger), + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).build() + return (result.outputs, set(result.jobs)) def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: """Run mosdepth for QC""" @@ -647,19 +594,15 @@ def merge_input_files( # Merge the sample_input into a single BAM merged_bam = self.tmp_dir.joinpath("longread_merged.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - driver.add_algo(ReadWriter(merged_bam)) - merge_job = Job( - Pipeline(Command(*driver.build_cmd())), - "merge-bam", - 0, + result = ReadWriterStage( + ctx=self.stage_context(), + inputs=sample_input, + output=merged_bam, + name="merge-bam", task_name="alignment", - ) - return (merged_bam, merge_job) + threads=0, # Run in the background + ).build() + return (merged_bam, result.jobs[0]) def pbsv( self, diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index cc0e2f1..a021dbf 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -8,8 +8,8 @@ import argparse import copy import json +import logging import pathlib -import shutil import sys from typing import Dict, List, Optional, Set, Tuple @@ -35,6 +35,12 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BwaExtractStage, + Minimap2RealignStage, + find_unzip, +) +from .stages.base import StageContext from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage from .stages.small_variants import DNAscopeStage, ModelApplyStage @@ -591,17 +597,6 @@ def configure(self) -> None: """Configure pipeline parameters""" pass - def find_unzip(self) -> str: - """The decompression tool for fastq input""" - unzip = "igzip" - if not shutil.which(unzip): - self.logger.info( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - return unzip - def build_dag(self) -> DAG: """Build the DAG for the hybrid-pangenome pipeline""" if not self.reference: @@ -657,9 +652,9 @@ def build_dag(self) -> DAG: calling_lr = list(self.lr_aln) realign_jobs: Set[Job] = set() if self.lr_align_input: - calling_lr, realign_jobs = self.lr_align_inputs() - for job in realign_jobs: - dag.add_job(job) + lr_result = self.lr_realign_stage(ctx).add_to(dag) + calling_lr = lr_result.outputs + realign_jobs = set(lr_result.jobs) haplotype_dependencies: Set[Job] = set() mm2_dependencies: Set[Job] = set() @@ -670,8 +665,10 @@ def build_dag(self) -> DAG: haplotype_dependencies.add(kmc_job) # BWA alignment and extraction - bwa_job = self.build_alignment_job(bwa_bam, ext_fastq) - dag.add_job(bwa_job) + bwa_result = self.bwa_extract_stage( + ctx, bwa_bam, ext_fastq + ).add_to(dag) + bwa_job = bwa_result.jobs[0] mm2_dependencies.add(bwa_job) # Do not run vg-haplotypes with bwa in low-mem environments total_mem_gb = total_memory() / (1024.0**3) @@ -900,7 +897,7 @@ def build_hybrid_kmc_job(self, kmer_prefix: pathlib.Path) -> Job: self.tmp_dir, memory=self.kmer_memory, threads=self.cores, - unzip=self.find_unzip(), + unzip=find_unzip(self.logger, logging.INFO), ), "kmc", 0, # run in the background @@ -947,78 +944,50 @@ def build_extract_kmc_jobs( ) return symlink_job, extract_kmc_job - def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: - """Align the long-read input files to the linear reference genome - with minimap2""" - assert self.output_vcf is not None - assert self.reference is not None + def lr_realign_stage(self, ctx: StageContext) -> Minimap2RealignStage: + """Re-align the long-read input to the linear reference genome""" assert self.model_bundle is not None - res: List[pathlib.Path] = [] - realign_jobs: Set[Job] = set() - suffix = "bam" if self.bam_format else "cram" - for i, input_aln in enumerate(self.lr_aln): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" - ) - ) - rg_lines = cmds.get_rg_lines(input_aln, self.dry_run) - realign_jobs.add( - Job( - cmds.cmd_samtools_fastq_minimap2( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_lines, - self.sample_sm, - self.lr_input_ref, - minimap2_model=self.model_bundle.joinpath( - "minimap2_lr.model" - ), - ), - f"bam-realign-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, realign_jobs) + return Minimap2RealignStage( + ctx=ctx, + inputs=self.lr_aln, + model_bundle=self.model_bundle, + sample_name=self.sample_sm, + bam_format=self.bam_format, + input_ref=self.lr_input_ref, + minimap2_model=self.model_bundle.joinpath("minimap2_lr.model"), + ) - def build_alignment_job( + def bwa_extract_stage( self, + ctx: StageContext, sample_bam: pathlib.Path, sample_fastq: pathlib.Path, - ) -> Job: - """Build the bwa alignment and read extraction job""" - assert self.reference is not None - assert self.model_bundle is not None + ) -> BwaExtractStage: + """The bwa alignment and read extraction stage. + The short reads are given the sample name and marked as short + reads (`LR:0`) so the hybrid caller can tell them apart. + """ + assert self.model_bundle is not None assert self.fastq_readgroup is not None rg = copy.deepcopy(self.fastq_readgroup) rg["SM"] = self.sample_sm rg["LR"] = "0" - bwa_job = Job( - cmds.cmd_bwa_extract( - sample_bam, - sample_fastq, - self.reference, - self.r1_fastq, - self.r2_fastq, - "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]), - self.model_bundle.joinpath(self.extract_model_name), - self.model_bundle.joinpath("bwa.model"), - self.cores, - unzip=self.find_unzip(), + return BwaExtractStage( + ctx=ctx, + output_bam=sample_bam, + output_fastq=sample_fastq, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroup=( + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]) ), - "bwa-extract", - self.cores, - task_name="alignment", + extract_model=self.model_bundle.joinpath(self.extract_model_name), + bwa_model=self.model_bundle.joinpath("bwa.model"), + unzip=find_unzip(self.logger, logging.INFO), ) - return bwa_job def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index ccd4877..35c81ef 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -5,6 +5,7 @@ import argparse import copy import json +import logging import pathlib import shutil import sys @@ -22,13 +23,17 @@ from .driver import ( BaseAlgo, DNAscope, - Driver, PangenomeSV, - ReadWriter, ) from .job import Job from .logging import get_logger from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BwaExtractStage, + ReadWriterStage, + find_unzip, +) +from .stages.base import StageContext from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage @@ -737,8 +742,10 @@ def build_first_dag(self) -> DAG: haplotype_dependencies.add(kmc_job) # BWA alignment and extraction - bwa_job = self.build_alignment_job(bwa_bam, ext_fastq) - dag.add_job(bwa_job) + bwa_result = self.bwa_extract_stage( + ctx, bwa_bam, ext_fastq + ).add_to(dag) + bwa_job = bwa_result.jobs[0] mm2_dependencies.add(bwa_job) bwa_lc_dependencies.add(bwa_job) # Do not run vg-haplotypes with bwa in low-mem environments @@ -1015,47 +1022,36 @@ def transfer_config(self) -> TransferConfig: fai_data=self.fai_data, ) - def build_alignment_job( + def bwa_extract_stage( self, + ctx: StageContext, sample_bam: pathlib.Path, sample_fastq: pathlib.Path, - ) -> Job: - """Build the alignment and extract jobs""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + ) -> BwaExtractStage: + """The bwa alignment and read extraction stage. + + The bwa alignment reuses the input readgroup ID with a `-bwa` + suffix, so it stays distinct from the lifted alignment's. + """ if not self.model_bundle: self.logger.error("model_bundle is required") sys.exit(2) - unzip = "igzip" - if not shutil.which(unzip): - self.logger.info( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - rg = copy.deepcopy(self.fastq_readgroup) rg["ID"] = rg["ID"] + "-bwa" - bwa_job = Job( - cmds.cmd_bwa_extract( - sample_bam, - sample_fastq, - self.reference, - self.r1_fastq, - self.r2_fastq, - "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]), - self.model_bundle.joinpath(self.extract_model_name), - self.model_bundle.joinpath("bwa.model"), - self.cores, - unzip=unzip, + return BwaExtractStage( + ctx=ctx, + output_bam=sample_bam, + output_fastq=sample_fastq, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroup=( + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]) ), - "bwa-extract", - self.cores, - task_name="alignment", + extract_model=self.model_bundle.joinpath(self.extract_model_name), + bwa_model=self.model_bundle.joinpath("bwa.model"), + unzip=find_unzip(self.logger, logging.INFO), ) - return bwa_job def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path @@ -1289,19 +1285,15 @@ def build_t1k_jobs( # Extract reads overlapping the locus to a BAM file with ReadWriter extracted_bam = self.tmp_dir.joinpath(f"sample_{tag}.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sr_alignments, - interval=locus, - ) - driver.add_algo(ReadWriter(extracted_bam)) - extract_job = Job( - Pipeline(Command(*driver.build_cmd())), - f"t1k-{tag}-extract", - self.cores, + extract_result = ReadWriterStage( + ctx=self.stage_context(), + inputs=sr_alignments, + output=extracted_bam, + name=f"t1k-{tag}-extract", task_name="t1k", - ) + interval=locus, + ).build() + extract_job = extract_result.jobs[0] t1k_job = Job( cmds.cmd_t1k( diff --git a/sentieon_cli/stages/alignment.py b/sentieon_cli/stages/alignment.py new file mode 100644 index 0000000..f474b42 --- /dev/null +++ b/sentieon_cli/stages/alignment.py @@ -0,0 +1,443 @@ +""" +Read alignment and read extraction + +Alignment jobs are the roots of every pipeline DAG: they read the run's +input files and depend on nothing. Because a few pipelines need the jobs +before they insert them, these stages split the work in two -- ``build`` +constructs the jobs, ``add_to`` inserts what ``build`` returned. +""" + +from abc import abstractmethod +from dataclasses import dataclass, field +import itertools +import logging +import pathlib +import shutil +from typing import Iterable, List, Optional, Union + +from .. import command_strings as cmds +from ..dag import DAG +from ..driver import ReadWriter +from ..job import Job +from .base import Stage, StageResult, driver_job + +# The `sentieon util sort` arguments every aligner defaults to +DEFAULT_UTIL_SORT_ARGS = "--cram_write_options version=3.0,compressor=rans" + + +def aln_suffix(bam_format: bool) -> str: + """The file extension for an aligned output file""" + return "bam" if bam_format else "cram" + + +def find_unzip(logger: logging.Logger, level: int = logging.WARNING) -> str: + """The decompression tool to read gzipped fastq with. + + ``igzip`` is considerably faster but is not always installed, so fall + back to ``gzip``. Callers differ on how loudly that is worth + reporting, hence ``level``. + """ + unzip = "igzip" + if not shutil.which(unzip): + logger.log( + level, + "igzip is recommended for decompression, but is not " + "available. Falling back to gzip.", + ) + unzip = "gzip" + return unzip + + +@dataclass(kw_only=True) +class AlignResult(StageResult): + """The jobs and outputs of an `AlignmentStage`. + + * ``outputs`` -- the aligned files, in input order. + * ``cleanup_paths`` -- files a later ``rm`` job should delete once the + alignment has been consumed; empty when the stage writes only + final outputs. + """ + + outputs: List[pathlib.Path] + cleanup_paths: List[pathlib.Path] + + +@dataclass(kw_only=True) +class AlignmentStage(Stage): + """A group of alignment jobs, all of them DAG roots. + + Subclasses implement :meth:`build`, which constructs the jobs and + inserts nothing. :meth:`add_to` then adds each of them with the same + dependencies, since no alignment job depends on another. + """ + + @abstractmethod + def build(self) -> AlignResult: + """Construct this stage's jobs without touching a DAG""" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> AlignResult: + deps = set(upstream) + result = self.build() + for job in result.jobs: + dag.add_job(job, deps) + return result + + +@dataclass(kw_only=True) +class BwaRealignStage(AlignmentStage): + """Re-align BAM/CRAM/uBAM/uCRAM input to the reference with bwa. + + The @RG lines of each input are read at build time and written to a + header file in the run's temporary directory, which the alignment + command passes to `samtools fastq`. + + When duplicate marking follows, the alignment is an intermediate: it + is written to the temporary directory as a lightly compressed BAM + instead of to the output directory. + """ + + inputs: List[pathlib.Path] + model_bundle: pathlib.Path + bam_format: bool = False + duplicate_marking: str = "markdup" + input_ref: Optional[pathlib.Path] = None + collate: bool = False + bwa_args: str = "" + bwa_k: str = "20000000" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + util_sort_args = self.util_sort_args + dedup_follows = self.duplicate_marking != "none" + if dedup_follows: + suffix = "bam" + util_sort_args += " --bam_compression 1 " + + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + cleanup_paths: List[pathlib.Path] = [] + for i, input_aln in enumerate(self.inputs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_bwa_sorted_{i}.{suffix}" + ) + ) + if dedup_follows: + out_aln = self.ctx.tmp_dir.joinpath(f"bwa_sorted_{i}.{suffix}") + rg_lines = cmds.get_rg_lines(input_aln, self.ctx.dry_run) + rg_header = self.ctx.tmp_dir.joinpath(f"input_{i}.hdr") + with open(rg_header, "w", encoding="utf-8") as rg_fh: + for line in rg_lines: + print(line, file=rg_fh) + jobs.append( + Job( + cmds.cmd_samtools_fastq_bwa( + out_aln, + input_aln, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + rg_header, + self.input_ref, + collate=self.collate, + bwa_args=self.bwa_args, + bwa_k=self.bwa_k, + util_sort_args=util_sort_args, + ), + f"bam-align-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) + if suffix == "cram": + cleanup_paths.append(pathlib.Path(str(out_aln) + ".crai")) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=cleanup_paths, + ) + + +@dataclass(kw_only=True) +class BwaFastqStage(AlignmentStage): + """Align short-read fastq pairs to the reference with bwa. + + With more than one NUMA node the reads of each pair are split across + the nodes: one job per (fastq pair, node), each pinned with `taskset` + and taking a share of the cores and a node token, so the scheduler + runs exactly one alignment per node. + """ + + r1_fastq: List[pathlib.Path] + r2_fastq: List[pathlib.Path] + readgroups: List[str] + model_bundle: pathlib.Path + numa_nodes: List[str] = field(default_factory=list) + bam_format: bool = False + duplicate_marking: str = "markdup" + unzip: str = "gzip" + bwa_args: str = "" + bwa_k: str = "20000000" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + n_alignment_jobs = max(1, len(self.numa_nodes)) + split_alignment = len(self.numa_nodes) > 1 + suffix = aln_suffix(self.bam_format) + util_sort_args = self.util_sort_args + dedup_follows = self.duplicate_marking != "none" + if dedup_follows: + suffix = "bam" + util_sort_args += " --bam_compression 1 " + + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + cleanup_paths: List[pathlib.Path] = [] + for i, (r1, r2, rg) in enumerate( + itertools.zip_longest( + self.r1_fastq, self.r2_fastq, self.readgroups + ) + ): + for j in range(n_alignment_jobs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_bwa_sorted_fq_{i}_{j}.{suffix}" + ) + ) + if dedup_follows: + out_aln = self.ctx.tmp_dir.joinpath( + f"bwa_sorted_fq_{i}_{j}.{suffix}" + ) + numa = self.numa_nodes[j] if split_alignment else None + split = f"{j}/{n_alignment_jobs}" if split_alignment else None + split_cores = max(1, int(self.ctx.cores / n_alignment_jobs)) + jobs.append( + Job( + cmds.cmd_fastq_bwa( + out_aln, + r1, + r2, + rg, + self.ctx.reference, + self.model_bundle, + split_cores, + self.unzip, + self.bwa_args, + self.bwa_k, + util_sort_args, + numa, + split, + ), + f"bam-align-{i}-{j}", + split_cores, + resources={f"node{j}": 1}, + task_name="alignment", + ) + ) + outputs.append(out_aln) + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) + if suffix == "cram": + cleanup_paths.append(pathlib.Path(str(out_aln) + ".crai")) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=cleanup_paths, + ) + + +@dataclass(kw_only=True) +class Minimap2RealignStage(AlignmentStage): + """Re-align long-read BAM/CRAM/uBAM/uCRAM input with minimap2. + + ``minimap2_model`` overrides the model the aligner picks up from the + bundle; the pangenome pipelines use it to select the long-read model. + """ + + inputs: List[pathlib.Path] + model_bundle: pathlib.Path + sample_name: str + bam_format: bool = False + input_ref: Optional[pathlib.Path] = None + fastq_taglist: str = "*" + minimap2_args: str = "-YL" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + minimap2_model: Optional[Union[pathlib.Path, str]] = None + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + for i, input_aln in enumerate(self.inputs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" + ) + ) + rg_lines = cmds.get_rg_lines(input_aln, self.ctx.dry_run) + jobs.append( + Job( + cmds.cmd_samtools_fastq_minimap2( + out_aln, + input_aln, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + rg_lines, + self.sample_name, + self.input_ref, + self.fastq_taglist, + self.minimap2_args, + self.util_sort_args, + self.minimap2_model, + ), + f"bam-realign-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class Minimap2FastqStage(AlignmentStage): + """Align long-read fastq to the reference with minimap2""" + + fastq: List[pathlib.Path] + readgroups: List[str] + model_bundle: pathlib.Path + bam_format: bool = False + unzip: str = "gzip" + minimap2_args: str = "-YL" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + for i, (fq, rg) in enumerate(zip(self.fastq, self.readgroups)): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_mm2_sorted_fq_{i}.{suffix}" + ) + ) + jobs.append( + Job( + cmds.cmd_fastq_minimap2( + out_aln, + fq, + rg, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + self.unzip, + self.minimap2_args, + self.util_sort_args, + ), + f"align-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class BwaExtractStage(AlignmentStage): + """Align short-read fastq with bwa and extract the reads a pangenome + pipeline needs, in a single pass. + + ``readgroup`` is the rendered `@RG` string; the pangenome pipelines + derive it differently from the input readgroup. + """ + + output_bam: pathlib.Path + output_fastq: pathlib.Path + r1_fastq: List[pathlib.Path] + r2_fastq: List[pathlib.Path] + readgroup: str + extract_model: pathlib.Path + bwa_model: pathlib.Path + unzip: str = "gzip" + name: str = "bwa-extract" + task_name: str = "alignment" + + def build(self) -> AlignResult: + job = Job( + cmds.cmd_bwa_extract( + self.output_bam, + self.output_fastq, + self.ctx.reference, + self.r1_fastq, + self.r2_fastq, + self.readgroup, + self.extract_model, + self.bwa_model, + self.ctx.cores, + unzip=self.unzip, + ), + self.name, + self.ctx.cores, + task_name=self.task_name, + ) + return AlignResult( + jobs=[job], + terminal={job}, + outputs=[self.output_bam, self.output_fastq], + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class ReadWriterStage(AlignmentStage): + """Rewrite alignments with `sentieon driver --algo ReadWriter`. + + With several inputs the driver merges them; with an ``interval`` it + extracts the reads overlapping that region. + """ + + inputs: List[pathlib.Path] + output: pathlib.Path + name: str + task_name: str + threads: Optional[int] = None + interval: Optional[Union[pathlib.Path, str]] = None + + def build(self) -> AlignResult: + job = driver_job( + self.ctx, + [ReadWriter(self.output)], + inputs=self.inputs, + interval=self.interval, + name=self.name, + task_name=self.task_name, + threads=self.threads, + ) + return AlignResult( + jobs=[job], + terminal={job}, + outputs=[self.output], + cleanup_paths=[], + ) diff --git a/tests/unit/test_stage_alignment.py b/tests/unit/test_stage_alignment.py new file mode 100644 index 0000000..3896437 --- /dev/null +++ b/tests/unit/test_stage_alignment.py @@ -0,0 +1,464 @@ +""" +Unit tests for the alignment and read-extraction stages +""" + +import logging +import pathlib +from unittest.mock import patch + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.alignment import ( + BwaExtractStage, + BwaFastqStage, + BwaRealignStage, + Minimap2FastqStage, + Minimap2RealignStage, + ReadWriterStage, + aln_suffix, + find_unzip, +) +from sentieon_cli.stages.base import StageContext, rm_job + +RG_LINES = ["@RG\tID:rg1\tSM:sample"] + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +@pytest.fixture +def rg_lines(): + """Stub out the `samtools view -H` call that reads the @RG lines""" + with patch( + "sentieon_cli.command_strings.get_rg_lines", return_value=RG_LINES + ) as mock: + yield mock + + +class TestHelpers: + """The shared helpers the stages and pipelines use""" + + def test_aln_suffix(self): + assert aln_suffix(True) == "bam" + assert aln_suffix(False) == "cram" + + def test_find_unzip_prefers_igzip(self): + with patch( + "sentieon_cli.stages.alignment.shutil.which", + return_value="/usr/bin/igzip", + ): + assert find_unzip(logging.getLogger("test-unzip")) == "igzip" + + def test_find_unzip_falls_back_to_gzip(self, caplog): + logger = logging.getLogger("test-unzip-fallback") + with patch( + "sentieon_cli.stages.alignment.shutil.which", return_value=None + ): + with caplog.at_level(logging.WARNING, logger=logger.name): + assert find_unzip(logger) == "gzip" + assert "igzip is recommended" in caplog.text + assert caplog.records[0].levelno == logging.WARNING + + def test_find_unzip_takes_the_log_level(self, caplog): + logger = logging.getLogger("test-unzip-level") + with patch( + "sentieon_cli.stages.alignment.shutil.which", return_value=None + ): + with caplog.at_level(logging.INFO, logger=logger.name): + assert find_unzip(logger, logging.INFO) == "gzip" + assert caplog.records[0].levelno == logging.INFO + + +class TestBwaRealignStage: + """Re-aligning BAM/CRAM input with bwa""" + + def make(self, tmp_path, **kwargs) -> BwaRealignStage: + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "in0.bam", tmp_path / "in1.bam"], + model_bundle=tmp_path / "model.bundle", + ) + defaults.update(kwargs) + return BwaRealignStage(**defaults) # type: ignore[arg-type] + + def test_one_job_per_input(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0", + "bam-align-1", + ] + for job in result.jobs: + assert job.threads == 4 + assert job.task_name == "alignment" + assert result.terminal == set(result.jobs) + + def test_header_files_are_written(self, tmp_path, rg_lines): + self.make(tmp_path).build() + + for i in range(2): + header = tmp_path / f"input_{i}.hdr" + assert header.read_text() == RG_LINES[0] + "\n" + assert rg_lines.call_count == 2 + + def test_duplicate_marking_writes_bam_to_the_tmp_dir( + self, tmp_path, rg_lines + ): + result = self.make(tmp_path).build() + + assert result.outputs == [ + tmp_path / "bwa_sorted_0.bam", + tmp_path / "bwa_sorted_1.bam", + ] + assert "--bam_compression 1" in str(result.jobs[0].shell) + + def test_without_duplicate_marking_writes_cram_output( + self, tmp_path, rg_lines + ): + result = self.make(tmp_path, duplicate_marking="none").build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_0.cram", + tmp_path / "output_bwa_sorted_1.cram", + ] + assert "--bam_compression 1" not in str(result.jobs[0].shell) + + def test_bam_format_without_duplicate_marking(self, tmp_path, rg_lines): + result = self.make( + tmp_path, duplicate_marking="none", bam_format=True + ).build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_0.bam", + tmp_path / "output_bwa_sorted_1.bam", + ] + + def test_bam_cleanup_paths(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert result.cleanup_paths == [ + tmp_path / "bwa_sorted_0.bam", + pathlib.Path(str(tmp_path / "bwa_sorted_0.bam") + ".bai"), + tmp_path / "bwa_sorted_1.bam", + pathlib.Path(str(tmp_path / "bwa_sorted_1.bam") + ".bai"), + ] + + def test_cram_cleanup_paths_add_the_crai(self, tmp_path, rg_lines): + result = self.make(tmp_path, duplicate_marking="none").build() + + out = str(tmp_path / "output_bwa_sorted_0.cram") + assert result.cleanup_paths[:3] == [ + pathlib.Path(out), + pathlib.Path(out + ".bai"), + pathlib.Path(out + ".crai"), + ] + + def test_collate_and_input_ref(self, tmp_path, rg_lines): + result = self.make( + tmp_path, + inputs=[tmp_path / "in0.cram"], + collate=True, + input_ref=tmp_path / "decode.fa", + ).build() + + shell = str(result.jobs[0].shell) + assert "samtools collate" in shell + assert f"--reference {tmp_path}/decode.fa" in shell + + def test_build_inserts_nothing(self, tmp_path, rg_lines): + dag = DAG() + self.make(tmp_path).build() + + assert not dag.ready_jobs and not dag.waiting_jobs + + +class TestBwaFastqStage: + """Aligning short-read fastq with bwa""" + + def make(self, tmp_path, **kwargs) -> BwaFastqStage: + defaults = dict( + ctx=make_ctx(tmp_path), + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroups=[r"@RG\tID:rg1\tSM:sample"], + model_bundle=tmp_path / "model.bundle", + ) + defaults.update(kwargs) + return BwaFastqStage(**defaults) # type: ignore[arg-type] + + def test_without_numa_one_job_takes_every_core(self, tmp_path): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == ["bam-align-0-0"] + job = result.jobs[0] + assert job.threads == 4 + assert job.resources == {"node0": 1} + assert job.task_name == "alignment" + assert "taskset" not in str(job.shell) + assert "--split" not in str(job.shell) + + def test_numa_splits_the_pair_across_the_nodes(self, tmp_path): + result = self.make( + tmp_path, ctx=make_ctx(tmp_path, cores=8), numa_nodes=["0-3", "4-7"] + ).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0-0", + "bam-align-0-1", + ] + assert [job.threads for job in result.jobs] == [4, 4] + assert [job.resources for job in result.jobs] == [ + {"node0": 1}, + {"node1": 1}, + ] + first, second = (str(job.shell) for job in result.jobs) + assert "taskset -c 0-3" in first and "0/2" in first + assert "taskset -c 4-7" in second and "1/2" in second + + def test_one_job_per_pair_and_node(self, tmp_path): + result = self.make( + tmp_path, + r1_fastq=[tmp_path / "a1.fq.gz", tmp_path / "b1.fq.gz"], + r2_fastq=[tmp_path / "a2.fq.gz", tmp_path / "b2.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + numa_nodes=["0-3", "4-7"], + ).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0-0", + "bam-align-0-1", + "bam-align-1-0", + "bam-align-1-1", + ] + + def test_no_fastq_builds_nothing(self, tmp_path): + result = self.make(tmp_path, r1_fastq=[], r2_fastq=[], readgroups=[]) + + built = result.build() + assert built.jobs == [] + assert built.outputs == [] + assert built.cleanup_paths == [] + + def test_unzip_reaches_the_command(self, tmp_path): + result = self.make(tmp_path, unzip="igzip").build() + + assert "igzip -dc" in str(result.jobs[0].shell) + + def test_outputs_and_cleanup_paths(self, tmp_path): + result = self.make(tmp_path, duplicate_marking="none").build() + + out = tmp_path / "output_bwa_sorted_fq_0_0.cram" + assert result.outputs == [out] + assert result.cleanup_paths == [ + out, + pathlib.Path(str(out) + ".bai"), + pathlib.Path(str(out) + ".crai"), + ] + + +class TestMinimap2RealignStage: + """Re-aligning long-read input with minimap2""" + + def make(self, tmp_path, **kwargs) -> Minimap2RealignStage: + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "lr.bam"], + model_bundle=tmp_path / "model.bundle", + sample_name="sample", + ) + defaults.update(kwargs) + return Minimap2RealignStage(**defaults) # type: ignore[arg-type] + + def test_job_metadata_and_outputs(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == ["bam-realign-0"] + assert result.jobs[0].threads == 4 + assert result.jobs[0].task_name == "alignment" + assert result.outputs == [tmp_path / "output_mm2_sorted_0.cram"] + assert result.cleanup_paths == [] + + def test_default_model_comes_from_the_bundle(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert f"-x {tmp_path}/model.bundle/minimap2.model" in str( + result.jobs[0].shell + ) + + def test_minimap2_model_overrides_the_bundle_default( + self, tmp_path, rg_lines + ): + model = tmp_path / "model.bundle" / "minimap2_lr.model" + result = self.make(tmp_path, minimap2_model=model).build() + + assert f"-x {model}" in str(result.jobs[0].shell) + + def test_bam_format_and_input_ref(self, tmp_path, rg_lines): + result = self.make( + tmp_path, bam_format=True, input_ref=tmp_path / "lr_ref.fa" + ).build() + + assert result.outputs == [tmp_path / "output_mm2_sorted_0.bam"] + assert f"--reference {tmp_path}/lr_ref.fa" in str(result.jobs[0].shell) + + +class TestMinimap2FastqStage: + """Aligning long-read fastq with minimap2""" + + def test_one_job_per_fastq(self, tmp_path): + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[tmp_path / "a.fq.gz", tmp_path / "b.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + model_bundle=tmp_path / "model.bundle", + unzip="igzip", + ).build() + + assert [job.name for job in result.jobs] == ["align-0", "align-1"] + assert result.outputs == [ + tmp_path / "output_mm2_sorted_fq_0.cram", + tmp_path / "output_mm2_sorted_fq_1.cram", + ] + assert "igzip -dc" in str(result.jobs[0].shell) + assert result.jobs[0].task_name == "alignment" + + def test_no_fastq_builds_nothing(self, tmp_path): + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[], + readgroups=[], + model_bundle=tmp_path / "model.bundle", + ).build() + + assert result.jobs == [] + assert result.outputs == [] + + +class TestBwaExtractStage: + """The pangenome bwa alignment and read-extraction pass""" + + def make(self, tmp_path, **kwargs) -> BwaExtractStage: + defaults = dict( + ctx=make_ctx(tmp_path), + output_bam=tmp_path / "sample-bwa.bam", + output_fastq=tmp_path / "sample-extract.fq.gz", + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroup=r"@RG\tID:rg1-bwa\tSM:sample", + extract_model=tmp_path / "model.bundle" / "extract.model", + bwa_model=tmp_path / "model.bundle" / "bwa.model", + ) + defaults.update(kwargs) + return BwaExtractStage(**defaults) # type: ignore[arg-type] + + def test_single_job(self, tmp_path): + result = self.make(tmp_path).build() + + assert len(result.jobs) == 1 + job = result.jobs[0] + assert job.name == "bwa-extract" + assert job.threads == 4 + assert job.task_name == "alignment" + assert result.outputs == [ + tmp_path / "sample-bwa.bam", + tmp_path / "sample-extract.fq.gz", + ] + assert result.cleanup_paths == [] + + def test_readgroup_is_used_verbatim(self, tmp_path): + result = self.make( + tmp_path, readgroup=r"@RG\tID:rg1\tSM:sample\tLR:0" + ).build() + + assert r"@RG\tID:rg1\tSM:sample\tLR:0" in str(result.jobs[0].shell) + + def test_unzip_reaches_the_command(self, tmp_path): + result = self.make(tmp_path, unzip="igzip").build() + + assert "igzip -dc" in str(result.jobs[0].shell) + + +class TestReadWriterStage: + """`sentieon driver --algo ReadWriter` merges and extracts""" + + def test_merge(self, tmp_path): + result = ReadWriterStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + output=tmp_path / "merged.bam", + name="merge-bam", + task_name="alignment", + threads=0, + ).build() + + job = result.jobs[0] + assert job.name == "merge-bam" + assert job.threads == 0 + assert job.task_name == "alignment" + assert str(job.shell) == ( + f"sentieon driver --input {tmp_path}/one.bam " + f"--input {tmp_path}/two.bam --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo ReadWriter {tmp_path}/merged.bam" + ) + assert result.outputs == [tmp_path / "merged.bam"] + + def test_string_interval_extracts_a_locus(self, tmp_path): + result = ReadWriterStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + output=tmp_path / "sample_hla.bam", + name="t1k-hla-extract", + task_name="t1k", + interval="chr6:28510020-33480577", + ).build() + + job = result.jobs[0] + assert job.name == "t1k-hla-extract" + assert job.threads == 4 # defaults to the run's core count + assert job.task_name == "t1k" + assert "--interval chr6:28510020-33480577" in str(job.shell) + + +class TestAddTo: + """`add_to` inserts what `build` returned""" + + def test_jobs_are_roots_without_upstream(self, tmp_path): + dag = DAG() + result = BwaFastqStage( + ctx=make_ctx(tmp_path, cores=8), + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroups=[r"@RG\tID:rg1"], + model_bundle=tmp_path / "model.bundle", + numa_nodes=["0-3", "4-7"], + ).add_to(dag) + + assert len(result.jobs) == 2 + for job in result.jobs: + assert job in dag.ready_jobs + assert not dag.waiting_jobs + + def test_every_job_waits_on_the_upstream_jobs(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[tmp_path / "a.fq.gz", tmp_path / "b.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + model_bundle=tmp_path / "model.bundle", + ).add_to(dag, [upstream]) + + for job in result.jobs: + assert dag.waiting_jobs[job] == {upstream} From 4c5408858bc0318c558c1cab16f3725095d4dc2c Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 01:59:41 -0700 Subject: [PATCH 09/20] Use the shared version-check and required-argument helpers Replace the repeated version-check loops with `require_versions` / `versions_available`, and the repeated "argument is required" exits with `BasePipeline.required` or the values already carried by the stage context. Drop `BasePipeline.cnv_sex_args`, which lost its last caller when CNV calling moved into a stage. The long-read and hybrid tests no longer patch `check_version` in the pipeline modules: they skip version checks, so the patches were inert, and the modules no longer import the function. The pangenome KMC test, which does exercise the check, patches it in `util` instead. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 63 +++---- sentieon_cli/dnascope_hybrid.py | 82 +++------ sentieon_cli/dnascope_longread.py | 133 +++++--------- sentieon_cli/hybrid_pangenome.py | 55 +++--- sentieon_cli/pipeline.py | 29 ++- sentieon_cli/sentieon_pangenome.py | 167 ++++++------------ tests/unit/test_cnv_sex.py | 7 - tests/unit/test_dnascope_hybrid_cnv.py | 10 +- tests/unit/test_dnascope_hybrid_pop_vcf.py | 6 +- tests/unit/test_dnascope_hybrid_stage1.py | 5 +- tests/unit/test_dnascope_longread_pop_vcf.py | 6 +- ...test_sentieon_pangenome_kmc_integration.py | 4 +- 12 files changed, 190 insertions(+), 377 deletions(-) diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 83397f7..7ed4ba8 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -47,10 +47,10 @@ TransferApplyStage, ) from .util import ( - check_version, library_preloaded, parse_rg_line, path_arg, + require_versions, split_alignment, total_memory, ) @@ -272,9 +272,7 @@ def __init__(self) -> None: self.no_split_alignment = False def validate(self) -> None: - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + self.required(self.output_vcf, "output_vcf") # uniquify pipeline attributes self.sr_r1_fastq = self.r1_fastq @@ -462,19 +460,14 @@ def add_sr_input_alignment( self, dag: DAG, ctx: StageContext ) -> AlignResult: """Align input BAM/CRAM/uBAM/uCRAM files with bwa""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") - if not self.skip_version_check: - for cmd, min_version in ALN_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(ALN_MIN_VERSIONS, skip=self.skip_version_check) return BwaRealignStage( ctx=ctx, inputs=self.sample_input, - model_bundle=self.model_bundle, + model_bundle=bundle, bam_format=self.bam_format, duplicate_marking=self.sr_duplicate_marking, input_ref=self.input_ref, @@ -488,26 +481,21 @@ def add_sr_fastq_alignment( self, dag: DAG, ctx: StageContext ) -> AlignResult: """Align fastq files to the reference genome using bwa""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") if not self.sr_r1_fastq and not self.sr_readgroups: return AlignResult( jobs=[], terminal=set(), outputs=[], cleanup_paths=[] ) - if not self.skip_version_check: - for cmd, min_version in FQ_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(FQ_MIN_VERSIONS, skip=self.skip_version_check) return BwaFastqStage( ctx=ctx, r1_fastq=self.sr_r1_fastq, r2_fastq=self.sr_r2_fastq, readgroups=self.sr_readgroups, - model_bundle=self.model_bundle, + model_bundle=bundle, numa_nodes=self.numa_nodes, bam_format=self.bam_format, duplicate_marking=self.sr_duplicate_marking, @@ -537,12 +525,9 @@ def add_sr_preprocessing( upstream: Set[Job], ) -> SrPreprocessing: """Mark duplicates and collect metrics from the short reads""" - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) suffix = "bam" if self.bam_format else "cram" - paths = MetricsPaths.from_output_vcf(self.output_vcf) + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) paths.ensure_dir(self.dry_run) if self.sr_duplicate_marking == "none": @@ -578,7 +563,7 @@ def add_sr_preprocessing( lc_extra_algos.extend(self.sr_metrics_algos(paths)) deduped = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") ) dedup_result = DedupStage( ctx=ctx, @@ -636,41 +621,33 @@ def add_small_variant_calling( upstream: Set[Job], ) -> None: """Call SNVs, indels, and SVs using DNAscope""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") - if not self.skip_version_check: - for cmd, min_version in VARIANTS_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(VARIANTS_MIN_VERSIONS, skip=self.skip_version_check) out_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) out_svs_tmp = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_svs_tmp.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_svs_tmp.vcf.gz") ) out_svs = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_svs.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_svs.vcf.gz") ) pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - model = self.model_bundle.joinpath("dnascope.model") + model = bundle.joinpath("dnascope.model") # Call variants with DNAscope emit_mode = "variant" - ds_out = self.output_vcf + ds_out = ctx.output_vcf tmp_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_tmp.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_tmp.vcf.gz") ) if self.gvcf: emit_mode = "gvcf" ds_out = out_gvcf tmp_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_tmp.g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_tmp.g.vcf.gz") ) algos: List[BaseAlgo] = [ @@ -716,7 +693,7 @@ def add_small_variant_calling( GVCFtyperStage( ctx=ctx, gvcf=out_gvcf, - output=self.output_vcf, + output=ctx.output_vcf, interval=self.bed, ).add_to(dag, transfer_apply.terminal) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 503f322..e31fd98 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -36,10 +36,10 @@ from .stages.transfer import TransferConfig, TransferStage from .util import ( __version__, - check_version, library_preloaded, parse_rg_line, path_arg, + require_versions, sample_sex_arg, split_alignment, vcf_id, @@ -396,10 +396,7 @@ def validate_cnv(self) -> None: # CNV calling now runs in the second DAG, so the version of the # driver is checked before the first DAG starts - if not self.skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) def validate_bundle(self) -> None: bundle_info_bytes = ar_load( @@ -571,15 +568,7 @@ def build_dag(self) -> DAG: self.logger.info("Building the DAG") dag = DAG() - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + self.required(self.model_bundle, "model_bundle") ctx = self.stage_context() rg_info = RgInfo( @@ -674,15 +663,7 @@ def build_second_dag(self) -> Optional[DAG]: """Build the second DAG, for sex-aware CNV calling""" if self.skip_cnv: return None - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") assert self.ploidy_json is not None ctx = self.stage_context() @@ -693,7 +674,7 @@ def build_second_dag(self) -> Optional[DAG]: CNVscopeStage( ctx=ctx, inputs=self.cnv_sr_aln, - model=self.model_bundle.joinpath("cnv.model"), + model=bundle.joinpath("cnv.model"), cnvscope_vcf=ctx.tmp_dir.joinpath("cnvscope.vcf.gz"), cnv_vcf=pathlib.Path( str(ctx.output_vcf).replace(".vcf.gz", ".cnv.vcf.gz") @@ -752,24 +733,13 @@ def add_variant_calling( the short reads there; a second pass recalls those regions, and the two call sets are merged, annotated and filtered. """ - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") - ref_fai = pathlib.Path(str(self.reference) + ".fai") + ref_fai = pathlib.Path(str(ctx.reference) + ".fai") - if not self.skip_version_check: - for cmd, min_version in CALLING_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CALLING_MIN_VERSIONS, skip=self.skip_version_check) - hybrid_model = self.model_bundle.joinpath("hybrid.model") + hybrid_model = bundle.joinpath("hybrid.model") # First pass - combined variant calling vcf_suffix = ".g.vcf.gz" if self.gvcf else ".vcf.gz" @@ -819,9 +789,7 @@ def add_variant_calling( ctx, [ HybridStage2( - model=self.model_bundle.joinpath( - "HybridStage2_region.model" - ), + model=bundle.joinpath("HybridStage2_region.model"), all_bed=mapq0_bed, ) ], @@ -869,7 +837,7 @@ def add_variant_calling( stage1_ins_fa = self.tmp_dir.joinpath("stage1_ins.fa") stage1_ins_bed = self.tmp_dir.joinpath("stage1_ins.bed") ins_driver = Driver( - reference=self.reference, + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0], input=lr_aln, @@ -878,7 +846,7 @@ def add_variant_calling( ins_driver.add_algo( HybridStage1( "-", - model=self.model_bundle.joinpath("HybridStage1_ins.model"), + model=bundle.joinpath("HybridStage1_ins.model"), fa_file=stage1_ins_fa, bed_file=stage1_ins_bed, ) @@ -901,7 +869,7 @@ def add_variant_calling( dag.add_job(stage1_fifo_job) stage1_driver = Driver( - reference=self.reference, + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0], input=lr_aln, @@ -911,7 +879,7 @@ def add_variant_calling( stage1_driver.add_algo( HybridStage1( stage1_fifo, - model=self.model_bundle.joinpath("HybridStage1.model"), + model=bundle.joinpath("HybridStage1.model"), hap_bam="-", hap_bed=stage1_hap_bed, hap_vcf=stage1_hap_vcf, @@ -938,12 +906,12 @@ def add_variant_calling( stage1_job = Job( cmds.hybrid_stage1( out_aln=stage1_bam, - reference=self.reference, + reference=ctx.reference, cores=self.cores, readgroup=f"@RG\\tID:hybrid-18893\\tSM:{self.hybrid_rg_sm}", ins_driver=ins_driver, hap_fastq_fifo=stage1_fifo, - bwa_model=self.model_bundle.joinpath("HybridStage1_bwa.model"), + bwa_model=bundle.joinpath("HybridStage1_bwa.model"), ), "first-stage", self.cores, @@ -964,7 +932,7 @@ def add_variant_calling( ctx, [ HybridStage2( - model=self.model_bundle.joinpath("HybridStage2.model"), + model=bundle.joinpath("HybridStage2.model"), hap_bed=stage1_hap_bed, unmap_bam=stage2_unmap_bam, alt_bam=stage2_alt_bam, @@ -985,10 +953,10 @@ def add_variant_calling( suffix = "bam" if self.bam_format else "cram" stage3_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_sr_realigned.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_sr_realigned.{suffix}") ) stage3_driver = Driver( - reference=self.reference, + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], input=lr_aln + sr_aln + [stage2_unmap_bam, stage2_alt_bam], @@ -998,13 +966,13 @@ def add_variant_calling( stage3_driver.add_algo( HybridStage3( "-", - model=self.model_bundle.joinpath("HybridStage3.model"), + model=bundle.joinpath("HybridStage3.model"), ) ) third_stage_job = Job( cmds.hybrid_stage3( stage3_aln, - reference=self.reference, + reference=ctx.reference, driver=stage3_driver, cores=self.cores, ), @@ -1077,7 +1045,7 @@ def add_variant_calling( ) anno_target = self.tmp_dir.joinpath("combined_tmp_anno.vcf.gz") if self.skip_model_apply and not self.pop_vcf: - anno_target = self.output_vcf + anno_target = ctx.output_vcf anno_job = Job( cmds.cmd_pyexec_hybrid_anno( @@ -1100,7 +1068,7 @@ def add_variant_calling( "combined_tmp_transfer.vcf.gz" ) if self.skip_model_apply: - transfer_target = self.output_vcf + transfer_target = ctx.output_vcf transfer = TransferStage( ctx=ctx, @@ -1126,9 +1094,9 @@ def add_variant_calling( # Final normalize norm_job = Job( cmds.filter_norm( - self.output_vcf, + ctx.output_vcf, apply_vcf, - self.reference, + ctx.reference, exclude_homref=not self.gvcf, ), "final-norm", diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index e3f8da5..2ebdb32 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -50,11 +50,12 @@ from .stages.transfer import TransferConfig, TransferStage from .util import ( __version__, - check_version, + library_preloaded, parse_rg_line, path_arg, - library_preloaded, + require_versions, vcf_id, + versions_available, ) TOOL_MIN_VERSIONS = { @@ -493,22 +494,15 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ Align reads to the reference genome using minimap2 """ - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in ALN_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + output_vcf = self.required(self.output_vcf, "output_vcf") + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(ALN_MIN_VERSIONS, skip=self.skip_version_check) result = Minimap2RealignStage( ctx=self.stage_context(), inputs=self.lr_aln, - model_bundle=self.model_bundle, - sample_name=self.output_vcf.name.replace(".vcf.gz", ""), + model_bundle=bundle, + sample_name=output_vcf.name.replace(".vcf.gz", ""), bam_format=self.bam_format, input_ref=self.lr_input_ref, fastq_taglist=self.fastq_taglist, @@ -521,22 +515,17 @@ def lr_align_fastq(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ Align fastq to the reference genome using minimap2 """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") if self.fastq is None and self.readgroups is None: return ([], set()) - if not self.skip_version_check: - for cmd, min_version in FQ_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(FQ_MIN_VERSIONS, skip=self.skip_version_check) result = Minimap2FastqStage( ctx=self.stage_context(), fastq=self.fastq, readgroups=self.readgroups, - model_bundle=self.model_bundle, + model_bundle=bundle, bam_format=self.bam_format, unzip=find_unzip(self.logger), minimap2_args=self.minimap2_args, @@ -547,19 +536,14 @@ def lr_align_fastq(self) -> Tuple[List[pathlib.Path], Set[Job]]: def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: """Run mosdepth for QC""" - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MOSDEPTH_MIN_VERSIONS.items() - ] - ): - self.logger.warning( - "Skipping mosdepth. mosdepth version %s or later " - "not found", - MOSDEPTH_MIN_VERSIONS["mosdepth"], - ) - return set() + if not versions_available( + MOSDEPTH_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping mosdepth. mosdepth version %s or later not found", + MOSDEPTH_MIN_VERSIONS["mosdepth"], + ) + return set() mosdepth_jobs = set() for i, input_file in enumerate(sample_input): @@ -587,10 +571,7 @@ def merge_input_files( """ Merge the aligned reads into a single BAM """ - if not self.skip_version_check: - for cmd, min_version in MERGE_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(MERGE_MIN_VERSIONS, skip=self.skip_version_check) # Merge the sample_input into a single BAM merged_bam = self.tmp_dir.joinpath("longread_merged.bam") @@ -611,10 +592,7 @@ def pbsv( """ Call SVs with pbsv """ - if not self.skip_version_check: - for cmd, min_version in PBSV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(PBSV_MIN_VERSIONS, skip=self.skip_version_check) # pbsv discover pbsv_discover_fn = str(self.output_vcf).replace( @@ -662,15 +640,14 @@ def hificnv( """ Call CNVs with HiFiCNV """ - if not self.skip_version_check: - for cmd, min_version in HIFICNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - self.logger.warning( - "Skipping hificnv. hificnv version %s or later not " - "found", - HIFICNV_MIN_VERSIONS["hificnv"], - ) - return None + if not versions_available( + HIFICNV_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping hificnv. hificnv version %s or later not found", + HIFICNV_MIN_VERSIONS["hificnv"], + ) + return None hifi_cnv_fn = str(self.output_vcf).replace(".vcf.gz", ".hificnv") hificnv_cmd = [ @@ -723,24 +700,13 @@ def add_small_variant_calling( the diploid HP model. The three call sets are patched, filtered and merged into the output VCF, with optional haploid calling on top. """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.skip_version_check: - for check_cmd, min_version in TOOL_MIN_VERSIONS.items(): - if not check_version(check_cmd, min_version): - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(TOOL_MIN_VERSIONS, skip=self.skip_version_check) # First pass - diploid calling diploid_gvcf_fn = self.tmp_dir.joinpath("out_diploid.g.vcf.gz") diploid_tmp_vcf = self.tmp_dir.joinpath("out_diploid_tmp.vcf.gz") - diploid_model = self.model_bundle.joinpath("diploid_model") + diploid_model = bundle.joinpath("diploid_model") diploid_algos: List[BaseAlgo] = [] if self.gvcf: diploid_algos.append( @@ -748,7 +714,7 @@ def add_small_variant_calling( diploid_gvcf_fn, dbsnp=self.dbsnp, emit_mode="gvcf", - model=self.model_bundle.joinpath("gvcf_model"), + model=bundle.joinpath("gvcf_model"), ) ) diploid_algos.append( @@ -847,7 +813,7 @@ def add_small_variant_calling( bed = self.tmp_dir.joinpath("reference.bed") fai_to_bed_job = Job( cmds.cmd_fai_to_bed( - pathlib.Path(str(self.reference) + ".fai"), + pathlib.Path(str(ctx.reference) + ".fai"), bed, ), "fai-to-bed", @@ -909,8 +875,8 @@ def add_small_variant_calling( repeat_model_deps.add(repeatmodel_job) # Second pass - phased variants - haploid_model = self.model_bundle.joinpath("haploid_model") - haploid_hp_model = self.model_bundle.joinpath("haploid_hp_model") + haploid_model = bundle.joinpath("haploid_model") + haploid_hp_model = bundle.joinpath("haploid_hp_model") for phase in (1, 2): hp_std_vcf = self.tmp_dir.joinpath( f"out_hap{phase}_nohp_tmp.vcf.gz" @@ -1022,7 +988,7 @@ def add_small_variant_calling( DNAscopeHP( diploid_unphased_hp, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("diploid_hp_model"), + model=bundle.joinpath("diploid_hp_model"), pcr_indel_model=self.repeat_model, ) ], @@ -1068,7 +1034,7 @@ def add_small_variant_calling( raw_vcf=diploid_unphased_patch, transfer=unphased_transfer, apply=ApplySpec( - model=self.model_bundle.joinpath("diploid_model_unphased"), + model=bundle.joinpath("diploid_model_unphased"), output=diploid_unphased, name="model-apply-unphased", ), @@ -1078,7 +1044,7 @@ def add_small_variant_calling( # merge calls to create the output diploid_merged_vcf = self.tmp_dir.joinpath("out_diploid_merged.vcf.gz") merge_out_vcf = ( - diploid_merged_vcf if self.haploid_bed else self.output_vcf + diploid_merged_vcf if self.haploid_bed else ctx.output_vcf ) merge_job = Job( cmds.cmd_pyexec_vcf_mod_merge( @@ -1101,7 +1067,7 @@ def add_small_variant_calling( if self.gvcf: gvcf_combine_job = Job( cmds.cmd_pyexec_gvcf_combine( - self.reference, + ctx.reference, str(diploid_gvcf_fn), str(merge_out_vcf), self.cores, @@ -1142,7 +1108,7 @@ def add_small_variant_calling( dbsnp=self.dbsnp, emit_mode="gvcf", ploidy=1, - model=self.model_bundle.joinpath("gvcf_model"), + model=bundle.joinpath("gvcf_model"), ) ) haploid_call = DNAscopeStage( @@ -1169,7 +1135,7 @@ def add_small_variant_calling( haploid_concat_job = Job( cmds.bcftools_concat( - self.output_vcf, + ctx.output_vcf, [diploid_merged_vcf, haploid_out_fn], ), "haploid-diploid-concat", @@ -1182,7 +1148,7 @@ def add_small_variant_calling( return output_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) diploid_gvcf = pathlib.Path( str(diploid_merged_vcf).replace(".vcf.gz", ".g.vcf.gz") @@ -1192,7 +1158,7 @@ def add_small_variant_calling( ) haploid_gvcf_combine_job = Job( cmds.cmd_pyexec_gvcf_combine( - self.reference, + ctx.reference, str(haploid_gvcf_fn), str(haploid_out_fn), self.cores, @@ -1226,13 +1192,8 @@ def call_svs( """ Call SVs using Sentieon LongReadSV """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in SV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(SV_MIN_VERSIONS, skip=self.skip_version_check) sv_vcf = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") @@ -1247,7 +1208,7 @@ def call_svs( driver.add_algo( LongReadSV( sv_vcf, - self.model_bundle.joinpath("longreadsv.model"), + bundle.joinpath("longreadsv.model"), ) ) longreadsv_job = Job( diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index a021dbf..1245b3b 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -48,9 +48,9 @@ from .util import ( __version__, check_kmc_patch, - check_version, parse_rg_line, path_arg, + require_versions, total_memory, vcf_id, ) @@ -258,19 +258,18 @@ def validate(self) -> None: self.collect_readgroups() self.validate_readgroups() - if not self.skip_version_check: - for cmd, min_version in HYBRID_PANGENOME_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions( + HYBRID_PANGENOME_MIN_VERSIONS, skip=self.skip_version_check + ) - if not check_kmc_patch("kmc"): - self.logger.error( - "Error: The 'kmc' executable in the PATH does not " - "support reading from stdin. Please ensure " - "you are using the patched version of KMC from " - "https://github.com/Sentieon/KMC/releases." - ) - sys.exit(2) + if not self.skip_version_check and not check_kmc_patch("kmc"): + self.logger.error( + "Error: The 'kmc' executable in the PATH does not " + "support reading from stdin. Please ensure " + "you are using the patched version of KMC from " + "https://github.com/Sentieon/KMC/releases." + ) + sys.exit(2) if self.bed is None: self.logger.info( @@ -599,35 +598,25 @@ def configure(self) -> None: def build_dag(self) -> DAG: """Build the DAG for the hybrid-pangenome pipeline""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.pop_vcf: - self.logger.error("pop_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + self.required(self.pop_vcf, "pop_vcf") self.logger.info("Building the hybrid-pangenome DAG") dag = DAG() ctx = self.stage_context() - ref_fai = pathlib.Path(str(self.reference) + ".fai") + ref_fai = pathlib.Path(str(ctx.reference) + ".fai") # Output files suffix = "bam" if self.bam_format else "cram" out_bwa_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") ) out_lift_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_lift_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_lift_deduped.{suffix}") ) sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") ) # Intermediate file paths @@ -762,7 +751,7 @@ def build_dag(self) -> DAG: # is assumed to be deduplicated already. # Emit Dedup metrics for the primary (bwa) short-read alignment # so they land in the metrics directory scanned by MultiQC. - paths = MetricsPaths.from_output_vcf(self.output_vcf) + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: paths.ensure_dir(self.dry_run) @@ -822,7 +811,7 @@ def build_dag(self) -> DAG: if self.skip_small_variants: return dag - model = self.model_bundle.joinpath("dnascope.model") + model = bundle.joinpath("dnascope.model") pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" call = DNAscopeStage( ctx=ctx, @@ -841,7 +830,7 @@ def build_dag(self) -> DAG: # transfer annotations from the pop_vcf transfer_target = ( - transfer_vcf if not self.skip_model_apply else self.output_vcf + transfer_vcf if not self.skip_model_apply else ctx.output_vcf ) transfer = TransferStage( ctx=ctx, @@ -855,7 +844,7 @@ def build_dag(self) -> DAG: ctx=ctx, model=model, vcf=transfer_vcf, - output=self.output_vcf, + output=ctx.output_vcf, ).add_to(dag, transfer.terminal) return dag diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 428f3af..ed17af3 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -11,7 +11,7 @@ import shutil import sys import time -from typing import Any, Dict, List, Optional, Tuple, TypeVar +from typing import Any, Dict, List, Optional, TypeVar import packaging.version @@ -32,10 +32,9 @@ from .util import ( SampleSex, __version__, - check_version, - cnvscope_sex_args, path_arg, tmp, + versions_available, ) MULTIQC_MIN_VERSION = { @@ -409,25 +408,17 @@ def validate_cnv_par(self, cnv_will_run: bool) -> None: ) sys.exit(2) - def cnv_sex_args(self) -> Tuple[Optional[str], Optional[pathlib.Path]]: - """The CNVscope `--sex` and `--par` arguments for this run""" - return cnvscope_sex_args(self.sample_sex, self.cnv_par_bed) - def multiqc(self) -> Optional[Job]: """Run MultiQC on the metrics files""" - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MULTIQC_MIN_VERSION.items() - ] - ): - self.logger.warning( - "Skipping MultiQC. MultiQC version %s or later not found", - MULTIQC_MIN_VERSION["multiqc"], - ) - return None + if not versions_available( + MULTIQC_MIN_VERSION, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping MultiQC. MultiQC version %s or later not found", + MULTIQC_MIN_VERSION["multiqc"], + ) + return None metrics_dir = MetricsPaths.from_output_vcf( pathlib.Path(str(self.output_vcf)) diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 35c81ef..9807b07 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -49,10 +49,10 @@ from .util import ( SampleSex, __version__, - check_version, check_kmc_patch, parse_rg_line, path_arg, + require_versions, sample_sex_arg, tmp, total_memory, @@ -375,20 +375,19 @@ def validate(self) -> None: self.validate_t1k() self.validate_cnv() - if not self.skip_version_check: - for cmd, min_version in SENT_PANGENOME_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions( + SENT_PANGENOME_MIN_VERSIONS, skip=self.skip_version_check + ) - if self.sample_input: - if not check_kmc_patch("kmc"): - self.logger.error( - "Error: The 'kmc' executable in the PATH does not " - "support reading from stdin. Please ensure " - "you are using the patched version of KMC from " - "https://github.com/Sentieon/KMC/releases." - ) - sys.exit(2) + if not self.skip_version_check and self.sample_input: + if not check_kmc_patch("kmc"): + self.logger.error( + "Error: The 'kmc' executable in the PATH does not " + "support reading from stdin. Please ensure " + "you are using the patched version of KMC from " + "https://github.com/Sentieon/KMC/releases." + ) + sys.exit(2) if self.bed is None: self.logger.info( @@ -461,10 +460,7 @@ def validate_segdup(self) -> None: ) sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in SEGDUP_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(SEGDUP_MIN_VERSION, skip=self.skip_version_check) def validate_expansion(self) -> None: if self.expansion_catalog is None: @@ -484,10 +480,7 @@ def validate_expansion(self) -> None: ) sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in EXPANSION_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(EXPANSION_MIN_VERSION, skip=self.skip_version_check) def validate_cnv(self) -> None: """Validate the arguments used for sex-aware CNV calling""" @@ -500,10 +493,7 @@ def validate_cnv(self) -> None: # A PAR BED file is required for CNV calling, whatever the sex self.validate_cnv_par(True) - if not self.skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) def validate_t1k(self) -> None: hla_requested = self.t1k_hla_seq is not None or ( @@ -529,18 +519,11 @@ def validate_t1k(self) -> None: if not (hla_requested or kir_requested): return - if not self.skip_version_check: - for cmd, min_version in T1K_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(T1K_MIN_VERSION, skip=self.skip_version_check) def validate_bundle(self) -> None: - if not self.pop_vcf: - self.logger.error("pop_vcf is required") - sys.exit(2) - if not self.gbz: - self.logger.error("gbz is required") - sys.exit(2) + pop_vcf = self.required(self.pop_vcf, "pop_vcf") + gbz = self.required(self.gbz, "gbz") bundle_info_bytes = ar_load( str(self.model_bundle) + "/bundle_info.json" ) @@ -573,7 +556,7 @@ def validate_bundle(self) -> None: if bundle_pipeline != "Sentieon pangenome": self.logger.error("The model bundle is for a different pipeline.") sys.exit(2) - if self.gbz.name != bundle_pangenome: + if gbz.name != bundle_pangenome: self.logger.warning( "The `--gbz` file name is not '%s'. " "This model is optimized for the %s pangenome.", @@ -613,7 +596,7 @@ def validate_bundle(self) -> None: ) if not self.skip_pop_vcf_id_check and not self.dry_run: - pop_vcf_id = vcf_id(self.pop_vcf) + pop_vcf_id = vcf_id(pop_vcf) if bundle_vcf_id != pop_vcf_id: self.logger.error( "The ID of the `--pop_vcf` does not match the model bundle" @@ -680,15 +663,7 @@ def build_dag(self) -> DAG: def build_first_dag(self) -> DAG: """Build the main DAG for the Sentieon pangenome pipeline""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") ctx = self.stage_context() @@ -698,13 +673,13 @@ def build_first_dag(self) -> DAG: # Output files suffix = "bam" if self.bam_format else "cram" out_bwa_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") ) out_mm2_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") ) out_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) # Intermediate file paths @@ -719,7 +694,7 @@ def build_first_dag(self) -> DAG: if not self.r1_fastq: # with bam/cram input, output the realigned bam/cram mm2_bam = pathlib.Path( - str(self.output_vcf).replace( + str(ctx.output_vcf).replace( ".vcf.gz", f"_mm2_deduped.{suffix}" ) ) @@ -770,8 +745,8 @@ def build_first_dag(self) -> DAG: kmer_prefix, ext_fastq, self.sample_input, - self.reference, - self.model_bundle.joinpath(self.extract_model_name), + ctx.reference, + bundle.joinpath(self.extract_model_name), self.tmp_dir, rw_bam, threads=self.cores, @@ -813,7 +788,7 @@ def build_first_dag(self) -> DAG: # Emit Dedup metrics for the primary (bwa) short-read alignment so # they land in the metrics directory scanned by MultiQC. - paths = MetricsPaths.from_output_vcf(self.output_vcf) + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: paths.ensure_dir(self.dry_run) @@ -877,7 +852,7 @@ def build_first_dag(self) -> DAG: # T1K HLA/KIR calling if self.t1k_hla_seq and self.t1k_hla_coord: out_hla = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_hla") + str(ctx.output_vcf).replace(".vcf.gz", "_hla") ) extract_job, t1k_job = self.build_t1k_jobs( out_hla, @@ -892,7 +867,7 @@ def build_first_dag(self) -> DAG: dag.add_job(t1k_job, {extract_job}) if self.t1k_kir_seq and self.t1k_kir_coord: out_kir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_kir") + str(ctx.output_vcf).replace(".vcf.gz", "_kir") ) extract_job, t1k_job = self.build_t1k_jobs( out_kir, @@ -910,7 +885,7 @@ def build_first_dag(self) -> DAG: sv_vcf = None if self.call_svs: sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") ) if self.skip_small_variants and not self.call_svs: @@ -920,7 +895,7 @@ def build_first_dag(self) -> DAG: if self.tech.upper() == "ULTIMA": read_filters.append("UltimaReadFilter") pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - model = self.model_bundle.joinpath("dnascope.model") + model = bundle.joinpath("dnascope.model") gfa_file = sample_gfa if self.call_svs else None algos: List[BaseAlgo] = [] @@ -960,8 +935,8 @@ def build_first_dag(self) -> DAG: return dag # When --gvcf is set, the model-apply / transfer outputs are - # gVCFs; GVCFtyper produces the final VCF at self.output_vcf. - small_variants_out = out_gvcf if self.gvcf else self.output_vcf + # gVCFs; GVCFtyper produces the final VCF at ctx.output_vcf. + small_variants_out = out_gvcf if self.gvcf else ctx.output_vcf snv_apply_vcf = self.tmp_dir.joinpath( "sample-snv_apply.g.vcf.gz" if self.gvcf @@ -1000,7 +975,7 @@ def build_first_dag(self) -> DAG: GVCFtyperStage( ctx=ctx, gvcf=out_gvcf, - output=self.output_vcf, + output=ctx.output_vcf, interval=self.bed, ).add_to(dag, {ad_update_job}) @@ -1033,9 +1008,7 @@ def bwa_extract_stage( The bwa alignment reuses the input readgroup ID with a `-bwa` suffix, so it stays distinct from the lifted alignment's. """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") rg = copy.deepcopy(self.fastq_readgroup) rg["ID"] = rg["ID"] + "-bwa" @@ -1048,8 +1021,8 @@ def bwa_extract_stage( readgroup=( "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]) ), - extract_model=self.model_bundle.joinpath(self.extract_model_name), - bwa_model=self.model_bundle.joinpath("bwa.model"), + extract_model=bundle.joinpath(self.extract_model_name), + bwa_model=bundle.joinpath("bwa.model"), unzip=find_unzip(self.logger, logging.INFO), ) @@ -1057,19 +1030,15 @@ def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path ) -> Job: """Build vg haplotypes job""" - if not self.hapl: - self.logger.error("hapl is required") - sys.exit(2) - if not self.gbz: - self.logger.error("gbz is required") - sys.exit(2) + hapl_file = self.required(self.hapl, "hapl") + gbz_file = self.required(self.gbz, "gbz") haplotypes_job = Job( cmds.cmd_vg_haplotypes( output_gbz, kmer_file, - self.hapl, - self.gbz, + hapl_file, + gbz_file, threads=self.cores, xargs=[ "--include-reference", @@ -1124,12 +1093,8 @@ def build_minimap2_lift_job( sample_gfa: pathlib.Path, ) -> Job: """Build minimap2 alignment with pgutil lift job""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + reference = self.required(self.reference, "reference") rg = ( self.fastq_readgroup @@ -1140,16 +1105,14 @@ def build_minimap2_lift_job( rg2["ID"] = rg2["ID"] + "-mm2" rg2["LR"] = "1" - mm2_model: Union[str, pathlib.Path] = self.model_bundle.joinpath( - "minimap2.model" - ) + mm2_model: Union[str, pathlib.Path] = bundle.joinpath("minimap2.model") mm2_job = Job( cmds.cmd_minimap2_lift( mm2_bam, sample_fasta, ext_fastq, sample_gfa, - self.reference, + reference, "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg2.items()]), mm2_model, threads=self.cores, @@ -1196,12 +1159,8 @@ def build_segdup_job( genes: Optional[str], ) -> Job: """Call variants in difficult SegDups""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + reference = self.required(self.reference, "reference") + bundle = self.required(self.model_bundle, "model_bundle") sex = "male" if self.sample_sex == SampleSex.MALE else "female" @@ -1215,8 +1174,8 @@ def build_segdup_job( cmds.cmd_segdup_caller( out_segdup, sr_alignment, - reference=self.reference, - sr_bundle=self.model_bundle, + reference=reference, + sr_bundle=bundle, input_vcf=input_vcf, sex=sex, genes=genes, @@ -1279,9 +1238,7 @@ def build_t1k_jobs( tag: str, ) -> Tuple[Job, Job]: """Extract reads at the T1K locus and genotype them with T1K""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + self.required(self.reference, "reference") # Extract reads overlapping the locus to a BAM file with ReadWriter extracted_bam = self.tmp_dir.joinpath(f"sample_{tag}.bam") @@ -1317,15 +1274,13 @@ def build_expansion_job( expansion_catalog: pathlib.Path, ) -> Job: """Identify repeat expansions""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + reference = self.required(self.reference, "reference") return Job( cmds.cmd_expansion_hunter( out_expansions, sr_alignment, - reference=self.reference, + reference=reference, variant_catalog=expansion_catalog, sex="male" if self.sample_sex == SampleSex.MALE else "female", threads=self.cores, @@ -1342,22 +1297,14 @@ def _add_cnv_jobs( cnv_input_bams: List[pathlib.Path], ) -> None: """Add CNV calling jobs to the DAG""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") ctx = self.stage_context() cnvscope_vcf = self.tmp_dir.joinpath("sample-cnvscope.vcf.gz") cnv_apply_vcf = self.tmp_dir.joinpath("sample-cnv_model_apply.vcf.gz") indel2cnv_vcf = self.tmp_dir.joinpath("sample-sv_cnv.vcf.gz") cnv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") ) # CNVscope on BWA deduped BAM, then CNVModelApply. The alignment @@ -1365,7 +1312,7 @@ def _add_cnv_jobs( cnv_result = CNVscopeStage( ctx=ctx, inputs=cnv_input_bams, - model=self.model_bundle.joinpath("cnv.model"), + model=bundle.joinpath("cnv.model"), cnvscope_vcf=cnvscope_vcf, cnv_vcf=cnv_apply_vcf, sample_sex=self.sample_sex, diff --git a/tests/unit/test_cnv_sex.py b/tests/unit/test_cnv_sex.py index 914d5a2..39efb2c 100644 --- a/tests/unit/test_cnv_sex.py +++ b/tests/unit/test_cnv_sex.py @@ -289,13 +289,6 @@ def test_validate_cnv_par_errors_in_a_dry_run_too(): assert excinfo.value.code == 2 -def test_cnv_sex_args_for_a_male_sample_with_a_par_bed(): - par_bed = pathlib.Path("/par.bed") - pipeline = _pipeline(sample_sex=SampleSex.MALE, cnv_par_bed=par_bed) - - assert pipeline.cnv_sex_args() == ("M", par_bed) - - def test_resolve_cnv_par_bed_prefers_the_supplied_file(tmp_path): par_bed = tmp_path.joinpath("custom_par.bed") par_bed.write_text("chrX\t10000\t2781479\n") diff --git a/tests/unit/test_dnascope_hybrid_cnv.py b/tests/unit/test_dnascope_hybrid_cnv.py index 924a712..b77903d 100644 --- a/tests/unit/test_dnascope_hybrid_cnv.py +++ b/tests/unit/test_dnascope_hybrid_cnv.py @@ -82,19 +82,13 @@ def create_pipeline(self): def build_dag(self, pipeline): """Build the first DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_dag() + dag = pipeline.build_dag() jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) return dag, {job.name: job for job in jobs} def build_second_dag(self, pipeline): """Build the second DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_second_dag() + dag = pipeline.build_second_dag() if dag is None: return None, {} jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) diff --git a/tests/unit/test_dnascope_hybrid_pop_vcf.py b/tests/unit/test_dnascope_hybrid_pop_vcf.py index 7add9f1..b7be312 100644 --- a/tests/unit/test_dnascope_hybrid_pop_vcf.py +++ b/tests/unit/test_dnascope_hybrid_pop_vcf.py @@ -135,8 +135,7 @@ def test_transfer_jobs_creation(self): pipeline.pop_vcf = self.mock_pop_vcf # Build DAG - with patch("sentieon_cli.dnascope_hybrid.check_version", return_value=True): - dag = pipeline.build_dag() + dag = pipeline.build_dag() # Verify transfer jobs exist job_names = [job.name for job in dag.waiting_jobs] @@ -150,8 +149,7 @@ def test_model_apply_input_with_pop_vcf(self): pipeline = self.create_pipeline() pipeline.pop_vcf = self.mock_pop_vcf - with patch("sentieon_cli.dnascope_hybrid.check_version", return_value=True): - dag = pipeline.build_dag() + dag = pipeline.build_dag() # Find model-apply job apply_job = None diff --git a/tests/unit/test_dnascope_hybrid_stage1.py b/tests/unit/test_dnascope_hybrid_stage1.py index 231fe8c..2c8c366 100644 --- a/tests/unit/test_dnascope_hybrid_stage1.py +++ b/tests/unit/test_dnascope_hybrid_stage1.py @@ -86,10 +86,7 @@ def create_pipeline(self): def build_dag(self, pipeline): """Build the DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_dag() + dag = pipeline.build_dag() jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) return dag, {job.name: job for job in jobs} diff --git a/tests/unit/test_dnascope_longread_pop_vcf.py b/tests/unit/test_dnascope_longread_pop_vcf.py index 0b65130..8376b7e 100644 --- a/tests/unit/test_dnascope_longread_pop_vcf.py +++ b/tests/unit/test_dnascope_longread_pop_vcf.py @@ -154,8 +154,7 @@ def test_transfer_jobs_creation(self): pipeline.pop_vcf = self.mock_pop_vcf # Mock methods to avoid real execution/checking - with patch("sentieon_cli.dnascope_longread.check_version", return_value=True), \ - patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ + with patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_fastq", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.mosdepth", return_value=set()), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.merge_input_files", return_value=(pathlib.Path("merged.bam"), MagicMock())), \ @@ -177,8 +176,7 @@ def test_model_apply_dependency(self): pipeline = self.create_pipeline() pipeline.pop_vcf = self.mock_pop_vcf - with patch("sentieon_cli.dnascope_longread.check_version", return_value=True), \ - patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ + with patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_fastq", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.mosdepth", return_value=set()), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.merge_input_files", return_value=(pathlib.Path("merged.bam"), MagicMock())), \ diff --git a/tests/unit/test_sentieon_pangenome_kmc_integration.py b/tests/unit/test_sentieon_pangenome_kmc_integration.py index cfe4e61..9c0402f 100644 --- a/tests/unit/test_sentieon_pangenome_kmc_integration.py +++ b/tests/unit/test_sentieon_pangenome_kmc_integration.py @@ -33,7 +33,7 @@ def test_validate_kmc_patch_success(self): pipeline.validate_ref = MagicMock() pipeline.collect_readgroups = MagicMock() - with patch("sentieon_cli.sentieon_pangenome.check_version", return_value=True), \ + with patch("sentieon_cli.util.check_version", return_value=True), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=True) as mock_check, \ patch("sys.exit") as mock_exit: @@ -59,7 +59,7 @@ def test_validate_kmc_patch_failure(self): pipeline.logger = MagicMock() - with patch("sentieon_cli.sentieon_pangenome.check_version", return_value=True), \ + with patch("sentieon_cli.util.check_version", return_value=True), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=False) as mock_check, \ patch("sys.exit") as mock_exit: From cfdc6a372a72153bf40795ffdd7e09b5fd859260 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 2 Sep 2026 09:57:00 -0700 Subject: [PATCH 10/20] Remove unused helpers Drop `cmd_bedtools_merge`, `cmd_vg_giraffe`, `cmd_vg_pack`, `cmd_sv_call`, `cmd_vg_surject` and `strip_ctg_prefix` from `command_strings`, `Shard.bcftools_str` from `shard`, and `get_read_length_aln` (with its regex and `shlex` import) from `util`. None had a caller in the package, the tests, or the docs. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/command_strings.py | 195 -------------------------------- sentieon_cli/shard.py | 3 - sentieon_cli/util.py | 44 ------- 3 files changed, 242 deletions(-) diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index 4cd0ba1..4ed974f 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -48,23 +48,6 @@ def cmd_bedtools_subtract( return Pipeline(Command(*cmd), file_output=unphased_bed) -def cmd_bedtools_merge( - in_bed: pathlib.Path, - out_bed: pathlib.Path, - distance: int = 0, -) -> Pipeline: - """Bedtools merge""" - cmd = [ - "bedtools", - "merge", - "-d", - str(distance), - "-i", - str(in_bed), - ] - return Pipeline(Command(*cmd), file_output=out_bed) - - def cmd_bedtools_slop( in_bed: pathlib.Path, out_bed: pathlib.Path, @@ -1041,184 +1024,6 @@ def cmd_vg_haplotypes( return Pipeline(Command(*cmd)) -def cmd_vg_giraffe( - sample_gam: pathlib.Path, - sample_pangenome: pathlib.Path, - fastq1: pathlib.Path, - fastq2: Optional[pathlib.Path], - readgroup="rg-1", - sample="sample", - threads=1, - max_fragment_length=3000, -) -> Pipeline: - vg_cmd = [ - "vg", - "giraffe", - "-t", - str(threads), - "-Z", - str(sample_pangenome), - "--read-group", - readgroup, - "--sample", - sample, - "-L", - str(max_fragment_length), - "--progress", - "-f", - str(fastq1), - ] - if fastq2: - vg_cmd.extend(["-f", str(fastq2)]) - - return Pipeline(Command(*vg_cmd), file_output=sample_gam) - - -def cmd_vg_pack( - sample_pack: pathlib.Path, - sample_gams: List[pathlib.Path], - gbz: pathlib.Path, - min_mapq=5, - threads=1, -) -> Pipeline: - cat_cmd = Command("cat", *[str(x) for x in sample_gams]) - vg_cmd = Command( - "vg", - "pack", - "-x", - str(gbz), - "-g", - "-", - "-o", - str(sample_pack), - "-Q", - str(min_mapq), - "--threads", - str(threads), - ) - - return Pipeline(cat_cmd, vg_cmd) - - -def cmd_sv_call( - out_svs: pathlib.Path, - sample_pack: pathlib.Path, - sv_header: pathlib.Path, - gbz: pathlib.Path, - snarls: pathlib.Path, - sample_name="", - ref_path="GRCh38", - min_indel_size=34, -) -> Pipeline: - # Call SVs with vg call - vg_call_cmd = Command( - "vg", - "call", - "-r", - str(snarls), - "-k", - str(sample_pack), - "-s", - sample_name, - "-z", - "--ref-sample", - ref_path, - "--progress", - str(gbz), - ) - - # Use bcftools to remove smaller variants - view_cmd = Command( - "bcftools", - "view", - "-i", - f"ABS(ILEN) > {min_indel_size}", - "-H", - "-", - ) - - # Rehead the vg call output - cat_cmd = Command( - "cat", - str(sv_header), - InputProcSub(Pipeline(vg_call_cmd, view_cmd)), - ) - - # Sort and compress the output - sort_cmd = Command( - "bcftools", - "sort", - "-O", - "v", - ) - convert_cmd = Command( - "sentieon", - "util", - "vcfconvert", - "-", - str(out_svs), - ) - return Pipeline(cat_cmd, sort_cmd, convert_cmd) - - -def cmd_vg_surject( - out_bam: pathlib.Path, - sample_gam: pathlib.Path, - xg: pathlib.Path, - surject_paths_dict: pathlib.Path, - threads=1, - surject_xargs: List[str] = [ - "--interleaved", - "--progress", - ], -) -> Pipeline: - surject_cmd_list = [ - "vg", - "surject", - "--sam-output", - "-t", - str(threads), - "-x", - str(xg), - "--into-paths", - shlex.quote(str(surject_paths_dict)), - ] - surject_cmd_list.extend(surject_xargs) - surject_cmd_list.append(str(sample_gam)) - - sort_cmd = Command( - "sentieon", - "util", - "sort", - "--sam2bam", - "-i", - "-", - "-o", - str(out_bam), - ) - - return Pipeline(Command(*surject_cmd_list), sort_cmd) - - -def strip_ctg_prefix( - output_header: pathlib.Path, - bam: pathlib.Path, - prefix: str, -) -> Pipeline: - # Also remove the M5 tag - samtools_cmd = Command("samtools", "view", "-H", str(bam)) - sed_cmd = Command( - "sed", - "-e", - f"s/\tSN:{prefix}/\tSN:/", - "-e", - "s/\tM5:[0-9a-zA-Z]*\t/\t/", - "-e", - "s/\tM5:[0-9a-zA-Z]*$//", - ) - return Pipeline(samtools_cmd, sed_cmd, file_output=output_header) - - def cmd_estimate_ploidy( output_json: pathlib.Path, aln_files: List[pathlib.Path], diff --git a/sentieon_cli/shard.py b/sentieon_cli/shard.py index 4506c40..9e509e5 100644 --- a/sentieon_cli/shard.py +++ b/sentieon_cli/shard.py @@ -93,9 +93,6 @@ class Shard(NamedTuple): def __str__(self) -> str: return f"{self.contig}:{self.start}-{self.stop}" - def bcftools_str(self) -> str: - return f"{{{self.contig}}}:{self.start}-{self.stop}" - def parse_fai(ref_fai: pathlib.Path) -> Dict[str, Dict[str, int]]: """Parse a faidx index""" diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 67fa1c4..7475a65 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -9,7 +9,6 @@ import os import pathlib import re -import shlex import shutil import subprocess as sp import sys @@ -31,7 +30,6 @@ PRELOAD_SEP_PAT = re.compile(PRELOAD_SEP) NUMA_NODE_PAT = re.compile(r"^NUMA node. CPU\(s\):\s+(?P.*)$") -READ_LENGTH_PAT = re.compile(r"SN\taverage length:\t(?P\d*)$") _UNSAFE = re.compile(r"[^A-Za-z0-9._-]") @@ -302,48 +300,6 @@ def parse_rg_line(rg_line: str) -> Dict[str, str]: return parsed -def get_read_length_aln( - aln: pathlib.Path, - reference: pathlib.Path, - n_reads: int = 100000, -) -> int: - """Get the average read length for an alignment file""" - cmds = [] - cmds.append( - [ - "samtools", - "view", - "-h", - "--reference", - shlex.quote(str(reference)), - shlex.quote(str(aln)), - ] - ) - cmds.append( - [ - "head", - "-n", - str(n_reads), - ] - ) - cmds.append(["samtools", "stats", "-"]) - all_cmds = [shlex.join(x) for x in cmds] - cmd = " | ".join(all_cmds) - res = sp.run( - cmd, - shell=True, - capture_output=True, - text=True, - executable="/bin/bash", - ) - - for line in res.stdout.split("\n"): - m = READ_LENGTH_PAT.match(line) - if m: - return int(m.groupdict()["length"]) - return 151 - - def vcf_id(in_vcf: pathlib.Path) -> Optional[str]: """Collect the SentieonVcfID header""" cmd = ["bcftools", "view", "-h", str(in_vcf)] From 98338cd1c74c8a0f04e9d9fa9e0622b883f42650 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 3 Sep 2026 20:48:58 -0700 Subject: [PATCH 11/20] Simplify the stages after review - `find_unzip` always warns; the per-caller log level is gone. - The bwa alignment stages report `cleanup_paths` only when the alignment is an intermediate that duplicate marking consumes, as the field's documentation already said; final outputs get none, and the pipelines build the `rm-*-aln` job only when there is something to remove. - `TransferApplyStage`'s DNAModelApply depends on the transfer's concat alone (the concat already depends on the raw-VCF producer), or on `upstream` when there is no transfer. The hybrid pipelines now use the composite too, and the long-read haplotype passes each run their own transfer-then-apply, so a haplotype's model-apply waits only on its own transfer. - `build_transfer_jobs` moves into `stages/transfer.py`, its only caller, and the top-level `transfer.py` is gone. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 23 ++-- sentieon_cli/dnascope_hybrid.py | 64 +++++----- sentieon_cli/dnascope_longread.py | 52 ++++---- sentieon_cli/hybrid_pangenome.py | 34 ++--- sentieon_cli/sentieon_pangenome.py | 3 +- sentieon_cli/stages/alignment.py | 22 ++-- sentieon_cli/stages/small_variants.py | 8 +- sentieon_cli/stages/transfer.py | 146 +++++++++++++++++++++- sentieon_cli/transfer.py | 152 ----------------------- tests/unit/test_dnascope_longread_dag.py | 26 ++-- tests/unit/test_stage_alignment.py | 35 +++--- tests/unit/test_stage_small_variants.py | 4 +- 12 files changed, 276 insertions(+), 293 deletions(-) delete mode 100644 sentieon_cli/transfer.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 7ed4ba8..431ea21 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -410,19 +410,14 @@ def build_dag(self) -> DAG: # Alignment align_jobs: Set[Job] = set() sample_input = copy.deepcopy(self.sample_input) - bam_rm_job = None + bam_cleanup: List[pathlib.Path] = [] if self.align or self.collate_align: aln_result = self.add_sr_input_alignment(dag, ctx) sample_input = aln_result.outputs align_jobs = set(aln_result.jobs) - bam_rm_job = rm_job(aln_result.cleanup_paths, "rm-bam-aln") + bam_cleanup = aln_result.cleanup_paths fq_result = self.add_sr_fastq_alignment(dag, ctx) align_fastq_jobs = set(fq_result.jobs) - fq_rm_job = ( - rm_job(fq_result.cleanup_paths, "rm-fq-aln") - if fq_result.cleanup_paths - else None - ) sample_input += fq_result.outputs # Dedup and metrics @@ -431,11 +426,17 @@ def build_dag(self) -> DAG: ) deduped = preprocessing.deduped dedup_job = preprocessing.dedup_job + # An alignment has cleanup paths only when it is an intermediate, + # which is exactly when duplicate marking -- and so `dedup_job` -- + # follows it if dedup_job: - if bam_rm_job: - dag.add_job(bam_rm_job, {dedup_job}) - if fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + if bam_cleanup: + dag.add_job(rm_job(bam_cleanup, "rm-bam-aln"), {dedup_job}) + if fq_result.cleanup_paths: + dag.add_job( + rm_job(fq_result.cleanup_paths, "rm-fq-aln"), + {dedup_job}, + ) # Small variants if not self.skip_small_variants: diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index e31fd98..fa767fe 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -32,8 +32,13 @@ from .stages.base import StageContext, driver_job, rm_job from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage from .stages.ploidy import PloidyStage -from .stages.small_variants import DNAscopeStage, ModelApplyStage -from .stages.transfer import TransferConfig, TransferStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + TransferApplyStage, + TransferSpec, +) +from .stages.transfer import TransferConfig from .util import ( __version__, library_preloaded, @@ -585,11 +590,6 @@ def build_dag(self) -> DAG: sr_aln = self.sr_aln fq_result = self.add_sr_fastq_alignment(dag, ctx) align_fastq_jobs = set(fq_result.jobs) - fq_rm_job = ( - rm_job(fq_result.cleanup_paths, "rm-fq-aln") - if fq_result.cleanup_paths - else None - ) sr_aln.extend(fq_result.outputs) # Short-read dedup @@ -598,8 +598,13 @@ def build_dag(self) -> DAG: ) sr_aln = preprocessing.deduped dedup_job = preprocessing.dedup_job - if dedup_job and fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + # The fastq alignment has cleanup paths only when it is an + # intermediate, which is exactly when duplicate marking -- and so + # `dedup_job` -- follows it + if dedup_job and fq_result.cleanup_paths: + dag.add_job( + rm_job(fq_result.cleanup_paths, "rm-fq-aln"), {dedup_job} + ) # Long-read alignment realign_jobs: Set[Job] = set() @@ -1061,36 +1066,37 @@ def add_variant_calling( ) dag.add_job(anno_job, {concat_job}) - input_to_apply = anno_target - apply_dependencies: Set[Job] = {anno_job} + if not self.pop_vcf and self.skip_model_apply: + # `anno_target` is already the output VCF + return + + # Transfer annotations and apply the model + transfer_spec: Optional[TransferSpec] = None if self.pop_vcf: transfer_target = self.tmp_dir.joinpath( "combined_tmp_transfer.vcf.gz" ) if self.skip_model_apply: transfer_target = ctx.output_vcf + transfer_spec = TransferSpec( + self.transfer_config(), transfer_target + ) - transfer = TransferStage( - ctx=ctx, - config=self.transfer_config(), - raw_vcf=anno_target, - out_vcf=transfer_target, - ).add_to(dag, {anno_job}) - input_to_apply = transfer_target - apply_dependencies = transfer.terminal + apply_spec: Optional[ApplySpec] = None + apply_vcf = self.tmp_dir.joinpath("combined_apply.vcf.gz") + if not self.skip_model_apply: + apply_spec = ApplySpec(hybrid_model, apply_vcf) + + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=anno_target, + transfer=transfer_spec, + apply=apply_spec, + ).add_to(dag, {anno_job}) if self.skip_model_apply: return - # Model Apply - apply_vcf = self.tmp_dir.joinpath("combined_apply.vcf.gz") - apply = ModelApplyStage( - ctx=ctx, - model=hybrid_model, - vcf=input_to_apply, - output=apply_vcf, - ).add_to(dag, apply_dependencies) - # Final normalize norm_job = Job( cmds.filter_norm( @@ -1103,4 +1109,4 @@ def add_variant_calling( 0, task_name="vcf-norm", ) - dag.add_job(norm_job, apply.terminal) + dag.add_job(norm_job, transfer_apply.terminal) diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index 2ebdb32..0ee2864 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -43,11 +43,10 @@ from .stages.small_variants import ( ApplySpec, DNAscopeStage, - ModelApplyStage, TransferApplyStage, TransferSpec, ) -from .stages.transfer import TransferConfig, TransferStage +from .stages.transfer import TransferConfig from .util import ( __version__, library_preloaded, @@ -940,28 +939,9 @@ def add_small_variant_calling( ) dag.add_job(haploid_patch_job, haploid_patch_deps) - # Transfer annotations to the patched VCFs. Both model-apply jobs - # wait on both transfers, as they did when the DAG was wired by hand. - second_ma_deps: Set[Job] = {haploid_patch_job} - if self.pop_vcf: - patch_unanno_vcfs = patch_vcfs.copy() - patch_vcfs = [ - self.tmp_dir.joinpath(f"out_hap{i}_patch_anno.vcf.gz") - for i in (1, 2) - ] - for i, (patch_unanno_vcf, patch_vcf) in enumerate( - zip(patch_unanno_vcfs, patch_vcfs), start=1 - ): - patch_transfer = TransferStage( - ctx=ctx, - config=self.transfer_config(), - raw_vcf=patch_unanno_vcf, - out_vcf=patch_vcf, - tag=f"hap{i}", - ).add_to(dag, {haploid_patch_job}) - second_ma_deps.update(patch_transfer.terminal) - - # apply trained model to the patched vcfs. + # Transfer annotations to each patched VCF, then apply the trained + # model. A haplotype's model-apply reads only that haplotype's + # VCF, so it waits on that haplotype's transfer alone. hap_vcfs = [ self.tmp_dir.joinpath(f"out_hap{i}.vcf.gz") for i in (1, 2) ] @@ -969,13 +949,25 @@ def add_small_variant_calling( for i, (patch_vcf, hap_vcf) in enumerate( zip(patch_vcfs, hap_vcfs), start=1 ): - hap_apply = ModelApplyStage( + patch_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + patch_transfer = TransferSpec( + config=self.transfer_config(), + out_vcf=self.tmp_dir.joinpath( + f"out_hap{i}_patch_anno.vcf.gz" + ), + tag=f"hap{i}", + ) + hap_apply = TransferApplyStage( ctx=ctx, - model=haploid_model, - vcf=patch_vcf, - output=hap_vcf, - name=f"model-apply-hap{i}", - ).add_to(dag, second_ma_deps) + raw_vcf=patch_vcf, + transfer=patch_transfer, + apply=ApplySpec( + model=haploid_model, + output=hap_vcf, + name=f"model-apply-hap{i}", + ), + ).add_to(dag, {haploid_patch_job}) merge_deps.update(hap_apply.terminal) # Second pass - unphased regions diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index 1245b3b..f6b09cb 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -8,7 +8,6 @@ import argparse import copy import json -import logging import pathlib import sys from typing import Dict, List, Optional, Set, Tuple @@ -43,8 +42,13 @@ from .stages.base import StageContext from .stages.dedup import DedupStage from .stages.metrics import MetricsPaths, MetricsStage -from .stages.small_variants import DNAscopeStage, ModelApplyStage -from .stages.transfer import TransferConfig, TransferStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + TransferApplyStage, + TransferSpec, +) +from .stages.transfer import TransferConfig from .util import ( __version__, check_kmc_patch, @@ -828,25 +832,21 @@ def build_dag(self) -> DAG: replace_rg=replace_rg, ).add_to(dag, calling_dependencies) - # transfer annotations from the pop_vcf + # transfer annotations from the pop_vcf, then apply the model transfer_target = ( transfer_vcf if not self.skip_model_apply else ctx.output_vcf ) - transfer = TransferStage( + TransferApplyStage( ctx=ctx, - config=self.transfer_config(), raw_vcf=raw_vcf, - out_vcf=transfer_target, + transfer=TransferSpec(self.transfer_config(), transfer_target), + apply=( + ApplySpec(model, ctx.output_vcf) + if not self.skip_model_apply + else None + ), ).add_to(dag, call.terminal) - if not self.skip_model_apply: - ModelApplyStage( - ctx=ctx, - model=model, - vcf=transfer_vcf, - output=ctx.output_vcf, - ).add_to(dag, transfer.terminal) - return dag def transfer_config(self) -> TransferConfig: @@ -886,7 +886,7 @@ def build_hybrid_kmc_job(self, kmer_prefix: pathlib.Path) -> Job: self.tmp_dir, memory=self.kmer_memory, threads=self.cores, - unzip=find_unzip(self.logger, logging.INFO), + unzip=find_unzip(self.logger), ), "kmc", 0, # run in the background @@ -975,7 +975,7 @@ def bwa_extract_stage( ), extract_model=self.model_bundle.joinpath(self.extract_model_name), bwa_model=self.model_bundle.joinpath("bwa.model"), - unzip=find_unzip(self.logger, logging.INFO), + unzip=find_unzip(self.logger), ) def build_haplotypes_job( diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 9807b07..1420360 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -5,7 +5,6 @@ import argparse import copy import json -import logging import pathlib import shutil import sys @@ -1023,7 +1022,7 @@ def bwa_extract_stage( ), extract_model=bundle.joinpath(self.extract_model_name), bwa_model=bundle.joinpath("bwa.model"), - unzip=find_unzip(self.logger, logging.INFO), + unzip=find_unzip(self.logger), ) def build_haplotypes_job( diff --git a/sentieon_cli/stages/alignment.py b/sentieon_cli/stages/alignment.py index f474b42..d12e6dc 100644 --- a/sentieon_cli/stages/alignment.py +++ b/sentieon_cli/stages/alignment.py @@ -30,17 +30,15 @@ def aln_suffix(bam_format: bool) -> str: return "bam" if bam_format else "cram" -def find_unzip(logger: logging.Logger, level: int = logging.WARNING) -> str: +def find_unzip(logger: logging.Logger) -> str: """The decompression tool to read gzipped fastq with. ``igzip`` is considerably faster but is not always installed, so fall - back to ``gzip``. Callers differ on how loudly that is worth - reporting, hence ``level``. + back to ``gzip``. """ unzip = "igzip" if not shutil.which(unzip): - logger.log( - level, + logger.warning( "igzip is recommended for decompression, but is not " "available. Falling back to gzip.", ) @@ -151,10 +149,9 @@ def build(self) -> AlignResult: ) ) outputs.append(out_aln) - cleanup_paths.append(out_aln) - cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - cleanup_paths.append(pathlib.Path(str(out_aln) + ".crai")) + if dedup_follows: + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) return AlignResult( jobs=jobs, @@ -241,10 +238,9 @@ def build(self) -> AlignResult: ) ) outputs.append(out_aln) - cleanup_paths.append(out_aln) - cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - cleanup_paths.append(pathlib.Path(str(out_aln) + ".crai")) + if dedup_follows: + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) return AlignResult( jobs=jobs, diff --git a/sentieon_cli/stages/small_variants.py b/sentieon_cli/stages/small_variants.py index 9757781..4880b55 100644 --- a/sentieon_cli/stages/small_variants.py +++ b/sentieon_cli/stages/small_variants.py @@ -174,9 +174,9 @@ class TransferApplyStage(Stage): """Post-process a raw VCF: transfer annotations, then apply the model. Either half may be omitted. When both run, DNAModelApply reads the - transfer output and waits on the transfer *and* on ``upstream`` -- - the raw-VCF producer stays a direct dependency, matching the DAGs the - pipelines built by hand. + transfer output and waits on the transfer alone -- the transfer's + concat job already depends on the raw-VCF producer. Without a + transfer, DNAModelApply waits on ``upstream``. """ raw_vcf: pathlib.Path @@ -206,7 +206,7 @@ def add_to( ).add_to(dag, deps) jobs.extend(transfer_result.jobs) apply_in = self.transfer.out_vcf - apply_deps = deps | transfer_result.terminal + apply_deps = transfer_result.terminal apply_job: Optional[Job] = None if self.apply is not None: diff --git a/sentieon_cli/stages/transfer.py b/sentieon_cli/stages/transfer.py index 7ba294b..fe9a0cd 100644 --- a/sentieon_cli/stages/transfer.py +++ b/sentieon_cli/stages/transfer.py @@ -4,14 +4,156 @@ from dataclasses import dataclass import pathlib -from typing import Dict, Iterable, List, Optional +import re +import subprocess as sp +import tempfile +from typing import Dict, Iterable, List, Optional, Set, Tuple +from importlib.resources import files + +from .. import command_strings as cmds from ..dag import DAG from ..job import Job +from ..logging import get_logger from ..shard import Shard -from ..transfer import build_transfer_jobs from .base import Stage, StageResult +logger = get_logger(__name__) + + +def build_transfer_jobs( + out_vcf: pathlib.Path, + pop_vcf: pathlib.Path, + raw_vcf: pathlib.Path, + base_tmp_dir: pathlib.Path, + shards: List[Shard], + pop_vcf_contigs: Dict[str, Optional[int]], + fai_data: Dict[str, Dict[str, int]], + dry_run: bool = False, + cores: int = 1, + *, + tag: Optional[str] = None, +) -> Tuple[List[Job], Job]: + """Transfer annotations from the pop_vcf to the raw_vcf. + + ``tag`` is inserted into the job names (``merge-trim-{tag}-...``) so a + pipeline that transfers annotations more than once in a run can tell + the two fan-outs apart. + """ + + name_base = "merge-trim" if tag is None else f"merge-trim-{tag}" + + # Get a unique tmpdir + tmp_dir_str = tempfile.mkdtemp(dir=base_tmp_dir) + tmp_dir = pathlib.Path(tmp_dir_str) + + # Generate merge rules from the population VCF + merge_rules = "AC_v20:sum,AF_v20:sum,AC_genomes:sum,AF_genomes:sum" + if not dry_run: + kvpat = re.compile(r'(.*?)=(".*?"|.*?)(?:,|$)') + cmd = ["bcftools", "view", "-h", str(pop_vcf)] + p = sp.run(cmd, capture_output=True, text=True) + id_fields: List[str] = [] + for line in p.stdout.split("\n"): + if not line.startswith("##INFO"): + continue + if ",Number=A" not in line: + continue + s = line.index("<") + e = line.index(">") + d = dict(kvpat.findall(line[s + 1 : e])) # noqa: E203 + id_fields.append(d["ID"]) + merge_rules = ",".join([x + ":sum" for x in id_fields]) + + # Merge VCFs by shards + sharded_vcfs: List[pathlib.Path] = [] + sharded_merge_jobs: List[Job] = [] + trim_script = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("trimalt.py")) + ).resolve() + seen_contigs: Set[str] = set() + for i, shard in enumerate(shards): + # Use a BED file for unusual contig names + subset_bed = tmp_dir.joinpath( + f"sample-dnascope_transfer-subset{i}.bed" + ) + + # Extract contigs not in the pop vcf as merge will fail + if shard.contig not in pop_vcf_contigs: + if shard.contig in seen_contigs: + continue + logger.info("Skipping transfer for contig: %s", shard.contig) + seen_contigs.add(shard.contig) + subset_vcf = tmp_dir.joinpath( + f"sample-dnascope_transfer-subset{i}.vcf.gz" + ) + + ctg_len = fai_data[shard.contig]["length"] + if not dry_run: + with open(subset_bed, "w") as fh: + print(f"{shard.contig}\t0\t{ctg_len}", file=fh) + + view_job = Job( + cmds.cmd_bcftools_view_regions( + subset_vcf, + raw_vcf, + regions_file=subset_bed, + ), + f"{name_base}-extra", + 1, + task_name="annotation-transfer", + ) + sharded_merge_jobs.append(view_job) + sharded_vcfs.append(subset_vcf) + else: + if not dry_run: + with open(subset_bed, "w") as fh: + print( + f"{shard.contig}\t{shard.start}\t{shard.stop}", + file=fh, + ) + + logger.debug("Transferring shard: %s", shard) + shard_vcf = tmp_dir.joinpath( + f"sample-dnascope_transfer-shard{i}.vcf.gz" + ) + merge_job = Job( + cmds.cmd_bcftools_merge_trim( + shard_vcf, + raw_vcf, + pop_vcf, + trim_script, + subset_bed, + merge_rules=merge_rules, + merge_xargs=[ + "--no-version", + "--regions-overlap", + "pos", + "-m", + "all", + ], + view_xargs=["--no-version"], + ), + f"{name_base}-{i}", + 1, + task_name="annotation-transfer", + ) + sharded_merge_jobs.append(merge_job) + sharded_vcfs.append(shard_vcf) + + # Concat all shards + concat_job = Job( + cmds.bcftools_concat( + out_vcf, + sharded_vcfs, + xargs=["--no-version", "--threads", str(cores)], + ), + f"{name_base}-concat", + cores, + task_name="annotation-transfer", + ) + return (sharded_merge_jobs, concat_job) + @dataclass(frozen=True) class TransferConfig: diff --git a/sentieon_cli/transfer.py b/sentieon_cli/transfer.py deleted file mode 100644 index 24f9c11..0000000 --- a/sentieon_cli/transfer.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Annotation transfer functionality -""" - -import pathlib -import re -import subprocess as sp -import tempfile -from typing import Dict, List, Optional, Set, Tuple - -from importlib.resources import files - -from . import command_strings as cmds -from .job import Job -from .logging import get_logger -from .shard import Shard - -logger = get_logger(__name__) - - -def build_transfer_jobs( - out_vcf: pathlib.Path, - pop_vcf: pathlib.Path, - raw_vcf: pathlib.Path, - base_tmp_dir: pathlib.Path, - shards: List[Shard], - pop_vcf_contigs: Dict[str, Optional[int]], - fai_data: Dict[str, Dict[str, int]], - dry_run: bool = False, - cores: int = 1, - *, - tag: Optional[str] = None, -) -> Tuple[List[Job], Job]: - """Transfer annotations from the pop_vcf to the raw_vcf. - - ``tag`` is inserted into the job names (``merge-trim-{tag}-...``) so a - pipeline that transfers annotations more than once in a run can tell - the two fan-outs apart. - """ - - name_base = "merge-trim" if tag is None else f"merge-trim-{tag}" - - # Get a unique tmpdir - tmp_dir_str = tempfile.mkdtemp(dir=base_tmp_dir) - tmp_dir = pathlib.Path(tmp_dir_str) - - # Generate merge rules from the population VCF - merge_rules = "AC_v20:sum,AF_v20:sum,AC_genomes:sum,AF_genomes:sum" - if not dry_run: - kvpat = re.compile(r'(.*?)=(".*?"|.*?)(?:,|$)') - cmd = ["bcftools", "view", "-h", str(pop_vcf)] - p = sp.run(cmd, capture_output=True, text=True) - id_fields: List[str] = [] - for line in p.stdout.split("\n"): - if not line.startswith("##INFO"): - continue - if ",Number=A" not in line: - continue - s = line.index("<") - e = line.index(">") - d = dict(kvpat.findall(line[s + 1 : e])) # noqa: E203 - id_fields.append(d["ID"]) - merge_rules = ",".join([x + ":sum" for x in id_fields]) - - # Merge VCFs by shards - sharded_vcfs: List[pathlib.Path] = [] - sharded_merge_jobs: List[Job] = [] - trim_script = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("trimalt.py")) - ).resolve() - seen_contigs: Set[str] = set() - for i, shard in enumerate(shards): - # Use a BED file for unusual contig names - subset_bed = tmp_dir.joinpath( - f"sample-dnascope_transfer-subset{i}.bed" - ) - - # Extract contigs not in the pop vcf as merge will fail - if shard.contig not in pop_vcf_contigs: - if shard.contig in seen_contigs: - continue - logger.info("Skipping transfer for contig: %s", shard.contig) - seen_contigs.add(shard.contig) - subset_vcf = tmp_dir.joinpath( - f"sample-dnascope_transfer-subset{i}.vcf.gz" - ) - - ctg_len = fai_data[shard.contig]["length"] - if not dry_run: - with open(subset_bed, "w") as fh: - print(f"{shard.contig}\t0\t{ctg_len}", file=fh) - - view_job = Job( - cmds.cmd_bcftools_view_regions( - subset_vcf, - raw_vcf, - regions_file=subset_bed, - ), - f"{name_base}-extra", - 1, - task_name="annotation-transfer", - ) - sharded_merge_jobs.append(view_job) - sharded_vcfs.append(subset_vcf) - else: - if not dry_run: - with open(subset_bed, "w") as fh: - print( - f"{shard.contig}\t{shard.start}\t{shard.stop}", - file=fh, - ) - - logger.debug("Transferring shard: %s", shard) - shard_vcf = tmp_dir.joinpath( - f"sample-dnascope_transfer-shard{i}.vcf.gz" - ) - merge_job = Job( - cmds.cmd_bcftools_merge_trim( - shard_vcf, - raw_vcf, - pop_vcf, - trim_script, - subset_bed, - merge_rules=merge_rules, - merge_xargs=[ - "--no-version", - "--regions-overlap", - "pos", - "-m", - "all", - ], - view_xargs=["--no-version"], - ), - f"{name_base}-{i}", - 1, - task_name="annotation-transfer", - ) - sharded_merge_jobs.append(merge_job) - sharded_vcfs.append(shard_vcf) - - # Concat all shards - concat_job = Job( - cmds.bcftools_concat( - out_vcf, - sharded_vcfs, - xargs=["--no-version", "--threads", str(cores)], - ), - f"{name_base}-concat", - cores, - task_name="annotation-transfer", - ) - return (sharded_merge_jobs, concat_job) diff --git a/tests/unit/test_dnascope_longread_dag.py b/tests/unit/test_dnascope_longread_dag.py index c48bcca..e239146 100644 --- a/tests/unit/test_dnascope_longread_dag.py +++ b/tests/unit/test_dnascope_longread_dag.py @@ -129,20 +129,26 @@ def test_merge_waits_on_the_three_model_apply_jobs(self, tmp_path): "model-apply-unphased", ] - def test_both_haplotype_applies_wait_on_both_transfers(self, tmp_path): + def test_each_haplotype_apply_waits_on_its_own_transfer(self, tmp_path): pop_vcf = tmp_path / "population.vcf.gz" pop_vcf.touch() dag = build_dag(tmp_path, pop_vcf=pop_vcf) - # Each model-apply reads only its own haplotype's transfer, but - # both wait on both concat jobs, as the hand-wired DAG did - for hap in ("model-apply-hap1", "model-apply-hap2"): - assert deps_of(dag, hap) == [ - "merge-trim-hap1-concat", - "merge-trim-hap2-concat", - "patch", + # A model-apply reads only its own haplotype's transfer, so it + # waits on that haplotype's concat alone + for i in (1, 2): + assert deps_of(dag, f"model-apply-hap{i}") == [ + f"merge-trim-hap{i}-concat", ] + def test_the_haplotype_applies_wait_on_the_patch_without_a_pop_vcf( + self, tmp_path + ): + dag = build_dag(tmp_path) + + for hap in ("model-apply-hap1", "model-apply-hap2"): + assert deps_of(dag, hap) == ["patch"] + def test_each_transfer_is_tagged(self, tmp_path): pop_vcf = tmp_path / "population.vcf.gz" pop_vcf.touch() @@ -157,12 +163,12 @@ def test_each_transfer_is_tagged(self, tmp_path): "merge-trim-hap2-concat", "merge-trim-unphased-concat", } + # With a transfer, the apply waits on the concat alone -- the + # concat already depends on the raw-VCF producer assert deps_of(dag, "model-apply-diploid") == [ - "dnascope-diploid", "merge-trim-diploid-concat", ] assert deps_of(dag, "model-apply-unphased") == [ - "diploid-patch", "merge-trim-unphased-concat", ] diff --git a/tests/unit/test_stage_alignment.py b/tests/unit/test_stage_alignment.py index 3896437..e4b7df2 100644 --- a/tests/unit/test_stage_alignment.py +++ b/tests/unit/test_stage_alignment.py @@ -69,15 +69,6 @@ def test_find_unzip_falls_back_to_gzip(self, caplog): assert "igzip is recommended" in caplog.text assert caplog.records[0].levelno == logging.WARNING - def test_find_unzip_takes_the_log_level(self, caplog): - logger = logging.getLogger("test-unzip-level") - with patch( - "sentieon_cli.stages.alignment.shutil.which", return_value=None - ): - with caplog.at_level(logging.INFO, logger=logger.name): - assert find_unzip(logger, logging.INFO) == "gzip" - assert caplog.records[0].levelno == logging.INFO - class TestBwaRealignStage: """Re-aligning BAM/CRAM input with bwa""" @@ -143,7 +134,7 @@ def test_bam_format_without_duplicate_marking(self, tmp_path, rg_lines): tmp_path / "output_bwa_sorted_1.bam", ] - def test_bam_cleanup_paths(self, tmp_path, rg_lines): + def test_intermediate_cleanup_paths(self, tmp_path, rg_lines): result = self.make(tmp_path).build() assert result.cleanup_paths == [ @@ -153,15 +144,10 @@ def test_bam_cleanup_paths(self, tmp_path, rg_lines): pathlib.Path(str(tmp_path / "bwa_sorted_1.bam") + ".bai"), ] - def test_cram_cleanup_paths_add_the_crai(self, tmp_path, rg_lines): + def test_final_outputs_have_no_cleanup_paths(self, tmp_path, rg_lines): result = self.make(tmp_path, duplicate_marking="none").build() - out = str(tmp_path / "output_bwa_sorted_0.cram") - assert result.cleanup_paths[:3] == [ - pathlib.Path(out), - pathlib.Path(out + ".bai"), - pathlib.Path(out + ".crai"), - ] + assert result.cleanup_paths == [] def test_collate_and_input_ref(self, tmp_path, rg_lines): result = self.make( @@ -254,17 +240,24 @@ def test_unzip_reaches_the_command(self, tmp_path): assert "igzip -dc" in str(result.jobs[0].shell) - def test_outputs_and_cleanup_paths(self, tmp_path): - result = self.make(tmp_path, duplicate_marking="none").build() + def test_intermediate_outputs_and_cleanup_paths(self, tmp_path): + result = self.make(tmp_path).build() - out = tmp_path / "output_bwa_sorted_fq_0_0.cram" + out = tmp_path / "bwa_sorted_fq_0_0.bam" assert result.outputs == [out] assert result.cleanup_paths == [ out, pathlib.Path(str(out) + ".bai"), - pathlib.Path(str(out) + ".crai"), ] + def test_final_outputs_have_no_cleanup_paths(self, tmp_path): + result = self.make(tmp_path, duplicate_marking="none").build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_fq_0_0.cram" + ] + assert result.cleanup_paths == [] + class TestMinimap2RealignStage: """Re-aligning long-read input with minimap2""" diff --git a/tests/unit/test_stage_small_variants.py b/tests/unit/test_stage_small_variants.py index 2a561ec..56ae89f 100644 --- a/tests/unit/test_stage_small_variants.py +++ b/tests/unit/test_stage_small_variants.py @@ -345,9 +345,9 @@ def test_both_halves(self, tmp_path): assert f"--vcf {tmp_path}/transfer.vcf.gz" in str( result.apply_job.shell ) - # ... and waits on both the transfer and the raw-VCF producer + # ... and waits on the transfer alone; the concat already depends + # on the raw-VCF producer assert dag.waiting_jobs[result.apply_job] == { - upstream, result.transfer.concat_job, } assert result.terminal == {result.apply_job} From 7ab0e4e1e9c71eb142dd76a078aa1deda677ed57 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 3 Sep 2026 22:13:55 -0700 Subject: [PATCH 12/20] Move short-read preprocessing, mosdepth and LongReadSV into stages `ShortReadPreprocessingStage` carries the duplicate-marking and metrics policy that lived in `DNAscopePipeline.add_sr_preprocessing`; `MosdepthStage` and `LongReadSVStage` replace the raw job construction in the long-read pipeline's `mosdepth` and `call_svs`. The alignment, mosdepth and LongReadSV version constants move next to the stages they guard. The pipeline methods remain as thin wrappers that hold the version-check policy, so the hybrid pipeline, which still inherits them, is unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 151 ++--------- sentieon_cli/dnascope_longread.py | 88 ++----- sentieon_cli/stages/alignment.py | 20 ++ sentieon_cli/stages/metrics.py | 52 ++++ sentieon_cli/stages/preprocessing.py | 179 +++++++++++++ sentieon_cli/stages/sv.py | 70 +++++ tests/unit/test_stage_metrics.py | 90 ++++++- tests/unit/test_stage_preprocessing.py | 342 +++++++++++++++++++++++++ tests/unit/test_stage_sv.py | 144 +++++++++++ 9 files changed, 940 insertions(+), 196 deletions(-) create mode 100644 sentieon_cli/stages/preprocessing.py create mode 100644 sentieon_cli/stages/sv.py create mode 100644 tests/unit/test_stage_preprocessing.py create mode 100644 tests/unit/test_stage_sv.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 431ea21..3d16a5d 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -4,7 +4,6 @@ import argparse import copy -from dataclasses import dataclass, field import itertools import os import pathlib @@ -16,30 +15,25 @@ from .dag import DAG from .driver import ( - AlignmentStat, BaseAlgo, - BaseDistributionByCycle, - CoverageMetrics, DNAscope, - GCBias, - HsMetricAlgo, - InsertSizeMetricAlgo, - MeanQualityByCycle, - QualDistribution, SVSolver, - WgsMetricsAlgo, ) from .job import Job from .pipeline import BasePipeline from .stages.alignment import ( + BWA_FASTQ_MIN_VERSIONS, + BWA_REALIGN_MIN_VERSIONS, AlignResult, BwaFastqStage, BwaRealignStage, find_unzip, ) from .stages.base import StageContext, driver_job, rm_job -from .stages.dedup import DedupStage -from .stages.metrics import MetricsPaths, MetricsStage +from .stages.preprocessing import ( + ShortReadPreprocessingStage, + SrPreprocessingResult, +) from .stages.small_variants import ( ApplySpec, DNAscopeStage, @@ -55,35 +49,11 @@ total_memory, ) -ALN_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), - "samtools": packaging.version.Version("1.16"), -} - -FQ_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - VARIANTS_MIN_VERSIONS = { "sentieon driver": packaging.version.Version("202308"), } -@dataclass -class SrPreprocessing: - """The short-read dedup and metrics jobs added to a DAG. - - * ``deduped`` -- the alignment to call variants from; the input files - unchanged when duplicate marking is off. - * ``dedup_job`` -- the Dedup job, absent when duplicate marking is off. - * ``qc_jobs`` -- every metrics-producing job, for MultiQC to wait on. - """ - - deduped: List[pathlib.Path] - dedup_job: Optional[Job] = None - qc_jobs: List[Job] = field(default_factory=list) - - class DNAscopePipeline(BasePipeline): """The DNAscope pipeline""" @@ -463,7 +433,9 @@ def add_sr_input_alignment( """Align input BAM/CRAM/uBAM/uCRAM files with bwa""" bundle = self.required(self.model_bundle, "model_bundle") - require_versions(ALN_MIN_VERSIONS, skip=self.skip_version_check) + require_versions( + BWA_REALIGN_MIN_VERSIONS, skip=self.skip_version_check + ) return BwaRealignStage( ctx=ctx, @@ -489,7 +461,7 @@ def add_sr_fastq_alignment( jobs=[], terminal=set(), outputs=[], cleanup_paths=[] ) - require_versions(FQ_MIN_VERSIONS, skip=self.skip_version_check) + require_versions(BWA_FASTQ_MIN_VERSIONS, skip=self.skip_version_check) return BwaFastqStage( ctx=ctx, @@ -506,113 +478,24 @@ def add_sr_fastq_alignment( util_sort_args=self.util_sort_args, ).add_to(dag) - def sr_metrics_algos(self, paths: MetricsPaths) -> List[BaseAlgo]: - """The metrics that do not need duplicate-marked reads""" - algos: List[BaseAlgo] = [ - MeanQualityByCycle(paths.mean_qual_by_cycle), - BaseDistributionByCycle(paths.base_distribution_by_cycle), - QualDistribution(paths.qual_distribution), - AlignmentStat(paths.alignment_stat), - ] - if self.assay == "WGS": - algos.append(GCBias(paths.gc_bias, summary=paths.gc_bias_summary)) - return algos - def add_sr_preprocessing( self, dag: DAG, ctx: StageContext, sample_input: List[pathlib.Path], upstream: Set[Job], - ) -> SrPreprocessing: + ) -> SrPreprocessingResult: """Mark duplicates and collect metrics from the short reads""" - suffix = "bam" if self.bam_format else "cram" - - paths = MetricsPaths.from_output_vcf(ctx.output_vcf) - paths.ensure_dir(self.dry_run) - - if self.sr_duplicate_marking == "none": - # Without duplicate marking there is nothing to collect after - # the fact, so every metric comes from one pass over the input - if self.skip_metrics: - return SrPreprocessing( - deduped=sample_input, dedup_job=None, qc_jobs=[] - ) - metrics_result = MetricsStage( - ctx=ctx, - inputs=sample_input, - algos=[ - InsertSizeMetricAlgo(paths.insert_size), - *self.sr_metrics_algos(paths), - ], - name="metrics", - task_name="metrics", - threads=ctx.cores, - ).add_to(dag, upstream) - return SrPreprocessing( - deduped=sample_input, - dedup_job=None, - qc_jobs=list(metrics_result.jobs), - ) - - # Metrics that ride along on the LocusCollector pass - lc_extra_algos: List[BaseAlgo] = [] - if not self.skip_metrics: - # Prefer to run InsertSizeMetricAlgo after duplicate marking - if self.assay == "WES" and not self.bed: - lc_extra_algos.append(InsertSizeMetricAlgo(paths.insert_size)) - lc_extra_algos.extend(self.sr_metrics_algos(paths)) - - deduped = pathlib.Path( - str(ctx.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") - ) - dedup_result = DedupStage( + return ShortReadPreprocessingStage( ctx=ctx, inputs=sample_input, - output=deduped, - score_file=paths.score, + duplicate_marking=self.sr_duplicate_marking, consensus=self.consensus, - rmdup=(self.sr_duplicate_marking == "rmdup"), - cram_write_options="version=3.0,compressor=rans", - dedup_metrics=paths.dedup_metrics, - lc_extra_algos=lc_extra_algos, + skip_metrics=self.skip_metrics, + assay=self.assay, + bed=self.bed, + bam_format=self.bam_format, ).add_to(dag, upstream) - qc_jobs: List[Job] = [dedup_result.lc_job] - - # HsMetricAlgo and WgsMetricsAlgo run after duplicate marking to - # account for duplicate reads - post_algos: List[BaseAlgo] = [] - rehead_metrics: Optional[pathlib.Path] = None - if not self.skip_metrics: - if self.assay == "WES" and self.bed: - post_algos = [ - HsMetricAlgo(paths.hybrid_selection, self.bed, self.bed), - InsertSizeMetricAlgo(paths.insert_size), - ] - elif self.assay == "WGS": - post_algos = [ - InsertSizeMetricAlgo(paths.insert_size), - WgsMetricsAlgo(paths.wgs, include_unpaired="true"), - CoverageMetrics(paths.coverage), - ] - # Rehead WGS metrics so they are recognized by MultiQC - rehead_metrics = paths.wgs - if post_algos: - metrics_result = MetricsStage( - ctx=ctx, - inputs=[deduped], - algos=post_algos, - interval=self.bed, - rehead_metrics=rehead_metrics, - threads=0, # Run metrics in the background - ).add_to(dag, {dedup_result.dedup_job}) - qc_jobs.extend(metrics_result.jobs) - - return SrPreprocessing( - deduped=[deduped], - dedup_job=dedup_result.dedup_job, - qc_jobs=qc_jobs, - ) def add_small_variant_calling( self, diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index 0ee2864..79e3a54 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -18,10 +18,8 @@ from .dag import DAG from .driver import ( BaseAlgo, - Driver, DNAscope, DNAscopeHP, - LongReadSV, RepeatModel, VariantPhaser, ) @@ -34,18 +32,22 @@ ) from .shell_pipeline import Command, Pipeline from .stages.alignment import ( + MINIMAP2_FASTQ_MIN_VERSIONS, + MINIMAP2_REALIGN_MIN_VERSIONS, Minimap2FastqStage, Minimap2RealignStage, ReadWriterStage, find_unzip, ) from .stages.base import StageContext, driver_job +from .stages.metrics import MOSDEPTH_MIN_VERSIONS, MosdepthStage from .stages.small_variants import ( ApplySpec, DNAscopeStage, TransferApplyStage, TransferSpec, ) +from .stages.sv import LONGREADSV_MIN_VERSIONS, LongReadSVStage from .stages.transfer import TransferConfig from .util import ( __version__, @@ -63,23 +65,6 @@ "bedtools": None, } -SV_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - -ALN_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), - "samtools": packaging.version.Version("1.16"), -} - -FQ_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - -MOSDEPTH_MIN_VERSIONS = { - "mosdepth": packaging.version.Version("0.2.6"), -} - MERGE_MIN_VERSIONS = { "sentieon driver": None, } @@ -495,7 +480,9 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ output_vcf = self.required(self.output_vcf, "output_vcf") bundle = self.required(self.model_bundle, "model_bundle") - require_versions(ALN_MIN_VERSIONS, skip=self.skip_version_check) + require_versions( + MINIMAP2_REALIGN_MIN_VERSIONS, skip=self.skip_version_check + ) result = Minimap2RealignStage( ctx=self.stage_context(), @@ -518,7 +505,9 @@ def lr_align_fastq(self) -> Tuple[List[pathlib.Path], Set[Job]]: if self.fastq is None and self.readgroups is None: return ([], set()) - require_versions(FQ_MIN_VERSIONS, skip=self.skip_version_check) + require_versions( + MINIMAP2_FASTQ_MIN_VERSIONS, skip=self.skip_version_check + ) result = Minimap2FastqStage( ctx=self.stage_context(), @@ -544,25 +533,14 @@ def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: ) return set() - mosdepth_jobs = set() - for i, input_file in enumerate(sample_input): - mosdepth_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_mosdepth_{i}") - ) - mosdepth_jobs.add( - Job( - cmds.cmd_mosdepth( - input_file, - mosdepth_dir, - fasta=self.reference, - threads=self.cores, - ), - f"mosdepth-{i}", - 0, # Run in background - task_name="metrics", - ) + return set( + MosdepthStage( + ctx=self.stage_context(), + inputs=sample_input, ) - return mosdepth_jobs + .build() + .jobs + ) def merge_input_files( self, sample_input: List[pathlib.Path] @@ -1185,28 +1163,16 @@ def call_svs( Call SVs using Sentieon LongReadSV """ bundle = self.required(self.model_bundle, "model_bundle") - require_versions(SV_MIN_VERSIONS, skip=self.skip_version_check) + require_versions(LONGREADSV_MIN_VERSIONS, skip=self.skip_version_check) - sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") - ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=replace_rg, - input=sample_input, - interval=self.bed, - ) - driver.add_algo( - LongReadSV( - sv_vcf, - bundle.joinpath("longreadsv.model"), + return ( + LongReadSVStage( + ctx=self.stage_context(), + inputs=sample_input, + model=bundle.joinpath("longreadsv.model"), + interval=self.bed, + replace_rg=replace_rg, ) + .build() + .job ) - longreadsv_job = Job( - Pipeline(Command(*driver.build_cmd())), - "LongReadSV", - self.cores, - task_name="sv-calling", - ) - return longreadsv_job diff --git a/sentieon_cli/stages/alignment.py b/sentieon_cli/stages/alignment.py index d12e6dc..ae6769a 100644 --- a/sentieon_cli/stages/alignment.py +++ b/sentieon_cli/stages/alignment.py @@ -15,6 +15,8 @@ import shutil from typing import Iterable, List, Optional, Union +import packaging.version + from .. import command_strings as cmds from ..dag import DAG from ..driver import ReadWriter @@ -24,6 +26,24 @@ # The `sentieon util sort` arguments every aligner defaults to DEFAULT_UTIL_SORT_ARGS = "--cram_write_options version=3.0,compressor=rans" +BWA_REALIGN_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), + "samtools": packaging.version.Version("1.16"), +} + +BWA_FASTQ_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + +MINIMAP2_REALIGN_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), + "samtools": packaging.version.Version("1.16"), +} + +MINIMAP2_FASTQ_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + def aln_suffix(bam_format: bool) -> str: """The file extension for an aligned output file""" diff --git a/sentieon_cli/stages/metrics.py b/sentieon_cli/stages/metrics.py index ee9d82b..e03c9a3 100644 --- a/sentieon_cli/stages/metrics.py +++ b/sentieon_cli/stages/metrics.py @@ -8,12 +8,19 @@ import sys from typing import Iterable, List, Optional +import packaging.version + +from .. import command_strings as cmds from ..dag import DAG from ..driver import BaseAlgo from ..job import Job from ..shell_pipeline import Command, Pipeline from .base import Stage, StageResult, driver_job +MOSDEPTH_MIN_VERSIONS = { + "mosdepth": packaging.version.Version("0.2.6"), +} + @dataclass(frozen=True) class MetricsPaths: @@ -179,3 +186,48 @@ def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> MetricsResult: metrics_job=metrics, rehead_job=rehead, ) + + +@dataclass(kw_only=True) +class MosdepthResult(StageResult): + """The jobs of `MosdepthStage`""" + + +@dataclass(kw_only=True) +class MosdepthStage(Stage): + """Coverage QC with mosdepth, one job per input file. + + The jobs are independent of each other, so ``build`` constructs them + all and ``add_to`` inserts each with the same dependencies. + """ + + inputs: List[pathlib.Path] + + def build(self) -> MosdepthResult: + """Construct this stage's jobs without touching a DAG""" + jobs: List[Job] = [] + for i, input_file in enumerate(self.inputs): + mosdepth_dir = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", f"_mosdepth_{i}") + ) + jobs.append( + Job( + cmds.cmd_mosdepth( + input_file, + mosdepth_dir, + fasta=self.ctx.reference, + threads=self.ctx.cores, + ), + f"mosdepth-{i}", + 0, # Run in background + task_name="metrics", + ) + ) + return MosdepthResult(jobs=jobs, terminal=set(jobs)) + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> MosdepthResult: + deps = set(upstream) + result = self.build() + for job in result.jobs: + dag.add_job(job, deps) + return result diff --git a/sentieon_cli/stages/preprocessing.py b/sentieon_cli/stages/preprocessing.py new file mode 100644 index 0000000..f2a29bd --- /dev/null +++ b/sentieon_cli/stages/preprocessing.py @@ -0,0 +1,179 @@ +""" +Short-read preprocessing: duplicate marking and metrics collection +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +from ..dag import DAG +from ..driver import ( + AlignmentStat, + BaseAlgo, + BaseDistributionByCycle, + CoverageMetrics, + GCBias, + HsMetricAlgo, + InsertSizeMetricAlgo, + MeanQualityByCycle, + QualDistribution, + WgsMetricsAlgo, +) +from ..job import Job +from .base import Stage, StageResult +from .dedup import DedupStage +from .metrics import MetricsPaths, MetricsStage + + +def pre_dedup_metrics_algos(paths: MetricsPaths, assay: str) -> List[BaseAlgo]: + """The metrics that do not need duplicate-marked reads""" + algos: List[BaseAlgo] = [ + MeanQualityByCycle(paths.mean_qual_by_cycle), + BaseDistributionByCycle(paths.base_distribution_by_cycle), + QualDistribution(paths.qual_distribution), + AlignmentStat(paths.alignment_stat), + ] + if assay == "WGS": + algos.append(GCBias(paths.gc_bias, summary=paths.gc_bias_summary)) + return algos + + +@dataclass(kw_only=True) +class SrPreprocessingResult(StageResult): + """The short-read dedup and metrics jobs added to a DAG. + + * ``deduped`` -- the alignment to call variants from; the input files + unchanged when duplicate marking is off. + * ``dedup_job`` -- the Dedup job, absent when duplicate marking is off. + * ``qc_jobs`` -- every metrics-producing job, for MultiQC to wait on. + """ + + deduped: List[pathlib.Path] + dedup_job: Optional[Job] + qc_jobs: List[Job] + + +@dataclass(kw_only=True) +class ShortReadPreprocessingStage(Stage): + """Mark duplicates and collect metrics from the short reads. + + Which metrics are collected, and whether they ride along on the + LocusCollector pass or run after duplicate marking, depends on the + assay and on whether a BED file restricts the run. + + ``terminal`` is the Dedup job, so downstream stages read the + deduplicated alignment. With ``duplicate_marking="none"`` there is no + Dedup job and the stage adds only a leaf metrics job that nothing + should wait on, so ``terminal`` is empty and callers depend on the + jobs that produced ``inputs`` instead. + """ + + inputs: List[pathlib.Path] + duplicate_marking: str = "markdup" + consensus: bool = False + skip_metrics: bool = False + assay: str = "WGS" + bed: Optional[pathlib.Path] = None + bam_format: bool = False + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> SrPreprocessingResult: + ctx = self.ctx + suffix = "bam" if self.bam_format else "cram" + + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) + paths.ensure_dir(ctx.dry_run) + + if self.duplicate_marking == "none": + # Without duplicate marking there is nothing to collect after + # the fact, so every metric comes from one pass over the input + if self.skip_metrics: + return SrPreprocessingResult( + jobs=[], + terminal=set(), + deduped=self.inputs, + dedup_job=None, + qc_jobs=[], + ) + metrics_result = MetricsStage( + ctx=ctx, + inputs=self.inputs, + algos=[ + InsertSizeMetricAlgo(paths.insert_size), + *pre_dedup_metrics_algos(paths, self.assay), + ], + name="metrics", + task_name="metrics", + threads=ctx.cores, + ).add_to(dag, upstream) + return SrPreprocessingResult( + jobs=list(metrics_result.jobs), + terminal=set(), + deduped=self.inputs, + dedup_job=None, + qc_jobs=list(metrics_result.jobs), + ) + + # Metrics that ride along on the LocusCollector pass + lc_extra_algos: List[BaseAlgo] = [] + if not self.skip_metrics: + # Prefer to run InsertSizeMetricAlgo after duplicate marking + if self.assay == "WES" and not self.bed: + lc_extra_algos.append(InsertSizeMetricAlgo(paths.insert_size)) + lc_extra_algos.extend(pre_dedup_metrics_algos(paths, self.assay)) + + deduped = pathlib.Path( + str(ctx.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") + ) + dedup_result = DedupStage( + ctx=ctx, + inputs=self.inputs, + output=deduped, + score_file=paths.score, + consensus=self.consensus, + rmdup=(self.duplicate_marking == "rmdup"), + cram_write_options="version=3.0,compressor=rans", + dedup_metrics=paths.dedup_metrics, + lc_extra_algos=lc_extra_algos, + ).add_to(dag, upstream) + jobs: List[Job] = list(dedup_result.jobs) + qc_jobs: List[Job] = [dedup_result.lc_job] + + # HsMetricAlgo and WgsMetricsAlgo run after duplicate marking to + # account for duplicate reads + post_algos: List[BaseAlgo] = [] + rehead_metrics: Optional[pathlib.Path] = None + if not self.skip_metrics: + if self.assay == "WES" and self.bed: + post_algos = [ + HsMetricAlgo(paths.hybrid_selection, self.bed, self.bed), + InsertSizeMetricAlgo(paths.insert_size), + ] + elif self.assay == "WGS": + post_algos = [ + InsertSizeMetricAlgo(paths.insert_size), + WgsMetricsAlgo(paths.wgs, include_unpaired="true"), + CoverageMetrics(paths.coverage), + ] + # Rehead WGS metrics so they are recognized by MultiQC + rehead_metrics = paths.wgs + if post_algos: + metrics_result = MetricsStage( + ctx=ctx, + inputs=[deduped], + algos=post_algos, + interval=self.bed, + rehead_metrics=rehead_metrics, + threads=0, # Run metrics in the background + ).add_to(dag, {dedup_result.dedup_job}) + jobs.extend(metrics_result.jobs) + qc_jobs.extend(metrics_result.jobs) + + return SrPreprocessingResult( + jobs=jobs, + terminal={dedup_result.dedup_job}, + deduped=[deduped], + dedup_job=dedup_result.dedup_job, + qc_jobs=qc_jobs, + ) diff --git a/sentieon_cli/stages/sv.py b/sentieon_cli/stages/sv.py new file mode 100644 index 0000000..03e0eef --- /dev/null +++ b/sentieon_cli/stages/sv.py @@ -0,0 +1,70 @@ +""" +Structural variant calling with LongReadSV +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +import packaging.version + +from ..dag import DAG +from ..driver import LongReadSV +from ..job import Job +from .base import Stage, StageResult, driver_job + +LONGREADSV_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + + +@dataclass(kw_only=True) +class LongReadSVResult(StageResult): + """The job and output of `LongReadSVStage`""" + + job: Job + sv_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class LongReadSVStage(Stage): + """Call SVs from long reads with Sentieon LongReadSV. + + The output VCF is derived from the run's output VCF, so the caller + only supplies the alignment and the model. + """ + + inputs: List[pathlib.Path] + model: pathlib.Path + interval: Optional[pathlib.Path] = None + replace_rg: Optional[List[List[str]]] = None + name: str = "LongReadSV" + task_name: str = "sv-calling" + + def build(self) -> LongReadSVResult: + """Construct this stage's job without touching a DAG""" + sv_vcf = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") + ) + job = driver_job( + self.ctx, + [LongReadSV(sv_vcf, self.model)], + inputs=self.inputs, + interval=self.interval, + replace_rg=self.replace_rg, + name=self.name, + task_name=self.task_name, + ) + return LongReadSVResult( + jobs=[job], + terminal={job}, + job=job, + sv_vcf=sv_vcf, + ) + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> LongReadSVResult: + result = self.build() + dag.add_job(result.job, set(upstream)) + return result diff --git a/tests/unit/test_stage_metrics.py b/tests/unit/test_stage_metrics.py index 1792560..7170b1d 100644 --- a/tests/unit/test_stage_metrics.py +++ b/tests/unit/test_stage_metrics.py @@ -12,7 +12,11 @@ WgsMetricsAlgo, ) from sentieon_cli.stages.base import StageContext, rm_job -from sentieon_cli.stages.metrics import MetricsPaths, MetricsStage +from sentieon_cli.stages.metrics import ( + MetricsPaths, + MetricsStage, + MosdepthStage, +) def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: @@ -242,3 +246,87 @@ def test_metrics_is_a_root_without_upstream(self, tmp_path): result = make_stage(tmp_path).add_to(dag) assert result.metrics_job in dag.ready_jobs + + +class TestMosdepth: + """Coverage QC, one job per input file""" + + def test_mosdepth_command(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(DAG()) + + (job,) = result.jobs + assert str(job.shell) == ( + f"mosdepth --fasta {tmp_path}/ref.fa --threads 4 " + "--by 500 --no-per-base --use-median " + "-T 1,3,5,10,15,20,30,40,50 " + f"{tmp_path}/output_mosdepth_0 {tmp_path}/sample.bam" + ) + + def test_one_job_per_input(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + assert [job.name for job in result.jobs] == [ + "mosdepth-0", + "mosdepth-1", + ] + assert f"{tmp_path}/output_mosdepth_0" in str(result.jobs[0].shell) + assert f"{tmp_path}/output_mosdepth_1" in str(result.jobs[1].shell) + + def test_job_metadata(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(DAG()) + + (job,) = result.jobs + assert job.task_name == "metrics" + # mosdepth runs in the background + assert job.threads == 0 + + def test_no_inputs_adds_nothing(self, tmp_path): + dag = DAG() + result = MosdepthStage(ctx=make_ctx(tmp_path), inputs=[]).add_to(dag) + + assert result.jobs == [] + assert result.terminal == set() + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_build_inserts_nothing(self, tmp_path): + dag = DAG() + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).build() + + assert result.terminal == set(result.jobs) + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_upstream_lands_on_every_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(dag, [upstream]) + + for job in result.jobs: + assert dag.waiting_jobs[job] == {upstream} + + def test_roots_without_upstream(self, tmp_path): + dag = DAG() + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(dag) + + assert result.jobs[0] in dag.ready_jobs diff --git a/tests/unit/test_stage_preprocessing.py b/tests/unit/test_stage_preprocessing.py new file mode 100644 index 0000000..6ccf8b7 --- /dev/null +++ b/tests/unit/test_stage_preprocessing.py @@ -0,0 +1,342 @@ +""" +Unit tests for the short-read preprocessing stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.metrics import MetricsPaths +from sentieon_cli.stages.preprocessing import ( + ShortReadPreprocessingStage, + pre_dedup_metrics_algos, +) + +# The pre-dedup metrics, in the order the stage collects them +PRE_DEDUP_ALGOS = [ + "MeanQualityByCycle", + "BaseDistributionByCycle", + "QualDistribution", + "AlignmentStat", +] + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage( + tmp_path: pathlib.Path, **kwargs +) -> ShortReadPreprocessingStage: + """A preprocessing stage over one input alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ) + defaults.update(kwargs) + return ShortReadPreprocessingStage(**defaults) # type: ignore[arg-type] + + +def algo_order(shell: str) -> list: + """The algos a driver command runs, in command-line order""" + tokens = shell.split() + return [ + tokens[i + 1] + for i, token in enumerate(tokens[:-1]) + if token == "--algo" + ] + + +class TestPreDedupAlgos: + """The metrics that do not need duplicate-marked reads""" + + def test_wgs_adds_gc_bias(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WGS") + + assert [algo.name for algo in algos] == PRE_DEDUP_ALGOS + ["GCBias"] + + def test_wes_omits_gc_bias(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WES") + + assert [algo.name for algo in algos] == PRE_DEDUP_ALGOS + + def test_the_outputs_come_from_the_metrics_paths(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WGS") + + assert algos[0].output == paths.mean_qual_by_cycle + assert algos[1].output == paths.base_distribution_by_cycle + assert algos[2].output == paths.qual_distribution + assert algos[3].output == paths.alignment_stat + assert algos[4].output == paths.gc_bias + assert algos[4].summary == paths.gc_bias_summary + + +class TestNoDuplicateMarking: + """`duplicate_marking="none"` collects everything in one pass""" + + def test_skip_metrics_adds_no_jobs(self, tmp_path): + dag = DAG() + inputs = [tmp_path / "sample.bam"] + result = make_stage( + tmp_path, + inputs=inputs, + duplicate_marking="none", + skip_metrics=True, + ).add_to(dag) + + assert result.jobs == [] + assert result.terminal == set() + assert result.qc_jobs == [] + assert result.dedup_job is None + assert result.deduped == inputs + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_one_metrics_job(self, tmp_path): + inputs = [tmp_path / "sample.bam"] + result = make_stage( + tmp_path, inputs=inputs, duplicate_marking="none" + ).add_to(DAG()) + + (metrics,) = result.jobs + assert metrics.name == "metrics" + assert metrics.task_name == "metrics" + # This pass reads the whole alignment, so it takes every core + assert metrics.threads == 4 + assert result.dedup_job is None + assert result.deduped == inputs + assert result.qc_jobs == [metrics] + + def test_wgs_algo_order(self, tmp_path): + result = make_stage(tmp_path, duplicate_marking="none").add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ["GCBias"] + ) + + def test_wes_omits_gc_bias(self, tmp_path): + result = make_stage( + tmp_path, duplicate_marking="none", assay="WES" + ).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ) + + def test_the_metrics_job_reads_the_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + duplicate_marking="none", + ).add_to(DAG()) + + shell = str(result.jobs[0].shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + def test_upstream_lands_on_the_metrics_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path, duplicate_marking="none").add_to( + dag, [upstream] + ) + + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + # Nothing should wait on this leaf + assert result.terminal == set() + + +class TestMarkdupWgs: + """The default: duplicate marking plus WGS metrics""" + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + lc, dedup, metrics, rehead = result.jobs + assert lc.name == "locuscollector" + assert dedup.name == "dedup" + assert metrics.name == "metrics" + assert rehead.name == "rehead-metrics" + assert result.dedup_job is dedup + assert result.terminal == {dedup} + assert result.qc_jobs == [lc, metrics, rehead] + assert result.deduped == [tmp_path / "output_deduped.cram"] + + def test_locuscollector_carries_the_pre_dedup_metrics(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["LocusCollector"] + PRE_DEDUP_ALGOS + ["GCBias"] + ) + + def test_dedup_writes_metrics_and_cram_options(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path).add_to(DAG()) + + shell = str(result.dedup_job.shell) + assert f"--metrics {paths.dedup_metrics}" in shell + assert "--cram_write_options version=3.0,compressor=rans" in shell + assert "--rmdup" not in shell + + def test_post_dedup_metrics(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + metrics = result.jobs[2] + assert algo_order(str(metrics.shell)) == [ + "InsertSizeMetricAlgo", + "WgsMetricsAlgo", + "CoverageMetrics", + ] + assert "--include_unpaired true" in str(metrics.shell) + # Metrics run in the background + assert metrics.threads == 0 + assert f"--input {tmp_path}/output_deduped.cram" in str(metrics.shell) + + def test_the_post_job_waits_on_dedup(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert dag.waiting_jobs[result.jobs[2]] == {result.dedup_job} + assert dag.waiting_jobs[result.jobs[3]] == {result.jobs[2]} + + def test_upstream_lands_on_locuscollector(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + + def test_insert_size_is_not_on_the_locuscollector_pass(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "InsertSizeMetricAlgo" not in algo_order( + str(result.jobs[0].shell) + ) + + +class TestWes: + """WES runs collect hybrid-selection metrics instead""" + + def test_with_a_bed(self, tmp_path): + bed = tmp_path / "regions.bed" + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path, assay="WES", bed=bed).add_to(DAG()) + + lc, dedup, metrics = result.jobs + assert algo_order(str(metrics.shell)) == [ + "HsMetricAlgo", + "InsertSizeMetricAlgo", + ] + assert f"--interval {bed}" in str(metrics.shell) + assert f"--targets_list {bed}" in str(metrics.shell) + assert str(paths.hybrid_selection) in str(metrics.shell) + # No WGS metrics means no rehead job + assert result.qc_jobs == [lc, metrics] + assert result.terminal == {dedup} + + def test_insert_size_is_not_on_the_locuscollector_pass_with_a_bed( + self, tmp_path + ): + result = make_stage( + tmp_path, assay="WES", bed=tmp_path / "regions.bed" + ).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["LocusCollector"] + PRE_DEDUP_ALGOS + ) + + def test_without_a_bed(self, tmp_path): + result = make_stage(tmp_path, assay="WES").add_to(DAG()) + + lc, dedup = result.jobs + # InsertSize moves onto the LocusCollector pass + assert algo_order(str(lc.shell)) == ( + ["LocusCollector", "InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ) + assert result.qc_jobs == [lc] + assert result.terminal == {dedup} + + +class TestOptions: + """The flags a caller passes through""" + + def test_rmdup(self, tmp_path): + result = make_stage(tmp_path, duplicate_marking="rmdup").add_to(DAG()) + + assert "--rmdup" in str(result.dedup_job.shell) + + def test_consensus(self, tmp_path): + result = make_stage(tmp_path, consensus=True).add_to(DAG()) + + assert "--algo LocusCollector --consensus" in str(result.jobs[0].shell) + + def test_bam_format(self, tmp_path): + result = make_stage(tmp_path, bam_format=True).add_to(DAG()) + + assert result.deduped == [tmp_path / "output_deduped.bam"] + + def test_skip_metrics_keeps_duplicate_marking(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path, skip_metrics=True).add_to(DAG()) + + lc, dedup = result.jobs + assert algo_order(str(lc.shell)) == ["LocusCollector"] + # Dedup still writes its own metrics + assert f"--metrics {paths.dedup_metrics}" in str(dedup.shell) + assert result.qc_jobs == [lc] + assert result.terminal == {dedup} + + +class TestMetricsDirectory: + """The metrics directory is created before the jobs are built""" + + def test_created_when_not_a_dry_run(self, tmp_path): + ctx = StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=4, + dry_run=False, + skip_version_check=True, + ) + make_stage(tmp_path, ctx=ctx).add_to(DAG()) + + assert (tmp_path / "output_metrics").is_dir() + + def test_not_created_on_a_dry_run(self, tmp_path): + make_stage(tmp_path).add_to(DAG()) + + assert not (tmp_path / "output_metrics").exists() + + def test_created_even_when_no_jobs_are_added(self, tmp_path): + ctx = StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=4, + dry_run=False, + skip_version_check=True, + ) + make_stage( + tmp_path, + ctx=ctx, + duplicate_marking="none", + skip_metrics=True, + ).add_to(DAG()) + + assert (tmp_path / "output_metrics").is_dir() diff --git a/tests/unit/test_stage_sv.py b/tests/unit/test_stage_sv.py new file mode 100644 index 0000000..c7f86aa --- /dev/null +++ b/tests/unit/test_stage_sv.py @@ -0,0 +1,144 @@ +""" +Unit tests for the long-read SV calling stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.sv import LongReadSVStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> LongReadSVStage: + """A LongReadSVStage over one alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + model=tmp_path / "bundle" / "longreadsv.model", + ) + defaults.update(kwargs) + return LongReadSVStage(**defaults) # type: ignore[arg-type] + + +class TestCommand: + """The `sentieon driver` command the stage builds""" + + def test_longreadsv_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LongReadSV " + f"--model {tmp_path}/bundle/longreadsv.model " + f"{tmp_path}/output.sv.vcf.gz" + ) + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.job.shell) + + def test_without_an_interval(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--interval" not in str(result.job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, inputs=[tmp_path / "one.bam", tmp_path / "two.bam"] + ).add_to(DAG()) + + shell = str(result.job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + def test_replace_rg_interleaves_with_the_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + replace_rg=[["rg1=@RG\\tID:rg1"], ["rg2=@RG\\tID:rg2"]], + ).add_to(DAG()) + + shell = str(result.job.shell) + assert ( + "--replace_rg 'rg1=@RG\\tID:rg1' " + f"--input {tmp_path}/one.bam " + "--replace_rg 'rg2=@RG\\tID:rg2' " + f"--input {tmp_path}/two.bam" + ) in shell + + def test_without_replace_rg(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--replace_rg" not in str(result.job.shell) + + +class TestOutput: + """The SV VCF is derived from the run's output VCF""" + + def test_sv_vcf(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.sv_vcf == tmp_path / "output.sv.vcf.gz" + assert str(result.sv_vcf) in str(result.job.shell) + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_defaults(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "LongReadSV" + assert result.job.task_name == "sv-calling" + assert result.job.threads == 4 + + def test_custom_name_and_task(self, tmp_path): + result = make_stage( + tmp_path, name="longreadsv-lr", task_name="sv" + ).add_to(DAG()) + + assert result.job.name == "longreadsv-lr" + assert result.job.task_name == "sv" + + +class TestDagWiring: + """`build` touches no DAG; `add_to` inserts the job""" + + def test_build_inserts_nothing(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).build() + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_upstream_lands_on_the_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs From b3289b7bdeba4f6b4abdfc2e3fec851e8aabe478 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 3 Sep 2026 22:21:32 -0700 Subject: [PATCH 13/20] Make the hybrid pipeline a direct BasePipeline subclass `DNAscopeHybridPipeline` inherited from both `DNAscopePipeline` and `DNAscopeLRPipeline` to reuse six methods. It now subclasses `BasePipeline`, declares every attribute its arguments need, and composes the short-read preprocessing, minimap2 realignment, mosdepth and LongReadSV stages through its own small wrappers. The bwt_max_mem computation becomes `util.set_bwt_max_mem`, shared with the dnascope pipeline, and the long-read `call_svs` loses the `replace_rg` parameter only the hybrid used. Two latent defects go away with the inheritance: `--lr_fastq_taglist` now reaches the minimap2 command (the inherited method read the long-read pipeline's `fastq_taglist`), and a hybrid built without the argument parser gets `minimap2_args="-Y"`, its documented default, instead of the long-read pipeline's `-YL`. A new test asserts, for every pipeline, that each argument is declared in `__init__` with its argparse default. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 28 +-- sentieon_cli/dnascope_hybrid.py | 177 +++++++++++++++++-- sentieon_cli/dnascope_longread.py | 7 +- sentieon_cli/util.py | 30 ++++ tests/unit/test_dnascope_hybrid_longread.py | 186 ++++++++++++++++++++ tests/unit/test_pipeline_params.py | 48 +++++ tests/unit/test_pipeline_validation.py | 39 ++-- 7 files changed, 455 insertions(+), 60 deletions(-) create mode 100644 tests/unit/test_dnascope_hybrid_longread.py create mode 100644 tests/unit/test_pipeline_params.py diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 3d16a5d..66a489f 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -45,6 +45,7 @@ parse_rg_line, path_arg, require_versions, + set_bwt_max_mem, split_alignment, total_memory, ) @@ -319,8 +320,10 @@ def configure_alignment(self) -> None: if shm_ok: os.environ["SENTIEON_TMPDIR"] = "/dev/shm" - self.set_bwt_max_mem( - total_input_size if shm_ok else 0, n_alignment_jobs + set_bwt_max_mem( + total_input_size if shm_ok else 0, + n_alignment_jobs, + override=self.bwt_max_mem, ) def total_input_size(self) -> int: @@ -350,27 +353,6 @@ def check_shm(self, total_input_size: int, n_alignment_jobs) -> bool: return True return False - def set_bwt_max_mem( - self, - total_input_size: int, - n_alignment_jobs: int = 1, - ): - """Set the bwt_max_mem environment variable""" - if self.bwt_max_mem: - os.environ["bwt_max_mem"] = self.bwt_max_mem - return - - total_mem = total_memory() - total_mem_gb = total_mem / (1024.0**3) - align_mem_gb = ( - total_mem_gb - 4 - total_input_size / (1024.0**3) * 2.3 - ) # some memory for other system processes - bwa_mem_gb = max( - int((align_mem_gb / n_alignment_jobs) - 6), 0 - ) # some memory for other alignment processes - self.logger.debug("Setting bwt_max_mem to: %sG", bwa_mem_gb) - os.environ["bwt_max_mem"] = f"{bwa_mem_gb}G" - def build_dag(self) -> DAG: """Build the DAG for the pipeline""" self.logger.info("Building the DAG") diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index fa767fe..bc25793 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -24,20 +24,32 @@ HybridStage2, HybridStage3, ) -from .dnascope import DNAscopePipeline -from .dnascope_longread import DNAscopeLRPipeline from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BWA_FASTQ_MIN_VERSIONS, + MINIMAP2_REALIGN_MIN_VERSIONS, + AlignResult, + BwaFastqStage, + Minimap2RealignStage, + find_unzip, +) from .stages.base import StageContext, driver_job, rm_job from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.metrics import MOSDEPTH_MIN_VERSIONS, MosdepthStage from .stages.ploidy import PloidyStage +from .stages.preprocessing import ( + ShortReadPreprocessingStage, + SrPreprocessingResult, +) from .stages.small_variants import ( ApplySpec, DNAscopeStage, TransferApplyStage, TransferSpec, ) +from .stages.sv import LONGREADSV_MIN_VERSIONS, LongReadSVStage from .stages.transfer import TransferConfig from .util import ( __version__, @@ -46,8 +58,10 @@ path_arg, require_versions, sample_sex_arg, + set_bwt_max_mem, split_alignment, vcf_id, + versions_available, ) from .shard import ( determine_shards_from_fai, @@ -126,7 +140,7 @@ def __init__( ) -class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): +class DNAscopeHybridPipeline(BasePipeline): """The DNAscope Hybrid pipeline""" params = copy.deepcopy(BasePipeline.params) @@ -319,24 +333,43 @@ class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): def __init__(self) -> None: super().__init__() + # Required arguments + self.lr_aln: List[pathlib.Path] = [] + self.model_bundle: Optional[pathlib.Path] = None + self.sr_aln: List[pathlib.Path] = [] self.sr_r1_fastq: List[pathlib.Path] = [] self.sr_r2_fastq: List[pathlib.Path] = [] self.sr_readgroups: List[str] = [] - self.sr_aln: List[pathlib.Path] = [] - self.sr_duplicate_marking = "markdup" - self.lr_aln: List[pathlib.Path] = [] + # Additional arguments + self.bam_format = False + self.bed: Optional[pathlib.Path] = None + self.dbsnp: Optional[pathlib.Path] = None + self.gvcf = False self.lr_align_input = False self.lr_input_ref: Optional[pathlib.Path] = None self.par_bed: Optional[pathlib.Path] = None self.pop_vcf: Optional[pathlib.Path] = None - self.bam_format = False self.rgsm: Optional[str] = None + self.skip_cnv = False + self.skip_metrics = False + self.skip_mosdepth = False + self.skip_multiqc = False + self.skip_svs = False + self.sr_duplicate_marking = "markdup" + # Hidden arguments + self.bwa_args = "" + self.bwa_k = 100000000 + self.bwt_max_mem: Optional[str] = None self.lr_fastq_taglist = "*" - self.sr_read_filter: Optional[str] = None self.lr_read_filter: Optional[str] = None - self.assay = "WGS" + self.minimap2_args = "-Y" + self.no_split_alignment = False self.skip_model_apply = False self.skip_pop_vcf_id_check = False + self.sr_read_filter: Optional[str] = None + self.util_sort_args = ( + "--cram_write_options version=3.0,compressor=rans" + ) # Stashed by `build_dag` for the second, sex-aware DAG self.ploidy_json: Optional[pathlib.Path] = None self.cnv_sr_aln: List[pathlib.Path] = [] @@ -549,7 +582,7 @@ def configure(self) -> None: self.numa_nodes = split_alignment(self.cores) n_alignment_jobs = max(1, len(self.numa_nodes)) - self.set_bwt_max_mem(0, n_alignment_jobs) + set_bwt_max_mem(0, n_alignment_jobs, override=self.bwt_max_mem) def configure_readgroups(self) -> None: self.lr_aln_readgroups = self.all_readgroups[0] @@ -610,14 +643,12 @@ def build_dag(self) -> DAG: realign_jobs: Set[Job] = set() lr_aln = self.lr_aln if self.lr_align_input: - lr_aln, realign_jobs = self.lr_align_inputs() - for job in realign_jobs: - dag.add_job(job) + realign_result = self.add_lr_realignment(dag, ctx) + lr_aln = realign_result.outputs + realign_jobs = set(realign_result.jobs) if not self.skip_mosdepth: - mosdepth_jobs = self.mosdepth(sample_input=lr_aln) - for job in mosdepth_jobs: - dag.add_job(job, realign_jobs) + self.add_mosdepth(dag, ctx, lr_aln, realign_jobs) if not self.skip_multiqc: multiqc_job = self.multiqc() @@ -625,10 +656,13 @@ def build_dag(self) -> DAG: dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) if not self.skip_svs: - longreadsv_job = self.call_svs( - lr_aln, replace_rg=rg_info.replace_rg_args[0] + self.add_sv_calling( + dag, + ctx, + lr_aln, + rg_info.replace_rg_args[0], + realign_jobs, ) - dag.add_job(longreadsv_job, realign_jobs) sr_preprocessing_jobs: Set[Job] = set() sr_preprocessing_jobs.update(align_fastq_jobs) @@ -664,6 +698,111 @@ def build_dag(self) -> DAG: return dag + def add_sr_fastq_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: + """Align the short-read fastq files with bwa""" + bundle = self.required(self.model_bundle, "model_bundle") + + if not self.sr_r1_fastq and not self.sr_readgroups: + return AlignResult( + jobs=[], terminal=set(), outputs=[], cleanup_paths=[] + ) + + require_versions(BWA_FASTQ_MIN_VERSIONS, skip=self.skip_version_check) + + return BwaFastqStage( + ctx=ctx, + r1_fastq=self.sr_r1_fastq, + r2_fastq=self.sr_r2_fastq, + readgroups=self.sr_readgroups, + model_bundle=bundle, + numa_nodes=self.numa_nodes, + bam_format=self.bam_format, + duplicate_marking=self.sr_duplicate_marking, + unzip=find_unzip(self.logger), + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_sr_preprocessing( + self, + dag: DAG, + ctx: StageContext, + sr_aln: List[pathlib.Path], + upstream: Set[Job], + ) -> SrPreprocessingResult: + """Mark duplicates and collect metrics from the short reads""" + return ShortReadPreprocessingStage( + ctx=ctx, + inputs=sr_aln, + duplicate_marking=self.sr_duplicate_marking, + consensus=False, + skip_metrics=self.skip_metrics, + assay="WGS", + bed=self.bed, + bam_format=self.bam_format, + ).add_to(dag, upstream) + + def add_lr_realignment(self, dag: DAG, ctx: StageContext) -> AlignResult: + """Re-align the long-read input files with minimap2""" + bundle = self.required(self.model_bundle, "model_bundle") + require_versions( + MINIMAP2_REALIGN_MIN_VERSIONS, skip=self.skip_version_check + ) + + return Minimap2RealignStage( + ctx=ctx, + inputs=self.lr_aln, + model_bundle=bundle, + sample_name=ctx.output_vcf.name.replace(".vcf.gz", ""), + bam_format=self.bam_format, + input_ref=self.lr_input_ref, + fastq_taglist=self.lr_fastq_taglist, + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_mosdepth( + self, + dag: DAG, + ctx: StageContext, + inputs: List[pathlib.Path], + upstream: Set[Job], + ) -> None: + """Run mosdepth for QC, when it is available""" + if not versions_available( + MOSDEPTH_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping mosdepth. mosdepth version %s or later not found", + MOSDEPTH_MIN_VERSIONS["mosdepth"], + ) + return + + MosdepthStage(ctx=ctx, inputs=inputs).add_to(dag, upstream) + + def add_sv_calling( + self, + dag: DAG, + ctx: StageContext, + inputs: List[pathlib.Path], + replace_rg: Optional[List[List[str]]], + upstream: Set[Job], + ) -> None: + """Call SVs from the long reads with Sentieon LongReadSV""" + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(LONGREADSV_MIN_VERSIONS, skip=self.skip_version_check) + + LongReadSVStage( + ctx=ctx, + inputs=inputs, + model=bundle.joinpath("longreadsv.model"), + interval=self.bed, + replace_rg=replace_rg, + ).add_to(dag, upstream) + def build_second_dag(self) -> Optional[DAG]: """Build the second DAG, for sex-aware CNV calling""" if self.skip_cnv: diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index 79e3a54..0102a77 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -1154,11 +1154,7 @@ def add_small_variant_calling( {haploid_gvcf_combine_job, gvcf_combine_job}, ) - def call_svs( - self, - sample_input: List[pathlib.Path], - replace_rg: Optional[List[List[str]]] = None, - ) -> Job: + def call_svs(self, sample_input: List[pathlib.Path]) -> Job: """ Call SVs using Sentieon LongReadSV """ @@ -1171,7 +1167,6 @@ def call_svs( inputs=sample_input, model=bundle.joinpath("longreadsv.model"), interval=self.bed, - replace_rg=replace_rg, ) .build() .job diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 7475a65..88e11f8 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -235,6 +235,36 @@ def total_memory() -> int: return total_mem +def set_bwt_max_mem( + total_input_size: int, + n_alignment_jobs: int = 1, + override: Optional[str] = None, +) -> str: + """Set the `bwt_max_mem` environment variable and return its value. + + `override` short-circuits the calculation, for the hidden + `--bwt_max_mem` argument. Otherwise the value is derived from the + memory available to the run, the size of the inputs staged in + memory, and the number of concurrent alignment jobs. + """ + if override: + os.environ["bwt_max_mem"] = override + return override + + total_mem = total_memory() + total_mem_gb = total_mem / (1024.0**3) + align_mem_gb = ( + total_mem_gb - 4 - total_input_size / (1024.0**3) * 2.3 + ) # some memory for other system processes + bwa_mem_gb = max( + int((align_mem_gb / n_alignment_jobs) - 6), 0 + ) # some memory for other alignment processes + logger.debug("Setting bwt_max_mem to: %sG", bwa_mem_gb) + bwt_max_mem = f"{bwa_mem_gb}G" + os.environ["bwt_max_mem"] = bwt_max_mem + return bwt_max_mem + + def find_numa_nodes() -> List[str]: """Find NUMA nodes on the system, if available""" numa_nodes = [] diff --git a/tests/unit/test_dnascope_hybrid_longread.py b/tests/unit/test_dnascope_hybrid_longread.py new file mode 100644 index 0000000..a1ff914 --- /dev/null +++ b/tests/unit/test_dnascope_hybrid_longread.py @@ -0,0 +1,186 @@ +""" +Unit tests for the DNAscope hybrid pipeline's long-read jobs + +The hybrid pipeline used to inherit its long-read realignment, mosdepth and +SV calling from `DNAscopeLRPipeline`; it now composes the same stages +itself, with its own argument defaults. +""" + +import pathlib +import tempfile +from abc import ABC +from unittest.mock import MagicMock, patch + +import pytest + +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.pipeline import BasePipeline + +RG_LINES = ["@RG\tID:lr_rg1\tSM:sample1"] + + +class TestDNAscopeHybridLongRead: + """The long-read half of the hybrid DAG""" + + def setup_method(self): + self.temp_dir = tempfile.mkdtemp() + self.mock_dir = pathlib.Path(self.temp_dir) + + self.mock_vcf = self.mock_dir / "output.vcf.gz" + self.mock_ref = self.mock_dir / "reference.fa" + self.mock_input_ref = self.mock_dir / "input_reference.fa" + self.mock_sr_aln = [self.mock_dir / "short.bam"] + self.mock_lr_aln = [self.mock_dir / "long.bam"] + self.mock_bundle = self.mock_dir / "model.bundle" + self.mock_bed = self.mock_dir / "interval.bed" + + for file_path in [ + self.mock_ref, + self.mock_input_ref, + self.mock_sr_aln[0], + self.mock_lr_aln[0], + self.mock_bundle, + self.mock_bed, + ]: + file_path.touch() + + def create_pipeline(self): + """Create a DNAscopeHybridPipeline that realigns its long reads""" + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + pipeline.logger = MagicMock() + + pipeline.output_vcf = self.mock_vcf + pipeline.reference = self.mock_ref + pipeline.model_bundle = self.mock_bundle + pipeline.bed = self.mock_bed + pipeline.cores = 2 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = self.mock_dir + pipeline.pop_vcf = None + + # State normally set by validate() + pipeline.fai_data = {"chr1": {"length": 1000}} + pipeline.shards = [MagicMock()] + pipeline.shards[0].contig = "chr1" + pipeline.shards[0].start = 1 + pipeline.shards[0].stop = 1000 + pipeline.pop_vcf_contigs = {"chr1": 1000} + pipeline.lr_aln_readgroups = [[{"ID": "lr_rg1", "SM": "sample1"}]] + pipeline.sr_aln_readgroups = [[{"ID": "sr_rg1", "SM": "sample1"}]] + pipeline.hybrid_rg_sm = "sample1" + pipeline.hybrid_set_rg = False + pipeline.shortread_tech = "Illumina" + pipeline.longread_tech = "ONT" + pipeline.sr_aln = self.mock_sr_aln + pipeline.lr_aln = self.mock_lr_aln + + # Realign the long reads + pipeline.lr_align_input = True + pipeline.lr_input_ref = self.mock_input_ref + + return pipeline + + def build_dag(self, pipeline): + """Build the DAG and index the jobs by name""" + with patch( + "sentieon_cli.command_strings.get_rg_lines", return_value=RG_LINES + ): + dag = pipeline.build_dag() + jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) + return dag, {job.name: job for job in jobs} + + def dep_names(self, dag, job): + """The names of a job's dependencies""" + return {dep.name for dep in dag.waiting_jobs.get(job, set())} + + def test_minimap2_args_default(self): + """The hybrid pipeline aligns with `-Y`, not the long-read `-YL`""" + pipeline = self.create_pipeline() + _dag, jobs = self.build_dag(pipeline) + + assert "bam-realign-0" in jobs + cmd_str = str(jobs["bam-realign-0"].shell) + assert " -Y " in cmd_str + assert "-YL" not in cmd_str + + def test_lr_fastq_taglist_is_used(self): + """`--lr_fastq_taglist` reaches the minimap2 command""" + pipeline = self.create_pipeline() + pipeline.lr_fastq_taglist = "RG,MM" + _dag, jobs = self.build_dag(pipeline) + + assert "-T RG,MM" in str(jobs["bam-realign-0"].shell) + + def test_long_read_jobs_wait_for_realignment(self): + """mosdepth and LongReadSV read the realigned long reads""" + pipeline = self.create_pipeline() + dag, jobs = self.build_dag(pipeline) + + assert self.dep_names(dag, jobs["mosdepth-0"]) == {"bam-realign-0"} + assert self.dep_names(dag, jobs["LongReadSV"]) == {"bam-realign-0"} + + def test_realigned_long_reads_are_called(self): + """The realigned alignment, not the input, is passed downstream""" + pipeline = self.create_pipeline() + _dag, jobs = self.build_dag(pipeline) + + realigned = str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") + for name in ("mosdepth-0", "LongReadSV", "dnascope-1"): + assert realigned in str(jobs[name].shell) + + +class TestDNAscopeHybridBases: + """The hybrid pipeline no longer inherits from the other pipelines""" + + def test_mro(self): + assert DNAscopeHybridPipeline.__mro__ == ( + DNAscopeHybridPipeline, + BasePipeline, + ABC, + object, + ) + + def test_not_a_dnascope_or_longread_pipeline(self): + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + assert not isinstance(pipeline, DNAscopePipeline) + assert not isinstance(pipeline, DNAscopeLRPipeline) + + @pytest.mark.parametrize( + "attribute", + [ + "sample_input", + "r1_fastq", + "r2_fastq", + "readgroups", + "duplicate_marking", + "align", + "collate_align", + "consensus", + "fastq", + "fastq_taglist", + "haploid_bed", + "input_ref", + "interval_padding", + "no_ramdisk", + "pcr_free", + "repeat_model", + "skip_small_variants", + "tech", + "use_pbsv", + "cnv_excluded_regions", + "assay", + ], + ) + def test_parent_only_attributes_are_gone(self, attribute): + """The attributes that came from the parent `__init__`s""" + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + assert not hasattr(pipeline, attribute) diff --git a/tests/unit/test_pipeline_params.py b/tests/unit/test_pipeline_params.py new file mode 100644 index 0000000..d0fb54e --- /dev/null +++ b/tests/unit/test_pipeline_params.py @@ -0,0 +1,48 @@ +""" +Every pipeline argument must be an attribute of a freshly built pipeline. + +`BasePipeline.handle_arguments` raises when a `params` key is missing from +the instance, and argparse leaves an unset argument as `None`, which it +skips -- so the `__init__` value is the effective default. These tests keep +the two in step. +""" + +from typing import Type + +import pytest + +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.hybrid_pangenome import HybridPangenome +from sentieon_cli.pipeline import BasePipeline +from sentieon_cli.sentieon_pangenome import SentieonPangenome + +PIPELINES = [ + DNAscopePipeline, + DNAscopeLRPipeline, + DNAscopeHybridPipeline, + SentieonPangenome, + HybridPangenome, +] + + +@pytest.mark.parametrize("cls", PIPELINES, ids=lambda c: c.__name__) +def test_every_argument_is_an_attribute(cls: Type[BasePipeline]): + """`handle_arguments` needs an attribute per param and positional""" + attributes = set(vars(cls())) + arguments = set(cls.params) | set(cls.positionals) + assert arguments <= attributes, sorted(arguments - attributes) + + +@pytest.mark.parametrize("cls", PIPELINES, ids=lambda c: c.__name__) +def test_argparse_defaults_match_the_attributes(cls: Type[BasePipeline]): + """An argparse default must match the value `__init__` assigns""" + pipeline = cls() + specs = list(cls.params.items()) + list(cls.positionals.items()) + mismatched = { + key: (getattr(pipeline, key), spec["default"]) + for key, spec in specs + if "default" in spec and getattr(pipeline, key) != spec["default"] + } + assert not mismatched diff --git a/tests/unit/test_pipeline_validation.py b/tests/unit/test_pipeline_validation.py index b3ae579..64fce04 100644 --- a/tests/unit/test_pipeline_validation.py +++ b/tests/unit/test_pipeline_validation.py @@ -16,6 +16,7 @@ from sentieon_cli.dnascope import DNAscopePipeline from sentieon_cli.dnascope_longread import DNAscopeLRPipeline from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.util import set_bwt_max_mem from tests.utils.test_helpers import create_mock_args @@ -431,21 +432,35 @@ def test_total_input_size_calculation(self): total_size = pipeline.total_input_size() assert total_size == 1500 # 1KB + 0.5KB - def test_memory_calculation_logic(self): - """Test bwt_max_mem calculation logic""" - pipeline = DNAscopePipeline() + def test_bwt_max_mem_override(self, monkeypatch): + """`--bwt_max_mem` short-circuits the calculation""" + monkeypatch.delenv("bwt_max_mem", raising=False) - # Setup logging first - args = create_mock_args() - pipeline.setup_logging(args) - - # Test with specific memory values - pipeline.bwt_max_mem = "10G" - pipeline.set_bwt_max_mem(0, 1) - - import os + assert set_bwt_max_mem(0, 1, override="10G") == "10G" assert os.environ.get("bwt_max_mem") == "10G" + @pytest.mark.parametrize( + "total_input_size,n_alignment_jobs,expected", + [ + (0, 1, "54G"), # 64 - 4 - 0 = 60; 60 / 1 - 6 = 54 + (0, 2, "24G"), # 60 / 2 - 6 = 24 + (30 * 1024**3, 1, "0G"), # 60 - 30 * 2.3 < 0, floored at 0 + ], + ) + def test_bwt_max_mem_calculation( + self, monkeypatch, total_input_size, n_alignment_jobs, expected + ): + """bwt_max_mem is derived from the memory and the input size""" + monkeypatch.setenv("bwt_max_mem", "unset") + + with patch( + "sentieon_cli.util.total_memory", return_value=64 * 1024**3 + ): + result = set_bwt_max_mem(total_input_size, n_alignment_jobs) + + assert result == expected + assert os.environ.get("bwt_max_mem") == expected + class TestValidateBwaIndex: """Test BasePipeline.validate_bwa_index()""" From 4ccade937306b79b303da0445ed4fac7bce4e141 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 3 Sep 2026 22:26:13 -0700 Subject: [PATCH 14/20] Drop the attribute renames from validate() The dnascope and long-read pipelines renamed their input attributes during validation (`r1_fastq` to `sr_r1_fastq`, `sample_input` to `lr_aln`, and so on) so that the hybrid pipeline could inherit from both without the names colliding. The hybrid no longer inherits from either, so each pipeline now reads its own argument names throughout. Two validation tests that set the renamed attributes before `validate()` ran, and so exercised the wrong branch, now assert the messages they were written for. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope.py | 38 +++++--------- sentieon_cli/dnascope_longread.py | 13 ++--- tests/unit/test_dag_construction.py | 8 +-- tests/unit/test_dnascope_longread_pop_vcf.py | 1 - tests/unit/test_pipeline_validation.py | 53 +++++++++++++------- 5 files changed, 54 insertions(+), 59 deletions(-) diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 66a489f..8f0729a 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -245,20 +245,8 @@ def __init__(self) -> None: def validate(self) -> None: self.required(self.output_vcf, "output_vcf") - # uniquify pipeline attributes - self.sr_r1_fastq = self.r1_fastq - self.sr_r2_fastq = self.r2_fastq - self.sr_readgroups = self.readgroups - self.sr_duplicate_marking = self.duplicate_marking - del self.r1_fastq - del self.r2_fastq - del self.readgroups - del self.duplicate_marking - # validate - if not self.sample_input and not ( - self.sr_r1_fastq and self.sr_readgroups - ): + if not self.sample_input and not (self.r1_fastq and self.readgroups): self.logger.error( "Please supply either the `--sample_input` or `--r1_fastq` " "and `--readgroups` arguments" @@ -285,21 +273,21 @@ def validate(self) -> None: "across decoy and unplaced contigs." ) - if len(self.sr_r1_fastq) != len(self.sr_readgroups): + if len(self.r1_fastq) != len(self.readgroups): self.logger.error( "The number of readgroups does not equal the number of fastq " "files" ) sys.exit(2) - for rg in self.sr_readgroups: + for rg in self.readgroups: try: parse_rg_line(rg.replace(r"\t", "\t")) except ValueError as e: self.logger.error("Invalid --readgroups value '%s': %s", rg, e) sys.exit(2) - if self.sr_r1_fastq or self.align or self.collate_align: + if self.r1_fastq or self.align or self.collate_align: self.validate_bwa_index() def configure(self) -> None: @@ -329,9 +317,7 @@ def configure_alignment(self) -> None: def total_input_size(self) -> int: """Find the total size of all inputs""" total_input_size = sum([x.stat().st_size for x in self.sample_input]) - for r1, r2 in itertools.zip_longest( - self.sr_r1_fastq, self.sr_r2_fastq - ): + for r1, r2 in itertools.zip_longest(self.r1_fastq, self.r2_fastq): for fq in (r1, r2): if isinstance(fq, pathlib.Path): total_input_size += fq.stat().st_size @@ -424,7 +410,7 @@ def add_sr_input_alignment( inputs=self.sample_input, model_bundle=bundle, bam_format=self.bam_format, - duplicate_marking=self.sr_duplicate_marking, + duplicate_marking=self.duplicate_marking, input_ref=self.input_ref, collate=self.collate_align, bwa_args=self.bwa_args, @@ -438,7 +424,7 @@ def add_sr_fastq_alignment( """Align fastq files to the reference genome using bwa""" bundle = self.required(self.model_bundle, "model_bundle") - if not self.sr_r1_fastq and not self.sr_readgroups: + if not self.r1_fastq and not self.readgroups: return AlignResult( jobs=[], terminal=set(), outputs=[], cleanup_paths=[] ) @@ -447,13 +433,13 @@ def add_sr_fastq_alignment( return BwaFastqStage( ctx=ctx, - r1_fastq=self.sr_r1_fastq, - r2_fastq=self.sr_r2_fastq, - readgroups=self.sr_readgroups, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroups=self.readgroups, model_bundle=bundle, numa_nodes=self.numa_nodes, bam_format=self.bam_format, - duplicate_marking=self.sr_duplicate_marking, + duplicate_marking=self.duplicate_marking, unzip=find_unzip(self.logger), bwa_args=self.bwa_args, bwa_k=str(self.bwa_k), @@ -471,7 +457,7 @@ def add_sr_preprocessing( return ShortReadPreprocessingStage( ctx=ctx, inputs=sample_input, - duplicate_marking=self.sr_duplicate_marking, + duplicate_marking=self.duplicate_marking, consensus=self.consensus, skip_metrics=self.skip_metrics, assay=self.assay, diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index 0102a77..f7b4c44 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -253,13 +253,8 @@ def __init__(self) -> None: self.use_pbsv = False def validate(self) -> None: - # uniquify pipeline attributes - self.lr_aln = self.sample_input - self.lr_input_ref = self.input_ref - del self.sample_input - # validate - if not (self.lr_aln or self.fastq): + if not (self.sample_input or self.fastq): self.logger.error( "Please supply either the `--sample_input` or `--fastq` and " "`--readgroups` arguments" @@ -433,7 +428,7 @@ def build_dag(self) -> DAG: dag = DAG() ctx = self.stage_context() - sample_input = self.lr_aln + sample_input = self.sample_input realign_jobs: Set[Job] = set() if self.align: sample_input, realign_jobs = self.lr_align_inputs() @@ -486,11 +481,11 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: result = Minimap2RealignStage( ctx=self.stage_context(), - inputs=self.lr_aln, + inputs=self.sample_input, model_bundle=bundle, sample_name=output_vcf.name.replace(".vcf.gz", ""), bam_format=self.bam_format, - input_ref=self.lr_input_ref, + input_ref=self.input_ref, fastq_taglist=self.fastq_taglist, minimap2_args=self.minimap2_args, util_sort_args=self.util_sort_args, diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index 1749f70..b7faa77 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -66,10 +66,10 @@ def create_basic_dnascope_pipeline(self): pipeline.reference = self.mock_ref pipeline.model_bundle = self.mock_bundle pipeline.sample_input = [self.mock_bam] - pipeline.sr_r1_fastq = [] - pipeline.sr_r2_fastq = [] - pipeline.sr_readgroups = [] - pipeline.sr_duplicate_marking = "markdup" + pipeline.r1_fastq = [] + pipeline.r2_fastq = [] + pipeline.readgroups = [] + pipeline.duplicate_marking = "markdup" pipeline.cores = 2 pipeline.dry_run = True # Important: use dry run to avoid actual execution pipeline.skip_version_check = True diff --git a/tests/unit/test_dnascope_longread_pop_vcf.py b/tests/unit/test_dnascope_longread_pop_vcf.py index 8376b7e..b96e613 100644 --- a/tests/unit/test_dnascope_longread_pop_vcf.py +++ b/tests/unit/test_dnascope_longread_pop_vcf.py @@ -75,7 +75,6 @@ def create_pipeline(self): # Mock sample inputs pipeline.sample_input = self.mock_aln - pipeline.lr_aln = self.mock_aln return pipeline diff --git a/tests/unit/test_pipeline_validation.py b/tests/unit/test_pipeline_validation.py index 64fce04..f628345 100644 --- a/tests/unit/test_pipeline_validation.py +++ b/tests/unit/test_pipeline_validation.py @@ -54,8 +54,8 @@ def test_valid_configuration_with_bam_input(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] # Should not raise any exceptions try: @@ -84,8 +84,8 @@ def test_missing_inputs_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] with pytest.raises(SystemExit): self.pipeline.validate() @@ -97,8 +97,8 @@ def test_invalid_output_extension_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] with pytest.raises(SystemExit): self.pipeline.validate() @@ -109,11 +109,19 @@ def test_mismatched_fastq_readgroups_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [] - self.pipeline.sr_r1_fastq = [self.mock_fastq] - self.pipeline.sr_readgroups = [] # Empty readgroups with non-empty fastq + # Two fastq files, but only one readgroup. Both must be non-empty to + # get past the "supply --sample_input or --r1_fastq" guard. + self.pipeline.r1_fastq = [self.mock_fastq, self.mock_fastq] + self.pipeline.readgroups = ["@RG\\tID:a\\tSM:s"] - with pytest.raises(SystemExit): - self.pipeline.validate() + with patch.object(self.pipeline.logger, "error") as mock_error: + with pytest.raises(SystemExit) as excinfo: + self.pipeline.validate() + + assert excinfo.value.code == 2 + mock_error.assert_any_call( + "The number of readgroups does not equal the number of fastq files" + ) def test_skip_multiqc_when_skip_metrics(self): """Test that skip_multiqc is set when skip_metrics is True""" @@ -121,8 +129,8 @@ def test_skip_multiqc_when_skip_metrics(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] self.pipeline.skip_metrics = True self.pipeline.skip_multiqc = False @@ -198,7 +206,7 @@ def test_valid_configuration_with_fastq_input(self): self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [] + self.pipeline.sample_input = [] self.pipeline.fastq = [self.mock_fastq] self.pipeline.readgroups = ["@RG\\tID:test\\tSM:sample"] @@ -272,22 +280,29 @@ def test_haploid_bed_requires_diploid_bed(self): self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [self.mock_bam] + self.pipeline.sample_input = [self.mock_bam] self.pipeline.fastq = [] self.pipeline.readgroups = [] self.pipeline.haploid_bed = self.mock_bed self.pipeline.bed = None # No diploid bed self.pipeline.skip_small_variants = False - with pytest.raises(SystemExit): - self.pipeline.validate() + with patch.object(self.pipeline.logger, "error") as mock_error: + with pytest.raises(SystemExit) as excinfo: + self.pipeline.validate() + + assert excinfo.value.code == 2 + mock_error.assert_any_call( + "Please supply a BED file of diploid regions to distinguish " + "haploid and diploid regions of the genome." + ) def test_mismatched_fastq_readgroups_raises_error(self): """Test that mismatched FASTQ and readgroup counts raise an error""" self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [] + self.pipeline.sample_input = [] self.pipeline.fastq = [self.mock_fastq] self.pipeline.readgroups = [] # Empty readgroups with non-empty fastq @@ -426,8 +441,8 @@ def test_total_input_size_calculation(self): fastq_file.write_bytes(b"y" * 500) # 0.5KB pipeline.sample_input = [bam_file] - pipeline.sr_r1_fastq = [fastq_file] - pipeline.sr_r2_fastq = [] + pipeline.r1_fastq = [fastq_file] + pipeline.r2_fastq = [] total_size = pipeline.total_input_size() assert total_size == 1500 # 1KB + 0.5KB From 329891252ec2fa8380f48c79e485928e0eadc5af Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 3 Sep 2026 22:32:17 -0700 Subject: [PATCH 15/20] Update the require_versions docstring The version constants no longer all live in the pipeline modules. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 88e11f8..e635991 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -146,7 +146,7 @@ def require_versions( `check_version` has already logged the reason, so the exit is silent. Pass `skip=True` (the pipelines' `--skip_version_check`) to do nothing. - A `Mapping` rather than a `Dict`, so the pipelines' module-level + A `Mapping` rather than a `Dict`, so the module-level `*_MIN_VERSIONS` constants -- some of which mypy infers as `Dict[str, Version]`, with no `None` entry -- are accepted. """ From 55ff1386b568e7b3b4e0f0cdd775bb3e9d9bcb4a Mon Sep 17 00:00:00 2001 From: Don Freed Date: Fri, 4 Sep 2026 09:57:54 -0700 Subject: [PATCH 16/20] Default the hybrid pipeline's minimap2 arguments to -YL The hybrid pipeline's long-read input may be ONT, whose very long reads can carry more than 65535 CIGAR operations, which the BAM format cannot store; `-L` moves such CIGAR strings into the `CG` tag. The hidden `--minimap2_args` default therefore becomes `-YL`, matching the long-read pipeline and the minimap2 stages, instead of `-Y`. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope_hybrid.py | 4 ++-- tests/unit/test_dnascope_hybrid_longread.py | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index bc25793..ab4dfd9 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -306,7 +306,7 @@ class DNAscopeHybridPipeline(BasePipeline): "minimap2_args": { # help="Extra arguments for sentieon minimap2", "help": argparse.SUPPRESS, - "default": "-Y", + "default": "-YL", }, "no_split_alignment": { "help": argparse.SUPPRESS, @@ -362,7 +362,7 @@ def __init__(self) -> None: self.bwt_max_mem: Optional[str] = None self.lr_fastq_taglist = "*" self.lr_read_filter: Optional[str] = None - self.minimap2_args = "-Y" + self.minimap2_args = "-YL" self.no_split_alignment = False self.skip_model_apply = False self.skip_pop_vcf_id_check = False diff --git a/tests/unit/test_dnascope_hybrid_longread.py b/tests/unit/test_dnascope_hybrid_longread.py index a1ff914..7807977 100644 --- a/tests/unit/test_dnascope_hybrid_longread.py +++ b/tests/unit/test_dnascope_hybrid_longread.py @@ -99,14 +99,25 @@ def dep_names(self, dag, job): return {dep.name for dep in dag.waiting_jobs.get(job, set())} def test_minimap2_args_default(self): - """The hybrid pipeline aligns with `-Y`, not the long-read `-YL`""" + """The hybrid pipeline aligns with `-YL`, like the long-read one + + `-L` moves CIGAR strings longer than 65535 operations - which ONT + reads can produce and BAM cannot store - into the `CG` tag, so + every minimap2 call that may see ONT input needs it. + """ pipeline = self.create_pipeline() _dag, jobs = self.build_dag(pipeline) assert "bam-realign-0" in jobs cmd_str = str(jobs["bam-realign-0"].shell) - assert " -Y " in cmd_str - assert "-YL" not in cmd_str + assert " -YL " in cmd_str + + def test_minimap2_args_match_the_long_read_pipeline(self): + """The hybrid and long-read defaults agree""" + assert ( + DNAscopeHybridPipeline().minimap2_args + == DNAscopeLRPipeline().minimap2_args + ) def test_lr_fastq_taglist_is_used(self): """`--lr_fastq_taglist` reaches the minimap2 command""" From c1066a741eff33ef68cbcbd85c8488d7ccaf3591 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Fri, 4 Sep 2026 10:38:43 -0700 Subject: [PATCH 17/20] Run the pangenome pipeline through BasePipeline.main `SentieonPangenome` overrode `main()` to parse the reference index and population VCF before validation, call `build_first_dag()` instead of `build_dag()`, and gate the second, sex-aware DAG itself. It now uses the base flow like every other pipeline: `validate()` parses the index, shards and VCF contigs (as the hybrid pangenome already did), the main DAG builder is `build_dag()`, and `build_second_dag()` returns `None` when no sex-aware caller is enabled and calls `get_sex` itself, as the hybrid DNAscope pipeline does. The reference index is validated once rather than twice. A new test asserts that no pipeline overrides `main()`. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/sentieon_pangenome.py | 75 ++++++------------- tests/unit/test_pipeline_lifecycle.py | 40 +++++++--- tests/unit/test_sentieon_pangenome.py | 67 +++++++---------- ...test_sentieon_pangenome_kmc_integration.py | 13 ++-- 4 files changed, 85 insertions(+), 110 deletions(-) diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 1420360..a41439e 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -6,9 +6,7 @@ import copy import json import pathlib -import shutil import sys -import time from typing import Dict, List, Optional, Set, Tuple, Union import packaging.version @@ -53,7 +51,6 @@ path_arg, require_versions, sample_sex_arg, - tmp, total_memory, vcf_id, ) @@ -269,7 +266,6 @@ def __init__(self) -> None: self.call_svs = False self.gvcf = False self.par_bed: Optional[pathlib.Path] = None - self.cnv_input_bams: List[pathlib.Path] = [] self.pangenome_ref_name = "GRCh38" self.extract_model_name = "extract.model" self.pangenome_contig_prefix = "GRCh38#0#" @@ -288,6 +284,10 @@ def __init__(self) -> None: self.skip_pop_vcf_id_check: bool = False self.skip_model_apply = False self.skip_small_variants = False + # Stashed by `build_dag` for the second, sex-aware DAG + self.ploidy_json: Optional[pathlib.Path] = None + self.sr_alignment: Optional[pathlib.Path] = None + self.cnv_input_bams: List[pathlib.Path] = [] def _cnv_in_second_dag(self) -> bool: """CNV calling runs in the second, sex-aware DAG""" @@ -301,55 +301,21 @@ def _needs_second_dag(self) -> bool: or self._cnv_in_second_dag() ) - def main(self, args: argparse.Namespace) -> None: - """Run the pipeline""" - self.handle_arguments(args) - self.setup_logging(args) - start_time = time.monotonic() - success = False - try: - self.validate_ref() - - self.fai_data = parse_fai( - pathlib.Path(str(self.reference) + ".fai") - ) - self.pop_vcf_contigs: Dict[str, Optional[int]] = {} - if self.pop_vcf: - self.pop_vcf_contigs = vcf_contigs(self.pop_vcf, self.dry_run) - self.logger.debug("VCF contigs are: %s", self.pop_vcf_contigs) - - self.validate() - self.start_run_logs() - self.shards = determine_shards_from_fai( - self.fai_data, 10 * 1000 * 1000 - ) - - tmp_dir_str = tmp() - self.tmp_dir = pathlib.Path(tmp_dir_str) - - try: - dag = self.build_first_dag() - executor = self.run(dag) - self.check_execution(dag, executor) - - if self._needs_second_dag(): - self.get_sex(self.ploidy_json) - dag = self.build_second_dag() - executor = self.run(dag) - self.check_execution(dag, executor) - finally: - if not self.retain_tmpdir: - shutil.rmtree(tmp_dir_str) - success = True - finally: - self.log_completion(success, start_time) - def validate(self) -> None: """Validate pipeline inputs""" + self.validate_ref() + self.fai_data = parse_fai(pathlib.Path(str(self.reference) + ".fai")) + self.shards = determine_shards_from_fai( + self.fai_data, 10 * 1000 * 1000 + ) + self.pop_vcf_contigs: Dict[str, Optional[int]] = {} + if self.pop_vcf: + self.pop_vcf_contigs = vcf_contigs(self.pop_vcf, self.dry_run) + self.logger.debug("VCF contigs are: %s", self.pop_vcf_contigs) + self.validate_bundle() self.validate_fastq_rg() self.validate_output_vcf() - self.validate_ref() self.collect_readgroups() if not self.sample_input and not self.r1_fastq: @@ -658,10 +624,7 @@ def configure(self) -> None: pass def build_dag(self) -> DAG: - return DAG() - - def build_first_dag(self) -> DAG: - """Build the main DAG for the Sentieon pangenome pipeline""" + """Build the first DAG for the Sentieon pangenome pipeline""" bundle = self.required(self.model_bundle, "model_bundle") ctx = self.stage_context() @@ -1185,8 +1148,14 @@ def build_segdup_job( task_name="segdup", ) - def build_second_dag(self) -> DAG: + def build_second_dag(self) -> Optional[DAG]: """Build the second DAG for sex-aware downstream tools""" + if not self._needs_second_dag(): + return None + + assert self.ploidy_json is not None + self.get_sex(self.ploidy_json) + self.logger.info("Building the second pangenome DAG") dag = DAG() diff --git a/tests/unit/test_pipeline_lifecycle.py b/tests/unit/test_pipeline_lifecycle.py index 9e84888..adf67e9 100644 --- a/tests/unit/test_pipeline_lifecycle.py +++ b/tests/unit/test_pipeline_lifecycle.py @@ -7,6 +7,7 @@ import logging import os import sys +from typing import Optional import pytest @@ -16,9 +17,14 @@ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), ) -from sentieon_cli import sentieon_pangenome # noqa: E402 from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.dnascope import DNAscopePipeline # noqa: E402 +from sentieon_cli.dnascope_hybrid import ( # noqa: E402 + DNAscopeHybridPipeline, +) +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline # noqa: E402 from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.hybrid_pangenome import HybridPangenome # noqa: E402 from sentieon_cli.job import Job # noqa: E402 from sentieon_cli.pipeline import BasePipeline # noqa: E402 from sentieon_cli.sentieon_pangenome import SentieonPangenome # noqa: E402 @@ -231,21 +237,20 @@ def __init__(self, fail_on=None): self.check_calls = 0 self.dags_built = [] - def validate_ref(self) -> None: - pass - def validate(self) -> None: pass def configure(self) -> None: pass - def build_first_dag(self) -> DAG: + def build_dag(self) -> DAG: self.dags_built.append("first") self.ploidy_json = self.tmp_dir.joinpath("ploidy.json") return DAG() - def build_second_dag(self) -> DAG: + def build_second_dag(self) -> Optional[DAG]: + if not self._needs_second_dag(): + return None self.dags_built.append("second") return DAG() @@ -264,18 +269,14 @@ def check_execution(self, dag, executor): def _stub_pangenome(monkeypatch, tmp_path, **kwargs): """A stubbed pangenome pipeline writing temp dirs into `tmp_path`""" monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) - monkeypatch.setattr(sentieon_pangenome, "parse_fai", lambda fai: {}) - monkeypatch.setattr( - sentieon_pangenome, "determine_shards_from_fai", lambda fai, size: [] - ) return _StubPangenome(**kwargs) def test_pangenome_main_cleans_up_tmpdir_when_the_first_dag_fails( tmp_path, monkeypatch ): - # SentieonPangenome overrides main() for its two-DAG flow; the temp - # directory has to be removed on the failure path too. + # SentieonPangenome runs the base two-DAG flow; the temp directory + # has to be removed on the failure path too. pipeline = _stub_pangenome(monkeypatch, tmp_path, fail_on=1) with pytest.raises(DagExecutionError): @@ -344,3 +345,18 @@ def test_check_execution_flags_unexecuted_jobs(): with pytest.raises(DagExecutionError, match=r"Job\(unexecuted-1\)"): pipeline.check_execution(dag, _StubExecutor()) + + +@pytest.mark.parametrize( + "pipeline_cls", + [ + DNAscopePipeline, + DNAscopeLRPipeline, + DNAscopeHybridPipeline, + SentieonPangenome, + HybridPangenome, + ], +) +def test_no_pipeline_overrides_main(pipeline_cls): + """Every pipeline runs through `BasePipeline.main`""" + assert "main" not in pipeline_cls.__dict__ diff --git a/tests/unit/test_sentieon_pangenome.py b/tests/unit/test_sentieon_pangenome.py index 5d17bb3..af96db3 100644 --- a/tests/unit/test_sentieon_pangenome.py +++ b/tests/unit/test_sentieon_pangenome.py @@ -96,7 +96,7 @@ def create_fastq_pipeline(self): """Create a fastq-input SentieonPangenome pipeline for testing. The default fixture uses BAM/CRAM input, which does not exercise the - dedup/metrics branch of ``build_first_dag``. This variant provides + dedup/metrics branch of ``build_dag``. This variant provides FASTQ input so the bwa/mm2 dedup jobs are created. """ mock_r1 = self.mock_dir / "sample_R1.fastq.gz" @@ -117,7 +117,7 @@ def test_dedup_metrics_output(self): """Dedup on the primary (bwa) alignment emits a --metrics file that lands in the metrics directory scanned by MultiQC.""" pipeline = self.create_fastq_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) assert "dedup-bwa" in job_names @@ -137,7 +137,7 @@ def test_metrics_input_bwa_only(self): """The metrics job reads only the bwa alignment; including the mm2 alignment would count the extracted reads it re-aligns twice.""" pipeline = self.create_fastq_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) metrics_job = next(j for j in all_jobs if j.name == "metrics") @@ -149,7 +149,7 @@ def test_dedup_metrics_skipped(self): """No Dedup --metrics output when metrics collection is skipped.""" pipeline = self.create_fastq_pipeline() pipeline.skip_metrics = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) bwa_dedup = next(j for j in all_jobs if j.name == "dedup-bwa") @@ -159,7 +159,7 @@ def test_model_apply_default(self): """Model apply runs by default and writes the intermediate, not the final output VCF.""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() # Verify DAG is created assert isinstance(dag, DAG) @@ -180,7 +180,7 @@ def test_skip_model_apply(self): """Test that model apply job is skipped when requested""" pipeline = self.create_pipeline() pipeline.skip_model_apply = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() # Verify DAG is created assert isinstance(dag, DAG) @@ -221,7 +221,7 @@ def test_gvcf_adds_gvcftyper_and_routes_outputs(self): updated gVCF to produce the final VCF at output_vcf.""" pipeline = self.create_pipeline() pipeline.gvcf = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] @@ -277,7 +277,7 @@ def test_ad_update_default(self): """A single AD-update job reads the model-apply intermediate and writes the final output VCF.""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) assert job_names.count("sad-lad-update") == 1 @@ -303,7 +303,7 @@ def test_no_ad_update_with_skip_small_variants(self): """No AD-update job when small variants are skipped""" pipeline = self.create_pipeline() pipeline.skip_small_variants = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) assert "sad-lad-update" not in job_names @@ -311,7 +311,7 @@ def test_no_ad_update_with_skip_small_variants(self): def test_no_gvcftyper_without_gvcf(self): """Without --gvcf, no GVCFtyper job is added""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] @@ -342,7 +342,7 @@ def test_call_svs(self): """Test that PangenomeSV is added when --call_svs is enabled""" pipeline = self.create_pipeline() pipeline.call_svs = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) @@ -375,7 +375,7 @@ def test_call_svs(self): def test_call_svs_disabled_by_default(self): """Test that PangenomeSV is not added by default""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list( dag.ready_jobs.keys() @@ -394,7 +394,7 @@ def test_skip_small_variants(self): """Test that DNAscope, transfer, and model-apply are skipped""" pipeline = self.create_pipeline() pipeline.skip_small_variants = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) @@ -411,7 +411,7 @@ def test_skip_small_variants_with_call_svs(self): pipeline = self.create_pipeline() pipeline.skip_small_variants = True pipeline.call_svs = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) @@ -451,7 +451,7 @@ def _build_cnv_dag(self, pipeline, sample_sex=SampleSex.FEMALE): The first DAG has to be built first so that the CNV input alignments are stashed for the second DAG. """ - pipeline.build_first_dag() + pipeline.build_dag() pipeline.sample_sex = sample_sex return pipeline.build_second_dag() @@ -505,7 +505,7 @@ def test_cnv_jobs_with_call_svs(self): """CNV jobs run in the second DAG when --call_svs is enabled""" pipeline = self.create_pipeline() pipeline.call_svs = True - first_dag = pipeline.build_first_dag() + first_dag = pipeline.build_dag() # CNV calling is sex-aware, so it is not in the first DAG first_job_names, _ = self._get_all_job_names(first_dag) @@ -525,7 +525,7 @@ def test_cnv_jobs_with_call_svs(self): def test_cnv_jobs_not_present_without_call_svs(self): """Test that CNV jobs are not added by default""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) assert "cnvscope" not in job_names @@ -533,13 +533,9 @@ def test_cnv_jobs_not_present_without_call_svs(self): assert "indel2cnv" not in job_names assert "combine-cnv" not in job_names - # Nor to the second DAG + # And no second DAG at all, as nothing consumes the sample sex pipeline.sample_sex = SampleSex.FEMALE - second_job_names, _ = self._get_all_job_names( - pipeline.build_second_dag() - ) - assert "cnvscope" not in second_job_names - assert "combine-cnv" not in second_job_names + assert pipeline.build_second_dag() is None def test_second_dag_needed_for_cnv_only(self): """CNV calling alone triggers the second DAG""" @@ -562,7 +558,7 @@ def test_ploidy_estimation_runs_with_sample_sex(self): pipeline.call_svs = True pipeline.sample_sex = SampleSex.FEMALE - job_names, _ = self._get_all_job_names(pipeline.build_first_dag()) + job_names, _ = self._get_all_job_names(pipeline.build_dag()) assert "estimate-ploidy" in job_names def test_ploidy_estimation_runs_without_sex_aware_callers(self): @@ -571,7 +567,7 @@ def test_ploidy_estimation_runs_without_sex_aware_callers(self): assert pipeline._needs_second_dag() is False job_names, all_jobs = self._get_all_job_names( - pipeline.build_first_dag() + pipeline.build_dag() ) assert "estimate-ploidy" in job_names @@ -586,7 +582,7 @@ def test_ploidy_estimation_runs_with_skip_small_variants(self): pipeline = self.create_pipeline() pipeline.skip_small_variants = True - job_names, _ = self._get_all_job_names(pipeline.build_first_dag()) + job_names, _ = self._get_all_job_names(pipeline.build_dag()) assert "estimate-ploidy" in job_names def test_cnv_jobs_with_skip_small_variants(self): @@ -607,7 +603,7 @@ def test_call_svs_without_cnv_model(self): pipeline = self.create_pipeline() pipeline.call_svs = True pipeline.has_cnv_model = False - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) @@ -623,13 +619,9 @@ def test_call_svs_without_cnv_model(self): assert "indel2cnv" not in job_names assert "combine-cnv" not in job_names - # And they are not in the second DAG either + # And there is no second DAG either pipeline.sample_sex = SampleSex.FEMALE - second_job_names, _ = self._get_all_job_names( - pipeline.build_second_dag() - ) - assert "cnvscope" not in second_job_names - assert "combine-cnv" not in second_job_names + assert pipeline.build_second_dag() is None def test_call_svs_without_cnv_model_skip_small_variants(self): """SV-only mode: SVs run but CNV jobs are skipped without cnv.model""" @@ -637,13 +629,8 @@ def test_call_svs_without_cnv_model_skip_small_variants(self): pipeline.skip_small_variants = True pipeline.call_svs = True pipeline.has_cnv_model = False - dag = self._build_cnv_dag(pipeline) - job_names, _ = self._get_all_job_names(dag) - assert "cnvscope" not in job_names - assert "cnv-model-apply" not in job_names - assert "indel2cnv" not in job_names - assert "combine-cnv" not in job_names + assert self._build_cnv_dag(pipeline) is None def test_cnvscope_command(self): """Test CNVscope driver command has correct algo and model""" @@ -746,7 +733,7 @@ def test_cnv_with_skip_model_apply(self): pipeline.call_svs = True pipeline.skip_model_apply = True first_job_names, _ = self._get_all_job_names( - pipeline.build_first_dag() + pipeline.build_dag() ) assert "model-apply" not in first_job_names diff --git a/tests/unit/test_sentieon_pangenome_kmc_integration.py b/tests/unit/test_sentieon_pangenome_kmc_integration.py index 9c0402f..eaad07e 100644 --- a/tests/unit/test_sentieon_pangenome_kmc_integration.py +++ b/tests/unit/test_sentieon_pangenome_kmc_integration.py @@ -21,10 +21,9 @@ def test_validate_kmc_patch_success(self): pipeline.reference = "ref" pipeline.pop_vcf = "pop" pipeline.gbz = "ref.grch38.gbz" - pipeline.fai_data = {} - pipeline.pop_vcf_contigs = {} - + # Mock dependencies + pipeline.logger = MagicMock() pipeline.handle_arguments = MagicMock() pipeline.setup_logging = MagicMock() pipeline.validate_bundle = MagicMock() @@ -34,6 +33,9 @@ def test_validate_kmc_patch_success(self): pipeline.collect_readgroups = MagicMock() with patch("sentieon_cli.util.check_version", return_value=True), \ + patch("sentieon_cli.sentieon_pangenome.parse_fai", return_value={}), \ + patch("sentieon_cli.sentieon_pangenome.determine_shards_from_fai", return_value=[]), \ + patch("sentieon_cli.sentieon_pangenome.vcf_contigs", return_value={}), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=True) as mock_check, \ patch("sys.exit") as mock_exit: @@ -54,12 +56,13 @@ def test_validate_kmc_patch_failure(self): pipeline.validate_ref = MagicMock() pipeline.collect_readgroups = MagicMock() pipeline.bed = "bed" - pipeline.fai_data = {} - pipeline.pop_vcf_contigs = {} pipeline.logger = MagicMock() with patch("sentieon_cli.util.check_version", return_value=True), \ + patch("sentieon_cli.sentieon_pangenome.parse_fai", return_value={}), \ + patch("sentieon_cli.sentieon_pangenome.determine_shards_from_fai", return_value=[]), \ + patch("sentieon_cli.sentieon_pangenome.vcf_contigs", return_value={}), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=False) as mock_check, \ patch("sys.exit") as mock_exit: From ac7c36ac37464ba27bd556c8156d46db595c6b61 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Fri, 4 Sep 2026 11:04:09 -0700 Subject: [PATCH 18/20] Collect the transfer inputs with TransferConfig.from_pipeline Four pipelines carried an identical `transfer_config()` helper building a `TransferConfig` from `pop_vcf`, `shards`, `pop_vcf_contigs` and `fai_data`. A `TransferInputs` protocol names those four attributes and `TransferConfig.from_pipeline` reads them structurally, raising a clear error when there is no population VCF. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/dnascope_hybrid.py | 16 +------------ sentieon_cli/dnascope_longread.py | 20 +++-------------- sentieon_cli/hybrid_pangenome.py | 17 +++----------- sentieon_cli/sentieon_pangenome.py | 16 +------------ sentieon_cli/stages/transfer.py | 34 +++++++++++++++++++++++++++- tests/unit/test_stage_transfer.py | 36 ++++++++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 62 deletions(-) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index ab4dfd9..d69090e 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -830,20 +830,6 @@ def build_second_dag(self) -> Optional[DAG]: ).add_to(dag) return dag - def transfer_config(self) -> TransferConfig: - """The inputs the annotation-transfer stage needs. - - Only valid once `--pop_vcf` is known to be set; the calling code - checks it first. - """ - assert self.pop_vcf is not None - return TransferConfig( - pop_vcf=self.pop_vcf, - shards=self.shards, - pop_vcf_contigs=self.pop_vcf_contigs, - fai_data=self.fai_data, - ) - def add_cleanup( self, dag: DAG, @@ -1218,7 +1204,7 @@ def add_variant_calling( if self.skip_model_apply: transfer_target = ctx.output_vcf transfer_spec = TransferSpec( - self.transfer_config(), transfer_target + TransferConfig.from_pipeline(self), transfer_target ) apply_spec: Optional[ApplySpec] = None diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index f7b4c44..d6f081e 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -643,20 +643,6 @@ def hificnv( ) return hificnv_job - def transfer_config(self) -> TransferConfig: - """The inputs the annotation-transfer stage needs. - - Only valid once `--pop_vcf` is known to be set; the calling code - checks it first. - """ - assert self.pop_vcf is not None - return TransferConfig( - pop_vcf=self.pop_vcf, - shards=self.shards, - pop_vcf_contigs=self.pop_vcf_contigs, - fai_data=self.fai_data, - ) - def add_small_variant_calling( self, dag: DAG, @@ -709,7 +695,7 @@ def add_small_variant_calling( diploid_transfer: Optional[TransferSpec] = None if self.pop_vcf: diploid_transfer = TransferSpec( - config=self.transfer_config(), + config=TransferConfig.from_pipeline(self), out_vcf=self.tmp_dir.joinpath("out_diploid_tmp_anno.vcf.gz"), tag="diploid", ) @@ -925,7 +911,7 @@ def add_small_variant_calling( patch_transfer: Optional[TransferSpec] = None if self.pop_vcf: patch_transfer = TransferSpec( - config=self.transfer_config(), + config=TransferConfig.from_pipeline(self), out_vcf=self.tmp_dir.joinpath( f"out_hap{i}_patch_anno.vcf.gz" ), @@ -988,7 +974,7 @@ def add_small_variant_calling( unphased_transfer: Optional[TransferSpec] = None if self.pop_vcf: unphased_transfer = TransferSpec( - config=self.transfer_config(), + config=TransferConfig.from_pipeline(self), out_vcf=self.tmp_dir.joinpath( "out_diploid_unphased_patch_anno.vcf.gz" ), diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index f6b09cb..259a026 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -839,7 +839,9 @@ def build_dag(self) -> DAG: TransferApplyStage( ctx=ctx, raw_vcf=raw_vcf, - transfer=TransferSpec(self.transfer_config(), transfer_target), + transfer=TransferSpec( + TransferConfig.from_pipeline(self), transfer_target + ), apply=( ApplySpec(model, ctx.output_vcf) if not self.skip_model_apply @@ -849,19 +851,6 @@ def build_dag(self) -> DAG: return dag - def transfer_config(self) -> TransferConfig: - """The inputs the annotation-transfer stage needs. - - `build_dag` requires `--pop_vcf` before it calls this. - """ - assert self.pop_vcf is not None - return TransferConfig( - pop_vcf=self.pop_vcf, - shards=self.shards, - pop_vcf_contigs=self.pop_vcf_contigs, - fai_data=self.fai_data, - ) - def lr_kmc_pairs(self) -> List[Tuple[pathlib.Path, pathlib.Path]]: """`(alignment, decode reference)` pairs for the long-read k-mer counting. diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index a41439e..044247a 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -909,7 +909,7 @@ def build_dag(self) -> DAG: transfer: Optional[TransferSpec] = None if self.pop_vcf: transfer = TransferSpec( - config=self.transfer_config(), + config=TransferConfig.from_pipeline(self), out_vcf=( transfer_vcf if not self.skip_model_apply @@ -945,20 +945,6 @@ def build_dag(self) -> DAG: return dag - def transfer_config(self) -> TransferConfig: - """The inputs the annotation-transfer stage needs. - - Only valid once `--pop_vcf` is known to be set; `validate_bundle` - requires it. - """ - assert self.pop_vcf is not None - return TransferConfig( - pop_vcf=self.pop_vcf, - shards=self.shards, - pop_vcf_contigs=self.pop_vcf_contigs, - fai_data=self.fai_data, - ) - def bwa_extract_stage( self, ctx: StageContext, diff --git a/sentieon_cli/stages/transfer.py b/sentieon_cli/stages/transfer.py index fe9a0cd..b6b9fff 100644 --- a/sentieon_cli/stages/transfer.py +++ b/sentieon_cli/stages/transfer.py @@ -7,7 +7,7 @@ import re import subprocess as sp import tempfile -from typing import Dict, Iterable, List, Optional, Set, Tuple +from typing import Dict, Iterable, List, Optional, Protocol, Set, Tuple from importlib.resources import files @@ -155,6 +155,20 @@ def build_transfer_jobs( return (sharded_merge_jobs, concat_job) +class TransferInputs(Protocol): + """A pipeline that carries the annotation transfer's inputs. + + Every pipeline that transfers annotations holds these four values under + these names, so `TransferConfig.from_pipeline` can collect them + structurally instead of each pipeline repeating the same helper. + """ + + pop_vcf: Optional[pathlib.Path] + shards: List[Shard] + pop_vcf_contigs: Dict[str, Optional[int]] + fai_data: Dict[str, Dict[str, int]] + + @dataclass(frozen=True) class TransferConfig: """The run-wide inputs the annotation transfer needs. @@ -169,6 +183,24 @@ class TransferConfig: pop_vcf_contigs: Dict[str, Optional[int]] fai_data: Dict[str, Dict[str, int]] + @classmethod + def from_pipeline(cls, pipeline: TransferInputs) -> "TransferConfig": + """Collect the transfer's inputs from a pipeline. + + The four values are set together while the pipeline validates its + arguments, so they are read back together here. Callers check + `pop_vcf` before asking for a transfer at all. + """ + pop_vcf = pipeline.pop_vcf + if pop_vcf is None: + raise ValueError("The annotation transfer needs a population VCF") + return cls( + pop_vcf=pop_vcf, + shards=pipeline.shards, + pop_vcf_contigs=pipeline.pop_vcf_contigs, + fai_data=pipeline.fai_data, + ) + @dataclass(kw_only=True) class TransferResult(StageResult): diff --git a/tests/unit/test_stage_transfer.py b/tests/unit/test_stage_transfer.py index ce3d807..d9c03f7 100644 --- a/tests/unit/test_stage_transfer.py +++ b/tests/unit/test_stage_transfer.py @@ -3,6 +3,9 @@ """ import pathlib +from types import SimpleNamespace + +import pytest from sentieon_cli.dag import DAG from sentieon_cli.shard import Shard @@ -131,3 +134,36 @@ def test_result_fields(self, tmp_path): assert result.jobs == [*result.shard_jobs, result.concat_job] assert result.terminal == {result.concat_job} + + +class TestFromPipeline: + """`TransferConfig.from_pipeline` collecting a pipeline's attributes""" + + @staticmethod + def make_pipeline(tmp_path: pathlib.Path, **kwargs) -> SimpleNamespace: + """A stand-in carrying the four `TransferInputs` attributes""" + config = make_config(tmp_path) + defaults = dict( + pop_vcf=config.pop_vcf, + shards=config.shards, + pop_vcf_contigs=config.pop_vcf_contigs, + fai_data=config.fai_data, + ) + defaults.update(kwargs) + return SimpleNamespace(**defaults) + + def test_collects_the_four_values(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + + config = TransferConfig.from_pipeline(pipeline) + + assert config.pop_vcf == pipeline.pop_vcf + assert config.shards == pipeline.shards + assert config.pop_vcf_contigs == pipeline.pop_vcf_contigs + assert config.fai_data == pipeline.fai_data + + def test_without_a_pop_vcf(self, tmp_path): + pipeline = self.make_pipeline(tmp_path, pop_vcf=None) + + with pytest.raises(ValueError, match="population VCF"): + TransferConfig.from_pipeline(pipeline) From f8f7280df3ae26febff3ad1fa3b8a0db0b9f32a8 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Fri, 4 Sep 2026 11:06:53 -0700 Subject: [PATCH 19/20] Let the hybrid pangenome pipeline use LongReadSVStage `LongReadSVStage` gains an explicit `output` (the derived `.sv.vcf.gz` path stays the default) and the LongReadSV algo options (`min_map_qual`, `min_sv_size`, `min_dp`, `min_af`), which is what the hybrid pangenome's graph-update SV call needed. Its hand-built `build_longreadsv_job` is gone; the job keeps its `longreadsv` name under the `pangenome-update` task and its command is unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/hybrid_pangenome.py | 41 +++++++------------------ sentieon_cli/stages/sv.py | 30 +++++++++++++++---- tests/unit/test_stage_sv.py | 51 ++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index 259a026..6649837 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -21,7 +21,6 @@ from .driver import ( DNAscope, Driver, - LongReadSV, PangenomeSV, PGHapUpdateAlgo, ) @@ -48,6 +47,7 @@ TransferApplyStage, TransferSpec, ) +from .stages.sv import LongReadSVStage from .stages.transfer import TransferConfig from .util import ( __version__, @@ -703,15 +703,22 @@ def build_dag(self) -> DAG: dag.add_job(update_raw_job, {gfa_job} | realign_jobs) # Call SVs from the long reads and collect graph update regions - longreadsv_job = self.build_longreadsv_job(longread_sv_vcf, calling_lr) - dag.add_job(longreadsv_job, realign_jobs) + longreadsv_result = LongReadSVStage( + ctx=ctx, + inputs=list(calling_lr), + model=bundle.joinpath("longreadsv.model"), + output=longread_sv_vcf, + min_sv_size=LONGREADSV_MIN_SV_SIZE, + name="longreadsv", + task_name="pangenome-update", + ).add_to(dag, realign_jobs) sv_bed_job = Job( cmds.cmd_longread_sv_bed(sv_bed, longread_sv_vcf, ref_fai), "longread-sv-bed", 0, task_name="pangenome-update", ) - dag.add_job(sv_bed_job, {longreadsv_job}) + dag.add_job(sv_bed_job, {longreadsv_result.job}) # Graph update with the SV BED update_job = self.build_graph_update_job( @@ -1040,32 +1047,6 @@ def build_graph_update_job( task_name="pangenome-update", ) - def build_longreadsv_job( - self, - out_vcf: pathlib.Path, - lr_aln: List[pathlib.Path], - ) -> Job: - """Call SVs from the long reads for the graph update""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=list(lr_aln), - ) - driver.add_algo( - LongReadSV( - out_vcf, - model=self.model_bundle.joinpath("longreadsv.model"), - min_sv_size=LONGREADSV_MIN_SV_SIZE, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "longreadsv", - self.cores, - task_name="pangenome-update", - ) - def build_minimap2_lift_job( self, out_bam: pathlib.Path, diff --git a/sentieon_cli/stages/sv.py b/sentieon_cli/stages/sv.py index 03e0eef..c722172 100644 --- a/sentieon_cli/stages/sv.py +++ b/sentieon_cli/stages/sv.py @@ -30,12 +30,19 @@ class LongReadSVResult(StageResult): class LongReadSVStage(Stage): """Call SVs from long reads with Sentieon LongReadSV. - The output VCF is derived from the run's output VCF, so the caller - only supplies the alignment and the model. + ``output`` names the SV VCF; when it is None the path is derived from + the run's output VCF, so the caller only supplies the alignment and + the model. The remaining fields are LongReadSV algo options, emitted + only when set. """ inputs: List[pathlib.Path] model: pathlib.Path + output: Optional[pathlib.Path] = None + min_map_qual: Optional[int] = None + min_sv_size: Optional[int] = None + min_dp: Optional[int] = None + min_af: Optional[float] = None interval: Optional[pathlib.Path] = None replace_rg: Optional[List[List[str]]] = None name: str = "LongReadSV" @@ -43,12 +50,23 @@ class LongReadSVStage(Stage): def build(self) -> LongReadSVResult: """Construct this stage's job without touching a DAG""" - sv_vcf = pathlib.Path( - str(self.ctx.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") - ) + sv_vcf = self.output + if sv_vcf is None: + sv_vcf = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") + ) job = driver_job( self.ctx, - [LongReadSV(sv_vcf, self.model)], + [ + LongReadSV( + sv_vcf, + model=self.model, + min_map_qual=self.min_map_qual, + min_sv_size=self.min_sv_size, + min_dp=self.min_dp, + min_af=self.min_af, + ) + ], inputs=self.inputs, interval=self.interval, replace_rg=self.replace_rg, diff --git a/tests/unit/test_stage_sv.py b/tests/unit/test_stage_sv.py index c7f86aa..388ce81 100644 --- a/tests/unit/test_stage_sv.py +++ b/tests/unit/test_stage_sv.py @@ -87,6 +87,49 @@ def test_without_replace_rg(self, tmp_path): assert "--replace_rg" not in str(result.job.shell) +class TestAlgoOptions: + """The optional LongReadSV algo arguments""" + + def test_min_sv_size(self, tmp_path): + result = make_stage(tmp_path, min_sv_size=20).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LongReadSV " + f"--model {tmp_path}/bundle/longreadsv.model " + f"--min_sv_size 20 " + f"{tmp_path}/output.sv.vcf.gz" + ) + + def test_min_map_qual(self, tmp_path): + result = make_stage(tmp_path, min_map_qual=30).add_to(DAG()) + + assert ( + f"--model {tmp_path}/bundle/longreadsv.model --min_map_qual 30 " + f"{tmp_path}/output.sv.vcf.gz" + ) in str(result.job.shell) + + def test_all_options(self, tmp_path): + result = make_stage( + tmp_path, + min_map_qual=30, + min_sv_size=20, + min_dp=5, + min_af=0.1, + ).add_to(DAG()) + + assert ( + "--min_map_qual 30 --min_sv_size 20 --min_dp 5 --min_af 0.1" + ) in str(result.job.shell) + + def test_options_default_to_absent(self, tmp_path): + shell = str(make_stage(tmp_path).add_to(DAG()).job.shell) + + for flag in ("min_map_qual", "min_sv_size", "min_dp", "min_af"): + assert f"--{flag}" not in shell + + class TestOutput: """The SV VCF is derived from the run's output VCF""" @@ -96,6 +139,14 @@ def test_sv_vcf(self, tmp_path): assert result.sv_vcf == tmp_path / "output.sv.vcf.gz" assert str(result.sv_vcf) in str(result.job.shell) + def test_an_explicit_output(self, tmp_path): + out_vcf = tmp_path / "tmp" / "sample-longread-sv.vcf.gz" + result = make_stage(tmp_path, output=out_vcf).add_to(DAG()) + + assert result.sv_vcf == out_vcf + assert str(result.job.shell).endswith(f" {out_vcf}") + assert "output.sv.vcf.gz" not in str(result.job.shell) + class TestJobMetadata: """Job name, task name and thread count""" From a494662b84085c9f425ee80516048af0c8321958 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Sat, 5 Sep 2026 11:17:46 -0700 Subject: [PATCH 20/20] Skip CNV calling in the hybrid smoke tests The smoke reference is not a recognised genome build, so no PAR BED resolves and CNV calling now fails validation up front. The smoke runs exercise alignment, duplicate marking and variant calling, not CNVscope, so they skip it. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1cee559..0a9ddac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -308,6 +308,7 @@ jobs: --sr_aln "tests/smoke/illumina.cram" \ --lr_aln "tests/smoke/sam ple.cram" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ --sr_duplicate_marking none \ "output_hybrid1.vcf.gz" if [ ! -f "output_hybrid1.vcf.gz" ]; then @@ -326,6 +327,7 @@ jobs: --lr_align_input \ --lr_input_ref "tests/smoke/r ef.fa" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ "output_hybrid2.vcf.gz" if [ ! -f "output_hybrid2.vcf.gz" ]; then exit 1 @@ -343,6 +345,7 @@ jobs: --sr_aln "tests/smoke/illumina.cram" "tests/smoke/illumina_2.bam" \ --lr_aln "tests/smoke/sam ple.cram" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ --sr_duplicate_marking none \ "output_hybrid3.vcf.gz" if [ ! -f "output_hybrid3.vcf.gz" ]; then