From 819060c48c2edcda0d68af0ee508f631f467b967 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Tue, 8 Sep 2026 10:28:31 -0700 Subject: [PATCH 1/4] Add segdup, ExpansionHunter, T1K and pangenome CNV stages Extract the optional pangenome callers into reusable stages so both pangenome pipelines can compose the same building blocks: `SegdupStage` (`stages/segdup.py`), `ExpansionHunterStage` (`stages/expansion.py`), `T1KStage` (`stages/t1k.py`, with the default HLA/KIR loci) and the composite `PangenomeCNVStage` in `stages/cnv.py` (CNVscope and CNVModelApply, indel2cnv.py on the PangenomeSV output, and combine_cnv.py over the two). `cmd_segdup_caller` gains optional `lr_alignments` / `lr_bundle` arguments for long-read calling; `cmd_pyexec_indel2cnv` and `cmd_pyexec_combine_cnv` move the two script invocations into `command_strings.py`. `util` gains `caller_sex_arg`, the shared `--sex male|female` mapping for ExpansionHunter and segdup-caller. No pipeline uses the new stages yet -- `sentieon_pangenome.py` keeps its inline builders, so the module-level version constants are duplicated for now -- and the DAG snapshots of all 88 configurations are identical. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/command_strings.py | 69 ++++++++++- sentieon_cli/stages/cnv.py | 108 ++++++++++++++++ sentieon_cli/stages/expansion.py | 71 +++++++++++ sentieon_cli/stages/segdup.py | 81 ++++++++++++ sentieon_cli/stages/t1k.py | 94 ++++++++++++++ sentieon_cli/util.py | 10 ++ tests/unit/test_cnv_sex.py | 14 +++ tests/unit/test_stage_cnv.py | 152 ++++++++++++++++++++++- tests/unit/test_stage_expansion.py | 130 +++++++++++++++++++ tests/unit/test_stage_segdup.py | 193 +++++++++++++++++++++++++++++ tests/unit/test_stage_t1k.py | 182 +++++++++++++++++++++++++++ 11 files changed, 1099 insertions(+), 5 deletions(-) create mode 100644 sentieon_cli/stages/expansion.py create mode 100644 sentieon_cli/stages/segdup.py create mode 100644 sentieon_cli/stages/t1k.py create mode 100644 tests/unit/test_stage_expansion.py create mode 100644 tests/unit/test_stage_segdup.py create mode 100644 tests/unit/test_stage_t1k.py diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index 0a0d132..ae5a5de 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -319,6 +319,46 @@ def cmd_pyexec_sad_lad_update( return Pipeline(Command(*cmd)) +def cmd_pyexec_indel2cnv( + out_vcf: pathlib.Path, + in_vcf: pathlib.Path, + reference: pathlib.Path, + indel2cnv_script: pathlib.Path, + threads: int, +) -> Pipeline: + """Convert PangenomeSV INDELs to CNV calls""" + cmd = [ + sys.executable, + str(indel2cnv_script), + str(reference), + str(in_vcf), + str(out_vcf), + "-t", + str(threads), + ] + return Pipeline(Command(*cmd)) + + +def cmd_pyexec_combine_cnv( + out_vcf: pathlib.Path, + cnv_vcf: pathlib.Path, + converted_vcf: pathlib.Path, + combine_script: pathlib.Path, +) -> Pipeline: + """Combine CNVscope and converted SV calls""" + cmd = [ + sys.executable, + str(combine_script), + "--cnv", + str(cnv_vcf), + "--converted", + str(converted_vcf), + "-o", + str(out_vcf), + ] + return Pipeline(Command(*cmd)) + + def hybrid_stage1_hap( out_hap_bam: pathlib.Path, stage1_driver: BaseDriver, @@ -1122,16 +1162,37 @@ def cmd_segdup_caller( sex: Optional[str] = None, genes: Optional[str] = None, overrides: Optional[List[str]] = None, + lr_alignments: Optional[pathlib.Path] = None, + lr_bundle: Optional[pathlib.Path] = None, ) -> Pipeline: + """Call variants in difficult segmental duplications. + + Long reads are optional; segdup-caller needs both the long-read + alignment and the long-read model bundle to use them. + """ + if (lr_alignments is None) != (lr_bundle is None): + raise ValueError( + "segdup-caller needs both `lr_alignments` and `lr_bundle` to " + "call with long reads" + ) + cmd = [ "segdup-caller", "--short", str(sr_alignments), - "--reference", - str(reference), - "--sr_model", - str(sr_bundle), ] + if lr_alignments is not None: + cmd.extend(["--long", str(lr_alignments)]) + cmd.extend( + [ + "--reference", + str(reference), + "--sr_model", + str(sr_bundle), + ] + ) + if lr_bundle is not None: + cmd.extend(["--lr_model", str(lr_bundle)]) if input_vcf is not None: cmd.extend(["--input_vcf", str(input_vcf)]) if sex is not None: diff --git a/sentieon_cli/stages/cnv.py b/sentieon_cli/stages/cnv.py index d8ff55b..d02e48e 100644 --- a/sentieon_cli/stages/cnv.py +++ b/sentieon_cli/stages/cnv.py @@ -3,11 +3,13 @@ """ from dataclasses import dataclass +from importlib.resources import files import pathlib from typing import Iterable, List, Optional import packaging.version +from .. import command_strings as cmds from ..dag import DAG from ..driver import CNVModelApply, CNVscope from ..job import Job @@ -96,3 +98,109 @@ def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> CNVResult: apply_job=apply_job, cnv_vcf=self.cnv_vcf, ) + + +@dataclass(kw_only=True) +class PangenomeCNVResult(StageResult): + """The jobs and output of `PangenomeCNVStage`""" + + cnvscope_job: Job + apply_job: Job + indel2cnv_job: Job + combine_job: Job + cnv_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class PangenomeCNVStage(Stage): + """CNV calling for the pangenome pipelines. + + CNVscope and CNVModelApply run on the short reads, the PangenomeSV + INDELs are converted to CNVs with `indel2cnv.py`, and the two call + sets are combined by `combine_cnv.py` into `_cnv.vcf.gz`. + Both inputs -- the alignments and the SV VCF -- come from an earlier + DAG, and the sample sex has to be known before the jobs are built. + """ + + inputs: List[pathlib.Path] + model: pathlib.Path + sv_vcf: pathlib.Path + sample_sex: Optional[SampleSex] = None + par_bed: Optional[pathlib.Path] = None + interval: Optional[pathlib.Path] = None + replace_rg: Optional[List[List[str]]] = None + task_name: str = "cnv" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> PangenomeCNVResult: + deps = set(upstream) + + cnvscope_vcf = self.ctx.tmp_dir.joinpath("sample-cnvscope.vcf.gz") + apply_vcf = self.ctx.tmp_dir.joinpath("sample-cnv_model_apply.vcf.gz") + converted_vcf = self.ctx.tmp_dir.joinpath("sample-sv_cnv.vcf.gz") + cnv_vcf = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") + ) + cnv_probes = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", "_cnv.probes") + ) + + cnv_result = CNVscopeStage( + ctx=self.ctx, + inputs=self.inputs, + model=self.model, + cnvscope_vcf=cnvscope_vcf, + cnv_vcf=apply_vcf, + sample_sex=self.sample_sex, + par_bed=self.par_bed, + dump_probes=cnv_probes, + interval=self.interval, + replace_rg=self.replace_rg, + task_name=self.task_name, + ).add_to(dag, deps) + + # Convert the PangenomeSV output to CNVs + indel2cnv_script = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("indel2cnv.py")) + ) + indel2cnv_job = Job( + cmds.cmd_pyexec_indel2cnv( + converted_vcf, + self.sv_vcf, + self.ctx.reference, + indel2cnv_script, + self.ctx.cores, + ), + "indel2cnv", + 0, + task_name=self.task_name, + ) + dag.add_job(indel2cnv_job, deps) + + # Combine the CNVModelApply output with the converted SVs + combine_script = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("combine_cnv.py")) + ) + combine_job = Job( + cmds.cmd_pyexec_combine_cnv( + cnv_vcf, + apply_vcf, + converted_vcf, + combine_script, + ), + "combine-cnv", + 0, + task_name=self.task_name, + ) + dag.add_job(combine_job, {cnv_result.apply_job, indel2cnv_job}) + + return PangenomeCNVResult( + jobs=[*cnv_result.jobs, indel2cnv_job, combine_job], + terminal={combine_job}, + cnvscope_job=cnv_result.cnvscope_job, + apply_job=cnv_result.apply_job, + indel2cnv_job=indel2cnv_job, + combine_job=combine_job, + cnv_vcf=cnv_vcf, + ) diff --git a/sentieon_cli/stages/expansion.py b/sentieon_cli/stages/expansion.py new file mode 100644 index 0000000..e75155c --- /dev/null +++ b/sentieon_cli/stages/expansion.py @@ -0,0 +1,71 @@ +""" +Repeat expansion genotyping with ExpansionHunter +""" + +from dataclasses import dataclass +import pathlib +from typing import Dict, Iterable, Optional + +import packaging.version + +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from ..util import SampleSex, caller_sex_arg +from .base import Stage, StageResult + +EXPANSION_MIN_VERSIONS: Dict[str, Optional[packaging.version.Version]] = { + "ExpansionHunter": None, +} + + +@dataclass(kw_only=True) +class ExpansionHunterResult(StageResult): + """The job and output of `ExpansionHunterStage`""" + + job: Job + output_prefix: pathlib.Path + + +@dataclass(kw_only=True) +class ExpansionHunterStage(Stage): + """Genotype repeat expansions with ExpansionHunter. + + ExpansionHunter is sex-aware, so callers run this stage after ploidy + estimation. ``output_prefix`` is the `--output-prefix` the tool + writes its files under. + """ + + alignment: pathlib.Path + variant_catalog: pathlib.Path + output_prefix: pathlib.Path + sample_sex: Optional[SampleSex] = None + name: str = "expansion-hunter" + task_name: str = "expansion-hunter" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> ExpansionHunterResult: + deps = set(upstream) + + job = Job( + cmds.cmd_expansion_hunter( + self.output_prefix, + self.alignment, + reference=self.ctx.reference, + variant_catalog=self.variant_catalog, + sex=caller_sex_arg(self.sample_sex), + threads=self.ctx.cores, + ), + self.name, + self.ctx.cores, + task_name=self.task_name, + ) + dag.add_job(job, deps) + + return ExpansionHunterResult( + jobs=[job], + terminal={job}, + job=job, + output_prefix=self.output_prefix, + ) diff --git a/sentieon_cli/stages/segdup.py b/sentieon_cli/stages/segdup.py new file mode 100644 index 0000000..5ecabb2 --- /dev/null +++ b/sentieon_cli/stages/segdup.py @@ -0,0 +1,81 @@ +""" +Variant calling in segmental duplications with segdup-caller +""" + +from dataclasses import dataclass +import pathlib +from typing import Dict, Iterable, Optional, Sequence + +import packaging.version + +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from ..util import SampleSex, caller_sex_arg +from .base import Stage, StageResult + +SEGDUP_MIN_VERSIONS: Dict[str, Optional[packaging.version.Version]] = { + # 0.7.0 adds the `--set` argument + "segdup-caller": packaging.version.Version("0.7.0"), +} + + +@dataclass(kw_only=True) +class SegdupResult(StageResult): + """The job and output of `SegdupStage`""" + + job: Job + output_dir: pathlib.Path + + +@dataclass(kw_only=True) +class SegdupStage(Stage): + """Call variants in difficult segmental duplications. + + segdup-caller is sex-aware, so callers run this stage after ploidy + estimation. Long reads are optional and need both ``lr_alignment`` + and ``lr_bundle``. ``genes`` is a comma-separated gene list; without + one the caller's default gene set is used. ``overrides`` become + ``--set KEY=VALUE`` arguments. + """ + + sr_alignment: pathlib.Path + sr_bundle: pathlib.Path + output_dir: pathlib.Path + input_vcf: Optional[pathlib.Path] = None + sample_sex: Optional[SampleSex] = None + genes: Optional[str] = None + lr_alignment: Optional[pathlib.Path] = None + lr_bundle: Optional[pathlib.Path] = None + overrides: Sequence[str] = () + name: str = "segdup-caller" + task_name: str = "segdup" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> SegdupResult: + deps = set(upstream) + + job = Job( + cmds.cmd_segdup_caller( + self.output_dir, + self.sr_alignment, + reference=self.ctx.reference, + sr_bundle=self.sr_bundle, + input_vcf=self.input_vcf, + sex=caller_sex_arg(self.sample_sex), + genes=self.genes, + overrides=list(self.overrides), + lr_alignments=self.lr_alignment, + lr_bundle=self.lr_bundle, + ), + self.name, + self.ctx.cores, + task_name=self.task_name, + ) + dag.add_job(job, deps) + + return SegdupResult( + jobs=[job], + terminal={job}, + job=job, + output_dir=self.output_dir, + ) diff --git a/sentieon_cli/stages/t1k.py b/sentieon_cli/stages/t1k.py new file mode 100644 index 0000000..6e2b264 --- /dev/null +++ b/sentieon_cli/stages/t1k.py @@ -0,0 +1,94 @@ +""" +HLA and KIR genotyping with T1K +""" + +from dataclasses import dataclass +import pathlib +from typing import Dict, Iterable, List, Optional + +import packaging.version + +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from .alignment import ReadWriterStage +from .base import Stage, StageResult + +T1K_MIN_VERSIONS: Dict[str, Optional[packaging.version.Version]] = { + "run-t1k": None, +} + +DEFAULT_T1K_HLA_LOCUS = "chr6:28510020-33480577" +DEFAULT_T1K_KIR_LOCUS = "chr19:53100000-55800000" + + +@dataclass(kw_only=True) +class T1KResult(StageResult): + """The jobs and outputs of `T1KStage`""" + + extract_job: Job + t1k_job: Job + extracted_bam: pathlib.Path + output_dir: pathlib.Path + + +@dataclass(kw_only=True) +class T1KStage(Stage): + """Extract the reads at a gene locus and genotype them with T1K. + + ``tag`` names the gene group ("hla" or "kir"); it labels the jobs and + the extracted BAM. ``preset`` is T1K's own `--preset` ("hla-wgs" or + "kir-wgs") and ``locus`` the region the reads are extracted from. + """ + + inputs: List[pathlib.Path] + gene_seq: pathlib.Path + gene_coord: pathlib.Path + locus: str + preset: str + tag: str + output_dir: pathlib.Path + task_name: str = "t1k" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> T1KResult: + deps = set(upstream) + + # Extract reads overlapping the locus to a BAM file with ReadWriter + extracted_bam = self.ctx.tmp_dir.joinpath(f"sample_{self.tag}.bam") + extract_job = ( + ReadWriterStage( + ctx=self.ctx, + inputs=self.inputs, + output=extracted_bam, + name=f"t1k-{self.tag}-extract", + task_name=self.task_name, + interval=self.locus, + ) + .build() + .jobs[0] + ) + dag.add_job(extract_job, deps) + + t1k_job = Job( + cmds.cmd_t1k( + self.output_dir, + extracted_bam, + gene_seq=self.gene_seq, + gene_coord=self.gene_coord, + preset=self.preset, + threads=self.ctx.cores, + ), + f"t1k-{self.tag}", + self.ctx.cores, + task_name=self.task_name, + ) + dag.add_job(t1k_job, {extract_job}) + + return T1KResult( + jobs=[extract_job, t1k_job], + terminal={t1k_job}, + extract_job=extract_job, + t1k_job=t1k_job, + extracted_bam=extracted_bam, + output_dir=self.output_dir, + ) diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 868dcd4..9dd6794 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -75,6 +75,16 @@ def cnvscope_sex_args( return (None, None) +def caller_sex_arg(sample_sex: Optional[SampleSex]) -> str: + """The `--sex male|female` value for ExpansionHunter and segdup-caller. + + Both tools accept only the two values, so an unknown sample sex + collapses to "female", unchanged from the previous + dnascope-pangenome behavior. + """ + return "male" if sample_sex is SampleSex.MALE else "female" + + def sanitize(component: str) -> str: """Restrict a path component to filesystem-safe characters""" return _UNSAFE.sub("-", component) diff --git a/tests/unit/test_cnv_sex.py b/tests/unit/test_cnv_sex.py index 27342ef..3fc4f1d 100644 --- a/tests/unit/test_cnv_sex.py +++ b/tests/unit/test_cnv_sex.py @@ -25,6 +25,7 @@ ) from sentieon_cli.util import ( SampleSex, + caller_sex_arg, cnvscope_sex_args, sample_sex_arg, ) @@ -210,6 +211,19 @@ def test_cnvscope_sex_args_without_a_known_sex(sex, messages): assert any("diploid" in msg for msg in messages) +# The ExpansionHunter and segdup-caller `--sex` argument + + +def test_caller_sex_arg_for_a_male_sample(): + assert caller_sex_arg(SampleSex.MALE) == "male" + + +@pytest.mark.parametrize("sex", [SampleSex.FEMALE, SampleSex.UNKNOWN, None]) +def test_caller_sex_arg_for_every_other_sample(sex): + # The two callers take only "male" or "female" + assert caller_sex_arg(sex) == "female" + + def test_cnvscope_algo_emits_the_sex_arguments(): cmd = CNVscope( pathlib.Path("out.vcf.gz"), diff --git a/tests/unit/test_stage_cnv.py b/tests/unit/test_stage_cnv.py index 5b97761..e0ee30c 100644 --- a/tests/unit/test_stage_cnv.py +++ b/tests/unit/test_stage_cnv.py @@ -2,8 +2,10 @@ Unit tests for the CNVscope stage """ +from importlib.resources import files import logging import pathlib +import sys from typing import List import packaging.version @@ -11,7 +13,11 @@ from sentieon_cli.dag import DAG from sentieon_cli.stages.base import StageContext, rm_job -from sentieon_cli.stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from sentieon_cli.stages.cnv import ( + CNV_MIN_VERSIONS, + CNVscopeStage, + PangenomeCNVStage, +) from sentieon_cli.util import SampleSex PACKAGE_LOGGER = "sentieon_cli" @@ -222,3 +228,147 @@ def test_result_fields(self, tmp_path): assert result.jobs == [result.cnvscope_job, result.apply_job] assert result.terminal == {result.apply_job} assert result.cnv_vcf == cnv_vcf + + +def make_pangenome_stage( + tmp_path: pathlib.Path, **kwargs +) -> PangenomeCNVStage: + """A PangenomeCNVStage over the short-read alignment and the SV VCF""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + model=tmp_path / "bundle" / "cnv.model", + sv_vcf=tmp_path / "output_sv.vcf.gz", + ) + defaults.update(kwargs) + return PangenomeCNVStage(**defaults) # type: ignore[arg-type] + + +def script_path(name: str) -> str: + """The packaged helper script, unresolved as the stage builds it""" + return str(files("sentieon_cli.scripts").joinpath(name)) + + +class TestPangenomeCNVStage: + """CNVscope, CNVModelApply, indel2cnv and combine_cnv together""" + + def test_job_names_threads_and_task(self, tmp_path): + result = make_pangenome_stage(tmp_path).add_to(DAG()) + + assert [job.name for job in result.jobs] == [ + "cnvscope", + "cnv-model-apply", + "indel2cnv", + "combine-cnv", + ] + assert [job.threads for job in result.jobs] == [4, 4, 0, 0] + for job in result.jobs: + assert job.task_name == "cnv" + + def test_derived_paths(self, tmp_path): + result = make_pangenome_stage(tmp_path).add_to(DAG()) + + # The intermediates live in the temporary directory; the CNV VCF + # and the probes file sit next to the output VCF + cnvscope_shell = str(result.cnvscope_job.shell) + assert f"{tmp_path}/sample-cnvscope.vcf.gz" in cnvscope_shell + assert f"--dump_probes {tmp_path}/output_cnv.probes" in cnvscope_shell + + apply_shell = str(result.apply_job.shell) + assert f"--vcf {tmp_path}/sample-cnvscope.vcf.gz" in apply_shell + assert f"{tmp_path}/sample-cnv_model_apply.vcf.gz" in apply_shell + + assert result.cnv_vcf == tmp_path / "output_cnv.vcf.gz" + + def test_indel2cnv_command(self, tmp_path): + result = make_pangenome_stage(tmp_path).add_to(DAG()) + + assert str(result.indel2cnv_job.shell) == ( + f"{sys.executable} {script_path('indel2cnv.py')} " + f"{tmp_path}/ref.fa {tmp_path}/output_sv.vcf.gz " + f"{tmp_path}/sample-sv_cnv.vcf.gz -t 4" + ) + + def test_combine_cnv_command(self, tmp_path): + result = make_pangenome_stage(tmp_path).add_to(DAG()) + + assert str(result.combine_job.shell) == ( + f"{sys.executable} {script_path('combine_cnv.py')} " + f"--cnv {tmp_path}/sample-cnv_model_apply.vcf.gz " + f"--converted {tmp_path}/sample-sv_cnv.vcf.gz " + f"-o {tmp_path}/output_cnv.vcf.gz" + ) + + def test_upstream_lands_on_the_entry_jobs(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_pangenome_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.cnvscope_job] == {upstream} + assert dag.waiting_jobs[result.indel2cnv_job] == {upstream} + assert dag.waiting_jobs[result.apply_job] == {result.cnvscope_job} + assert dag.waiting_jobs[result.combine_job] == { + result.apply_job, + result.indel2cnv_job, + } + + def test_entry_jobs_are_roots_without_upstream(self, tmp_path): + dag = DAG() + result = make_pangenome_stage(tmp_path).add_to(dag) + + assert result.cnvscope_job in dag.ready_jobs + assert result.indel2cnv_job in dag.ready_jobs + + def test_interval_and_replace_rg_reach_cnvscope_only(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_pangenome_stage( + tmp_path, + interval=bed, + replace_rg=[["rg1=ID:rg1\\tSM:sample"]], + ).add_to(DAG()) + + cnvscope_shell = str(result.cnvscope_job.shell) + assert f"--interval {bed}" in cnvscope_shell + assert "--replace_rg 'rg1=ID:rg1\\tSM:sample'" in cnvscope_shell + for job in ( + result.apply_job, + result.indel2cnv_job, + result.combine_job, + ): + assert "--interval" not in str(job.shell) + assert "--replace_rg" not in str(job.shell) + + def test_sex_arguments_reach_cnvscope(self, tmp_path): + par_bed = tmp_path / "par.bed" + result = make_pangenome_stage( + tmp_path, sample_sex=SampleSex.MALE, par_bed=par_bed + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex M" in shell + assert f"--par {par_bed}" in shell + + def test_female_sample_needs_no_par(self, tmp_path): + result = make_pangenome_stage( + tmp_path, + sample_sex=SampleSex.FEMALE, + par_bed=tmp_path / "par.bed", + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex F" in shell + assert "--par" not in shell + + def test_result_fields(self, tmp_path): + result = make_pangenome_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [ + result.cnvscope_job, + result.apply_job, + result.indel2cnv_job, + result.combine_job, + ] + assert result.terminal == {result.combine_job} + assert result.cnv_vcf == tmp_path / "output_cnv.vcf.gz" diff --git a/tests/unit/test_stage_expansion.py b/tests/unit/test_stage_expansion.py new file mode 100644 index 0000000..78026b3 --- /dev/null +++ b/tests/unit/test_stage_expansion.py @@ -0,0 +1,130 @@ +""" +Unit tests for the ExpansionHunter stage +""" + +import pathlib + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.expansion import ( + EXPANSION_MIN_VERSIONS, + ExpansionHunterStage, +) +from sentieon_cli.util import SampleSex + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> ExpansionHunterStage: + """An ExpansionHunterStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + alignment=tmp_path / "sample.cram", + variant_catalog=tmp_path / "catalog.json", + output_prefix=tmp_path / "output_expansion", + ) + defaults.update(kwargs) + return ExpansionHunterStage(**defaults) # type: ignore[arg-type] + + +class TestMinVersions: + """ExpansionHunter has no minimum version""" + + def test_no_minimum_version(self): + assert EXPANSION_MIN_VERSIONS == {"ExpansionHunter": None} + + +class TestCommands: + """The `ExpansionHunter` command the stage builds""" + + def test_expansion_hunter_command(self, tmp_path): + result = make_stage(tmp_path, sample_sex=SampleSex.MALE).add_to(DAG()) + + assert str(result.job.shell) == ( + f"ExpansionHunter --reads {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa " + f"--variant-catalog {tmp_path}/catalog.json " + "--sex male --threads 4 " + f"--output-prefix {tmp_path}/output_expansion" + ) + + @pytest.mark.parametrize( + "sample_sex,expected", + [ + (SampleSex.MALE, "male"), + (SampleSex.FEMALE, "female"), + (SampleSex.UNKNOWN, "female"), + (None, "female"), + ], + ) + def test_sex_argument(self, tmp_path, sample_sex, expected): + result = make_stage(tmp_path, sample_sex=sample_sex).add_to(DAG()) + + assert f"--sex {expected} " in str(result.job.shell) + + def test_threads_follow_the_run_cores(self, tmp_path): + result = make_stage(tmp_path, ctx=make_ctx(tmp_path, cores=16)).add_to( + DAG() + ) + + assert "--threads 16" in str(result.job.shell) + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_default_names_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "expansion-hunter" + assert result.job.task_name == "expansion-hunter" + assert result.job.threads == 4 + + def test_overridden_names(self, tmp_path): + result = make_stage( + tmp_path, name="expansions", task_name="expansions" + ).add_to(DAG()) + + assert result.job.name == "expansions" + assert result.job.task_name == "expansions" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_becomes_the_dependencies(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + output_prefix = tmp_path / "output_expansion" + result = make_stage(tmp_path, output_prefix=output_prefix).add_to( + DAG() + ) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + assert result.output_prefix == output_prefix diff --git a/tests/unit/test_stage_segdup.py b/tests/unit/test_stage_segdup.py new file mode 100644 index 0000000..35c7106 --- /dev/null +++ b/tests/unit/test_stage_segdup.py @@ -0,0 +1,193 @@ +""" +Unit tests for the segdup-caller stage +""" + +import pathlib + +import packaging.version +import pytest + +from sentieon_cli import command_strings as cmds +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.segdup import ( + SEGDUP_MIN_VERSIONS, + SegdupStage, +) +from sentieon_cli.util import SampleSex + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> SegdupStage: + """A SegdupStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + sr_alignment=tmp_path / "sample.cram", + sr_bundle=tmp_path / "bundle", + output_dir=tmp_path / "output_segdups", + input_vcf=tmp_path / "output.vcf.gz", + sample_sex=SampleSex.MALE, + ) + defaults.update(kwargs) + return SegdupStage(**defaults) # type: ignore[arg-type] + + +class TestMinVersions: + """The version constant lives next to the stage""" + + def test_set_argument_needs_0_7_0(self): + assert SEGDUP_MIN_VERSIONS["segdup-caller"] == ( + packaging.version.Version("0.7.0") + ) + + +class TestCommands: + """The `segdup-caller` commands the stage builds""" + + def test_short_read_command(self, tmp_path): + # The command dnascope-pangenome builds today + result = make_stage(tmp_path, genes="CFH,CYP2D6,SMN1").add_to(DAG()) + + assert str(result.job.shell) == ( + f"segdup-caller --short {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa " + f"--sr_model {tmp_path}/bundle " + f"--input_vcf {tmp_path}/output.vcf.gz " + "--sex male --genes CFH,CYP2D6,SMN1 " + f"--outdir {tmp_path}/output_segdups" + ) + + def test_long_read_command(self, tmp_path): + result = make_stage( + tmp_path, + lr_alignment=tmp_path / "long.cram", + lr_bundle=tmp_path / "bundle", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"segdup-caller --short {tmp_path}/sample.cram " + f"--long {tmp_path}/long.cram " + f"--reference {tmp_path}/ref.fa " + f"--sr_model {tmp_path}/bundle " + f"--lr_model {tmp_path}/bundle " + f"--input_vcf {tmp_path}/output.vcf.gz " + "--sex male " + f"--outdir {tmp_path}/output_segdups" + ) + + def test_overrides_become_set_arguments(self, tmp_path): + result = make_stage( + tmp_path, + overrides=["main.min_map_qual=30", "main.other=1"], + ).add_to(DAG()) + + assert "--set main.min_map_qual=30 --set main.other=1 --outdir" in str( + result.job.shell + ) + + def test_without_an_input_vcf_or_genes(self, tmp_path): + result = make_stage(tmp_path, input_vcf=None).add_to(DAG()) + + shell = str(result.job.shell) + assert "--input_vcf" not in shell + assert "--genes" not in shell + assert "--set" not in shell + + @pytest.mark.parametrize( + "sample_sex,expected", + [ + (SampleSex.MALE, "male"), + (SampleSex.FEMALE, "female"), + (SampleSex.UNKNOWN, "female"), + (None, "female"), + ], + ) + def test_sex_argument(self, tmp_path, sample_sex, expected): + result = make_stage(tmp_path, sample_sex=sample_sex).add_to(DAG()) + + assert f"--sex {expected} " in str(result.job.shell) + + +class TestLongReadValidation: + """Long-read calling needs both the alignment and the model bundle""" + + def test_alignment_without_a_bundle(self, tmp_path): + with pytest.raises(ValueError): + cmds.cmd_segdup_caller( + tmp_path / "out", + tmp_path / "sample.cram", + reference=tmp_path / "ref.fa", + sr_bundle=tmp_path / "bundle", + lr_alignments=tmp_path / "long.cram", + ) + + def test_bundle_without_an_alignment(self, tmp_path): + with pytest.raises(ValueError): + cmds.cmd_segdup_caller( + tmp_path / "out", + tmp_path / "sample.cram", + reference=tmp_path / "ref.fa", + sr_bundle=tmp_path / "bundle", + lr_bundle=tmp_path / "bundle", + ) + + def test_the_stage_propagates_the_error(self, tmp_path): + with pytest.raises(ValueError): + make_stage(tmp_path, lr_bundle=tmp_path / "bundle").add_to(DAG()) + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_default_names_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "segdup-caller" + assert result.job.task_name == "segdup" + assert result.job.threads == 4 + + def test_overridden_names(self, tmp_path): + result = make_stage( + tmp_path, name="segdups", task_name="segdups" + ).add_to(DAG()) + + assert result.job.name == "segdups" + assert result.job.task_name == "segdups" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_becomes_the_dependencies(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + output_dir = tmp_path / "output_segdups" + result = make_stage(tmp_path, output_dir=output_dir).add_to(DAG()) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + assert result.output_dir == output_dir diff --git a/tests/unit/test_stage_t1k.py b/tests/unit/test_stage_t1k.py new file mode 100644 index 0000000..b08b265 --- /dev/null +++ b/tests/unit/test_stage_t1k.py @@ -0,0 +1,182 @@ +""" +Unit tests for the T1K HLA/KIR genotyping stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.t1k import ( + DEFAULT_T1K_HLA_LOCUS, + DEFAULT_T1K_KIR_LOCUS, + T1K_MIN_VERSIONS, + T1KStage, +) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> T1KStage: + """A T1KStage genotyping HLA, unless the caller says otherwise""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + gene_seq=tmp_path / "hla_seq.fa", + gene_coord=tmp_path / "hla_coord.fa", + locus=DEFAULT_T1K_HLA_LOCUS, + preset="hla-wgs", + tag="hla", + output_dir=tmp_path / "output_hla", + ) + defaults.update(kwargs) + return T1KStage(**defaults) # type: ignore[arg-type] + + +def make_kir_stage(tmp_path: pathlib.Path, **kwargs) -> T1KStage: + """A T1KStage genotyping KIR""" + return make_stage( + tmp_path, + gene_seq=tmp_path / "kir_seq.fa", + gene_coord=tmp_path / "kir_coord.fa", + locus=DEFAULT_T1K_KIR_LOCUS, + preset="kir-wgs", + tag="kir", + output_dir=tmp_path / "output_kir", + **kwargs, + ) + + +class TestConstants: + """The version constant and the default loci live with the stage""" + + def test_no_minimum_version(self): + assert T1K_MIN_VERSIONS == {"run-t1k": None} + + def test_default_loci(self): + assert DEFAULT_T1K_HLA_LOCUS == "chr6:28510020-33480577" + assert DEFAULT_T1K_KIR_LOCUS == "chr19:53100000-55800000" + + +class TestCommands: + """The extraction and `run-t1k` commands the stage builds""" + + def test_hla_extract_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.extract_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + "--interval chr6:28510020-33480577 " + f"--algo ReadWriter {tmp_path}/sample_hla.bam" + ) + + def test_hla_t1k_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.t1k_job.shell) == ( + "run-t1k --abnormalUnmapFlag -t 4 --preset hla-wgs " + f"-f {tmp_path}/hla_seq.fa -c {tmp_path}/hla_coord.fa " + f"--od {tmp_path}/output_hla -b {tmp_path}/sample_hla.bam" + ) + + def test_kir_extract_command(self, tmp_path): + result = make_kir_stage(tmp_path).add_to(DAG()) + + assert str(result.extract_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + "--interval chr19:53100000-55800000 " + f"--algo ReadWriter {tmp_path}/sample_kir.bam" + ) + + def test_kir_t1k_command(self, tmp_path): + result = make_kir_stage(tmp_path).add_to(DAG()) + + assert str(result.t1k_job.shell) == ( + "run-t1k --abnormalUnmapFlag -t 4 --preset kir-wgs " + f"-f {tmp_path}/kir_seq.fa -c {tmp_path}/kir_coord.fa " + f"--od {tmp_path}/output_kir -b {tmp_path}/sample_kir.bam" + ) + + def test_multiple_inputs_are_merged_by_the_extract(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + ).add_to(DAG()) + + shell = str(result.extract_job.shell) + assert f"--input {tmp_path}/one.cram" in shell + assert f"--input {tmp_path}/two.cram" in shell + + def test_a_custom_locus(self, tmp_path): + result = make_stage(tmp_path, locus="chr6:1-100").add_to(DAG()) + + assert "--interval chr6:1-100 " in str(result.extract_job.shell) + + +class TestJobMetadata: + """Job names, task name and thread counts""" + + def test_hla_job_names(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.extract_job.name == "t1k-hla-extract" + assert result.t1k_job.name == "t1k-hla" + + def test_kir_job_names(self, tmp_path): + result = make_kir_stage(tmp_path).add_to(DAG()) + + assert result.extract_job.name == "t1k-kir-extract" + assert result.t1k_job.name == "t1k-kir" + + def test_task_name_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "t1k" + assert job.threads == 4 + + def test_overridden_task_name(self, tmp_path): + result = make_stage(tmp_path, task_name="hla").add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "hla" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_the_extract_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.extract_job] == {upstream} + assert dag.waiting_jobs[result.t1k_job] == {result.extract_job} + + def test_the_extract_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.extract_job in dag.ready_jobs + assert dag.waiting_jobs[result.t1k_job] == {result.extract_job} + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [result.extract_job, result.t1k_job] + assert result.terminal == {result.t1k_job} + assert result.extracted_bam == tmp_path / "sample_hla.bam" + assert result.output_dir == tmp_path / "output_hla" From 6e3bc766221242ce6339d8d0756580f72374e5fe Mon Sep 17 00:00:00 2001 From: Don Freed Date: Tue, 8 Sep 2026 10:36:56 -0700 Subject: [PATCH 2/4] Compose the dnascope-pangenome callers from the shared stages `SentieonPangenome` now builds its optional callers from the stages added in the previous commit, and everything both pangenome pipelines will need moved up to `BasePangenome`. `BasePangenome` gains the shared caller arguments -- `par_bed`, `sample_sex`, `segdup_caller`, `expansion_catalog` and the six `t1k_*` (the two loci defaulting to the constants in `stages/t1k.py`) -- with the matching `__init__` attributes, `has_cnv_model`, and the DAG-2 stashes `ploidy_json` / `sr_alignments` / `sr_replace_rg` (`sr_replace_rg` is unused until the hybrid pipeline needs it). `validate_t1k` moved up unchanged, and four thin helpers build the stages: `output_path(suffix)`, `add_t1k(dag, ctx, inputs, upstream)`, `add_expansion(dag, ctx, alignment)`, `add_segdup(dag, ctx, sr_alignment, *, lr_alignment, lr_bundle, overrides)` and `add_pangenome_cnv(dag, ctx, sv_vcf, inputs, *, replace_rg, interval)`. `--call_svs` stays on `SentieonPangenome`. `hybrid-pangenome` inherits the new flags, which are accepted but build no jobs yet. Deleted from `sentieon_pangenome.py`: the `SEGDUP_MIN_VERSION` / `EXPANSION_MIN_VERSION` / `T1K_MIN_VERSION` constants (now `*_MIN_VERSIONS` next to their stages) and the default T1K loci, the moved params and `__init__` lines, `validate_t1k`, `build_segdup_job`, `build_t1k_jobs`, `build_expansion_job`, `_add_cnv_jobs`, `_build_indel2cnv_job`, `_build_combine_cnv_job`, and the dead `self.rw_bam` that `build_dag` assigned but nothing read. The DAG snapshot over all 88 configurations is identical, including `compare --with-ids`. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/base_pangenome.py | 253 ++++++++++++++- sentieon_cli/sentieon_pangenome.py | 438 ++------------------------ tests/unit/test_sentieon_pangenome.py | 260 ++++++++++++--- 3 files changed, 496 insertions(+), 455 deletions(-) diff --git a/sentieon_cli/base_pangenome.py b/sentieon_cli/base_pangenome.py index f21e9a6..bf7f90f 100644 --- a/sentieon_cli/base_pangenome.py +++ b/sentieon_cli/base_pangenome.py @@ -4,9 +4,11 @@ import copy import pathlib -from typing import List, Optional +import sys +from typing import Iterable, List, Optional, Sequence from . import command_strings as cmds +from .dag import DAG from .driver import ( AlignmentStat, BaseAlgo, @@ -20,8 +22,19 @@ ) from .job import Job from .pipeline import BasePipeline +from .stages.base import StageContext +from .stages.cnv import PangenomeCNVResult, PangenomeCNVStage +from .stages.expansion import ExpansionHunterResult, ExpansionHunterStage from .stages.metrics import MetricsPaths -from .util import path_arg +from .stages.segdup import SegdupResult, SegdupStage +from .stages.t1k import ( + DEFAULT_T1K_HLA_LOCUS, + DEFAULT_T1K_KIR_LOCUS, + T1K_MIN_VERSIONS, + T1KResult, + T1KStage, +) +from .util import path_arg, require_versions, sample_sex_arg class BasePangenome(BasePipeline): @@ -82,6 +95,84 @@ class BasePangenome(BasePipeline): "help": "Use arguments for PCR-free data processing", "action": "store_true", }, + "par_bed": { + "help": ( + "A BED file of the pseudo-autosomal regions (PAR), used " + "for sex-aware CNV calling of male samples. Overrides the " + "PAR BED file selected for the reference genome." + ), + "type": path_arg(exists=True, is_file=True), + }, + "sample_sex": { + "help": ( + "The sample sex, used by the sex-aware callers. " + "Supplying this argument overrides the sex estimated " + "from read coverage." + ), + "metavar": "{male,female}", + "type": sample_sex_arg, + }, + "segdup_caller": { + "nargs": "*", + "help": ( + "Call variants in difficult segmental duplications with " + "segdup-caller. Supply the flag with no arguments to run " + "the caller's default gene set. Supply a comma-separated " + "list of gene names (e.g. 'CFH,CYP2D6,SMN1') to restrict " + "calling to those genes." + ), + }, + "expansion_catalog": { + "help": ( + "An ExpansionHunter variant catalog. Required for short " + "tandem repeat expansion calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_hla_seq": { + "help": ( + "The DNA HLA seq FASTA file for T1K. Required for HLA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_hla_coord": { + "help": ( + "The DNA HLA coord FASTA file for T1K. Required for HLA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_hla_locus": { + "default": DEFAULT_T1K_HLA_LOCUS, + "help": ( + "Reference interval covering the HLA locus. Reads " + "overlapping this region are extracted before being " + f"passed to T1K (default: {DEFAULT_T1K_HLA_LOCUS})." + ), + }, + "t1k_kir_seq": { + "help": ( + "The DNA KIR seq FASTA file for T1K. Required for KIR " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_kir_coord": { + "help": ( + "The DNA KIR coord FASTA file for T1K. Required for KIR " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_kir_locus": { + "default": DEFAULT_T1K_KIR_LOCUS, + "help": ( + "Reference interval covering the KIR locus. Reads " + "overlapping this region are extracted before being " + f"passed to T1K (default: {DEFAULT_T1K_KIR_LOCUS})." + ), + }, } ) @@ -98,6 +189,164 @@ def __init__(self) -> None: self.dbsnp: Optional[pathlib.Path] = None self.kmer_memory = 30 self.pcr_free = False + self.par_bed: Optional[pathlib.Path] = None + self.segdup_caller: Optional[List[str]] = None + self.expansion_catalog: Optional[pathlib.Path] = None + self.t1k_hla_seq: Optional[pathlib.Path] = None + self.t1k_hla_coord: Optional[pathlib.Path] = None + self.t1k_hla_locus: str = DEFAULT_T1K_HLA_LOCUS + self.t1k_kir_seq: Optional[pathlib.Path] = None + self.t1k_kir_coord: Optional[pathlib.Path] = None + self.t1k_kir_locus: str = DEFAULT_T1K_KIR_LOCUS + self.has_cnv_model = False + # Stashed by `build_dag` for the second, sex-aware DAG + self.ploidy_json: Optional[pathlib.Path] = None + self.sr_alignments: List[pathlib.Path] = [] + self.sr_replace_rg: Optional[List[List[str]]] = None + + def validate_t1k(self) -> None: + hla_requested = self.t1k_hla_seq is not None or ( + self.t1k_hla_coord is not None + ) + if hla_requested and not (self.t1k_hla_seq and self.t1k_hla_coord): + self.logger.error( + "For HLA calling, both `--t1k_hla_seq` and `--t1k_hla_coord` " + "must be supplied." + ) + sys.exit(2) + + kir_requested = self.t1k_kir_seq is not None or ( + self.t1k_kir_coord is not None + ) + if kir_requested and not (self.t1k_kir_seq and self.t1k_kir_coord): + self.logger.error( + "For KIR calling, both `--t1k_kir_seq` and `--t1k_kir_coord` " + "must be supplied." + ) + sys.exit(2) + + if not (hla_requested or kir_requested): + return + + require_versions(T1K_MIN_VERSIONS, skip=self.skip_version_check) + + def output_path(self, suffix: str) -> pathlib.Path: + """A path next to the output VCF, with `.vcf.gz` replaced""" + output_vcf = self.required(self.output_vcf, "output_vcf") + return pathlib.Path(str(output_vcf).replace(".vcf.gz", suffix)) + + def add_t1k( + self, + dag: DAG, + ctx: StageContext, + inputs: List[pathlib.Path], + upstream: Iterable[Job] = (), + ) -> List[T1KResult]: + """Genotype every enabled T1K gene group from `inputs`""" + deps = set(upstream) + gene_groups = ( + ( + "hla", + "hla-wgs", + self.t1k_hla_seq, + self.t1k_hla_coord, + self.t1k_hla_locus, + ), + ( + "kir", + "kir-wgs", + self.t1k_kir_seq, + self.t1k_kir_coord, + self.t1k_kir_locus, + ), + ) + + results: List[T1KResult] = [] + for tag, preset, gene_seq, gene_coord, locus in gene_groups: + if not (gene_seq and gene_coord): + continue + results.append( + T1KStage( + ctx=ctx, + inputs=inputs, + gene_seq=gene_seq, + gene_coord=gene_coord, + locus=locus, + preset=preset, + tag=tag, + output_dir=self.output_path(f"_{tag}"), + ).add_to(dag, deps) + ) + return results + + def add_expansion( + self, + dag: DAG, + ctx: StageContext, + alignment: pathlib.Path, + ) -> Optional[ExpansionHunterResult]: + """Genotype repeat expansions, when a catalog was supplied""" + if self.expansion_catalog is None: + return None + + return ExpansionHunterStage( + ctx=ctx, + alignment=alignment, + variant_catalog=self.expansion_catalog, + output_prefix=self.output_path("_expansion"), + sample_sex=self.sample_sex, + ).add_to(dag) + + def add_segdup( + self, + dag: DAG, + ctx: StageContext, + sr_alignment: pathlib.Path, + *, + lr_alignment: Optional[pathlib.Path] = None, + lr_bundle: Optional[pathlib.Path] = None, + overrides: Sequence[str] = (), + ) -> Optional[SegdupResult]: + """Call segmental duplications, when `--segdup_caller` was set""" + if self.segdup_caller is None: + return None + + return SegdupStage( + ctx=ctx, + sr_alignment=sr_alignment, + sr_bundle=self.required(self.model_bundle, "model_bundle"), + output_dir=self.output_path("_segdups"), + input_vcf=ctx.output_vcf, + sample_sex=self.sample_sex, + genes=",".join(self.segdup_caller) or None, + lr_alignment=lr_alignment, + lr_bundle=lr_bundle, + overrides=overrides, + ).add_to(dag) + + def add_pangenome_cnv( + self, + dag: DAG, + ctx: StageContext, + sv_vcf: pathlib.Path, + inputs: List[pathlib.Path], + *, + replace_rg: Optional[List[List[str]]] = None, + interval: Optional[pathlib.Path] = None, + ) -> PangenomeCNVResult: + """Call CNVs from the short reads and the PangenomeSV output""" + bundle = self.required(self.model_bundle, "model_bundle") + + return PangenomeCNVStage( + ctx=ctx, + inputs=inputs, + model=bundle.joinpath("cnv.model"), + sv_vcf=sv_vcf, + sample_sex=self.sample_sex, + par_bed=self.cnv_par_bed, + interval=interval, + replace_rg=replace_rg, + ).add_to(dag) def build_kmc_job( self, kmer_prefix: pathlib.Path, job_threads: int diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 74d7f75..65c2375 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -7,7 +7,7 @@ import json import pathlib import sys -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Union import packaging.version @@ -27,14 +27,15 @@ from .shell_pipeline import Command, Pipeline from .stages.alignment import ( BwaExtractStage, - ReadWriterStage, find_unzip, ) from .stages.base import StageContext -from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.cnv import CNV_MIN_VERSIONS from .stages.dedup import DedupStage +from .stages.expansion import EXPANSION_MIN_VERSIONS from .stages.metrics import MetricsPaths, MetricsStage from .stages.ploidy import PloidyStage +from .stages.segdup import SEGDUP_MIN_VERSIONS from .stages.small_variants import ( ApplySpec, DNAscopeStage, @@ -44,13 +45,11 @@ ) from .stages.transfer import TransferConfig from .util import ( - SampleSex, __version__, check_kmc_patch, parse_rg_line, path_arg, require_versions, - sample_sex_arg, total_memory, vcf_id, ) @@ -69,21 +68,6 @@ "samtools": packaging.version.Version("1.16"), } -SEGDUP_MIN_VERSION = { - "segdup-caller": packaging.version.Version("0.7.0"), -} - -EXPANSION_MIN_VERSION = { - "ExpansionHunter": None, -} - -T1K_MIN_VERSION = { - "run-t1k": None, -} - -DEFAULT_T1K_HLA_LOCUS = "chr6:28510020-33480577" -DEFAULT_T1K_KIR_LOCUS = "chr19:53100000-55800000" - logger = get_logger(__name__) @@ -145,23 +129,6 @@ class SentieonPangenome(BasePangenome): "Prefix to strip from pangenome contig names (GRCh38#0#)" ), }, - "par_bed": { - "help": ( - "A BED file of the pseudo-autosomal regions (PAR), used " - "for sex-aware CNV calling of male samples. Overrides the " - "PAR BED file selected for the reference genome." - ), - "type": path_arg(exists=True, is_file=True), - }, - "sample_sex": { - "help": ( - "The sample sex, used by the sex-aware callers. " - "Supplying this argument overrides the sex estimated " - "from read coverage." - ), - "metavar": "{male,female}", - "type": sample_sex_arg, - }, "skip_metrics": { "help": "Skip metrics collection and multiQC", "action": "store_true", @@ -170,67 +137,6 @@ class SentieonPangenome(BasePangenome): "help": "Skip multiQC report generation", "action": "store_true", }, - "segdup_caller": { - "nargs": "*", - "help": ( - "Call variants in difficult segmental duplications with " - "segdup-caller. Supply the flag with no arguments to run " - "the caller's default gene set. Supply a comma-separated " - "list of gene names (e.g. 'CFH,CYP2D6,SMN1') to restrict " - "calling to those genes." - ), - }, - "expansion_catalog": { - "help": ( - "An ExpansionHunter variant catalog. Required for short " - "tandem repeat expansion calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_hla_seq": { - "help": ( - "The DNA HLA seq FASTA file for T1K. Required for HLA " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_hla_coord": { - "help": ( - "The DNA HLA coord FASTA file for T1K. Required for HLA " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_hla_locus": { - "default": DEFAULT_T1K_HLA_LOCUS, - "help": ( - "Reference interval covering the HLA locus. Reads " - "overlapping this region are extracted before being " - f"passed to T1K (default: {DEFAULT_T1K_HLA_LOCUS})." - ), - }, - "t1k_kir_seq": { - "help": ( - "The DNA KIR seq FASTA file for T1K. Required for KIR " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_kir_coord": { - "help": ( - "The DNA KIR coord FASTA file for T1K. Required for KIR " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_kir_locus": { - "default": DEFAULT_T1K_KIR_LOCUS, - "help": ( - "Reference interval covering the KIR locus. Reads " - "overlapping this region are extracted before being " - f"passed to T1K (default: {DEFAULT_T1K_KIR_LOCUS})." - ), - }, # Hidden arguments "skip_contig_checks": { "help": argparse.SUPPRESS, @@ -265,29 +171,16 @@ def __init__(self) -> None: self.bed: Optional[pathlib.Path] = None self.call_svs = False self.gvcf = False - self.par_bed: Optional[pathlib.Path] = None self.pangenome_ref_name = "GRCh38" self.extract_model_name = "extract.model" self.pangenome_contig_prefix = "GRCh38#0#" self.skip_metrics = False self.skip_multiqc = False - self.segdup_caller: Optional[List[str]] = None - self.expansion_catalog: Optional[pathlib.Path] = None - self.t1k_hla_seq: Optional[pathlib.Path] = None - self.t1k_hla_coord: Optional[pathlib.Path] = None - self.t1k_hla_locus: str = DEFAULT_T1K_HLA_LOCUS - self.t1k_kir_seq: Optional[pathlib.Path] = None - self.t1k_kir_coord: Optional[pathlib.Path] = None - self.t1k_kir_locus: str = DEFAULT_T1K_KIR_LOCUS self.skip_contig_checks: bool = False self.skip_pangenome_name_checks: bool = False self.skip_pop_vcf_id_check: bool = False self.skip_model_apply = False self.skip_small_variants = False - # Stashed by `build_dag` for the second, sex-aware DAG - self.ploidy_json: Optional[pathlib.Path] = None - self.sr_alignment: Optional[pathlib.Path] = None - self.cnv_input_bams: List[pathlib.Path] = [] def _cnv_in_second_dag(self) -> bool: """CNV calling runs in the second, sex-aware DAG""" @@ -425,7 +318,7 @@ def validate_segdup(self) -> None: ) sys.exit(2) - require_versions(SEGDUP_MIN_VERSION, skip=self.skip_version_check) + require_versions(SEGDUP_MIN_VERSIONS, skip=self.skip_version_check) def validate_expansion(self) -> None: if self.expansion_catalog is None: @@ -445,7 +338,7 @@ def validate_expansion(self) -> None: ) sys.exit(2) - require_versions(EXPANSION_MIN_VERSION, skip=self.skip_version_check) + require_versions(EXPANSION_MIN_VERSIONS, skip=self.skip_version_check) def validate_cnv(self) -> None: """Validate the arguments used for sex-aware CNV calling""" @@ -460,32 +353,6 @@ def validate_cnv(self) -> None: require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) - def validate_t1k(self) -> None: - hla_requested = self.t1k_hla_seq is not None or ( - self.t1k_hla_coord is not None - ) - if hla_requested and not (self.t1k_hla_seq and self.t1k_hla_coord): - self.logger.error( - "For HLA calling, both `--t1k_hla_seq` and `--t1k_hla_coord` " - "must be supplied." - ) - sys.exit(2) - - kir_requested = self.t1k_kir_seq is not None or ( - self.t1k_kir_coord is not None - ) - if kir_requested and not (self.t1k_kir_seq and self.t1k_kir_coord): - self.logger.error( - "For KIR calling, both `--t1k_kir_seq` and `--t1k_kir_coord` " - "must be supplied." - ) - sys.exit(2) - - if not (hla_requested or kir_requested): - return - - require_versions(T1K_MIN_VERSION, skip=self.skip_version_check) - def validate_bundle(self) -> None: pop_vcf = self.required(self.pop_vcf, "pop_vcf") gbz = self.required(self.gbz, "gbz") @@ -662,8 +529,6 @@ def build_dag(self) -> DAG: ) raw_vcf = self.tmp_dir.joinpath("sample-dnascope.vcf.gz") transfer_vcf = self.tmp_dir.joinpath("sample-dnascope_transfer.vcf.gz") - # - Ensure we have a bam file for t1k - self.rw_bam = self.tmp_dir.joinpath("sample_deduped.bam") bwa_lc_dependencies: Set[Job] = set() haplotype_dependencies: Set[Job] = set() @@ -742,7 +607,7 @@ def build_dag(self) -> DAG: dnascope_dependencies.add(mm2_job) # With fastq input, perform dedup and metrics - cnv_input_bams: List[pathlib.Path] = [] + sr_alignments: List[pathlib.Path] = [] cnvscope_deps: Set[Job] = set() if self.r1_fastq: dnascope_bams.append(out_bwa_aln) @@ -779,7 +644,7 @@ def build_dag(self) -> DAG: ).add_to(dag, {mm2_job}) dnascope_dependencies.add(mm2_dedup.dedup_job) - cnv_input_bams = [out_bwa_aln] + sr_alignments = [out_bwa_aln] cnvscope_deps = {bwa_dedup_job} if not self.skip_metrics: @@ -795,53 +660,23 @@ def build_dag(self) -> DAG: dag.add_job(multiqc_job, metrics_result.terminal) else: dnascope_bams.append(mm2_bam) - cnv_input_bams = list(self.sample_input) + sr_alignments = list(self.sample_input) - # Stash the short-read alignment for the second DAG - self.sr_alignment = cnv_input_bams[0] - self.cnv_input_bams = cnv_input_bams + # Stash the short-read alignments for the second DAG + self.sr_alignments = sr_alignments # Estimate the sample ploidy and sex. The JSON output is always # written; `--sample_sex` takes precedence for the sex used by # the sex-aware callers. ploidy_result = PloidyStage( ctx=ctx, - inputs=[cnv_input_bams[0]], + inputs=[self.sr_alignments[0]], reference_build=self.reference_build, ).add_to(dag, cnvscope_deps) self.ploidy_json = ploidy_result.ploidy_json # T1K HLA/KIR calling - if self.t1k_hla_seq and self.t1k_hla_coord: - out_hla = pathlib.Path( - str(ctx.output_vcf).replace(".vcf.gz", "_hla") - ) - extract_job, t1k_job = self.build_t1k_jobs( - out_hla, - cnv_input_bams, - self.t1k_hla_seq, - self.t1k_hla_coord, - self.t1k_hla_locus, - "hla-wgs", - tag="hla", - ) - dag.add_job(extract_job, cnvscope_deps) - dag.add_job(t1k_job, {extract_job}) - if self.t1k_kir_seq and self.t1k_kir_coord: - out_kir = pathlib.Path( - str(ctx.output_vcf).replace(".vcf.gz", "_kir") - ) - extract_job, t1k_job = self.build_t1k_jobs( - out_kir, - cnv_input_bams, - self.t1k_kir_seq, - self.t1k_kir_coord, - self.t1k_kir_locus, - "kir-wgs", - tag="kir", - ) - dag.add_job(extract_job, cnvscope_deps) - dag.add_job(t1k_job, {extract_job}) + self.add_t1k(dag, ctx, self.sr_alignments, cnvscope_deps) # DNAscope calling with bwa and mm2 input sv_vcf = None @@ -1099,41 +934,6 @@ def build_count_ad_update_job( task_name="ad-update", ) - def build_segdup_job( - self, - out_segdup: pathlib.Path, - sr_alignment: pathlib.Path, - input_vcf: pathlib.Path, - genes: Optional[str], - ) -> Job: - """Call variants in difficult SegDups""" - reference = self.required(self.reference, "reference") - bundle = self.required(self.model_bundle, "model_bundle") - - sex = "male" if self.sample_sex == SampleSex.MALE else "female" - - # segdup-caller's default `main.min_map_qual` of 45 is too strict - # for Ultima alignments. - overrides = [] - if self.tech.upper() == "ULTIMA": - overrides.append("main.min_map_qual=30") - - return Job( - cmds.cmd_segdup_caller( - out_segdup, - sr_alignment, - reference=reference, - sr_bundle=bundle, - input_vcf=input_vcf, - sex=sex, - genes=genes, - overrides=overrides, - ), - "segdup-caller", - self.cores, - task_name="segdup", - ) - def build_second_dag(self) -> Optional[DAG]: """Build the second DAG for sex-aware downstream tools""" if not self._needs_second_dag(): @@ -1144,201 +944,33 @@ def build_second_dag(self) -> Optional[DAG]: self.logger.info("Building the second pangenome DAG") dag = DAG() + ctx = self.stage_context() # CNV calling with CNVscope, using the sample sex if self._cnv_in_second_dag(): - sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") - ) - self._add_cnv_jobs(dag, sv_vcf, self.cnv_input_bams) - - if self.expansion_catalog and self.sr_alignment: - out_expansions = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_expansion") - ) - expansion_job = self.build_expansion_job( - out_expansions, self.sr_alignment, self.expansion_catalog + self.add_pangenome_cnv( + dag, + ctx, + self.output_path("_sv.vcf.gz"), + self.sr_alignments, + interval=self.bed, ) - dag.add_job(expansion_job) - # SegDup calling consumes the small-variant VCF and the inferred sex - if ( - self.segdup_caller is not None - and self.sr_alignment - and self.output_vcf - ): - out_segdup = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_segdups") - ) - genes = ",".join(self.segdup_caller) or None - segdup_job = self.build_segdup_job( - out_segdup, - self.sr_alignment, - self.output_vcf, - genes, + if self.sr_alignments: + self.add_expansion(dag, ctx, self.sr_alignments[0]) + + # SegDup calling consumes the small-variant VCF and the + # inferred sex. segdup-caller's default `main.min_map_qual` + # of 45 is too strict for Ultima alignments. + self.add_segdup( + dag, + ctx, + self.sr_alignments[0], + overrides=( + ["main.min_map_qual=30"] + if self.tech.upper() == "ULTIMA" + else () + ), ) - dag.add_job(segdup_job) return dag - - def build_t1k_jobs( - self, - out_basename: pathlib.Path, - sr_alignments: List[pathlib.Path], - gene_seq: pathlib.Path, - gene_coord: pathlib.Path, - locus: str, - preset: str, - tag: str, - ) -> Tuple[Job, Job]: - """Extract reads at the T1K locus and genotype them with T1K""" - self.required(self.reference, "reference") - - # Extract reads overlapping the locus to a BAM file with ReadWriter - extracted_bam = self.tmp_dir.joinpath(f"sample_{tag}.bam") - extract_result = ReadWriterStage( - ctx=self.stage_context(), - inputs=sr_alignments, - output=extracted_bam, - name=f"t1k-{tag}-extract", - task_name="t1k", - interval=locus, - ).build() - extract_job = extract_result.jobs[0] - - t1k_job = Job( - cmds.cmd_t1k( - out_basename, - extracted_bam, - gene_seq=gene_seq, - gene_coord=gene_coord, - preset=preset, - threads=self.cores, - ), - f"t1k-{tag}", - self.cores, - task_name="t1k", - ) - return (extract_job, t1k_job) - - def build_expansion_job( - self, - out_expansions: pathlib.Path, - sr_alignment: pathlib.Path, - expansion_catalog: pathlib.Path, - ) -> Job: - """Identify repeat expansions""" - reference = self.required(self.reference, "reference") - - return Job( - cmds.cmd_expansion_hunter( - out_expansions, - sr_alignment, - reference=reference, - variant_catalog=expansion_catalog, - sex="male" if self.sample_sex == SampleSex.MALE else "female", - threads=self.cores, - ), - "expansion-hunter", - self.cores, - task_name="expansion-hunter", - ) - - def _add_cnv_jobs( - self, - dag: DAG, - sv_vcf: pathlib.Path, - cnv_input_bams: List[pathlib.Path], - ) -> None: - """Add CNV calling jobs to the DAG""" - bundle = self.required(self.model_bundle, "model_bundle") - - ctx = self.stage_context() - cnvscope_vcf = self.tmp_dir.joinpath("sample-cnvscope.vcf.gz") - cnv_apply_vcf = self.tmp_dir.joinpath("sample-cnv_model_apply.vcf.gz") - indel2cnv_vcf = self.tmp_dir.joinpath("sample-sv_cnv.vcf.gz") - cnv_vcf = pathlib.Path( - str(ctx.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") - ) - cnv_probes = pathlib.Path( - str(ctx.output_vcf).replace(".vcf.gz", "_cnv.probes") - ) - - # CNVscope on BWA deduped BAM, then CNVModelApply. The alignment - # and the PangenomeSV output were both written by the first DAG. - cnv_result = CNVscopeStage( - ctx=ctx, - inputs=cnv_input_bams, - model=bundle.joinpath("cnv.model"), - cnvscope_vcf=cnvscope_vcf, - cnv_vcf=cnv_apply_vcf, - sample_sex=self.sample_sex, - par_bed=self.cnv_par_bed, - dump_probes=cnv_probes, - interval=self.bed, - ).add_to(dag) - - # Convert PangenomeSV output to CNVs - indel2cnv_job = self._build_indel2cnv_job(indel2cnv_vcf, sv_vcf) - dag.add_job(indel2cnv_job) - - # Combine CNVModelApply output with converted SVs - combine_job = self._build_combine_cnv_job( - cnv_vcf, cnv_apply_vcf, indel2cnv_vcf - ) - dag.add_job(combine_job, {cnv_result.apply_job, indel2cnv_job}) - - def _build_indel2cnv_job( - self, - output_vcf: pathlib.Path, - input_vcf: pathlib.Path, - ) -> Job: - """Convert PangenomeSV INDELs to CNV calls""" - indel2cnv_script = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("indel2cnv.py")) - ) - # Run in background - return Job( - Pipeline( - Command( - sys.executable, - str(indel2cnv_script), - str(self.reference), - str(input_vcf), - str(output_vcf), - "-t", - str(self.cores), - ) - ), - "indel2cnv", - 0, - task_name="cnv", - ) - - def _build_combine_cnv_job( - self, - output_vcf: pathlib.Path, - cnv_vcf: pathlib.Path, - converted_vcf: pathlib.Path, - ) -> Job: - """Combine CNVscope and converted SV calls""" - combine_script = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("combine_cnv.py")) - ) - return Job( - Pipeline( - Command( - sys.executable, - str(combine_script), - "--cnv", - str(cnv_vcf), - "--converted", - str(converted_vcf), - "-o", - str(output_vcf), - ) - ), - "combine-cnv", - 0, - task_name="cnv", - ) diff --git a/tests/unit/test_sentieon_pangenome.py b/tests/unit/test_sentieon_pangenome.py index a619f96..ca6e9fa 100644 --- a/tests/unit/test_sentieon_pangenome.py +++ b/tests/unit/test_sentieon_pangenome.py @@ -11,7 +11,9 @@ from unittest.mock import patch, MagicMock # Add the parent directory to the path to import sentieon_cli -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +) from sentieon_cli.sentieon_pangenome import SentieonPangenome from sentieon_cli.util import SampleSex @@ -53,10 +55,10 @@ def setup_method(self): def create_pipeline(self): """Create a SentieonPangenome pipeline for testing""" pipeline = SentieonPangenome() - + # Setup mocks pipeline.logger = MagicMock() - + # Configure arguments pipeline.output_vcf = self.mock_vcf pipeline.reference = self.mock_ref @@ -72,14 +74,14 @@ def create_pipeline(self): pipeline.skip_pangenome_name_checks = True pipeline.skip_pop_vcf_id_check = True pipeline.tmp_dir = self.mock_dir - + # Mock parsing fai pipeline.fai_data = {"chr1": {"length": 1000}} pipeline.shards = [MagicMock()] pipeline.shards[0].contig = "chr1" pipeline.shards[0].start = 1 pipeline.shards[0].stop = 1000 - + # Mock pop vcf contigs pipeline.pop_vcf_contigs = {"chr1": 1000} @@ -200,16 +202,16 @@ def test_skip_model_apply(self): assert concat_job is not None # Check that the first argument (output file) is set to the intermediate - args_str = " ".join([str(arg) for arg in concat_job.shell.nodes[0].args]) + args_str = " ".join( + [str(arg) for arg in concat_job.shell.nodes[0].args] + ) assert str(self.snv_apply_vcf(pipeline)) in args_str assert str(pipeline.output_vcf) not in args_str # The AD-update job writes the final output and depends # on the concat job _, all_jobs = self._get_all_job_names(dag) - ad_update_job = next( - j for j in all_jobs if j.name == "sad-lad-update" - ) + ad_update_job = next(j for j in all_jobs if j.name == "sad-lad-update") update_cmd = str(ad_update_job.shell) assert f"--input_vcf {self.snv_apply_vcf(pipeline)}" in update_cmd assert f"--output_vcf {pipeline.output_vcf}" in update_cmd @@ -247,9 +249,7 @@ def test_gvcf_adds_gvcftyper_and_routes_outputs(self): # The update job rewrites it to the final .g.vcf.gz apply_job = next(j for j in all_jobs if j.name == "model-apply") - ad_update_job = next( - j for j in all_jobs if j.name == "sad-lad-update" - ) + ad_update_job = next(j for j in all_jobs if j.name == "sad-lad-update") update_cmd = str(ad_update_job.shell) assert f"--input_vcf {apply_gvcf}" in update_cmd assert f"--output_vcf {expected_gvcf}" in update_cmd @@ -282,9 +282,7 @@ def test_ad_update_default(self): job_names, all_jobs = self._get_all_job_names(dag) assert job_names.count("sad-lad-update") == 1 - ad_update_job = next( - j for j in all_jobs if j.name == "sad-lad-update" - ) + ad_update_job = next(j for j in all_jobs if j.name == "sad-lad-update") cmd_str = str(ad_update_job.shell) assert "sad_lad_update.py" in cmd_str assert f"--input_vcf {self.snv_apply_vcf(pipeline)}" in cmd_str @@ -317,17 +315,24 @@ def test_no_gvcftyper_without_gvcf(self): job_names = [job.name for job in all_jobs] assert "gvcftyper" not in job_names + def _second_dag_job(self, pipeline, name, sample_sex=SampleSex.FEMALE): + """Build both DAGs and return the named job of the second. + + The first DAG has to be built first so that the short-read + alignments and the ploidy JSON are stashed for the second. + """ + pipeline.build_dag() + pipeline.sample_sex = sample_sex + dag = pipeline.build_second_dag() + _, all_jobs = self._get_all_job_names(dag) + return next(j for j in all_jobs if j.name == name) + def build_segdup_cmd(self, tech): """Build the segdup-caller command string for a given platform""" pipeline = self.create_pipeline() pipeline.tech = tech - pipeline.sample_sex = SampleSex.FEMALE - job = pipeline.build_segdup_job( - self.mock_dir / "output_segdups", - self.mock_bam, - self.mock_vcf, - None, - ) + pipeline.segdup_caller = [] + job = self._second_dag_job(pipeline, "segdup-caller") return str(job.shell) def test_segdup_ultima_lowers_min_map_qual(self): @@ -338,6 +343,180 @@ def test_segdup_no_override_for_short_reads(self): """Non-Ultima input leaves the segdup-caller defaults alone""" assert "--set" not in self.build_segdup_cmd("Illumina") + def test_segdup_command(self): + """The short-read segdup-caller command built by the second DAG""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = ["CFH", "CYP2D6", "SMN1"] + job = self._second_dag_job(pipeline, "segdup-caller") + + assert str(job.shell) == ( + f"segdup-caller --short {self.mock_bam} " + f"--reference {self.mock_ref} " + f"--sr_model {self.mock_bundle} " + f"--input_vcf {self.mock_vcf} " + "--sex female " + "--genes CFH,CYP2D6,SMN1 " + f"--outdir {self.mock_dir}/output_segdups" + ) + assert job.name == "segdup-caller" + assert job.task_name == "segdup" + assert job.threads == pipeline.cores + + def test_segdup_without_genes(self): + """An empty gene list runs segdup-caller's own default set""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + job = self._second_dag_job(pipeline, "segdup-caller") + + assert "--genes" not in str(job.shell) + + def test_segdup_male_sample(self): + """The estimated sex reaches segdup-caller""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + job = self._second_dag_job( + pipeline, "segdup-caller", sample_sex=SampleSex.MALE + ) + + assert "--sex male" in str(job.shell) + + def test_no_segdup_job_by_default(self): + """Without `--segdup_caller` there is no segdup job, or DAG 2""" + pipeline = self.create_pipeline() + pipeline.build_dag() + pipeline.sample_sex = SampleSex.FEMALE + + assert pipeline.build_second_dag() is None + + def test_expansion_hunter_command(self): + """The ExpansionHunter command built by the second DAG""" + catalog = self.mock_dir / "catalog.json" + catalog.touch() + + pipeline = self.create_pipeline() + pipeline.expansion_catalog = catalog + job = self._second_dag_job(pipeline, "expansion-hunter") + + assert str(job.shell) == ( + f"ExpansionHunter --reads {self.mock_bam} " + f"--reference {self.mock_ref} " + f"--variant-catalog {catalog} " + "--sex female " + f"--threads {pipeline.cores} " + f"--output-prefix {self.mock_dir}/output_expansion" + ) + assert job.name == "expansion-hunter" + assert job.task_name == "expansion-hunter" + assert job.threads == pipeline.cores + + def enable_t1k(self, pipeline): + """Supply the T1K HLA and KIR reference files""" + for name in ("hla_seq", "hla_coord", "kir_seq", "kir_coord"): + (self.mock_dir / f"{name}.fa").touch() + pipeline.t1k_hla_seq = self.mock_dir / "hla_seq.fa" + pipeline.t1k_hla_coord = self.mock_dir / "hla_coord.fa" + pipeline.t1k_kir_seq = self.mock_dir / "kir_seq.fa" + pipeline.t1k_kir_coord = self.mock_dir / "kir_coord.fa" + return pipeline + + def _get_deps(self, dag, job): + """The dependencies of a job in a DAG""" + return dag.waiting_jobs.get(job, set()) + + def _get_dep_names(self, dag, job): + """The names of a job's dependencies""" + return {dep.name for dep in self._get_deps(dag, job)} + + def test_t1k_jobs_in_the_first_dag(self): + """Both T1K gene groups add an extract and a genotyping job""" + pipeline = self.enable_t1k(self.create_pipeline()) + dag = pipeline.build_dag() + + job_names, all_jobs = self._get_all_job_names(dag) + for name in ( + "t1k-hla-extract", + "t1k-hla", + "t1k-kir-extract", + "t1k-kir", + ): + assert name in job_names + + for tag in ("hla", "kir"): + extract = next( + j for j in all_jobs if j.name == f"t1k-{tag}-extract" + ) + t1k = next(j for j in all_jobs if j.name == f"t1k-{tag}") + assert self._get_deps(dag, t1k) == {extract} + # With BAM/CRAM input the extract reads the user's alignment + assert self._get_deps(dag, extract) == set() + assert extract.task_name == "t1k" + assert t1k.task_name == "t1k" + + def test_t1k_extract_command(self): + """The HLA extract runs ReadWriter over the T1K locus""" + pipeline = self.enable_t1k(self.create_pipeline()) + dag = pipeline.build_dag() + + _, all_jobs = self._get_all_job_names(dag) + extract = next(j for j in all_jobs if j.name == "t1k-hla-extract") + assert str(extract.shell) == ( + f"sentieon driver --input {self.mock_bam} " + f"--reference {self.mock_ref} --thread_count {pipeline.cores} " + "--interval chr6:28510020-33480577 " + f"--algo ReadWriter {self.mock_dir}/sample_hla.bam" + ) + + def test_t1k_command(self): + """run-t1k reads the extracted BAM and writes next to the VCF""" + pipeline = self.enable_t1k(self.create_pipeline()) + dag = pipeline.build_dag() + + _, all_jobs = self._get_all_job_names(dag) + t1k = next(j for j in all_jobs if j.name == "t1k-kir") + assert str(t1k.shell) == ( + f"run-t1k --abnormalUnmapFlag -t {pipeline.cores} " + "--preset kir-wgs " + f"-f {self.mock_dir}/kir_seq.fa " + f"-c {self.mock_dir}/kir_coord.fa " + f"--od {self.mock_dir}/output_kir " + f"-b {self.mock_dir}/sample_kir.bam" + ) + + def test_t1k_extract_depends_on_dedup_with_fastq_input(self): + """With fastq input the extract waits for the bwa alignment""" + pipeline = self.enable_t1k(self.create_fastq_pipeline()) + dag = pipeline.build_dag() + + _, all_jobs = self._get_all_job_names(dag) + extract = next(j for j in all_jobs if j.name == "t1k-hla-extract") + assert self._get_dep_names(dag, extract) == {"dedup-bwa"} + assert str(self.mock_dir / "output_bwa_deduped.cram") in str( + extract.shell + ) + + def test_no_t1k_jobs_by_default(self): + """T1K runs only when its reference files are supplied""" + pipeline = self.create_pipeline() + job_names, _ = self._get_all_job_names(pipeline.build_dag()) + + assert not [name for name in job_names if name.startswith("t1k")] + + def test_sr_alignments_stashed_with_bam_input(self): + """The second DAG reads the short-read alignments off the pipeline""" + pipeline = self.create_pipeline() + pipeline.build_dag() + + assert pipeline.sr_alignments == [self.mock_bam] + + def test_sr_alignments_stashed_with_fastq_input(self): + """With fastq input it is the deduplicated bwa alignment""" + pipeline = self.create_fastq_pipeline() + pipeline.build_dag() + + assert pipeline.sr_alignments == [ + self.mock_dir / "output_bwa_deduped.cram" + ] + def test_call_svs(self): """Test that PangenomeSV is added when --call_svs is enabled""" pipeline = self.create_pipeline() @@ -347,9 +526,7 @@ def test_call_svs(self): assert isinstance(dag, DAG) # The dnascope job should exist - all_jobs = list(dag.waiting_jobs.keys()) + list( - dag.ready_jobs.keys() - ) + all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] assert "dnascope" in job_names @@ -367,9 +544,7 @@ def test_call_svs(self): assert "--gfa_file" in cmd_str # SV output should use _sv.vcf.gz suffix - sv_vcf = str(pipeline.output_vcf).replace( - ".vcf.gz", "_sv.vcf.gz" - ) + sv_vcf = str(pipeline.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") assert sv_vcf in cmd_str def test_call_svs_disabled_by_default(self): @@ -377,9 +552,7 @@ def test_call_svs_disabled_by_default(self): pipeline = self.create_pipeline() dag = pipeline.build_dag() - all_jobs = list(dag.waiting_jobs.keys()) + list( - dag.ready_jobs.keys() - ) + all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) dnascope_job = None for job in all_jobs: if job.name == "dnascope": @@ -398,9 +571,7 @@ def test_skip_small_variants(self): assert isinstance(dag, DAG) - all_jobs = list(dag.waiting_jobs.keys()) + list( - dag.ready_jobs.keys() - ) + all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] assert "dnascope" not in job_names assert "model-apply" not in job_names @@ -415,9 +586,7 @@ def test_skip_small_variants_with_call_svs(self): assert isinstance(dag, DAG) - all_jobs = list(dag.waiting_jobs.keys()) + list( - dag.ready_jobs.keys() - ) + all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] # The driver job should still run for SV calling @@ -440,9 +609,7 @@ def test_skip_small_variants_with_call_svs(self): def _get_all_job_names(self, dag): """Helper to get all job names from a DAG""" - all_jobs = list(dag.waiting_jobs.keys()) + list( - dag.ready_jobs.keys() - ) + all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) return [job.name for job in all_jobs], all_jobs def _build_cnv_dag(self, pipeline, sample_sex=SampleSex.FEMALE): @@ -566,9 +733,7 @@ def test_ploidy_estimation_runs_without_sex_aware_callers(self): pipeline = self.create_pipeline() assert pipeline._needs_second_dag() is False - job_names, all_jobs = self._get_all_job_names( - pipeline.build_dag() - ) + job_names, all_jobs = self._get_all_job_names(pipeline.build_dag()) assert "estimate-ploidy" in job_names ploidy_job = next(j for j in all_jobs if j.name == "estimate-ploidy") @@ -725,9 +890,7 @@ def test_combine_cnv_command(self): assert "--cnv" in cmd_str assert "--converted" in cmd_str # Output should use _cnv.vcf.gz suffix - cnv_vcf = str(pipeline.output_vcf).replace( - ".vcf.gz", "_cnv.vcf.gz" - ) + cnv_vcf = str(pipeline.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") assert cnv_vcf in cmd_str def test_cnv_with_skip_model_apply(self): @@ -735,9 +898,7 @@ def test_cnv_with_skip_model_apply(self): pipeline = self.create_pipeline() pipeline.call_svs = True pipeline.skip_model_apply = True - first_job_names, _ = self._get_all_job_names( - pipeline.build_dag() - ) + first_job_names, _ = self._get_all_job_names(pipeline.build_dag()) assert "model-apply" not in first_job_names pipeline.sample_sex = SampleSex.FEMALE @@ -748,4 +909,3 @@ def test_cnv_with_skip_model_apply(self): assert "cnv-model-apply" in job_names assert "indel2cnv" in job_names assert "combine-cnv" in job_names - From 179ba3f7fe4171dd139c5edaad31866f55d03216 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Tue, 8 Sep 2026 10:43:15 -0700 Subject: [PATCH 3/4] Validate the optional callers and estimate ploidy in hybrid-pangenome Give `hybrid-pangenome` the argument surface, validation, and first-DAG jobs of the optional callers that `dnascope-pangenome` already has. The new `--call_cnvs` flag opts into CNVscope calling, and the shared `--sample_sex`, `--par_bed`, `--segdup_caller`, `--expansion_catalog` and `--t1k_*` arguments come from `BasePangenome`. `validate_bundle` now records whether the bundle carries a 'cnv.model' file and requires `diploid_model` when `--segdup_caller` is set, since segdup-caller reads it from the bundle passed as its `--lr_model`. `validate()` runs `validate_segdup`, `validate_expansion`, `validate_t1k` and `validate_cnv` after the readgroup checks. segdup-caller and ExpansionHunter accept a single `--sr_aln` file (segdup-caller also a single `--lr_aln`), `--segdup_caller` cannot be combined with `--skip_small_variants`, and `--call_cnvs` requires SV calling and a bundle CNV model. CNV runs resolve the PAR BED file during validation and exit when none is available; a run without `--bed` warns that CNVscope will cover every contig. `--bed` now restricts CNV calling as well as small-variant calling. The first DAG always adds the `PloidyStage`, writing `_ploidy.json` whatever the enabled callers, and runs T1K HLA/KIR on the short reads. It also stashes the short-read alignments, their `--replace_rg` rows, and the single long-read alignment for the second DAG. `--call_cnvs`, `--segdup_caller` and `--expansion_catalog` are validated but build no second-DAG jobs yet; `build_second_dag` follows in the next commit. The DAG snapshots gain one `estimate-ploidy` row per hybrid-pangenome config and are otherwise identical. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/hybrid_pangenome.py | 147 +++++++++- tests/unit/test_hybrid_pangenome.py | 439 +++++++++++++++++++++++++++- 2 files changed, 580 insertions(+), 6 deletions(-) diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index 4f5f2f9..ce0a292 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -40,8 +40,12 @@ find_unzip, ) from .stages.base import StageContext +from .stages.cnv import CNV_MIN_VERSIONS from .stages.dedup import DedupStage +from .stages.expansion import EXPANSION_MIN_VERSIONS from .stages.metrics import MetricsPaths, MetricsStage +from .stages.ploidy import PloidyStage +from .stages.segdup import SEGDUP_MIN_VERSIONS from .stages.small_variants import ( ApplySpec, DNAscopeStage, @@ -119,11 +123,19 @@ class HybridPangenome(BasePangenome): "flags": ["-b", "--bed"], "help": ( "Region BED file. Supplying this file will limit " - "small-variant calling to the intervals inside the BED " - "file." + "small-variant and CNV calling to the intervals inside " + "the BED file." ), "type": path_arg(exists=True, is_file=True), }, + "call_cnvs": { + "help": ( + "Call copy number variants with CNVscope and the " + "PangenomeSV output. Requires a model bundle with a " + "'cnv.model' file." + ), + "action": "store_true", + }, "lr_align_input": { "help": ( "Align the `--lr_aln` input files to the linear " @@ -208,6 +220,7 @@ def __init__(self) -> None: self.pop_vcf: Optional[pathlib.Path] = None self.readgroup: Optional[str] = None self.bed: Optional[pathlib.Path] = None + self.call_cnvs = False self.lr_align_input = False self.lr_input_ref: Optional[pathlib.Path] = None self.pangenome_contig_prefix = "GRCh38#0#" @@ -228,6 +241,21 @@ def __init__(self) -> None: self.skip_pop_vcf_id_check = False self.skip_small_variants = False self.skip_svs = False + # The single long-read alignment, stashed by `build_dag` for the + # second, sex-aware DAG + self.lr_alignment: Optional[pathlib.Path] = None + + def _cnv_in_second_dag(self) -> bool: + """CNV calling runs in the second, sex-aware DAG""" + return self.call_cnvs and self.has_cnv_model + + def _needs_second_dag(self) -> bool: + """The run has jobs that depend on the estimated sample sex""" + return bool( + self.expansion_catalog + or self.segdup_caller is not None + or self._cnv_in_second_dag() + ) def validate(self) -> None: """Validate pipeline inputs""" @@ -270,6 +298,11 @@ def validate(self) -> None: self.collect_readgroups() self.validate_readgroups() + self.validate_segdup() + self.validate_expansion() + self.validate_t1k() + self.validate_cnv() + require_versions( HYBRID_PANGENOME_MIN_VERSIONS, skip=self.skip_version_check ) @@ -336,9 +369,86 @@ def validate(self) -> None: sys.exit(2) # After the version gate and the contig checks, so the tool choice - # is only logged for runs that proceed + # is only logged for runs that proceed. The `reference_build` it + # sets is the value `validate_cnv` already resolved. self.resolve_gfa2fa_tool() + def validate_segdup(self) -> None: + """Validate the arguments used by segdup-caller""" + if self.segdup_caller is None: + return + + if len(self.sr_aln) > 1: + self.logger.error( + "`--segdup_caller` accepts only a single `--sr_aln` " + "BAM/CRAM file." + ) + sys.exit(2) + + if len(self.lr_aln) > 1: + self.logger.error( + "`--segdup_caller` accepts only a single `--lr_aln` " + "BAM/CRAM file." + ) + sys.exit(2) + + if self.skip_small_variants: + self.logger.error( + "`--segdup_caller` requires the small-variant VCF as " + "`--input_vcf` and cannot be combined with " + "`--skip_small_variants`." + ) + sys.exit(2) + + require_versions(SEGDUP_MIN_VERSIONS, skip=self.skip_version_check) + + def validate_expansion(self) -> None: + """Validate the arguments used by ExpansionHunter""" + if self.expansion_catalog is None: + return + + if len(self.sr_aln) > 1: + self.logger.error( + "`--expansion_catalog` accepts only a single `--sr_aln` " + "BAM/CRAM file." + ) + sys.exit(2) + + require_versions(EXPANSION_MIN_VERSIONS, skip=self.skip_version_check) + + def validate_cnv(self) -> None: + """Validate the arguments used for sex-aware CNV calling""" + if self.call_cnvs and self.skip_svs: + self.logger.error( + "`--call_cnvs` requires SV calling and cannot be combined " + "with `--skip_svs`." + ) + sys.exit(2) + + # `validate_bundle` has already set `self.has_cnv_model` + if self.call_cnvs and not self.has_cnv_model: + self.logger.error( + "`--call_cnvs` requires a model bundle with a 'cnv.model' " + "file." + ) + sys.exit(2) + + if self.call_cnvs and self.bed is None: + self.logger.warning( + "No `--bed` supplied; CNVscope will run on every contig in " + "the reference, including decoy and unplaced contigs" + ) + + cnv_will_run = self._cnv_in_second_dag() + self.resolve_cnv_par_bed(self.fai_data, self.par_bed, cnv_will_run) + if not cnv_will_run: + return + + # A PAR BED file is required for CNV calling, whatever the sex + self.validate_cnv_par(True) + + require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) + def validate_sr_inputs(self) -> None: """Validate the short-read input arguments. @@ -425,6 +535,10 @@ def validate_bundle(self) -> None: required_members.add("bwa.model") if self.lr_align_input: required_members.add("minimap2_lr.model") + if self.segdup_caller is not None: + # segdup-caller reads `/diploid_model` and the + # pipeline passes this bundle as its `--lr_model` + required_members.add("diploid_model") missing_members = required_members - bundle_members if missing_members: self.logger.error( @@ -433,6 +547,8 @@ def validate_bundle(self) -> None: ) sys.exit(2) + self.has_cnv_model = "cnv.model" in bundle_members + bundle_vcf_id = bundle_info.get("SentieonVcfID") if ( bundle_vcf_id @@ -853,15 +969,40 @@ def build_dag(self) -> DAG: calling_bams = [out_bwa_aln, out_lift_aln] + calling_lr calling_dependencies = {bwa_dedup_job, lift_dedup_job} + sr_alignments = [out_bwa_aln] + sr_deps: Set[Job] = {bwa_dedup_job} else: # Aligned short reads are used as-is; duplicate, secondary, and # supplementary reads are excluded from the lifted alignment # during read extraction calling_bams = list(self.sr_aln) + [out_lift_aln] + calling_lr calling_dependencies = {mm2_job} + sr_alignments = list(self.sr_aln) + sr_deps = set() calling_dependencies |= realign_jobs replace_rg = self.build_replace_rg() + # Stash the short-read inputs for the second, sex-aware DAG. The + # `replace_rg` rows follow the calling input order, so the + # short-read rows are the leading ones; all-empty rows are dropped. + sr_replace_rg = replace_rg[: len(sr_alignments)] + self.sr_alignments = sr_alignments + self.sr_replace_rg = sr_replace_rg if any(sr_replace_rg) else None + self.lr_alignment = calling_lr[0] if len(calling_lr) == 1 else None + + # Estimate the sample ploidy and sex. The JSON output is always + # written; `--sample_sex` takes precedence for the sex used by the + # sex-aware callers. + ploidy_result = PloidyStage( + ctx=ctx, + inputs=[sr_alignments[0]], + reference_build=self.reference_build, + ).add_to(dag, sr_deps) + self.ploidy_json = ploidy_result.ploidy_json + + # T1K HLA/KIR calling from the short reads + self.add_t1k(dag, ctx, sr_alignments, sr_deps) + if not self.skip_svs: pangenomesv_job = self.build_pangenomesv_job( sv_vcf, diff --git a/tests/unit/test_hybrid_pangenome.py b/tests/unit/test_hybrid_pangenome.py index be2dd3f..ed7cd1b 100644 --- a/tests/unit/test_hybrid_pangenome.py +++ b/tests/unit/test_hybrid_pangenome.py @@ -160,6 +160,7 @@ def test_dag_jobs_present(self): "locuscollector-lift", "dedup-lift", "metrics", + "estimate-ploidy", "pangenome-sv", "dnascope", "model-apply", @@ -645,6 +646,7 @@ def test_aligned_dag_jobs(self): "vg-haplotypes", "graph-update", "mm2-lift", + "estimate-ploidy", "pangenome-sv", "dnascope", "model-apply", @@ -717,9 +719,7 @@ def test_aligned_dag_dependencies(self): "extract-kmc" } assert "extract-kmc" in self._get_dep_names(dag, all_jobs, "mm2-lift") - assert self._get_dep_names(dag, all_jobs, "dnascope") == { - "mm2-lift" - } + assert self._get_dep_names(dag, all_jobs, "dnascope") == {"mm2-lift"} def test_aligned_lift_output(self): """The lifted alignment is the final short-read output""" @@ -1035,3 +1035,436 @@ def test_collect_readgroups_parses_input_headers(self, monkeypatch): assert pipeline.lr_readgroups == [[{"ID": "lr-rg1", "SM": "sample1"}]] pipeline.validate_readgroups() assert pipeline.sample_sm == "sample1" + + # Ploidy estimation + + def test_estimate_ploidy_job_with_fastq_input(self): + """Ploidy is estimated from the deduplicated bwa alignment""" + pipeline = self.create_pipeline() + dag = pipeline.build_dag() + _, all_jobs = self._get_all_job_names(dag) + + bwa_aln = str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + job = self._get_job(all_jobs, "estimate-ploidy") + assert "estimate_ploidy.py" in str(job.shell) + assert f"-i {bwa_aln}" in str(job.shell) + assert job.task_name == "ploidy" + assert job.threads == 0 + assert self._get_dep_names(dag, all_jobs, "estimate-ploidy") == { + "dedup-bwa" + } + + def test_estimate_ploidy_job_with_aligned_input(self): + """Aligned short reads are used as-is, without dependencies""" + pipeline = self.create_aligned_pipeline() + dag = pipeline.build_dag() + _, all_jobs = self._get_all_job_names(dag) + + job = self._get_job(all_jobs, "estimate-ploidy") + assert f"-i {self.mock_sr_bam}" in str(job.shell) + assert self._get_dep_names(dag, all_jobs, "estimate-ploidy") == set() + + def test_ploidy_json_stashed(self): + """The ploidy JSON path is stashed for the second DAG""" + pipeline = self.create_pipeline() + pipeline.build_dag() + + assert pipeline.ploidy_json == pathlib.Path( + str(self.mock_vcf).replace(".vcf.gz", "_ploidy.json") + ) + + def test_estimate_ploidy_with_skip_small_variants(self): + """Ploidy estimation runs before the small-variant early return""" + pipeline = self.create_pipeline() + pipeline.skip_small_variants = True + job_names, _ = self._get_all_job_names(pipeline.build_dag()) + + assert "estimate-ploidy" in job_names + + # T1K HLA/KIR genotyping + + def enable_t1k(self, pipeline): + """Supply the T1K reference files""" + for attr, name in ( + ("t1k_hla_seq", "hla_seq.fa"), + ("t1k_hla_coord", "hla_coord.fa"), + ("t1k_kir_seq", "kir_seq.fa"), + ("t1k_kir_coord", "kir_coord.fa"), + ): + path = self.mock_dir / name + path.touch() + setattr(pipeline, attr, path) + return pipeline + + def test_no_t1k_jobs_by_default(self): + """T1K runs only when its reference files are supplied""" + pipeline = self.create_pipeline() + job_names, _ = self._get_all_job_names(pipeline.build_dag()) + + assert not [name for name in job_names if name.startswith("t1k")] + + def test_t1k_jobs_with_fastq_input(self): + """T1K genotypes the deduplicated short reads""" + pipeline = self.enable_t1k(self.create_pipeline()) + dag = pipeline.build_dag() + job_names, all_jobs = self._get_all_job_names(dag) + + for name in ( + "t1k-hla-extract", + "t1k-hla", + "t1k-kir-extract", + "t1k-kir", + ): + assert name in job_names, f"missing job: {name}" + + bwa_aln = str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + extract = self._get_job(all_jobs, "t1k-hla-extract") + assert str(extract.shell) == ( + f"sentieon driver --input {bwa_aln} " + f"--reference {self.mock_ref} --thread_count {pipeline.cores} " + "--interval chr6:28510020-33480577 " + f"--algo ReadWriter {self.mock_dir}/sample_hla.bam" + ) + assert str(self._get_job(all_jobs, "t1k-hla").shell) == ( + f"run-t1k --abnormalUnmapFlag -t {pipeline.cores} " + "--preset hla-wgs " + f"-f {self.mock_dir}/hla_seq.fa " + f"-c {self.mock_dir}/hla_coord.fa " + f"--od {self.mock_dir}/output_hla " + f"-b {self.mock_dir}/sample_hla.bam" + ) + assert str(self._get_job(all_jobs, "t1k-kir").shell) == ( + f"run-t1k --abnormalUnmapFlag -t {pipeline.cores} " + "--preset kir-wgs " + f"-f {self.mock_dir}/kir_seq.fa " + f"-c {self.mock_dir}/kir_coord.fa " + f"--od {self.mock_dir}/output_kir " + f"-b {self.mock_dir}/sample_kir.bam" + ) + + assert self._get_dep_names(dag, all_jobs, "t1k-hla") == { + "t1k-hla-extract" + } + assert self._get_dep_names(dag, all_jobs, "t1k-hla-extract") == { + "dedup-bwa" + } + + def test_t1k_jobs_with_aligned_input(self): + """T1K extracts from the `--sr_aln` input without dependencies""" + pipeline = self.enable_t1k(self.create_aligned_pipeline()) + dag = pipeline.build_dag() + _, all_jobs = self._get_all_job_names(dag) + + extract = self._get_job(all_jobs, "t1k-kir-extract") + assert str(extract.shell) == ( + f"sentieon driver --input {self.mock_sr_bam} " + f"--reference {self.mock_ref} --thread_count {pipeline.cores} " + "--interval chr19:53100000-55800000 " + f"--algo ReadWriter {self.mock_dir}/sample_kir.bam" + ) + assert self._get_dep_names(dag, all_jobs, "t1k-kir-extract") == set() + assert self._get_dep_names(dag, all_jobs, "t1k-kir") == { + "t1k-kir-extract" + } + + # The second-DAG stashes + + def test_stashes_with_fastq_input(self): + """The bwa alignment is stashed without a `--replace_rg` row""" + pipeline = self.create_pipeline() + pipeline.build_dag() + + bwa_aln = pathlib.Path( + str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + ) + assert pipeline.sr_alignments == [bwa_aln] + assert pipeline.sr_replace_rg is None + assert pipeline.lr_alignment == self.mock_lr_bam + + def test_stashes_with_aligned_input(self): + """The `--sr_aln` input is stashed with its `--replace_rg` row""" + pipeline = self.create_aligned_pipeline() + pipeline.build_dag() + + assert pipeline.sr_alignments == [self.mock_sr_bam] + assert pipeline.sr_replace_rg == [ + [pipeline._replace_rg_arg({"ID": "sr-rg1", "SM": "sample1"}, "0")] + ] + assert pipeline.sr_replace_rg == [ + [r"sr-rg1=ID:sr-rg1\tSM:sample1\tLR:0"] + ] + + def test_lr_alignment_stash_with_realignment(self): + """The realigned long-read alignment is stashed""" + pipeline = self.create_lr_realign_pipeline() + pipeline.build_dag() + + assert pipeline.lr_alignment == pathlib.Path( + str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") + ) + + def test_lr_alignment_stash_with_two_inputs(self): + """segdup-caller takes a single long-read alignment""" + lr_bam2 = self.mock_dir / "longreads2.bam" + lr_bam2.touch() + + pipeline = self.create_pipeline() + pipeline.lr_aln = [self.mock_lr_bam, lr_bam2] + pipeline.lr_readgroups = [ + [{"ID": "lr-rg1", "SM": "sample1"}], + [{"ID": "lr-rg2", "SM": "sample1"}], + ] + pipeline.build_dag() + + assert pipeline.lr_alignment is None + + def test_build_second_dag_not_implemented_yet(self): + """The sex-aware callers do not build jobs yet""" + pipeline = self.create_pipeline() + pipeline.build_dag() + + assert pipeline.build_second_dag() is None + + # Second-DAG gates + + @pytest.mark.parametrize( + "call_cnvs,has_cnv_model,expected", + [ + (False, False, False), + (False, True, False), + (True, False, False), + (True, True, True), + ], + ) + def test_cnv_in_second_dag(self, call_cnvs, has_cnv_model, expected): + """CNVs are called with `--call_cnvs` and a bundle CNV model""" + pipeline = self.create_pipeline() + pipeline.call_cnvs = call_cnvs + pipeline.has_cnv_model = has_cnv_model + + assert pipeline._cnv_in_second_dag() is expected + + def test_needs_second_dag(self): + """Each sex-aware caller requires the second DAG""" + pipeline = self.create_pipeline() + assert pipeline._needs_second_dag() is False + + pipeline = self.create_pipeline() + pipeline.expansion_catalog = self.mock_bed + assert pipeline._needs_second_dag() is True + + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + assert pipeline._needs_second_dag() is True + + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + assert pipeline._needs_second_dag() is True + + # `--call_cnvs` without a CNV model is rejected during validation + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + assert pipeline._needs_second_dag() is False + + # Optional caller validation + + def test_validate_segdup_single_input(self): + """One short-read and one long-read alignment are accepted""" + pipeline = self.create_aligned_pipeline() + pipeline.segdup_caller = [] + + pipeline.validate_segdup() # no SystemExit + + def test_validate_segdup_two_sr_inputs(self): + """segdup-caller takes a single `--sr_aln` file""" + pipeline = self.create_aligned_pipeline() + pipeline.segdup_caller = [] + pipeline.sr_aln = [self.mock_sr_bam, self.mock_sr_bam] + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_segdup() + assert excinfo.value.code == 2 + + def test_validate_segdup_two_lr_inputs(self): + """segdup-caller takes a single `--lr_aln` file""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = ["CFH"] + pipeline.lr_aln = [self.mock_lr_bam, self.mock_lr_bam] + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_segdup() + assert excinfo.value.code == 2 + + def test_validate_segdup_skip_small_variants(self): + """segdup-caller reads the small-variant VCF""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + pipeline.skip_small_variants = True + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_segdup() + assert excinfo.value.code == 2 + + def test_validate_segdup_without_the_argument(self): + """Nothing is validated without `--segdup_caller`""" + pipeline = self.create_pipeline() + pipeline.skip_small_variants = True + + pipeline.validate_segdup() # no SystemExit + + def test_validate_expansion_two_sr_inputs(self): + """ExpansionHunter takes a single `--sr_aln` file""" + pipeline = self.create_aligned_pipeline() + pipeline.expansion_catalog = self.mock_bed + pipeline.sr_aln = [self.mock_sr_bam, self.mock_sr_bam] + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_expansion() + assert excinfo.value.code == 2 + + def test_validate_expansion_single_input(self): + """A single short-read alignment is accepted""" + pipeline = self.create_aligned_pipeline() + pipeline.expansion_catalog = self.mock_bed + + pipeline.validate_expansion() # no SystemExit + + def test_validate_cnv_requires_sv_calling(self): + """indel2cnv reads the PangenomeSV output""" + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + pipeline.skip_svs = True + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_cnv() + assert excinfo.value.code == 2 + + def test_validate_cnv_requires_a_cnv_model(self): + """`--call_cnvs` needs a bundle with a 'cnv.model' file""" + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = False + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_cnv() + assert excinfo.value.code == 2 + + def test_validate_cnv_requires_a_par_bed(self): + # The mock reference is not a recognized build, so no packaged + # PAR BED file can be selected and validation stops the run + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_cnv() + assert excinfo.value.code == 2 + + def test_validate_cnv_accepts_the_par_bed_argument(self): + par_bed = self.mock_dir / "par.bed" + par_bed.write_text("chrX\t10000\t2781479\n") + + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + pipeline.par_bed = par_bed + + pipeline.validate_cnv() # no SystemExit + + assert pipeline.cnv_par_bed == par_bed + + def test_validate_cnv_needs_no_par_bed_without_call_cnvs(self): + pipeline = self.create_pipeline() + pipeline.has_cnv_model = True + + pipeline.validate_cnv() # no SystemExit + + assert pipeline.cnv_par_bed is None + + def test_validate_cnv_warns_without_a_bed(self): + """CNVscope runs across every contig without `--bed`""" + par_bed = self.mock_dir / "par.bed" + par_bed.write_text("chrX\t10000\t2781479\n") + + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + pipeline.par_bed = par_bed + pipeline.bed = None + + pipeline.validate_cnv() + + assert any( + "No `--bed` supplied" in str(call.args[0]) + for call in pipeline.logger.warning.call_args_list + ) + + def test_validate_cnv_does_not_warn_with_a_bed(self): + par_bed = self.mock_dir / "par.bed" + par_bed.write_text("chrX\t10000\t2781479\n") + + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + pipeline.par_bed = par_bed + + pipeline.validate_cnv() + + pipeline.logger.warning.assert_not_called() + + # Model bundle validation + + def bundle_pipeline(self, monkeypatch, members): + """A pipeline whose bundle holds `members`""" + pipeline = self.create_pipeline() + pipeline.skip_pop_vcf_id_check = True + + def fake_ar_load(path): + if str(path).endswith("bundle_info.json"): + return b'{"pipeline": "Hybrid pangenome"}' + return list(members) + + monkeypatch.setattr(hybrid_pangenome, "ar_load", fake_ar_load) + return pipeline + + BUNDLE_MEMBERS = [ + "dnascope.model", + "longreadsv.model", + "minimap2.model", + "extract.model", + "bwa.model", + ] + + def test_validate_bundle_sets_has_cnv_model(self, monkeypatch): + pipeline = self.bundle_pipeline( + monkeypatch, self.BUNDLE_MEMBERS + ["cnv.model"] + ) + pipeline.validate_bundle() + assert pipeline.has_cnv_model is True + + pipeline = self.bundle_pipeline(monkeypatch, self.BUNDLE_MEMBERS) + pipeline.validate_bundle() + assert pipeline.has_cnv_model is False + + def test_validate_bundle_requires_the_diploid_model(self, monkeypatch): + """segdup-caller reads the bundle's `diploid_model`""" + pipeline = self.bundle_pipeline(monkeypatch, self.BUNDLE_MEMBERS) + pipeline.segdup_caller = [] + + with pytest.raises(SystemExit) as excinfo: + pipeline.validate_bundle() + assert excinfo.value.code == 2 + + pipeline = self.bundle_pipeline( + monkeypatch, self.BUNDLE_MEMBERS + ["diploid_model"] + ) + pipeline.segdup_caller = [] + pipeline.validate_bundle() # no SystemExit + + def test_validate_bundle_diploid_model_not_required_by_default( + self, monkeypatch + ): + pipeline = self.bundle_pipeline(monkeypatch, self.BUNDLE_MEMBERS) + pipeline.validate_bundle() # no SystemExit From 3ff98a9c83fb393a3247b7ecc853794b0e0bd4ec Mon Sep 17 00:00:00 2001 From: Don Freed Date: Tue, 8 Sep 2026 10:49:12 -0700 Subject: [PATCH 4/4] Call CNVs, repeat expansions and segdups in the hybrid-pangenome second DAG `HybridPangenome.build_second_dag` builds the sex-aware callers once ploidy estimation has run, using the short-read alignments, the sample sex and the long-read alignment stashed by `build_dag`. With `--call_cnvs` the DAG holds the `PangenomeCNVStage` jobs: CNVscope over the short reads only -- restricted by `--bed` and carrying the short-read `--replace_rg` rows for `--sr_aln` input -- CNVModelApply, `indel2cnv.py` over the PangenomeSV VCF, and `combine_cnv.py` writing `_cnv.vcf.gz`. `--expansion_catalog` adds ExpansionHunter over the short reads. `--segdup_caller` adds segdup-caller with both data types (`--short` and `--long`) and this pipeline's model bundle as both `--sr_model` and `--lr_model`, since it ships the `diploid_model`. The DAG-snapshot harness gains seven hybrid-pangenome configs for the new callers; every pre-existing config is unchanged. Co-Authored-By: Claude Fable 5.1 --- sentieon_cli/hybrid_pangenome.py | 39 ++++ tests/unit/test_hybrid_pangenome.py | 326 +++++++++++++++++++++++++++- 2 files changed, 361 insertions(+), 4 deletions(-) diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index ce0a292..3c556ea 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -1051,6 +1051,45 @@ def build_dag(self) -> DAG: return dag + def build_second_dag(self) -> Optional[DAG]: + """Build the second DAG for sex-aware downstream tools""" + if not self._needs_second_dag(): + return None + + assert self.ploidy_json is not None + self.get_sex(self.ploidy_json) + + self.logger.info("Building the second hybrid-pangenome DAG") + dag = DAG() + ctx = self.stage_context() + + # CNV calling with CNVscope on the short reads, using the sample sex + if self._cnv_in_second_dag(): + self.add_pangenome_cnv( + dag, + ctx, + self.output_path("_sv.vcf.gz"), + self.sr_alignments, + replace_rg=self.sr_replace_rg, + interval=self.bed, + ) + + if self.sr_alignments: + self.add_expansion(dag, ctx, self.sr_alignments[0]) + + # segdup-caller consumes the small-variant VCF, the sample sex, + # and the long reads. The bundle serves as both the short- and + # long-read model, as it ships the `diploid_model`. + self.add_segdup( + dag, + ctx, + self.sr_alignments[0], + lr_alignment=self.lr_alignment, + lr_bundle=self.model_bundle if self.lr_alignment else None, + ) + + return dag + def lr_kmc_pairs(self) -> List[Tuple[pathlib.Path, pathlib.Path]]: """`(alignment, decode reference)` pairs for the long-read k-mer counting. diff --git a/tests/unit/test_hybrid_pangenome.py b/tests/unit/test_hybrid_pangenome.py index ed7cd1b..29d3ad8 100644 --- a/tests/unit/test_hybrid_pangenome.py +++ b/tests/unit/test_hybrid_pangenome.py @@ -2,10 +2,13 @@ Unit tests for the HybridPangenome pipeline logic """ +from importlib.resources import files +import logging import os import pathlib import sys import tempfile +from typing import List from unittest.mock import MagicMock import packaging.version @@ -21,6 +24,32 @@ from sentieon_cli.hybrid_pangenome import HybridPangenome from sentieon_cli.command_strings import LONGREAD_SV_BED_AWK from sentieon_cli.dag import DAG +from sentieon_cli.util import SampleSex + +PACKAGE_LOGGER = "sentieon_cli" + + +class _RecordingHandler(logging.Handler): + """Collects formatted messages from the package logger.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages: List[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def messages(): + """`caplog` does not see the package logger, which does not propagate""" + logger = logging.getLogger(PACKAGE_LOGGER) + handler = _RecordingHandler() + logger.addHandler(handler) + try: + yield handler.messages + finally: + logger.removeHandler(handler) class TestHybridPangenome: @@ -1218,12 +1247,301 @@ def test_lr_alignment_stash_with_two_inputs(self): assert pipeline.lr_alignment is None - def test_build_second_dag_not_implemented_yet(self): - """The sex-aware callers do not build jobs yet""" - pipeline = self.create_pipeline() + # The second, sex-aware DAG + + def _build_second_dag(self, pipeline, sample_sex=SampleSex.FEMALE): + """Build both DAGs and return the second. + + The first DAG has to be built first so that the short-read + alignments and the ploidy JSON are stashed for the second. + """ pipeline.build_dag() + pipeline.sample_sex = sample_sex + return pipeline.build_second_dag() + + def _second_dag_job(self, pipeline, name, sample_sex=SampleSex.FEMALE): + """Build both DAGs and return the named job of the second""" + dag = self._build_second_dag(pipeline, sample_sex) + _, all_jobs = self._get_all_job_names(dag) + return self._get_job(all_jobs, name) + + def enable_cnv(self, pipeline): + """Call CNVs with a bundle CNV model and a PAR BED file""" + par_bed = self.mock_dir / "par.bed" + par_bed.write_text("chrX\t10000\t2781479\n") + pipeline.call_cnvs = True + pipeline.has_cnv_model = True + pipeline.cnv_par_bed = par_bed + return pipeline + + def script_path(self, name): + """A path to a packaged helper script, as the stages build it""" + return pathlib.Path(str(files("sentieon_cli.scripts").joinpath(name))) + + CNV_JOBS = ("cnvscope", "cnv-model-apply", "indel2cnv", "combine-cnv") + + def test_no_second_dag_without_callers(self): + """No second DAG when nothing consumes the sample sex""" + pipeline = self.create_pipeline() + + assert self._build_second_dag(pipeline) is None + + def test_no_second_dag_without_a_cnv_model(self): + """`--call_cnvs` without a bundle CNV model is rejected earlier""" + pipeline = self.create_pipeline() + pipeline.call_cnvs = True + pipeline.has_cnv_model = False + + assert self._build_second_dag(pipeline) is None + + def test_second_dag_for_each_caller(self): + """Every sex-aware caller on its own builds the second DAG""" + pipeline = self.create_pipeline() + pipeline.expansion_catalog = self.mock_bed + job_names, _ = self._get_all_job_names( + self._build_second_dag(pipeline) + ) + assert job_names == ["expansion-hunter"] + + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + job_names, _ = self._get_all_job_names( + self._build_second_dag(pipeline) + ) + assert job_names == ["segdup-caller"] + + pipeline = self.enable_cnv(self.create_pipeline()) + job_names, _ = self._get_all_job_names( + self._build_second_dag(pipeline) + ) + assert sorted(job_names) == sorted(self.CNV_JOBS) + + def test_cnv_jobs_in_the_second_dag(self): + """The CNV jobs and their dependencies""" + pipeline = self.enable_cnv(self.create_pipeline()) + first_dag = pipeline.build_dag() + + # CNV calling is sex-aware, so it is not in the first DAG + first_job_names, _ = self._get_all_job_names(first_dag) + for name in self.CNV_JOBS: + assert name not in first_job_names + + pipeline.sample_sex = SampleSex.FEMALE + dag = pipeline.build_second_dag() + job_names, all_jobs = self._get_all_job_names(dag) + assert sorted(job_names) == sorted(self.CNV_JOBS) + + assert self._get_dep_names(dag, all_jobs, "cnvscope") == set() + assert self._get_dep_names(dag, all_jobs, "indel2cnv") == set() + assert self._get_dep_names(dag, all_jobs, "cnv-model-apply") == { + "cnvscope" + } + assert self._get_dep_names(dag, all_jobs, "combine-cnv") == { + "cnv-model-apply", + "indel2cnv", + } + + for name in self.CNV_JOBS: + job = self._get_job(all_jobs, name) + assert job.task_name == "cnv", name + for name in ("cnvscope", "cnv-model-apply"): + assert self._get_job(all_jobs, name).threads == pipeline.cores + for name in ("indel2cnv", "combine-cnv"): + assert self._get_job(all_jobs, name).threads == 0 + + def test_cnvscope_command_with_fastq_input(self): + """CNVscope runs on the deduplicated short reads""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job(pipeline, "cnvscope") + + bwa_aln = str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + probes = str(self.mock_vcf).replace(".vcf.gz", "_cnv.probes") + assert str(job.shell) == ( + f"sentieon driver --input {bwa_aln} " + f"--reference {self.mock_ref} --thread_count {pipeline.cores} " + f"--interval {self.mock_bed} " + f"--algo CNVscope --model {self.mock_bundle}/cnv.model " + "--sex F " + f"--dump_probes {probes} " + f"{self.mock_dir}/sample-cnvscope.vcf.gz" + ) + # The pipeline-generated alignment carries the LR attribute already + assert "--replace_rg" not in str(job.shell) + + def test_cnvscope_command_with_aligned_input(self): + """The `--sr_aln` input is used with its `--replace_rg` row""" + pipeline = self.enable_cnv(self.create_aligned_pipeline()) + job = self._second_dag_job(pipeline, "cnvscope") + + probes = str(self.mock_vcf).replace(".vcf.gz", "_cnv.probes") + assert str(job.shell) == ( + "sentieon driver " + r"--replace_rg 'sr-rg1=ID:sr-rg1\tSM:sample1\tLR:0' " + f"--input {self.mock_sr_bam} " + f"--reference {self.mock_ref} --thread_count {pipeline.cores} " + f"--interval {self.mock_bed} " + f"--algo CNVscope --model {self.mock_bundle}/cnv.model " + "--sex F " + f"--dump_probes {probes} " + f"{self.mock_dir}/sample-cnvscope.vcf.gz" + ) + + def test_cnvscope_without_a_bed(self): + """Without `--bed`, CNVscope runs across the whole reference""" + pipeline = self.enable_cnv(self.create_pipeline()) + pipeline.bed = None + job = self._second_dag_job(pipeline, "cnvscope") + + assert "--interval" not in str(job.shell) + + def test_cnv_model_apply_command(self): + """CNVModelApply filters the CNVscope output""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job(pipeline, "cnv-model-apply") + + assert str(job.shell) == ( + f"sentieon driver --reference {self.mock_ref} " + f"--thread_count {pipeline.cores} " + f"--algo CNVModelApply --model {self.mock_bundle}/cnv.model " + f"--vcf {self.mock_dir}/sample-cnvscope.vcf.gz " + f"{self.mock_dir}/sample-cnv_model_apply.vcf.gz" + ) + # The BED restricts CNVscope, not the model apply + assert "--interval" not in str(job.shell) + + def test_cnvscope_male_sample(self): + """A male sample is called with the PAR BED file""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job( + pipeline, "cnvscope", sample_sex=SampleSex.MALE + ) + + assert "--sex M" in str(job.shell) + assert f"--par {pipeline.cnv_par_bed}" in str(job.shell) + + def test_cnvscope_female_sample(self): + """A female sample does not need the PAR regions""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job( + pipeline, "cnvscope", sample_sex=SampleSex.FEMALE + ) + + assert "--sex F" in str(job.shell) + assert "--par" not in str(job.shell) + + def test_cnvscope_unknown_sex(self, messages): + """An unknown sex calls a diploid genome, with a warning""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job( + pipeline, "cnvscope", sample_sex=SampleSex.UNKNOWN + ) + + assert "--sex" not in str(job.shell) + assert "--par" not in str(job.shell) + assert any("diploid" in msg for msg in messages) + + def test_indel2cnv_command(self): + """The PangenomeSV output is converted to CNVs""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job(pipeline, "indel2cnv") + + sv_vcf = str(self.mock_vcf).replace(".vcf.gz", "_sv.vcf.gz") + assert str(job.shell) == ( + f"{sys.executable} {self.script_path('indel2cnv.py')} " + f"{self.mock_ref} {sv_vcf} " + f"{self.mock_dir}/sample-sv_cnv.vcf.gz -t {pipeline.cores}" + ) + + def test_combine_cnv_command(self): + """The CNV and converted SV calls are combined""" + pipeline = self.enable_cnv(self.create_pipeline()) + job = self._second_dag_job(pipeline, "combine-cnv") + + cnv_vcf = str(self.mock_vcf).replace(".vcf.gz", "_cnv.vcf.gz") + assert str(job.shell) == ( + f"{sys.executable} {self.script_path('combine_cnv.py')} " + f"--cnv {self.mock_dir}/sample-cnv_model_apply.vcf.gz " + f"--converted {self.mock_dir}/sample-sv_cnv.vcf.gz " + f"-o {cnv_vcf}" + ) + + def test_expansion_hunter_command(self): + """ExpansionHunter genotypes the short reads""" + catalog = self.mock_dir / "catalog.json" + catalog.touch() + + pipeline = self.create_pipeline() + pipeline.expansion_catalog = catalog + dag = self._build_second_dag(pipeline) + _, all_jobs = self._get_all_job_names(dag) + job = self._get_job(all_jobs, "expansion-hunter") + + bwa_aln = str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + assert str(job.shell) == ( + f"ExpansionHunter --reads {bwa_aln} " + f"--reference {self.mock_ref} " + f"--variant-catalog {catalog} " + "--sex female " + f"--threads {pipeline.cores} " + f"--output-prefix {self.mock_dir}/output_expansion" + ) + assert job.task_name == "expansion-hunter" + assert job.threads == pipeline.cores + assert self._get_dep_names(dag, all_jobs, "expansion-hunter") == set() + + def test_segdup_command_with_fastq_input(self): + """segdup-caller uses the short and long reads, and one bundle""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = ["CFH", "CYP2D6", "SMN1"] + first_job_names, _ = self._get_all_job_names(pipeline.build_dag()) + assert "segdup-caller" not in first_job_names + + pipeline.sample_sex = SampleSex.MALE + dag = pipeline.build_second_dag() + _, all_jobs = self._get_all_job_names(dag) + job = self._get_job(all_jobs, "segdup-caller") + + bwa_aln = str(self.mock_vcf).replace(".vcf.gz", "_bwa_deduped.cram") + assert str(job.shell) == ( + f"segdup-caller --short {bwa_aln} " + f"--long {self.mock_lr_bam} " + f"--reference {self.mock_ref} " + f"--sr_model {self.mock_bundle} " + f"--lr_model {self.mock_bundle} " + f"--input_vcf {self.mock_vcf} " + "--sex male " + "--genes CFH,CYP2D6,SMN1 " + f"--outdir {self.mock_dir}/output_segdups" + ) + assert job.task_name == "segdup" + assert job.threads == pipeline.cores + assert self._get_dep_names(dag, all_jobs, "segdup-caller") == set() + + def test_segdup_without_genes(self): + """An empty gene list runs segdup-caller's own default set""" + pipeline = self.create_pipeline() + pipeline.segdup_caller = [] + job = self._second_dag_job(pipeline, "segdup-caller") + + assert "--genes" not in str(job.shell) + assert "--sex female" in str(job.shell) + + def test_segdup_command_with_aligned_input(self): + """The `--sr_aln` input is passed to segdup-caller as-is""" + pipeline = self.create_aligned_pipeline() + pipeline.segdup_caller = [] + job = self._second_dag_job(pipeline, "segdup-caller") + + assert f"--short {self.mock_sr_bam} " in str(job.shell) + + def test_segdup_command_with_realigned_long_reads(self): + """The realigned long-read alignment reaches segdup-caller""" + pipeline = self.create_lr_realign_pipeline() + pipeline.segdup_caller = [] + job = self._second_dag_job(pipeline, "segdup-caller") - assert pipeline.build_second_dag() is None + lr_aln = str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") + assert f"--long {lr_aln} " in str(job.shell) # Second-DAG gates