diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1cee559..0a9ddac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -308,6 +308,7 @@ jobs: --sr_aln "tests/smoke/illumina.cram" \ --lr_aln "tests/smoke/sam ple.cram" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ --sr_duplicate_marking none \ "output_hybrid1.vcf.gz" if [ ! -f "output_hybrid1.vcf.gz" ]; then @@ -326,6 +327,7 @@ jobs: --lr_align_input \ --lr_input_ref "tests/smoke/r ef.fa" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ "output_hybrid2.vcf.gz" if [ ! -f "output_hybrid2.vcf.gz" ]; then exit 1 @@ -343,6 +345,7 @@ jobs: --sr_aln "tests/smoke/illumina.cram" "tests/smoke/illumina_2.bam" \ --lr_aln "tests/smoke/sam ple.cram" \ -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --skip_cnv \ --sr_duplicate_marking none \ "output_hybrid3.vcf.gz" if [ ! -f "output_hybrid3.vcf.gz" ]; then diff --git a/sentieon_cli/base_pangenome.py b/sentieon_cli/base_pangenome.py index c617c48..f21e9a6 100644 --- a/sentieon_cli/base_pangenome.py +++ b/sentieon_cli/base_pangenome.py @@ -4,28 +4,23 @@ import copy import pathlib -import sys -from typing import List, Optional, Tuple - -from importlib.resources import files +from typing import List, Optional from . import command_strings as cmds from .driver import ( AlignmentStat, + BaseAlgo, BaseDistributionByCycle, CoverageMetrics, - Dedup, - Driver, GCBias, InsertSizeMetricAlgo, - LocusCollector, MeanQualityByCycle, QualDistribution, WgsMetricsAlgo, ) from .job import Job from .pipeline import BasePipeline -from .shell_pipeline import Command, Pipeline +from .stages.metrics import MetricsPaths from .util import path_arg @@ -141,130 +136,15 @@ def build_kmc_job( return kmc_job - def build_dedup_job( - self, - output_bam, - input_bam: List[pathlib.Path], - tag: str, - left_align_rgid: Optional[str] = None, - metrics: Optional[pathlib.Path] = None, - ) -> Tuple[Job, Job]: - """Build deduplication job""" - score_file = self.tmp_dir.joinpath(f"sample-{tag}-score.txt.gz") - - read_filters = [] - if left_align_rgid: - read_filters.append( - f"IndelLeftAlignReadTransform,rgid={left_align_rgid}" - ) - - # LocusCollector + Dedup - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bam, - read_filter=read_filters, - ) - driver.add_algo(LocusCollector(score_file)) - - lc_job = Job( - Pipeline(Command(*driver.build_cmd())), - f"locuscollector-{tag}", - self.cores, - task_name="dedup", - ) - - driver2 = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bam, - read_filter=read_filters, - ) - driver2.add_algo(Dedup(output_bam, score_file, metrics=metrics)) - - dedup_job = Job( - Pipeline(Command(*driver2.build_cmd())), - f"dedup-{tag}", - self.cores, - task_name="dedup", - ) - - return lc_job, dedup_job - - def build_metrics_job( - self, - sample_input: List[pathlib.Path], - ) -> Tuple[Job, Job]: - """Build a metrics job""" - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - - # Create the metrics directory - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - metric_base = sample_name + ".txt" - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - - is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") - mqbc_metrics = metrics_dir.joinpath( - metric_base + ".mean_qual_by_cycle.txt" - ) - bdbc_metrics = metrics_dir.joinpath( - metric_base + ".base_distribution_by_cycle.txt" - ) - qualdist_metrics = metrics_dir.joinpath( - metric_base + ".qual_distribution.txt" - ) - as_metrics = metrics_dir.joinpath(metric_base + ".alignment_stat.txt") - coverage_metrics = metrics_dir.joinpath("coverage") - - # WGS metrics - wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") - gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - driver.add_algo(MeanQualityByCycle(mqbc_metrics)) - driver.add_algo(BaseDistributionByCycle(bdbc_metrics)) - driver.add_algo(QualDistribution(qualdist_metrics)) - driver.add_algo(AlignmentStat(as_metrics)) - driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) - driver.add_algo(WgsMetricsAlgo(wgs_metrics, include_unpaired="true")) - driver.add_algo(CoverageMetrics(coverage_metrics)) - - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) - - rehead_script = pathlib.Path( - str( - files("sentieon_cli.scripts").joinpath("rehead_wgs_metrics.py") - ) - ) - rehead_job = Job( - Pipeline( - Command( - sys.executable, - str(rehead_script), - "--metrics_file", - str(wgs_metrics), - ) - ), - "Rehead metrics", - 0, - task_name="metrics", - ) - return (metrics_job, rehead_job) + def pangenome_metrics_algos(self, paths: MetricsPaths) -> List[BaseAlgo]: + """The metrics collected from a deduplicated pangenome alignment""" + return [ + InsertSizeMetricAlgo(paths.insert_size), + MeanQualityByCycle(paths.mean_qual_by_cycle), + BaseDistributionByCycle(paths.base_distribution_by_cycle), + QualDistribution(paths.qual_distribution), + AlignmentStat(paths.alignment_stat), + GCBias(paths.gc_bias, summary=paths.gc_bias_summary), + WgsMetricsAlgo(paths.wgs, include_unpaired="true"), + CoverageMetrics(paths.coverage), + ] diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index 4cd0ba1..4ed974f 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -48,23 +48,6 @@ def cmd_bedtools_subtract( return Pipeline(Command(*cmd), file_output=unphased_bed) -def cmd_bedtools_merge( - in_bed: pathlib.Path, - out_bed: pathlib.Path, - distance: int = 0, -) -> Pipeline: - """Bedtools merge""" - cmd = [ - "bedtools", - "merge", - "-d", - str(distance), - "-i", - str(in_bed), - ] - return Pipeline(Command(*cmd), file_output=out_bed) - - def cmd_bedtools_slop( in_bed: pathlib.Path, out_bed: pathlib.Path, @@ -1041,184 +1024,6 @@ def cmd_vg_haplotypes( return Pipeline(Command(*cmd)) -def cmd_vg_giraffe( - sample_gam: pathlib.Path, - sample_pangenome: pathlib.Path, - fastq1: pathlib.Path, - fastq2: Optional[pathlib.Path], - readgroup="rg-1", - sample="sample", - threads=1, - max_fragment_length=3000, -) -> Pipeline: - vg_cmd = [ - "vg", - "giraffe", - "-t", - str(threads), - "-Z", - str(sample_pangenome), - "--read-group", - readgroup, - "--sample", - sample, - "-L", - str(max_fragment_length), - "--progress", - "-f", - str(fastq1), - ] - if fastq2: - vg_cmd.extend(["-f", str(fastq2)]) - - return Pipeline(Command(*vg_cmd), file_output=sample_gam) - - -def cmd_vg_pack( - sample_pack: pathlib.Path, - sample_gams: List[pathlib.Path], - gbz: pathlib.Path, - min_mapq=5, - threads=1, -) -> Pipeline: - cat_cmd = Command("cat", *[str(x) for x in sample_gams]) - vg_cmd = Command( - "vg", - "pack", - "-x", - str(gbz), - "-g", - "-", - "-o", - str(sample_pack), - "-Q", - str(min_mapq), - "--threads", - str(threads), - ) - - return Pipeline(cat_cmd, vg_cmd) - - -def cmd_sv_call( - out_svs: pathlib.Path, - sample_pack: pathlib.Path, - sv_header: pathlib.Path, - gbz: pathlib.Path, - snarls: pathlib.Path, - sample_name="", - ref_path="GRCh38", - min_indel_size=34, -) -> Pipeline: - # Call SVs with vg call - vg_call_cmd = Command( - "vg", - "call", - "-r", - str(snarls), - "-k", - str(sample_pack), - "-s", - sample_name, - "-z", - "--ref-sample", - ref_path, - "--progress", - str(gbz), - ) - - # Use bcftools to remove smaller variants - view_cmd = Command( - "bcftools", - "view", - "-i", - f"ABS(ILEN) > {min_indel_size}", - "-H", - "-", - ) - - # Rehead the vg call output - cat_cmd = Command( - "cat", - str(sv_header), - InputProcSub(Pipeline(vg_call_cmd, view_cmd)), - ) - - # Sort and compress the output - sort_cmd = Command( - "bcftools", - "sort", - "-O", - "v", - ) - convert_cmd = Command( - "sentieon", - "util", - "vcfconvert", - "-", - str(out_svs), - ) - return Pipeline(cat_cmd, sort_cmd, convert_cmd) - - -def cmd_vg_surject( - out_bam: pathlib.Path, - sample_gam: pathlib.Path, - xg: pathlib.Path, - surject_paths_dict: pathlib.Path, - threads=1, - surject_xargs: List[str] = [ - "--interleaved", - "--progress", - ], -) -> Pipeline: - surject_cmd_list = [ - "vg", - "surject", - "--sam-output", - "-t", - str(threads), - "-x", - str(xg), - "--into-paths", - shlex.quote(str(surject_paths_dict)), - ] - surject_cmd_list.extend(surject_xargs) - surject_cmd_list.append(str(sample_gam)) - - sort_cmd = Command( - "sentieon", - "util", - "sort", - "--sam2bam", - "-i", - "-", - "-o", - str(out_bam), - ) - - return Pipeline(Command(*surject_cmd_list), sort_cmd) - - -def strip_ctg_prefix( - output_header: pathlib.Path, - bam: pathlib.Path, - prefix: str, -) -> Pipeline: - # Also remove the M5 tag - samtools_cmd = Command("samtools", "view", "-H", str(bam)) - sed_cmd = Command( - "sed", - "-e", - f"s/\tSN:{prefix}/\tSN:/", - "-e", - "s/\tM5:[0-9a-zA-Z]*\t/\t/", - "-e", - "s/\tM5:[0-9a-zA-Z]*$//", - ) - return Pipeline(samtools_cmd, sed_cmd, file_output=output_header) - - def cmd_estimate_ploidy( output_json: pathlib.Path, aln_files: List[pathlib.Path], diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 2a3d837..8f0729a 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -5,71 +5,55 @@ import argparse import copy import itertools -import multiprocessing as mp import os import pathlib import shutil import sys -from typing import Any, List, Optional, Set, Tuple +from typing import List, Optional, Set import packaging.version -from importlib.resources import files - -from . import command_strings as cmds from .dag import DAG from .driver import ( - AlignmentStat, - BaseDistributionByCycle, - CNVscope, - CNVModelApply, - CoverageMetrics, - Dedup, - DNAModelApply, + BaseAlgo, DNAscope, - Driver, - GCBias, - GVCFtyper, - HsMetricAlgo, - InsertSizeMetricAlgo, - MeanQualityByCycle, - LocusCollector, - QualDistribution, SVSolver, - WgsMetricsAlgo, ) from .job import Job from .pipeline import BasePipeline -from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BWA_FASTQ_MIN_VERSIONS, + BWA_REALIGN_MIN_VERSIONS, + AlignResult, + BwaFastqStage, + BwaRealignStage, + find_unzip, +) +from .stages.base import StageContext, driver_job, rm_job +from .stages.preprocessing import ( + ShortReadPreprocessingStage, + SrPreprocessingResult, +) +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + GVCFtyperStage, + TransferApplyStage, +) from .util import ( - SampleSex, - check_version, - cnvscope_sex_args, library_preloaded, parse_rg_line, path_arg, + require_versions, + set_bwt_max_mem, split_alignment, total_memory, ) -ALN_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), - "samtools": packaging.version.Version("1.16"), -} - -FQ_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - VARIANTS_MIN_VERSIONS = { "sentieon driver": packaging.version.Version("202308"), } -CNV_MIN_VERSIONS = { - # 202503.04 adds the CNVscope `--sex` and `--par` arguments - "sentieon driver": packaging.version.Version("202503.04"), -} - class DNAscopePipeline(BasePipeline): """The DNAscope pipeline""" @@ -235,7 +219,7 @@ def __init__(self) -> None: self.model_bundle: Optional[pathlib.Path] = None self.dbsnp: Optional[pathlib.Path] = None self.bed: Optional[pathlib.Path] = None - self.interval_padding = 0 + self.interval_padding: Optional[int] = None self.pcr_free = False self.gvcf = False self.duplicate_marking = "markdup" @@ -259,24 +243,10 @@ def __init__(self) -> None: self.no_split_alignment = False def validate(self) -> None: - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - - # uniquify pipeline attributes - self.sr_r1_fastq = self.r1_fastq - self.sr_r2_fastq = self.r2_fastq - self.sr_readgroups = self.readgroups - self.sr_duplicate_marking = self.duplicate_marking - del self.r1_fastq - del self.r2_fastq - del self.readgroups - del self.duplicate_marking + self.required(self.output_vcf, "output_vcf") # validate - if not self.sample_input and not ( - self.sr_r1_fastq and self.sr_readgroups - ): + if not self.sample_input and not (self.r1_fastq and self.readgroups): self.logger.error( "Please supply either the `--sample_input` or `--r1_fastq` " "and `--readgroups` arguments" @@ -303,21 +273,21 @@ def validate(self) -> None: "across decoy and unplaced contigs." ) - if len(self.sr_r1_fastq) != len(self.sr_readgroups): + if len(self.r1_fastq) != len(self.readgroups): self.logger.error( "The number of readgroups does not equal the number of fastq " "files" ) sys.exit(2) - for rg in self.sr_readgroups: + for rg in self.readgroups: try: parse_rg_line(rg.replace(r"\t", "\t")) except ValueError as e: self.logger.error("Invalid --readgroups value '%s': %s", rg, e) sys.exit(2) - if self.sr_r1_fastq or self.align or self.collate_align: + if self.r1_fastq or self.align or self.collate_align: self.validate_bwa_index() def configure(self) -> None: @@ -338,16 +308,16 @@ def configure_alignment(self) -> None: if shm_ok: os.environ["SENTIEON_TMPDIR"] = "/dev/shm" - self.set_bwt_max_mem( - total_input_size if shm_ok else 0, n_alignment_jobs + set_bwt_max_mem( + total_input_size if shm_ok else 0, + n_alignment_jobs, + override=self.bwt_max_mem, ) def total_input_size(self) -> int: """Find the total size of all inputs""" total_input_size = sum([x.stat().st_size for x in self.sample_input]) - for r1, r2 in itertools.zip_longest( - self.sr_r1_fastq, self.sr_r2_fastq - ): + for r1, r2 in itertools.zip_longest(self.r1_fastq, self.r2_fastq): for fq in (r1, r2): if isinstance(fq, pathlib.Path): total_input_size += fq.stat().st_size @@ -369,509 +339,170 @@ def check_shm(self, total_input_size: int, n_alignment_jobs) -> bool: return True return False - def set_bwt_max_mem( - self, - total_input_size: int, - n_alignment_jobs: int = 1, - ): - """Set the bwt_max_mem environment variable""" - if self.bwt_max_mem: - os.environ["bwt_max_mem"] = self.bwt_max_mem - return - - total_mem = total_memory() - total_mem_gb = total_mem / (1024.0**3) - align_mem_gb = ( - total_mem_gb - 4 - total_input_size / (1024.0**3) * 2.3 - ) # some memory for other system processes - bwa_mem_gb = max( - int((align_mem_gb / n_alignment_jobs) - 6), 0 - ) # some memory for other alignment processes - self.logger.debug("Setting bwt_max_mem to: %sG", bwa_mem_gb) - os.environ["bwt_max_mem"] = f"{bwa_mem_gb}G" - def build_dag(self) -> DAG: """Build the DAG for the pipeline""" self.logger.info("Building the DAG") dag = DAG() + ctx = self.stage_context() # Alignment align_jobs: Set[Job] = set() sample_input = copy.deepcopy(self.sample_input) - bam_rm_job = None + bam_cleanup: List[pathlib.Path] = [] if self.align or self.collate_align: - sample_input, align_jobs, bam_rm_job = self.sr_align_inputs() - for job in align_jobs: - dag.add_job(job) - aligned_fastq, align_fastq_jobs, fq_rm_job = self.sr_align_fastq() - for job in align_fastq_jobs: - dag.add_job(job) - sample_input += aligned_fastq + aln_result = self.add_sr_input_alignment(dag, ctx) + sample_input = aln_result.outputs + align_jobs = set(aln_result.jobs) + bam_cleanup = aln_result.cleanup_paths + fq_result = self.add_sr_fastq_alignment(dag, ctx) + align_fastq_jobs = set(fq_result.jobs) + sample_input += fq_result.outputs # Dedup and metrics - ( - deduped, - lc_job, - dedup_job, - metrics_job, - rehead_job, - ) = self.dedup_and_metrics(sample_input) - if lc_job: - dag.add_job(lc_job, align_jobs.union(align_fastq_jobs)) - if dedup_job: - dag.add_job(dedup_job, {lc_job}) - if metrics_job: - dag.add_job(metrics_job, {dedup_job}) - if rehead_job: - dag.add_job(rehead_job, {metrics_job}) - if bam_rm_job: - dag.add_job(bam_rm_job, {dedup_job}) - if fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + preprocessing = self.add_sr_preprocessing( + dag, ctx, sample_input, align_jobs.union(align_fastq_jobs) + ) + deduped = preprocessing.deduped + dedup_job = preprocessing.dedup_job + # An alignment has cleanup paths only when it is an intermediate, + # which is exactly when duplicate marking -- and so `dedup_job` -- + # follows it + if dedup_job: + if bam_cleanup: + dag.add_job(rm_job(bam_cleanup, "rm-bam-aln"), {dedup_job}) + if fq_result.cleanup_paths: + dag.add_job( + rm_job(fq_result.cleanup_paths, "rm-fq-aln"), + {dedup_job}, + ) # Small variants if not self.skip_small_variants: - ( - call_job, - apply_job, - rm_job, - gvcftyper_job, - svsolver_job, - sv_rm_job, - ) = self.sr_call_variants(deduped) call_dependencies: Set[Job] = set() if dedup_job: call_dependencies.add(dedup_job) else: call_dependencies.update(align_jobs) call_dependencies.update(align_fastq_jobs) - dag.add_job(call_job, call_dependencies) - dag.add_job(apply_job, {call_job}) - dag.add_job(rm_job, {apply_job}) - if gvcftyper_job: - dag.add_job(gvcftyper_job, {apply_job}) - if svsolver_job and sv_rm_job: - dag.add_job(svsolver_job, {call_job}) - dag.add_job(sv_rm_job, {svsolver_job}) + self.add_small_variant_calling( + dag, ctx, deduped, call_dependencies + ) # Multiqc if not self.skip_multiqc: multiqc_job = self.multiqc() - multiqc_dependencies: Set[Job] = set() - if lc_job: - multiqc_dependencies.add(lc_job) - if metrics_job: - multiqc_dependencies.add(metrics_job) - if rehead_job: - multiqc_dependencies.add(rehead_job) - if multiqc_job: - dag.add_job(multiqc_job, multiqc_dependencies) + dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) return dag - def sr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job], Job]: + def add_sr_input_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: """Align input BAM/CRAM/uBAM/uCRAM files with bwa""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - if not self.skip_version_check: - for cmd, min_version in ALN_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - res: List[pathlib.Path] = [] - suffix = "bam" if self.bam_format else "cram" - util_sort_args = self.util_sort_args - if self.sr_duplicate_marking != "none": - suffix = "bam" - util_sort_args += " --bam_compression 1 " - jobs = set() - align_outputs: List[pathlib.Path] = [] - for i, input_aln in enumerate(self.sample_input): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_bwa_sorted_{i}.{suffix}" - ) - ) - if self.sr_duplicate_marking != "none": - out_aln = self.tmp_dir.joinpath(f"bwa_sorted_{i}.{suffix}") - rg_lines = cmds.get_rg_lines(input_aln, self.dry_run) - rg_header = self.tmp_dir.joinpath(f"input_{i}.hdr") - with open(rg_header, "w", encoding="utf-8") as rg_fh: - for line in rg_lines: - print(line, file=rg_fh) - job = Job( - cmds.cmd_samtools_fastq_bwa( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_header, - self.input_ref, - collate=self.collate_align, - bwa_args=self.bwa_args, - bwa_k=str(self.bwa_k), - util_sort_args=util_sort_args, - ), - f"bam-align-{i}", - self.cores, - task_name="alignment", - ) - res.append(out_aln) - jobs.add(job) - align_outputs.append(out_aln) - align_outputs.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - align_outputs.append(pathlib.Path(str(out_aln) + ".crai")) + bundle = self.required(self.model_bundle, "model_bundle") - # Create an unscheduled job to remove the aligned inputs - rm_job = Job( - Pipeline( - Command("rm", *[str(x) for x in align_outputs], fail_ok=True) - ), - "rm-bam-aln", - 0, - task_name="cleanup", + require_versions( + BWA_REALIGN_MIN_VERSIONS, skip=self.skip_version_check ) - return (res, jobs, rm_job) - - def sr_align_fastq( - self, - ) -> Tuple[List[pathlib.Path], Set[Job], Optional[Job]]: + return BwaRealignStage( + ctx=ctx, + inputs=self.sample_input, + model_bundle=bundle, + bam_format=self.bam_format, + duplicate_marking=self.duplicate_marking, + input_ref=self.input_ref, + collate=self.collate_align, + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_sr_fastq_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: """Align fastq files to the reference genome using bwa""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - res: List[pathlib.Path] = [] - jobs: Set[Job] = set() - if not self.sr_r1_fastq and not self.sr_readgroups: - return (res, jobs, None) - - if not self.skip_version_check: - for cmd, min_version in FQ_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - unzip = "igzip" - if not shutil.which(unzip): - self.logger.warning( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" + bundle = self.required(self.model_bundle, "model_bundle") - n_alignment_jobs = max(1, len(self.numa_nodes)) - suffix = "bam" if self.bam_format else "cram" - util_sort_args = self.util_sort_args - if self.sr_duplicate_marking != "none": - suffix = "bam" - util_sort_args += " --bam_compression 1 " - align_outputs: List[pathlib.Path] = [] - for i, (r1, r2, rg) in enumerate( - itertools.zip_longest( - self.sr_r1_fastq, self.sr_r2_fastq, self.sr_readgroups + if not self.r1_fastq and not self.readgroups: + return AlignResult( + jobs=[], terminal=set(), outputs=[], cleanup_paths=[] ) - ): - for j in range(n_alignment_jobs): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_bwa_sorted_fq_{i}_{j}.{suffix}" - ) - ) - if self.sr_duplicate_marking != "none": - out_aln = self.tmp_dir.joinpath( - f"bwa_sorted_fq_{i}_{j}.{suffix}" - ) - numa = self.numa_nodes[j] if len(self.numa_nodes) > 1 else None - split = ( - f"{j}/{n_alignment_jobs}" - if len(self.numa_nodes) > 1 - else None - ) - split_cores = max(1, int(self.cores / n_alignment_jobs)) - job = Job( - cmds.cmd_fastq_bwa( - out_aln, - r1, - r2, - rg, - self.reference, - self.model_bundle, - split_cores, - unzip, - self.bwa_args, - str(self.bwa_k), - util_sort_args, - numa, - split, - ), - f"bam-align-{i}-{j}", - split_cores, - resources={f"node{j}": 1}, - task_name="alignment", - ) - res.append(out_aln) - jobs.add(job) - align_outputs.append(out_aln) - align_outputs.append(pathlib.Path(str(out_aln) + ".bai")) - if suffix == "cram": - align_outputs.append(pathlib.Path(str(out_aln) + ".crai")) - - # Create an unscheduled job to remove the aligned inputs - rm_job = Job( - Pipeline( - Command("rm", *[str(x) for x in align_outputs], fail_ok=True) - ), - "rm-fq-aln", - 0, - task_name="cleanup", - ) - - return (res, jobs, rm_job) - def dedup_and_metrics( + require_versions(BWA_FASTQ_MIN_VERSIONS, skip=self.skip_version_check) + + return BwaFastqStage( + ctx=ctx, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroups=self.readgroups, + model_bundle=bundle, + numa_nodes=self.numa_nodes, + bam_format=self.bam_format, + duplicate_marking=self.duplicate_marking, + unzip=find_unzip(self.logger), + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_sr_preprocessing( self, - sample_input, - ) -> Tuple[ - List[pathlib.Path], - Optional[Job], - Optional[Job], - Optional[Job], - Optional[Job], - ]: - """Perform dedup and metrics collection""" - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - suffix = "bam" if self.bam_format else "cram" - - # Create the metrics directory - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - metric_base = sample_name + ".txt" - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - - # LocusCollector and Metrics - out_score = metrics_dir.joinpath(metric_base + ".score.txt.gz") - is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") - mqbc_metrics = metrics_dir.joinpath( - metric_base + ".mean_qual_by_cycle.txt" - ) - bdbc_metrics = metrics_dir.joinpath( - metric_base + ".base_distribution_by_cycle.txt" - ) - qualdist_metrics = metrics_dir.joinpath( - metric_base + ".qual_distribution.txt" - ) - as_metrics = metrics_dir.joinpath(metric_base + ".alignment_stat.txt") - coverage_metrics = metrics_dir.joinpath("coverage") - - # WES metrics - hs_metrics = metrics_dir.joinpath( - metric_base + ".hybrid-selection.txt" - ) - - # WGS metrics - wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") - gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") - - lc_job = None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - if self.sr_duplicate_marking != "none": - driver.add_algo( - LocusCollector( - out_score, - consensus=self.consensus, - ) - ) - - # Prefer to run InsertSizeMetricAlgo after duplicate marking - if not self.skip_metrics and ( - (self.assay == "WES" and not self.bed) - or self.sr_duplicate_marking == "none" - ): - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - - if not self.skip_metrics: - driver.add_algo(MeanQualityByCycle(mqbc_metrics)) - driver.add_algo(BaseDistributionByCycle(bdbc_metrics)) - driver.add_algo(QualDistribution(qualdist_metrics)) - driver.add_algo(AlignmentStat(as_metrics)) - if self.assay == "WGS": - driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) - - if not (self.sr_duplicate_marking == "none" and self.skip_metrics): - lc_job = Job( - Pipeline(Command(*driver.build_cmd())), - "locuscollector", - self.cores, - task_name="dedup", - ) - - if self.sr_duplicate_marking == "none": - return (sample_input, lc_job, None, None, None) - - # Dedup - deduped = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") - ) - dedup_metrics = metrics_dir.joinpath( - metric_base + ".dedup_metrics.txt" - ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - driver.add_algo( - Dedup( - deduped, - out_score, - cram_write_options="version=3.0,compressor=rans", - metrics=dedup_metrics, - rmdup=(self.sr_duplicate_marking == "rmdup"), - ) - ) - dedup_job = Job( - Pipeline(Command(*driver.build_cmd())), - "dedup", - self.cores, - task_name="dedup", - ) - - if self.skip_metrics: - return ([deduped], lc_job, dedup_job, None, None) - - # Run HsMetricAlgo after duplicate marking to account for - # duplicate reads - metrics_job = None - rehead_job = None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=( - [deduped] - if self.sr_duplicate_marking != "none" - else sample_input - ), - interval=self.bed, - ) - if self.assay == "WES" and self.bed: - driver.add_algo(HsMetricAlgo(hs_metrics, self.bed, self.bed)) - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) # Run metrics in the background - - # Run WgsMetricsAlgo after duplicate marking to account for - # duplicate reads - if self.assay == "WGS": - driver.add_algo(InsertSizeMetricAlgo(is_metrics)) - driver.add_algo( - WgsMetricsAlgo(wgs_metrics, include_unpaired="true") - ) - driver.add_algo(CoverageMetrics(coverage_metrics)) - metrics_job = Job( - Pipeline(Command(*driver.build_cmd())), - "metrics", - 0, - task_name="metrics", - ) # Run metrics in the background - - # Rehead WGS metrics so they are recognized by MultiQC - rehead_script = pathlib.Path( - str( - files("sentieon_cli.scripts").joinpath( - "rehead_wgs_metrics.py" - ) - ) - ) - rehead_job = Job( - Pipeline( - Command( - sys.executable, - str(rehead_script), - "--metrics_file", - str(wgs_metrics), - ) - ), - "Rehead metrics", - 0, - task_name="metrics", - ) - return ([deduped], lc_job, dedup_job, metrics_job, rehead_job) - - def sr_call_variants( + dag: DAG, + ctx: StageContext, + sample_input: List[pathlib.Path], + upstream: Set[Job], + ) -> SrPreprocessingResult: + """Mark duplicates and collect metrics from the short reads""" + return ShortReadPreprocessingStage( + ctx=ctx, + inputs=sample_input, + duplicate_marking=self.duplicate_marking, + consensus=self.consensus, + skip_metrics=self.skip_metrics, + assay=self.assay, + bed=self.bed, + bam_format=self.bam_format, + ).add_to(dag, upstream) + + def add_small_variant_calling( self, + dag: DAG, + ctx: StageContext, deduped: List[pathlib.Path], - ) -> Tuple[Job, Job, Job, Optional[Job], Optional[Job], Optional[Job]]: + upstream: Set[Job], + ) -> None: """Call SNVs, indels, and SVs using DNAscope""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") - if not self.skip_version_check: - for cmd, min_version in VARIANTS_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(VARIANTS_MIN_VERSIONS, skip=self.skip_version_check) out_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) out_svs_tmp = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_svs_tmp.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_svs_tmp.vcf.gz") ) out_svs = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_svs.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_svs.vcf.gz") ) - emit_mode = "gvcf" if self.gvcf else "variant" pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - model = self.model_bundle.joinpath("dnascope.model") + model = bundle.joinpath("dnascope.model") # Call variants with DNAscope emit_mode = "variant" - ds_out = self.output_vcf + ds_out = ctx.output_vcf tmp_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_tmp.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_tmp.vcf.gz") ) if self.gvcf: emit_mode = "gvcf" ds_out = out_gvcf tmp_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_tmp.g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_tmp.g.vcf.gz") ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=deduped, - interval=self.bed, - interval_padding=self.interval_padding, - ) - driver.add_algo( + + algos: List[BaseAlgo] = [ DNAscope( tmp_vcf, dbsnp=self.dbsnp, @@ -879,178 +510,57 @@ def sr_call_variants( pcr_indel_model=pcr_indel_model, model=model, ) - ) + ] if not self.skip_svs: - driver.add_algo( + algos.append( DNAscope( out_svs_tmp, dbsnp=self.dbsnp, var_type="BND", ) ) - call_job = Job( - Pipeline(Command(*driver.build_cmd())), - "variant-calling", - self.cores, - task_name="variant-calling", - ) + call = DNAscopeStage( + ctx=ctx, + algos=algos, + inputs=deduped, + interval=self.bed, + interval_padding=self.interval_padding, + ).add_to(dag, upstream) # Genotyping and filtering with DNAModelApply - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model, - tmp_vcf, - ds_out, - ) - ) - apply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=tmp_vcf, + apply=ApplySpec(model=model, output=ds_out), + ).add_to(dag, call.terminal) # Remove the tmp_vcf - rm_cmd = ["rm", str(tmp_vcf), str(tmp_vcf) + ".tbi"] - rm_job = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), - "rm-tmp-vcf", - 0, - task_name="cleanup", + dag.add_job( + rm_job([tmp_vcf, str(tmp_vcf) + ".tbi"], "rm-tmp-vcf"), + transfer_apply.terminal, ) # Genotype gVCFs - gvcftyper_job = None if self.gvcf: - driver = Driver( - reference=self.reference, - thread_count=self.cores, + GVCFtyperStage( + ctx=ctx, + gvcf=out_gvcf, + output=ctx.output_vcf, interval=self.bed, - ) - driver.add_algo( - GVCFtyper( - output=self.output_vcf, - vcf=out_gvcf, - ) - ) - gvcftyper_job = Job( - Pipeline(Command(*driver.build_cmd())), - "gvcftyper", - self.cores, - task_name="gvcftyper", - ) + ).add_to(dag, transfer_apply.terminal) # Call SVs - svsolver_job = None - sv_rm_job = None if not self.skip_svs: - driver = Driver( - reference=self.reference, - thread_count=self.cores, + svsolver_job = driver_job( + ctx, + [SVSolver(output=out_svs, vcf=out_svs_tmp)], interval=self.bed, - ) - driver.add_algo( - SVSolver( - output=out_svs, - vcf=out_svs_tmp, - ) - ) - svsolver_job = Job( - Pipeline(Command(*driver.build_cmd())), - "svsolver", + threads=1, + name="svsolver", task_name="sv-calling", ) - sv_rm_job = Job( - Pipeline( - Command( - "rm", - str(out_svs_tmp), - str(out_svs_tmp) + ".tbi", - fail_ok=True, - ) - ), - "rm-tmp-sv", - 0, - task_name="cleanup", + dag.add_job(svsolver_job, call.terminal) + dag.add_job( + rm_job([out_svs_tmp, str(out_svs_tmp) + ".tbi"], "rm-tmp-sv"), + {svsolver_job}, ) - - return ( - call_job, - apply_job, - rm_job, - gvcftyper_job, - svsolver_job, - sv_rm_job, - ) - - -def call_cnvs( - tmp_dir: pathlib.Path, - output_vcf: pathlib.Path, - reference: pathlib.Path, - sample_input: List[pathlib.Path], - model_bundle: pathlib.Path, - bed: Optional[pathlib.Path] = None, - cores: int = mp.cpu_count(), - skip_version_check: bool = False, - replace_rg: Optional[List[List[str]]] = None, - sample_sex: Optional[SampleSex] = None, - par_bed: Optional[pathlib.Path] = None, - **_kwargs: Any, -) -> Tuple[Job, Job]: - """ - Call CNVs using CNVscope - """ - if not skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - sex, par = cnvscope_sex_args(sample_sex, par_bed) - cnvscope_vcf = tmp_dir.joinpath("cnvscope.vcf.gz") - cnv_vcf = pathlib.Path(str(output_vcf).replace(".vcf.gz", ".cnv.vcf.gz")) - driver = Driver( - reference=reference, - thread_count=cores, - replace_rg=replace_rg, - input=sample_input, - interval=bed, - ) - driver.add_algo( - CNVscope( - cnvscope_vcf, - model_bundle.joinpath("cnv.model"), - sex=sex, - par=par, - ) - ) - cnvscope_job = Job( - Pipeline(Command(*driver.build_cmd())), - "CNVscope", - cores, - task_name="cnv", - ) - - driver = Driver( - reference=reference, - thread_count=cores, - ) - driver.add_algo( - CNVModelApply( - cnv_vcf, - model_bundle.joinpath("cnv.model"), - vcf=cnvscope_vcf, - ) - ) - cnvmodelapply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "CNVModelApply", - cores, - task_name="cnv", - ) - - return (cnvscope_job, cnvmodelapply_job) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 45a8cbc..d69090e 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -7,7 +7,7 @@ import json import pathlib import sys -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple import packaging.version @@ -18,36 +18,56 @@ from . import command_strings as cmds from .dag import DAG from .driver import ( - BaseDriver, Driver, DNAscope, - DNAModelApply, HybridStage1, HybridStage2, HybridStage3, ) -from .dnascope import call_cnvs, CNV_MIN_VERSIONS, DNAscopePipeline -from .dnascope_longread import DNAscopeLRPipeline from .job import Job from .pipeline import BasePipeline from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BWA_FASTQ_MIN_VERSIONS, + MINIMAP2_REALIGN_MIN_VERSIONS, + AlignResult, + BwaFastqStage, + Minimap2RealignStage, + find_unzip, +) +from .stages.base import StageContext, driver_job, rm_job +from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.metrics import MOSDEPTH_MIN_VERSIONS, MosdepthStage +from .stages.ploidy import PloidyStage +from .stages.preprocessing import ( + ShortReadPreprocessingStage, + SrPreprocessingResult, +) +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + TransferApplyStage, + TransferSpec, +) +from .stages.sv import LONGREADSV_MIN_VERSIONS, LongReadSVStage +from .stages.transfer import TransferConfig from .util import ( __version__, - check_version, library_preloaded, parse_rg_line, path_arg, + require_versions, sample_sex_arg, + set_bwt_max_mem, split_alignment, vcf_id, + versions_available, ) from .shard import ( determine_shards_from_fai, parse_fai, - ploidy_contigs_for_build, vcf_contigs, ) -from .transfer import build_transfer_jobs logger = get_logger(__name__) @@ -120,7 +140,7 @@ def __init__( ) -class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): +class DNAscopeHybridPipeline(BasePipeline): """The DNAscope Hybrid pipeline""" params = copy.deepcopy(BasePipeline.params) @@ -286,7 +306,7 @@ class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): "minimap2_args": { # help="Extra arguments for sentieon minimap2", "help": argparse.SUPPRESS, - "default": "-Y", + "default": "-YL", }, "no_split_alignment": { "help": argparse.SUPPRESS, @@ -313,24 +333,43 @@ class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): def __init__(self) -> None: super().__init__() + # Required arguments + self.lr_aln: List[pathlib.Path] = [] + self.model_bundle: Optional[pathlib.Path] = None + self.sr_aln: List[pathlib.Path] = [] self.sr_r1_fastq: List[pathlib.Path] = [] self.sr_r2_fastq: List[pathlib.Path] = [] self.sr_readgroups: List[str] = [] - self.sr_aln: List[pathlib.Path] = [] - self.sr_duplicate_marking = "markdup" - self.lr_aln: List[pathlib.Path] = [] + # Additional arguments + self.bam_format = False + self.bed: Optional[pathlib.Path] = None + self.dbsnp: Optional[pathlib.Path] = None + self.gvcf = False self.lr_align_input = False self.lr_input_ref: Optional[pathlib.Path] = None self.par_bed: Optional[pathlib.Path] = None self.pop_vcf: Optional[pathlib.Path] = None - self.bam_format = False self.rgsm: Optional[str] = None + self.skip_cnv = False + self.skip_metrics = False + self.skip_mosdepth = False + self.skip_multiqc = False + self.skip_svs = False + self.sr_duplicate_marking = "markdup" + # Hidden arguments + self.bwa_args = "" + self.bwa_k = 100000000 + self.bwt_max_mem: Optional[str] = None self.lr_fastq_taglist = "*" - self.sr_read_filter: Optional[str] = None self.lr_read_filter: Optional[str] = None - self.assay = "WGS" + self.minimap2_args = "-YL" + self.no_split_alignment = False self.skip_model_apply = False self.skip_pop_vcf_id_check = False + self.sr_read_filter: Optional[str] = None + self.util_sort_args = ( + "--cram_write_options version=3.0,compressor=rans" + ) # Stashed by `build_dag` for the second, sex-aware DAG self.ploidy_json: Optional[pathlib.Path] = None self.cnv_sr_aln: List[pathlib.Path] = [] @@ -395,10 +434,7 @@ def validate_cnv(self) -> None: # CNV calling now runs in the second DAG, so the version of the # driver is checked before the first DAG starts - if not self.skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) def validate_bundle(self) -> None: bundle_info_bytes = ar_load( @@ -546,7 +582,7 @@ def configure(self) -> None: self.numa_nodes = split_alignment(self.cores) n_alignment_jobs = max(1, len(self.numa_nodes)) - self.set_bwt_max_mem(0, n_alignment_jobs) + set_bwt_max_mem(0, n_alignment_jobs, override=self.bwt_max_mem) def configure_readgroups(self) -> None: self.lr_aln_readgroups = self.all_readgroups[0] @@ -570,15 +606,8 @@ def build_dag(self) -> DAG: self.logger.info("Building the DAG") dag = DAG() - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + self.required(self.model_bundle, "model_bundle") + ctx = self.stage_context() rg_info = RgInfo( self.lr_aln_readgroups, @@ -592,68 +621,48 @@ def build_dag(self) -> DAG: # Short-read alignment sr_aln = self.sr_aln - ( - aligned_fastq, - align_fastq_jobs, - fq_rm_job, - ) = self.sr_align_fastq() - for job in align_fastq_jobs: - dag.add_job(job) - sr_aln.extend(aligned_fastq) + fq_result = self.add_sr_fastq_alignment(dag, ctx) + align_fastq_jobs = set(fq_result.jobs) + sr_aln.extend(fq_result.outputs) # Short-read dedup - ( - deduped, - lc_job, - dedup_job, - metrics_job, - rehead_job, - ) = self.dedup_and_metrics( - sample_input=sr_aln, + preprocessing = self.add_sr_preprocessing( + dag, ctx, sr_aln, set(align_fastq_jobs) ) - sr_aln = deduped - if lc_job: - dag.add_job(lc_job, align_fastq_jobs) - if dedup_job: - dag.add_job(dedup_job, {lc_job}) - if metrics_job and not self.skip_metrics: - dag.add_job(metrics_job, {dedup_job}) - if rehead_job: - dag.add_job(rehead_job, {metrics_job}) - if fq_rm_job: - dag.add_job(fq_rm_job, {dedup_job}) + sr_aln = preprocessing.deduped + dedup_job = preprocessing.dedup_job + # The fastq alignment has cleanup paths only when it is an + # intermediate, which is exactly when duplicate marking -- and so + # `dedup_job` -- follows it + if dedup_job and fq_result.cleanup_paths: + dag.add_job( + rm_job(fq_result.cleanup_paths, "rm-fq-aln"), {dedup_job} + ) # Long-read alignment realign_jobs: Set[Job] = set() lr_aln = self.lr_aln if self.lr_align_input: - lr_aln, realign_jobs = self.lr_align_inputs() - for job in realign_jobs: - dag.add_job(job) + realign_result = self.add_lr_realignment(dag, ctx) + lr_aln = realign_result.outputs + realign_jobs = set(realign_result.jobs) if not self.skip_mosdepth: - mosdepth_jobs = self.mosdepth(sample_input=lr_aln) - for job in mosdepth_jobs: - dag.add_job(job, realign_jobs) + self.add_mosdepth(dag, ctx, lr_aln, realign_jobs) if not self.skip_multiqc: multiqc_job = self.multiqc() - multiqc_dependencies: Set[Job] = set() - if lc_job: - multiqc_dependencies.add(lc_job) - if metrics_job: - multiqc_dependencies.add(metrics_job) - if rehead_job: - multiqc_dependencies.add(rehead_job) - if multiqc_job: - dag.add_job(multiqc_job, multiqc_dependencies) + dag.add_job(multiqc_job, set(preprocessing.qc_jobs)) if not self.skip_svs: - longreadsv_job = self.call_svs( - lr_aln, replace_rg=rg_info.replace_rg_args[0] + self.add_sv_calling( + dag, + ctx, + lr_aln, + rg_info.replace_rg_args[0], + realign_jobs, ) - dag.add_job(longreadsv_job, realign_jobs) sr_preprocessing_jobs: Set[Job] = set() sr_preprocessing_jobs.update(align_fastq_jobs) @@ -662,15 +671,12 @@ def build_dag(self) -> DAG: # Estimate the sample ploidy and sex. The JSON output is always # written; `--sample_sex` takes precedence for the sex used by # sex-aware CNV calling. - self.ploidy_json = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_ploidy.json") - ) - ploidy_job = self.build_ploidy_job( - self.ploidy_json, - sr_aln, - ploidy_contigs_for_build(self.reference_build), - ) - dag.add_job(ploidy_job, sr_preprocessing_jobs) + ploidy_result = PloidyStage( + ctx=ctx, + inputs=sr_aln, + reference_build=self.reference_build, + ).add_to(dag, sr_preprocessing_jobs) + self.ploidy_json = ploidy_result.ploidy_json if not self.skip_cnv: # CNV calling is sex-aware and runs in the second DAG @@ -681,185 +687,213 @@ def build_dag(self) -> DAG: else None ) - ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat, - apply_job, - norm_job, - ) = self.call_variants(sr_aln, lr_aln, rg_info) - dag.add_job(call_job, realign_jobs | sr_preprocessing_jobs) - dag.add_job(select_job, {call_job}) - dag.add_job(mapq0_job, realign_jobs | sr_preprocessing_jobs) - dag.add_job(mapq0_slop_job, {mapq0_job}) - dag.add_job(cat_merge_job, {mapq0_slop_job, select_job}) - # The haplotype job writes the fifo read by the first-stage job - dag.add_job(stage1_fifo_job) - dag.add_job(stage1_hap_job, {stage1_fifo_job, cat_merge_job}) - dag.add_job(stage1_job, {stage1_fifo_job, cat_merge_job}) - dag.add_job(second_stage_job, {stage1_job, stage1_hap_job}) - dag.add_job(third_stage_job, {second_stage_job}) - dag.add_job(call2_job, {third_stage_job}) - dag.add_job(subset_job, {second_stage_job}) - dag.add_job(concat_job, {subset_job, call2_job}) - dag.add_job(anno_job, {concat_job}) - - apply_dependencies = {anno_job} - if transfer_jobs and transfer_concat: - for job in transfer_jobs: - dag.add_job(job, {anno_job}) - dag.add_job(transfer_concat, set(transfer_jobs)) - apply_dependencies = {transfer_concat} - - if apply_job: - dag.add_job(apply_job, apply_dependencies) - if apply_job and norm_job: - dag.add_job(norm_job, {apply_job}) - - # Remove intermediate files during processing - if not self.retain_tmpdir: - dag.add_job(rm_job1, {cat_merge_job}) - dag.add_job(rm_job2, {stage1_job, stage1_hap_job}) - dag.add_job(rm_job3, {second_stage_job}) - dag.add_job(rm_job4, {third_stage_job}) - dag.add_job(rm_job5, {concat_job}) + self.add_variant_calling( + dag, + ctx, + sr_aln, + lr_aln, + rg_info, + realign_jobs | sr_preprocessing_jobs, + ) return dag + def add_sr_fastq_alignment( + self, dag: DAG, ctx: StageContext + ) -> AlignResult: + """Align the short-read fastq files with bwa""" + bundle = self.required(self.model_bundle, "model_bundle") + + if not self.sr_r1_fastq and not self.sr_readgroups: + return AlignResult( + jobs=[], terminal=set(), outputs=[], cleanup_paths=[] + ) + + require_versions(BWA_FASTQ_MIN_VERSIONS, skip=self.skip_version_check) + + return BwaFastqStage( + ctx=ctx, + r1_fastq=self.sr_r1_fastq, + r2_fastq=self.sr_r2_fastq, + readgroups=self.sr_readgroups, + model_bundle=bundle, + numa_nodes=self.numa_nodes, + bam_format=self.bam_format, + duplicate_marking=self.sr_duplicate_marking, + unzip=find_unzip(self.logger), + bwa_args=self.bwa_args, + bwa_k=str(self.bwa_k), + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_sr_preprocessing( + self, + dag: DAG, + ctx: StageContext, + sr_aln: List[pathlib.Path], + upstream: Set[Job], + ) -> SrPreprocessingResult: + """Mark duplicates and collect metrics from the short reads""" + return ShortReadPreprocessingStage( + ctx=ctx, + inputs=sr_aln, + duplicate_marking=self.sr_duplicate_marking, + consensus=False, + skip_metrics=self.skip_metrics, + assay="WGS", + bed=self.bed, + bam_format=self.bam_format, + ).add_to(dag, upstream) + + def add_lr_realignment(self, dag: DAG, ctx: StageContext) -> AlignResult: + """Re-align the long-read input files with minimap2""" + bundle = self.required(self.model_bundle, "model_bundle") + require_versions( + MINIMAP2_REALIGN_MIN_VERSIONS, skip=self.skip_version_check + ) + + return Minimap2RealignStage( + ctx=ctx, + inputs=self.lr_aln, + model_bundle=bundle, + sample_name=ctx.output_vcf.name.replace(".vcf.gz", ""), + bam_format=self.bam_format, + input_ref=self.lr_input_ref, + fastq_taglist=self.lr_fastq_taglist, + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).add_to(dag) + + def add_mosdepth( + self, + dag: DAG, + ctx: StageContext, + inputs: List[pathlib.Path], + upstream: Set[Job], + ) -> None: + """Run mosdepth for QC, when it is available""" + if not versions_available( + MOSDEPTH_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping mosdepth. mosdepth version %s or later not found", + MOSDEPTH_MIN_VERSIONS["mosdepth"], + ) + return + + MosdepthStage(ctx=ctx, inputs=inputs).add_to(dag, upstream) + + def add_sv_calling( + self, + dag: DAG, + ctx: StageContext, + inputs: List[pathlib.Path], + replace_rg: Optional[List[List[str]]], + upstream: Set[Job], + ) -> None: + """Call SVs from the long reads with Sentieon LongReadSV""" + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(LONGREADSV_MIN_VERSIONS, skip=self.skip_version_check) + + LongReadSVStage( + ctx=ctx, + inputs=inputs, + model=bundle.joinpath("longreadsv.model"), + interval=self.bed, + replace_rg=replace_rg, + ).add_to(dag, upstream) + def build_second_dag(self) -> Optional[DAG]: """Build the second DAG, for sex-aware CNV calling""" if self.skip_cnv: return None - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") assert self.ploidy_json is not None + ctx = self.stage_context() self.get_sex(self.ploidy_json) self.logger.info("Building the DNAscope Hybrid CNV DAG") dag = DAG() - cnvscope_job, cnvmodelapply_job = call_cnvs( - self.tmp_dir, - self.output_vcf, - self.reference, - self.cnv_sr_aln, - self.model_bundle, - self.bed, - self.cores, - self.skip_version_check, - replace_rg=self.cnv_replace_rg, + CNVscopeStage( + ctx=ctx, + inputs=self.cnv_sr_aln, + model=bundle.joinpath("cnv.model"), + cnvscope_vcf=ctx.tmp_dir.joinpath("cnvscope.vcf.gz"), + cnv_vcf=pathlib.Path( + str(ctx.output_vcf).replace(".vcf.gz", ".cnv.vcf.gz") + ), sample_sex=self.sample_sex, par_bed=self.cnv_par_bed, - ) - dag.add_job(cnvscope_job) - dag.add_job(cnvmodelapply_job, {cnvscope_job}) + interval=self.bed, + replace_rg=self.cnv_replace_rg, + ).add_to(dag) return dag - def call_variants( + def add_cleanup( self, + dag: DAG, + paths: List[pathlib.Path], + name: str, + upstream: Set[Job], + ) -> None: + """Remove intermediate files once `upstream` has finished. + + A no-op under `--retain_tmpdir`, which keeps everything the run + wrote in the temporary directory. + """ + if self.retain_tmpdir: + return + dag.add_job(rm_job(paths, name), upstream) + + def add_variant_calling( + self, + dag: DAG, + ctx: StageContext, sr_aln: List[pathlib.Path], lr_aln: List[pathlib.Path], rg_info: RgInfo, - **_kwargs: Any, - ) -> Tuple[ - Job, # call_job - Job, # select_job - Job, # mapq0_job - Job, # mapq0_slop_job - Job, # cat_merge_job - Job, # rm_job1 - Job, # stage1_fifo_job - Job, # stage1_hap_job - Job, # stage1_job - Job, # rm_job2 - Job, # second_stage_job - Job, # rm_job3 - Job, # third_stage_job - Job, # rm_job4 - Job, # call2_job - Job, # subset_job - Job, # concat_job - Job, # rm_job5 - Job, # anno_job - Optional[List[Job]], # transfer_jobs - Optional[Job], # transfer_concat_job - Optional[Job], # apply_job - Optional[Job], # norm_job - ]: + upstream: Set[Job], + ) -> None: """ Call SNVs and indels using the DNAscope hybrid pipeline + + A first calling pass over the input alignments picks the regions + that need realignment; the three hybrid-realignment stages realign + the short reads there; a second pass recalls those regions, and the + two call sets are merged, annotated and filtered. """ - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") - ref_fai = pathlib.Path(str(self.reference) + ".fai") + ref_fai = pathlib.Path(str(ctx.reference) + ".fai") - if not self.skip_version_check: - for cmd, min_version in CALLING_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CALLING_MIN_VERSIONS, skip=self.skip_version_check) + + hybrid_model = bundle.joinpath("hybrid.model") # First pass - combined variant calling vcf_suffix = ".g.vcf.gz" if self.gvcf else ".vcf.gz" combined_vcf = self.tmp_dir.joinpath("initial" + vcf_suffix) emit_mode = "gvcf" if self.gvcf else None - driver: BaseDriver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], - input=lr_aln + sr_aln, + call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + combined_vcf, + dbsnp=self.dbsnp, + model=hybrid_model, + pcr_indel_model="none", + emit_mode=emit_mode, + ) + ], + inputs=lr_aln + sr_aln, interval=self.bed, read_filter=rg_info.ultima_read_filter + rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, - ) - driver.add_algo( - DNAscope( - combined_vcf, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("hybrid.model"), - pcr_indel_model="none", - emit_mode=emit_mode, - ) - ) - call_job = Job( - Pipeline(Command(*driver.build_cmd())), - "calling-1", - self.cores, - task_name="variant-calling", - ) + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], + name="dnascope-1", + ).add_to(dag, upstream) # Region selection hybrid_select = pathlib.Path( @@ -878,27 +912,24 @@ def call_variants( 0, task_name="region-selection", ) + dag.add_job(select_job, call.terminal) mapq0_bed = self.tmp_dir.joinpath("hybrid_mapq0.bed") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], - input=lr_aln + sr_aln, - read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, - ) - driver.add_algo( - HybridStage2( - model=self.model_bundle.joinpath("HybridStage2_region.model"), - all_bed=mapq0_bed, - ) - ) - mapq0_job = Job( - Pipeline(Command(*driver.build_cmd())), - "mapq0-bed", - self.cores, + mapq0_job = driver_job( + ctx, + [ + HybridStage2( + model=bundle.joinpath("HybridStage2_region.model"), + all_bed=mapq0_bed, + ) + ], + name="mapq0-bed", task_name="region-selection", + inputs=lr_aln + sr_aln, + read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], ) + dag.add_job(mapq0_job, upstream) mapq0_slop_bed = self.tmp_dir.joinpath("hybrid_mapq0.ex1000.bed") mapq0_slop_job = Job( @@ -912,6 +943,7 @@ def call_variants( 0, task_name="region-selection", ) + dag.add_job(mapq0_slop_job, {mapq0_job}) diff_bed = self.tmp_dir.joinpath("merged_diff.bed") cat_merge_job = Job( @@ -924,18 +956,18 @@ def call_variants( 0, task_name="region-selection", ) - rm_cmd = ["rm", str(selected_bed), str(mapq0_slop_bed)] - rm_job1 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(cat_merge_job, {mapq0_slop_job, select_job}) + self.add_cleanup( + dag, + [selected_bed, mapq0_slop_bed], "rm-tmp1", - 0, - task_name="cleanup", + {cat_merge_job}, ) stage1_ins_fa = self.tmp_dir.joinpath("stage1_ins.fa") stage1_ins_bed = self.tmp_dir.joinpath("stage1_ins.bed") ins_driver = Driver( - reference=self.reference, + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0], input=lr_aln, @@ -944,7 +976,7 @@ def call_variants( ins_driver.add_algo( HybridStage1( "-", - model=self.model_bundle.joinpath("HybridStage1_ins.model"), + model=bundle.joinpath("HybridStage1_ins.model"), fa_file=stage1_ins_fa, bed_file=stage1_ins_bed, ) @@ -963,9 +995,11 @@ def call_variants( 1, task_name="hybrid-realignment", ) + # The haplotype job writes the fifo read by the first-stage job + dag.add_job(stage1_fifo_job) stage1_driver = Driver( - reference=self.reference, + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0], input=lr_aln, @@ -975,7 +1009,7 @@ def call_variants( stage1_driver.add_algo( HybridStage1( stage1_fifo, - model=self.model_bundle.joinpath("HybridStage1.model"), + model=bundle.joinpath("HybridStage1.model"), hap_bam="-", hap_bed=stage1_hap_bed, hap_vcf=stage1_hap_vcf, @@ -996,129 +1030,113 @@ def call_variants( 0, task_name="hybrid-realignment", ) + dag.add_job(stage1_hap_job, {stage1_fifo_job, cat_merge_job}) stage1_bam = self.tmp_dir.joinpath("hybrid_stage1.bam") stage1_job = Job( cmds.hybrid_stage1( out_aln=stage1_bam, - reference=self.reference, + reference=ctx.reference, cores=self.cores, readgroup=f"@RG\\tID:hybrid-18893\\tSM:{self.hybrid_rg_sm}", ins_driver=ins_driver, hap_fastq_fifo=stage1_fifo, - bwa_model=self.model_bundle.joinpath("HybridStage1_bwa.model"), + bwa_model=bundle.joinpath("HybridStage1_bwa.model"), ), "first-stage", self.cores, task_name="hybrid-realignment", ) - rm_cmd = [ - "rm", - str(stage1_ins_fa), - str(stage1_ins_bed), - str(stage1_hap_vcf), - ] - rm_job2 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(stage1_job, {stage1_fifo_job, cat_merge_job}) + self.add_cleanup( + dag, + [stage1_ins_fa, stage1_ins_bed, stage1_hap_vcf], "rm-tmp2", - 0, - task_name="cleanup", + {stage1_job, stage1_hap_job}, ) stage2_bed = self.tmp_dir.joinpath("hybrid_stage2.bed") stage2_unmap_bam = self.tmp_dir.joinpath("hybrid_stage2_unmap.bam") stage2_alt_bam = self.tmp_dir.joinpath("hybrid_stage2_alt.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=[stage1_bam, stage1_hap_bam], - ) - driver.add_algo( - HybridStage2( - model=self.model_bundle.joinpath("HybridStage2.model"), - hap_bed=stage1_hap_bed, - unmap_bam=stage2_unmap_bam, - alt_bam=stage2_alt_bam, - all_bed=stage2_bed, - ) - ) - second_stage_job = Job( - Pipeline(Command(*driver.build_cmd())), - "second-stage", - self.cores, + second_stage_job = driver_job( + ctx, + [ + HybridStage2( + model=bundle.joinpath("HybridStage2.model"), + hap_bed=stage1_hap_bed, + unmap_bam=stage2_unmap_bam, + alt_bam=stage2_alt_bam, + all_bed=stage2_bed, + ) + ], + name="second-stage", task_name="hybrid-realignment", + inputs=[stage1_bam, stage1_hap_bam], ) - - rm_cmd = ["rm", str(stage1_bam), str(stage1_hap_bam)] - rm_job3 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(second_stage_job, {stage1_job, stage1_hap_job}) + self.add_cleanup( + dag, + [stage1_bam, stage1_hap_bam], "rm-tmp3", - 0, - task_name="cleanup", + {second_stage_job}, ) suffix = "bam" if self.bam_format else "cram" stage3_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_sr_realigned.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_sr_realigned.{suffix}") ) - driver = Driver( - reference=self.reference, + stage3_driver = Driver( + reference=ctx.reference, thread_count=self.cores, replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], input=lr_aln + sr_aln + [stage2_unmap_bam, stage2_alt_bam], interval=stage2_bed, read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, ) - driver.add_algo( + stage3_driver.add_algo( HybridStage3( "-", - model=self.model_bundle.joinpath("HybridStage3.model"), + model=bundle.joinpath("HybridStage3.model"), ) ) third_stage_job = Job( cmds.hybrid_stage3( stage3_aln, - reference=self.reference, - driver=driver, + reference=ctx.reference, + driver=stage3_driver, cores=self.cores, ), "third-stage", self.cores, task_name="hybrid-realignment", ) - rm_cmd = ["rm", str(stage2_unmap_bam), str(stage2_alt_bam)] - rm_job4 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(third_stage_job, {second_stage_job}) + self.add_cleanup( + dag, + [stage2_unmap_bam, stage2_alt_bam], "rm-tmp4", - 0, - task_name="cleanup", + {third_stage_job}, ) # pass 2 of variant calling pass2_vcf = self.tmp_dir.joinpath("hybrid_pass2.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=rg_info.replace_rg_args[0], - input=lr_aln + [stage3_aln], + call2 = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + pass2_vcf, + dbsnp=self.dbsnp, + model=hybrid_model, + pcr_indel_model="none", + emit_mode=emit_mode, + ) + ], + inputs=lr_aln + [stage3_aln], interval=stage2_bed, read_filter=rg_info.ultima_read_filter + rg_info.lr_rg_read_filter, - ) - driver.add_algo( - DNAscope( - pass2_vcf, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("hybrid.model"), - pcr_indel_model="none", - emit_mode=emit_mode, - ) - ) - call2_job = Job( - Pipeline(Command(*driver.build_cmd())), - "call2", - self.cores, - task_name="variant-calling", - ) + replace_rg=rg_info.replace_rg_args[0], + name="dnascope-2", + ).add_to(dag, {third_stage_job}) # Merge and normalize the VCFs subset_vcf = self.tmp_dir.joinpath("mix_subset.vcf.gz") @@ -1133,6 +1151,7 @@ def call_variants( 0, task_name="vcf-merge", ) + dag.add_job(subset_job, {second_stage_job}) concat_job = Job( cmds.bcftools_concat( combined_tmp_vcf, @@ -1142,12 +1161,12 @@ def call_variants( 0, task_name="vcf-merge", ) - rm_cmd = ["rm", str(combined_vcf), str(subset_vcf), str(pass2_vcf)] - rm_job5 = Job( - Pipeline(Command(*rm_cmd, fail_ok=True)), + dag.add_job(concat_job, {subset_job} | call2.terminal) + self.add_cleanup( + dag, + [combined_vcf, subset_vcf, pass2_vcf], "rm-tmp5", - 0, - task_name="cleanup", + {concat_job}, ) # Annotate the output VCF @@ -1156,7 +1175,7 @@ def call_variants( ) anno_target = self.tmp_dir.joinpath("combined_tmp_anno.vcf.gz") if self.skip_model_apply and not self.pop_vcf: - anno_target = self.output_vcf + anno_target = ctx.output_vcf anno_job = Job( cmds.cmd_pyexec_hybrid_anno( @@ -1170,113 +1189,49 @@ def call_variants( 0, task_name="annotation", ) + dag.add_job(anno_job, {concat_job}) - transfer_jobs: Optional[List[Job]] = None - transfer_concat_job: Optional[Job] = None - input_to_apply = anno_target + if not self.pop_vcf and self.skip_model_apply: + # `anno_target` is already the output VCF + return + # Transfer annotations and apply the model + transfer_spec: Optional[TransferSpec] = None if self.pop_vcf: transfer_target = self.tmp_dir.joinpath( "combined_tmp_transfer.vcf.gz" ) if self.skip_model_apply: - transfer_target = self.output_vcf - - transfer_jobs, transfer_concat_job = build_transfer_jobs( - transfer_target, - self.pop_vcf, - anno_target, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, - ) - input_to_apply = transfer_target - - if self.skip_model_apply: - return ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat_job, - None, - None, + transfer_target = ctx.output_vcf + transfer_spec = TransferSpec( + TransferConfig.from_pipeline(self), transfer_target ) - # Model Apply + apply_spec: Optional[ApplySpec] = None apply_vcf = self.tmp_dir.joinpath("combined_apply.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - interval=self.bed, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("hybrid.model"), - vcf=input_to_apply, - output=apply_vcf, - ) - ) - apply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) + if not self.skip_model_apply: + apply_spec = ApplySpec(hybrid_model, apply_vcf) + + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=anno_target, + transfer=transfer_spec, + apply=apply_spec, + ).add_to(dag, {anno_job}) + + if self.skip_model_apply: + return # Final normalize norm_job = Job( cmds.filter_norm( - self.output_vcf, + ctx.output_vcf, apply_vcf, - self.reference, + ctx.reference, exclude_homref=not self.gvcf, ), "final-norm", 0, task_name="vcf-norm", ) - return ( - call_job, - select_job, - mapq0_job, - mapq0_slop_job, - cat_merge_job, - rm_job1, - stage1_fifo_job, - stage1_hap_job, - stage1_job, - rm_job2, - second_stage_job, - rm_job3, - third_stage_job, - rm_job4, - call2_job, - subset_job, - concat_job, - rm_job5, - anno_job, - transfer_jobs, - transfer_concat_job, - apply_job, - norm_job, - ) + dag.add_job(norm_job, transfer_apply.terminal) diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index ca4aeb0..d6f081e 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -4,10 +4,8 @@ import argparse import copy -from dataclasses import dataclass import json import pathlib -import shutil import sys from typing import Dict, List, Optional, Set, Tuple @@ -19,12 +17,9 @@ from .archive import ar_load from .dag import DAG from .driver import ( - Driver, + BaseAlgo, DNAscope, DNAscopeHP, - DNAModelApply, - LongReadSV, - ReadWriter, RepeatModel, VariantPhaser, ) @@ -36,14 +31,32 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline -from .transfer import build_transfer_jobs +from .stages.alignment import ( + MINIMAP2_FASTQ_MIN_VERSIONS, + MINIMAP2_REALIGN_MIN_VERSIONS, + Minimap2FastqStage, + Minimap2RealignStage, + ReadWriterStage, + find_unzip, +) +from .stages.base import StageContext, driver_job +from .stages.metrics import MOSDEPTH_MIN_VERSIONS, MosdepthStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + TransferApplyStage, + TransferSpec, +) +from .stages.sv import LONGREADSV_MIN_VERSIONS, LongReadSVStage +from .stages.transfer import TransferConfig from .util import ( __version__, - check_version, + library_preloaded, parse_rg_line, path_arg, - library_preloaded, + require_versions, vcf_id, + versions_available, ) TOOL_MIN_VERSIONS = { @@ -52,23 +65,6 @@ "bedtools": None, } -SV_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - -ALN_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), - "samtools": packaging.version.Version("1.16"), -} - -FQ_MIN_VERSIONS = { - "sentieon driver": packaging.version.Version("202308"), -} - -MOSDEPTH_MIN_VERSIONS = { - "mosdepth": packaging.version.Version("0.2.6"), -} - MERGE_MIN_VERSIONS = { "sentieon driver": None, } @@ -82,39 +78,6 @@ } -@dataclass -class LRCallVariantsResult: - """Results from lr_call_variants""" - - first_calling_job: Job - diploid_transfer_jobs: List[Job] - diploid_concat_job: Optional[Job] - first_modelapply_job: Job - phaser_job: Job - bcftools_subset_phased_job: Optional[Job] - fai_to_bed_job: Optional[Job] - bcftools_subtract_job: Job - repeatmodel_job: Optional[Job] - bcftools_subset_unphased_job: Job - second_calling_job: Set[Job] - haploid_patch_job: Job - patch_transfer_jobs: List[List[Job]] - patch_concat_jobs: List[Job] - second_modelapply_job: Set[Job] - calling_unphased_job: Job - diploid_patch_job: Job - unphased_transfer_jobs: List[Job] - unphased_concat_job: Optional[Job] - modelapply_unphased_job: Job - merge_job: Job - gvcf_combine_job: Optional[Job] - haploid_calling_job: Optional[Job] - haploid_patch2_job: Optional[Job] - haploid_concat_job: Optional[Job] - haploid_gvcf_combine_job: Optional[Job] - haploid_gvcf_concat_job: Optional[Job] - - class DNAscopeLRPipeline(BasePipeline): """The DNAscope LongRead pipeline""" @@ -290,13 +253,8 @@ def __init__(self) -> None: self.use_pbsv = False def validate(self) -> None: - # uniquify pipeline attributes - self.lr_aln = self.sample_input - self.lr_input_ref = self.input_ref - del self.sample_input - # validate - if not (self.lr_aln or self.fastq): + if not (self.sample_input or self.fastq): self.logger.error( "Please supply either the `--sample_input` or `--fastq` and " "`--readgroups` arguments" @@ -468,8 +426,9 @@ def configure(self) -> None: def build_dag(self) -> DAG: self.logger.info("Building the DAG") dag = DAG() + ctx = self.stage_context() - sample_input = self.lr_aln + sample_input = self.sample_input realign_jobs: Set[Job] = set() if self.align: sample_input, realign_jobs = self.lr_align_inputs() @@ -500,122 +459,10 @@ def build_dag(self) -> DAG: dag.add_job(hificnv_job, {merge_job}) if not self.skip_small_variants: - cv = self.lr_call_variants(sample_input) - dag.add_job(cv.first_calling_job, realign_jobs.union(align_jobs)) - - first_ma_deps = {cv.first_calling_job} - if cv.diploid_transfer_jobs and cv.diploid_concat_job: - for job in cv.diploid_transfer_jobs: - dag.add_job(job, {cv.first_calling_job}) - dag.add_job( - cv.diploid_concat_job, set(cv.diploid_transfer_jobs) - ) - first_ma_deps.add(cv.diploid_concat_job) - - dag.add_job(cv.first_modelapply_job, first_ma_deps) - dag.add_job(cv.phaser_job, {cv.first_modelapply_job}) - - haploid_patch_deps: Set[Job] = set() - if cv.bcftools_subset_phased_job: - dag.add_job(cv.bcftools_subset_phased_job, {cv.phaser_job}) - haploid_patch_deps.add(cv.bcftools_subset_phased_job) - - subtract_deps: Set[Job] = {cv.phaser_job} - if cv.fai_to_bed_job: - dag.add_job(cv.fai_to_bed_job) - subtract_deps.add(cv.fai_to_bed_job) - dag.add_job(cv.bcftools_subtract_job, subtract_deps) - dag.add_job( - cv.bcftools_subset_unphased_job, - {cv.bcftools_subtract_job}, + self.add_small_variant_calling( + dag, ctx, sample_input, realign_jobs.union(align_jobs) ) - second_pass_deps: Set[Job] = {cv.phaser_job} - calling_unphased_deps: Set[Job] = {cv.bcftools_subtract_job} - haploid_calling_deps: Set[Job] = set() - if cv.repeatmodel_job: - dag.add_job(cv.repeatmodel_job, {cv.phaser_job}) - second_pass_deps.add(cv.repeatmodel_job) - calling_unphased_deps.add(cv.repeatmodel_job) - haploid_calling_deps.add(cv.repeatmodel_job) - - merge_deps: Set[Job] = set() - for job in cv.second_calling_job: - dag.add_job(job, second_pass_deps) - haploid_patch_deps.add(job) - dag.add_job(cv.haploid_patch_job, haploid_patch_deps) - - second_ma_deps = {cv.haploid_patch_job} - if cv.patch_transfer_jobs: # pop_vcf - for patch_transfer_hap, patch_concat in zip( - cv.patch_transfer_jobs, cv.patch_concat_jobs - ): - for job in patch_transfer_hap: - dag.add_job(job, {cv.haploid_patch_job}) - dag.add_job(patch_concat, set(patch_transfer_hap)) - second_ma_deps.add(patch_concat) - - for job in cv.second_modelapply_job: - dag.add_job(job, second_ma_deps) - merge_deps.add(job) - - dag.add_job(cv.calling_unphased_job, calling_unphased_deps) - dag.add_job( - cv.diploid_patch_job, - { - cv.bcftools_subset_unphased_job, - cv.calling_unphased_job, - }, - ) - - unphased_ma_deps = {cv.diploid_patch_job} - if cv.unphased_transfer_jobs and cv.unphased_concat_job: # pop_vcf - for job in cv.unphased_transfer_jobs: - dag.add_job(job, {cv.diploid_patch_job}) - dag.add_job( - cv.unphased_concat_job, set(cv.unphased_transfer_jobs) - ) - unphased_ma_deps.add(cv.unphased_concat_job) - - dag.add_job(cv.modelapply_unphased_job, unphased_ma_deps) - merge_deps.add(cv.modelapply_unphased_job) - dag.add_job(cv.merge_job, merge_deps) - - if cv.gvcf_combine_job: - dag.add_job(cv.gvcf_combine_job, {cv.merge_job}) - - if ( - cv.haploid_calling_job - and cv.haploid_patch2_job - and cv.haploid_concat_job - ): - dag.add_job(cv.haploid_calling_job, haploid_calling_deps) - dag.add_job( - cv.haploid_patch2_job, - {cv.haploid_calling_job}, - ) - dag.add_job( - cv.haploid_concat_job, - {cv.haploid_patch2_job, cv.merge_job}, - ) - - if ( - cv.haploid_gvcf_combine_job - and cv.haploid_gvcf_concat_job - and cv.gvcf_combine_job - ): - dag.add_job( - cv.haploid_gvcf_combine_job, - {cv.haploid_patch2_job}, - ) - dag.add_job( - cv.haploid_gvcf_concat_job, - { - cv.haploid_gvcf_combine_job, - cv.gvcf_combine_job, - }, - ) - if not self.skip_svs: longreadsv_job = self.call_svs(sample_input) dag.add_job(longreadsv_job, realign_jobs.union(align_jobs)) @@ -626,150 +473,69 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ Align reads to the reference genome using minimap2 """ - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in ALN_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - res: List[pathlib.Path] = [] - suffix = "bam" if self.bam_format else "cram" - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - realign_jobs = set() - for i, input_aln in enumerate(self.lr_aln): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" - ) - ) - rg_lines = cmds.get_rg_lines( - input_aln, - self.dry_run, - ) + output_vcf = self.required(self.output_vcf, "output_vcf") + bundle = self.required(self.model_bundle, "model_bundle") + require_versions( + MINIMAP2_REALIGN_MIN_VERSIONS, skip=self.skip_version_check + ) - realign_jobs.add( - Job( - cmds.cmd_samtools_fastq_minimap2( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_lines, - sample_name, - self.lr_input_ref, - self.fastq_taglist, - self.minimap2_args, - self.util_sort_args, - ), - f"bam-realign-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, realign_jobs) + result = Minimap2RealignStage( + ctx=self.stage_context(), + inputs=self.sample_input, + model_bundle=bundle, + sample_name=output_vcf.name.replace(".vcf.gz", ""), + bam_format=self.bam_format, + input_ref=self.input_ref, + fastq_taglist=self.fastq_taglist, + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).build() + return (result.outputs, set(result.jobs)) def lr_align_fastq(self) -> Tuple[List[pathlib.Path], Set[Job]]: """ Align fastq to the reference genome using minimap2 """ - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - res: List[pathlib.Path] = [] + bundle = self.required(self.model_bundle, "model_bundle") if self.fastq is None and self.readgroups is None: - return (res, set()) + return ([], set()) - if not self.skip_version_check: - for cmd, min_version in FQ_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions( + MINIMAP2_FASTQ_MIN_VERSIONS, skip=self.skip_version_check + ) - unzip = "igzip" - if not shutil.which(unzip): - self.logger.warning( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - - suffix = "bam" if self.bam_format else "cram" - align_jobs = set() - for i, (fq, rg) in enumerate(zip(self.fastq, self.readgroups)): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_fq_{i}.{suffix}" - ) - ) - align_jobs.add( - Job( - cmds.cmd_fastq_minimap2( - out_aln, - fq, - rg, - self.reference, - self.model_bundle, - self.cores, - unzip, - self.minimap2_args, - self.util_sort_args, - ), - f"align-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, align_jobs) + result = Minimap2FastqStage( + ctx=self.stage_context(), + fastq=self.fastq, + readgroups=self.readgroups, + model_bundle=bundle, + bam_format=self.bam_format, + unzip=find_unzip(self.logger), + minimap2_args=self.minimap2_args, + util_sort_args=self.util_sort_args, + ).build() + return (result.outputs, set(result.jobs)) def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: """Run mosdepth for QC""" - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MOSDEPTH_MIN_VERSIONS.items() - ] - ): - self.logger.warning( - "Skipping mosdepth. mosdepth version %s or later " - "not found", - MOSDEPTH_MIN_VERSIONS["mosdepth"], - ) - return set() - - mosdepth_jobs = set() - for i, input_file in enumerate(sample_input): - mosdepth_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_mosdepth_{i}") + if not versions_available( + MOSDEPTH_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping mosdepth. mosdepth version %s or later not found", + MOSDEPTH_MIN_VERSIONS["mosdepth"], ) - mosdepth_jobs.add( - Job( - cmds.cmd_mosdepth( - input_file, - mosdepth_dir, - fasta=self.reference, - threads=self.cores, - ), - f"mosdepth-{i}", - 0, # Run in background - task_name="metrics", - ) + return set() + + return set( + MosdepthStage( + ctx=self.stage_context(), + inputs=sample_input, ) - return mosdepth_jobs + .build() + .jobs + ) def merge_input_files( self, sample_input: List[pathlib.Path] @@ -777,26 +543,19 @@ def merge_input_files( """ Merge the aligned reads into a single BAM """ - if not self.skip_version_check: - for cmd, min_version in MERGE_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(MERGE_MIN_VERSIONS, skip=self.skip_version_check) # Merge the sample_input into a single BAM merged_bam = self.tmp_dir.joinpath("longread_merged.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - ) - driver.add_algo(ReadWriter(merged_bam)) - merge_job = Job( - Pipeline(Command(*driver.build_cmd())), - "merge-bam", - 0, + result = ReadWriterStage( + ctx=self.stage_context(), + inputs=sample_input, + output=merged_bam, + name="merge-bam", task_name="alignment", - ) - return (merged_bam, merge_job) + threads=0, # Run in the background + ).build() + return (merged_bam, result.jobs[0]) def pbsv( self, @@ -805,10 +564,7 @@ def pbsv( """ Call SVs with pbsv """ - if not self.skip_version_check: - for cmd, min_version in PBSV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(PBSV_MIN_VERSIONS, skip=self.skip_version_check) # pbsv discover pbsv_discover_fn = str(self.output_vcf).replace( @@ -856,15 +612,14 @@ def hificnv( """ Call CNVs with HiFiCNV """ - if not self.skip_version_check: - for cmd, min_version in HIFICNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - self.logger.warning( - "Skipping hificnv. hificnv version %s or later not " - "found", - HIFICNV_MIN_VERSIONS["hificnv"], - ) - return None + if not versions_available( + HIFICNV_MIN_VERSIONS, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping hificnv. hificnv version %s or later not found", + HIFICNV_MIN_VERSIONS["hificnv"], + ) + return None hifi_cnv_fn = str(self.output_vcf).replace(".vcf.gz", ".hificnv") hificnv_cmd = [ @@ -888,102 +643,72 @@ def hificnv( ) return hificnv_job - def lr_call_variants( + def add_small_variant_calling( self, + dag: DAG, + ctx: StageContext, sample_input: List[pathlib.Path], - ) -> LRCallVariantsResult: + upstream: Set[Job], + ) -> None: """ Call SNVs and indels using the DNAscope LongRead pipeline + + A diploid first pass phases the reads; the phased regions are then + called once per haplotype and the unphased regions are called with + the diploid HP model. The three call sets are patched, filtered and + merged into the output VCF, with optional haploid calling on top. """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.skip_version_check: - for check_cmd, min_version in TOOL_MIN_VERSIONS.items(): - if not check_version(check_cmd, min_version): - sys.exit(2) - - transfer_xargs = ( - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, - ) + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(TOOL_MIN_VERSIONS, skip=self.skip_version_check) # First pass - diploid calling diploid_gvcf_fn = self.tmp_dir.joinpath("out_diploid.g.vcf.gz") diploid_tmp_vcf = self.tmp_dir.joinpath("out_diploid_tmp.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.bed, - ) + diploid_model = bundle.joinpath("diploid_model") + diploid_algos: List[BaseAlgo] = [] if self.gvcf: - driver.add_algo( + diploid_algos.append( DNAscope( diploid_gvcf_fn, dbsnp=self.dbsnp, emit_mode="gvcf", - model=self.model_bundle.joinpath("gvcf_model"), + model=bundle.joinpath("gvcf_model"), ) ) - driver.add_algo( + diploid_algos.append( DNAscope( diploid_tmp_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("diploid_model"), + model=diploid_model, ) ) - first_calling_job = Job( - Pipeline(Command(*driver.build_cmd())), - "first-pass", - self.cores, - task_name="variant-calling", - ) - - # Transfer annotations to the tmp vcf - diploid_transfer_jobs: List[Job] = [] - diploid_concat_job: Optional[Job] = None - if self.pop_vcf: - diploid_unanno_tmp_vcf = diploid_tmp_vcf - diploid_tmp_vcf = self.tmp_dir.joinpath( - "out_diploid_tmp_anno.vcf.gz" - ) - diploid_transfer_jobs, diploid_concat_job = build_transfer_jobs( - diploid_tmp_vcf, - self.pop_vcf, - diploid_unanno_tmp_vcf, - *transfer_xargs, - ) + first_call = DNAscopeStage( + ctx=ctx, + algos=diploid_algos, + inputs=sample_input, + interval=self.bed, + name="dnascope-diploid", + ).add_to(dag, upstream) - # DNAModelApply on the diploid VCF + # Transfer annotations to the tmp vcf, then apply the diploid model diploid_vcf = self.tmp_dir.joinpath("out_diploid.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("diploid_model"), - diploid_tmp_vcf, - diploid_vcf, + diploid_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + diploid_transfer = TransferSpec( + config=TransferConfig.from_pipeline(self), + out_vcf=self.tmp_dir.joinpath("out_diploid_tmp_anno.vcf.gz"), + tag="diploid", ) - ) - first_modelapply_job = Job( - Pipeline(Command(*driver.build_cmd())), - "first-modelapply", - self.cores, - task_name="model-apply", - ) + first_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=diploid_tmp_vcf, + transfer=diploid_transfer, + apply=ApplySpec( + model=diploid_model, + output=diploid_vcf, + name="model-apply-diploid", + ), + ).add_to(dag, first_call.terminal) # Phasing and RepeatModel phased_bed = self.tmp_dir.joinpath("out_diploid_phased.bed") @@ -996,28 +721,24 @@ def lr_call_variants( phased_phased = self.tmp_dir.joinpath( "out_diploid_phased_phased.vcf.gz" ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.bed, - ) - driver.add_algo( - VariantPhaser( - diploid_vcf, - phased_vcf, - out_bed=phased_bed, - out_ext=phased_ext, - ) - ) - phaser_job = Job( - Pipeline(Command(*driver.build_cmd())), - "variantphaser", - self.cores, + phaser_job = driver_job( + ctx, + [ + VariantPhaser( + diploid_vcf, + phased_vcf, + out_bed=phased_bed, + out_ext=phased_ext, + ) + ], + name="variantphaser", task_name="phasing", + inputs=sample_input, + interval=self.bed, ) + dag.add_job(phaser_job, first_apply.terminal) - bcftools_subset_phased_job = None + haploid_patch_deps: Set[Job] = set() if self.tech.upper() == "ONT": bcftools_subset_phased_job = Job( Pipeline( @@ -1040,21 +761,25 @@ def lr_call_variants( 0, task_name="phasing", ) + dag.add_job(bcftools_subset_phased_job, {phaser_job}) + haploid_patch_deps.add(bcftools_subset_phased_job) - fai_to_bed_job = None + subtract_deps: Set[Job] = {phaser_job} if self.bed: bed = self.bed else: bed = self.tmp_dir.joinpath("reference.bed") fai_to_bed_job = Job( cmds.cmd_fai_to_bed( - pathlib.Path(str(self.reference) + ".fai"), + pathlib.Path(str(ctx.reference) + ".fai"), bed, ), "fai-to-bed", 0, task_name="phasing", ) + dag.add_job(fai_to_bed_job) + subtract_deps.add(fai_to_bed_job) bcftools_subtract_job = Job( cmds.cmd_bedtools_subtract(bed, phased_bed, unphased_bed), @@ -1062,33 +787,7 @@ def lr_call_variants( 0, task_name="phasing", ) - - repeatmodel_job = None - if not self.repeat_model: - self.repeat_model = self.tmp_dir.joinpath("out_repeat.model") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=phased_bed, - read_filter=[ - f"PhasedReadFilter,phased_vcf={phased_ext}," - "phase_select=tag" - ], # noqa: E501 - ) - driver.add_algo( - RepeatModel( - self.repeat_model, - phased=True, - read_flag_mask="drop=supplementary", - ) - ) - repeatmodel_job = Job( - Pipeline(Command(*driver.build_cmd())), - "repeatmodel", - self.cores, - task_name="repeat-model", - ) + dag.add_job(bcftools_subtract_job, subtract_deps) bcftools_subset_unphased_job = Job( Pipeline( @@ -1107,50 +806,70 @@ def lr_call_variants( 0, task_name="phasing", ) + dag.add_job(bcftools_subset_unphased_job, {bcftools_subtract_job}) + + repeat_model_deps: Set[Job] = set() + if not self.repeat_model: + self.repeat_model = self.tmp_dir.joinpath("out_repeat.model") + repeatmodel_job = driver_job( + ctx, + [ + RepeatModel( + self.repeat_model, + phased=True, + read_flag_mask="drop=supplementary", + ) + ], + name="repeatmodel", + task_name="repeat-model", + inputs=sample_input, + interval=phased_bed, + read_filter=[ + f"PhasedReadFilter,phased_vcf={phased_ext}," + "phase_select=tag" + ], # noqa: E501 + ) + dag.add_job(repeatmodel_job, {phaser_job}) + repeat_model_deps.add(repeatmodel_job) # Second pass - phased variants - second_calling_job = set() + haploid_model = bundle.joinpath("haploid_model") + haploid_hp_model = bundle.joinpath("haploid_hp_model") for phase in (1, 2): hp_std_vcf = self.tmp_dir.joinpath( f"out_hap{phase}_nohp_tmp.vcf.gz" ) hp_vcf = self.tmp_dir.joinpath(f"out_hap{phase}_tmp.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=phased_bed, - read_filter=[ - f"PhasedReadFilter,phased_vcf={phased_ext}" - f",phase_select={phase}" - ], - ) - + phase_algos: List[BaseAlgo] = [] if self.tech.upper() == "HIFI": # ONT doesn't do DNAscope in 2nd pass. - driver.add_algo( + phase_algos.append( DNAscope( hp_std_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_model"), + model=haploid_model, ) ) - driver.add_algo( + phase_algos.append( DNAscopeHP( hp_vcf, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_hp_model"), + model=haploid_hp_model, pcr_indel_model=self.repeat_model, ) ) - second_calling_job.add( - Job( - Pipeline(Command(*driver.build_cmd())), - "second-pass", - self.cores, - task_name="variant-calling", - ) - ) + hap_call = DNAscopeStage( + ctx=ctx, + algos=phase_algos, + inputs=sample_input, + interval=phased_bed, + read_filter=[ + f"PhasedReadFilter,phased_vcf={phased_ext}" + f",phase_select={phase}" + ], + name=f"dnascope-hap{phase}", + ).add_to(dag, {phaser_job} | repeat_model_deps) + haploid_patch_deps.update(hap_call.terminal) kwargs: Dict[str, str] = dict() kwargs["gvcf_combine_py"] = str( @@ -1177,133 +896,106 @@ def lr_call_variants( self.cores, task_name="variant-patch", ) + dag.add_job(haploid_patch_job, haploid_patch_deps) - # Transfer annotations to the patched VCFs - patch_transfer_jobs = [] - patch_concat_jobs = [] - if self.pop_vcf: - patch_unanno_vcfs = patch_vcfs.copy() - patch_vcfs = [ - self.tmp_dir.joinpath(f"out_hap{i}_patch_anno.vcf.gz") - for i in (1, 2) - ] - for patch_unanno_vcf, patch_vcf in zip( - patch_unanno_vcfs, patch_vcfs - ): - transfer_jobs, concat_job = build_transfer_jobs( - patch_vcf, - self.pop_vcf, - patch_unanno_vcf, - *transfer_xargs, - ) - patch_transfer_jobs.append(transfer_jobs) - patch_concat_jobs.append(concat_job) - - # apply trained model to the patched vcfs. + # Transfer annotations to each patched VCF, then apply the trained + # model. A haplotype's model-apply reads only that haplotype's + # VCF, so it waits on that haplotype's transfer alone. hap_vcfs = [ self.tmp_dir.joinpath(f"out_hap{i}.vcf.gz") for i in (1, 2) ] - second_modelapply_job = set() - for patch_vcf, hap_vcf in zip(patch_vcfs, hap_vcfs): - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("haploid_model"), - patch_vcf, - hap_vcf, - ) - ) - second_modelapply_job.add( - Job( - Pipeline(Command(*driver.build_cmd())), - "second-modelapply", - self.cores, - task_name="model-apply", + merge_deps: Set[Job] = set() + for i, (patch_vcf, hap_vcf) in enumerate( + zip(patch_vcfs, hap_vcfs), start=1 + ): + patch_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + patch_transfer = TransferSpec( + config=TransferConfig.from_pipeline(self), + out_vcf=self.tmp_dir.joinpath( + f"out_hap{i}_patch_anno.vcf.gz" + ), + tag=f"hap{i}", ) - ) + hap_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=patch_vcf, + transfer=patch_transfer, + apply=ApplySpec( + model=haploid_model, + output=hap_vcf, + name=f"model-apply-hap{i}", + ), + ).add_to(dag, {haploid_patch_job}) + merge_deps.update(hap_apply.terminal) # Second pass - unphased regions diploid_unphased_hp = self.tmp_dir.joinpath( "out_diploid_phased_unphased_hp.vcf.gz" ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, + unphased_call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscopeHP( + diploid_unphased_hp, + dbsnp=self.dbsnp, + model=bundle.joinpath("diploid_hp_model"), + pcr_indel_model=self.repeat_model, + ) + ], + inputs=sample_input, interval=unphased_bed, - ) - driver.add_algo( - DNAscopeHP( - diploid_unphased_hp, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("diploid_hp_model"), - pcr_indel_model=self.repeat_model, - ) - ) - calling_unphased_job = Job( - Pipeline(Command(*driver.build_cmd())), - "calling-unphased", - self.cores, - task_name="variant-calling", - ) + name="dnascope-unphased", + ).add_to(dag, {bcftools_subtract_job} | repeat_model_deps) # Patch DNA and DNAHP variants diploid_unphased_patch = self.tmp_dir.joinpath( "out_diploid_unphased_patch.vcf.gz" ) - cmd = cmds.cmd_pyexec_vcf_mod_patch( - str(diploid_unphased_patch), - str(phased_unphased), - str(diploid_unphased_hp), + diploid_patch_job = Job( + cmds.cmd_pyexec_vcf_mod_patch( + str(diploid_unphased_patch), + str(phased_unphased), + str(diploid_unphased_hp), + self.cores, + kwargs, + ), + "diploid-patch", self.cores, - kwargs, + task_name="variant-patch", ) - diploid_patch_job = Job( - cmd, "diploid-patch", self.cores, task_name="variant-patch" + dag.add_job( + diploid_patch_job, + {bcftools_subset_unphased_job} | unphased_call.terminal, ) - # Transfer annotations to the diploid VCF - unphased_transfer_jobs: List[Job] = [] - unphased_concat_job: Optional[Job] = None - if self.pop_vcf: - diploid_unphased_unanno = diploid_unphased_patch - diploid_unphased_patch = self.tmp_dir.joinpath( - "out_diploid_unphased_patch_anno.vcf.gz" - ) - unphased_transfer_jobs, unphased_concat_job = build_transfer_jobs( - diploid_unphased_patch, - self.pop_vcf, - diploid_unphased_unanno, - *transfer_xargs, - ) - - # DNAModelApply on the unphased VCF + # Transfer annotations to the diploid VCF, then apply the model diploid_unphased = self.tmp_dir.joinpath("out_diploid_unphased.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - self.model_bundle.joinpath("diploid_model_unphased"), - diploid_unphased_patch, - diploid_unphased, + unphased_transfer: Optional[TransferSpec] = None + if self.pop_vcf: + unphased_transfer = TransferSpec( + config=TransferConfig.from_pipeline(self), + out_vcf=self.tmp_dir.joinpath( + "out_diploid_unphased_patch_anno.vcf.gz" + ), + tag="unphased", ) - ) - modelapply_unphased_job = Job( - Pipeline(Command(*driver.build_cmd())), - "modelapply-unphased", - self.cores, - task_name="model-apply", - ) + unphased_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=diploid_unphased_patch, + transfer=unphased_transfer, + apply=ApplySpec( + model=bundle.joinpath("diploid_model_unphased"), + output=diploid_unphased, + name="model-apply-unphased", + ), + ).add_to(dag, {diploid_patch_job}) + merge_deps.update(unphased_apply.terminal) # merge calls to create the output diploid_merged_vcf = self.tmp_dir.joinpath("out_diploid_merged.vcf.gz") merge_out_vcf = ( - diploid_merged_vcf if self.haploid_bed else self.output_vcf + diploid_merged_vcf if self.haploid_bed else ctx.output_vcf ) merge_job = Job( cmds.cmd_pyexec_vcf_mod_merge( @@ -1320,12 +1012,13 @@ def lr_call_variants( self.cores, task_name="vcf-merge", ) + dag.add_job(merge_job, merge_deps) gvcf_combine_job = None if self.gvcf: gvcf_combine_job = Job( cmds.cmd_pyexec_gvcf_combine( - self.reference, + ctx.reference, str(diploid_gvcf_fn), str(merge_out_vcf), self.cores, @@ -1335,175 +1028,127 @@ def lr_call_variants( 0, task_name="gvcf", ) + dag.add_job(gvcf_combine_job, {merge_job}) - haploid_calling_job = None - haploid_patch2_job = None - haploid_concat_job = None - haploid_gvcf_combine_job = None - haploid_gvcf_concat_job = None - if self.haploid_bed: - # Haploid variant calling - haploid_fn = self.tmp_dir.joinpath("haploid.vcf.gz") - haploid_gvcf_fn = self.tmp_dir.joinpath("haploid.g.vcf.gz") - haploid_hp_fn = self.tmp_dir.joinpath("haploid_hp.vcf.gz") - haploid_out_fn = self.tmp_dir.joinpath("haploid_patched.vcf.gz") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sample_input, - interval=self.haploid_bed, - ) - driver.add_algo( + if not self.haploid_bed: + return + + # Haploid variant calling. This job depends only on the repeat + # model, not on the alignment jobs. + haploid_fn = self.tmp_dir.joinpath("haploid.vcf.gz") + haploid_gvcf_fn = self.tmp_dir.joinpath("haploid.g.vcf.gz") + haploid_hp_fn = self.tmp_dir.joinpath("haploid_hp.vcf.gz") + haploid_out_fn = self.tmp_dir.joinpath("haploid_patched.vcf.gz") + haploid_algos: List[BaseAlgo] = [ + DNAscope( + haploid_fn, + dbsnp=self.dbsnp, + model=haploid_model, + ), + DNAscopeHP( + haploid_hp_fn, + dbsnp=self.dbsnp, + model=haploid_hp_model, + pcr_indel_model=self.repeat_model, + ), + ] + if self.gvcf: + haploid_algos.append( DNAscope( - haploid_fn, - dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_model"), - ) - ) - driver.add_algo( - DNAscopeHP( - haploid_hp_fn, + haploid_gvcf_fn, dbsnp=self.dbsnp, - model=self.model_bundle.joinpath("haploid_hp_model"), - pcr_indel_model=self.repeat_model, + emit_mode="gvcf", + ploidy=1, + model=bundle.joinpath("gvcf_model"), ) ) - if self.gvcf: - driver.add_algo( - DNAscope( - haploid_gvcf_fn, - dbsnp=self.dbsnp, - emit_mode="gvcf", - ploidy=1, - model=self.model_bundle.joinpath("gvcf_model"), - ) - ) - haploid_calling_job = Job( - Pipeline(Command(*driver.build_cmd())), - "haploid-calling", + haploid_call = DNAscopeStage( + ctx=ctx, + algos=haploid_algos, + inputs=sample_input, + interval=self.haploid_bed, + name="dnascope-haploid", + ).add_to(dag, repeat_model_deps) + + haploid_patch2_job = Job( + cmds.cmd_pyexec_vcf_mod_haploid_patch2( + str(haploid_out_fn), + str(haploid_fn), + str(haploid_hp_fn), self.cores, - task_name="variant-calling", - ) + kwargs, + ), + "haploid-patch2", + self.cores, + task_name="variant-patch", + ) + dag.add_job(haploid_patch2_job, haploid_call.terminal) - haploid_patch2_job = Job( - cmds.cmd_pyexec_vcf_mod_haploid_patch2( - str(haploid_out_fn), - str(haploid_fn), - str(haploid_hp_fn), - self.cores, - kwargs, - ), - "haploid-patch2", + haploid_concat_job = Job( + cmds.bcftools_concat( + ctx.output_vcf, + [diploid_merged_vcf, haploid_out_fn], + ), + "haploid-diploid-concat", + 0, + task_name="vcf-merge", + ) + dag.add_job(haploid_concat_job, {haploid_patch2_job, merge_job}) + + if not self.gvcf or gvcf_combine_job is None: + return + + output_gvcf = pathlib.Path( + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + ) + diploid_gvcf = pathlib.Path( + str(diploid_merged_vcf).replace(".vcf.gz", ".g.vcf.gz") + ) + haploid_gvcf = pathlib.Path( + str(haploid_out_fn).replace(".vcf.gz", ".g.vcf.gz") + ) + haploid_gvcf_combine_job = Job( + cmds.cmd_pyexec_gvcf_combine( + ctx.reference, + str(haploid_gvcf_fn), + str(haploid_out_fn), self.cores, - task_name="variant-patch", - ) - haploid_concat_job = Job( - cmds.bcftools_concat( - self.output_vcf, - [diploid_merged_vcf, haploid_out_fn], - ), - "haploid-diploid-concat", - 0, - task_name="vcf-merge", - ) + kwargs, + ), + "haploid-gvcf-combine", + 0, + task_name="gvcf", + ) + dag.add_job(haploid_gvcf_combine_job, {haploid_patch2_job}) - if self.gvcf: - output_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") - ) - diploid_gvcf = pathlib.Path( - str(diploid_merged_vcf).replace(".vcf.gz", ".g.vcf.gz") - ) - haploid_gvcf = pathlib.Path( - str(haploid_out_fn).replace(".vcf.gz", ".g.vcf.gz") - ) - haploid_gvcf_combine_job = Job( - cmds.cmd_pyexec_gvcf_combine( - self.reference, - str(haploid_gvcf_fn), - str(haploid_out_fn), - self.cores, - kwargs, - ), - "haploid-gvcf-combine", - 0, - task_name="gvcf", - ) - haploid_gvcf_concat_job = Job( - cmds.bcftools_concat( - output_gvcf, - [diploid_gvcf, haploid_gvcf], - ), - "haploid-gvcf-concat", - 0, - task_name="gvcf", - ) - return LRCallVariantsResult( - first_calling_job, - diploid_transfer_jobs, - diploid_concat_job, - first_modelapply_job, - phaser_job, - bcftools_subset_phased_job, - fai_to_bed_job, - bcftools_subtract_job, - repeatmodel_job, - bcftools_subset_unphased_job, - second_calling_job, - haploid_patch_job, - patch_transfer_jobs, - patch_concat_jobs, - second_modelapply_job, - calling_unphased_job, - diploid_patch_job, - unphased_transfer_jobs, - unphased_concat_job, - modelapply_unphased_job, - merge_job, - gvcf_combine_job, - haploid_calling_job, - haploid_patch2_job, - haploid_concat_job, - haploid_gvcf_combine_job, + haploid_gvcf_concat_job = Job( + cmds.bcftools_concat( + output_gvcf, + [diploid_gvcf, haploid_gvcf], + ), + "haploid-gvcf-concat", + 0, + task_name="gvcf", + ) + dag.add_job( haploid_gvcf_concat_job, + {haploid_gvcf_combine_job, gvcf_combine_job}, ) - def call_svs( - self, - sample_input: List[pathlib.Path], - replace_rg: Optional[List[List[str]]] = None, - ) -> Job: + def call_svs(self, sample_input: List[pathlib.Path]) -> Job: """ Call SVs using Sentieon LongReadSV """ - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in SV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") - ) - driver = Driver( - reference=self.reference, - thread_count=self.cores, - replace_rg=replace_rg, - input=sample_input, - interval=self.bed, - ) - driver.add_algo( - LongReadSV( - sv_vcf, - self.model_bundle.joinpath("longreadsv.model"), + bundle = self.required(self.model_bundle, "model_bundle") + require_versions(LONGREADSV_MIN_VERSIONS, skip=self.skip_version_check) + + return ( + LongReadSVStage( + ctx=self.stage_context(), + inputs=sample_input, + model=bundle.joinpath("longreadsv.model"), + interval=self.bed, ) + .build() + .job ) - longreadsv_job = Job( - Pipeline(Command(*driver.build_cmd())), - "LongReadSV", - self.cores, - task_name="sv-calling", - ) - return longreadsv_job diff --git a/sentieon_cli/driver.py b/sentieon_cli/driver.py index 4574df8..e442a89 100644 --- a/sentieon_cli/driver.py +++ b/sentieon_cli/driver.py @@ -575,7 +575,7 @@ def __init__( reference: Optional[pathlib.Path] = None, thread_count: Optional[int] = None, interval: Optional[Union[pathlib.Path, str]] = None, - interval_padding: int = 0, + interval_padding: Optional[int] = None, read_filter: Optional[List[str]] = None, replace_rg: Optional[List[List[str]]] = None, input: Optional[List[pathlib.Path]] = None, diff --git a/sentieon_cli/hybrid_pangenome.py b/sentieon_cli/hybrid_pangenome.py index 0f33b8e..6649837 100644 --- a/sentieon_cli/hybrid_pangenome.py +++ b/sentieon_cli/hybrid_pangenome.py @@ -9,7 +9,6 @@ import copy import json import pathlib -import shutil import sys from typing import Dict, List, Optional, Set, Tuple @@ -20,10 +19,8 @@ from .base_pangenome import BasePangenome from .dag import DAG from .driver import ( - DNAModelApply, DNAscope, Driver, - LongReadSV, PangenomeSV, PGHapUpdateAlgo, ) @@ -36,13 +33,28 @@ vcf_contigs, ) from .shell_pipeline import Command, Pipeline -from .transfer import build_transfer_jobs +from .stages.alignment import ( + BwaExtractStage, + Minimap2RealignStage, + find_unzip, +) +from .stages.base import StageContext +from .stages.dedup import DedupStage +from .stages.metrics import MetricsPaths, MetricsStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + TransferApplyStage, + TransferSpec, +) +from .stages.sv import LongReadSVStage +from .stages.transfer import TransferConfig from .util import ( __version__, check_kmc_patch, - check_version, parse_rg_line, path_arg, + require_versions, total_memory, vcf_id, ) @@ -250,19 +262,18 @@ def validate(self) -> None: self.collect_readgroups() self.validate_readgroups() - if not self.skip_version_check: - for cmd, min_version in HYBRID_PANGENOME_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions( + HYBRID_PANGENOME_MIN_VERSIONS, skip=self.skip_version_check + ) - if not check_kmc_patch("kmc"): - self.logger.error( - "Error: The 'kmc' executable in the PATH does not " - "support reading from stdin. Please ensure " - "you are using the patched version of KMC from " - "https://github.com/Sentieon/KMC/releases." - ) - sys.exit(2) + if not self.skip_version_check and not check_kmc_patch("kmc"): + self.logger.error( + "Error: The 'kmc' executable in the PATH does not " + "support reading from stdin. Please ensure " + "you are using the patched version of KMC from " + "https://github.com/Sentieon/KMC/releases." + ) + sys.exit(2) if self.bed is None: self.logger.info( @@ -589,47 +600,27 @@ def configure(self) -> None: """Configure pipeline parameters""" pass - def find_unzip(self) -> str: - """The decompression tool for fastq input""" - unzip = "igzip" - if not shutil.which(unzip): - self.logger.info( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" - return unzip - def build_dag(self) -> DAG: """Build the DAG for the hybrid-pangenome pipeline""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) - if not self.pop_vcf: - self.logger.error("pop_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + self.required(self.pop_vcf, "pop_vcf") self.logger.info("Building the hybrid-pangenome DAG") dag = DAG() + ctx = self.stage_context() - ref_fai = pathlib.Path(str(self.reference) + ".fai") + ref_fai = pathlib.Path(str(ctx.reference) + ".fai") # Output files suffix = "bam" if self.bam_format else "cram" out_bwa_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") ) out_lift_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_lift_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_lift_deduped.{suffix}") ) sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") ) # Intermediate file paths @@ -654,9 +645,9 @@ def build_dag(self) -> DAG: calling_lr = list(self.lr_aln) realign_jobs: Set[Job] = set() if self.lr_align_input: - calling_lr, realign_jobs = self.lr_align_inputs() - for job in realign_jobs: - dag.add_job(job) + lr_result = self.lr_realign_stage(ctx).add_to(dag) + calling_lr = lr_result.outputs + realign_jobs = set(lr_result.jobs) haplotype_dependencies: Set[Job] = set() mm2_dependencies: Set[Job] = set() @@ -667,8 +658,10 @@ def build_dag(self) -> DAG: haplotype_dependencies.add(kmc_job) # BWA alignment and extraction - bwa_job = self.build_alignment_job(bwa_bam, ext_fastq) - dag.add_job(bwa_job) + bwa_result = self.bwa_extract_stage( + ctx, bwa_bam, ext_fastq + ).add_to(dag) + bwa_job = bwa_result.jobs[0] mm2_dependencies.add(bwa_job) # Do not run vg-haplotypes with bwa in low-mem environments total_mem_gb = total_memory() / (1024.0**3) @@ -710,15 +703,22 @@ def build_dag(self) -> DAG: dag.add_job(update_raw_job, {gfa_job} | realign_jobs) # Call SVs from the long reads and collect graph update regions - longreadsv_job = self.build_longreadsv_job(longread_sv_vcf, calling_lr) - dag.add_job(longreadsv_job, realign_jobs) + longreadsv_result = LongReadSVStage( + ctx=ctx, + inputs=list(calling_lr), + model=bundle.joinpath("longreadsv.model"), + output=longread_sv_vcf, + min_sv_size=LONGREADSV_MIN_SV_SIZE, + name="longreadsv", + task_name="pangenome-update", + ).add_to(dag, realign_jobs) sv_bed_job = Job( cmds.cmd_longread_sv_bed(sv_bed, longread_sv_vcf, ref_fai), "longread-sv-bed", 0, task_name="pangenome-update", ) - dag.add_job(sv_bed_job, {longreadsv_job}) + dag.add_job(sv_bed_job, {longreadsv_result.job}) # Graph update with the SV BED update_job = self.build_graph_update_job( @@ -762,42 +762,42 @@ def build_dag(self) -> DAG: # is assumed to be deduplicated already. # Emit Dedup metrics for the primary (bwa) short-read alignment # so they land in the metrics directory scanned by MultiQC. + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - dedup_metrics = metrics_dir.joinpath( - sample_name + ".txt.dedup_metrics.txt" - ) - - bwa_lc_job, bwa_dedup_job = self.build_dedup_job( - out_bwa_aln, [bwa_bam], "bwa", metrics=dedup_metrics - ) - lift_lc_job, lift_dedup_job = self.build_dedup_job( - out_lift_aln, - [lift_bam], - "lift", - ) - dag.add_job(bwa_lc_job, {bwa_job}) - dag.add_job(bwa_dedup_job, {bwa_lc_job}) - dag.add_job(lift_lc_job, {mm2_job}) - dag.add_job(lift_dedup_job, {lift_lc_job}) + paths.ensure_dir(self.dry_run) + dedup_metrics = paths.dedup_metrics + + bwa_dedup = DedupStage( + ctx=ctx, + tag="bwa", + inputs=[bwa_bam], + output=out_bwa_aln, + score_file=self.tmp_dir.joinpath("sample-bwa-score.txt.gz"), + dedup_metrics=dedup_metrics, + ).add_to(dag, {bwa_job}) + bwa_dedup_job = bwa_dedup.dedup_job + lift_dedup = DedupStage( + ctx=ctx, + tag="lift", + inputs=[lift_bam], + output=out_lift_aln, + score_file=self.tmp_dir.joinpath("sample-lift-score.txt.gz"), + ).add_to(dag, {mm2_job}) + lift_dedup_job = lift_dedup.dedup_job # Alignment metrics from the deduplicated short reads if not self.skip_metrics: - metrics_job, rehead_job = self.build_metrics_job( - [out_bwa_aln, out_lift_aln], - ) - dag.add_job(metrics_job, {bwa_dedup_job, lift_dedup_job}) - dag.add_job(rehead_job, {metrics_job}) + metrics_result = MetricsStage( + ctx=ctx, + inputs=[out_bwa_aln, out_lift_aln], + algos=self.pangenome_metrics_algos(paths), + rehead_metrics=paths.wgs, + ).add_to(dag, {bwa_dedup_job, lift_dedup_job}) if not self.skip_multiqc: multiqc_job = self.multiqc() if multiqc_job: - dag.add_job(multiqc_job, {rehead_job}) + dag.add_job(multiqc_job, metrics_result.terminal) calling_bams = [out_bwa_aln, out_lift_aln] + calling_lr calling_dependencies = {bwa_dedup_job, lift_dedup_job} @@ -822,37 +822,39 @@ def build_dag(self) -> DAG: if self.skip_small_variants: return dag - dnascope_job = self.build_dnascope_job( - raw_vcf, - calling_bams, - replace_rg, - ) - dag.add_job(dnascope_job, calling_dependencies) + model = bundle.joinpath("dnascope.model") + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" + call = DNAscopeStage( + ctx=ctx, + algos=[ + DNAscope( + raw_vcf, + model=model, + pcr_indel_model=pcr_indel_model, + dbsnp=self.dbsnp, + ) + ], + inputs=calling_bams, + interval=self.bed, + replace_rg=replace_rg, + ).add_to(dag, calling_dependencies) - # transfer annotations from the pop_vcf + # transfer annotations from the pop_vcf, then apply the model transfer_target = ( - transfer_vcf if not self.skip_model_apply else self.output_vcf - ) - transfer_jobs, concat_job = build_transfer_jobs( - transfer_target, - self.pop_vcf, - raw_vcf, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, + transfer_vcf if not self.skip_model_apply else ctx.output_vcf ) - for job in transfer_jobs: - dag.add_job(job, {dnascope_job}) - dag.add_job(concat_job, set(transfer_jobs)) - - if not self.skip_model_apply: - apply_job = self.build_dnamodelapply_job( - transfer_vcf, self.output_vcf - ) - dag.add_job(apply_job, {concat_job}) + TransferApplyStage( + ctx=ctx, + raw_vcf=raw_vcf, + transfer=TransferSpec( + TransferConfig.from_pipeline(self), transfer_target + ), + apply=( + ApplySpec(model, ctx.output_vcf) + if not self.skip_model_apply + else None + ), + ).add_to(dag, call.terminal) return dag @@ -880,7 +882,7 @@ def build_hybrid_kmc_job(self, kmer_prefix: pathlib.Path) -> Job: self.tmp_dir, memory=self.kmer_memory, threads=self.cores, - unzip=self.find_unzip(), + unzip=find_unzip(self.logger), ), "kmc", 0, # run in the background @@ -927,78 +929,50 @@ def build_extract_kmc_jobs( ) return symlink_job, extract_kmc_job - def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: - """Align the long-read input files to the linear reference genome - with minimap2""" - assert self.output_vcf is not None - assert self.reference is not None + def lr_realign_stage(self, ctx: StageContext) -> Minimap2RealignStage: + """Re-align the long-read input to the linear reference genome""" assert self.model_bundle is not None - res: List[pathlib.Path] = [] - realign_jobs: Set[Job] = set() - suffix = "bam" if self.bam_format else "cram" - for i, input_aln in enumerate(self.lr_aln): - out_aln = pathlib.Path( - str(self.output_vcf).replace( - ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" - ) - ) - rg_lines = cmds.get_rg_lines(input_aln, self.dry_run) - realign_jobs.add( - Job( - cmds.cmd_samtools_fastq_minimap2( - out_aln, - input_aln, - self.reference, - self.model_bundle, - self.cores, - rg_lines, - self.sample_sm, - self.lr_input_ref, - minimap2_model=self.model_bundle.joinpath( - "minimap2_lr.model" - ), - ), - f"bam-realign-{i}", - self.cores, - task_name="alignment", - ) - ) - res.append(out_aln) - return (res, realign_jobs) + return Minimap2RealignStage( + ctx=ctx, + inputs=self.lr_aln, + model_bundle=self.model_bundle, + sample_name=self.sample_sm, + bam_format=self.bam_format, + input_ref=self.lr_input_ref, + minimap2_model=self.model_bundle.joinpath("minimap2_lr.model"), + ) - def build_alignment_job( + def bwa_extract_stage( self, + ctx: StageContext, sample_bam: pathlib.Path, sample_fastq: pathlib.Path, - ) -> Job: - """Build the bwa alignment and read extraction job""" - assert self.reference is not None - assert self.model_bundle is not None + ) -> BwaExtractStage: + """The bwa alignment and read extraction stage. + The short reads are given the sample name and marked as short + reads (`LR:0`) so the hybrid caller can tell them apart. + """ + assert self.model_bundle is not None assert self.fastq_readgroup is not None rg = copy.deepcopy(self.fastq_readgroup) rg["SM"] = self.sample_sm rg["LR"] = "0" - bwa_job = Job( - cmds.cmd_bwa_extract( - sample_bam, - sample_fastq, - self.reference, - self.r1_fastq, - self.r2_fastq, - "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]), - self.model_bundle.joinpath(self.extract_model_name), - self.model_bundle.joinpath("bwa.model"), - self.cores, - unzip=self.find_unzip(), + return BwaExtractStage( + ctx=ctx, + output_bam=sample_bam, + output_fastq=sample_fastq, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroup=( + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]) ), - "bwa-extract", - self.cores, - task_name="alignment", + extract_model=self.model_bundle.joinpath(self.extract_model_name), + bwa_model=self.model_bundle.joinpath("bwa.model"), + unzip=find_unzip(self.logger), ) - return bwa_job def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path @@ -1073,32 +1047,6 @@ def build_graph_update_job( task_name="pangenome-update", ) - def build_longreadsv_job( - self, - out_vcf: pathlib.Path, - lr_aln: List[pathlib.Path], - ) -> Job: - """Call SVs from the long reads for the graph update""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=list(lr_aln), - ) - driver.add_algo( - LongReadSV( - out_vcf, - model=self.model_bundle.joinpath("longreadsv.model"), - min_sv_size=LONGREADSV_MIN_SV_SIZE, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "longreadsv", - self.cores, - task_name="pangenome-update", - ) - def build_minimap2_lift_job( self, out_bam: pathlib.Path, @@ -1203,59 +1151,3 @@ def build_pangenomesv_job( self.cores, task_name="sv-calling", ) - - def build_dnascope_job( - self, - out_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - replace_rg: List[List[str]], - ) -> Job: - """Call small variants with the original, lifted, and long reads""" - assert self.model_bundle is not None - pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - replace_rg=replace_rg, - ) - driver.add_algo( - DNAscope( - out_vcf, - model=self.model_bundle.joinpath("dnascope.model"), - pcr_indel_model=pcr_indel_model, - dbsnp=self.dbsnp, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "dnascope-raw", - self.cores, - task_name="variant-calling", - ) - - def build_dnamodelapply_job( - self, - in_vcf: pathlib.Path, - out_vcf: pathlib.Path, - ) -> Job: - """Apply the DNAscope model""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("dnascope.model"), - vcf=in_vcf, - output=out_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index f6d3cd6..ed17af3 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -11,12 +11,10 @@ import shutil import sys import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, TypeVar import packaging.version -from importlib.resources import files - from . import command_strings as cmds from .dag import DAG from .exceptions import DagExecutionError @@ -26,17 +24,17 @@ from .run_logs import RunLogs from .scheduler import ThreadScheduler from .shard import ( - PloidyContigs, detect_reference_build, par_bed_for_build, ) +from .stages.base import StageContext +from .stages.metrics import MetricsPaths from .util import ( SampleSex, __version__, - check_version, - cnvscope_sex_args, path_arg, tmp, + versions_available, ) MULTIQC_MIN_VERSION = { @@ -45,6 +43,8 @@ BWA_INDEX_SUFFIXES = (".amb", ".ann", ".bwt", ".pac", ".sa") +T = TypeVar("T") + class BasePipeline(ABC): """A pipeline base class""" @@ -246,6 +246,47 @@ def check_execution( f"Ready jobs: {dag.ready_jobs}\n" ) + def required(self, value: Optional[T], name: str) -> T: + """Return `value`, ending the run when it was not supplied. + + Narrows an optional pipeline attribute to its value, so callers do + not have to repeat the "missing argument" check. + """ + if value is None: + self.logger.error("%s is required", name) + sys.exit(2) + return value + + def stage_context(self) -> StageContext: + """The run-wide settings this pipeline's stages need. + + Built fresh on every call, so it always reflects the pipeline's + current attributes. Only available once `main` has created the + run's temporary directory. + """ + if self.reference is None: + self.logger.error("reference is required") + sys.exit(2) + if self.output_vcf is None: + self.logger.error("output_vcf is required") + sys.exit(2) + tmp_dir: Optional[pathlib.Path] = getattr(self, "tmp_dir", None) + if tmp_dir is None: + self.logger.error( + "The temporary directory has not been created yet; " + "`stage_context` is only available after `main` sets " + "`tmp_dir`" + ) + sys.exit(2) + return StageContext( + reference=self.reference, + output_vcf=self.output_vcf, + tmp_dir=tmp_dir, + cores=self.cores, + dry_run=self.dry_run, + skip_version_check=self.skip_version_check, + ) + @abstractmethod def validate(self) -> None: pass @@ -312,33 +353,6 @@ def build_second_dag(self) -> Optional[DAG]: """ return None - def build_ploidy_job( - self, - ploidy_json: pathlib.Path, - deduped_bam: List[pathlib.Path], - ploidy_contigs: Optional[PloidyContigs] = None, - ) -> Job: - """Estimate sample ploidy and sex""" - estimate_ploidy = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) - ).resolve() - ploidy_contigs = ploidy_contigs or PloidyContigs() - ploidy_job = Job( - cmds.cmd_estimate_ploidy( - ploidy_json, - deduped_bam, - estimate_ploidy, - contigs=ploidy_contigs.contigs, - autosomes=ploidy_contigs.autosomes, - x_contig=ploidy_contigs.x_contig, - y_contig=ploidy_contigs.y_contig, - ), - "estimate-ploidy", - 0, - task_name="ploidy", - ) - return ploidy_job - def get_sex(self, ploidy_json: pathlib.Path) -> None: """Retrieve the sample sex""" if self.sample_sex is not None: @@ -394,29 +408,21 @@ def validate_cnv_par(self, cnv_will_run: bool) -> None: ) sys.exit(2) - def cnv_sex_args(self) -> Tuple[Optional[str], Optional[pathlib.Path]]: - """The CNVscope `--sex` and `--par` arguments for this run""" - return cnvscope_sex_args(self.sample_sex, self.cnv_par_bed) - def multiqc(self) -> Optional[Job]: """Run MultiQC on the metrics files""" - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MULTIQC_MIN_VERSION.items() - ] - ): - self.logger.warning( - "Skipping MultiQC. MultiQC version %s or later not found", - MULTIQC_MIN_VERSION["multiqc"], - ) - return None + if not versions_available( + MULTIQC_MIN_VERSION, skip=self.skip_version_check + ): + self.logger.warning( + "Skipping MultiQC. MultiQC version %s or later not found", + MULTIQC_MIN_VERSION["multiqc"], + ) + return None - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) + metrics_dir = MetricsPaths.from_output_vcf( + pathlib.Path(str(self.output_vcf)) + ).metrics_dir multiqc_job = Job( cmds.cmd_multiqc( metrics_dir, diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py index 8099897..044247a 100644 --- a/sentieon_cli/sentieon_pangenome.py +++ b/sentieon_cli/sentieon_pangenome.py @@ -6,9 +6,7 @@ import copy import json import pathlib -import shutil import sys -import time from typing import Dict, List, Optional, Set, Tuple, Union import packaging.version @@ -19,29 +17,40 @@ from .archive import ar_load from .base_pangenome import BasePangenome from .dag import DAG -from .dnascope import CNV_MIN_VERSIONS from .driver import ( - CNVModelApply, - CNVscope, - DNAModelApply, + BaseAlgo, DNAscope, - Driver, - GVCFtyper, PangenomeSV, - ReadWriter, ) from .job import Job from .logging import get_logger from .shell_pipeline import Command, Pipeline +from .stages.alignment import ( + BwaExtractStage, + ReadWriterStage, + find_unzip, +) +from .stages.base import StageContext +from .stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from .stages.dedup import DedupStage +from .stages.metrics import MetricsPaths, MetricsStage +from .stages.ploidy import PloidyStage +from .stages.small_variants import ( + ApplySpec, + DNAscopeStage, + GVCFtyperStage, + TransferApplyStage, + TransferSpec, +) +from .stages.transfer import TransferConfig from .util import ( SampleSex, __version__, - check_version, check_kmc_patch, parse_rg_line, path_arg, + require_versions, sample_sex_arg, - tmp, total_memory, vcf_id, ) @@ -49,10 +58,8 @@ GRCH38_CONTIGS, determine_shards_from_fai, parse_fai, - ploidy_contigs_for_build, vcf_contigs, ) -from .transfer import build_transfer_jobs SENT_PANGENOME_MIN_VERSIONS = { "kmc": None, @@ -259,7 +266,6 @@ def __init__(self) -> None: self.call_svs = False self.gvcf = False self.par_bed: Optional[pathlib.Path] = None - self.cnv_input_bams: List[pathlib.Path] = [] self.pangenome_ref_name = "GRCh38" self.extract_model_name = "extract.model" self.pangenome_contig_prefix = "GRCh38#0#" @@ -278,6 +284,10 @@ def __init__(self) -> None: self.skip_pop_vcf_id_check: bool = False self.skip_model_apply = False self.skip_small_variants = False + # Stashed by `build_dag` for the second, sex-aware DAG + self.ploidy_json: Optional[pathlib.Path] = None + self.sr_alignment: Optional[pathlib.Path] = None + self.cnv_input_bams: List[pathlib.Path] = [] def _cnv_in_second_dag(self) -> bool: """CNV calling runs in the second, sex-aware DAG""" @@ -291,55 +301,21 @@ def _needs_second_dag(self) -> bool: or self._cnv_in_second_dag() ) - def main(self, args: argparse.Namespace) -> None: - """Run the pipeline""" - self.handle_arguments(args) - self.setup_logging(args) - start_time = time.monotonic() - success = False - try: - self.validate_ref() - - self.fai_data = parse_fai( - pathlib.Path(str(self.reference) + ".fai") - ) - self.pop_vcf_contigs: Dict[str, Optional[int]] = {} - if self.pop_vcf: - self.pop_vcf_contigs = vcf_contigs(self.pop_vcf, self.dry_run) - self.logger.debug("VCF contigs are: %s", self.pop_vcf_contigs) - - self.validate() - self.start_run_logs() - self.shards = determine_shards_from_fai( - self.fai_data, 10 * 1000 * 1000 - ) - - tmp_dir_str = tmp() - self.tmp_dir = pathlib.Path(tmp_dir_str) - - try: - dag = self.build_first_dag() - executor = self.run(dag) - self.check_execution(dag, executor) - - if self._needs_second_dag(): - self.get_sex(self.ploidy_json) - dag = self.build_second_dag() - executor = self.run(dag) - self.check_execution(dag, executor) - finally: - if not self.retain_tmpdir: - shutil.rmtree(tmp_dir_str) - success = True - finally: - self.log_completion(success, start_time) - def validate(self) -> None: """Validate pipeline inputs""" + self.validate_ref() + self.fai_data = parse_fai(pathlib.Path(str(self.reference) + ".fai")) + self.shards = determine_shards_from_fai( + self.fai_data, 10 * 1000 * 1000 + ) + self.pop_vcf_contigs: Dict[str, Optional[int]] = {} + if self.pop_vcf: + self.pop_vcf_contigs = vcf_contigs(self.pop_vcf, self.dry_run) + self.logger.debug("VCF contigs are: %s", self.pop_vcf_contigs) + self.validate_bundle() self.validate_fastq_rg() self.validate_output_vcf() - self.validate_ref() self.collect_readgroups() if not self.sample_input and not self.r1_fastq: @@ -364,20 +340,19 @@ def validate(self) -> None: self.validate_t1k() self.validate_cnv() - if not self.skip_version_check: - for cmd, min_version in SENT_PANGENOME_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions( + SENT_PANGENOME_MIN_VERSIONS, skip=self.skip_version_check + ) - if self.sample_input: - if not check_kmc_patch("kmc"): - self.logger.error( - "Error: The 'kmc' executable in the PATH does not " - "support reading from stdin. Please ensure " - "you are using the patched version of KMC from " - "https://github.com/Sentieon/KMC/releases." - ) - sys.exit(2) + if not self.skip_version_check and self.sample_input: + if not check_kmc_patch("kmc"): + self.logger.error( + "Error: The 'kmc' executable in the PATH does not " + "support reading from stdin. Please ensure " + "you are using the patched version of KMC from " + "https://github.com/Sentieon/KMC/releases." + ) + sys.exit(2) if self.bed is None: self.logger.info( @@ -450,10 +425,7 @@ def validate_segdup(self) -> None: ) sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in SEGDUP_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(SEGDUP_MIN_VERSION, skip=self.skip_version_check) def validate_expansion(self) -> None: if self.expansion_catalog is None: @@ -473,10 +445,7 @@ def validate_expansion(self) -> None: ) sys.exit(2) - if not self.skip_version_check: - for cmd, min_version in EXPANSION_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(EXPANSION_MIN_VERSION, skip=self.skip_version_check) def validate_cnv(self) -> None: """Validate the arguments used for sex-aware CNV calling""" @@ -489,10 +458,7 @@ def validate_cnv(self) -> None: # A PAR BED file is required for CNV calling, whatever the sex self.validate_cnv_par(True) - if not self.skip_version_check: - for cmd, min_version in CNV_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(CNV_MIN_VERSIONS, skip=self.skip_version_check) def validate_t1k(self) -> None: hla_requested = self.t1k_hla_seq is not None or ( @@ -518,18 +484,11 @@ def validate_t1k(self) -> None: if not (hla_requested or kir_requested): return - if not self.skip_version_check: - for cmd, min_version in T1K_MIN_VERSION.items(): - if not check_version(cmd, min_version): - sys.exit(2) + require_versions(T1K_MIN_VERSION, skip=self.skip_version_check) def validate_bundle(self) -> None: - if not self.pop_vcf: - self.logger.error("pop_vcf is required") - sys.exit(2) - if not self.gbz: - self.logger.error("gbz is required") - sys.exit(2) + pop_vcf = self.required(self.pop_vcf, "pop_vcf") + gbz = self.required(self.gbz, "gbz") bundle_info_bytes = ar_load( str(self.model_bundle) + "/bundle_info.json" ) @@ -562,7 +521,7 @@ def validate_bundle(self) -> None: if bundle_pipeline != "Sentieon pangenome": self.logger.error("The model bundle is for a different pipeline.") sys.exit(2) - if self.gbz.name != bundle_pangenome: + if gbz.name != bundle_pangenome: self.logger.warning( "The `--gbz` file name is not '%s'. " "This model is optimized for the %s pangenome.", @@ -602,7 +561,7 @@ def validate_bundle(self) -> None: ) if not self.skip_pop_vcf_id_check and not self.dry_run: - pop_vcf_id = vcf_id(self.pop_vcf) + pop_vcf_id = vcf_id(pop_vcf) if bundle_vcf_id != pop_vcf_id: self.logger.error( "The ID of the `--pop_vcf` does not match the model bundle" @@ -665,19 +624,10 @@ def configure(self) -> None: pass def build_dag(self) -> DAG: - return DAG() + """Build the first DAG for the Sentieon pangenome pipeline""" + bundle = self.required(self.model_bundle, "model_bundle") - def build_first_dag(self) -> DAG: - """Build the main DAG for the Sentieon pangenome pipeline""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + ctx = self.stage_context() self.logger.info("Building the Sentieon pangenome DAG") dag = DAG() @@ -685,16 +635,13 @@ def build_first_dag(self) -> DAG: # Output files suffix = "bam" if self.bam_format else "cram" out_bwa_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") + str(ctx.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") ) out_mm2_aln = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") - ) - self.ploidy_json = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_ploidy.json") + str(ctx.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") ) out_gvcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", ".g.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", ".g.vcf.gz") ) # Intermediate file paths @@ -709,7 +656,7 @@ def build_first_dag(self) -> DAG: if not self.r1_fastq: # with bam/cram input, output the realigned bam/cram mm2_bam = pathlib.Path( - str(self.output_vcf).replace( + str(ctx.output_vcf).replace( ".vcf.gz", f"_mm2_deduped.{suffix}" ) ) @@ -732,8 +679,10 @@ def build_first_dag(self) -> DAG: haplotype_dependencies.add(kmc_job) # BWA alignment and extraction - bwa_job = self.build_alignment_job(bwa_bam, ext_fastq) - dag.add_job(bwa_job) + bwa_result = self.bwa_extract_stage( + ctx, bwa_bam, ext_fastq + ).add_to(dag) + bwa_job = bwa_result.jobs[0] mm2_dependencies.add(bwa_job) bwa_lc_dependencies.add(bwa_job) # Do not run vg-haplotypes with bwa in low-mem environments @@ -758,8 +707,8 @@ def build_first_dag(self) -> DAG: kmer_prefix, ext_fastq, self.sample_input, - self.reference, - self.model_bundle.joinpath(self.extract_model_name), + ctx.reference, + bundle.joinpath(self.extract_model_name), self.tmp_dir, rw_bam, threads=self.cores, @@ -801,47 +750,49 @@ def build_first_dag(self) -> DAG: # Emit Dedup metrics for the primary (bwa) short-read alignment so # they land in the metrics directory scanned by MultiQC. + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) dedup_metrics: Optional[pathlib.Path] = None if not self.skip_metrics: - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - if not self.dry_run: - metrics_dir.mkdir(exist_ok=True) - sample_name = self.output_vcf.name.replace(".vcf.gz", "") - dedup_metrics = metrics_dir.joinpath( - sample_name + ".txt.dedup_metrics.txt" - ) - - bwa_lc_job, bwa_dedup_job = self.build_dedup_job( - out_bwa_aln, [bwa_bam], "bwa", metrics=dedup_metrics - ) - mm2_lc_job, mm2_dedup_job = self.build_dedup_job( - out_mm2_aln, - [mm2_bam], - "mm2", - left_align_rgid=f"{self.fastq_readgroup['ID']}-mm2", - ) - dag.add_job(bwa_lc_job, bwa_lc_dependencies) - dag.add_job(bwa_dedup_job, {bwa_lc_job}) + paths.ensure_dir(self.dry_run) + dedup_metrics = paths.dedup_metrics + + bwa_dedup = DedupStage( + ctx=ctx, + tag="bwa", + inputs=[bwa_bam], + output=out_bwa_aln, + score_file=self.tmp_dir.joinpath("sample-bwa-score.txt.gz"), + dedup_metrics=dedup_metrics, + ).add_to(dag, bwa_lc_dependencies) + bwa_dedup_job = bwa_dedup.dedup_job dnascope_dependencies.add(bwa_dedup_job) - dag.add_job(mm2_lc_job, {mm2_job}) - dag.add_job(mm2_dedup_job, {mm2_lc_job}) - dnascope_dependencies.add(mm2_dedup_job) + mm2_dedup = DedupStage( + ctx=ctx, + tag="mm2", + inputs=[mm2_bam], + output=out_mm2_aln, + score_file=self.tmp_dir.joinpath("sample-mm2-score.txt.gz"), + read_filters=[ + "IndelLeftAlignReadTransform,rgid=" + f"{self.fastq_readgroup['ID']}-mm2" + ], + ).add_to(dag, {mm2_job}) + dnascope_dependencies.add(mm2_dedup.dedup_job) cnv_input_bams = [out_bwa_aln] cnvscope_deps = {bwa_dedup_job} if not self.skip_metrics: - metrics_job, rehead_job = self.build_metrics_job( - [out_bwa_aln], - ) - dag.add_job(metrics_job, {bwa_dedup_job}) - dag.add_job(rehead_job, {metrics_job}) + metrics_result = MetricsStage( + ctx=ctx, + inputs=[out_bwa_aln], + algos=self.pangenome_metrics_algos(paths), + rehead_metrics=paths.wgs, + ).add_to(dag, {bwa_dedup_job}) if not self.skip_multiqc: multiqc_job = self.multiqc() if multiqc_job: - dag.add_job(multiqc_job, {rehead_job}) + dag.add_job(multiqc_job, metrics_result.terminal) else: dnascope_bams.append(mm2_bam) cnv_input_bams = list(self.sample_input) @@ -853,17 +804,17 @@ def build_first_dag(self) -> DAG: # Estimate the sample ploidy and sex. The JSON output is always # written; `--sample_sex` takes precedence for the sex used by # the sex-aware callers. - ploidy_job = self.build_ploidy_job( - self.ploidy_json, - [cnv_input_bams[0]], - ploidy_contigs_for_build(self.reference_build), - ) - dag.add_job(ploidy_job, cnvscope_deps) + ploidy_result = PloidyStage( + ctx=ctx, + inputs=[cnv_input_bams[0]], + reference_build=self.reference_build, + ).add_to(dag, cnvscope_deps) + self.ploidy_json = ploidy_result.ploidy_json # T1K HLA/KIR calling if self.t1k_hla_seq and self.t1k_hla_coord: out_hla = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_hla") + str(ctx.output_vcf).replace(".vcf.gz", "_hla") ) extract_job, t1k_job = self.build_t1k_jobs( out_hla, @@ -878,7 +829,7 @@ def build_first_dag(self) -> DAG: dag.add_job(t1k_job, {extract_job}) if self.t1k_kir_seq and self.t1k_kir_coord: out_kir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_kir") + str(ctx.output_vcf).replace(".vcf.gz", "_kir") ) extract_job, t1k_job = self.build_t1k_jobs( out_kir, @@ -896,152 +847,146 @@ def build_first_dag(self) -> DAG: sv_vcf = None if self.call_svs: sv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_sv.vcf.gz") ) if self.skip_small_variants and not self.call_svs: return dag - if self.skip_small_variants and self.call_svs and sv_vcf: - # SV calling only - dnascope_job = self.build_dnascope_job( - raw_vcf, - dnascope_bams, - sv_vcf=sv_vcf, - gfa_file=sample_gfa, + read_filters: List[str] = [] + if self.tech.upper() == "ULTIMA": + read_filters.append("UltimaReadFilter") + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" + model = bundle.joinpath("dnascope.model") + gfa_file = sample_gfa if self.call_svs else None + + algos: List[BaseAlgo] = [] + if not self.skip_small_variants: + algos.append( + DNAscope( + raw_vcf, + model=model, + pcr_indel_model=pcr_indel_model, + dbsnp=self.dbsnp, + emit_mode="gvcf" if self.gvcf else "variant", + ) ) - dag.add_job(dnascope_job, dnascope_dependencies) - # CNV calling is sex-aware and runs in the second DAG + if sv_vcf and gfa_file: + algos.append( + PangenomeSV( + sv_vcf, + gfa_file=gfa_file, + prefix=self.pangenome_contig_prefix, + ) + ) + call = DNAscopeStage( + ctx=ctx, + algos=algos, + inputs=dnascope_bams, + interval=self.bed, + read_filter=read_filters, + ).add_to(dag, dnascope_dependencies) + + if self.skip_small_variants: + # SV calling only. CNV calling is sex-aware and runs in the + # second DAG return dag - apply_dependencies = set() - dnascope_job = self.build_dnascope_job( - raw_vcf, - dnascope_bams, - sv_vcf=sv_vcf, - gfa_file=sample_gfa if self.call_svs else None, - ) - dag.add_job(dnascope_job, dnascope_dependencies) - apply_dependencies.add(dnascope_job) + if self.skip_model_apply and not self.pop_vcf: + # Nothing post-processes the raw VCF + return dag # When --gvcf is set, the model-apply / transfer outputs are - # gVCFs; GVCFtyper produces the final VCF at self.output_vcf. - small_variants_out = out_gvcf if self.gvcf else self.output_vcf + # gVCFs; GVCFtyper produces the final VCF at ctx.output_vcf. + small_variants_out = out_gvcf if self.gvcf else ctx.output_vcf snv_apply_vcf = self.tmp_dir.joinpath( "sample-snv_apply.g.vcf.gz" if self.gvcf else "sample-snv_apply.vcf.gz" ) - # transfer annotations from the pop_vcf + # Transfer annotations from the pop_vcf, then apply the model + transfer: Optional[TransferSpec] = None if self.pop_vcf: - transfer_jobs, concat_job = build_transfer_jobs( - ( + transfer = TransferSpec( + config=TransferConfig.from_pipeline(self), + out_vcf=( transfer_vcf if not self.skip_model_apply - else (snv_apply_vcf) + else snv_apply_vcf ), - self.pop_vcf, - raw_vcf, - self.tmp_dir, - self.shards, - self.pop_vcf_contigs, - self.fai_data, - self.dry_run, - self.cores, ) - for job in transfer_jobs: - dag.add_job(job, {dnascope_job}) - dag.add_job(concat_job, set(transfer_jobs)) - apply_dependencies.add(concat_job) - - small_variants_last_job: Optional[Job] = None + apply_spec: Optional[ApplySpec] = None if not self.skip_model_apply: - # DNAModelApply - apply_job = self.build_dnamodelapply_job( - transfer_vcf, snv_apply_vcf - ) - dag.add_job(apply_job, apply_dependencies) - small_variants_last_job = apply_job - elif self.pop_vcf: - small_variants_last_job = concat_job + apply_spec = ApplySpec(model=model, output=snv_apply_vcf) + transfer_apply = TransferApplyStage( + ctx=ctx, + raw_vcf=raw_vcf, + transfer=transfer, + apply=apply_spec, + ).add_to(dag, call.terminal) # Update the overestimated AD/DP of the joint pileup - if small_variants_last_job is not None: - ad_update_job = self.build_count_ad_update_job( - small_variants_out, snv_apply_vcf - ) - dag.add_job(ad_update_job, {small_variants_last_job}) - small_variants_last_job = ad_update_job + ad_update_job = self.build_count_ad_update_job( + small_variants_out, snv_apply_vcf + ) + dag.add_job(ad_update_job, transfer_apply.terminal) # Genotype the gVCF to also produce a regular VCF at output_vcf - if self.gvcf and small_variants_last_job is not None: - gvcftyper_job = self.build_gvcftyper_job(self.output_vcf, out_gvcf) - dag.add_job(gvcftyper_job, {small_variants_last_job}) + if self.gvcf: + GVCFtyperStage( + ctx=ctx, + gvcf=out_gvcf, + output=ctx.output_vcf, + interval=self.bed, + ).add_to(dag, {ad_update_job}) # CNV calling is sex-aware and runs in the second DAG return dag - def build_alignment_job( + def bwa_extract_stage( self, + ctx: StageContext, sample_bam: pathlib.Path, sample_fastq: pathlib.Path, - ) -> Job: - """Build the alignment and extract jobs""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + ) -> BwaExtractStage: + """The bwa alignment and read extraction stage. - unzip = "igzip" - if not shutil.which(unzip): - self.logger.info( - "igzip is recommended for decompression, but is not " - "available. Falling back to gzip." - ) - unzip = "gzip" + The bwa alignment reuses the input readgroup ID with a `-bwa` + suffix, so it stays distinct from the lifted alignment's. + """ + bundle = self.required(self.model_bundle, "model_bundle") rg = copy.deepcopy(self.fastq_readgroup) rg["ID"] = rg["ID"] + "-bwa" - bwa_job = Job( - cmds.cmd_bwa_extract( - sample_bam, - sample_fastq, - self.reference, - self.r1_fastq, - self.r2_fastq, - "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]), - self.model_bundle.joinpath(self.extract_model_name), - self.model_bundle.joinpath("bwa.model"), - self.cores, - unzip=unzip, + return BwaExtractStage( + ctx=ctx, + output_bam=sample_bam, + output_fastq=sample_fastq, + r1_fastq=self.r1_fastq, + r2_fastq=self.r2_fastq, + readgroup=( + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]) ), - "bwa-extract", - self.cores, - task_name="alignment", + extract_model=bundle.joinpath(self.extract_model_name), + bwa_model=bundle.joinpath("bwa.model"), + unzip=find_unzip(self.logger), ) - return bwa_job def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path ) -> Job: """Build vg haplotypes job""" - if not self.hapl: - self.logger.error("hapl is required") - sys.exit(2) - if not self.gbz: - self.logger.error("gbz is required") - sys.exit(2) + hapl_file = self.required(self.hapl, "hapl") + gbz_file = self.required(self.gbz, "gbz") haplotypes_job = Job( cmds.cmd_vg_haplotypes( output_gbz, kmer_file, - self.hapl, - self.gbz, + hapl_file, + gbz_file, threads=self.cores, xargs=[ "--include-reference", @@ -1096,12 +1041,8 @@ def build_minimap2_lift_job( sample_gfa: pathlib.Path, ) -> Job: """Build minimap2 alignment with pgutil lift job""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + reference = self.required(self.reference, "reference") rg = ( self.fastq_readgroup @@ -1112,16 +1053,14 @@ def build_minimap2_lift_job( rg2["ID"] = rg2["ID"] + "-mm2" rg2["LR"] = "1" - mm2_model: Union[str, pathlib.Path] = self.model_bundle.joinpath( - "minimap2.model" - ) + mm2_model: Union[str, pathlib.Path] = bundle.joinpath("minimap2.model") mm2_job = Job( cmds.cmd_minimap2_lift( mm2_bam, sample_fasta, ext_fastq, sample_gfa, - self.reference, + reference, "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg2.items()]), mm2_model, threads=self.cores, @@ -1133,109 +1072,6 @@ def build_minimap2_lift_job( ) return mm2_job - def build_dnascope_job( - self, - out_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - sv_vcf: Optional[pathlib.Path] = None, - gfa_file: Optional[pathlib.Path] = None, - ) -> Job: - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - read_filters = [] - if self.tech.upper() == "ULTIMA": - read_filters.append("UltimaReadFilter") - - pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - read_filter=read_filters, - ) - if not self.skip_small_variants: - driver.add_algo( - DNAscope( - out_vcf, - model=self.model_bundle.joinpath("dnascope.model"), - pcr_indel_model=pcr_indel_model, - dbsnp=self.dbsnp, - emit_mode="gvcf" if self.gvcf else "variant", - ) - ) - if sv_vcf and gfa_file: - driver.add_algo( - PangenomeSV( - sv_vcf, - gfa_file=gfa_file, - prefix=self.pangenome_contig_prefix, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "dnascope-raw", - self.cores, - task_name="variant-calling", - ) - - def build_dnamodelapply_job( - self, - in_vcf: pathlib.Path, - out_vcf: pathlib.Path, - ) -> Job: - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - DNAModelApply( - model=self.model_bundle.joinpath("dnascope.model"), - vcf=in_vcf, - output=out_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "model-apply", - self.cores, - task_name="model-apply", - ) - - def build_gvcftyper_job( - self, - out_vcf: pathlib.Path, - in_gvcf: pathlib.Path, - ) -> Job: - """Genotype a gVCF to produce a VCF""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - - driver = Driver( - reference=self.reference, - thread_count=self.cores, - interval=self.bed, - ) - driver.add_algo( - GVCFtyper( - output=out_vcf, - vcf=in_gvcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "gvcftyper", - self.cores, - task_name="gvcftyper", - ) - def build_count_ad_update_job( self, out_vcf: pathlib.Path, @@ -1271,12 +1107,8 @@ def build_segdup_job( genes: Optional[str], ) -> Job: """Call variants in difficult SegDups""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) + reference = self.required(self.reference, "reference") + bundle = self.required(self.model_bundle, "model_bundle") sex = "male" if self.sample_sex == SampleSex.MALE else "female" @@ -1290,8 +1122,8 @@ def build_segdup_job( cmds.cmd_segdup_caller( out_segdup, sr_alignment, - reference=self.reference, - sr_bundle=self.model_bundle, + reference=reference, + sr_bundle=bundle, input_vcf=input_vcf, sex=sex, genes=genes, @@ -1302,8 +1134,14 @@ def build_segdup_job( task_name="segdup", ) - def build_second_dag(self) -> DAG: + def build_second_dag(self) -> Optional[DAG]: """Build the second DAG for sex-aware downstream tools""" + if not self._needs_second_dag(): + return None + + assert self.ploidy_json is not None + self.get_sex(self.ploidy_json) + self.logger.info("Building the second pangenome DAG") dag = DAG() @@ -1354,25 +1192,19 @@ def build_t1k_jobs( tag: str, ) -> Tuple[Job, Job]: """Extract reads at the T1K locus and genotype them with T1K""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + self.required(self.reference, "reference") # Extract reads overlapping the locus to a BAM file with ReadWriter extracted_bam = self.tmp_dir.joinpath(f"sample_{tag}.bam") - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=sr_alignments, - interval=locus, - ) - driver.add_algo(ReadWriter(extracted_bam)) - extract_job = Job( - Pipeline(Command(*driver.build_cmd())), - f"t1k-{tag}-extract", - self.cores, + extract_result = ReadWriterStage( + ctx=self.stage_context(), + inputs=sr_alignments, + output=extracted_bam, + name=f"t1k-{tag}-extract", task_name="t1k", - ) + interval=locus, + ).build() + extract_job = extract_result.jobs[0] t1k_job = Job( cmds.cmd_t1k( @@ -1396,15 +1228,13 @@ def build_expansion_job( expansion_catalog: pathlib.Path, ) -> Job: """Identify repeat expansions""" - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) + reference = self.required(self.reference, "reference") return Job( cmds.cmd_expansion_hunter( out_expansions, sr_alignment, - reference=self.reference, + reference=reference, variant_catalog=expansion_catalog, sex="male" if self.sample_sex == SampleSex.MALE else "female", threads=self.cores, @@ -1421,33 +1251,28 @@ def _add_cnv_jobs( cnv_input_bams: List[pathlib.Path], ) -> None: """Add CNV calling jobs to the DAG""" - if not self.model_bundle: - self.logger.error("model_bundle is required") - sys.exit(2) - if not self.reference: - self.logger.error("reference is required") - sys.exit(2) - if not self.output_vcf: - self.logger.error("output_vcf is required") - sys.exit(2) + bundle = self.required(self.model_bundle, "model_bundle") + ctx = self.stage_context() cnvscope_vcf = self.tmp_dir.joinpath("sample-cnvscope.vcf.gz") cnv_apply_vcf = self.tmp_dir.joinpath("sample-cnv_model_apply.vcf.gz") indel2cnv_vcf = self.tmp_dir.joinpath("sample-sv_cnv.vcf.gz") cnv_vcf = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") + str(ctx.output_vcf).replace(".vcf.gz", "_cnv.vcf.gz") ) - # CNVscope on BWA deduped BAM. The alignment and the PangenomeSV - # output were both written by the first DAG. - cnvscope_job = self._build_cnvscope_job(cnvscope_vcf, cnv_input_bams) - dag.add_job(cnvscope_job) - - # CNVModelApply on CNVscope output - cnv_model_apply_job = self._build_cnv_model_apply_job( - cnv_apply_vcf, cnvscope_vcf - ) - dag.add_job(cnv_model_apply_job, {cnvscope_job}) + # CNVscope on BWA deduped BAM, then CNVModelApply. The alignment + # and the PangenomeSV output were both written by the first DAG. + cnv_result = CNVscopeStage( + ctx=ctx, + inputs=cnv_input_bams, + model=bundle.joinpath("cnv.model"), + cnvscope_vcf=cnvscope_vcf, + cnv_vcf=cnv_apply_vcf, + sample_sex=self.sample_sex, + par_bed=self.cnv_par_bed, + interval=self.bed, + ).add_to(dag) # Convert PangenomeSV output to CNVs indel2cnv_job = self._build_indel2cnv_job(indel2cnv_vcf, sv_vcf) @@ -1457,61 +1282,7 @@ def _add_cnv_jobs( combine_job = self._build_combine_cnv_job( cnv_vcf, cnv_apply_vcf, indel2cnv_vcf ) - dag.add_job(combine_job, {cnv_model_apply_job, indel2cnv_job}) - - def _build_cnvscope_job( - self, - output_vcf: pathlib.Path, - input_bams: List[pathlib.Path], - ) -> Job: - """Build a CNVscope job""" - assert self.model_bundle is not None - sex, par = self.cnv_sex_args() - driver = Driver( - reference=self.reference, - thread_count=self.cores, - input=input_bams, - interval=self.bed, - ) - driver.add_algo( - CNVscope( - output=output_vcf, - model=self.model_bundle.joinpath("cnv.model"), - sex=sex, - par=par, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "cnvscope", - self.cores, - task_name="cnv", - ) - - def _build_cnv_model_apply_job( - self, - output_vcf: pathlib.Path, - input_vcf: pathlib.Path, - ) -> Job: - """Build a CNVModelApply job""" - assert self.model_bundle is not None - driver = Driver( - reference=self.reference, - thread_count=self.cores, - ) - driver.add_algo( - CNVModelApply( - output=output_vcf, - model=self.model_bundle.joinpath("cnv.model"), - vcf=input_vcf, - ) - ) - return Job( - Pipeline(Command(*driver.build_cmd())), - "cnv-model-apply", - self.cores, - task_name="cnv", - ) + dag.add_job(combine_job, {cnv_result.apply_job, indel2cnv_job}) def _build_indel2cnv_job( self, diff --git a/sentieon_cli/shard.py b/sentieon_cli/shard.py index 4506c40..9e509e5 100644 --- a/sentieon_cli/shard.py +++ b/sentieon_cli/shard.py @@ -93,9 +93,6 @@ class Shard(NamedTuple): def __str__(self) -> str: return f"{self.contig}:{self.start}-{self.stop}" - def bcftools_str(self) -> str: - return f"{{{self.contig}}}:{self.start}-{self.stop}" - def parse_fai(ref_fai: pathlib.Path) -> Dict[str, Dict[str, int]]: """Parse a faidx index""" diff --git a/sentieon_cli/stages/__init__.py b/sentieon_cli/stages/__init__.py new file mode 100644 index 0000000..e74d5da --- /dev/null +++ b/sentieon_cli/stages/__init__.py @@ -0,0 +1 @@ +"""Reusable pipeline stages""" diff --git a/sentieon_cli/stages/alignment.py b/sentieon_cli/stages/alignment.py new file mode 100644 index 0000000..ae6769a --- /dev/null +++ b/sentieon_cli/stages/alignment.py @@ -0,0 +1,459 @@ +""" +Read alignment and read extraction + +Alignment jobs are the roots of every pipeline DAG: they read the run's +input files and depend on nothing. Because a few pipelines need the jobs +before they insert them, these stages split the work in two -- ``build`` +constructs the jobs, ``add_to`` inserts what ``build`` returned. +""" + +from abc import abstractmethod +from dataclasses import dataclass, field +import itertools +import logging +import pathlib +import shutil +from typing import Iterable, List, Optional, Union + +import packaging.version + +from .. import command_strings as cmds +from ..dag import DAG +from ..driver import ReadWriter +from ..job import Job +from .base import Stage, StageResult, driver_job + +# The `sentieon util sort` arguments every aligner defaults to +DEFAULT_UTIL_SORT_ARGS = "--cram_write_options version=3.0,compressor=rans" + +BWA_REALIGN_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), + "samtools": packaging.version.Version("1.16"), +} + +BWA_FASTQ_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + +MINIMAP2_REALIGN_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), + "samtools": packaging.version.Version("1.16"), +} + +MINIMAP2_FASTQ_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + + +def aln_suffix(bam_format: bool) -> str: + """The file extension for an aligned output file""" + return "bam" if bam_format else "cram" + + +def find_unzip(logger: logging.Logger) -> str: + """The decompression tool to read gzipped fastq with. + + ``igzip`` is considerably faster but is not always installed, so fall + back to ``gzip``. + """ + unzip = "igzip" + if not shutil.which(unzip): + logger.warning( + "igzip is recommended for decompression, but is not " + "available. Falling back to gzip.", + ) + unzip = "gzip" + return unzip + + +@dataclass(kw_only=True) +class AlignResult(StageResult): + """The jobs and outputs of an `AlignmentStage`. + + * ``outputs`` -- the aligned files, in input order. + * ``cleanup_paths`` -- files a later ``rm`` job should delete once the + alignment has been consumed; empty when the stage writes only + final outputs. + """ + + outputs: List[pathlib.Path] + cleanup_paths: List[pathlib.Path] + + +@dataclass(kw_only=True) +class AlignmentStage(Stage): + """A group of alignment jobs, all of them DAG roots. + + Subclasses implement :meth:`build`, which constructs the jobs and + inserts nothing. :meth:`add_to` then adds each of them with the same + dependencies, since no alignment job depends on another. + """ + + @abstractmethod + def build(self) -> AlignResult: + """Construct this stage's jobs without touching a DAG""" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> AlignResult: + deps = set(upstream) + result = self.build() + for job in result.jobs: + dag.add_job(job, deps) + return result + + +@dataclass(kw_only=True) +class BwaRealignStage(AlignmentStage): + """Re-align BAM/CRAM/uBAM/uCRAM input to the reference with bwa. + + The @RG lines of each input are read at build time and written to a + header file in the run's temporary directory, which the alignment + command passes to `samtools fastq`. + + When duplicate marking follows, the alignment is an intermediate: it + is written to the temporary directory as a lightly compressed BAM + instead of to the output directory. + """ + + inputs: List[pathlib.Path] + model_bundle: pathlib.Path + bam_format: bool = False + duplicate_marking: str = "markdup" + input_ref: Optional[pathlib.Path] = None + collate: bool = False + bwa_args: str = "" + bwa_k: str = "20000000" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + util_sort_args = self.util_sort_args + dedup_follows = self.duplicate_marking != "none" + if dedup_follows: + suffix = "bam" + util_sort_args += " --bam_compression 1 " + + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + cleanup_paths: List[pathlib.Path] = [] + for i, input_aln in enumerate(self.inputs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_bwa_sorted_{i}.{suffix}" + ) + ) + if dedup_follows: + out_aln = self.ctx.tmp_dir.joinpath(f"bwa_sorted_{i}.{suffix}") + rg_lines = cmds.get_rg_lines(input_aln, self.ctx.dry_run) + rg_header = self.ctx.tmp_dir.joinpath(f"input_{i}.hdr") + with open(rg_header, "w", encoding="utf-8") as rg_fh: + for line in rg_lines: + print(line, file=rg_fh) + jobs.append( + Job( + cmds.cmd_samtools_fastq_bwa( + out_aln, + input_aln, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + rg_header, + self.input_ref, + collate=self.collate, + bwa_args=self.bwa_args, + bwa_k=self.bwa_k, + util_sort_args=util_sort_args, + ), + f"bam-align-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + if dedup_follows: + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=cleanup_paths, + ) + + +@dataclass(kw_only=True) +class BwaFastqStage(AlignmentStage): + """Align short-read fastq pairs to the reference with bwa. + + With more than one NUMA node the reads of each pair are split across + the nodes: one job per (fastq pair, node), each pinned with `taskset` + and taking a share of the cores and a node token, so the scheduler + runs exactly one alignment per node. + """ + + r1_fastq: List[pathlib.Path] + r2_fastq: List[pathlib.Path] + readgroups: List[str] + model_bundle: pathlib.Path + numa_nodes: List[str] = field(default_factory=list) + bam_format: bool = False + duplicate_marking: str = "markdup" + unzip: str = "gzip" + bwa_args: str = "" + bwa_k: str = "20000000" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + n_alignment_jobs = max(1, len(self.numa_nodes)) + split_alignment = len(self.numa_nodes) > 1 + suffix = aln_suffix(self.bam_format) + util_sort_args = self.util_sort_args + dedup_follows = self.duplicate_marking != "none" + if dedup_follows: + suffix = "bam" + util_sort_args += " --bam_compression 1 " + + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + cleanup_paths: List[pathlib.Path] = [] + for i, (r1, r2, rg) in enumerate( + itertools.zip_longest( + self.r1_fastq, self.r2_fastq, self.readgroups + ) + ): + for j in range(n_alignment_jobs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_bwa_sorted_fq_{i}_{j}.{suffix}" + ) + ) + if dedup_follows: + out_aln = self.ctx.tmp_dir.joinpath( + f"bwa_sorted_fq_{i}_{j}.{suffix}" + ) + numa = self.numa_nodes[j] if split_alignment else None + split = f"{j}/{n_alignment_jobs}" if split_alignment else None + split_cores = max(1, int(self.ctx.cores / n_alignment_jobs)) + jobs.append( + Job( + cmds.cmd_fastq_bwa( + out_aln, + r1, + r2, + rg, + self.ctx.reference, + self.model_bundle, + split_cores, + self.unzip, + self.bwa_args, + self.bwa_k, + util_sort_args, + numa, + split, + ), + f"bam-align-{i}-{j}", + split_cores, + resources={f"node{j}": 1}, + task_name="alignment", + ) + ) + outputs.append(out_aln) + if dedup_follows: + cleanup_paths.append(out_aln) + cleanup_paths.append(pathlib.Path(str(out_aln) + ".bai")) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=cleanup_paths, + ) + + +@dataclass(kw_only=True) +class Minimap2RealignStage(AlignmentStage): + """Re-align long-read BAM/CRAM/uBAM/uCRAM input with minimap2. + + ``minimap2_model`` overrides the model the aligner picks up from the + bundle; the pangenome pipelines use it to select the long-read model. + """ + + inputs: List[pathlib.Path] + model_bundle: pathlib.Path + sample_name: str + bam_format: bool = False + input_ref: Optional[pathlib.Path] = None + fastq_taglist: str = "*" + minimap2_args: str = "-YL" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + minimap2_model: Optional[Union[pathlib.Path, str]] = None + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + for i, input_aln in enumerate(self.inputs): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_mm2_sorted_{i}.{suffix}" + ) + ) + rg_lines = cmds.get_rg_lines(input_aln, self.ctx.dry_run) + jobs.append( + Job( + cmds.cmd_samtools_fastq_minimap2( + out_aln, + input_aln, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + rg_lines, + self.sample_name, + self.input_ref, + self.fastq_taglist, + self.minimap2_args, + self.util_sort_args, + self.minimap2_model, + ), + f"bam-realign-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class Minimap2FastqStage(AlignmentStage): + """Align long-read fastq to the reference with minimap2""" + + fastq: List[pathlib.Path] + readgroups: List[str] + model_bundle: pathlib.Path + bam_format: bool = False + unzip: str = "gzip" + minimap2_args: str = "-YL" + util_sort_args: str = DEFAULT_UTIL_SORT_ARGS + + def build(self) -> AlignResult: + suffix = aln_suffix(self.bam_format) + jobs: List[Job] = [] + outputs: List[pathlib.Path] = [] + for i, (fq, rg) in enumerate(zip(self.fastq, self.readgroups)): + out_aln = pathlib.Path( + str(self.ctx.output_vcf).replace( + ".vcf.gz", f"_mm2_sorted_fq_{i}.{suffix}" + ) + ) + jobs.append( + Job( + cmds.cmd_fastq_minimap2( + out_aln, + fq, + rg, + self.ctx.reference, + self.model_bundle, + self.ctx.cores, + self.unzip, + self.minimap2_args, + self.util_sort_args, + ), + f"align-{i}", + self.ctx.cores, + task_name="alignment", + ) + ) + outputs.append(out_aln) + + return AlignResult( + jobs=jobs, + terminal=set(jobs), + outputs=outputs, + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class BwaExtractStage(AlignmentStage): + """Align short-read fastq with bwa and extract the reads a pangenome + pipeline needs, in a single pass. + + ``readgroup`` is the rendered `@RG` string; the pangenome pipelines + derive it differently from the input readgroup. + """ + + output_bam: pathlib.Path + output_fastq: pathlib.Path + r1_fastq: List[pathlib.Path] + r2_fastq: List[pathlib.Path] + readgroup: str + extract_model: pathlib.Path + bwa_model: pathlib.Path + unzip: str = "gzip" + name: str = "bwa-extract" + task_name: str = "alignment" + + def build(self) -> AlignResult: + job = Job( + cmds.cmd_bwa_extract( + self.output_bam, + self.output_fastq, + self.ctx.reference, + self.r1_fastq, + self.r2_fastq, + self.readgroup, + self.extract_model, + self.bwa_model, + self.ctx.cores, + unzip=self.unzip, + ), + self.name, + self.ctx.cores, + task_name=self.task_name, + ) + return AlignResult( + jobs=[job], + terminal={job}, + outputs=[self.output_bam, self.output_fastq], + cleanup_paths=[], + ) + + +@dataclass(kw_only=True) +class ReadWriterStage(AlignmentStage): + """Rewrite alignments with `sentieon driver --algo ReadWriter`. + + With several inputs the driver merges them; with an ``interval`` it + extracts the reads overlapping that region. + """ + + inputs: List[pathlib.Path] + output: pathlib.Path + name: str + task_name: str + threads: Optional[int] = None + interval: Optional[Union[pathlib.Path, str]] = None + + def build(self) -> AlignResult: + job = driver_job( + self.ctx, + [ReadWriter(self.output)], + inputs=self.inputs, + interval=self.interval, + name=self.name, + task_name=self.task_name, + threads=self.threads, + ) + return AlignResult( + jobs=[job], + terminal={job}, + outputs=[self.output], + cleanup_paths=[], + ) diff --git a/sentieon_cli/stages/base.py b/sentieon_cli/stages/base.py new file mode 100644 index 0000000..ecee86f --- /dev/null +++ b/sentieon_cli/stages/base.py @@ -0,0 +1,127 @@ +""" +Building blocks for pipeline stages + +A *stage* is a reusable piece of a pipeline: it knows how to build the jobs +for one operation (dedup, metrics, small-variant calling, ...) and how to +wire them into a :class:`~sentieon_cli.dag.DAG`. Stages are configured with +plain dataclass fields plus a :class:`StageContext` of run-wide settings. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional, Sequence, Set, Union + +from ..dag import DAG +from ..driver import BaseAlgo, Driver +from ..job import Job +from ..shell_pipeline import Command, Pipeline + + +@dataclass(frozen=True) +class StageContext: + """Run-wide settings every stage needs.""" + + reference: pathlib.Path + output_vcf: pathlib.Path + tmp_dir: pathlib.Path + cores: int + dry_run: bool + skip_version_check: bool + + +@dataclass(kw_only=True) +class StageResult: + """The jobs a stage added to the DAG. + + * ``jobs`` -- every job the stage inserted, in insertion order. It + exists for the occasional edge that has to attach to a job in the + middle of a stage rather than to its output. + * ``terminal`` -- the stage's sink job(s), the ones that carry its + declared outputs. Downstream stages depend on these. + + Subclasses add named fields for the files the stage produces. + """ + + jobs: List[Job] + terminal: Set[Job] + + +@dataclass(kw_only=True) +class Stage(ABC): + """A reusable group of jobs that inserts itself into a DAG. + + Subclasses are dataclasses (``@dataclass(kw_only=True)``) that add + their own configuration fields and return a :class:`StageResult` + subclass naming the outputs they produce. + + The :meth:`add_to` contract: + + * ``upstream`` is any iterable of jobs already in the DAG; it is + consumed once into a set. + * Every *entry* job -- an in-stage job with no in-stage dependency -- + is inserted with that set as its dependencies. Edges between the + stage's own jobs are added on top of that. + * Insertion is topological (a job is added only after the jobs it + depends on) and deterministic: the same stage and DAG produce the + same jobs, in the same order, every time. + * The jobs are created inside ``add_to``; constructing a stage has no + side effects and assigns no job ids. + * ``add_to`` may be called only once per stage instance. Job identity + is the shell pipeline, so a second call raises in + :meth:`~sentieon_cli.dag.DAG.add_job`. + """ + + ctx: StageContext + + @abstractmethod + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> StageResult: + """Add this stage's jobs to ``dag``, downstream of ``upstream``""" + + +def driver_job( + ctx: StageContext, + algos: Sequence[BaseAlgo], + *, + name: str, + task_name: str, + threads: Optional[int] = None, + inputs: Optional[List[pathlib.Path]] = None, + interval: Optional[Union[pathlib.Path, str]] = None, + interval_padding: Optional[int] = None, + read_filter: Optional[List[str]] = None, + replace_rg: Optional[List[List[str]]] = None, +) -> Job: + """A job running `sentieon driver` with ``algos``. + + ``threads`` defaults to the run's core count; the driver arguments are + passed through untouched so the command matches what the pipelines + build by hand. + """ + driver = Driver( + reference=ctx.reference, + thread_count=ctx.cores, + interval=interval, + interval_padding=interval_padding, + read_filter=read_filter, + replace_rg=replace_rg, + input=inputs, + ) + for algo in algos: + driver.add_algo(algo) + return Job( + Pipeline(Command(*driver.build_cmd())), + name, + ctx.cores if threads is None else threads, + task_name=task_name, + ) + + +def rm_job(paths: Iterable[Union[pathlib.Path, str]], name: str) -> Job: + """A job removing intermediate files, tolerating missing ones""" + return Job( + Pipeline(Command("rm", *[str(p) for p in paths], fail_ok=True)), + name, + 0, + task_name="cleanup", + ) diff --git a/sentieon_cli/stages/cnv.py b/sentieon_cli/stages/cnv.py new file mode 100644 index 0000000..91904ad --- /dev/null +++ b/sentieon_cli/stages/cnv.py @@ -0,0 +1,95 @@ +""" +CNV calling with CNVscope +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +import packaging.version + +from ..dag import DAG +from ..driver import CNVModelApply, CNVscope +from ..job import Job +from ..util import SampleSex, cnvscope_sex_args +from .base import Stage, StageResult, driver_job + +CNV_MIN_VERSIONS = { + # 202503.04 adds the CNVscope `--sex` and `--par` arguments + "sentieon driver": packaging.version.Version("202503.04"), +} + + +@dataclass(kw_only=True) +class CNVResult(StageResult): + """The jobs and output of `CNVscopeStage`""" + + cnvscope_job: Job + apply_job: Job + cnv_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class CNVscopeStage(Stage): + """Call CNVs with CNVscope, then filter them with CNVModelApply. + + The sample sex has to be known before the jobs are built, so callers + run this stage after ploidy estimation. Version checks stay with the + pipelines, which run them during validation. + """ + + inputs: List[pathlib.Path] + model: pathlib.Path + cnvscope_vcf: pathlib.Path + cnv_vcf: pathlib.Path + sample_sex: Optional[SampleSex] = None + par_bed: Optional[pathlib.Path] = None + interval: Optional[pathlib.Path] = None + replace_rg: Optional[List[List[str]]] = None + name: str = "cnvscope" + apply_name: str = "cnv-model-apply" + task_name: str = "cnv" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> CNVResult: + deps = set(upstream) + sex, par = cnvscope_sex_args(self.sample_sex, self.par_bed) + + cnvscope = driver_job( + self.ctx, + [ + CNVscope( + self.cnvscope_vcf, + self.model, + sex=sex, + par=par, + ) + ], + inputs=self.inputs, + interval=self.interval, + replace_rg=self.replace_rg, + name=self.name, + task_name=self.task_name, + ) + apply_job = driver_job( + self.ctx, + [ + CNVModelApply( + self.cnv_vcf, + self.model, + vcf=self.cnvscope_vcf, + ) + ], + name=self.apply_name, + task_name=self.task_name, + ) + + dag.add_job(cnvscope, deps) + dag.add_job(apply_job, {cnvscope}) + + return CNVResult( + jobs=[cnvscope, apply_job], + terminal={apply_job}, + cnvscope_job=cnvscope, + apply_job=apply_job, + cnv_vcf=self.cnv_vcf, + ) diff --git a/sentieon_cli/stages/dedup.py b/sentieon_cli/stages/dedup.py new file mode 100644 index 0000000..534dd19 --- /dev/null +++ b/sentieon_cli/stages/dedup.py @@ -0,0 +1,94 @@ +""" +Duplicate marking with LocusCollector and Dedup +""" + +from dataclasses import dataclass, field +import pathlib +from typing import Iterable, List, Optional + +from ..dag import DAG +from ..driver import BaseAlgo, Dedup, LocusCollector +from ..job import Job +from .base import Stage, StageResult, driver_job + + +@dataclass(kw_only=True) +class DedupResult(StageResult): + """The jobs and output of `DedupStage`""" + + lc_job: Job + dedup_job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class DedupStage(Stage): + """Mark duplicates: LocusCollector collects the scores, Dedup applies + them. + + Both passes read the same input files with the same read filters, so + the two jobs only differ in their algo. Metrics that are cheap to + collect before duplicate marking can ride along on the LocusCollector + pass through ``lc_extra_algos``. + """ + + inputs: List[pathlib.Path] + output: pathlib.Path + score_file: pathlib.Path + tag: Optional[str] = None + consensus: bool = False + rmdup: bool = False + cram_write_options: Optional[str] = None + dedup_metrics: Optional[pathlib.Path] = None + read_filters: List[str] = field(default_factory=list) + lc_extra_algos: List[BaseAlgo] = field(default_factory=list) + threads: Optional[int] = None + task_name: str = "dedup" + + def _name(self, base: str) -> str: + return base if self.tag is None else f"{base}-{self.tag}" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> DedupResult: + deps = set(upstream) + read_filter = self.read_filters or None + + lc_job = driver_job( + self.ctx, + [ + LocusCollector(self.score_file, consensus=self.consensus), + *self.lc_extra_algos, + ], + inputs=self.inputs, + read_filter=read_filter, + name=self._name("locuscollector"), + task_name=self.task_name, + threads=self.threads, + ) + dedup_job = driver_job( + self.ctx, + [ + Dedup( + self.output, + self.score_file, + cram_write_options=self.cram_write_options, + metrics=self.dedup_metrics, + rmdup=self.rmdup, + ) + ], + inputs=self.inputs, + read_filter=read_filter, + name=self._name("dedup"), + task_name=self.task_name, + threads=self.threads, + ) + + dag.add_job(lc_job, deps) + dag.add_job(dedup_job, {lc_job}) + + return DedupResult( + jobs=[lc_job, dedup_job], + terminal={dedup_job}, + lc_job=lc_job, + dedup_job=dedup_job, + output=self.output, + ) diff --git a/sentieon_cli/stages/metrics.py b/sentieon_cli/stages/metrics.py new file mode 100644 index 0000000..e03c9a3 --- /dev/null +++ b/sentieon_cli/stages/metrics.py @@ -0,0 +1,233 @@ +""" +Alignment metrics collection +""" + +from dataclasses import dataclass +from importlib.resources import files +import pathlib +import sys +from typing import Iterable, List, Optional + +import packaging.version + +from .. import command_strings as cmds +from ..dag import DAG +from ..driver import BaseAlgo +from ..job import Job +from ..shell_pipeline import Command, Pipeline +from .base import Stage, StageResult, driver_job + +MOSDEPTH_MIN_VERSIONS = { + "mosdepth": packaging.version.Version("0.2.6"), +} + + +@dataclass(frozen=True) +class MetricsPaths: + """The metrics directory and the files the pipelines write into it. + + Every metrics file is named after the output VCF, so the whole set is + derived once, from :meth:`from_output_vcf`, and passed around instead + of being rebuilt at each call site. + """ + + metrics_dir: pathlib.Path + metric_base: str + + @classmethod + def from_output_vcf(cls, output_vcf: pathlib.Path) -> "MetricsPaths": + """The metrics paths belonging to an output VCF""" + return cls( + metrics_dir=pathlib.Path( + str(output_vcf).replace(".vcf.gz", "_metrics") + ), + metric_base=output_vcf.name.replace(".vcf.gz", "") + ".txt", + ) + + def ensure_dir(self, dry_run: bool) -> None: + """Create the metrics directory, unless this is a dry run""" + if not dry_run: + self.metrics_dir.mkdir(exist_ok=True) + + def _metric(self, suffix: str) -> pathlib.Path: + return self.metrics_dir.joinpath(self.metric_base + suffix) + + @property + def score(self) -> pathlib.Path: + """The LocusCollector score file""" + return self._metric(".score.txt.gz") + + @property + def insert_size(self) -> pathlib.Path: + """InsertSizeMetricAlgo output""" + return self._metric(".insert_size.txt") + + @property + def mean_qual_by_cycle(self) -> pathlib.Path: + """MeanQualityByCycle output""" + return self._metric(".mean_qual_by_cycle.txt") + + @property + def base_distribution_by_cycle(self) -> pathlib.Path: + """BaseDistributionByCycle output""" + return self._metric(".base_distribution_by_cycle.txt") + + @property + def qual_distribution(self) -> pathlib.Path: + """QualDistribution output""" + return self._metric(".qual_distribution.txt") + + @property + def alignment_stat(self) -> pathlib.Path: + """AlignmentStat output""" + return self._metric(".alignment_stat.txt") + + @property + def coverage(self) -> pathlib.Path: + """CoverageMetrics output prefix (not named after the VCF)""" + return self.metrics_dir.joinpath("coverage") + + @property + def hybrid_selection(self) -> pathlib.Path: + """HsMetricAlgo output, for WES runs""" + return self._metric(".hybrid-selection.txt") + + @property + def wgs(self) -> pathlib.Path: + """WgsMetricsAlgo output, for WGS runs""" + return self._metric(".wgs.txt") + + @property + def gc_bias(self) -> pathlib.Path: + """GCBias output""" + return self._metric(".gc_bias.txt") + + @property + def gc_bias_summary(self) -> pathlib.Path: + """The GCBias summary output""" + return self._metric(".gc_bias_summary.txt") + + @property + def dedup_metrics(self) -> pathlib.Path: + """The Dedup metrics output""" + return self._metric(".dedup_metrics.txt") + + +@dataclass(kw_only=True) +class MetricsResult(StageResult): + """The jobs of `MetricsStage`""" + + metrics_job: Job + rehead_job: Optional[Job] + + +@dataclass(kw_only=True) +class MetricsStage(Stage): + """Collect alignment metrics with a single `sentieon driver` pass. + + The caller supplies the ordered algo list, since which metrics are + worth collecting depends on the assay and on whether the alignment + has been deduplicated yet. When ``rehead_metrics`` names a WGS + metrics file, a second job rewrites its header so MultiQC recognizes + it. + """ + + inputs: List[pathlib.Path] + algos: List[BaseAlgo] + interval: Optional[pathlib.Path] = None + rehead_metrics: Optional[pathlib.Path] = None + name: str = "metrics" + task_name: str = "metrics" + threads: int = 0 + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> MetricsResult: + deps = set(upstream) + + metrics = driver_job( + self.ctx, + self.algos, + inputs=self.inputs, + interval=self.interval, + name=self.name, + task_name=self.task_name, + threads=self.threads, + ) + dag.add_job(metrics, deps) + jobs = [metrics] + + rehead: Optional[Job] = None + if self.rehead_metrics is not None: + rehead_script = pathlib.Path( + str( + files("sentieon_cli.scripts").joinpath( + "rehead_wgs_metrics.py" + ) + ) + ) + rehead = Job( + Pipeline( + Command( + sys.executable, + str(rehead_script), + "--metrics_file", + str(self.rehead_metrics), + ) + ), + "rehead-metrics", + 0, + task_name="metrics", + ) + dag.add_job(rehead, {metrics}) + jobs.append(rehead) + + return MetricsResult( + jobs=jobs, + terminal={rehead} if rehead else {metrics}, + metrics_job=metrics, + rehead_job=rehead, + ) + + +@dataclass(kw_only=True) +class MosdepthResult(StageResult): + """The jobs of `MosdepthStage`""" + + +@dataclass(kw_only=True) +class MosdepthStage(Stage): + """Coverage QC with mosdepth, one job per input file. + + The jobs are independent of each other, so ``build`` constructs them + all and ``add_to`` inserts each with the same dependencies. + """ + + inputs: List[pathlib.Path] + + def build(self) -> MosdepthResult: + """Construct this stage's jobs without touching a DAG""" + jobs: List[Job] = [] + for i, input_file in enumerate(self.inputs): + mosdepth_dir = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", f"_mosdepth_{i}") + ) + jobs.append( + Job( + cmds.cmd_mosdepth( + input_file, + mosdepth_dir, + fasta=self.ctx.reference, + threads=self.ctx.cores, + ), + f"mosdepth-{i}", + 0, # Run in background + task_name="metrics", + ) + ) + return MosdepthResult(jobs=jobs, terminal=set(jobs)) + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> MosdepthResult: + deps = set(upstream) + result = self.build() + for job in result.jobs: + dag.add_job(job, deps) + return result diff --git a/sentieon_cli/stages/ploidy.py b/sentieon_cli/stages/ploidy.py new file mode 100644 index 0000000..49e87da --- /dev/null +++ b/sentieon_cli/stages/ploidy.py @@ -0,0 +1,67 @@ +""" +Sample ploidy and sex estimation +""" + +from dataclasses import dataclass +from importlib.resources import files +import pathlib +from typing import Iterable, List, Optional + +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from ..shard import ploidy_contigs_for_build +from .base import Stage, StageResult + + +@dataclass(kw_only=True) +class PloidyResult(StageResult): + """The job and output of `PloidyStage`""" + + job: Job + ploidy_json: pathlib.Path + + +@dataclass(kw_only=True) +class PloidyStage(Stage): + """Estimate the sample ploidy and sex from an alignment. + + The JSON report is written next to the output VCF and is read after + the DAG has run, by the pipelines that call variants with a + sex-aware caller. + """ + + inputs: List[pathlib.Path] + reference_build: Optional[str] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> PloidyResult: + deps = set(upstream) + ploidy_json = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", "_ploidy.json") + ) + estimate_ploidy = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) + ).resolve() + ploidy_contigs = ploidy_contigs_for_build(self.reference_build) + job = Job( + cmds.cmd_estimate_ploidy( + ploidy_json, + self.inputs, + estimate_ploidy, + contigs=ploidy_contigs.contigs, + autosomes=ploidy_contigs.autosomes, + x_contig=ploidy_contigs.x_contig, + y_contig=ploidy_contigs.y_contig, + ), + "estimate-ploidy", + 0, + task_name="ploidy", + ) + dag.add_job(job, deps) + + return PloidyResult( + jobs=[job], + terminal={job}, + job=job, + ploidy_json=ploidy_json, + ) diff --git a/sentieon_cli/stages/preprocessing.py b/sentieon_cli/stages/preprocessing.py new file mode 100644 index 0000000..f2a29bd --- /dev/null +++ b/sentieon_cli/stages/preprocessing.py @@ -0,0 +1,179 @@ +""" +Short-read preprocessing: duplicate marking and metrics collection +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +from ..dag import DAG +from ..driver import ( + AlignmentStat, + BaseAlgo, + BaseDistributionByCycle, + CoverageMetrics, + GCBias, + HsMetricAlgo, + InsertSizeMetricAlgo, + MeanQualityByCycle, + QualDistribution, + WgsMetricsAlgo, +) +from ..job import Job +from .base import Stage, StageResult +from .dedup import DedupStage +from .metrics import MetricsPaths, MetricsStage + + +def pre_dedup_metrics_algos(paths: MetricsPaths, assay: str) -> List[BaseAlgo]: + """The metrics that do not need duplicate-marked reads""" + algos: List[BaseAlgo] = [ + MeanQualityByCycle(paths.mean_qual_by_cycle), + BaseDistributionByCycle(paths.base_distribution_by_cycle), + QualDistribution(paths.qual_distribution), + AlignmentStat(paths.alignment_stat), + ] + if assay == "WGS": + algos.append(GCBias(paths.gc_bias, summary=paths.gc_bias_summary)) + return algos + + +@dataclass(kw_only=True) +class SrPreprocessingResult(StageResult): + """The short-read dedup and metrics jobs added to a DAG. + + * ``deduped`` -- the alignment to call variants from; the input files + unchanged when duplicate marking is off. + * ``dedup_job`` -- the Dedup job, absent when duplicate marking is off. + * ``qc_jobs`` -- every metrics-producing job, for MultiQC to wait on. + """ + + deduped: List[pathlib.Path] + dedup_job: Optional[Job] + qc_jobs: List[Job] + + +@dataclass(kw_only=True) +class ShortReadPreprocessingStage(Stage): + """Mark duplicates and collect metrics from the short reads. + + Which metrics are collected, and whether they ride along on the + LocusCollector pass or run after duplicate marking, depends on the + assay and on whether a BED file restricts the run. + + ``terminal`` is the Dedup job, so downstream stages read the + deduplicated alignment. With ``duplicate_marking="none"`` there is no + Dedup job and the stage adds only a leaf metrics job that nothing + should wait on, so ``terminal`` is empty and callers depend on the + jobs that produced ``inputs`` instead. + """ + + inputs: List[pathlib.Path] + duplicate_marking: str = "markdup" + consensus: bool = False + skip_metrics: bool = False + assay: str = "WGS" + bed: Optional[pathlib.Path] = None + bam_format: bool = False + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> SrPreprocessingResult: + ctx = self.ctx + suffix = "bam" if self.bam_format else "cram" + + paths = MetricsPaths.from_output_vcf(ctx.output_vcf) + paths.ensure_dir(ctx.dry_run) + + if self.duplicate_marking == "none": + # Without duplicate marking there is nothing to collect after + # the fact, so every metric comes from one pass over the input + if self.skip_metrics: + return SrPreprocessingResult( + jobs=[], + terminal=set(), + deduped=self.inputs, + dedup_job=None, + qc_jobs=[], + ) + metrics_result = MetricsStage( + ctx=ctx, + inputs=self.inputs, + algos=[ + InsertSizeMetricAlgo(paths.insert_size), + *pre_dedup_metrics_algos(paths, self.assay), + ], + name="metrics", + task_name="metrics", + threads=ctx.cores, + ).add_to(dag, upstream) + return SrPreprocessingResult( + jobs=list(metrics_result.jobs), + terminal=set(), + deduped=self.inputs, + dedup_job=None, + qc_jobs=list(metrics_result.jobs), + ) + + # Metrics that ride along on the LocusCollector pass + lc_extra_algos: List[BaseAlgo] = [] + if not self.skip_metrics: + # Prefer to run InsertSizeMetricAlgo after duplicate marking + if self.assay == "WES" and not self.bed: + lc_extra_algos.append(InsertSizeMetricAlgo(paths.insert_size)) + lc_extra_algos.extend(pre_dedup_metrics_algos(paths, self.assay)) + + deduped = pathlib.Path( + str(ctx.output_vcf).replace(".vcf.gz", f"_deduped.{suffix}") + ) + dedup_result = DedupStage( + ctx=ctx, + inputs=self.inputs, + output=deduped, + score_file=paths.score, + consensus=self.consensus, + rmdup=(self.duplicate_marking == "rmdup"), + cram_write_options="version=3.0,compressor=rans", + dedup_metrics=paths.dedup_metrics, + lc_extra_algos=lc_extra_algos, + ).add_to(dag, upstream) + jobs: List[Job] = list(dedup_result.jobs) + qc_jobs: List[Job] = [dedup_result.lc_job] + + # HsMetricAlgo and WgsMetricsAlgo run after duplicate marking to + # account for duplicate reads + post_algos: List[BaseAlgo] = [] + rehead_metrics: Optional[pathlib.Path] = None + if not self.skip_metrics: + if self.assay == "WES" and self.bed: + post_algos = [ + HsMetricAlgo(paths.hybrid_selection, self.bed, self.bed), + InsertSizeMetricAlgo(paths.insert_size), + ] + elif self.assay == "WGS": + post_algos = [ + InsertSizeMetricAlgo(paths.insert_size), + WgsMetricsAlgo(paths.wgs, include_unpaired="true"), + CoverageMetrics(paths.coverage), + ] + # Rehead WGS metrics so they are recognized by MultiQC + rehead_metrics = paths.wgs + if post_algos: + metrics_result = MetricsStage( + ctx=ctx, + inputs=[deduped], + algos=post_algos, + interval=self.bed, + rehead_metrics=rehead_metrics, + threads=0, # Run metrics in the background + ).add_to(dag, {dedup_result.dedup_job}) + jobs.extend(metrics_result.jobs) + qc_jobs.extend(metrics_result.jobs) + + return SrPreprocessingResult( + jobs=jobs, + terminal={dedup_result.dedup_job}, + deduped=[deduped], + dedup_job=dedup_result.dedup_job, + qc_jobs=qc_jobs, + ) diff --git a/sentieon_cli/stages/small_variants.py b/sentieon_cli/stages/small_variants.py new file mode 100644 index 0000000..4880b55 --- /dev/null +++ b/sentieon_cli/stages/small_variants.py @@ -0,0 +1,235 @@ +""" +Small-variant calling, annotation transfer, and genotyping +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional, Set, Union + +from ..dag import DAG +from ..driver import BaseAlgo, DNAModelApply, GVCFtyper +from ..job import Job +from .base import Stage, StageResult, driver_job +from .transfer import TransferConfig, TransferResult, TransferStage + + +@dataclass(kw_only=True) +class DriverResult(StageResult): + """The job of a single-job `sentieon driver` stage""" + + job: Job + + +@dataclass(kw_only=True) +class DriverStage(Stage): + """A single `sentieon driver` invocation, wired into the DAG. + + The driver arguments are passed through untouched; a stage that needs + more structure than "run these algos" subclasses this one and fixes + the defaults it cares about. + """ + + algos: List[BaseAlgo] + name: str + task_name: str + threads: Optional[int] = None + inputs: Optional[List[pathlib.Path]] = None + interval: Optional[Union[pathlib.Path, str]] = None + interval_padding: Optional[int] = None + read_filter: Optional[List[str]] = None + replace_rg: Optional[List[List[str]]] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> DriverResult: + job = driver_job( + self.ctx, + self.algos, + name=self.name, + task_name=self.task_name, + threads=self.threads, + inputs=self.inputs, + interval=self.interval, + interval_padding=self.interval_padding, + read_filter=self.read_filter, + replace_rg=self.replace_rg, + ) + dag.add_job(job, set(upstream)) + return DriverResult(jobs=[job], terminal={job}, job=job) + + +@dataclass(kw_only=True) +class DNAscopeStage(DriverStage): + """The DNAscope-family calling pass. + + The caller supplies the algos in the order the driver should see + them -- SNV/indel calling first, then a BND or PangenomeSV pass in + the same pileup. + """ + + name: str = "dnascope" + task_name: str = "variant-calling" + + +@dataclass(kw_only=True) +class ModelApplyResult(StageResult): + """The job and output of `ModelApplyStage`""" + + job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class ModelApplyStage(Stage): + """Genotype and filter a raw VCF with DNAModelApply""" + + model: pathlib.Path + vcf: pathlib.Path + output: pathlib.Path + name: str = "model-apply" + task_name: str = "model-apply" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> ModelApplyResult: + job = driver_job( + self.ctx, + [DNAModelApply(self.model, self.vcf, self.output)], + name=self.name, + task_name=self.task_name, + ) + dag.add_job(job, set(upstream)) + return ModelApplyResult( + jobs=[job], + terminal={job}, + job=job, + output=self.output, + ) + + +@dataclass(kw_only=True) +class GVCFtyperResult(StageResult): + """The job and output of `GVCFtyperStage`""" + + job: Job + output: pathlib.Path + + +@dataclass(kw_only=True) +class GVCFtyperStage(Stage): + """Genotype a gVCF to produce a VCF""" + + gvcf: pathlib.Path + output: pathlib.Path + interval: Optional[pathlib.Path] = None + name: str = "gvcftyper" + task_name: str = "gvcftyper" + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> GVCFtyperResult: + job = driver_job( + self.ctx, + [GVCFtyper(output=self.output, vcf=self.gvcf)], + interval=self.interval, + name=self.name, + task_name=self.task_name, + ) + dag.add_job(job, set(upstream)) + return GVCFtyperResult( + jobs=[job], + terminal={job}, + job=job, + output=self.output, + ) + + +@dataclass(frozen=True) +class TransferSpec: + """Ask `TransferApplyStage` for an annotation transfer""" + + config: TransferConfig + out_vcf: pathlib.Path + tag: Optional[str] = None + + +@dataclass(frozen=True) +class ApplySpec: + """Ask `TransferApplyStage` for a DNAModelApply pass""" + + model: pathlib.Path + output: pathlib.Path + name: str = "model-apply" + + +@dataclass(kw_only=True) +class TransferApplyResult(StageResult): + """The jobs and output of `TransferApplyStage`""" + + transfer: Optional[TransferResult] + apply_job: Optional[Job] + output: pathlib.Path + + +@dataclass(kw_only=True) +class TransferApplyStage(Stage): + """Post-process a raw VCF: transfer annotations, then apply the model. + + Either half may be omitted. When both run, DNAModelApply reads the + transfer output and waits on the transfer alone -- the transfer's + concat job already depends on the raw-VCF producer. Without a + transfer, DNAModelApply waits on ``upstream``. + """ + + raw_vcf: pathlib.Path + transfer: Optional[TransferSpec] = None + apply: Optional[ApplySpec] = None + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> TransferApplyResult: + deps = set(upstream) + if self.transfer is None and self.apply is None: + raise ValueError( + "TransferApplyStage needs a transfer, an apply, or both" + ) + + jobs: List[Job] = [] + transfer_result: Optional[TransferResult] = None + apply_in = self.raw_vcf + apply_deps = deps + if self.transfer is not None: + transfer_result = TransferStage( + ctx=self.ctx, + config=self.transfer.config, + raw_vcf=self.raw_vcf, + out_vcf=self.transfer.out_vcf, + tag=self.transfer.tag, + ).add_to(dag, deps) + jobs.extend(transfer_result.jobs) + apply_in = self.transfer.out_vcf + apply_deps = transfer_result.terminal + + apply_job: Optional[Job] = None + if self.apply is not None: + apply_result = ModelApplyStage( + ctx=self.ctx, + model=self.apply.model, + vcf=apply_in, + output=self.apply.output, + name=self.apply.name, + ).add_to(dag, apply_deps) + apply_job = apply_result.job + jobs.append(apply_job) + terminal: Set[Job] = {apply_job} + output = self.apply.output + else: + assert transfer_result is not None + terminal = transfer_result.terminal + output = transfer_result.out_vcf + + return TransferApplyResult( + jobs=jobs, + terminal=terminal, + transfer=transfer_result, + apply_job=apply_job, + output=output, + ) diff --git a/sentieon_cli/stages/sv.py b/sentieon_cli/stages/sv.py new file mode 100644 index 0000000..c722172 --- /dev/null +++ b/sentieon_cli/stages/sv.py @@ -0,0 +1,88 @@ +""" +Structural variant calling with LongReadSV +""" + +from dataclasses import dataclass +import pathlib +from typing import Iterable, List, Optional + +import packaging.version + +from ..dag import DAG +from ..driver import LongReadSV +from ..job import Job +from .base import Stage, StageResult, driver_job + +LONGREADSV_MIN_VERSIONS = { + "sentieon driver": packaging.version.Version("202308"), +} + + +@dataclass(kw_only=True) +class LongReadSVResult(StageResult): + """The job and output of `LongReadSVStage`""" + + job: Job + sv_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class LongReadSVStage(Stage): + """Call SVs from long reads with Sentieon LongReadSV. + + ``output`` names the SV VCF; when it is None the path is derived from + the run's output VCF, so the caller only supplies the alignment and + the model. The remaining fields are LongReadSV algo options, emitted + only when set. + """ + + inputs: List[pathlib.Path] + model: pathlib.Path + output: Optional[pathlib.Path] = None + min_map_qual: Optional[int] = None + min_sv_size: Optional[int] = None + min_dp: Optional[int] = None + min_af: Optional[float] = None + interval: Optional[pathlib.Path] = None + replace_rg: Optional[List[List[str]]] = None + name: str = "LongReadSV" + task_name: str = "sv-calling" + + def build(self) -> LongReadSVResult: + """Construct this stage's job without touching a DAG""" + sv_vcf = self.output + if sv_vcf is None: + sv_vcf = pathlib.Path( + str(self.ctx.output_vcf).replace(".vcf.gz", ".sv.vcf.gz") + ) + job = driver_job( + self.ctx, + [ + LongReadSV( + sv_vcf, + model=self.model, + min_map_qual=self.min_map_qual, + min_sv_size=self.min_sv_size, + min_dp=self.min_dp, + min_af=self.min_af, + ) + ], + inputs=self.inputs, + interval=self.interval, + replace_rg=self.replace_rg, + name=self.name, + task_name=self.task_name, + ) + return LongReadSVResult( + jobs=[job], + terminal={job}, + job=job, + sv_vcf=sv_vcf, + ) + + def add_to( + self, dag: DAG, upstream: Iterable[Job] = () + ) -> LongReadSVResult: + result = self.build() + dag.add_job(result.job, set(upstream)) + return result diff --git a/sentieon_cli/transfer.py b/sentieon_cli/stages/transfer.py similarity index 52% rename from sentieon_cli/transfer.py rename to sentieon_cli/stages/transfer.py index ed082fb..b6b9fff 100644 --- a/sentieon_cli/transfer.py +++ b/sentieon_cli/stages/transfer.py @@ -1,19 +1,22 @@ """ -Annotation transfer functionality +Annotation transfer from a population VCF """ +from dataclasses import dataclass import pathlib import re import subprocess as sp import tempfile -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, Iterable, List, Optional, Protocol, Set, Tuple from importlib.resources import files -from . import command_strings as cmds -from .job import Job -from .logging import get_logger -from .shard import Shard +from .. import command_strings as cmds +from ..dag import DAG +from ..job import Job +from ..logging import get_logger +from ..shard import Shard +from .base import Stage, StageResult logger = get_logger(__name__) @@ -28,8 +31,17 @@ def build_transfer_jobs( fai_data: Dict[str, Dict[str, int]], dry_run: bool = False, cores: int = 1, + *, + tag: Optional[str] = None, ) -> Tuple[List[Job], Job]: - """Transfer annotations from the pop_vcf to the raw_vcf""" + """Transfer annotations from the pop_vcf to the raw_vcf. + + ``tag`` is inserted into the job names (``merge-trim-{tag}-...``) so a + pipeline that transfers annotations more than once in a run can tell + the two fan-outs apart. + """ + + name_base = "merge-trim" if tag is None else f"merge-trim-{tag}" # Get a unique tmpdir tmp_dir_str = tempfile.mkdtemp(dir=base_tmp_dir) @@ -87,7 +99,7 @@ def build_transfer_jobs( raw_vcf, regions_file=subset_bed, ), - "merge-trim-extra", + f"{name_base}-extra", 1, task_name="annotation-transfer", ) @@ -122,7 +134,7 @@ def build_transfer_jobs( ], view_xargs=["--no-version"], ), - f"merge-trim-{i}", + f"{name_base}-{i}", 1, task_name="annotation-transfer", ) @@ -136,8 +148,107 @@ def build_transfer_jobs( sharded_vcfs, xargs=["--no-version", "--threads", str(cores)], ), - "merge-trim-concat", + f"{name_base}-concat", cores, task_name="annotation-transfer", ) return (sharded_merge_jobs, concat_job) + + +class TransferInputs(Protocol): + """A pipeline that carries the annotation transfer's inputs. + + Every pipeline that transfers annotations holds these four values under + these names, so `TransferConfig.from_pipeline` can collect them + structurally instead of each pipeline repeating the same helper. + """ + + pop_vcf: Optional[pathlib.Path] + shards: List[Shard] + pop_vcf_contigs: Dict[str, Optional[int]] + fai_data: Dict[str, Dict[str, int]] + + +@dataclass(frozen=True) +class TransferConfig: + """The run-wide inputs the annotation transfer needs. + + Every pipeline that transfers annotations builds the same fan-out from + the same four values, so they travel together instead of being threaded + through each call site. + """ + + pop_vcf: pathlib.Path + shards: List[Shard] + pop_vcf_contigs: Dict[str, Optional[int]] + fai_data: Dict[str, Dict[str, int]] + + @classmethod + def from_pipeline(cls, pipeline: TransferInputs) -> "TransferConfig": + """Collect the transfer's inputs from a pipeline. + + The four values are set together while the pipeline validates its + arguments, so they are read back together here. Callers check + `pop_vcf` before asking for a transfer at all. + """ + pop_vcf = pipeline.pop_vcf + if pop_vcf is None: + raise ValueError("The annotation transfer needs a population VCF") + return cls( + pop_vcf=pop_vcf, + shards=pipeline.shards, + pop_vcf_contigs=pipeline.pop_vcf_contigs, + fai_data=pipeline.fai_data, + ) + + +@dataclass(kw_only=True) +class TransferResult(StageResult): + """The jobs and output of `TransferStage`""" + + shard_jobs: List[Job] + concat_job: Job + out_vcf: pathlib.Path + + +@dataclass(kw_only=True) +class TransferStage(Stage): + """Transfer annotations from the population VCF onto a raw VCF. + + One job per genomic shard merges the shard's annotations, then a + single concat job assembles ``out_vcf``. ``tag`` disambiguates the + job names when a pipeline transfers more than once in a run. + """ + + config: TransferConfig + raw_vcf: pathlib.Path + out_vcf: pathlib.Path + tag: Optional[str] = None + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> TransferResult: + deps = set(upstream) + + shard_jobs, concat_job = build_transfer_jobs( + self.out_vcf, + self.config.pop_vcf, + self.raw_vcf, + self.ctx.tmp_dir, + self.config.shards, + self.config.pop_vcf_contigs, + self.config.fai_data, + self.ctx.dry_run, + self.ctx.cores, + tag=self.tag, + ) + + for job in shard_jobs: + dag.add_job(job, deps) + dag.add_job(concat_job, set(shard_jobs)) + + return TransferResult( + jobs=[*shard_jobs, concat_job], + terminal={concat_job}, + shard_jobs=shard_jobs, + concat_job=concat_job, + out_vcf=self.out_vcf, + ) diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index 2499def..e635991 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -9,11 +9,11 @@ import os import pathlib import re -import shlex import shutil import subprocess as sp +import sys import tempfile -from typing import Callable, Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Mapping, Optional, Tuple import packaging.version @@ -30,7 +30,6 @@ PRELOAD_SEP_PAT = re.compile(PRELOAD_SEP) NUMA_NODE_PAT = re.compile(r"^NUMA node. CPU\(s\):\s+(?P.*)$") -READ_LENGTH_PAT = re.compile(r"SN\taverage length:\t(?P\d*)$") _UNSAFE = re.compile(r"[^A-Za-z0-9._-]") @@ -137,6 +136,45 @@ def check_version( return True +def require_versions( + min_versions: Mapping[str, Optional[packaging.version.Version]], + *, + skip: bool = False, +) -> None: + """Exit unless every executable meets its minimum version. + + `check_version` has already logged the reason, so the exit is silent. + Pass `skip=True` (the pipelines' `--skip_version_check`) to do nothing. + + A `Mapping` rather than a `Dict`, so the module-level + `*_MIN_VERSIONS` constants -- some of which mypy infers as + `Dict[str, Version]`, with no `None` entry -- are accepted. + """ + if skip: + return + for cmd, min_version in min_versions.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + +def versions_available( + min_versions: Mapping[str, Optional[packaging.version.Version]], + *, + skip: bool = False, +) -> bool: + """Whether every executable meets its minimum version. + + For optional tools, where a missing or outdated executable skips a step + rather than ending the run. `skip=True` reports them as available. + """ + if skip: + return True + return all( + check_version(cmd, min_version) + for cmd, min_version in min_versions.items() + ) + + def path_arg( exists: Optional[bool] = None, is_dir: Optional[bool] = None, @@ -197,6 +235,36 @@ def total_memory() -> int: return total_mem +def set_bwt_max_mem( + total_input_size: int, + n_alignment_jobs: int = 1, + override: Optional[str] = None, +) -> str: + """Set the `bwt_max_mem` environment variable and return its value. + + `override` short-circuits the calculation, for the hidden + `--bwt_max_mem` argument. Otherwise the value is derived from the + memory available to the run, the size of the inputs staged in + memory, and the number of concurrent alignment jobs. + """ + if override: + os.environ["bwt_max_mem"] = override + return override + + total_mem = total_memory() + total_mem_gb = total_mem / (1024.0**3) + align_mem_gb = ( + total_mem_gb - 4 - total_input_size / (1024.0**3) * 2.3 + ) # some memory for other system processes + bwa_mem_gb = max( + int((align_mem_gb / n_alignment_jobs) - 6), 0 + ) # some memory for other alignment processes + logger.debug("Setting bwt_max_mem to: %sG", bwa_mem_gb) + bwt_max_mem = f"{bwa_mem_gb}G" + os.environ["bwt_max_mem"] = bwt_max_mem + return bwt_max_mem + + def find_numa_nodes() -> List[str]: """Find NUMA nodes on the system, if available""" numa_nodes = [] @@ -262,48 +330,6 @@ def parse_rg_line(rg_line: str) -> Dict[str, str]: return parsed -def get_read_length_aln( - aln: pathlib.Path, - reference: pathlib.Path, - n_reads: int = 100000, -) -> int: - """Get the average read length for an alignment file""" - cmds = [] - cmds.append( - [ - "samtools", - "view", - "-h", - "--reference", - shlex.quote(str(reference)), - shlex.quote(str(aln)), - ] - ) - cmds.append( - [ - "head", - "-n", - str(n_reads), - ] - ) - cmds.append(["samtools", "stats", "-"]) - all_cmds = [shlex.join(x) for x in cmds] - cmd = " | ".join(all_cmds) - res = sp.run( - cmd, - shell=True, - capture_output=True, - text=True, - executable="/bin/bash", - ) - - for line in res.stdout.split("\n"): - m = READ_LENGTH_PAT.match(line) - if m: - return int(m.groupdict()["length"]) - return 151 - - def vcf_id(in_vcf: pathlib.Path) -> Optional[str]: """Collect the SentieonVcfID header""" cmd = ["bcftools", "view", "-h", str(in_vcf)] diff --git a/tests/integration/test_pipeline_execution.py b/tests/integration/test_pipeline_execution.py index eba087e..9ed9e6a 100644 --- a/tests/integration/test_pipeline_execution.py +++ b/tests/integration/test_pipeline_execution.py @@ -152,7 +152,7 @@ def test_hybrid_dry_run_execution_rgsm(self): all_jobs = [job for job in list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys())] # with --rgsm should use replace_rg during first pass - first_pass_job = [job for job in all_jobs if job.name == "calling-1"][0] + first_pass_job = [job for job in all_jobs if job.name == "dnascope-1"][0] assert "--replace_rg" in first_pass_job.shell.nodes[0].args # CNV calling is sex-aware and runs in the second DAG @@ -160,7 +160,7 @@ def test_hybrid_dry_run_execution_rgsm(self): second_jobs = list(second_dag.waiting_jobs.keys()) + list(second_dag.ready_jobs.keys()) # should use replace_rg in CNV calling - cnv_job = [job for job in second_jobs if job.name == "CNVscope"][0] + cnv_job = [job for job in second_jobs if job.name == "cnvscope"][0] assert "--replace_rg" in cnv_job.shell.nodes[0].args # should use replace_rg in SV calling @@ -192,7 +192,7 @@ def test_dnascope_command_parameters(self): dag = pipeline.build_dag() # Find variant calling job - variant_job = DAGAnalyzer.get_job_by_name(dag, "variant-calling") + variant_job = DAGAnalyzer.get_job_by_name(dag, "dnascope") assert variant_job is not None, "Should have variant calling job" # Check that command includes expected parameters @@ -364,7 +364,7 @@ def test_variant_calling_dependencies(self): dag = pipeline.build_dag() # Variant calling should depend on either dedup or input processing - variant_jobs = DAGAnalyzer.get_jobs_by_type(dag, "variant") + variant_jobs = DAGAnalyzer.get_jobs_by_type(dag, "dnascope") assert len(variant_jobs) > 0, "Should have variant calling jobs" diff --git a/tests/unit/test_cnv_sex.py b/tests/unit/test_cnv_sex.py index fa256b0..39efb2c 100644 --- a/tests/unit/test_cnv_sex.py +++ b/tests/unit/test_cnv_sex.py @@ -12,10 +12,11 @@ import pytest from sentieon_cli import shard -from sentieon_cli.dnascope import call_cnvs from sentieon_cli.driver import CNVscope from sentieon_cli.pipeline import BasePipeline from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext +from sentieon_cli.stages.cnv import CNVscopeStage from sentieon_cli.shard import ( BUILD_SIGNATURES, detect_reference_build, @@ -288,13 +289,6 @@ def test_validate_cnv_par_errors_in_a_dry_run_too(): assert excinfo.value.code == 2 -def test_cnv_sex_args_for_a_male_sample_with_a_par_bed(): - par_bed = pathlib.Path("/par.bed") - pipeline = _pipeline(sample_sex=SampleSex.MALE, cnv_par_bed=par_bed) - - assert pipeline.cnv_sex_args() == ("M", par_bed) - - def test_resolve_cnv_par_bed_prefers_the_supplied_file(tmp_path): par_bed = tmp_path.joinpath("custom_par.bed") par_bed.write_text("chrX\t10000\t2781479\n") @@ -335,22 +329,31 @@ def test_resolve_cnv_par_bed_skips_the_lookup_without_cnv_calling(messages): assert not any("PAR" in msg for msg in messages) -# `call_cnvs` passes the sample sex through to CNVscope +# `CNVscopeStage` passes the sample sex through to CNVscope def _cnvscope_cmd(**kwargs) -> str: - cnvscope_job, _ = call_cnvs( - pathlib.Path("/tmp/cnv"), - pathlib.Path("/out/sample.vcf.gz"), - pathlib.Path("/ref/reference.fa"), - [pathlib.Path("/in/sample.cram")], - pathlib.Path("/bundle/model.bundle"), - None, - mp.cpu_count(), - True, # skip_version_check - **kwargs, + tmp_dir = pathlib.Path("/tmp/cnv") + output_vcf = pathlib.Path("/out/sample.vcf.gz") + ctx = StageContext( + reference=pathlib.Path("/ref/reference.fa"), + output_vcf=output_vcf, + tmp_dir=tmp_dir, + cores=mp.cpu_count(), + dry_run=True, + skip_version_check=True, ) - return str(cnvscope_job.shell) + result = CNVscopeStage( + ctx=ctx, + inputs=[pathlib.Path("/in/sample.cram")], + model=pathlib.Path("/bundle/model.bundle/cnv.model"), + cnvscope_vcf=tmp_dir.joinpath("cnvscope.vcf.gz"), + cnv_vcf=pathlib.Path( + str(output_vcf).replace(".vcf.gz", ".cnv.vcf.gz") + ), + **kwargs, + ).add_to(DAG()) + return str(result.cnvscope_job.shell) def test_call_cnvs_passes_the_sex_of_a_male_sample(): diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index c0df2cf..b7faa77 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -66,10 +66,10 @@ def create_basic_dnascope_pipeline(self): pipeline.reference = self.mock_ref pipeline.model_bundle = self.mock_bundle pipeline.sample_input = [self.mock_bam] - pipeline.sr_r1_fastq = [] - pipeline.sr_r2_fastq = [] - pipeline.sr_readgroups = [] - pipeline.sr_duplicate_marking = "markdup" + pipeline.r1_fastq = [] + pipeline.r2_fastq = [] + pipeline.readgroups = [] + pipeline.duplicate_marking = "markdup" pipeline.cores = 2 pipeline.dry_run = True # Important: use dry run to avoid actual execution pipeline.skip_version_check = True @@ -166,7 +166,7 @@ def test_skip_small_variants_dag(self, mock_lib_preloaded): job_names = [job.name for job in all_jobs] # Should not have variant calling jobs - variant_job_keywords = ["variant-calling", "model-apply", "gvcftyper"] + variant_job_keywords = ["dnascope", "model-apply", "gvcftyper"] for keyword in variant_job_keywords: assert not any(keyword in name for name in job_names) diff --git a/tests/unit/test_dnascope_hybrid_cnv.py b/tests/unit/test_dnascope_hybrid_cnv.py index 20c4597..b77903d 100644 --- a/tests/unit/test_dnascope_hybrid_cnv.py +++ b/tests/unit/test_dnascope_hybrid_cnv.py @@ -82,19 +82,13 @@ def create_pipeline(self): def build_dag(self, pipeline): """Build the first DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_dag() + dag = pipeline.build_dag() jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) return dag, {job.name: job for job in jobs} def build_second_dag(self, pipeline): """Build the second DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_second_dag() + dag = pipeline.build_second_dag() if dag is None: return None, {} jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) @@ -105,14 +99,14 @@ def test_cnv_jobs_run_in_the_second_dag(self): pipeline.sample_sex = SampleSex.FEMALE _dag, jobs = self.build_dag(pipeline) - assert "CNVscope" not in jobs - assert "CNVModelApply" not in jobs + assert "cnvscope" not in jobs + assert "cnv-model-apply" not in jobs second_dag, second_jobs = self.build_second_dag(pipeline) - assert "CNVscope" in second_jobs - assert "CNVModelApply" in second_jobs - assert second_dag.waiting_jobs[second_jobs["CNVModelApply"]] == { - second_jobs["CNVscope"] + assert "cnvscope" in second_jobs + assert "cnv-model-apply" in second_jobs + assert second_dag.waiting_jobs[second_jobs["cnv-model-apply"]] == { + second_jobs["cnvscope"] } def test_the_first_dag_estimates_the_sample_ploidy(self): @@ -164,7 +158,7 @@ def test_a_female_sample_is_called_without_a_par_bed(self): self.build_dag(pipeline) _dag, jobs = self.build_second_dag(pipeline) - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex F" in cmd_str assert "--par" not in cmd_str @@ -175,7 +169,7 @@ def test_a_male_sample_is_called_with_the_par_bed(self): self.build_dag(pipeline) _dag, jobs = self.build_second_dag(pipeline) - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex M" in cmd_str assert f"--par {self.mock_par_bed}" in cmd_str @@ -187,7 +181,7 @@ def test_a_dry_run_assumes_a_male_sample(self): _dag, jobs = self.build_second_dag(pipeline) assert pipeline.sample_sex is SampleSex.MALE - cmd_str = str(jobs["CNVscope"].shell) + cmd_str = str(jobs["cnvscope"].shell) assert "--sex M" in cmd_str assert f"--par {self.mock_par_bed}" in cmd_str diff --git a/tests/unit/test_dnascope_hybrid_longread.py b/tests/unit/test_dnascope_hybrid_longread.py new file mode 100644 index 0000000..7807977 --- /dev/null +++ b/tests/unit/test_dnascope_hybrid_longread.py @@ -0,0 +1,197 @@ +""" +Unit tests for the DNAscope hybrid pipeline's long-read jobs + +The hybrid pipeline used to inherit its long-read realignment, mosdepth and +SV calling from `DNAscopeLRPipeline`; it now composes the same stages +itself, with its own argument defaults. +""" + +import pathlib +import tempfile +from abc import ABC +from unittest.mock import MagicMock, patch + +import pytest + +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.pipeline import BasePipeline + +RG_LINES = ["@RG\tID:lr_rg1\tSM:sample1"] + + +class TestDNAscopeHybridLongRead: + """The long-read half of the hybrid DAG""" + + def setup_method(self): + self.temp_dir = tempfile.mkdtemp() + self.mock_dir = pathlib.Path(self.temp_dir) + + self.mock_vcf = self.mock_dir / "output.vcf.gz" + self.mock_ref = self.mock_dir / "reference.fa" + self.mock_input_ref = self.mock_dir / "input_reference.fa" + self.mock_sr_aln = [self.mock_dir / "short.bam"] + self.mock_lr_aln = [self.mock_dir / "long.bam"] + self.mock_bundle = self.mock_dir / "model.bundle" + self.mock_bed = self.mock_dir / "interval.bed" + + for file_path in [ + self.mock_ref, + self.mock_input_ref, + self.mock_sr_aln[0], + self.mock_lr_aln[0], + self.mock_bundle, + self.mock_bed, + ]: + file_path.touch() + + def create_pipeline(self): + """Create a DNAscopeHybridPipeline that realigns its long reads""" + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + pipeline.logger = MagicMock() + + pipeline.output_vcf = self.mock_vcf + pipeline.reference = self.mock_ref + pipeline.model_bundle = self.mock_bundle + pipeline.bed = self.mock_bed + pipeline.cores = 2 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = self.mock_dir + pipeline.pop_vcf = None + + # State normally set by validate() + pipeline.fai_data = {"chr1": {"length": 1000}} + pipeline.shards = [MagicMock()] + pipeline.shards[0].contig = "chr1" + pipeline.shards[0].start = 1 + pipeline.shards[0].stop = 1000 + pipeline.pop_vcf_contigs = {"chr1": 1000} + pipeline.lr_aln_readgroups = [[{"ID": "lr_rg1", "SM": "sample1"}]] + pipeline.sr_aln_readgroups = [[{"ID": "sr_rg1", "SM": "sample1"}]] + pipeline.hybrid_rg_sm = "sample1" + pipeline.hybrid_set_rg = False + pipeline.shortread_tech = "Illumina" + pipeline.longread_tech = "ONT" + pipeline.sr_aln = self.mock_sr_aln + pipeline.lr_aln = self.mock_lr_aln + + # Realign the long reads + pipeline.lr_align_input = True + pipeline.lr_input_ref = self.mock_input_ref + + return pipeline + + def build_dag(self, pipeline): + """Build the DAG and index the jobs by name""" + with patch( + "sentieon_cli.command_strings.get_rg_lines", return_value=RG_LINES + ): + dag = pipeline.build_dag() + jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) + return dag, {job.name: job for job in jobs} + + def dep_names(self, dag, job): + """The names of a job's dependencies""" + return {dep.name for dep in dag.waiting_jobs.get(job, set())} + + def test_minimap2_args_default(self): + """The hybrid pipeline aligns with `-YL`, like the long-read one + + `-L` moves CIGAR strings longer than 65535 operations - which ONT + reads can produce and BAM cannot store - into the `CG` tag, so + every minimap2 call that may see ONT input needs it. + """ + pipeline = self.create_pipeline() + _dag, jobs = self.build_dag(pipeline) + + assert "bam-realign-0" in jobs + cmd_str = str(jobs["bam-realign-0"].shell) + assert " -YL " in cmd_str + + def test_minimap2_args_match_the_long_read_pipeline(self): + """The hybrid and long-read defaults agree""" + assert ( + DNAscopeHybridPipeline().minimap2_args + == DNAscopeLRPipeline().minimap2_args + ) + + def test_lr_fastq_taglist_is_used(self): + """`--lr_fastq_taglist` reaches the minimap2 command""" + pipeline = self.create_pipeline() + pipeline.lr_fastq_taglist = "RG,MM" + _dag, jobs = self.build_dag(pipeline) + + assert "-T RG,MM" in str(jobs["bam-realign-0"].shell) + + def test_long_read_jobs_wait_for_realignment(self): + """mosdepth and LongReadSV read the realigned long reads""" + pipeline = self.create_pipeline() + dag, jobs = self.build_dag(pipeline) + + assert self.dep_names(dag, jobs["mosdepth-0"]) == {"bam-realign-0"} + assert self.dep_names(dag, jobs["LongReadSV"]) == {"bam-realign-0"} + + def test_realigned_long_reads_are_called(self): + """The realigned alignment, not the input, is passed downstream""" + pipeline = self.create_pipeline() + _dag, jobs = self.build_dag(pipeline) + + realigned = str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") + for name in ("mosdepth-0", "LongReadSV", "dnascope-1"): + assert realigned in str(jobs[name].shell) + + +class TestDNAscopeHybridBases: + """The hybrid pipeline no longer inherits from the other pipelines""" + + def test_mro(self): + assert DNAscopeHybridPipeline.__mro__ == ( + DNAscopeHybridPipeline, + BasePipeline, + ABC, + object, + ) + + def test_not_a_dnascope_or_longread_pipeline(self): + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + assert not isinstance(pipeline, DNAscopePipeline) + assert not isinstance(pipeline, DNAscopeLRPipeline) + + @pytest.mark.parametrize( + "attribute", + [ + "sample_input", + "r1_fastq", + "r2_fastq", + "readgroups", + "duplicate_marking", + "align", + "collate_align", + "consensus", + "fastq", + "fastq_taglist", + "haploid_bed", + "input_ref", + "interval_padding", + "no_ramdisk", + "pcr_free", + "repeat_model", + "skip_small_variants", + "tech", + "use_pbsv", + "cnv_excluded_regions", + "assay", + ], + ) + def test_parent_only_attributes_are_gone(self, attribute): + """The attributes that came from the parent `__init__`s""" + with patch("sys.exit"): + pipeline = DNAscopeHybridPipeline() + + assert not hasattr(pipeline, attribute) diff --git a/tests/unit/test_dnascope_hybrid_pop_vcf.py b/tests/unit/test_dnascope_hybrid_pop_vcf.py index 7add9f1..b7be312 100644 --- a/tests/unit/test_dnascope_hybrid_pop_vcf.py +++ b/tests/unit/test_dnascope_hybrid_pop_vcf.py @@ -135,8 +135,7 @@ def test_transfer_jobs_creation(self): pipeline.pop_vcf = self.mock_pop_vcf # Build DAG - with patch("sentieon_cli.dnascope_hybrid.check_version", return_value=True): - dag = pipeline.build_dag() + dag = pipeline.build_dag() # Verify transfer jobs exist job_names = [job.name for job in dag.waiting_jobs] @@ -150,8 +149,7 @@ def test_model_apply_input_with_pop_vcf(self): pipeline = self.create_pipeline() pipeline.pop_vcf = self.mock_pop_vcf - with patch("sentieon_cli.dnascope_hybrid.check_version", return_value=True): - dag = pipeline.build_dag() + dag = pipeline.build_dag() # Find model-apply job apply_job = None diff --git a/tests/unit/test_dnascope_hybrid_stage1.py b/tests/unit/test_dnascope_hybrid_stage1.py index 6b72aeb..2c8c366 100644 --- a/tests/unit/test_dnascope_hybrid_stage1.py +++ b/tests/unit/test_dnascope_hybrid_stage1.py @@ -86,10 +86,7 @@ def create_pipeline(self): def build_dag(self, pipeline): """Build the DAG and index the jobs by name""" - with patch( - "sentieon_cli.dnascope_hybrid.check_version", return_value=True - ): - dag = pipeline.build_dag() + dag = pipeline.build_dag() jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) return dag, {job.name: job for job in jobs} @@ -170,6 +167,28 @@ def test_stage1_dependencies(self): "first-stage-hap", } + def test_cleanup_jobs(self): + """Each cleanup job waits for the jobs writing what it removes""" + pipeline = self.create_pipeline() + dag, jobs = self.build_dag(pipeline) + + assert self.dep_names(dag, jobs["rm-tmp1"]) == {"concat-merge-bed"} + assert self.dep_names(dag, jobs["rm-tmp2"]) == { + "first-stage", + "first-stage-hap", + } + assert self.dep_names(dag, jobs["rm-tmp3"]) == {"second-stage"} + assert self.dep_names(dag, jobs["rm-tmp4"]) == {"third-stage"} + assert self.dep_names(dag, jobs["rm-tmp5"]) == {"concat-calls"} + + def test_retain_tmpdir_skips_cleanup(self): + """`--retain_tmpdir` keeps the intermediate files""" + pipeline = self.create_pipeline() + pipeline.retain_tmpdir = True + _dag, jobs = self.build_dag(pipeline) + + assert [name for name in jobs if name.startswith("rm-tmp")] == [] + def test_second_stage_inputs(self): """The second stage consumes the sorted haplotype BAM""" pipeline = self.create_pipeline() diff --git a/tests/unit/test_dnascope_longread_dag.py b/tests/unit/test_dnascope_longread_dag.py new file mode 100644 index 0000000..e239146 --- /dev/null +++ b/tests/unit/test_dnascope_longread_dag.py @@ -0,0 +1,224 @@ +""" +The non-obvious edges of the DNAscope LongRead small-variant DAG +""" + +import json +import pathlib +from typing import Dict, List, Set +from unittest.mock import patch + +from sentieon_cli.dag import DAG +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.job import Job + +from tests.utils.test_helpers import create_mock_args + +BUNDLE_MEMBERS = [ + "diploid_hp_model", + "diploid_model", + "diploid_model_unphased", + "gvcf_model", + "haploid_hp_model", + "haploid_model", + "longreadsv.model", + "minimap2.model", +] + +BUNDLE_INFO = { + "platform": "HiFi", + "minScriptVersion": "1.5.2", + "pipeline": "DNAscope LongRead", +} + + +def build_dag(tmp_path: pathlib.Path, **kwargs) -> DAG: + """Build the first DAG of a HiFi long-read run""" + reference = tmp_path / "reference.fa" + reference.touch() + (tmp_path / "reference.fa.fai").write_text( + "chr1\t1000\t10\t60\t61\nchr2\t2000\t1030\t60\t61\n" + ) + bam = tmp_path / "longread.bam" + bam.touch() + bundle = tmp_path / "model.bundle" + bundle.touch() + bed = tmp_path / "regions.bed" + bed.touch() + + pipeline = DNAscopeLRPipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.output_vcf = tmp_path / "output.vcf.gz" + pipeline.reference = reference + pipeline.model_bundle = bundle + pipeline.sample_input = [bam] + pipeline.fastq = [] + pipeline.readgroups = [] + pipeline.bed = bed + pipeline.cores = 2 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = tmp_path + # Only the small-variant jobs are under test + pipeline.skip_mosdepth = True + pipeline.skip_cnv = True + pipeline.skip_svs = True + for key, value in kwargs.items(): + setattr(pipeline, key, value) + + bundle_info = dict(BUNDLE_INFO) + if pipeline.pop_vcf: + bundle_info["SentieonVcfID"] = "test-pop-vcf" + pipeline.skip_pop_vcf_id_check = True + + with patch("sentieon_cli.dnascope_longread.ar_load") as mock_ar_load: + mock_ar_load.side_effect = [ + BUNDLE_MEMBERS, + json.dumps(bundle_info).encode(), + ] + pipeline.validate() + pipeline.configure() + return pipeline.build_dag() + + +def deps_of(dag: DAG, name: str) -> List[str]: + """The sorted dependency names of the single job called `name`""" + return sorted(job.name for job in dag.waiting_jobs[job_named(dag, name)]) + + +def job_named(dag: DAG, name: str) -> Job: + """The one job called `name`""" + matches = [job for job in all_jobs(dag) if job.name == name] + assert len(matches) == 1, f"{name}: {len(matches)} jobs" + return matches[0] + + +def all_jobs(dag: DAG) -> List[Job]: + return list(dag.waiting_jobs) + list(dag.ready_jobs) + + +def roots(dag: DAG) -> Set[str]: + """The names of the jobs with no dependencies""" + names = {job.name for job in dag.ready_jobs} + names.update( + job.name for job, deps in dag.waiting_jobs.items() if not deps + ) + return names + + +class TestSmallVariantEdges: + """The edges `add_small_variant_calling` wires by hand""" + + def test_the_calling_passes_follow_the_phaser(self, tmp_path): + dag = build_dag(tmp_path) + + assert deps_of(dag, "variantphaser") == ["model-apply-diploid"] + for hap in ("dnascope-hap1", "dnascope-hap2"): + assert deps_of(dag, hap) == ["repeatmodel", "variantphaser"] + assert deps_of(dag, "dnascope-unphased") == [ + "bedtools-subtract", + "repeatmodel", + ] + assert deps_of(dag, "patch") == ["dnascope-hap1", "dnascope-hap2"] + + def test_merge_waits_on_the_three_model_apply_jobs(self, tmp_path): + dag = build_dag(tmp_path) + + assert deps_of(dag, "merge") == [ + "model-apply-hap1", + "model-apply-hap2", + "model-apply-unphased", + ] + + def test_each_haplotype_apply_waits_on_its_own_transfer(self, tmp_path): + pop_vcf = tmp_path / "population.vcf.gz" + pop_vcf.touch() + dag = build_dag(tmp_path, pop_vcf=pop_vcf) + + # A model-apply reads only its own haplotype's transfer, so it + # waits on that haplotype's concat alone + for i in (1, 2): + assert deps_of(dag, f"model-apply-hap{i}") == [ + f"merge-trim-hap{i}-concat", + ] + + def test_the_haplotype_applies_wait_on_the_patch_without_a_pop_vcf( + self, tmp_path + ): + dag = build_dag(tmp_path) + + for hap in ("model-apply-hap1", "model-apply-hap2"): + assert deps_of(dag, hap) == ["patch"] + + def test_each_transfer_is_tagged(self, tmp_path): + pop_vcf = tmp_path / "population.vcf.gz" + pop_vcf.touch() + dag = build_dag(tmp_path, pop_vcf=pop_vcf) + + concats = { + job.name for job in all_jobs(dag) if job.name.endswith("-concat") + } + assert concats == { + "merge-trim-diploid-concat", + "merge-trim-hap1-concat", + "merge-trim-hap2-concat", + "merge-trim-unphased-concat", + } + # With a transfer, the apply waits on the concat alone -- the + # concat already depends on the raw-VCF producer + assert deps_of(dag, "model-apply-diploid") == [ + "merge-trim-diploid-concat", + ] + assert deps_of(dag, "model-apply-unphased") == [ + "merge-trim-unphased-concat", + ] + + def test_fai_to_bed_is_a_root_without_a_bed(self, tmp_path): + dag = build_dag(tmp_path, bed=None) + + assert "fai-to-bed" in roots(dag) + assert deps_of(dag, "bedtools-subtract") == [ + "fai-to-bed", + "variantphaser", + ] + + def test_haploid_calling_only_follows_the_repeat_model(self, tmp_path): + haploid_bed = tmp_path / "haploid.bed" + haploid_bed.touch() + dag = build_dag(tmp_path, haploid_bed=haploid_bed) + + # It does not wait on the alignment jobs, only on the repeat model + assert deps_of(dag, "dnascope-haploid") == ["repeatmodel"] + assert deps_of(dag, "haploid-patch2") == ["dnascope-haploid"] + assert deps_of(dag, "haploid-diploid-concat") == [ + "haploid-patch2", + "merge", + ] + + def test_haploid_calling_is_a_root_with_a_repeat_model(self, tmp_path): + haploid_bed = tmp_path / "haploid.bed" + haploid_bed.touch() + repeat_model = tmp_path / "given_repeat.model" + repeat_model.touch() + dag = build_dag( + tmp_path, haploid_bed=haploid_bed, repeat_model=repeat_model + ) + + assert "repeatmodel" not in {job.name for job in all_jobs(dag)} + assert "dnascope-haploid" in roots(dag) + + +class TestJobIdentity: + """Job names and task names of the migrated stages""" + + def test_task_names(self, tmp_path): + dag = build_dag(tmp_path) + + tasks: Dict[str, str] = { + job.name: job.task_name for job in all_jobs(dag) + } + assert tasks["dnascope-diploid"] == "variant-calling" + assert tasks["dnascope-hap1"] == "variant-calling" + assert tasks["dnascope-unphased"] == "variant-calling" + assert tasks["model-apply-diploid"] == "model-apply" + assert tasks["model-apply-hap2"] == "model-apply" + assert tasks["repeatmodel"] == "repeat-model" diff --git a/tests/unit/test_dnascope_longread_pop_vcf.py b/tests/unit/test_dnascope_longread_pop_vcf.py index bdcfd6c..b96e613 100644 --- a/tests/unit/test_dnascope_longread_pop_vcf.py +++ b/tests/unit/test_dnascope_longread_pop_vcf.py @@ -75,7 +75,6 @@ def create_pipeline(self): # Mock sample inputs pipeline.sample_input = self.mock_aln - pipeline.lr_aln = self.mock_aln return pipeline @@ -154,8 +153,7 @@ def test_transfer_jobs_creation(self): pipeline.pop_vcf = self.mock_pop_vcf # Mock methods to avoid real execution/checking - with patch("sentieon_cli.dnascope_longread.check_version", return_value=True), \ - patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ + with patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_fastq", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.mosdepth", return_value=set()), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.merge_input_files", return_value=(pathlib.Path("merged.bam"), MagicMock())), \ @@ -170,15 +168,14 @@ def test_transfer_jobs_creation(self): # Expect merge-trim jobs assert any("merge-trim" in name for name in job_names) - assert "merge-trim-concat" in job_names + assert "merge-trim-diploid-concat" in job_names def test_model_apply_dependency(self): """Test that model apply uses the transfer output when pop_vcf is present""" pipeline = self.create_pipeline() pipeline.pop_vcf = self.mock_pop_vcf - with patch("sentieon_cli.dnascope_longread.check_version", return_value=True), \ - patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ + with patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_inputs", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.lr_align_fastq", return_value=([], set())), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.mosdepth", return_value=set()), \ patch("sentieon_cli.dnascope_longread.DNAscopeLRPipeline.merge_input_files", return_value=(pathlib.Path("merged.bam"), MagicMock())), \ @@ -188,16 +185,16 @@ def test_model_apply_dependency(self): dag = pipeline.build_dag() - # Find first-modelapply job + # Find the diploid model-apply job apply_job = None for job in dag.waiting_jobs: - if job.name == "first-modelapply": + if job.name == "model-apply-diploid": apply_job = job break assert apply_job is not None - # Check dependency: apply_job should depend on merge-trim-concat (transfer_vcf_job) + # Check dependency: apply_job should depend on the transfer concat job deps = dag.waiting_jobs[apply_job] dep_names = [j.name for j in deps] - assert "merge-trim-concat" in dep_names + assert "merge-trim-diploid-concat" in dep_names diff --git a/tests/unit/test_hybrid_pangenome.py b/tests/unit/test_hybrid_pangenome.py index 42c836c..f3f25e1 100644 --- a/tests/unit/test_hybrid_pangenome.py +++ b/tests/unit/test_hybrid_pangenome.py @@ -159,7 +159,7 @@ def test_dag_jobs_present(self): "dedup-lift", "metrics", "pangenome-sv", - "dnascope-raw", + "dnascope", "model-apply", ): assert name in job_names, f"missing job: {name}" @@ -273,7 +273,7 @@ def test_metrics_job(self): ) assert "--algo GCBias" in cmd_str assert "--algo WgsMetricsAlgo" in cmd_str - assert "Rehead metrics" in job_names + assert "rehead-metrics" in job_names def test_skip_metrics(self): """skip_metrics removes the metrics jobs and the Dedup metrics""" @@ -283,7 +283,7 @@ def test_skip_metrics(self): job_names, all_jobs = self._get_all_job_names(dag) assert "metrics" not in job_names - assert "Rehead metrics" not in job_names + assert "rehead-metrics" not in job_names assert "multiqc" not in job_names # Deduplication still runs assert "dedup-bwa" in job_names @@ -366,7 +366,7 @@ def test_calling_inputs_and_replace_rg(self): lift_aln = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.cram") replace_arg = r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" - for job_name in ("pangenome-sv", "dnascope-raw"): + for job_name in ("pangenome-sv", "dnascope"): cmd_str = str(self._get_job(all_jobs, job_name).shell) assert bwa_aln in cmd_str assert lift_aln in cmd_str @@ -421,7 +421,7 @@ def test_dnascope_command(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert "--algo DNAscope" in cmd_str assert "dnascope.model" in cmd_str assert "--pcr_indel_model CONSERVATIVE" in cmd_str @@ -435,7 +435,7 @@ def test_pcr_free(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert "--pcr_indel_model NONE" in cmd_str def test_bam_format(self): @@ -451,7 +451,7 @@ def test_bam_format(self): assert bwa_bam in str(self._get_job(all_jobs, "dedup-bwa").shell) assert lift_bam in str(self._get_job(all_jobs, "dedup-lift").shell) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert bwa_bam in cmd_str assert lift_bam in cmd_str @@ -484,7 +484,7 @@ def test_skip_svs(self): job_names, _ = self._get_all_job_names(dag) assert "pangenome-sv" not in job_names - assert "dnascope-raw" in job_names + assert "dnascope" in job_names assert "graph-update" in job_names def test_skip_small_variants(self): @@ -494,7 +494,7 @@ def test_skip_small_variants(self): dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) - assert "dnascope-raw" not in job_names + assert "dnascope" not in job_names assert "model-apply" not in job_names assert "merge-trim-concat" not in job_names assert "pangenome-sv" in job_names @@ -513,7 +513,7 @@ def test_multiple_lr_inputs(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" in cmd_str assert r"lr-rg2=ID:lr-rg2\tSM:sample1\tLR:1" in cmd_str assert str(lr_bam2) in cmd_str @@ -526,7 +526,7 @@ def test_rgsm_overrides_sm(self): dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:override_sm\tLR:1" in cmd_str bwa_cmd = str(self._get_job(all_jobs, "bwa-extract").shell) @@ -548,7 +548,7 @@ def test_aligned_dag_jobs(self): "graph-update", "mm2-lift", "pangenome-sv", - "dnascope-raw", + "dnascope", "model-apply", ): assert name in job_names, f"missing job: {name}" @@ -561,7 +561,7 @@ def test_aligned_dag_jobs(self): "locuscollector-lift", "dedup-lift", "metrics", - "Rehead metrics", + "rehead-metrics", "multiqc", ): assert name not in job_names, f"unexpected job: {name}" @@ -619,7 +619,7 @@ def test_aligned_dag_dependencies(self): "extract-kmc" } assert "extract-kmc" in self._get_dep_names(dag, all_jobs, "mm2-lift") - assert self._get_dep_names(dag, all_jobs, "dnascope-raw") == { + assert self._get_dep_names(dag, all_jobs, "dnascope") == { "mm2-lift" } @@ -646,7 +646,7 @@ def test_aligned_replace_rg(self): sr_arg = r"sr-rg1=ID:sr-rg1\tSM:sample1\tLR:0" lr_arg = r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert cmd_str.count("--replace_rg") == 2 # Each row precedes the input file it applies to assert cmd_str.index(sr_arg) < cmd_str.index(str(self.mock_sr_bam)) @@ -669,7 +669,7 @@ def test_aligned_bam_format(self): lift_bam = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.bam") assert lift_bam in str(self._get_job(all_jobs, "mm2-lift").shell) - assert lift_bam in str(self._get_job(all_jobs, "dnascope-raw").shell) + assert lift_bam in str(self._get_job(all_jobs, "dnascope").shell) # Short-read input validation @@ -753,19 +753,19 @@ def test_lr_realign_downstream(self): "graph-update-raw", "graph-update", "pangenome-sv", - "dnascope-raw", + "dnascope", ): cmd_str = str(self._get_job(all_jobs, job_name).shell) assert realigned in cmd_str, job_name assert str(self.mock_lr_bam) not in cmd_str, job_name - for job_name in ("longreadsv", "graph-update-raw", "dnascope-raw"): + for job_name in ("longreadsv", "graph-update-raw", "dnascope"): assert "bam-realign-0" in self._get_dep_names( dag, all_jobs, job_name ), job_name # The readgroups of the realigned input are unchanged - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert r"lr-rg1=ID:lr-rg1\tSM:sample1\tLR:1" in cmd_str def test_lr_realign_kmc_reads_original_input(self): @@ -803,13 +803,13 @@ def test_aligned_sr_with_lr_realign(self): realigned = str(self.mock_vcf).replace(".vcf.gz", "_mm2_sorted_0.cram") lift_cram = str(self.mock_vcf).replace(".vcf.gz", "_lift_deduped.cram") - cmd_str = str(self._get_job(all_jobs, "dnascope-raw").shell) + cmd_str = str(self._get_job(all_jobs, "dnascope").shell) assert ( cmd_str.index(str(self.mock_sr_bam)) < cmd_str.index(lift_cram) < cmd_str.index(realigned) ) - assert self._get_dep_names(dag, all_jobs, "dnascope-raw") == { + assert self._get_dep_names(dag, all_jobs, "dnascope") == { "mm2-lift", "bam-realign-0", } diff --git a/tests/unit/test_pipeline_lifecycle.py b/tests/unit/test_pipeline_lifecycle.py index 9e84888..adf67e9 100644 --- a/tests/unit/test_pipeline_lifecycle.py +++ b/tests/unit/test_pipeline_lifecycle.py @@ -7,6 +7,7 @@ import logging import os import sys +from typing import Optional import pytest @@ -16,9 +17,14 @@ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), ) -from sentieon_cli import sentieon_pangenome # noqa: E402 from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.dnascope import DNAscopePipeline # noqa: E402 +from sentieon_cli.dnascope_hybrid import ( # noqa: E402 + DNAscopeHybridPipeline, +) +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline # noqa: E402 from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.hybrid_pangenome import HybridPangenome # noqa: E402 from sentieon_cli.job import Job # noqa: E402 from sentieon_cli.pipeline import BasePipeline # noqa: E402 from sentieon_cli.sentieon_pangenome import SentieonPangenome # noqa: E402 @@ -231,21 +237,20 @@ def __init__(self, fail_on=None): self.check_calls = 0 self.dags_built = [] - def validate_ref(self) -> None: - pass - def validate(self) -> None: pass def configure(self) -> None: pass - def build_first_dag(self) -> DAG: + def build_dag(self) -> DAG: self.dags_built.append("first") self.ploidy_json = self.tmp_dir.joinpath("ploidy.json") return DAG() - def build_second_dag(self) -> DAG: + def build_second_dag(self) -> Optional[DAG]: + if not self._needs_second_dag(): + return None self.dags_built.append("second") return DAG() @@ -264,18 +269,14 @@ def check_execution(self, dag, executor): def _stub_pangenome(monkeypatch, tmp_path, **kwargs): """A stubbed pangenome pipeline writing temp dirs into `tmp_path`""" monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) - monkeypatch.setattr(sentieon_pangenome, "parse_fai", lambda fai: {}) - monkeypatch.setattr( - sentieon_pangenome, "determine_shards_from_fai", lambda fai, size: [] - ) return _StubPangenome(**kwargs) def test_pangenome_main_cleans_up_tmpdir_when_the_first_dag_fails( tmp_path, monkeypatch ): - # SentieonPangenome overrides main() for its two-DAG flow; the temp - # directory has to be removed on the failure path too. + # SentieonPangenome runs the base two-DAG flow; the temp directory + # has to be removed on the failure path too. pipeline = _stub_pangenome(monkeypatch, tmp_path, fail_on=1) with pytest.raises(DagExecutionError): @@ -344,3 +345,18 @@ def test_check_execution_flags_unexecuted_jobs(): with pytest.raises(DagExecutionError, match=r"Job\(unexecuted-1\)"): pipeline.check_execution(dag, _StubExecutor()) + + +@pytest.mark.parametrize( + "pipeline_cls", + [ + DNAscopePipeline, + DNAscopeLRPipeline, + DNAscopeHybridPipeline, + SentieonPangenome, + HybridPangenome, + ], +) +def test_no_pipeline_overrides_main(pipeline_cls): + """Every pipeline runs through `BasePipeline.main`""" + assert "main" not in pipeline_cls.__dict__ diff --git a/tests/unit/test_pipeline_params.py b/tests/unit/test_pipeline_params.py new file mode 100644 index 0000000..d0fb54e --- /dev/null +++ b/tests/unit/test_pipeline_params.py @@ -0,0 +1,48 @@ +""" +Every pipeline argument must be an attribute of a freshly built pipeline. + +`BasePipeline.handle_arguments` raises when a `params` key is missing from +the instance, and argparse leaves an unset argument as `None`, which it +skips -- so the `__init__` value is the effective default. These tests keep +the two in step. +""" + +from typing import Type + +import pytest + +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.dnascope_longread import DNAscopeLRPipeline +from sentieon_cli.hybrid_pangenome import HybridPangenome +from sentieon_cli.pipeline import BasePipeline +from sentieon_cli.sentieon_pangenome import SentieonPangenome + +PIPELINES = [ + DNAscopePipeline, + DNAscopeLRPipeline, + DNAscopeHybridPipeline, + SentieonPangenome, + HybridPangenome, +] + + +@pytest.mark.parametrize("cls", PIPELINES, ids=lambda c: c.__name__) +def test_every_argument_is_an_attribute(cls: Type[BasePipeline]): + """`handle_arguments` needs an attribute per param and positional""" + attributes = set(vars(cls())) + arguments = set(cls.params) | set(cls.positionals) + assert arguments <= attributes, sorted(arguments - attributes) + + +@pytest.mark.parametrize("cls", PIPELINES, ids=lambda c: c.__name__) +def test_argparse_defaults_match_the_attributes(cls: Type[BasePipeline]): + """An argparse default must match the value `__init__` assigns""" + pipeline = cls() + specs = list(cls.params.items()) + list(cls.positionals.items()) + mismatched = { + key: (getattr(pipeline, key), spec["default"]) + for key, spec in specs + if "default" in spec and getattr(pipeline, key) != spec["default"] + } + assert not mismatched diff --git a/tests/unit/test_pipeline_validation.py b/tests/unit/test_pipeline_validation.py index b3ae579..f628345 100644 --- a/tests/unit/test_pipeline_validation.py +++ b/tests/unit/test_pipeline_validation.py @@ -16,6 +16,7 @@ from sentieon_cli.dnascope import DNAscopePipeline from sentieon_cli.dnascope_longread import DNAscopeLRPipeline from sentieon_cli.dnascope_hybrid import DNAscopeHybridPipeline +from sentieon_cli.util import set_bwt_max_mem from tests.utils.test_helpers import create_mock_args @@ -53,8 +54,8 @@ def test_valid_configuration_with_bam_input(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] # Should not raise any exceptions try: @@ -83,8 +84,8 @@ def test_missing_inputs_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] with pytest.raises(SystemExit): self.pipeline.validate() @@ -96,8 +97,8 @@ def test_invalid_output_extension_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] with pytest.raises(SystemExit): self.pipeline.validate() @@ -108,11 +109,19 @@ def test_mismatched_fastq_readgroups_raises_error(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [] - self.pipeline.sr_r1_fastq = [self.mock_fastq] - self.pipeline.sr_readgroups = [] # Empty readgroups with non-empty fastq + # Two fastq files, but only one readgroup. Both must be non-empty to + # get past the "supply --sample_input or --r1_fastq" guard. + self.pipeline.r1_fastq = [self.mock_fastq, self.mock_fastq] + self.pipeline.readgroups = ["@RG\\tID:a\\tSM:s"] - with pytest.raises(SystemExit): - self.pipeline.validate() + with patch.object(self.pipeline.logger, "error") as mock_error: + with pytest.raises(SystemExit) as excinfo: + self.pipeline.validate() + + assert excinfo.value.code == 2 + mock_error.assert_any_call( + "The number of readgroups does not equal the number of fastq files" + ) def test_skip_multiqc_when_skip_metrics(self): """Test that skip_multiqc is set when skip_metrics is True""" @@ -120,8 +129,8 @@ def test_skip_multiqc_when_skip_metrics(self): self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle self.pipeline.sample_input = [self.mock_bam] - self.pipeline.sr_r1_fastq = [] - self.pipeline.sr_readgroups = [] + self.pipeline.r1_fastq = [] + self.pipeline.readgroups = [] self.pipeline.skip_metrics = True self.pipeline.skip_multiqc = False @@ -197,7 +206,7 @@ def test_valid_configuration_with_fastq_input(self): self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [] + self.pipeline.sample_input = [] self.pipeline.fastq = [self.mock_fastq] self.pipeline.readgroups = ["@RG\\tID:test\\tSM:sample"] @@ -271,22 +280,29 @@ def test_haploid_bed_requires_diploid_bed(self): self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [self.mock_bam] + self.pipeline.sample_input = [self.mock_bam] self.pipeline.fastq = [] self.pipeline.readgroups = [] self.pipeline.haploid_bed = self.mock_bed self.pipeline.bed = None # No diploid bed self.pipeline.skip_small_variants = False - with pytest.raises(SystemExit): - self.pipeline.validate() + with patch.object(self.pipeline.logger, "error") as mock_error: + with pytest.raises(SystemExit) as excinfo: + self.pipeline.validate() + + assert excinfo.value.code == 2 + mock_error.assert_any_call( + "Please supply a BED file of diploid regions to distinguish " + "haploid and diploid regions of the genome." + ) def test_mismatched_fastq_readgroups_raises_error(self): """Test that mismatched FASTQ and readgroup counts raise an error""" self.pipeline.output_vcf = self.mock_vcf self.pipeline.reference = self.mock_ref self.pipeline.model_bundle = self.mock_bundle - self.pipeline.lr_aln = [] + self.pipeline.sample_input = [] self.pipeline.fastq = [self.mock_fastq] self.pipeline.readgroups = [] # Empty readgroups with non-empty fastq @@ -425,27 +441,41 @@ def test_total_input_size_calculation(self): fastq_file.write_bytes(b"y" * 500) # 0.5KB pipeline.sample_input = [bam_file] - pipeline.sr_r1_fastq = [fastq_file] - pipeline.sr_r2_fastq = [] + pipeline.r1_fastq = [fastq_file] + pipeline.r2_fastq = [] total_size = pipeline.total_input_size() assert total_size == 1500 # 1KB + 0.5KB - def test_memory_calculation_logic(self): - """Test bwt_max_mem calculation logic""" - pipeline = DNAscopePipeline() - - # Setup logging first - args = create_mock_args() - pipeline.setup_logging(args) - - # Test with specific memory values - pipeline.bwt_max_mem = "10G" - pipeline.set_bwt_max_mem(0, 1) + def test_bwt_max_mem_override(self, monkeypatch): + """`--bwt_max_mem` short-circuits the calculation""" + monkeypatch.delenv("bwt_max_mem", raising=False) - import os + assert set_bwt_max_mem(0, 1, override="10G") == "10G" assert os.environ.get("bwt_max_mem") == "10G" + @pytest.mark.parametrize( + "total_input_size,n_alignment_jobs,expected", + [ + (0, 1, "54G"), # 64 - 4 - 0 = 60; 60 / 1 - 6 = 54 + (0, 2, "24G"), # 60 / 2 - 6 = 24 + (30 * 1024**3, 1, "0G"), # 60 - 30 * 2.3 < 0, floored at 0 + ], + ) + def test_bwt_max_mem_calculation( + self, monkeypatch, total_input_size, n_alignment_jobs, expected + ): + """bwt_max_mem is derived from the memory and the input size""" + monkeypatch.setenv("bwt_max_mem", "unset") + + with patch( + "sentieon_cli.util.total_memory", return_value=64 * 1024**3 + ): + result = set_bwt_max_mem(total_input_size, n_alignment_jobs) + + assert result == expected + assert os.environ.get("bwt_max_mem") == expected + class TestValidateBwaIndex: """Test BasePipeline.validate_bwa_index()""" diff --git a/tests/unit/test_sentieon_pangenome.py b/tests/unit/test_sentieon_pangenome.py index f1d459e..af96db3 100644 --- a/tests/unit/test_sentieon_pangenome.py +++ b/tests/unit/test_sentieon_pangenome.py @@ -96,7 +96,7 @@ def create_fastq_pipeline(self): """Create a fastq-input SentieonPangenome pipeline for testing. The default fixture uses BAM/CRAM input, which does not exercise the - dedup/metrics branch of ``build_first_dag``. This variant provides + dedup/metrics branch of ``build_dag``. This variant provides FASTQ input so the bwa/mm2 dedup jobs are created. """ mock_r1 = self.mock_dir / "sample_R1.fastq.gz" @@ -117,7 +117,7 @@ def test_dedup_metrics_output(self): """Dedup on the primary (bwa) alignment emits a --metrics file that lands in the metrics directory scanned by MultiQC.""" pipeline = self.create_fastq_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) assert "dedup-bwa" in job_names @@ -137,7 +137,7 @@ def test_metrics_input_bwa_only(self): """The metrics job reads only the bwa alignment; including the mm2 alignment would count the extracted reads it re-aligns twice.""" pipeline = self.create_fastq_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) metrics_job = next(j for j in all_jobs if j.name == "metrics") @@ -149,7 +149,7 @@ def test_dedup_metrics_skipped(self): """No Dedup --metrics output when metrics collection is skipped.""" pipeline = self.create_fastq_pipeline() pipeline.skip_metrics = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() _, all_jobs = self._get_all_job_names(dag) bwa_dedup = next(j for j in all_jobs if j.name == "dedup-bwa") @@ -159,7 +159,7 @@ def test_model_apply_default(self): """Model apply runs by default and writes the intermediate, not the final output VCF.""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() # Verify DAG is created assert isinstance(dag, DAG) @@ -180,7 +180,7 @@ def test_skip_model_apply(self): """Test that model apply job is skipped when requested""" pipeline = self.create_pipeline() pipeline.skip_model_apply = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() # Verify DAG is created assert isinstance(dag, DAG) @@ -221,7 +221,7 @@ def test_gvcf_adds_gvcftyper_and_routes_outputs(self): updated gVCF to produce the final VCF at output_vcf.""" pipeline = self.create_pipeline() pipeline.gvcf = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] @@ -236,10 +236,10 @@ def test_gvcf_adds_gvcftyper_and_routes_outputs(self): assert apply_gvcf.endswith("sample-snv_apply.g.vcf.gz") # DNAscope emits gVCF; model-apply writes gVCF - for name in ("dnascope-raw", "model-apply"): + for name in ("dnascope", "model-apply"): job = next(j for j in all_jobs if j.name == name) cmd_str = str(job.shell) - if name == "dnascope-raw": + if name == "dnascope": assert "--emit_mode gvcf" in cmd_str else: assert apply_gvcf in cmd_str @@ -277,7 +277,7 @@ def test_ad_update_default(self): """A single AD-update job reads the model-apply intermediate and writes the final output VCF.""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) assert job_names.count("sad-lad-update") == 1 @@ -303,7 +303,7 @@ def test_no_ad_update_with_skip_small_variants(self): """No AD-update job when small variants are skipped""" pipeline = self.create_pipeline() pipeline.skip_small_variants = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) assert "sad-lad-update" not in job_names @@ -311,7 +311,7 @@ def test_no_ad_update_with_skip_small_variants(self): def test_no_gvcftyper_without_gvcf(self): """Without --gvcf, no GVCFtyper job is added""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) job_names = [job.name for job in all_jobs] @@ -342,22 +342,22 @@ def test_call_svs(self): """Test that PangenomeSV is added when --call_svs is enabled""" pipeline = self.create_pipeline() pipeline.call_svs = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) - # The dnascope-raw job should exist + # The dnascope job should exist all_jobs = list(dag.waiting_jobs.keys()) + list( dag.ready_jobs.keys() ) job_names = [job.name for job in all_jobs] - assert "dnascope-raw" in job_names + assert "dnascope" in job_names - # Find the dnascope-raw job and check its command includes + # Find the dnascope job and check its command includes # both DNAscope and PangenomeSV dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -375,14 +375,14 @@ def test_call_svs(self): def test_call_svs_disabled_by_default(self): """Test that PangenomeSV is not added by default""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() all_jobs = list(dag.waiting_jobs.keys()) + list( dag.ready_jobs.keys() ) dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -394,7 +394,7 @@ def test_skip_small_variants(self): """Test that DNAscope, transfer, and model-apply are skipped""" pipeline = self.create_pipeline() pipeline.skip_small_variants = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) @@ -402,7 +402,7 @@ def test_skip_small_variants(self): dag.ready_jobs.keys() ) job_names = [job.name for job in all_jobs] - assert "dnascope-raw" not in job_names + assert "dnascope" not in job_names assert "model-apply" not in job_names assert "merge-trim-concat" not in job_names @@ -411,7 +411,7 @@ def test_skip_small_variants_with_call_svs(self): pipeline = self.create_pipeline() pipeline.skip_small_variants = True pipeline.call_svs = True - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() assert isinstance(dag, DAG) @@ -421,7 +421,7 @@ def test_skip_small_variants_with_call_svs(self): job_names = [job.name for job in all_jobs] # The driver job should still run for SV calling - assert "dnascope-raw" in job_names + assert "dnascope" in job_names # Transfer and model-apply should be skipped assert "model-apply" not in job_names @@ -430,7 +430,7 @@ def test_skip_small_variants_with_call_svs(self): # The driver command should have PangenomeSV but NOT DNAscope dnascope_job = None for job in all_jobs: - if job.name == "dnascope-raw": + if job.name == "dnascope": dnascope_job = job break assert dnascope_job is not None @@ -451,7 +451,7 @@ def _build_cnv_dag(self, pipeline, sample_sex=SampleSex.FEMALE): The first DAG has to be built first so that the CNV input alignments are stashed for the second DAG. """ - pipeline.build_first_dag() + pipeline.build_dag() pipeline.sample_sex = sample_sex return pipeline.build_second_dag() @@ -505,7 +505,7 @@ def test_cnv_jobs_with_call_svs(self): """CNV jobs run in the second DAG when --call_svs is enabled""" pipeline = self.create_pipeline() pipeline.call_svs = True - first_dag = pipeline.build_first_dag() + first_dag = pipeline.build_dag() # CNV calling is sex-aware, so it is not in the first DAG first_job_names, _ = self._get_all_job_names(first_dag) @@ -525,7 +525,7 @@ def test_cnv_jobs_with_call_svs(self): def test_cnv_jobs_not_present_without_call_svs(self): """Test that CNV jobs are not added by default""" pipeline = self.create_pipeline() - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, _ = self._get_all_job_names(dag) assert "cnvscope" not in job_names @@ -533,13 +533,9 @@ def test_cnv_jobs_not_present_without_call_svs(self): assert "indel2cnv" not in job_names assert "combine-cnv" not in job_names - # Nor to the second DAG + # And no second DAG at all, as nothing consumes the sample sex pipeline.sample_sex = SampleSex.FEMALE - second_job_names, _ = self._get_all_job_names( - pipeline.build_second_dag() - ) - assert "cnvscope" not in second_job_names - assert "combine-cnv" not in second_job_names + assert pipeline.build_second_dag() is None def test_second_dag_needed_for_cnv_only(self): """CNV calling alone triggers the second DAG""" @@ -562,7 +558,7 @@ def test_ploidy_estimation_runs_with_sample_sex(self): pipeline.call_svs = True pipeline.sample_sex = SampleSex.FEMALE - job_names, _ = self._get_all_job_names(pipeline.build_first_dag()) + job_names, _ = self._get_all_job_names(pipeline.build_dag()) assert "estimate-ploidy" in job_names def test_ploidy_estimation_runs_without_sex_aware_callers(self): @@ -571,7 +567,7 @@ def test_ploidy_estimation_runs_without_sex_aware_callers(self): assert pipeline._needs_second_dag() is False job_names, all_jobs = self._get_all_job_names( - pipeline.build_first_dag() + pipeline.build_dag() ) assert "estimate-ploidy" in job_names @@ -586,7 +582,7 @@ def test_ploidy_estimation_runs_with_skip_small_variants(self): pipeline = self.create_pipeline() pipeline.skip_small_variants = True - job_names, _ = self._get_all_job_names(pipeline.build_first_dag()) + job_names, _ = self._get_all_job_names(pipeline.build_dag()) assert "estimate-ploidy" in job_names def test_cnv_jobs_with_skip_small_variants(self): @@ -607,13 +603,13 @@ def test_call_svs_without_cnv_model(self): pipeline = self.create_pipeline() pipeline.call_svs = True pipeline.has_cnv_model = False - dag = pipeline.build_first_dag() + dag = pipeline.build_dag() job_names, all_jobs = self._get_all_job_names(dag) # PangenomeSV still runs - assert "dnascope-raw" in job_names - dnascope_job = next(j for j in all_jobs if j.name == "dnascope-raw") + assert "dnascope" in job_names + dnascope_job = next(j for j in all_jobs if j.name == "dnascope") cmd_str = str(dnascope_job.shell) assert "--algo PangenomeSV" in cmd_str @@ -623,13 +619,9 @@ def test_call_svs_without_cnv_model(self): assert "indel2cnv" not in job_names assert "combine-cnv" not in job_names - # And they are not in the second DAG either + # And there is no second DAG either pipeline.sample_sex = SampleSex.FEMALE - second_job_names, _ = self._get_all_job_names( - pipeline.build_second_dag() - ) - assert "cnvscope" not in second_job_names - assert "combine-cnv" not in second_job_names + assert pipeline.build_second_dag() is None def test_call_svs_without_cnv_model_skip_small_variants(self): """SV-only mode: SVs run but CNV jobs are skipped without cnv.model""" @@ -637,13 +629,8 @@ def test_call_svs_without_cnv_model_skip_small_variants(self): pipeline.skip_small_variants = True pipeline.call_svs = True pipeline.has_cnv_model = False - dag = self._build_cnv_dag(pipeline) - job_names, _ = self._get_all_job_names(dag) - assert "cnvscope" not in job_names - assert "cnv-model-apply" not in job_names - assert "indel2cnv" not in job_names - assert "combine-cnv" not in job_names + assert self._build_cnv_dag(pipeline) is None def test_cnvscope_command(self): """Test CNVscope driver command has correct algo and model""" @@ -746,7 +733,7 @@ def test_cnv_with_skip_model_apply(self): pipeline.call_svs = True pipeline.skip_model_apply = True first_job_names, _ = self._get_all_job_names( - pipeline.build_first_dag() + pipeline.build_dag() ) assert "model-apply" not in first_job_names diff --git a/tests/unit/test_sentieon_pangenome_kmc_integration.py b/tests/unit/test_sentieon_pangenome_kmc_integration.py index cfe4e61..eaad07e 100644 --- a/tests/unit/test_sentieon_pangenome_kmc_integration.py +++ b/tests/unit/test_sentieon_pangenome_kmc_integration.py @@ -21,10 +21,9 @@ def test_validate_kmc_patch_success(self): pipeline.reference = "ref" pipeline.pop_vcf = "pop" pipeline.gbz = "ref.grch38.gbz" - pipeline.fai_data = {} - pipeline.pop_vcf_contigs = {} - + # Mock dependencies + pipeline.logger = MagicMock() pipeline.handle_arguments = MagicMock() pipeline.setup_logging = MagicMock() pipeline.validate_bundle = MagicMock() @@ -33,7 +32,10 @@ def test_validate_kmc_patch_success(self): pipeline.validate_ref = MagicMock() pipeline.collect_readgroups = MagicMock() - with patch("sentieon_cli.sentieon_pangenome.check_version", return_value=True), \ + with patch("sentieon_cli.util.check_version", return_value=True), \ + patch("sentieon_cli.sentieon_pangenome.parse_fai", return_value={}), \ + patch("sentieon_cli.sentieon_pangenome.determine_shards_from_fai", return_value=[]), \ + patch("sentieon_cli.sentieon_pangenome.vcf_contigs", return_value={}), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=True) as mock_check, \ patch("sys.exit") as mock_exit: @@ -54,12 +56,13 @@ def test_validate_kmc_patch_failure(self): pipeline.validate_ref = MagicMock() pipeline.collect_readgroups = MagicMock() pipeline.bed = "bed" - pipeline.fai_data = {} - pipeline.pop_vcf_contigs = {} pipeline.logger = MagicMock() - with patch("sentieon_cli.sentieon_pangenome.check_version", return_value=True), \ + with patch("sentieon_cli.util.check_version", return_value=True), \ + patch("sentieon_cli.sentieon_pangenome.parse_fai", return_value={}), \ + patch("sentieon_cli.sentieon_pangenome.determine_shards_from_fai", return_value=[]), \ + patch("sentieon_cli.sentieon_pangenome.vcf_contigs", return_value={}), \ patch("sentieon_cli.sentieon_pangenome.check_kmc_patch", return_value=False) as mock_check, \ patch("sys.exit") as mock_exit: diff --git a/tests/unit/test_stage_alignment.py b/tests/unit/test_stage_alignment.py new file mode 100644 index 0000000..e4b7df2 --- /dev/null +++ b/tests/unit/test_stage_alignment.py @@ -0,0 +1,457 @@ +""" +Unit tests for the alignment and read-extraction stages +""" + +import logging +import pathlib +from unittest.mock import patch + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.alignment import ( + BwaExtractStage, + BwaFastqStage, + BwaRealignStage, + Minimap2FastqStage, + Minimap2RealignStage, + ReadWriterStage, + aln_suffix, + find_unzip, +) +from sentieon_cli.stages.base import StageContext, rm_job + +RG_LINES = ["@RG\tID:rg1\tSM:sample"] + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +@pytest.fixture +def rg_lines(): + """Stub out the `samtools view -H` call that reads the @RG lines""" + with patch( + "sentieon_cli.command_strings.get_rg_lines", return_value=RG_LINES + ) as mock: + yield mock + + +class TestHelpers: + """The shared helpers the stages and pipelines use""" + + def test_aln_suffix(self): + assert aln_suffix(True) == "bam" + assert aln_suffix(False) == "cram" + + def test_find_unzip_prefers_igzip(self): + with patch( + "sentieon_cli.stages.alignment.shutil.which", + return_value="/usr/bin/igzip", + ): + assert find_unzip(logging.getLogger("test-unzip")) == "igzip" + + def test_find_unzip_falls_back_to_gzip(self, caplog): + logger = logging.getLogger("test-unzip-fallback") + with patch( + "sentieon_cli.stages.alignment.shutil.which", return_value=None + ): + with caplog.at_level(logging.WARNING, logger=logger.name): + assert find_unzip(logger) == "gzip" + assert "igzip is recommended" in caplog.text + assert caplog.records[0].levelno == logging.WARNING + + +class TestBwaRealignStage: + """Re-aligning BAM/CRAM input with bwa""" + + def make(self, tmp_path, **kwargs) -> BwaRealignStage: + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "in0.bam", tmp_path / "in1.bam"], + model_bundle=tmp_path / "model.bundle", + ) + defaults.update(kwargs) + return BwaRealignStage(**defaults) # type: ignore[arg-type] + + def test_one_job_per_input(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0", + "bam-align-1", + ] + for job in result.jobs: + assert job.threads == 4 + assert job.task_name == "alignment" + assert result.terminal == set(result.jobs) + + def test_header_files_are_written(self, tmp_path, rg_lines): + self.make(tmp_path).build() + + for i in range(2): + header = tmp_path / f"input_{i}.hdr" + assert header.read_text() == RG_LINES[0] + "\n" + assert rg_lines.call_count == 2 + + def test_duplicate_marking_writes_bam_to_the_tmp_dir( + self, tmp_path, rg_lines + ): + result = self.make(tmp_path).build() + + assert result.outputs == [ + tmp_path / "bwa_sorted_0.bam", + tmp_path / "bwa_sorted_1.bam", + ] + assert "--bam_compression 1" in str(result.jobs[0].shell) + + def test_without_duplicate_marking_writes_cram_output( + self, tmp_path, rg_lines + ): + result = self.make(tmp_path, duplicate_marking="none").build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_0.cram", + tmp_path / "output_bwa_sorted_1.cram", + ] + assert "--bam_compression 1" not in str(result.jobs[0].shell) + + def test_bam_format_without_duplicate_marking(self, tmp_path, rg_lines): + result = self.make( + tmp_path, duplicate_marking="none", bam_format=True + ).build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_0.bam", + tmp_path / "output_bwa_sorted_1.bam", + ] + + def test_intermediate_cleanup_paths(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert result.cleanup_paths == [ + tmp_path / "bwa_sorted_0.bam", + pathlib.Path(str(tmp_path / "bwa_sorted_0.bam") + ".bai"), + tmp_path / "bwa_sorted_1.bam", + pathlib.Path(str(tmp_path / "bwa_sorted_1.bam") + ".bai"), + ] + + def test_final_outputs_have_no_cleanup_paths(self, tmp_path, rg_lines): + result = self.make(tmp_path, duplicate_marking="none").build() + + assert result.cleanup_paths == [] + + def test_collate_and_input_ref(self, tmp_path, rg_lines): + result = self.make( + tmp_path, + inputs=[tmp_path / "in0.cram"], + collate=True, + input_ref=tmp_path / "decode.fa", + ).build() + + shell = str(result.jobs[0].shell) + assert "samtools collate" in shell + assert f"--reference {tmp_path}/decode.fa" in shell + + def test_build_inserts_nothing(self, tmp_path, rg_lines): + dag = DAG() + self.make(tmp_path).build() + + assert not dag.ready_jobs and not dag.waiting_jobs + + +class TestBwaFastqStage: + """Aligning short-read fastq with bwa""" + + def make(self, tmp_path, **kwargs) -> BwaFastqStage: + defaults = dict( + ctx=make_ctx(tmp_path), + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroups=[r"@RG\tID:rg1\tSM:sample"], + model_bundle=tmp_path / "model.bundle", + ) + defaults.update(kwargs) + return BwaFastqStage(**defaults) # type: ignore[arg-type] + + def test_without_numa_one_job_takes_every_core(self, tmp_path): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == ["bam-align-0-0"] + job = result.jobs[0] + assert job.threads == 4 + assert job.resources == {"node0": 1} + assert job.task_name == "alignment" + assert "taskset" not in str(job.shell) + assert "--split" not in str(job.shell) + + def test_numa_splits_the_pair_across_the_nodes(self, tmp_path): + result = self.make( + tmp_path, ctx=make_ctx(tmp_path, cores=8), numa_nodes=["0-3", "4-7"] + ).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0-0", + "bam-align-0-1", + ] + assert [job.threads for job in result.jobs] == [4, 4] + assert [job.resources for job in result.jobs] == [ + {"node0": 1}, + {"node1": 1}, + ] + first, second = (str(job.shell) for job in result.jobs) + assert "taskset -c 0-3" in first and "0/2" in first + assert "taskset -c 4-7" in second and "1/2" in second + + def test_one_job_per_pair_and_node(self, tmp_path): + result = self.make( + tmp_path, + r1_fastq=[tmp_path / "a1.fq.gz", tmp_path / "b1.fq.gz"], + r2_fastq=[tmp_path / "a2.fq.gz", tmp_path / "b2.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + numa_nodes=["0-3", "4-7"], + ).build() + + assert [job.name for job in result.jobs] == [ + "bam-align-0-0", + "bam-align-0-1", + "bam-align-1-0", + "bam-align-1-1", + ] + + def test_no_fastq_builds_nothing(self, tmp_path): + result = self.make(tmp_path, r1_fastq=[], r2_fastq=[], readgroups=[]) + + built = result.build() + assert built.jobs == [] + assert built.outputs == [] + assert built.cleanup_paths == [] + + def test_unzip_reaches_the_command(self, tmp_path): + result = self.make(tmp_path, unzip="igzip").build() + + assert "igzip -dc" in str(result.jobs[0].shell) + + def test_intermediate_outputs_and_cleanup_paths(self, tmp_path): + result = self.make(tmp_path).build() + + out = tmp_path / "bwa_sorted_fq_0_0.bam" + assert result.outputs == [out] + assert result.cleanup_paths == [ + out, + pathlib.Path(str(out) + ".bai"), + ] + + def test_final_outputs_have_no_cleanup_paths(self, tmp_path): + result = self.make(tmp_path, duplicate_marking="none").build() + + assert result.outputs == [ + tmp_path / "output_bwa_sorted_fq_0_0.cram" + ] + assert result.cleanup_paths == [] + + +class TestMinimap2RealignStage: + """Re-aligning long-read input with minimap2""" + + def make(self, tmp_path, **kwargs) -> Minimap2RealignStage: + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "lr.bam"], + model_bundle=tmp_path / "model.bundle", + sample_name="sample", + ) + defaults.update(kwargs) + return Minimap2RealignStage(**defaults) # type: ignore[arg-type] + + def test_job_metadata_and_outputs(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert [job.name for job in result.jobs] == ["bam-realign-0"] + assert result.jobs[0].threads == 4 + assert result.jobs[0].task_name == "alignment" + assert result.outputs == [tmp_path / "output_mm2_sorted_0.cram"] + assert result.cleanup_paths == [] + + def test_default_model_comes_from_the_bundle(self, tmp_path, rg_lines): + result = self.make(tmp_path).build() + + assert f"-x {tmp_path}/model.bundle/minimap2.model" in str( + result.jobs[0].shell + ) + + def test_minimap2_model_overrides_the_bundle_default( + self, tmp_path, rg_lines + ): + model = tmp_path / "model.bundle" / "minimap2_lr.model" + result = self.make(tmp_path, minimap2_model=model).build() + + assert f"-x {model}" in str(result.jobs[0].shell) + + def test_bam_format_and_input_ref(self, tmp_path, rg_lines): + result = self.make( + tmp_path, bam_format=True, input_ref=tmp_path / "lr_ref.fa" + ).build() + + assert result.outputs == [tmp_path / "output_mm2_sorted_0.bam"] + assert f"--reference {tmp_path}/lr_ref.fa" in str(result.jobs[0].shell) + + +class TestMinimap2FastqStage: + """Aligning long-read fastq with minimap2""" + + def test_one_job_per_fastq(self, tmp_path): + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[tmp_path / "a.fq.gz", tmp_path / "b.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + model_bundle=tmp_path / "model.bundle", + unzip="igzip", + ).build() + + assert [job.name for job in result.jobs] == ["align-0", "align-1"] + assert result.outputs == [ + tmp_path / "output_mm2_sorted_fq_0.cram", + tmp_path / "output_mm2_sorted_fq_1.cram", + ] + assert "igzip -dc" in str(result.jobs[0].shell) + assert result.jobs[0].task_name == "alignment" + + def test_no_fastq_builds_nothing(self, tmp_path): + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[], + readgroups=[], + model_bundle=tmp_path / "model.bundle", + ).build() + + assert result.jobs == [] + assert result.outputs == [] + + +class TestBwaExtractStage: + """The pangenome bwa alignment and read-extraction pass""" + + def make(self, tmp_path, **kwargs) -> BwaExtractStage: + defaults = dict( + ctx=make_ctx(tmp_path), + output_bam=tmp_path / "sample-bwa.bam", + output_fastq=tmp_path / "sample-extract.fq.gz", + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroup=r"@RG\tID:rg1-bwa\tSM:sample", + extract_model=tmp_path / "model.bundle" / "extract.model", + bwa_model=tmp_path / "model.bundle" / "bwa.model", + ) + defaults.update(kwargs) + return BwaExtractStage(**defaults) # type: ignore[arg-type] + + def test_single_job(self, tmp_path): + result = self.make(tmp_path).build() + + assert len(result.jobs) == 1 + job = result.jobs[0] + assert job.name == "bwa-extract" + assert job.threads == 4 + assert job.task_name == "alignment" + assert result.outputs == [ + tmp_path / "sample-bwa.bam", + tmp_path / "sample-extract.fq.gz", + ] + assert result.cleanup_paths == [] + + def test_readgroup_is_used_verbatim(self, tmp_path): + result = self.make( + tmp_path, readgroup=r"@RG\tID:rg1\tSM:sample\tLR:0" + ).build() + + assert r"@RG\tID:rg1\tSM:sample\tLR:0" in str(result.jobs[0].shell) + + def test_unzip_reaches_the_command(self, tmp_path): + result = self.make(tmp_path, unzip="igzip").build() + + assert "igzip -dc" in str(result.jobs[0].shell) + + +class TestReadWriterStage: + """`sentieon driver --algo ReadWriter` merges and extracts""" + + def test_merge(self, tmp_path): + result = ReadWriterStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + output=tmp_path / "merged.bam", + name="merge-bam", + task_name="alignment", + threads=0, + ).build() + + job = result.jobs[0] + assert job.name == "merge-bam" + assert job.threads == 0 + assert job.task_name == "alignment" + assert str(job.shell) == ( + f"sentieon driver --input {tmp_path}/one.bam " + f"--input {tmp_path}/two.bam --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo ReadWriter {tmp_path}/merged.bam" + ) + assert result.outputs == [tmp_path / "merged.bam"] + + def test_string_interval_extracts_a_locus(self, tmp_path): + result = ReadWriterStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + output=tmp_path / "sample_hla.bam", + name="t1k-hla-extract", + task_name="t1k", + interval="chr6:28510020-33480577", + ).build() + + job = result.jobs[0] + assert job.name == "t1k-hla-extract" + assert job.threads == 4 # defaults to the run's core count + assert job.task_name == "t1k" + assert "--interval chr6:28510020-33480577" in str(job.shell) + + +class TestAddTo: + """`add_to` inserts what `build` returned""" + + def test_jobs_are_roots_without_upstream(self, tmp_path): + dag = DAG() + result = BwaFastqStage( + ctx=make_ctx(tmp_path, cores=8), + r1_fastq=[tmp_path / "r1.fq.gz"], + r2_fastq=[tmp_path / "r2.fq.gz"], + readgroups=[r"@RG\tID:rg1"], + model_bundle=tmp_path / "model.bundle", + numa_nodes=["0-3", "4-7"], + ).add_to(dag) + + assert len(result.jobs) == 2 + for job in result.jobs: + assert job in dag.ready_jobs + assert not dag.waiting_jobs + + def test_every_job_waits_on_the_upstream_jobs(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = Minimap2FastqStage( + ctx=make_ctx(tmp_path), + fastq=[tmp_path / "a.fq.gz", tmp_path / "b.fq.gz"], + readgroups=[r"@RG\tID:a", r"@RG\tID:b"], + model_bundle=tmp_path / "model.bundle", + ).add_to(dag, [upstream]) + + for job in result.jobs: + assert dag.waiting_jobs[job] == {upstream} diff --git a/tests/unit/test_stage_base.py b/tests/unit/test_stage_base.py new file mode 100644 index 0000000..46468ed --- /dev/null +++ b/tests/unit/test_stage_base.py @@ -0,0 +1,364 @@ +""" +Unit tests for the pipeline stage building blocks +""" + +import argparse +import dataclasses +import pathlib +from typing import Iterable +from unittest.mock import patch + +import packaging.version +import pytest + +from sentieon_cli import util +from sentieon_cli.dag import DAG +from sentieon_cli.dnascope import DNAscopePipeline +from sentieon_cli.driver import LocusCollector +from sentieon_cli.job import Job +from sentieon_cli.stages.base import ( + Stage, + StageContext, + StageResult, + driver_job, + rm_job, +) +from sentieon_cli.util import require_versions, versions_available + + +def create_mock_args(loglevel: str = "WARNING") -> argparse.Namespace: + """The argparse namespace `setup_logging` needs""" + return argparse.Namespace(loglevel=loglevel) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +@dataclasses.dataclass(kw_only=True) +class ToyStage(Stage): + """Two jobs: `b` depends on `a`, `a` is the only entry job""" + + tag: str = "toy" + + def add_to(self, dag: DAG, upstream: Iterable[Job] = ()) -> StageResult: + deps = set(upstream) + job_a = rm_job( + [self.ctx.tmp_dir / f"{self.tag}-a"], f"{self.tag}-first" + ) + job_b = rm_job( + [self.ctx.tmp_dir / f"{self.tag}-b"], f"{self.tag}-second" + ) + dag.add_job(job_a, deps) + dag.add_job(job_b, {job_a}) + return StageResult(jobs=[job_a, job_b], terminal={job_b}) + + +class TestStageContext: + """The run-wide settings object""" + + def test_fields(self, tmp_path): + ctx = make_ctx(tmp_path) + assert ctx.reference == tmp_path / "ref.fa" + assert ctx.output_vcf == tmp_path / "output.vcf.gz" + assert ctx.tmp_dir == tmp_path + assert ctx.cores == 4 + assert ctx.dry_run is True + assert ctx.skip_version_check is True + + def test_frozen(self, tmp_path): + ctx = make_ctx(tmp_path) + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.cores = 8 # type: ignore[misc] + + +class TestStage: + """The `add_to` contract""" + + def test_upstream_lands_on_entry_jobs_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag, [upstream]) + + job_a, job_b = result.jobs + assert result.jobs == [job_a, job_b] + assert result.terminal == {job_b} + # The entry job gets the upstream dependencies... + assert dag.waiting_jobs[job_a] == {upstream} + # ...and the internal edge is added on top of them. + assert dag.waiting_jobs[job_b] == {job_a} + + def test_no_upstream_makes_the_entry_job_ready(self, tmp_path): + dag = DAG() + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag) + + job_a, job_b = result.jobs + assert job_a in dag.ready_jobs + assert dag.waiting_jobs[job_b] == {job_a} + + def test_upstream_iterable_consumed_once(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = ToyStage(ctx=make_ctx(tmp_path)).add_to(dag, iter([upstream])) + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + + def test_add_to_twice_raises(self, tmp_path): + dag = DAG() + stage = ToyStage(ctx=make_ctx(tmp_path)) + stage.add_to(dag) + # Job identity is the shell pipeline, so the same jobs collide + with pytest.raises(ValueError): + stage.add_to(dag) + + def test_constructing_a_stage_adds_nothing(self, tmp_path): + dag = DAG() + ToyStage(ctx=make_ctx(tmp_path)) + assert not dag.waiting_jobs + assert not dag.ready_jobs + + +class TestDriverJob: + """The `sentieon driver` job helper""" + + def test_basic_command(self, tmp_path): + ctx = make_ctx(tmp_path, cores=8) + job = driver_job( + ctx, + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + inputs=[tmp_path / "sample.bam"], + ) + expected = ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 8 " + f"--algo LocusCollector {tmp_path}/score.txt.gz" + ) + assert str(job.shell) == expected + assert job.name == "locuscollector" + assert job.task_name == "dedup" + assert job.threads == 8 + + def test_no_interval_padding_by_default(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + ) + assert "--interval_padding" not in str(job.shell) + assert "--interval" not in str(job.shell) + assert "--read_filter" not in str(job.shell) + assert "--input" not in str(job.shell) + + def test_interval_and_padding(self, tmp_path): + bed = tmp_path / "regions.bed" + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + interval=bed, + interval_padding=5, + ) + assert f"--interval {bed} --interval_padding 5" in str(job.shell) + + def test_interval_as_a_string(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="readwriter", + task_name="extract", + interval="chr6:28510020-33480577", + ) + assert "--interval chr6:28510020-33480577" in str(job.shell) + + def test_threads_zero(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + threads=0, + ) + # `threads` is the scheduling budget only; the driver still gets + # the run's core count. + assert job.threads == 0 + assert "--thread_count 4" in str(job.shell) + + def test_read_filter_repeats_the_flag(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + read_filter=["QualCalFilter,table=t", "IndelLeftAlign,rgid=rg"], + ) + assert ( + "--read_filter QualCalFilter,table=t " + "--read_filter IndelLeftAlign,rgid=rg" in str(job.shell) + ) + + def test_replace_rg_interleaves_with_input(self, tmp_path): + bam1 = tmp_path / "one.bam" + bam2 = tmp_path / "two.bam" + job = driver_job( + make_ctx(tmp_path), + [LocusCollector(tmp_path / "score.txt.gz")], + name="locuscollector", + task_name="dedup", + inputs=[bam1, bam2], + replace_rg=[["rg1=@RG\\tID:rg1"], ["rg2=@RG\\tID:rg2"]], + ) + shell = str(job.shell) + assert ( + f"--replace_rg 'rg1=@RG\\tID:rg1' --input {bam1} " + f"--replace_rg 'rg2=@RG\\tID:rg2' --input {bam2}" in shell + ) + + def test_algos_are_added_in_order(self, tmp_path): + job = driver_job( + make_ctx(tmp_path), + [ + LocusCollector(tmp_path / "first.txt.gz"), + LocusCollector(tmp_path / "second.txt.gz"), + ], + name="locuscollector", + task_name="dedup", + ) + shell = str(job.shell) + assert shell.index("first.txt.gz") < shell.index("second.txt.gz") + + +class TestRmJob: + """The cleanup job helper""" + + def test_command_and_metadata(self, tmp_path): + vcf = tmp_path / "tmp.vcf.gz" + job = rm_job([vcf, str(vcf) + ".tbi"], "rm-tmp-vcf") + assert str(job.shell) == f"rm {vcf} {vcf}.tbi" + assert job.name == "rm-tmp-vcf" + assert job.task_name == "cleanup" + assert job.threads == 0 + assert job.shell.nodes[0].fail_ok is True + + +class TestRequireVersions: + """`require_versions` and `versions_available`""" + + min_versions = {"sentieon driver": packaging.version.Version("202308")} + + def test_require_skips(self): + with patch.object(util, "check_version") as check: + require_versions(self.min_versions, skip=True) + check.assert_not_called() + + def test_require_passes(self): + with patch.object(util, "check_version", return_value=True) as check: + require_versions(self.min_versions) + check.assert_called_once_with( + "sentieon driver", packaging.version.Version("202308") + ) + + def test_require_exits_on_failure(self): + with patch.object(util, "check_version", return_value=False): + with pytest.raises(SystemExit) as excinfo: + require_versions(self.min_versions) + assert excinfo.value.code == 2 + + def test_require_exits_on_the_first_failure(self): + versions = { + "samtools": packaging.version.Version("1.16"), + "bcftools": packaging.version.Version("1.10"), + } + with patch.object(util, "check_version", return_value=False) as check: + with pytest.raises(SystemExit): + require_versions(versions) + assert check.call_count == 1 + + def test_available_skips(self): + with patch.object(util, "check_version") as check: + assert versions_available(self.min_versions, skip=True) is True + check.assert_not_called() + + def test_available_true(self): + with patch.object(util, "check_version", return_value=True): + assert versions_available(self.min_versions) is True + + def test_available_false(self): + with patch.object(util, "check_version", return_value=False): + assert versions_available(self.min_versions) is False + + +class TestPipelineStageContext: + """`BasePipeline.stage_context` and `BasePipeline.required`""" + + def make_pipeline(self, tmp_path) -> DNAscopePipeline: + pipeline = DNAscopePipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.reference = tmp_path / "ref.fa" + pipeline.output_vcf = tmp_path / "output.vcf.gz" + pipeline.cores = 3 + pipeline.dry_run = True + pipeline.skip_version_check = True + pipeline.tmp_dir = tmp_path + return pipeline + + def test_stage_context(self, tmp_path): + ctx = self.make_pipeline(tmp_path).stage_context() + assert ctx == StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=3, + dry_run=True, + skip_version_check=True, + ) + + def test_missing_reference(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + pipeline.reference = None + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_missing_output_vcf(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + pipeline.output_vcf = None + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_missing_tmp_dir(self, tmp_path): + pipeline = DNAscopePipeline() + pipeline.setup_logging(create_mock_args()) + pipeline.reference = tmp_path / "ref.fa" + pipeline.output_vcf = tmp_path / "output.vcf.gz" + assert not hasattr(pipeline, "tmp_dir") + with pytest.raises(SystemExit) as excinfo: + pipeline.stage_context() + assert excinfo.value.code == 2 + + def test_required_returns_the_value(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + assert pipeline.required(pipeline.output_vcf, "output_vcf") == ( + tmp_path / "output.vcf.gz" + ) + + def test_required_exits_when_missing(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + with pytest.raises(SystemExit) as excinfo: + pipeline.required(None, "model_bundle") + assert excinfo.value.code == 2 diff --git a/tests/unit/test_stage_cnv.py b/tests/unit/test_stage_cnv.py new file mode 100644 index 0000000..3fbf962 --- /dev/null +++ b/tests/unit/test_stage_cnv.py @@ -0,0 +1,217 @@ +""" +Unit tests for the CNVscope stage +""" + +import logging +import pathlib +from typing import List + +import packaging.version +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.cnv import CNV_MIN_VERSIONS, CNVscopeStage +from sentieon_cli.util import SampleSex + +PACKAGE_LOGGER = "sentieon_cli" + + +class _RecordingHandler(logging.Handler): + """Collects formatted messages from the package logger.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages: List[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def messages(): + """`caplog` does not see the package logger, which does not propagate""" + logger = logging.getLogger(PACKAGE_LOGGER) + handler = _RecordingHandler() + logger.addHandler(handler) + try: + yield handler.messages + finally: + logger.removeHandler(handler) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> CNVscopeStage: + """A CNVscopeStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + model=tmp_path / "bundle" / "cnv.model", + cnvscope_vcf=tmp_path / "cnvscope.vcf.gz", + cnv_vcf=tmp_path / "output.cnv.vcf.gz", + ) + defaults.update(kwargs) + return CNVscopeStage(**defaults) # type: ignore[arg-type] + + +class TestMinVersions: + """The version constant moved here with the stage""" + + def test_cnvscope_sex_arguments_need_202503_04(self): + assert CNV_MIN_VERSIONS["sentieon driver"] == ( + packaging.version.Version("202503.04") + ) + + +class TestCommands: + """The `sentieon driver` commands the stage builds""" + + def test_cnvscope_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.cnvscope_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo CNVscope --model {tmp_path}/bundle/cnv.model " + f"{tmp_path}/cnvscope.vcf.gz" + ) + + def test_cnv_model_apply_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + # CNVModelApply reads the CNVscope VCF; it takes no alignment + assert str(result.apply_job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo CNVModelApply " + f"--model {tmp_path}/bundle/cnv.model " + f"--vcf {tmp_path}/cnvscope.vcf.gz " + f"{tmp_path}/output.cnv.vcf.gz" + ) + + def test_interval_is_passed_to_cnvscope_only(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.cnvscope_job.shell) + assert "--interval" not in str(result.apply_job.shell) + + def test_replace_rg_is_passed_to_cnvscope_only(self, tmp_path): + result = make_stage( + tmp_path, + replace_rg=[["rg1=ID:rg1\\tSM:sample"]], + ).add_to(DAG()) + + assert "--replace_rg 'rg1=ID:rg1\\tSM:sample'" in str( + result.cnvscope_job.shell + ) + assert "--replace_rg" not in str(result.apply_job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + +class TestSexArguments: + """`--sex` and `--par` follow the sample sex""" + + def test_male_sample(self, tmp_path): + par_bed = tmp_path / "par.bed" + result = make_stage( + tmp_path, sample_sex=SampleSex.MALE, par_bed=par_bed + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex M" in shell + assert f"--par {par_bed}" in shell + + def test_female_sample(self, tmp_path): + # A female sample does not need the PAR regions + result = make_stage( + tmp_path, + sample_sex=SampleSex.FEMALE, + par_bed=tmp_path / "par.bed", + ).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex F" in shell + assert "--par" not in shell + + @pytest.mark.parametrize("sample_sex", [None, SampleSex.UNKNOWN]) + def test_without_a_known_sex(self, tmp_path, sample_sex, messages): + result = make_stage(tmp_path, sample_sex=sample_sex).add_to(DAG()) + + shell = str(result.cnvscope_job.shell) + assert "--sex" not in shell + assert "--par" not in shell + assert any("diploid" in msg for msg in messages) + + +class TestJobMetadata: + """Job names, task name and thread counts""" + + def test_default_names(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.cnvscope_job.name == "cnvscope" + assert result.apply_job.name == "cnv-model-apply" + + def test_overridden_names(self, tmp_path): + result = make_stage( + tmp_path, name="CNVscope", apply_name="CNVModelApply" + ).add_to(DAG()) + + assert result.cnvscope_job.name == "CNVscope" + assert result.apply_job.name == "CNVModelApply" + + def test_task_name_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "cnv" + assert job.threads == 4 + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_cnvscope_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.cnvscope_job] == {upstream} + assert dag.waiting_jobs[result.apply_job] == {result.cnvscope_job} + + def test_cnvscope_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.cnvscope_job in dag.ready_jobs + assert dag.waiting_jobs[result.apply_job] == {result.cnvscope_job} + + def test_result_fields(self, tmp_path): + cnv_vcf = tmp_path / "output.cnv.vcf.gz" + result = make_stage(tmp_path, cnv_vcf=cnv_vcf).add_to(DAG()) + + assert result.jobs == [result.cnvscope_job, result.apply_job] + assert result.terminal == {result.apply_job} + assert result.cnv_vcf == cnv_vcf diff --git a/tests/unit/test_stage_dedup.py b/tests/unit/test_stage_dedup.py new file mode 100644 index 0000000..b48081e --- /dev/null +++ b/tests/unit/test_stage_dedup.py @@ -0,0 +1,200 @@ +""" +Unit tests for the duplicate-marking stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import AlignmentStat, GCBias +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.dedup import DedupStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> DedupStage: + """A DedupStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + output=tmp_path / "deduped.cram", + score_file=tmp_path / "score.txt.gz", + ) + defaults.update(kwargs) + return DedupStage(**defaults) # type: ignore[arg-type] + + +class TestCommands: + """The `sentieon driver` commands the stage builds""" + + def test_locuscollector_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.lc_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LocusCollector {tmp_path}/score.txt.gz" + ) + + def test_dedup_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + # The defaults produce the shortest possible Dedup command + assert str(result.dedup_job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo Dedup --score_info {tmp_path}/score.txt.gz " + f"{tmp_path}/deduped.cram" + ) + + def test_consensus_reads(self, tmp_path): + result = make_stage(tmp_path, consensus=True).add_to(DAG()) + + assert "--algo LocusCollector --consensus" in str(result.lc_job.shell) + assert "--consensus" not in str(result.dedup_job.shell) + + def test_rmdup_and_cram_write_options(self, tmp_path): + result = make_stage( + tmp_path, + rmdup=True, + cram_write_options="version=3.0,compressor=rans", + ).add_to(DAG()) + + shell = str(result.dedup_job.shell) + assert "--cram_write_options version=3.0,compressor=rans" in shell + assert "--rmdup" in shell + # The options belong to Dedup alone + assert "--rmdup" not in str(result.lc_job.shell) + + def test_dedup_metrics(self, tmp_path): + metrics = tmp_path / "sample.dedup_metrics.txt" + result = make_stage(tmp_path, dedup_metrics=metrics).add_to(DAG()) + + assert f"--metrics {metrics}" in str(result.dedup_job.shell) + + def test_without_optional_dedup_arguments(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + shell = str(result.dedup_job.shell) + for flag in ("--cram_write_options", "--metrics", "--rmdup"): + assert flag not in shell + + def test_read_filters_reach_both_passes(self, tmp_path): + read_filter = "IndelLeftAlignReadTransform,rgid=rg1-mm2" + result = make_stage(tmp_path, read_filters=[read_filter]).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + assert f"--read_filter {read_filter}" in str(job.shell) + + def test_no_read_filters_emits_nothing(self, tmp_path): + result = make_stage(tmp_path, read_filters=[]).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + assert "--read_filter" not in str(job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + for job in (result.lc_job, result.dedup_job): + shell = str(job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + +class TestExtraAlgos: + """Metrics that ride along on the LocusCollector pass""" + + def test_extra_algos_follow_locuscollector(self, tmp_path): + result = make_stage( + tmp_path, + lc_extra_algos=[ + AlignmentStat(tmp_path / "as.txt"), + GCBias(tmp_path / "gc.txt", summary=tmp_path / "sum.txt"), + ], + ).add_to(DAG()) + + shell = str(result.lc_job.shell) + assert shell.index("--algo LocusCollector") < shell.index( + "--algo AlignmentStat" + ) + assert shell.index("--algo AlignmentStat") < shell.index( + "--algo GCBias" + ) + # They stay off the Dedup pass + assert "--algo AlignmentStat" not in str(result.dedup_job.shell) + + +class TestJobMetadata: + """Job names, task name and thread counts""" + + def test_default_names(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.lc_job.name == "locuscollector" + assert result.dedup_job.name == "dedup" + + def test_tagged_names(self, tmp_path): + result = make_stage(tmp_path, tag="bwa").add_to(DAG()) + + assert result.lc_job.name == "locuscollector-bwa" + assert result.dedup_job.name == "dedup-bwa" + + def test_threads_default_to_the_core_count(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.threads == 4 + assert job.task_name == "dedup" + + def test_overridden_threads_and_task_name(self, tmp_path): + result = make_stage( + tmp_path, threads=1, task_name="preprocess" + ).add_to(DAG()) + + for job in result.jobs: + assert job.threads == 1 + assert job.task_name == "preprocess" + # The driver still uses every core + assert "--thread_count 4" in str(result.lc_job.shell) + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_locuscollector_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.lc_job] == {upstream} + assert dag.waiting_jobs[result.dedup_job] == {result.lc_job} + + def test_locuscollector_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.lc_job in dag.ready_jobs + assert dag.waiting_jobs[result.dedup_job] == {result.lc_job} + + def test_result_fields(self, tmp_path): + output = tmp_path / "deduped.cram" + result = make_stage(tmp_path, output=output).add_to(DAG()) + + assert result.jobs == [result.lc_job, result.dedup_job] + assert result.terminal == {result.dedup_job} + assert result.output == output diff --git a/tests/unit/test_stage_metrics.py b/tests/unit/test_stage_metrics.py new file mode 100644 index 0000000..7170b1d --- /dev/null +++ b/tests/unit/test_stage_metrics.py @@ -0,0 +1,332 @@ +""" +Unit tests for the metrics stage and the metrics paths +""" + +import pathlib +import sys + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import ( + AlignmentStat, + InsertSizeMetricAlgo, + WgsMetricsAlgo, +) +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.metrics import ( + MetricsPaths, + MetricsStage, + MosdepthStage, +) + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> MetricsStage: + """A MetricsStage with the paths a caller supplies""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "deduped.cram"], + algos=[AlignmentStat(tmp_path / "as.txt")], + ) + defaults.update(kwargs) + return MetricsStage(**defaults) # type: ignore[arg-type] + + +class TestMetricsPaths: + """Every metrics file is derived from the output VCF""" + + def test_directory_and_base(self): + paths = MetricsPaths.from_output_vcf( + pathlib.Path("/data/run/sample.vcf.gz") + ) + + assert paths.metrics_dir == pathlib.Path("/data/run/sample_metrics") + assert paths.metric_base == "sample.txt" + + def test_metric_files(self): + paths = MetricsPaths.from_output_vcf(pathlib.Path("/data/out.vcf.gz")) + base = pathlib.Path("/data/out_metrics") + + assert paths.score == base / "out.txt.score.txt.gz" + assert paths.insert_size == base / "out.txt.insert_size.txt" + assert ( + paths.mean_qual_by_cycle == base / "out.txt.mean_qual_by_cycle.txt" + ) + assert paths.base_distribution_by_cycle == ( + base / "out.txt.base_distribution_by_cycle.txt" + ) + assert paths.qual_distribution == ( + base / "out.txt.qual_distribution.txt" + ) + assert paths.alignment_stat == base / "out.txt.alignment_stat.txt" + assert paths.hybrid_selection == base / "out.txt.hybrid-selection.txt" + assert paths.wgs == base / "out.txt.wgs.txt" + assert paths.gc_bias == base / "out.txt.gc_bias.txt" + assert paths.gc_bias_summary == base / "out.txt.gc_bias_summary.txt" + assert paths.dedup_metrics == base / "out.txt.dedup_metrics.txt" + + def test_coverage_is_not_named_after_the_vcf(self): + paths = MetricsPaths.from_output_vcf(pathlib.Path("/data/out.vcf.gz")) + + assert paths.coverage == pathlib.Path("/data/out_metrics/coverage") + + def test_ensure_dir_creates_the_directory(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.ensure_dir(dry_run=False) + + assert paths.metrics_dir.is_dir() + + def test_ensure_dir_creates_nothing_on_a_dry_run(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.ensure_dir(dry_run=True) + + assert not paths.metrics_dir.exists() + + def test_ensure_dir_tolerates_an_existing_directory(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + paths.metrics_dir.mkdir() + paths.ensure_dir(dry_run=False) + + assert paths.metrics_dir.is_dir() + + +class TestCommands: + """The commands the stage builds""" + + def test_metrics_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.metrics_job.shell) == ( + f"sentieon driver --input {tmp_path}/deduped.cram " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo AlignmentStat --adapter_seq '' {tmp_path}/as.txt" + ) + + def test_algo_order_is_the_callers(self, tmp_path): + result = make_stage( + tmp_path, + algos=[ + InsertSizeMetricAlgo(tmp_path / "is.txt"), + WgsMetricsAlgo(tmp_path / "wgs.txt", include_unpaired="true"), + AlignmentStat(tmp_path / "as.txt"), + ], + ).add_to(DAG()) + + shell = str(result.metrics_job.shell) + assert shell.index("--algo InsertSizeMetricAlgo") < shell.index( + "--algo WgsMetricsAlgo" + ) + assert shell.index("--algo WgsMetricsAlgo") < shell.index( + "--algo AlignmentStat" + ) + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.metrics_job.shell) + + def test_without_an_interval(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--interval" not in str(result.metrics_job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + ).add_to(DAG()) + + shell = str(result.metrics_job.shell) + assert f"--input {tmp_path}/one.cram" in shell + assert f"--input {tmp_path}/two.cram" in shell + + +class TestRehead: + """The optional WGS-metrics rehead job""" + + def test_rehead_command(self, tmp_path): + wgs = tmp_path / "output.txt.wgs.txt" + result = make_stage(tmp_path, rehead_metrics=wgs).add_to(DAG()) + + assert result.rehead_job is not None + shell = str(result.rehead_job.shell) + assert shell.startswith(sys.executable) + assert "rehead_wgs_metrics.py" in shell + assert f"--metrics_file {wgs}" in shell + + def test_rehead_job_metadata(self, tmp_path): + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(DAG()) + + assert result.rehead_job is not None + assert result.rehead_job.name == "rehead-metrics" + assert result.rehead_job.threads == 0 + assert result.rehead_job.task_name == "metrics" + + def test_no_rehead_job_by_default(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.rehead_job is None + assert result.jobs == [result.metrics_job] + assert result.terminal == {result.metrics_job} + + def test_terminal_is_the_rehead_job(self, tmp_path): + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(DAG()) + + assert result.jobs == [result.metrics_job, result.rehead_job] + assert result.terminal == {result.rehead_job} + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_defaults(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.metrics_job.name == "metrics" + assert result.metrics_job.task_name == "metrics" + # Metrics run in the background by default + assert result.metrics_job.threads == 0 + + def test_custom_name_task_and_threads(self, tmp_path): + result = make_stage( + tmp_path, + name="sr-metrics", + task_name="qc", + threads=4, + ).add_to(DAG()) + + assert result.metrics_job.name == "sr-metrics" + assert result.metrics_job.task_name == "qc" + assert result.metrics_job.threads == 4 + + def test_rehead_keeps_the_metrics_task(self, tmp_path): + result = make_stage( + tmp_path, + name="sr-metrics", + task_name="qc", + rehead_metrics=tmp_path / "wgs.txt", + ).add_to(DAG()) + + assert result.rehead_job is not None + assert result.rehead_job.task_name == "metrics" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_lands_on_the_metrics_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage( + tmp_path, rehead_metrics=tmp_path / "wgs.txt" + ).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.metrics_job] == {upstream} + assert dag.waiting_jobs[result.rehead_job] == {result.metrics_job} + + def test_metrics_is_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.metrics_job in dag.ready_jobs + + +class TestMosdepth: + """Coverage QC, one job per input file""" + + def test_mosdepth_command(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(DAG()) + + (job,) = result.jobs + assert str(job.shell) == ( + f"mosdepth --fasta {tmp_path}/ref.fa --threads 4 " + "--by 500 --no-per-base --use-median " + "-T 1,3,5,10,15,20,30,40,50 " + f"{tmp_path}/output_mosdepth_0 {tmp_path}/sample.bam" + ) + + def test_one_job_per_input(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + assert [job.name for job in result.jobs] == [ + "mosdepth-0", + "mosdepth-1", + ] + assert f"{tmp_path}/output_mosdepth_0" in str(result.jobs[0].shell) + assert f"{tmp_path}/output_mosdepth_1" in str(result.jobs[1].shell) + + def test_job_metadata(self, tmp_path): + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(DAG()) + + (job,) = result.jobs + assert job.task_name == "metrics" + # mosdepth runs in the background + assert job.threads == 0 + + def test_no_inputs_adds_nothing(self, tmp_path): + dag = DAG() + result = MosdepthStage(ctx=make_ctx(tmp_path), inputs=[]).add_to(dag) + + assert result.jobs == [] + assert result.terminal == set() + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_build_inserts_nothing(self, tmp_path): + dag = DAG() + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).build() + + assert result.terminal == set(result.jobs) + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_upstream_lands_on_every_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(dag, [upstream]) + + for job in result.jobs: + assert dag.waiting_jobs[job] == {upstream} + + def test_roots_without_upstream(self, tmp_path): + dag = DAG() + result = MosdepthStage( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ).add_to(dag) + + assert result.jobs[0] in dag.ready_jobs diff --git a/tests/unit/test_stage_ploidy.py b/tests/unit/test_stage_ploidy.py new file mode 100644 index 0000000..d696903 --- /dev/null +++ b/tests/unit/test_stage_ploidy.py @@ -0,0 +1,130 @@ +""" +Unit tests for the ploidy estimation stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.ploidy import PloidyStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> PloidyStage: + """A PloidyStage over one alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.cram"], + ) + defaults.update(kwargs) + return PloidyStage(**defaults) # type: ignore[arg-type] + + +class TestOutput: + """The ploidy JSON sits next to the output VCF""" + + def test_ploidy_json_derivation(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.ploidy_json == tmp_path / "output_ploidy.json" + assert str(result.ploidy_json) in str(result.job.shell) + + +class TestCommand: + """The `estimate_ploidy.py` command line""" + + def test_script_and_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + ).add_to(DAG()) + + cmd = str(result.job.shell) + assert cmd.startswith("python3 ") + assert "estimate_ploidy.py" in cmd + assert f"-i {tmp_path}/one.bam {tmp_path}/two.bam" in cmd + assert f"--outfile {tmp_path}/output_ploidy.json" in cmd + + def test_the_script_defaults_apply_without_a_build(self, tmp_path): + # The `estimate_ploidy.py` defaults are the chr-prefixed names + cmd = str(make_stage(tmp_path).add_to(DAG()).job.shell) + + assert "--contigs" not in cmd + assert "--autosomes" not in cmd + assert "--x_contig" not in cmd + assert "--y_contig" not in cmd + + def test_an_unrecognized_build_uses_the_defaults(self, tmp_path): + cmd = str( + make_stage(tmp_path, reference_build="hg38") + .add_to(DAG()) + .job.shell + ) + + assert "--contigs" not in cmd + assert "--x_contig" not in cmd + + def test_b37_contig_names(self, tmp_path): + cmd = str( + make_stage(tmp_path, reference_build="b37").add_to(DAG()).job.shell + ) + + assert "--contigs 1 2 " in cmd + assert "--autosomes 1 2 " in cmd + # The sex chromosomes are in `--contigs` but not in `--autosomes` + assert ( + "--contigs " + + " ".join([str(i) for i in range(1, 23)] + ["X", "Y"]) + in cmd + ) + assert "--autosomes " + " ".join([str(i) for i in range(1, 23)]) in cmd + assert "--x_contig X" in cmd + assert "--y_contig Y" in cmd + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_metadata(self, tmp_path): + job = make_stage(tmp_path).add_to(DAG()).job + + assert job.name == "estimate-ploidy" + assert job.task_name == "ploidy" + # The script is single-threaded and does not reserve a core + assert job.threads == 0 + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_upstream_edge(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} diff --git a/tests/unit/test_stage_preprocessing.py b/tests/unit/test_stage_preprocessing.py new file mode 100644 index 0000000..6ccf8b7 --- /dev/null +++ b/tests/unit/test_stage_preprocessing.py @@ -0,0 +1,342 @@ +""" +Unit tests for the short-read preprocessing stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.metrics import MetricsPaths +from sentieon_cli.stages.preprocessing import ( + ShortReadPreprocessingStage, + pre_dedup_metrics_algos, +) + +# The pre-dedup metrics, in the order the stage collects them +PRE_DEDUP_ALGOS = [ + "MeanQualityByCycle", + "BaseDistributionByCycle", + "QualDistribution", + "AlignmentStat", +] + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage( + tmp_path: pathlib.Path, **kwargs +) -> ShortReadPreprocessingStage: + """A preprocessing stage over one input alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + ) + defaults.update(kwargs) + return ShortReadPreprocessingStage(**defaults) # type: ignore[arg-type] + + +def algo_order(shell: str) -> list: + """The algos a driver command runs, in command-line order""" + tokens = shell.split() + return [ + tokens[i + 1] + for i, token in enumerate(tokens[:-1]) + if token == "--algo" + ] + + +class TestPreDedupAlgos: + """The metrics that do not need duplicate-marked reads""" + + def test_wgs_adds_gc_bias(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WGS") + + assert [algo.name for algo in algos] == PRE_DEDUP_ALGOS + ["GCBias"] + + def test_wes_omits_gc_bias(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WES") + + assert [algo.name for algo in algos] == PRE_DEDUP_ALGOS + + def test_the_outputs_come_from_the_metrics_paths(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + algos = pre_dedup_metrics_algos(paths, "WGS") + + assert algos[0].output == paths.mean_qual_by_cycle + assert algos[1].output == paths.base_distribution_by_cycle + assert algos[2].output == paths.qual_distribution + assert algos[3].output == paths.alignment_stat + assert algos[4].output == paths.gc_bias + assert algos[4].summary == paths.gc_bias_summary + + +class TestNoDuplicateMarking: + """`duplicate_marking="none"` collects everything in one pass""" + + def test_skip_metrics_adds_no_jobs(self, tmp_path): + dag = DAG() + inputs = [tmp_path / "sample.bam"] + result = make_stage( + tmp_path, + inputs=inputs, + duplicate_marking="none", + skip_metrics=True, + ).add_to(dag) + + assert result.jobs == [] + assert result.terminal == set() + assert result.qc_jobs == [] + assert result.dedup_job is None + assert result.deduped == inputs + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_one_metrics_job(self, tmp_path): + inputs = [tmp_path / "sample.bam"] + result = make_stage( + tmp_path, inputs=inputs, duplicate_marking="none" + ).add_to(DAG()) + + (metrics,) = result.jobs + assert metrics.name == "metrics" + assert metrics.task_name == "metrics" + # This pass reads the whole alignment, so it takes every core + assert metrics.threads == 4 + assert result.dedup_job is None + assert result.deduped == inputs + assert result.qc_jobs == [metrics] + + def test_wgs_algo_order(self, tmp_path): + result = make_stage(tmp_path, duplicate_marking="none").add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ["GCBias"] + ) + + def test_wes_omits_gc_bias(self, tmp_path): + result = make_stage( + tmp_path, duplicate_marking="none", assay="WES" + ).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ) + + def test_the_metrics_job_reads_the_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + duplicate_marking="none", + ).add_to(DAG()) + + shell = str(result.jobs[0].shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + def test_upstream_lands_on_the_metrics_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path, duplicate_marking="none").add_to( + dag, [upstream] + ) + + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + # Nothing should wait on this leaf + assert result.terminal == set() + + +class TestMarkdupWgs: + """The default: duplicate marking plus WGS metrics""" + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + lc, dedup, metrics, rehead = result.jobs + assert lc.name == "locuscollector" + assert dedup.name == "dedup" + assert metrics.name == "metrics" + assert rehead.name == "rehead-metrics" + assert result.dedup_job is dedup + assert result.terminal == {dedup} + assert result.qc_jobs == [lc, metrics, rehead] + assert result.deduped == [tmp_path / "output_deduped.cram"] + + def test_locuscollector_carries_the_pre_dedup_metrics(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["LocusCollector"] + PRE_DEDUP_ALGOS + ["GCBias"] + ) + + def test_dedup_writes_metrics_and_cram_options(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path).add_to(DAG()) + + shell = str(result.dedup_job.shell) + assert f"--metrics {paths.dedup_metrics}" in shell + assert "--cram_write_options version=3.0,compressor=rans" in shell + assert "--rmdup" not in shell + + def test_post_dedup_metrics(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + metrics = result.jobs[2] + assert algo_order(str(metrics.shell)) == [ + "InsertSizeMetricAlgo", + "WgsMetricsAlgo", + "CoverageMetrics", + ] + assert "--include_unpaired true" in str(metrics.shell) + # Metrics run in the background + assert metrics.threads == 0 + assert f"--input {tmp_path}/output_deduped.cram" in str(metrics.shell) + + def test_the_post_job_waits_on_dedup(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert dag.waiting_jobs[result.jobs[2]] == {result.dedup_job} + assert dag.waiting_jobs[result.jobs[3]] == {result.jobs[2]} + + def test_upstream_lands_on_locuscollector(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.jobs[0]] == {upstream} + + def test_insert_size_is_not_on_the_locuscollector_pass(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "InsertSizeMetricAlgo" not in algo_order( + str(result.jobs[0].shell) + ) + + +class TestWes: + """WES runs collect hybrid-selection metrics instead""" + + def test_with_a_bed(self, tmp_path): + bed = tmp_path / "regions.bed" + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path, assay="WES", bed=bed).add_to(DAG()) + + lc, dedup, metrics = result.jobs + assert algo_order(str(metrics.shell)) == [ + "HsMetricAlgo", + "InsertSizeMetricAlgo", + ] + assert f"--interval {bed}" in str(metrics.shell) + assert f"--targets_list {bed}" in str(metrics.shell) + assert str(paths.hybrid_selection) in str(metrics.shell) + # No WGS metrics means no rehead job + assert result.qc_jobs == [lc, metrics] + assert result.terminal == {dedup} + + def test_insert_size_is_not_on_the_locuscollector_pass_with_a_bed( + self, tmp_path + ): + result = make_stage( + tmp_path, assay="WES", bed=tmp_path / "regions.bed" + ).add_to(DAG()) + + assert algo_order(str(result.jobs[0].shell)) == ( + ["LocusCollector"] + PRE_DEDUP_ALGOS + ) + + def test_without_a_bed(self, tmp_path): + result = make_stage(tmp_path, assay="WES").add_to(DAG()) + + lc, dedup = result.jobs + # InsertSize moves onto the LocusCollector pass + assert algo_order(str(lc.shell)) == ( + ["LocusCollector", "InsertSizeMetricAlgo"] + PRE_DEDUP_ALGOS + ) + assert result.qc_jobs == [lc] + assert result.terminal == {dedup} + + +class TestOptions: + """The flags a caller passes through""" + + def test_rmdup(self, tmp_path): + result = make_stage(tmp_path, duplicate_marking="rmdup").add_to(DAG()) + + assert "--rmdup" in str(result.dedup_job.shell) + + def test_consensus(self, tmp_path): + result = make_stage(tmp_path, consensus=True).add_to(DAG()) + + assert "--algo LocusCollector --consensus" in str(result.jobs[0].shell) + + def test_bam_format(self, tmp_path): + result = make_stage(tmp_path, bam_format=True).add_to(DAG()) + + assert result.deduped == [tmp_path / "output_deduped.bam"] + + def test_skip_metrics_keeps_duplicate_marking(self, tmp_path): + paths = MetricsPaths.from_output_vcf(tmp_path / "output.vcf.gz") + result = make_stage(tmp_path, skip_metrics=True).add_to(DAG()) + + lc, dedup = result.jobs + assert algo_order(str(lc.shell)) == ["LocusCollector"] + # Dedup still writes its own metrics + assert f"--metrics {paths.dedup_metrics}" in str(dedup.shell) + assert result.qc_jobs == [lc] + assert result.terminal == {dedup} + + +class TestMetricsDirectory: + """The metrics directory is created before the jobs are built""" + + def test_created_when_not_a_dry_run(self, tmp_path): + ctx = StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=4, + dry_run=False, + skip_version_check=True, + ) + make_stage(tmp_path, ctx=ctx).add_to(DAG()) + + assert (tmp_path / "output_metrics").is_dir() + + def test_not_created_on_a_dry_run(self, tmp_path): + make_stage(tmp_path).add_to(DAG()) + + assert not (tmp_path / "output_metrics").exists() + + def test_created_even_when_no_jobs_are_added(self, tmp_path): + ctx = StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=4, + dry_run=False, + skip_version_check=True, + ) + make_stage( + tmp_path, + ctx=ctx, + duplicate_marking="none", + skip_metrics=True, + ).add_to(DAG()) + + assert (tmp_path / "output_metrics").is_dir() diff --git a/tests/unit/test_stage_small_variants.py b/tests/unit/test_stage_small_variants.py new file mode 100644 index 0000000..56ae89f --- /dev/null +++ b/tests/unit/test_stage_small_variants.py @@ -0,0 +1,385 @@ +""" +Unit tests for the small-variant stages +""" + +import pathlib + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.driver import DNAscope, SVSolver +from sentieon_cli.shard import Shard +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.small_variants import ( + ApplySpec, + DNAscopeStage, + DriverStage, + GVCFtyperStage, + ModelApplyStage, + TransferApplyStage, + TransferSpec, +) +from sentieon_cli.stages.transfer import TransferConfig + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + # `build_transfer_jobs` shells out to bcftools unless this is set + dry_run=True, + skip_version_check=True, + ) + + +def make_call_stage(tmp_path: pathlib.Path, **kwargs) -> DNAscopeStage: + """A DNAscopeStage calling SNVs and indels""" + defaults = dict( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + inputs=[tmp_path / "sample.cram"], + ) + defaults.update(kwargs) + return DNAscopeStage(**defaults) # type: ignore[arg-type] + + +def make_transfer_config(tmp_path: pathlib.Path) -> TransferConfig: + """One shard the pop VCF has, one it lacks""" + return TransferConfig( + pop_vcf=tmp_path / "pop.vcf.gz", + shards=[Shard("chr1", 0, 1000), Shard("chrUn_extra", 0, 500)], + pop_vcf_contigs={"chr1": 1000}, + fai_data={ + "chr1": {"length": 1000}, + "chrUn_extra": {"length": 500}, + }, + ) + + +class TestDriverStage: + """The generic single-job driver stage""" + + def test_minimal_command(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[ + SVSolver( + output=tmp_path / "sv.vcf.gz", + vcf=tmp_path / "sv_tmp.vcf.gz", + ) + ], + name="svsolver", + task_name="sv-calling", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo SVSolver " + f"--vcf {tmp_path}/sv_tmp.vcf.gz {tmp_path}/sv.vcf.gz" + ) + + def test_name_task_name_and_default_threads(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + name="a-name", + task_name="a-task", + ).add_to(DAG()) + + assert result.job.name == "a-name" + assert result.job.task_name == "a-task" + assert result.job.threads == 4 + + def test_threads_override_leaves_the_driver_alone(self, tmp_path): + result = DriverStage( + ctx=make_ctx(tmp_path), + algos=[DNAscope(tmp_path / "raw.vcf.gz")], + name="a-name", + task_name="a-task", + threads=1, + ).add_to(DAG()) + + assert result.job.threads == 1 + assert "--thread_count 4" in str(result.job.shell) + + def test_result_fields(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + + +class TestDNAscopeStage: + """The DNAscope calling pass""" + + def test_default_name_and_task_name(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "dnascope" + assert result.job.task_name == "variant-calling" + + def test_inputs_and_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_call_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + interval=bed, + ).add_to(DAG()) + + shell = str(result.job.shell) + assert f"--input {tmp_path}/one.cram" in shell + assert f"--input {tmp_path}/two.cram" in shell + assert f"--interval {bed}" in shell + + def test_interval_padding_is_absent_unless_set(self, tmp_path): + result = make_call_stage(tmp_path).add_to(DAG()) + + assert "--interval_padding" not in str(result.job.shell) + + def test_interval_padding(self, tmp_path): + result = make_call_stage(tmp_path, interval_padding=100).add_to(DAG()) + + assert "--interval_padding 100" in str(result.job.shell) + + def test_read_filter(self, tmp_path): + result = make_call_stage( + tmp_path, read_filter=["UltimaReadFilter"] + ).add_to(DAG()) + + assert "--read_filter UltimaReadFilter" in str(result.job.shell) + + def test_empty_read_filter_emits_nothing(self, tmp_path): + result = make_call_stage(tmp_path, read_filter=[]).add_to(DAG()) + + assert "--read_filter" not in str(result.job.shell) + + def test_replace_rg_precedes_its_input(self, tmp_path): + result = make_call_stage( + tmp_path, + inputs=[tmp_path / "one.cram", tmp_path / "two.cram"], + replace_rg=[["rg1=rg1-lr"], ["rg2=rg2-lr"]], + ).add_to(DAG()) + + assert str(result.job.shell).startswith( + f"sentieon driver --replace_rg rg1=rg1-lr " + f"--input {tmp_path}/one.cram " + f"--replace_rg rg2=rg2-lr --input {tmp_path}/two.cram" + ) + + def test_algos_keep_their_order(self, tmp_path): + result = make_call_stage( + tmp_path, + algos=[ + DNAscope(tmp_path / "raw.vcf.gz"), + DNAscope(tmp_path / "sv_tmp.vcf.gz", var_type="BND"), + ], + ).add_to(DAG()) + + shell = str(result.job.shell) + assert shell.index("raw.vcf.gz") < shell.index("--var_type BND") + + def test_upstream_becomes_the_dependency(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_call_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + +class TestModelApplyStage: + """DNAModelApply""" + + def test_command(self, tmp_path): + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "bundle" / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=tmp_path / "applied.vcf.gz", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo DNAModelApply " + f"--model {tmp_path}/bundle/dnascope.model " + f"--vcf {tmp_path}/raw.vcf.gz " + f"{tmp_path}/applied.vcf.gz" + ) + + def test_defaults_and_result(self, tmp_path): + output = tmp_path / "applied.vcf.gz" + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=output, + ).add_to(DAG()) + + assert result.job.name == "model-apply" + assert result.job.task_name == "model-apply" + assert result.job.threads == 4 + assert result.output == output + assert result.terminal == {result.job} + + def test_renamed_job(self, tmp_path): + result = ModelApplyStage( + ctx=make_ctx(tmp_path), + model=tmp_path / "dnascope.model", + vcf=tmp_path / "raw.vcf.gz", + output=tmp_path / "applied.vcf.gz", + name="snv-apply", + ).add_to(DAG()) + + assert result.job.name == "snv-apply" + + +class TestGVCFtyperStage: + """Genotyping a gVCF""" + + def test_command_without_an_interval(self, tmp_path): + result = GVCFtyperStage( + ctx=make_ctx(tmp_path), + gvcf=tmp_path / "sample.g.vcf.gz", + output=tmp_path / "sample.vcf.gz", + ).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --reference {tmp_path}/ref.fa " + f"--thread_count 4 --algo GVCFtyper " + f"--vcf {tmp_path}/sample.g.vcf.gz " + f"{tmp_path}/sample.vcf.gz" + ) + assert result.job.name == "gvcftyper" + assert result.job.task_name == "gvcftyper" + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = GVCFtyperStage( + ctx=make_ctx(tmp_path), + gvcf=tmp_path / "sample.g.vcf.gz", + output=tmp_path / "sample.vcf.gz", + interval=bed, + ).add_to(DAG()) + + assert f"--interval {bed}" in str(result.job.shell) + assert result.output == tmp_path / "sample.vcf.gz" + + +class TestTransferApplyStage: + """Transfer, apply, or both""" + + def test_neither_half_is_an_error(self, tmp_path): + with pytest.raises(ValueError): + TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + ).add_to(DAG()) + + def test_apply_only_reads_the_raw_vcf(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.transfer is None + assert result.apply_job is not None + assert f"--vcf {tmp_path}/raw.vcf.gz" in str(result.apply_job.shell) + assert dag.waiting_jobs[result.apply_job] == {upstream} + assert result.terminal == {result.apply_job} + assert result.output == tmp_path / "applied.vcf.gz" + + def test_transfer_only(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.apply_job is None + assert result.transfer is not None + assert result.terminal == {result.transfer.concat_job} + assert result.output == tmp_path / "transfer.vcf.gz" + for job in result.transfer.shard_jobs: + assert dag.waiting_jobs[job] == {upstream} + + def test_both_halves(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + ), + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + ), + ).add_to(dag, [upstream]) + + assert result.transfer is not None + assert result.apply_job is not None + # DNAModelApply reads the transfer output + assert f"--vcf {tmp_path}/transfer.vcf.gz" in str( + result.apply_job.shell + ) + # ... and waits on the transfer alone; the concat already depends + # on the raw-VCF producer + assert dag.waiting_jobs[result.apply_job] == { + result.transfer.concat_job, + } + assert result.terminal == {result.apply_job} + assert result.jobs == [ + *result.transfer.jobs, + result.apply_job, + ] + + def test_transfer_tag_reaches_the_job_names(self, tmp_path): + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + transfer=TransferSpec( + config=make_transfer_config(tmp_path), + out_vcf=tmp_path / "transfer.vcf.gz", + tag="hp", + ), + ).add_to(DAG()) + + assert result.transfer is not None + assert result.transfer.concat_job.name == "merge-trim-hp-concat" + + def test_apply_name_override(self, tmp_path): + result = TransferApplyStage( + ctx=make_ctx(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + apply=ApplySpec( + model=tmp_path / "dnascope.model", + output=tmp_path / "applied.vcf.gz", + name="snv-apply", + ), + ).add_to(DAG()) + + assert result.apply_job is not None + assert result.apply_job.name == "snv-apply" diff --git a/tests/unit/test_stage_sv.py b/tests/unit/test_stage_sv.py new file mode 100644 index 0000000..388ce81 --- /dev/null +++ b/tests/unit/test_stage_sv.py @@ -0,0 +1,195 @@ +""" +Unit tests for the long-read SV calling stage +""" + +import pathlib + +from sentieon_cli.dag import DAG +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.sv import LongReadSVStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + dry_run=True, + skip_version_check=True, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> LongReadSVStage: + """A LongReadSVStage over one alignment""" + defaults = dict( + ctx=make_ctx(tmp_path), + inputs=[tmp_path / "sample.bam"], + model=tmp_path / "bundle" / "longreadsv.model", + ) + defaults.update(kwargs) + return LongReadSVStage(**defaults) # type: ignore[arg-type] + + +class TestCommand: + """The `sentieon driver` command the stage builds""" + + def test_longreadsv_command(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LongReadSV " + f"--model {tmp_path}/bundle/longreadsv.model " + f"{tmp_path}/output.sv.vcf.gz" + ) + + def test_interval(self, tmp_path): + bed = tmp_path / "regions.bed" + result = make_stage(tmp_path, interval=bed).add_to(DAG()) + + assert f"--interval {bed}" in str(result.job.shell) + + def test_without_an_interval(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--interval" not in str(result.job.shell) + + def test_multiple_inputs(self, tmp_path): + result = make_stage( + tmp_path, inputs=[tmp_path / "one.bam", tmp_path / "two.bam"] + ).add_to(DAG()) + + shell = str(result.job.shell) + assert f"--input {tmp_path}/one.bam" in shell + assert f"--input {tmp_path}/two.bam" in shell + + def test_replace_rg_interleaves_with_the_inputs(self, tmp_path): + result = make_stage( + tmp_path, + inputs=[tmp_path / "one.bam", tmp_path / "two.bam"], + replace_rg=[["rg1=@RG\\tID:rg1"], ["rg2=@RG\\tID:rg2"]], + ).add_to(DAG()) + + shell = str(result.job.shell) + assert ( + "--replace_rg 'rg1=@RG\\tID:rg1' " + f"--input {tmp_path}/one.bam " + "--replace_rg 'rg2=@RG\\tID:rg2' " + f"--input {tmp_path}/two.bam" + ) in shell + + def test_without_replace_rg(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert "--replace_rg" not in str(result.job.shell) + + +class TestAlgoOptions: + """The optional LongReadSV algo arguments""" + + def test_min_sv_size(self, tmp_path): + result = make_stage(tmp_path, min_sv_size=20).add_to(DAG()) + + assert str(result.job.shell) == ( + f"sentieon driver --input {tmp_path}/sample.bam " + f"--reference {tmp_path}/ref.fa --thread_count 4 " + f"--algo LongReadSV " + f"--model {tmp_path}/bundle/longreadsv.model " + f"--min_sv_size 20 " + f"{tmp_path}/output.sv.vcf.gz" + ) + + def test_min_map_qual(self, tmp_path): + result = make_stage(tmp_path, min_map_qual=30).add_to(DAG()) + + assert ( + f"--model {tmp_path}/bundle/longreadsv.model --min_map_qual 30 " + f"{tmp_path}/output.sv.vcf.gz" + ) in str(result.job.shell) + + def test_all_options(self, tmp_path): + result = make_stage( + tmp_path, + min_map_qual=30, + min_sv_size=20, + min_dp=5, + min_af=0.1, + ).add_to(DAG()) + + assert ( + "--min_map_qual 30 --min_sv_size 20 --min_dp 5 --min_af 0.1" + ) in str(result.job.shell) + + def test_options_default_to_absent(self, tmp_path): + shell = str(make_stage(tmp_path).add_to(DAG()).job.shell) + + for flag in ("min_map_qual", "min_sv_size", "min_dp", "min_af"): + assert f"--{flag}" not in shell + + +class TestOutput: + """The SV VCF is derived from the run's output VCF""" + + def test_sv_vcf(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.sv_vcf == tmp_path / "output.sv.vcf.gz" + assert str(result.sv_vcf) in str(result.job.shell) + + def test_an_explicit_output(self, tmp_path): + out_vcf = tmp_path / "tmp" / "sample-longread-sv.vcf.gz" + result = make_stage(tmp_path, output=out_vcf).add_to(DAG()) + + assert result.sv_vcf == out_vcf + assert str(result.job.shell).endswith(f" {out_vcf}") + assert "output.sv.vcf.gz" not in str(result.job.shell) + + +class TestJobMetadata: + """Job name, task name and thread count""" + + def test_defaults(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.job.name == "LongReadSV" + assert result.job.task_name == "sv-calling" + assert result.job.threads == 4 + + def test_custom_name_and_task(self, tmp_path): + result = make_stage( + tmp_path, name="longreadsv-lr", task_name="sv" + ).add_to(DAG()) + + assert result.job.name == "longreadsv-lr" + assert result.job.task_name == "sv" + + +class TestDagWiring: + """`build` touches no DAG; `add_to` inserts the job""" + + def test_build_inserts_nothing(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).build() + + assert result.jobs == [result.job] + assert result.terminal == {result.job} + assert not dag.ready_jobs + assert not dag.waiting_jobs + + def test_upstream_lands_on_the_job(self, tmp_path): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + assert dag.waiting_jobs[result.job] == {upstream} + + def test_a_root_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + assert result.job in dag.ready_jobs diff --git a/tests/unit/test_stage_transfer.py b/tests/unit/test_stage_transfer.py new file mode 100644 index 0000000..d9c03f7 --- /dev/null +++ b/tests/unit/test_stage_transfer.py @@ -0,0 +1,169 @@ +""" +Unit tests for the annotation-transfer stage +""" + +import pathlib +from types import SimpleNamespace + +import pytest + +from sentieon_cli.dag import DAG +from sentieon_cli.shard import Shard +from sentieon_cli.stages.base import StageContext, rm_job +from sentieon_cli.stages.transfer import TransferConfig, TransferStage + + +def make_ctx(tmp_path: pathlib.Path, cores: int = 4) -> StageContext: + """A StageContext over a temporary directory""" + return StageContext( + reference=tmp_path / "ref.fa", + output_vcf=tmp_path / "output.vcf.gz", + tmp_dir=tmp_path, + cores=cores, + # `build_transfer_jobs` shells out to bcftools unless this is set + dry_run=True, + skip_version_check=True, + ) + + +def make_config(tmp_path: pathlib.Path) -> TransferConfig: + """Two shards on a contig the pop VCF has, one on a contig it lacks""" + return TransferConfig( + pop_vcf=tmp_path / "pop.vcf.gz", + shards=[ + Shard("chr1", 0, 1000), + Shard("chr1", 1000, 2000), + Shard("chrUn_extra", 0, 500), + ], + pop_vcf_contigs={"chr1": 2000}, + fai_data={ + "chr1": {"length": 2000}, + "chrUn_extra": {"length": 500}, + }, + ) + + +def make_stage(tmp_path: pathlib.Path, **kwargs) -> TransferStage: + """A TransferStage over `make_config`""" + defaults = dict( + ctx=make_ctx(tmp_path), + config=make_config(tmp_path), + raw_vcf=tmp_path / "raw.vcf.gz", + out_vcf=tmp_path / "transfer.vcf.gz", + ) + defaults.update(kwargs) + return TransferStage(**defaults) # type: ignore[arg-type] + + +class TestJobs: + """The jobs the fan-out produces""" + + def test_one_job_per_shard_plus_the_extra_contig(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert [job.name for job in result.shard_jobs] == [ + "merge-trim-0", + "merge-trim-1", + "merge-trim-extra", + ] + assert result.concat_job.name == "merge-trim-concat" + + def test_task_name_and_threads(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.jobs: + assert job.task_name == "annotation-transfer" + for job in result.shard_jobs: + assert job.threads == 1 + # The concat runs with every core + assert result.concat_job.threads == 4 + + def test_shard_jobs_read_the_raw_vcf(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + for job in result.shard_jobs: + assert str(tmp_path / "raw.vcf.gz") in str(job.shell) + + def test_concat_writes_the_output(self, tmp_path): + out_vcf = tmp_path / "annotated.vcf.gz" + result = make_stage(tmp_path, out_vcf=out_vcf).add_to(DAG()) + + assert str(out_vcf) in str(result.concat_job.shell) + assert result.out_vcf == out_vcf + + +class TestTag: + """`tag` disambiguates two fan-outs in one run""" + + def test_tagged_names(self, tmp_path): + result = make_stage(tmp_path, tag="hp").add_to(DAG()) + + assert [job.name for job in result.shard_jobs] == [ + "merge-trim-hp-0", + "merge-trim-hp-1", + "merge-trim-hp-extra", + ] + assert result.concat_job.name == "merge-trim-hp-concat" + + +class TestDagWiring: + """The edges the stage inserts""" + + def test_shards_depend_on_upstream_and_concat_on_the_shards( + self, tmp_path + ): + dag = DAG() + upstream = rm_job([tmp_path / "upstream"], "upstream") + dag.add_job(upstream) + + result = make_stage(tmp_path).add_to(dag, [upstream]) + + for job in result.shard_jobs: + assert dag.waiting_jobs[job] == {upstream} + assert dag.waiting_jobs[result.concat_job] == set(result.shard_jobs) + + def test_shards_are_roots_without_upstream(self, tmp_path): + dag = DAG() + result = make_stage(tmp_path).add_to(dag) + + for job in result.shard_jobs: + assert job in dag.ready_jobs + + def test_result_fields(self, tmp_path): + result = make_stage(tmp_path).add_to(DAG()) + + assert result.jobs == [*result.shard_jobs, result.concat_job] + assert result.terminal == {result.concat_job} + + +class TestFromPipeline: + """`TransferConfig.from_pipeline` collecting a pipeline's attributes""" + + @staticmethod + def make_pipeline(tmp_path: pathlib.Path, **kwargs) -> SimpleNamespace: + """A stand-in carrying the four `TransferInputs` attributes""" + config = make_config(tmp_path) + defaults = dict( + pop_vcf=config.pop_vcf, + shards=config.shards, + pop_vcf_contigs=config.pop_vcf_contigs, + fai_data=config.fai_data, + ) + defaults.update(kwargs) + return SimpleNamespace(**defaults) + + def test_collects_the_four_values(self, tmp_path): + pipeline = self.make_pipeline(tmp_path) + + config = TransferConfig.from_pipeline(pipeline) + + assert config.pop_vcf == pipeline.pop_vcf + assert config.shards == pipeline.shards + assert config.pop_vcf_contigs == pipeline.pop_vcf_contigs + assert config.fai_data == pipeline.fai_data + + def test_without_a_pop_vcf(self, tmp_path): + pipeline = self.make_pipeline(tmp_path, pop_vcf=None) + + with pytest.raises(ValueError, match="population VCF"): + TransferConfig.from_pipeline(pipeline)