Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 10 additions & 24 deletions sentieon_cli/command_strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@

logger = get_logger(__name__)

# Requested OS pipe-buffer size (bytes) for throughput-bound alignment
# pipelines. Applied best-effort: the kernel caps unprivileged requests at
# fs.pipe-max-size and non-Linux platforms keep their default size.
ALN_PIPE_SIZE = 268435456


def cmd_fai_to_bed(
fai: pathlib.Path,
Expand Down Expand Up @@ -596,9 +601,6 @@ def cmd_samtools_fastq_bwa(
if input_ref:
ref_cmd = ["--reference", str(reference)]

pipebuf_cmd = Command(
"perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)"
)
if collate:
collate_cmd = Command(
"samtools",
Expand Down Expand Up @@ -667,20 +669,18 @@ def cmd_samtools_fastq_bwa(
)
if collate_cmd:
return Pipeline(
pipebuf_cmd,
collate_cmd,
fastq_cmd,
bwa_cmd,
sort_cmd,
skip_pipe=(0,),
pipe_size=ALN_PIPE_SIZE,
)
else:
return Pipeline(
pipebuf_cmd,
fastq_cmd,
bwa_cmd,
sort_cmd,
skip_pipe=(0,),
pipe_size=ALN_PIPE_SIZE,
)


Expand Down Expand Up @@ -746,21 +746,10 @@ def cmd_fastq_bwa(
split: Optional[str] = None,
) -> Pipeline:
"""Align an input fastq file with bwa"""
pipebuf_cmd = Command(
"perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)"
)
r1_unzip = Pipeline(
pipebuf_cmd,
Command(str(unzip), "-dc", str(r1)),
skip_pipe=(0,),
)
r1_unzip = Pipeline(Command(str(unzip), "-dc", str(r1)))
r2_unzip = None
if r2:
r2_unzip = Pipeline(
pipebuf_cmd,
Command(str(unzip), "-dc", str(r2)),
skip_pipe=(0,),
)
r2_unzip = Pipeline(Command(str(unzip), "-dc", str(r2)))
bwa_cmd_args: List[Union[str, InputProcSub]] = [
*(["taskset", "-c", numa] if numa else []),
"sentieon",
Expand All @@ -780,7 +769,6 @@ def cmd_fastq_bwa(
]
if split:
extract_cmd = Pipeline(
pipebuf_cmd,
Command(
"sentieon",
"fqidx",
Expand All @@ -792,7 +780,6 @@ def cmd_fastq_bwa(
InputProcSub(r1_unzip),
*([InputProcSub(r2_unzip)] if r2_unzip else []),
),
skip_pipe=(0,),
)
bwa_cmd_args.append(InputProcSub(extract_cmd))
else:
Expand All @@ -816,10 +803,9 @@ def cmd_fastq_bwa(
*util_sort_args.split(),
)
return Pipeline(
pipebuf_cmd,
Command(str(bwa_cmd_args[0]), *bwa_cmd_args[1:]),
sort_cmd,
skip_pipe=(0,),
pipe_size=ALN_PIPE_SIZE,
)


Expand Down
93 changes: 50 additions & 43 deletions sentieon_cli/dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from __future__ import annotations

from typing import Dict, Generator, List, Optional, Set
from typing import Dict, Iterable, List, Optional, Set

from .exceptions import DagExecutionError
from .job import Job
Expand All @@ -25,54 +25,61 @@ def __init__(self) -> None:
self.ready_jobs: Dict[Job, None] = {}
self.finished_jobs: List[Job] = []

def add_job(self, job: Job, dependencies: Optional[Set[Job]] = None):
"""Add a job to the DAG"""
if dependencies:
for dependency in dependencies:
if (
dependency not in self.waiting_jobs
and dependency not in self.ready_jobs
):
raise ValueError(
f"Dependency '{dependency}' is not in the DAG"
)
def add_job(
self, job: Job, dependencies: Optional[Iterable[Job]] = None
) -> None:
"""Add a job to the DAG.

if isinstance(dependencies, set) and len(dependencies) > 0:
self.waiting_jobs[job] = dependencies.copy()
for dependency in dependencies:
``dependencies`` may be any iterable of jobs already in the DAG;
it is consumed once and stored as a private set.
"""
deps: Set[Job] = (
set(dependencies) if dependencies is not None else set()
)
if job in self.waiting_jobs or job in self.ready_jobs:
raise ValueError(
f"{job} is already in the DAG; jobs are identified by their "
f"shell pipeline, so the same pipeline cannot be added twice."
)
for dependency in deps:
if (
dependency not in self.waiting_jobs
and dependency not in self.ready_jobs
):
raise ValueError(
f"Dependency '{dependency}' is not in the DAG"
)

if deps:
self.waiting_jobs[job] = deps
for dependency in deps:
downstream = self.planned_jobs.setdefault(dependency, [])
downstream.append(job)
else:
self.ready_jobs[job] = None

def update_dag(
self,
) -> Generator[Dict[Job, None], Optional[Job], None]:
"""Update the DAG with a finished job"""
finished_job = yield self.ready_jobs.copy()
def mark_finished(self, job: Job) -> Dict[Job, None]:
"""Record a finished job and return the jobs it unblocked.

while True:
logger.debug("Waiting jobs: %s", self.waiting_jobs)
logger.debug("Planned jobs: %s", self.planned_jobs)
logger.debug("Ready jobs: %s", self.ready_jobs)
logger.debug("Finished jobs: %s", self.finished_jobs)
logger.debug("Newly finished: %s", finished_job)
The returned mapping is used as an ordered set (values are ``None``).
Raises :class:`~sentieon_cli.exceptions.DagExecutionError` if ``job``
was not currently ready.
"""
if job not in self.ready_jobs:
raise DagExecutionError(
f"Finished job '{job}' was not ready for execution"
)

new_ready_jobs: Dict[Job, None] = {}
if isinstance(finished_job, Job):
if finished_job not in self.ready_jobs:
raise DagExecutionError(
f"Finished job '{finished_job}' was not ready for "
"execution"
)
logger.debug("Marking finished: %s", job)
del self.ready_jobs[job]
self.finished_jobs.append(job)

del self.ready_jobs[finished_job]
self.finished_jobs.append(finished_job)
for dependency in self.planned_jobs.get(finished_job, []):
upstream = self.waiting_jobs[dependency]
upstream.remove(finished_job)
if len(upstream) < 1:
self.ready_jobs[dependency] = None
new_ready_jobs[dependency] = None
del self.waiting_jobs[dependency]
finished_job = yield new_ready_jobs
new_ready_jobs: Dict[Job, None] = {}
for dependency in self.planned_jobs.get(job, []):
upstream = self.waiting_jobs[dependency]
upstream.remove(job)
if len(upstream) < 1:
self.ready_jobs[dependency] = None
new_ready_jobs[dependency] = None
del self.waiting_jobs[dependency]
return new_ready_jobs
Loading
Loading