From 8d3d68e0b36f8a5ee1b01abad6e647cf45be48c6 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 12 Aug 2026 12:46:22 -0700 Subject: [PATCH 1/4] Fix shell-pipeline correctness bugs and size internal pipes natively Correctness fixes in the pipeline AST: - Command.__eq__ no longer mixes in self.proc (runtime state), so a command's identity is stable once its job runs. exec_kwargs (env, cwd) now participates in __eq__/__hash__ (canonicalised for order-independence) and renders into str() as a conventional bash prefix, so two jobs differing only in env/cwd no longer collide. - ProcSub.__eq__ requires the same subclass: <(x) != >(x). - Pipeline file/stream conflicts raise ValueError instead of calling sys.exit(), and the guards test `is not None` so fd 0 is not skipped. Fix a deadlock when a proc-sub FIFO is never opened: if the outer command failed to spawn or exited early, the background task's blocking open() never returned and Context.cleanup() hung the whole run, including the Ctrl-C path. cleanup() now unblocks still-unopened FIFOs (holding both ends non-blocking), lets the unblocked task return without spawning, and re-raises task exceptions only after all resources are released. Replace the injected `perl -MFcntl` pipe-buffer command with a native Pipeline(pipe_size=...) option. The injected command received the inherited stdout rather than the internal pipe it was meant to grow, so F_SETPIPE_SZ hit a non-pipe fd and the pipe kept its default 64 KiB size. The pipes are created with os.pipe() here, so they are now sized directly (best-effort; logged fallback on non-Linux or over fs.pipe-max-size) before the children inherit them. skip_pipe existed only for the injection and is removed; the bwa alignment pipelines in command_strings.py request the intended 256 MiB via pipe_size. Co-Authored-By: Claude Fable 5 --- sentieon_cli/command_strings.py | 34 ++-- sentieon_cli/shell_pipeline.py | 274 ++++++++++++++++++++--------- tests/unit/test_shell_pipeline.py | 275 ++++++++++++++++++++++++++++-- 3 files changed, 468 insertions(+), 115 deletions(-) diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index 4bdebde..2595fe1 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -18,6 +18,11 @@ logger = get_logger(__name__) +# Requested OS pipe-buffer size (bytes) for throughput-bound alignment +# pipelines. Applied best-effort: the kernel caps unprivileged requests at +# fs.pipe-max-size and non-Linux platforms keep their default size. +ALN_PIPE_SIZE = 268435456 + def cmd_fai_to_bed( fai: pathlib.Path, @@ -596,9 +601,6 @@ def cmd_samtools_fastq_bwa( if input_ref: ref_cmd = ["--reference", str(reference)] - pipebuf_cmd = Command( - "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" - ) if collate: collate_cmd = Command( "samtools", @@ -667,20 +669,18 @@ def cmd_samtools_fastq_bwa( ) if collate_cmd: return Pipeline( - pipebuf_cmd, collate_cmd, fastq_cmd, bwa_cmd, sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) else: return Pipeline( - pipebuf_cmd, fastq_cmd, bwa_cmd, sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) @@ -746,21 +746,10 @@ def cmd_fastq_bwa( split: Optional[str] = None, ) -> Pipeline: """Align an input fastq file with bwa""" - pipebuf_cmd = Command( - "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" - ) - r1_unzip = Pipeline( - pipebuf_cmd, - Command(str(unzip), "-dc", str(r1)), - skip_pipe=(0,), - ) + r1_unzip = Pipeline(Command(str(unzip), "-dc", str(r1))) r2_unzip = None if r2: - r2_unzip = Pipeline( - pipebuf_cmd, - Command(str(unzip), "-dc", str(r2)), - skip_pipe=(0,), - ) + r2_unzip = Pipeline(Command(str(unzip), "-dc", str(r2))) bwa_cmd_args: List[Union[str, InputProcSub]] = [ *(["taskset", "-c", numa] if numa else []), "sentieon", @@ -780,7 +769,6 @@ def cmd_fastq_bwa( ] if split: extract_cmd = Pipeline( - pipebuf_cmd, Command( "sentieon", "fqidx", @@ -792,7 +780,6 @@ def cmd_fastq_bwa( InputProcSub(r1_unzip), *([InputProcSub(r2_unzip)] if r2_unzip else []), ), - skip_pipe=(0,), ) bwa_cmd_args.append(InputProcSub(extract_cmd)) else: @@ -816,10 +803,9 @@ def cmd_fastq_bwa( *util_sort_args.split(), ) return Pipeline( - pipebuf_cmd, Command(str(bwa_cmd_args[0]), *bwa_cmd_args[1:]), sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) diff --git a/sentieon_cli/shell_pipeline.py b/sentieon_cli/shell_pipeline.py index 02eb082..2469484 100644 --- a/sentieon_cli/shell_pipeline.py +++ b/sentieon_cli/shell_pipeline.py @@ -1,21 +1,77 @@ -"""Complex shell pipelines with process substitution""" +"""Complex shell pipelines with process substitution. + +POSIX only: pipelines are wired together with OS pipes and FIFOs, and process +substitution relies on named pipes, so this module does not work on Windows. +""" from __future__ import annotations import asyncio +import dataclasses +import fcntl import io import os import pathlib import tempfile import shlex -import sys from abc import ABC, abstractmethod -from typing import Any, Dict, IO, Iterable, List, Optional, Union +from typing import Any, Dict, IO, List, Optional, Union from .logging import get_logger logger = get_logger(__name__) +def _freeze(value: Any) -> Any: + """Return an order-independent, hashable view of ``value``. + + ``exec_kwargs`` (and a nested ``env`` dict) have no stable iteration + order, so hashing their raw form would let two *equal* commands hash + differently, breaking the ``a == b implies hash(a) == hash(b)`` + contract the DAG relies on. Canonicalise dicts/sets recursively. + """ + if isinstance(value, dict): + return frozenset((k, _freeze(v)) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return tuple(_freeze(v) for v in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(v) for v in value) + return value + + +def _set_pipe_size(fd: int, size: int) -> None: + """Best-effort enlarge the OS pipe buffer backing ``fd``. + + Lets a throughput-bound pipeline hold more in-flight data between + stages. ``F_SETPIPE_SZ`` is Linux-only and, for unprivileged + processes, capped by ``/proc/sys/fs/pipe-max-size`` (exceeding it + raises EPERM) -- both deployment concerns, so a failure is logged and + the pipe keeps its default size rather than aborting the run. A no-op + where the constant is unavailable (non-Linux), preserving POSIX + portability. + """ + set_pipe_sz = getattr(fcntl, "F_SETPIPE_SZ", None) + if set_pipe_sz is None: + return + try: + fcntl.fcntl(fd, set_pipe_sz, size) + except OSError as exc: + logger.warning( + "could not set pipe size to %d (%s); using the default. " + "Raise fs.pipe-max-size to allow larger buffers.", + size, + exc, + ) + + +@dataclasses.dataclass +class _ProcSubWait: + """A proc-sub FIFO whose blocking open may need unblocking.""" + + fifo_path: str + opened: bool = False # run_inner's open() returned (set pre-await) + unblocked: bool = False # cleanup already opened both ends + + class Context: """Holds shared resources.""" @@ -23,11 +79,15 @@ def __init__(self) -> None: # This directory is already unique per pipeline run self.temp_dir = tempfile.TemporaryDirectory() # Background tasks to wait for (e.g., proc sub processes) - self.tasks: List[asyncio.Task] = [] + self.tasks: List[asyncio.Task[Any]] = [] # Hold the commands run in this context self.commands: List[Command] = [] self._counter = 0 # Monotonic counter self.file_handles: List[io.IOBase] = [] + # Set by cleanup(); a proc sub whose FIFO open was unblocked by + # cleanup must not spawn its inner command. + self.closing: bool = False + self._procsub_waits: List[_ProcSubWait] = [] def get_new_fifo(self) -> str: """Creates a new unique FIFO and returns its path.""" @@ -38,11 +98,48 @@ def get_new_fifo(self) -> str: return path async def cleanup(self) -> None: - if self.tasks: - await asyncio.gather(*self.tasks) + """Wait for background tasks, then release all resources. + + A proc sub's blocking FIFO open() never returns if the outer + command exited without opening the other end, so cleanup first + unblocks those opens, then waits for the tasks. Task exceptions + are re-raised only after every resource has been released. + """ + self.closing = True + unblock_fds: List[int] = [] + try: + while True: + for wait in self._procsub_waits: + if wait.opened or wait.unblocked: + continue + wait.unblocked = True + # Hold both ends open: this unblocks a blocked open() + # of either end, and read-then-write cannot raise + # ENXIO. + unblock_fds.append( + os.open(wait.fifo_path, os.O_RDONLY | os.O_NONBLOCK) + ) + unblock_fds.append( + os.open(wait.fifo_path, os.O_WRONLY | os.O_NONBLOCK) + ) + # Tasks can register new proc subs (nested pipelines), so + # re-snapshot until everything is done. + pending = [t for t in self.tasks if not t.done()] + if not pending: + break + await asyncio.gather(*pending, return_exceptions=True) + finally: + for fd in unblock_fds: + os.close(fd) for fh in self.file_handles: fh.close() self.temp_dir.cleanup() + for task in self.tasks: + if task.cancelled(): + continue + exc = task.exception() + if exc is not None and not isinstance(exc, asyncio.CancelledError): + raise exc class ShellNode(ABC): @@ -56,8 +153,8 @@ def __init__( self, executable: str, *args: str | ProcSub, - fail_ok=False, - exec_kwargs: Optional[Dict] = None, + fail_ok: bool = False, + exec_kwargs: Optional[Dict[str, Any]] = None, ) -> None: self.executable = executable self.args = list(args) @@ -68,9 +165,9 @@ def __init__( async def run( self, context: Context, - stdin: Union[IO, int, None] = None, - stdout: Union[IO, int, None] = None, - stderr: Union[IO, int, None] = None, + stdin: Union[IO[Any], int, None] = None, + stdout: Union[IO[Any], int, None] = None, + stderr: Union[IO[Any], int, None] = None, ) -> asyncio.subprocess.Process: # 1. Resolve Arguments (Handle Process Substitutions) final_args = [self.executable] @@ -96,27 +193,47 @@ async def run( return self.proc def __str__(self) -> str: - return shlex.join([self.executable] + [str(x) for x in self.args]) + rendered = shlex.join([self.executable] + [str(x) for x in self.args]) + # Render env/cwd so the command is self-contained bash (backends + # submit str(job.shell)). Other exec_kwargs (close_fds, ...) have no + # shell rendering and are omitted. env is a conventional K=V prefix + # that merges into the inherited environment. + env = self.exec_kwargs.get("env") + if env: + assignments = " ".join( + f"{key}={shlex.quote(str(val))}" for key, val in env.items() + ) + rendered = f"{assignments} {rendered}" + cwd = self.exec_kwargs.get("cwd") + if cwd: + rendered = f"(cd {shlex.quote(str(cwd))} && {rendered})" + return rendered def __repr__(self) -> str: return ( f"{self.__class__.__name__}({self.executable}, " + ", ".join([repr(x) for x in self.args]) - + f", fail_ok={self.fail_ok})" + + f", fail_ok={self.fail_ok}, " + + f"exec_kwargs={self.exec_kwargs!r})" ) def __eq__(self, other: object) -> bool: if not isinstance(other, Command): return NotImplemented + # Identity is the *definition* of the command: not self.proc, which + # is runtime state that would mutate identity once the job runs. return ( self.executable == other.executable and self.args == other.args and self.fail_ok == other.fail_ok - and self.proc == other.proc + and self.exec_kwargs == other.exec_kwargs ) def __hash__(self) -> int: - return hash(tuple([self.executable, self.fail_ok] + self.args)) + return hash( + tuple([self.executable, self.fail_ok] + self.args) + + (_freeze(self.exec_kwargs),) + ) class Pipeline(ShellNode): @@ -125,39 +242,40 @@ class Pipeline(ShellNode): def __init__( self, *nodes: Command, - skip_pipe: Optional[Iterable[int]] = None, + pipe_size: Optional[int] = None, file_input: Optional[pathlib.Path] = None, file_output: Optional[pathlib.Path] = None, ): self.nodes = list(nodes) if not self.nodes: raise ValueError("Pipeline nodes cannot be empty") - self.skip_pipe = set(skip_pipe) if skip_pipe else set() + # Best-effort OS pipe-buffer size (bytes) for every internal pipe; + # None keeps the kernel default. A local-execution tuning hint, so + # it is not part of pipeline identity (like Job.threads). + self.pipe_size = pipe_size self.file_input = file_input self.file_output = file_output async def run( self, context: Context, - stdin: Union[IO, int, None] = None, - stdout: Union[IO, int, None] = None, - stderr: Union[IO, int, None] = None, + stdin: Union[IO[Any], int, None] = None, + stdout: Union[IO[Any], int, None] = None, + stderr: Union[IO[Any], int, None] = None, ) -> asyncio.subprocess.Process: - # We cannot have handles from both files and to run() - if self.file_input and stdin: - logger.error( - "Pipeline %s with both file and stdin input. %s", - self, - str(stdin), + # We cannot have handles from both files and to run(). + # Test `is not None`, not truthiness: an int fd of 0 is a valid + # stream but falsy. + if self.file_input is not None and stdin is not None: + raise ValueError( + f"Pipeline {self} was given both a file_input and a stdin " + f"stream ({stdin!r}); provide only one." ) - sys.exit(1) - if self.file_output and stdout: - logger.error( - "Pipeline %s with both file and stdout output. %s", - self, - str(stdout), + if self.file_output is not None and stdout is not None: + raise ValueError( + f"Pipeline {self} was given both a file_output and a stdout " + f"stream ({stdout!r}); provide only one." ) - sys.exit(1) processes = [] # We need to close these FDs in the parent after passing them to @@ -166,19 +284,18 @@ async def run( # pass the file handle as input to the pipeline current_stdin = stdin - if self.file_input: + if self.file_input is not None: current_stdin = open(self.file_input, "rb") context.file_handles.append(current_stdin) try: # Chain all commands except the last one for i in range(len(self.nodes) - 1): - read_fd = None - write_fd = None - if i not in self.skip_pipe: - # Create OS pipe - read_fd, write_fd = os.pipe() - fds_to_close.extend([read_fd, write_fd]) + # Create the OS pipe linking this command to the next. + read_fd, write_fd = os.pipe() + fds_to_close.extend([read_fd, write_fd]) + if self.pipe_size is not None: + _set_pipe_size(write_fd, self.pipe_size) # Run the node (Command or sub-Pipeline) # stdout goes to the write-end of the pipe @@ -190,16 +307,15 @@ async def run( ) processes.append(proc) - if write_fd: - # Close write_fd in parent (child has it now) - os.close(write_fd) - fds_to_close.remove(write_fd) + # Close write_fd in parent (child has it now) + os.close(write_fd) + fds_to_close.remove(write_fd) # Next command reads from the read-end current_stdin = read_fd # Send the pipeline output to a file - if self.file_output: + if self.file_output is not None: stdout = open(self.file_output, "wb") context.file_handles.append(stdout) @@ -226,13 +342,7 @@ def __str__(self) -> str: res = [] if self.file_input: res.append(f"<'{self.file_input}' ") - res.append(str(self.nodes[0])) - for i, node in enumerate(self.nodes[1:]): - if i in self.skip_pipe: - res.append("; ") - else: - res.append(" | ") - res.append(str(node)) + res.append(" | ".join(str(node) for node in self.nodes)) if self.file_output: res.append(f" >'{self.file_output}'") return "".join(res) @@ -241,26 +351,23 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" + ", ".join(repr(x) for x in self.nodes) - + f"skip_pipe={self.skip_pipe},file_input={self.file_input}," + + f", pipe_size={self.pipe_size}, " + + f"file_input={self.file_input}, " + f"file_output={self.file_output})" ) def __eq__(self, other: object) -> bool: if not isinstance(other, Pipeline): return NotImplemented + # pipe_size is a tuning hint, not part of identity (see __init__). return ( self.nodes == other.nodes - and self.skip_pipe == other.skip_pipe and self.file_input == other.file_input and self.file_output == other.file_output ) def __hash__(self) -> int: attrs: List[Any] = self.nodes + [self.file_input, self.file_output] - if self.skip_pipe is None: - attrs.append(None) - else: - attrs.append(tuple(self.skip_pipe)) return hash(tuple(attrs)) @@ -282,7 +389,8 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: if not isinstance(other, ProcSub): return NotImplemented - return self.node == other.node + # An input and an output proc sub are not the same: <(x) != >(x). + return type(self) is type(other) and self.node == other.node def __hash__(self) -> int: return hash((self.__class__.__name__, self.node)) @@ -296,22 +404,26 @@ class InputProcSub(ProcSub): """ async def setup(self, context: Context) -> str: - self.fifo_path = context.get_new_fifo() + fifo_path = context.get_new_fifo() + self.fifo_path = fifo_path + wait = _ProcSubWait(fifo_path) # We must open the FIFO for writing. # WARNING: opening a FIFO blocks until the other end is opened. # We run the open() in a thread to prevent blocking the async event # loop. - def open_fifo_write(): - if not self.fifo_path: - raise RuntimeError("FIFO path not set") - return open(self.fifo_path, "w") + def open_fifo_write() -> IO[Any]: + return open(fifo_path, "w") # Start the task that waits for the open, then runs the process - async def run_inner(): - # This awaits until the outer command opens the file for reading! + async def run_inner() -> None: + # This awaits until the outer command opens the file for + # reading (or until cleanup() unblocks the open)! write_handle = await asyncio.to_thread(open_fifo_write) + wait.opened = True try: + if context.closing: + return # cleanup unblocked us; do not spawn proc = await self.node.run( context, stdout=write_handle, @@ -321,10 +433,10 @@ async def run_inner(): write_handle.close() # Schedule this task in the background - task = asyncio.create_task(run_inner()) - context.tasks.append(task) + context.tasks.append(asyncio.create_task(run_inner())) + context._procsub_waits.append(wait) - return self.fifo_path + return fifo_path def __str__(self) -> str: return "<(" + str(self.node) + ")" @@ -338,26 +450,30 @@ class OutputProcSub(ProcSub): """ async def setup(self, context: Context) -> str: - self.fifo_path = context.get_new_fifo() + fifo_path = context.get_new_fifo() + self.fifo_path = fifo_path + wait = _ProcSubWait(fifo_path) - def open_fifo_read(): - if not self.fifo_path: - raise RuntimeError("FIFO path not set") - return open(self.fifo_path, "r") + def open_fifo_read() -> IO[Any]: + return open(fifo_path, "r") - async def run_inner(): - # This awaits until the outer command opens the file for writing! + async def run_inner() -> None: + # This awaits until the outer command opens the file for + # writing (or until cleanup() unblocks the open)! read_handle = await asyncio.to_thread(open_fifo_read) + wait.opened = True try: + if context.closing: + return # cleanup unblocked us; do not spawn proc = await self.node.run(context, stdin=read_handle) await proc.wait() finally: read_handle.close() - task = asyncio.create_task(run_inner()) - context.tasks.append(task) + context.tasks.append(asyncio.create_task(run_inner())) + context._procsub_waits.append(wait) - return self.fifo_path + return fifo_path def __str__(self) -> str: return ">(" + str(self.node) + ")" diff --git a/tests/unit/test_shell_pipeline.py b/tests/unit/test_shell_pipeline.py index 943d5b9..38ec2bd 100644 --- a/tests/unit/test_shell_pipeline.py +++ b/tests/unit/test_shell_pipeline.py @@ -3,6 +3,7 @@ """ import asyncio +import fcntl import os import pathlib import sys @@ -31,7 +32,6 @@ async def test_simple_command(): cmd = Command("echo", "hello world") context = Context() - # Use a file for stdout with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: proc = await cmd.run(context, stdout=stdout_file) await proc.wait() @@ -73,7 +73,6 @@ async def test_pipeline_with_file_io(): """Test a pipeline with file input and output""" context = Context() - # Create input and output files with tempfile.NamedTemporaryFile(mode="w", delete=False) as infile: infile.write("hello from file") infile_path = pathlib.Path(infile.name) @@ -82,12 +81,16 @@ async def test_pipeline_with_file_io(): cmd1 = Command("cat") cmd2 = Command("wc", "-c") - pipeline = Pipeline(cmd1, cmd2, file_input=infile_path, file_output=outfile_path) + pipeline = Pipeline( + cmd1, + cmd2, + file_input=infile_path, + file_output=outfile_path, + ) proc = await pipeline.run(context) await proc.wait() - # Verify output file content with open(outfile_path, "r") as f: output = f.read().strip() @@ -132,18 +135,14 @@ async def test_output_process_substitution(): """Test output process substitution: tee >(wc -c)""" context = Context() - # We will write the output of wc -c to a file to check it outfile_path = pathlib.Path(context.temp_dir.name) / "wc_output.txt" - # The pipeline will be: echo "hello" | tee >(wc -c > outfile.txt) + # echo "hello" | tee >(wc -c > outfile.txt) # The main output of the pipeline should be "hello" - # The output of the process substitution should be the character count "6" - - # Inner pipeline for process substitution + # The output of the process substitution should be the char count. wc_cmd = Command("wc", "-c") wc_pipeline = Pipeline(wc_cmd, file_output=outfile_path) - # Main pipeline echo_cmd = Command("echo", "hello") tee_cmd = Command("tee", OutputProcSub(wc_pipeline)) main_pipeline = Pipeline(echo_cmd, tee_cmd) @@ -158,10 +157,8 @@ async def test_output_process_substitution(): stdout_file.seek(0) main_output = stdout_file.read().strip() - # Verify main output assert main_output == "hello" - # Verify process substitution output with open(outfile_path, "r") as f: wc_output = f.read().strip() @@ -171,3 +168,257 @@ async def test_output_process_substitution(): await context.cleanup() os.remove(stdout_file.name) assert proc.returncode == 0 + + +@pytest.mark.asyncio +async def test_pipe_size_enlarges_internal_pipes(tmp_path): + """pipe_size raises F_SETPIPE_SZ on the pipeline's internal pipes, + which the downstream child inherits.""" + if not hasattr(fcntl, "F_SETPIPE_SZ"): + pytest.skip("F_SETPIPE_SZ not available (non-Linux)") + try: + pipe_max = int(open("/proc/sys/fs/pipe-max-size").read()) + except OSError: + pytest.skip("cannot read fs.pipe-max-size") + target = 128 * 1024 # a power of two, so the kernel sets it exactly + if pipe_max < target: + pytest.skip("fs.pipe-max-size too small for this test") + + report = tmp_path / "size.txt" + # The first stage's stdout is the internal pipe to `cat`; report that + # pipe's buffer size (1032 == F_GETPIPE_SZ) rather than writing to it. + reporter = ( + "import fcntl, sys; " + "open(sys.argv[1], 'w').write(str(fcntl.fcntl(1, 1032)))" + ) + context = Context() + pipeline = Pipeline( + Command(sys.executable, "-c", reporter, str(report)), + Command("cat"), + pipe_size=target, + ) + proc = await pipeline.run(context) + await proc.wait() + for sub in context.commands: + if sub.proc: + await sub.proc.wait() + await context.cleanup() + + assert int(report.read_text()) == target + + +def test_command_and_pipeline_hash_identity(): + """Equal commands/pipelines hash equally (DAG relies on this).""" + c1 = Command("echo", "x") + c2 = Command("echo", "x") + c3 = Command("echo", "y") + assert c1 == c2 + assert hash(c1) == hash(c2) + assert c1 != c3 + + p1 = Pipeline(Command("echo", "x"), Command("cat")) + p2 = Pipeline(Command("echo", "x"), Command("cat")) + p3 = Pipeline(Command("echo", "x")) + assert p1 == p2 + assert hash(p1) == hash(p2) + assert p1 != p3 + + +def test_procsub_input_and_output_are_distinct(): + """<(x) and >(x) wrap the same pipeline but are not the same node: they + must be unequal and hash differently.""" + p = Pipeline(Command("echo", "x")) + assert InputProcSub(p) != OutputProcSub(p) + assert hash(InputProcSub(p)) != hash(OutputProcSub(p)) + # Each still equals another of its own kind wrapping an equal pipeline. + other = Pipeline(Command("echo", "x")) + assert InputProcSub(p) == InputProcSub(other) + assert hash(InputProcSub(p)) == hash(InputProcSub(other)) + + +def test_command_identity_is_stable_across_running(): + """A Command's identity must not change once it has started: __eq__ + used to mix in self.proc, so running a job mutated its identity.""" + c1 = Command("echo", "x") + c2 = Command("echo", "x") + assert c1 == c2 and hash(c1) == hash(c2) + c1.proc = object() # stand in for a started asyncio process + assert c1 == c2 + assert hash(c1) == hash(c2) + + +def test_command_identity_includes_exec_kwargs(): + """exec_kwargs participates in identity: commands differing only in + env/cwd are distinct, and equal env hashes the same regardless of the + dict's insertion order.""" + assert Command("run", exec_kwargs={"cwd": "/a"}) != Command( + "run", exec_kwargs={"cwd": "/b"} + ) + assert Command("run", exec_kwargs={"cwd": "/a"}) != Command("run") + e1 = Command("run", exec_kwargs={"env": {"A": "1", "B": "2"}}) + e2 = Command("run", exec_kwargs={"env": {"B": "2", "A": "1"}}) + assert e1 == e2 + assert hash(e1) == hash(e2) + + +def test_str_renders_env_and_cwd(): + """str() renders exec_kwargs env/cwd as a conventional bash prefix, + quoting values that need it.""" + env_cmd = Command("samtools", "sort", exec_kwargs={"env": {"REF": "hg"}}) + assert str(env_cmd) == "REF=hg samtools sort" + cwd_cmd = Command("samtools", "sort", exec_kwargs={"cwd": "/data"}) + assert str(cwd_cmd) == "(cd /data && samtools sort)" + both = Command( + "run", exec_kwargs={"cwd": "/data dir", "env": {"K": "v w"}} + ) + assert str(both) == "(cd '/data dir' && K='v w' run)" + + +def test_empty_pipeline_raises(): + with pytest.raises(ValueError): + Pipeline() + + +@pytest.mark.asyncio +async def test_file_output_and_stdout_conflict_raises(): + """A file/stdout conflict raises ValueError (not sys.exit).""" + context = Context() + out = pathlib.Path(context.temp_dir.name) / "out.txt" + pipeline = Pipeline(Command("echo", "x"), file_output=out) + with pytest.raises(ValueError): + await pipeline.run(context, stdout=1) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_file_input_and_stdin_fd0_conflict_raises(tmp_path): + """file_input plus a stdin of fd 0 conflicts: fd 0 is a valid stream but + falsy, so the guard must test `is not None`, not truthiness.""" + inp = tmp_path / "in.txt" + inp.write_text("data") + context = Context() + pipeline = Pipeline(Command("cat"), file_input=inp) + with pytest.raises(ValueError): + await pipeline.run(context, stdin=0) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_file_input_and_stdin_conflict_raises(): + """A file/stdin conflict raises ValueError (not sys.exit).""" + context = Context() + infile = pathlib.Path(context.temp_dir.name) / "in.txt" + pipeline = Pipeline(Command("cat"), file_input=infile) + with pytest.raises(ValueError): + await pipeline.run(context, stdin=1) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_unopened_input_procsub(): + """cleanup() must not hang when an <(...) FIFO was never opened.""" + context = Context() + proc_sub = InputProcSub(Pipeline(Command("echo", "x"))) + fifo_path = await proc_sub.setup(context) + assert os.path.exists(fifo_path) + + await asyncio.wait_for(context.cleanup(), timeout=10) + + # The inner command never spawned: nothing opened the FIFO. + assert context.commands == [] + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_unopened_output_procsub(): + """cleanup() must not hang when a >(...) FIFO was never opened.""" + context = Context() + proc_sub = OutputProcSub(Pipeline(Command("cat"))) + await proc_sub.setup(context) + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert context.commands == [] + + +@pytest.mark.asyncio +async def test_cleanup_after_outer_exits_without_opening(): + """An outer command that exits without opening its proc-sub FIFO + (here: false ignores its arguments) must not hang cleanup().""" + context = Context() + cmd = Command("false", InputProcSub(Pipeline(Command("echo", "x")))) + proc = await cmd.run(context) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert [c.executable for c in context.commands] == ["false"] + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_multiple_procsubs(): + """Several never-opened proc subs are all unblocked by cleanup().""" + context = Context() + cmd = Command( + "false", + InputProcSub(Pipeline(Command("echo", "x"))), + OutputProcSub(Pipeline(Command("cat"))), + ) + proc = await cmd.run(context) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert all(task.done() for task in context.tasks) + + +@pytest.mark.asyncio +async def test_cleanup_still_raises_inner_launch_failure(): + """An inner command that cannot be spawned still propagates from + cleanup(), but only after all resources have been released.""" + context = Context() + cmd = Command( + "cat", InputProcSub(Pipeline(Command("no_such_cmd_zzz"))) + ) + proc = await cmd.run(context) + await proc.wait() + + with pytest.raises(FileNotFoundError): + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert not os.path.exists(context.temp_dir.name) + + +@pytest.mark.asyncio +async def test_cleanup_does_not_stall_running_inner_writer(): + """cleanup() must not touch the FIFO of a proc sub whose inner command + is already running; holding a read end there would prevent the EPIPE + that lets the inner writer exit.""" + context = Context() + cmd = Command( + "sh", + "-c", + 'sleep 0.5 <"$1"', + "sh", + InputProcSub(Pipeline(Command("yes"))), + ) + proc = await cmd.run(context) + # Wait until the inner command has spawned (sh + yes). + async with asyncio.timeout(5): + while len(context.commands) < 2: + await asyncio.sleep(0.01) + + await asyncio.wait_for(context.cleanup(), timeout=10) + assert await proc.wait() == 0 + + +@pytest.mark.asyncio +async def test_cleanup_twice_is_safe(): + """A second cleanup() call is a harmless no-op.""" + context = Context() + cmd = Command("cat", InputProcSub(Pipeline(Command("echo", "hello")))) + proc = await cmd.run(context, stdout=asyncio.subprocess.DEVNULL) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + await asyncio.wait_for(context.cleanup(), timeout=10) + assert proc.returncode == 0 From a34554fec59930acd5ae29b7a6b12e03147030dd Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 12 Aug 2026 12:52:50 -0700 Subject: [PATCH 2/4] Rework the DAG/scheduler/executor engine layer Replace the undocumented send(None)/send(job) generator protocol between the DAG, scheduler, and executors with an explicit, typed interface: DAG.mark_finished(job) returns the jobs a completion unblocked, and BaseScheduler (new ABC) exposes start() / job_finished(job) with the contract documented and covered by a conformance-test harness. Make interrupt handling asyncio-correct and opt-in. The old handler ran time.sleep(10) and asyncio.run(context.cleanup()) inside a signal handler while the executor's event loop was running -- asyncio.run raises RuntimeError from within a running loop, and the blocking sleep froze it. Handlers (SIGINT/SIGTERM) are now installed on the running loop via loop.add_signal_handler only for the duration of a run (and only on the main thread), the previous handlers are restored afterwards, and shutdown is fully async: SIGTERM -> bounded grace period -> SIGKILL -> awaited cleanup. BasePipeline opts in, preserving Ctrl-C behavior. Executor correctness fixes: - A job that fails to launch is recorded in jobs_with_errors and stops new launches, instead of being dropped silently; the catch is narrowed to OSError so genuine bugs still propagate. - A later pipeline stage failing to spawn now terminates the stages that already started (e.g. `sleep 600 | no_such_cmd` no longer leaks sleep past the end of the run). - A proc-sub inner command that fails to spawn fails its own job instead of aborting the whole run mid-flight; teardown paths log (rather than re-raise) launch errors surfaced from cleanup so sibling contexts are still released. - A sub-command killed by SIGPIPE (downstream reader exited early) is no longer treated as a job failure, matching non-pipefail bash. Scheduler/DAG validation: - Jobs that can never be scheduled (threads over budget, or a managed resource request over capacity) raise DagExecutionError up front instead of stalling the run silently. - Adding a second job with an identical pipeline raises instead of silently collapsing onto the first. - DAG.add_job accepts any iterable of dependencies; a list or generator used to pass validation but was silently discarded, leaving the job immediately ready. Jobs may now declare inputs/outputs metadata: declared inputs are verified before a run and declared outputs after (StorageProvider seam, local presence checks), and a failed or interrupted job's declared outputs are deleted so a partial file cannot be mistaken for a good one. DAG.skip_satisfied plus the has_all_outputs predicate can prune already-satisfied jobs before a run, groundwork for a future resume option. The long-dead Job.run() method is removed. Co-Authored-By: Claude Fable 5 --- sentieon_cli/dag.py | 140 ++++-- sentieon_cli/executor.py | 475 +++++++++++++++---- sentieon_cli/job.py | 79 ++-- sentieon_cli/pipeline.py | 9 +- sentieon_cli/scheduler.py | 125 +++-- sentieon_cli/storage.py | 46 ++ tests/integration/test_executor.py | 636 ++++++++++++++++++++++++-- tests/integration/test_signals.py | 217 +++++++++ tests/unit/test_dag.py | 279 +++++++++++ tests/unit/test_dag_construction.py | 7 +- tests/unit/test_job.py | 52 +++ tests/unit/test_scheduler.py | 104 +++++ tests/unit/test_scheduler_contract.py | 109 +++++ tests/unit/test_storage.py | 49 ++ 14 files changed, 2070 insertions(+), 257 deletions(-) create mode 100644 sentieon_cli/storage.py create mode 100644 tests/integration/test_signals.py create mode 100644 tests/unit/test_dag.py create mode 100644 tests/unit/test_job.py create mode 100644 tests/unit/test_scheduler.py create mode 100644 tests/unit/test_scheduler_contract.py create mode 100644 tests/unit/test_storage.py diff --git a/sentieon_cli/dag.py b/sentieon_cli/dag.py index b3997b6..13c3343 100644 --- a/sentieon_cli/dag.py +++ b/sentieon_cli/dag.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import Dict, Generator, List, Optional, Set +from typing import Callable, Dict, Iterable, List, Optional, Set from .exceptions import DagExecutionError from .job import Job @@ -25,54 +25,100 @@ def __init__(self) -> None: self.ready_jobs: Dict[Job, None] = {} self.finished_jobs: List[Job] = [] - def add_job(self, job: Job, dependencies: Optional[Set[Job]] = None): - """Add a job to the DAG""" - if dependencies: - for dependency in dependencies: - if ( - dependency not in self.waiting_jobs - and dependency not in self.ready_jobs - ): - raise ValueError( - f"Dependency '{dependency}' is not in the DAG" - ) - - if isinstance(dependencies, set) and len(dependencies) > 0: - self.waiting_jobs[job] = dependencies.copy() - for dependency in dependencies: + def add_job( + self, job: Job, dependencies: Optional[Iterable[Job]] = None + ) -> None: + """Add a job to the DAG. + + ``dependencies`` may be any iterable of jobs already in the DAG; + it is consumed once and stored as a private set. + """ + deps: Set[Job] = ( + set(dependencies) if dependencies is not None else set() + ) + if job in self.waiting_jobs or job in self.ready_jobs: + raise ValueError( + f"{job} is already in the DAG; jobs are identified by their " + f"shell pipeline, so the same pipeline cannot be added twice." + ) + for dependency in deps: + if ( + dependency not in self.waiting_jobs + and dependency not in self.ready_jobs + ): + raise ValueError( + f"Dependency '{dependency}' is not in the DAG" + ) + + if deps: + self.waiting_jobs[job] = deps + for dependency in deps: downstream = self.planned_jobs.setdefault(dependency, []) downstream.append(job) else: self.ready_jobs[job] = None - def update_dag( - self, - ) -> Generator[Dict[Job, None], Optional[Job], None]: - """Update the DAG with a finished job""" - finished_job = yield self.ready_jobs.copy() - - while True: - logger.debug("Waiting jobs: %s", self.waiting_jobs) - logger.debug("Planned jobs: %s", self.planned_jobs) - logger.debug("Ready jobs: %s", self.ready_jobs) - logger.debug("Finished jobs: %s", self.finished_jobs) - logger.debug("Newly finished: %s", finished_job) - - new_ready_jobs: Dict[Job, None] = {} - if isinstance(finished_job, Job): - if finished_job not in self.ready_jobs: - raise DagExecutionError( - f"Finished job '{finished_job}' was not ready for " - "execution" - ) - - del self.ready_jobs[finished_job] - self.finished_jobs.append(finished_job) - for dependency in self.planned_jobs.get(finished_job, []): - upstream = self.waiting_jobs[dependency] - upstream.remove(finished_job) - if len(upstream) < 1: - self.ready_jobs[dependency] = None - new_ready_jobs[dependency] = None - del self.waiting_jobs[dependency] - finished_job = yield new_ready_jobs + def mark_finished(self, job: Job) -> Dict[Job, None]: + """Record a finished job and return the jobs it unblocked. + + The returned mapping is used as an ordered set (values are ``None``). + Raises :class:`~sentieon_cli.exceptions.DagExecutionError` if ``job`` + was not currently ready. + """ + if job not in self.ready_jobs: + raise DagExecutionError( + f"Finished job '{job}' was not ready for execution" + ) + + logger.debug("Marking finished: %s", job) + del self.ready_jobs[job] + self.finished_jobs.append(job) + + new_ready_jobs: Dict[Job, None] = {} + for dependency in self.planned_jobs.get(job, []): + upstream = self.waiting_jobs[dependency] + upstream.remove(job) + if len(upstream) < 1: + self.ready_jobs[dependency] = None + new_ready_jobs[dependency] = None + del self.waiting_jobs[dependency] + return new_ready_jobs + + def skip_satisfied(self, is_satisfied: Callable[[Job], bool]) -> List[Job]: + """Mark already-satisfied jobs finished before a run (resume). + + Walks the ready frontier: a job is skipped iff ``is_satisfied(job)`` + returns True and every one of its dependencies was itself skipped. + A job behind an unsatisfied ancestor is never offered to the + predicate -- it only becomes ready once all its dependencies are + finished, and during this walk "finished" means "skipped". + + Skipped jobs land in :attr:`finished_jobs`, so an executor never + sees them; unsatisfied jobs are left in place untouched. Call this + before handing the DAG to a scheduler. + + Returns the skipped jobs in topological order. + """ + skipped: List[Job] = [] + frontier: List[Job] = list(self.ready_jobs) + i = 0 + while i < len(frontier): + job = frontier[i] + i += 1 + if is_satisfied(job): + skipped.append(job) + frontier.extend(self.mark_finished(job)) + return skipped + + +def has_all_outputs(job: Job) -> bool: + """The default resume predicate for :meth:`DAG.skip_satisfied`. + + True iff ``job`` declares at least one output and every declared + output exists on the local filesystem. Jobs with no declared outputs + always return False -- they cannot be verified, so they must rerun. + Zero-byte files count as present; a dangling symlink does not + (``Path.exists()`` follows links) -- matching + :class:`~sentieon_cli.storage.LocalStorageProvider` semantics. + """ + return bool(job.outputs) and all(path.exists() for path in job.outputs) diff --git a/sentieon_cli/executor.py b/sentieon_cli/executor.py index 4d60f3e..ad0e39d 100644 --- a/sentieon_cli/executor.py +++ b/sentieon_cli/executor.py @@ -1,30 +1,71 @@ """Execute jobs""" import asyncio -import asyncio.subprocess +import contextlib import signal import sys +import threading import time -from typing import Any, List, Tuple +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Tuple from .job import Job from .logging import get_logger -from .scheduler import ThreadScheduler +from .scheduler import BaseScheduler from .shell_pipeline import Context +from .storage import LocalStorageProvider, StorageProvider logger = get_logger(__name__) -class BaseExecutor: +def _signal_procs(context: Context, sig: int) -> None: + """Send a signal to every live sub-process in a context.""" + for subcommand in context.commands: + proc = subcommand.proc + if proc is not None and proc.returncode is None: + try: + proc.send_signal(sig) + except ProcessLookupError: + pass + + +def _kill_survivors(context: Context) -> None: + """SIGKILL any sub-process that is still running.""" + for subcommand in context.commands: + proc = subcommand.proc + if proc is not None and proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + + +async def _cleanup_quietly(context: Context) -> None: + """Release a context, logging -- not raising -- a surfaced launch failure. + + A proc-sub inner command that fails to spawn surfaces its OSError from + ``context.cleanup()``'s gather. On the abort and shutdown paths the owning + job is already recorded (or the whole run is being torn down), so a stored + OSError must not escape ``execute()``; log it and release the rest. A + non-OSError stays a bug and propagates, matching ``_drive``'s per-job + handling. + """ + try: + await context.cleanup() + except OSError as exc: + logger.error("error releasing resources during teardown: %s", exc) + + +class BaseExecutor(ABC): """Execute jobs""" - def __init__(self, scheduler: ThreadScheduler): + def __init__(self, scheduler: BaseScheduler): self.scheduler = scheduler self.jobs_with_errors: List[Job] = [] + @abstractmethod def execute(self) -> None: """Execute jobs from the DAG""" - raise NotImplementedError class DryRunExecutor(BaseExecutor): @@ -35,8 +76,7 @@ def run_job(self, job: Job) -> None: print(job.shell) def execute(self) -> None: - scheduler_gen = self.scheduler.schedule() - ready_jobs = scheduler_gen.send(None) + ready_jobs = self.scheduler.start() for job in ready_jobs: self.run_job(job) @@ -45,18 +85,134 @@ def execute(self) -> None: ready_jobs = { new_job for completed_job in finished_jobs - for new_job in scheduler_gen.send(completed_job) + for new_job in self.scheduler.job_finished(completed_job) } for job in ready_jobs: self.run_job(job) -class LocalExecutor(BaseExecutor): - """Run jobs locally""" +class AsyncExecutor(BaseExecutor, ABC): + """Base for executors that run an asyncio event loop. + + Provides the ``execute()`` entry point and opt-in SIGINT/SIGTERM handling + with graceful shutdown; subclasses implement :meth:`_drive` (the run loop) + and :meth:`_shutdown` (teardown after an interrupt). - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) + Set ``install_signal_handlers=True`` to install handlers (on the main + thread only) for the duration of a run; the previous handlers are + restored afterwards, so a caller's signal state is left untouched. + """ + + def __init__( + self, + scheduler: BaseScheduler, + *, + install_signal_handlers: bool = False, + ) -> None: + super().__init__(scheduler) + self.install_signal_handlers = install_signal_handlers self.start_new_jobs = True + + def _install_signal_handlers( + self, + loop: asyncio.AbstractEventLoop, + stop_event: asyncio.Event, + ) -> Callable[[], None]: + """Install SIGINT/SIGTERM handlers on the running loop. + + Returns a callable that removes the handlers and restores the + previous ones. A no-op is returned when handlers are disabled, or + when we are not on the main thread (Python can only manage signals + there, so we skip rather than crash). + """ + if not self.install_signal_handlers: + return lambda: None + if threading.current_thread() is not threading.main_thread(): + logger.debug("Signal handlers require the main thread; skipping.") + return lambda: None + + def on_signal(signum: int) -> None: + logger.error( + "Received %s; terminating jobs.", + signal.Signals(signum).name, + ) + self.start_new_jobs = False + stop_event.set() + + signums = (signal.SIGINT, signal.SIGTERM) + previous: Dict[int, Any] = { + signum: signal.getsignal(signum) for signum in signums + } + installed: List[int] = [] + for signum in signums: + try: + loop.add_signal_handler(signum, on_signal, signum) + installed.append(signum) + except (NotImplementedError, RuntimeError, ValueError) as exc: + logger.debug( + "Could not install handler for %s: %s", signum, exc + ) + + def restore() -> None: + for signum in installed: + try: + loop.remove_signal_handler(signum) + except (RuntimeError, ValueError): + pass + handler = previous[signum] + if handler is not None: + try: + signal.signal(signum, handler) + except (OSError, ValueError, TypeError): + pass + + return restore + + def execute(self) -> None: + """Execute jobs from the DAG""" + asyncio.run(self._execute()) + + async def _execute(self) -> None: + """Execute jobs from the DAG""" + self.jobs_with_errors = [] + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + restore = self._install_signal_handlers(loop, stop_event) + try: + await self._drive(stop_event) + if stop_event.is_set(): + await self._shutdown() + finally: + restore() + + @abstractmethod + async def _drive(self, stop_event: asyncio.Event) -> None: + """Submit/run jobs until the DAG drains or a stop is requested.""" + + @abstractmethod + async def _shutdown(self) -> None: + """Tear down whatever is still running after a stop request.""" + + +class LocalExecutor(AsyncExecutor): + """Run jobs locally as async subprocesses.""" + + def __init__( + self, + scheduler: BaseScheduler, + *, + install_signal_handlers: bool = False, + shutdown_grace_period: float = 10.0, + storage: Optional[StorageProvider] = None, + ) -> None: + super().__init__( + scheduler, + install_signal_handlers=install_signal_handlers, + ) + self.shutdown_grace_period = shutdown_grace_period + self.storage: StorageProvider = ( + storage if storage is not None else LocalStorageProvider() + ) self.running: List[ Tuple[ Job, @@ -65,26 +221,6 @@ def __init__(self, *args: Any, **kwargs: Any): int, ] ] = [] - signal.signal(signal.SIGINT, self.exit_gracefully) - signal.signal(signal.SIGTERM, self.exit_gracefully) - - def exit_gracefully(self, signum, frame) -> None: - logger.error("Termination signal detected. Terminating jobs.") - self.start_new_jobs = False - for running_job in self.running: - for subcommand in running_job[1].commands: - if not subcommand.proc: - continue - subcommand.proc.send_signal(signal=signal.SIGTERM) - time.sleep(10) - for running_job in self.running: - context = running_job[1] - for subcommand in context.commands: - if not subcommand.proc: - continue - if subcommand.proc.returncode is None: - subcommand.proc.kill() - asyncio.run(context.cleanup()) async def run_job(self, job: Job) -> None: """Run a job""" @@ -93,6 +229,8 @@ async def run_job(self, job: Job) -> None: context = Context() start_time = time.monotonic_ns() try: + for path in job.inputs: + await self.storage.stage_in(path) proc = await cmd.run( context, stderr=sys.stderr, @@ -105,85 +243,224 @@ async def run_job(self, job: Job) -> None: start_time, ) ) - except Exception as e: + except OSError as e: + # OSError covers the expected launch failures (command not found, + # permission denied, pipe/FIFO errors). Anything else is a bug in + # the caller's pipeline and is left to propagate. logger.error("failed to start command: %s", job.shell) logger.error("Error: %s", str(e)) - await context.cleanup() + self.jobs_with_errors.append(job) + self.start_new_jobs = False + # A later pipeline stage can fail to spawn after earlier stages + # are already running; terminate them so they do not outlive the + # workflow, then release the remaining resources. A proc-sub inner + # launch failure can also surface from cleanup() here (narrow + # race): the job is already recorded, so swallow it. + await self._terminate_context(context) + await _cleanup_quietly(context) + self._remove_outputs(job) - def execute(self) -> None: - """Execute jobs from the DAG""" - asyncio.run(self._execute()) + async def _terminate_context(self, context: Context) -> None: + """Tear down processes already spawned in an aborted context. - async def _execute(self) -> None: - """Execute jobs from the DAG""" - self.jobs_with_errors: List[Job] = [] - scheduler_gen = self.scheduler.schedule() - ready_jobs = scheduler_gen.send(None) + Used when a job aborts mid-launch (a later pipeline stage failed to + spawn): earlier stages are already running and would otherwise be + leaked. SIGTERM them, allow a grace period, then SIGKILL any + survivors, waiting for each to be reaped. + """ + live = [ + subcommand.proc + for subcommand in context.commands + if subcommand.proc is not None + and subcommand.proc.returncode is None + ] + if not live: + return + _signal_procs(context, signal.SIGTERM) + waits = [asyncio.create_task(proc.wait()) for proc in live] + await asyncio.wait(waits, timeout=self.shutdown_grace_period) + _kill_survivors(context) + await asyncio.wait(waits) + + def _remove_outputs(self, job: Job) -> None: + """Delete a failed job's declared outputs. + + A partial or unverified output must not survive a failure: resume + (:meth:`~sentieon_cli.dag.DAG.skip_satisfied`) treats an existing + declared output as proof that its producer succeeded. Directories are + deliberately not removed (never recurse into user paths); a warning + is logged instead. + """ + for path in job.outputs: + try: + path.unlink() + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "could not remove output %s of failed job %s: %s", + path, + job, + exc, + ) + else: + logger.info("Removed output of failed job %s: %s", job, path) + + async def _drive(self, stop_event: asyncio.Event) -> None: + """Schedule and run jobs until the DAG drains or we are stopped.""" + ready_jobs = self.scheduler.start() for job in ready_jobs: + if not self.start_new_jobs: + # A job in this batch failed to launch; drain, don't start + # the rest -- matching the policy the main loop applies. + break await self.run_job(job) - while self.running: - done, _running = await asyncio.wait( - [job[2] for job in self.running], - return_when=asyncio.FIRST_COMPLETED, - ) + stop_task = asyncio.ensure_future(stop_event.wait()) + try: + while self.running and not stop_event.is_set(): + wait_on: List[Any] = [job[2] for job in self.running] + wait_on.append(stop_task) + done, _pending = await asyncio.wait( + wait_on, + return_when=asyncio.FIRST_COMPLETED, + ) - finished_jobs = [ - self.running.pop(i) - for i in reversed(range(len(self.running))) - if self.running[i][2] in done - ] + if stop_event.is_set(): + break - # Check job execution - for job, context, _task, start_time in finished_jobs: - end_time = time.monotonic_ns() - total_seconds = (end_time - start_time) / 1e9 - - # Check if the command failed - cmd_failed = False - for subcommand in context.commands: - if not subcommand.proc: - logger.error( - "Sub-command has no process: %s", - subcommand, - ) - cmd_failed = True - continue - ret = ( - await subcommand.proc.wait() - ) # Wait on all sub-commands - if ret != 0 and not subcommand.fail_ok: - logger.error( - f"Return code {ret} for sub-command: {subcommand}" - ) - cmd_failed = True - if ret == -9: + finished_jobs = [ + self.running.pop(i) + for i in reversed(range(len(self.running))) + if self.running[i][2] in done + ] + + # Check job execution + for job, context, _task, start_time in finished_jobs: + end_time = time.monotonic_ns() + total_seconds = (end_time - start_time) / 1e9 + + # Check if the command failed + cmd_failed = False + for subcommand in context.commands: + if not subcommand.proc: + logger.error( + "Sub-command has no process: %s", + subcommand, + ) + cmd_failed = True + continue + ret = ( + await subcommand.proc.wait() + ) # Wait on all sub-commands + # A subcommand killed by SIGPIPE is not a failure: + # it was writing to a pipe whose reader exited early + # (e.g. `... | head`), which is normal in default + # (non-pipefail) bash. Real crashes surface as other + # non-zero codes and still fail the job. + if ret == -signal.SIGPIPE: + logger.debug( + "Sub-command exited on SIGPIPE (ok): %s", + subcommand, + ) + elif ret != 0 and not subcommand.fail_ok: logger.error( - "Sub-command received SIGKILL. " - "Possible out-of-memory?" + f"Return code {ret} for sub-command: " + f"{subcommand}" ) + cmd_failed = True + if ret == -9: + logger.error( + "Sub-command received SIGKILL. " + "Possible out-of-memory?" + ) + + if not cmd_failed: + for path in job.outputs: + try: + await self.storage.stage_out(path) + except OSError as exc: + logger.error( + "missing output %s for %s: %s", + path, + job.shell, + exc, + ) + cmd_failed = True + + # Release resources before recording the verdict. A + # proc-sub inner command that failed to launch surfaces + # its OSError from the background task here; treat it as + # a launch failure for this job rather than letting it + # abort the run while siblings are still mid-flight. + try: + await context.cleanup() + except OSError as exc: + logger.error("failed to start command: %s", job.shell) + logger.error("Error: %s", str(exc)) + cmd_failed = True + + if cmd_failed: + logger.error("Command failure: %s", job.shell) + self.jobs_with_errors.append(job) + self.start_new_jobs = False + self._remove_outputs(job) + else: + logger.info( + f"Finished command in " + f"{total_seconds:.2f}: {job.shell}" + ) + + if not self.start_new_jobs: + # Don't start new jobs + continue - if cmd_failed: - logger.error("Command failure: %s", job.shell) - self.jobs_with_errors.append(job) - self.start_new_jobs = False - else: - logger.info( - f"Finished command in " - f"{total_seconds:.2f}: {job.shell}" + ready_jobs = { + new_job + for completed_job in finished_jobs + for new_job in self.scheduler.job_finished( + completed_job[0] ) - await context.cleanup() + } - if not self.start_new_jobs: - # Don't start new jobs - continue + # Run the ready jobs + for job in ready_jobs: + if not self.start_new_jobs: + # A job in this batch failed to launch; drain the + # rest of the run without starting more. + break + await self.run_job(job) + finally: + stop_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await stop_task - ready_jobs = { - new_job - for completed_job in finished_jobs - for new_job in scheduler_gen.send(completed_job[0]) - } + async def _shutdown(self) -> None: + """Terminate running jobs, escalate to SIGKILL, then clean up.""" + running = self.running + self.running = [] - # Run the ready jobs - for job in ready_jobs: - await self.run_job(job) + for _job, context, _task, _start in running: + _signal_procs(context, signal.SIGTERM) + + tasks = [tup[2] for tup in running] + if tasks: + await asyncio.wait(tasks, timeout=self.shutdown_grace_period) + + for _job, context, _task, _start in running: + _kill_survivors(context) + if tasks: + await asyncio.wait(tasks) + + for _job, context, _task, _start in running: + # A proc-sub inner launch failure can surface from cleanup() as we + # tear down after an interrupt; log it rather than let it escape, + # and keep releasing the remaining contexts. + await _cleanup_quietly(context) + + # Interrupted jobs never had their outputs verified; a partial file + # must not survive to fool a later resume. Jobs that completed in the + # same wake-up as the stop signal are conservatively included -- they + # were never verified either. + for job, _context, _task, _start in running: + self._remove_outputs(job) diff --git a/sentieon_cli/job.py b/sentieon_cli/job.py index dea8343..9d7d4d2 100644 --- a/sentieon_cli/job.py +++ b/sentieon_cli/job.py @@ -2,19 +2,29 @@ Job objects """ -import asyncio -import sys -import time -from typing import Dict, Optional +import pathlib +from typing import Dict, Iterable, List, Optional -from .logging import get_logger -from .shell_pipeline import Context, Pipeline - -logger = get_logger(__name__) +from .shell_pipeline import Pipeline class Job: - """A job for execution""" + """A unit of work: a shell pipeline plus its execution metadata. + + Fields: + + * ``shell`` -- the pipeline to run. + * ``name`` -- a human-readable label (not part of identity). + * ``threads`` -- CPU threads the job needs (a local scheduling budget). + * ``resources`` -- named resource counts (e.g. NUMA-node tokens). + * ``inputs`` / ``outputs`` -- local files the job reads / writes; when + declared, the executor verifies them around the run and removes the + outputs of a failed attempt. + * ``image`` -- container image (or environment) to run in on a backend. + + A job's identity is keyed only on ``shell``: two jobs with the same + pipeline are equal (and collide in a DAG) regardless of the other fields. + """ def __init__( self, @@ -22,53 +32,36 @@ def __init__( name: str, threads: int = 1, resources: Optional[Dict[str, int]] = None, - ): + *, + inputs: Optional[Iterable[pathlib.Path]] = None, + outputs: Optional[Iterable[pathlib.Path]] = None, + image: Optional[str] = None, + ) -> None: self.shell = pipeline self.name = name self.threads = threads self.resources = {} if resources is None else resources + self.inputs: List[pathlib.Path] = ( + [] if inputs is None else list(inputs) + ) + self.outputs: List[pathlib.Path] = ( + [] if outputs is None else list(outputs) + ) + self.image = image - def __hash__(self): + def __hash__(self) -> int: return hash(self.shell) - def __eq__(self, other: object): + def __eq__(self, other: object) -> bool: if isinstance(other, Job): return self.shell == other.shell return False - def __ne__(self, other: object): + def __ne__(self, other: object) -> bool: return not self == other - def __repr__(self): + def __repr__(self) -> str: return f"Job({self.name})" - def __str__(self): + def __str__(self) -> str: return f"Job({self.name})" - - def run(self, dry_run: bool = False): - """Run a command""" - if dry_run: - print(self.shell) - return - - logger.info("running command: %s", self.shell) - t0 = time.time() - context = Context() - try: - asyncio.run( - self.shell.run(context, stdout=sys.stdout, stderr=sys.stderr) - ) - for subcommand in context.commands: - if not subcommand.proc: - logger.error("subcommand has no process: %s", subcommand) - continue - ret = asyncio.run(subcommand.proc.wait()) - if ret != 0 and not subcommand.fail_ok: - logger.error( - f"subcommand failed with code {ret}: {subcommand}" - ) - except Exception as e: - logger.error(f"Failed to run command: {e}") - finally: - asyncio.run(context.cleanup()) - logger.info("finished in: %s seconds", f"{time.time() - t0:.1f}") diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 39672c1..ddb67e1 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -250,8 +250,13 @@ def run(self, dag: DAG) -> BaseExecutor: ) self.logger.debug("Creating the executor") - Executor = DryRunExecutor if self.dry_run else LocalExecutor - executor = Executor(scheduler) + executor: BaseExecutor + if self.dry_run: + executor = DryRunExecutor(scheduler) + else: + # Handle Ctrl-C/SIGTERM by terminating running jobs gracefully; + # the handlers are installed only for the duration of the run. + executor = LocalExecutor(scheduler, install_signal_handlers=True) self.logger.info("Starting execution") executor.execute() diff --git a/sentieon_cli/scheduler.py b/sentieon_cli/scheduler.py index ffc0bb8..4ed5329 100644 --- a/sentieon_cli/scheduler.py +++ b/sentieon_cli/scheduler.py @@ -1,15 +1,48 @@ """Schedule jobs""" -from typing import Dict, Generator, Optional, Set +from abc import ABC, abstractmethod +from typing import Dict, Optional, Set from .dag import DAG +from .exceptions import DagExecutionError from .job import Job from .logging import get_logger logger = get_logger(__name__) -class ThreadScheduler: +class BaseScheduler(ABC): + """Selects which ready jobs an executor may run, subject to a policy. + + An executor drives a scheduler with two calls: + + 1. :meth:`start` once, to get the first batch of jobs to run. + 2. :meth:`job_finished` for each job as it completes, to get the jobs + that its completion unblocked (possibly an empty set). + + Contract / invariants: + + * :meth:`start` is called exactly once, before any :meth:`job_finished`. + * Every job passed to :meth:`job_finished` was previously returned by + :meth:`start` or :meth:`job_finished`, and is reported at most once. + * A scheduler instance drives a single run; do not reuse it. + * :meth:`start` may raise + :class:`~sentieon_cli.exceptions.DagExecutionError` if the DAG + contains a job that can never be scheduled. + * The executor stops when it has no running jobs and the scheduler + returns no new ones. + """ + + @abstractmethod + def start(self) -> Set[Job]: + """Return the first batch of jobs to run.""" + + @abstractmethod + def job_finished(self, job: Job) -> Set[Job]: + """Record a finished job and return the jobs it unblocked.""" + + +class ThreadScheduler(BaseScheduler): """Schedule jobs as threads are available""" def __init__( @@ -17,41 +50,65 @@ def __init__( dag: DAG, threads: int = 1, resources: Optional[Dict[str, int]] = None, - ): + ) -> None: self.dag = dag self.threads = threads self.available_threads = threads self.resources = {} if resources is None else resources + # Jobs that are ready to run but have not yet been scheduled. + self._ready: Dict[Job, None] = {} + + def _check_feasible(self) -> None: + """Ensure every job can eventually be scheduled. + + A job that requests more threads than the total budget, or more of a + managed resource than exists, can never be released and would + otherwise stall the run silently. + """ + jobs = list(self.dag.waiting_jobs) + list(self.dag.ready_jobs) + for job in jobs: + if job.threads > self.threads: + raise DagExecutionError( + f"{job} requests {job.threads} threads but the scheduler " + f"budget is only {self.threads}" + ) + for resource, needed in job.resources.items(): + capacity = self.resources.get(resource) + if capacity is not None and needed > capacity: + raise DagExecutionError( + f"{job} requests {needed} of resource '{resource}' " + f"but the scheduler capacity is only {capacity}" + ) + + def _fit(self) -> Set[Job]: + """Reserve budget for as many ready jobs as fit; return that set.""" + logger.debug("Ready pool: %s", self._ready) - def schedule(self) -> Generator[Set[Job], Optional[Job], None]: - """Schedule a job for execution""" - dag_gen = self.dag.update_dag() - ready_jobs = dag_gen.send(None) - - while True: - logger.debug("Ready jobs: %s", ready_jobs) - - scheduled_jobs: Set[Job] = set() - for ready_job in ready_jobs: - if self.available_threads - ready_job.threads >= 0 and all( - [ - available - ready_job.resources.get(resource, 0) >= 0 - for resource, available in self.resources.items() - ] - ): - self.available_threads -= ready_job.threads - for resource, used in ready_job.resources.items(): - if resource in self.resources: - self.resources[resource] -= used - scheduled_jobs.add(ready_job) - - for job in scheduled_jobs: - del ready_jobs[job] - - finished_job = yield scheduled_jobs - if isinstance(finished_job, Job): - self.available_threads += finished_job.threads - for resource, used in finished_job.resources.items(): + scheduled_jobs: Set[Job] = set() + for ready_job in self._ready: + if self.available_threads - ready_job.threads >= 0 and all( + available - ready_job.resources.get(resource, 0) >= 0 + for resource, available in self.resources.items() + ): + self.available_threads -= ready_job.threads + for resource, used in ready_job.resources.items(): if resource in self.resources: - self.resources[resource] += used - ready_jobs.update(dag_gen.send(finished_job)) + self.resources[resource] -= used + scheduled_jobs.add(ready_job) + + for job in scheduled_jobs: + del self._ready[job] + return scheduled_jobs + + def start(self) -> Set[Job]: + self._check_feasible() + self._ready = dict(self.dag.ready_jobs) + return self._fit() + + def job_finished(self, job: Job) -> Set[Job]: + self.available_threads += job.threads + for resource, used in job.resources.items(): + if resource in self.resources: + self.resources[resource] += used + self._ready.update(self.dag.mark_finished(job)) + return self._fit() diff --git a/sentieon_cli/storage.py b/sentieon_cli/storage.py new file mode 100644 index 0000000..5fb6263 --- /dev/null +++ b/sentieon_cli/storage.py @@ -0,0 +1,46 @@ +"""Storage staging for job inputs and outputs.""" + +import pathlib +from abc import ABC, abstractmethod + + +class StorageProvider(ABC): + """Stage a job's declared files in before it runs and out after. + + On a shared filesystem the files are already local, so staging + degenerates to verifying presence (see :class:`LocalStorageProvider`). + A cloud provider would instead pull inputs from and push outputs to + object storage, deriving each object's URI from a configured prefix and + the local path. The methods are async so they compose with the async + executors. + """ + + @abstractmethod + async def stage_in(self, local: pathlib.Path) -> None: + """Make ``local`` available before a job runs. + + Pull it from storage, or verify it is present. Raise if it cannot be + made available. + """ + + @abstractmethod + async def stage_out(self, local: pathlib.Path) -> None: + """Persist ``local`` after a job runs. + + Push it to storage, or verify it was produced. Raise if it is missing. + """ + + +class LocalStorageProvider(StorageProvider): + """Shared-filesystem provider: no data movement, just verify presence. + + A file counts as present when it exists, even if it is empty (zero bytes). + """ + + async def stage_in(self, local: pathlib.Path) -> None: + if not local.exists(): + raise FileNotFoundError(f"missing input: {local}") + + async def stage_out(self, local: pathlib.Path) -> None: + if not local.exists(): + raise FileNotFoundError(f"missing output: {local}") diff --git a/tests/integration/test_executor.py b/tests/integration/test_executor.py index 7199ecc..4f0a7f4 100644 --- a/tests/integration/test_executor.py +++ b/tests/integration/test_executor.py @@ -1,13 +1,13 @@ """ -Integration tests for the Executor +Integration tests for the executors. """ -import io +import asyncio import os import pathlib import sys import tempfile -from unittest.mock import patch +import threading import pytest @@ -16,8 +16,13 @@ 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) ) -from sentieon_cli.dag import DAG # noqa: E402 -from sentieon_cli.executor import LocalExecutor # noqa: E402 +from sentieon_cli.dag import DAG, has_all_outputs # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.executor import ( # noqa: E402 + DryRunExecutor, + LocalExecutor, + _cleanup_quietly, +) from sentieon_cli.job import Job # noqa: E402 from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 from sentieon_cli.shell_pipeline import ( # noqa: E402 @@ -30,64 +35,64 @@ def test_local_executor_simple_job(): """Test LocalExecutor with a single, simple job""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() - job = Job(Pipeline(Command("echo", "hello executor"), file_output=cmd_out), "echo-job") + job = Job( + Pipeline(Command("echo", "hello executor"), file_output=cmd_out), + "echo-job", + ) dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() - cmd_out_str = open(cmd_out).read() - assert "hello executor" in cmd_out_str + assert "hello executor" in cmd_out.read_text() assert len(executor.jobs_with_errors) == 0 def test_local_executor_pipeline_job(): """Test LocalExecutor with a job containing a pipeline""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() - pipeline = Pipeline(Command("echo", "hello pipeline"), Command("cat"), file_output=cmd_out) + pipeline = Pipeline( + Command("echo", "hello pipeline"), + Command("cat"), + file_output=cmd_out, + ) job = Job(pipeline, "pipeline-job") dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() - cmd_out_str = open(cmd_out).read() - assert "hello pipeline" in cmd_out_str + assert "hello pipeline" in cmd_out.read_text() assert len(executor.jobs_with_errors) == 0 + def test_local_executor_failing_job(): """Test LocalExecutor with a job that fails""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_in = tmp_dir / "test_in.txt" + cmd_in = pathlib.Path(tmp_dir_str) / "test_in.txt" dag = DAG() - # This command will fail + # This command will fail: the input file does not exist job = Job(Pipeline(Command("cat", str(cmd_in))), "failing-job") dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() assert len(executor.jobs_with_errors) == 1 assert executor.jobs_with_errors[0] == job + def test_local_executor_proc_sub_job(): """Test LocalExecutor with a job using process substitution""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() command = Command( "diff", @@ -100,18 +105,593 @@ def test_local_executor_proc_sub_job(): # Need at least 2 threads for the two proc subs scheduler = ThreadScheduler(dag, 2) executor = LocalExecutor(scheduler) - executor.execute() # diff will write to stdout and fail - output = open(cmd_out).read() + output = cmd_out.read_text() assert "1c1" in output assert "< a" in output assert "> b" in output - # The job should be marked as an error because the diff command fails + # The job is marked as an error because the diff command fails assert len(executor.jobs_with_errors) == 1 -if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file +def test_dry_run_executor_prints_without_running(capsys): + """DryRunExecutor prints the shell but does not execute it""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + cmd_out = pathlib.Path(tmp_dir_str) / "out.txt" + dag = DAG() + job = Job( + Pipeline(Command("echo", "dryrun"), file_output=cmd_out), + "dry-job", + ) + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = DryRunExecutor(scheduler) + executor.execute() + + printed = capsys.readouterr().out + assert "echo" in printed + assert "dryrun" in printed + assert not cmd_out.exists() + assert len(executor.jobs_with_errors) == 0 + + +def test_dry_run_executor_drains_dependencies(capsys): + """DryRunExecutor walks the whole DAG, including dependent jobs""" + dag = DAG() + a = Job(Pipeline(Command("echo", "a")), "a") + b = Job(Pipeline(Command("echo", "b")), "b") + dag.add_job(a) + dag.add_job(b, {a}) + + scheduler = ThreadScheduler(dag, 4) + executor = DryRunExecutor(scheduler) + executor.execute() + + printed = capsys.readouterr().out + assert "echo a" in printed + assert "echo b" in printed + + +def test_fail_ok_subcommand_does_not_fail_job(): + """A failing but fail_ok sub-command does not mark the job as errored""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + missing = tmp_dir / "missing.txt" + out = tmp_dir / "out.txt" + + # `cat missing | wc -c`; cat fails (no file) but is tolerated. + dag = DAG() + job = Job( + Pipeline( + Command("cat", str(missing), fail_ok=True), + Command("wc", "-c"), + file_output=out, + ), + "tolerant-job", + ) + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + executor.execute() + + assert len(executor.jobs_with_errors) == 0 + + +def test_independent_jobs_all_complete(): + """Two independent jobs both run and produce output""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + o1 = tmp_dir / "o1.txt" + o2 = tmp_dir / "o2.txt" + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "1"), file_output=o1), "j1")) + dag.add_job(Job(Pipeline(Command("echo", "2"), file_output=o2), "j2")) + + scheduler = ThreadScheduler(dag, 2) + executor = LocalExecutor(scheduler) + executor.execute() + + assert o1.read_text().strip() == "1" + assert o2.read_text().strip() == "2" + assert len(executor.jobs_with_errors) == 0 + + +def test_dependent_job_runs_after_dependency(): + """A dependent job runs and both jobs complete successfully""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + first = tmp_dir / "first.txt" + second = tmp_dir / "second.txt" + dag = DAG() + a = Job(Pipeline(Command("echo", "a"), file_output=first), "a") + b = Job(Pipeline(Command("echo", "b"), file_output=second), "b") + dag.add_job(a) + dag.add_job(b, {a}) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + executor.execute() + + assert first.read_text().strip() == "a" + assert second.read_text().strip() == "b" + assert len(executor.jobs_with_errors) == 0 + + +def test_job_that_fails_to_start_is_recorded(): + """A job whose command cannot be spawned is recorded as an error.""" + dag = DAG() + job = Job(Pipeline(Command("no_such_cmd_zzz")), "ghost") + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + executor.execute() + assert job in executor.jobs_with_errors + + +def test_launch_loop_stops_after_a_failed_launch(): + """Once a job in a ready batch fails to launch, the loop stops starting + the rest of the batch instead of launching all of it.""" + dag = DAG() + for i in range(5): + # Distinct args -> distinct pipeline identities (else the DAG + # rejects them as duplicates). + dag.add_job(Job(Pipeline(Command("true", str(i))), f"j{i}")) + executor = LocalExecutor(ThreadScheduler(dag, 5)) + + launched = [] + + async def fake_run_job(job): + launched.append(job) + executor.start_new_jobs = False # simulate a launch failure + + executor.run_job = fake_run_job + executor.execute() + + # The guard stopped the loop after the first job rather than starting + # all five. + assert len(launched) == 1 + + +def test_infeasible_job_raises_instead_of_deadlocking(): + """A job larger than the budget raises rather than stalling silently.""" + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + LocalExecutor(ThreadScheduler(dag, 2)).execute() + + +def test_dry_run_also_rejects_infeasible_job(): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + DryRunExecutor(ThreadScheduler(dag, 2)).execute() + + +def test_non_oserror_on_start_propagates(monkeypatch): + """A non-OSError while starting a job is a bug and must propagate.""" + + async def boom(self, *args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr("sentieon_cli.shell_pipeline.Pipeline.run", boom) + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "j")) + with pytest.raises(RuntimeError): + LocalExecutor(ThreadScheduler(dag, 1)).execute() + + +def test_missing_input_fails_before_run(tmp_path): + """A declared input that is missing fails the job before it runs.""" + missing = tmp_path / "in.txt" + marker = tmp_path / "ran.txt" + dag = DAG() + job = Job( + Pipeline(Command("touch", str(marker))), + "j", + inputs=[missing], + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert not marker.exists() # the command never ran + + +def test_present_inputs_and_outputs_allow_success(tmp_path): + """Empty (zero-byte) declared inputs/outputs still count as present.""" + inp = tmp_path / "in.txt" + inp.touch() # empty input file + out = tmp_path / "out.txt" + dag = DAG() + job = Job( + Pipeline(Command("touch", str(out))), # produces an empty output + "j", + inputs=[inp], + outputs=[out], + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert executor.jobs_with_errors == [] + assert out.exists() + + +def test_missing_output_fails_after_run(tmp_path): + """A declared output the command does not produce fails the job.""" + missing = tmp_path / "never.txt" + dag = DAG() + job = Job(Pipeline(Command("true")), "j", outputs=[missing]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + + +def test_failed_job_outputs_are_removed(tmp_path): + """A failed job's declared outputs are deleted from disk.""" + out = tmp_path / "out.txt" + out.write_text("stale") + dag = DAG() + job = Job(Pipeline(Command("false")), "j", outputs=[out]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert not out.exists() + + +def test_missing_output_failure_removes_produced_siblings(tmp_path): + """A missing declared output deletes the outputs that were produced.""" + produced = tmp_path / "out1.txt" + never = tmp_path / "out2.txt" + dag = DAG() + job = Job( + Pipeline(Command("touch", str(produced))), + "j", + outputs=[produced, never], + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert not produced.exists() + + +def test_launch_failure_removes_outputs(tmp_path): + """A job that fails to launch has its declared outputs deleted.""" + out = tmp_path / "out.txt" + out.write_text("stale") + dag = DAG() + job = Job(Pipeline(Command("no_such_cmd_zzz")), "j", outputs=[out]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert not out.exists() + + +def test_missing_input_failure_removes_outputs(tmp_path): + """Any failed attempt clears the job's declared outputs -- even a + missing-input failure where the command never ran.""" + missing = tmp_path / "in.txt" + out = tmp_path / "out.txt" + out.write_text("stale") + dag = DAG() + job = Job(Pipeline(Command("true")), "j", inputs=[missing], outputs=[out]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert not out.exists() + + +def test_successful_job_keeps_outputs(tmp_path): + """A successful job's outputs are never deleted.""" + out = tmp_path / "out.txt" + dag = DAG() + job = Job(Pipeline(Command("touch", str(out))), "j", outputs=[out]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert executor.jobs_with_errors == [] + assert out.exists() + + +def test_failed_job_directory_output_is_left_in_place(tmp_path): + """Directories declared as outputs are never removed on failure.""" + out_dir = tmp_path / "out_dir" + out_dir.mkdir() + (out_dir / "keep.txt").write_text("keep") + dag = DAG() + job = Job(Pipeline(Command("false")), "j", outputs=[out_dir]) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert job in executor.jobs_with_errors + assert out_dir.is_dir() + assert (out_dir / "keep.txt").read_text() == "keep" + + +def test_local_executor_empty_dag_completes(): + """An empty DAG (e.g. everything skipped by resume) runs cleanly.""" + executor = LocalExecutor(ThreadScheduler(DAG(), 1)) + executor.execute() + assert executor.jobs_with_errors == [] + + +def test_dry_run_empty_dag_completes(capsys): + """DryRunExecutor handles an empty DAG without printing anything.""" + executor = DryRunExecutor(ThreadScheduler(DAG(), 1)) + executor.execute() + assert capsys.readouterr().out == "" + assert executor.jobs_with_errors == [] + + +def test_resume_after_failure_reruns_only_failed_subtree(tmp_path): + """After a partial failure, skip_satisfied + has_all_outputs resumes + the run from the surviving intermediate files.""" + a_out = tmp_path / "a.txt" + b_out = tmp_path / "b.txt" + + def build_dag(b_cmd): + dag = DAG() + a = Job( + Pipeline(Command("echo", "a"), file_output=a_out), + "a", + outputs=[a_out], + ) + b = Job(Pipeline(b_cmd), "b", outputs=[b_out]) + dag.add_job(a) + dag.add_job(b, {a}) + return dag, a, b + + # First run: a succeeds, b fails (and b's partial output is deleted). + b_out.write_text("partial") + dag, _a, b = build_dag(Command("false")) + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert b in executor.jobs_with_errors + assert a_out.exists() + assert not b_out.exists() + + # Second run: rebuild the DAG with b fixed; resume skips a. + dag, a, b = build_dag(Command("touch", str(b_out))) + skipped = dag.skip_satisfied(has_all_outputs) + assert skipped == [a] + executor = LocalExecutor(ThreadScheduler(dag, 1)) + executor.execute() + assert executor.jobs_with_errors == [] + assert b_out.exists() + assert a_out.read_text().strip() == "a" # a was not rerun over + + +def _execute_bounded(executor, timeout=15): + """Run executor.execute() with a hang guard.""" + holder = {} + + def run(): + try: + executor.execute() + except BaseException as exc: # pragma: no cover - regression guard + holder["exc"] = exc + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout=timeout) + assert not thread.is_alive(), "executor.execute() hung" + assert "exc" not in holder, holder.get("exc") + + +def test_procsub_job_with_early_exiting_outer_does_not_hang(): + """An outer command that exits without opening its proc-sub FIFO must + not hang the executor (regression: cleanup deadlock).""" + dag = DAG() + job = Job( + Pipeline( + Command("false", InputProcSub(Pipeline(Command("echo", "x")))) + ), + "early-exit", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_job_with_unspawnable_outer_does_not_hang(): + """A missing outer executable with a proc-sub argument must not hang + the executor's error path (regression: cleanup deadlock).""" + dag = DAG() + job = Job( + Pipeline( + Command( + "no_such_cmd_zzz", + InputProcSub(Pipeline(Command("echo", "x"))), + ) + ), + "ghost-with-procsub", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_inner_that_fails_to_launch_is_recorded(): + """An inner proc-sub command that cannot be spawned fails its job + instead of crashing the executor (regression: a raw FileNotFoundError + from the background task propagated out of execute()).""" + dag = DAG() + job = Job( + Pipeline( + Command( + "cat", + InputProcSub(Pipeline(Command("no_such_cmd_zzz"))), + ) + ), + "inner-fail", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_inner_failure_does_not_skip_sibling_bookkeeping(): + """A job whose inner proc-sub fails to launch must not abort the run + for independent sibling jobs (regression: the exception escaped mid- + flight and skipped other jobs' bookkeeping).""" + dag = DAG() + bad = Job( + Pipeline( + Command( + "cat", + InputProcSub(Pipeline(Command("no_such_cmd_zzz"))), + ) + ), + "inner-fail", + ) + good = Job(Pipeline(Command("true")), "sibling") + dag.add_job(bad) + dag.add_job(good) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert bad in executor.jobs_with_errors + # The sibling ran to completion and was recorded as a success, not + # silently dropped when the bad job's failure surfaced. + assert good not in executor.jobs_with_errors + + +def test_mid_pipeline_spawn_failure_terminates_running_stages(monkeypatch): + """When a later pipeline stage fails to spawn, earlier stages that are + already running must be terminated, not leaked (regression: a + Pipeline(sleep 600, no_such_cmd) left sleep alive past the workflow).""" + import sentieon_cli.executor as executor_mod + + contexts = [] + real_context = executor_mod.Context + + class RecordingContext(real_context): # type: ignore[valid-type,misc] + def __init__(self) -> None: + super().__init__() + contexts.append(self) + + monkeypatch.setattr(executor_mod, "Context", RecordingContext) + + dag = DAG() + job = Job( + Pipeline( + Command("sleep", "600"), + Command("no_such_cmd_zzz"), + ), + "leak", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + + assert job in executor.jobs_with_errors + # The first stage spawned; every spawned proc must be reaped, not left + # running to outlive the workflow. + spawned = [sub.proc for c in contexts for sub in c.commands if sub.proc] + assert spawned, "expected the first stage to have spawned" + for proc in spawned: + assert proc.returncode is not None + + +def test_sigpipe_producer_does_not_fail_job(tmp_path): + """A producer killed by SIGPIPE when a consumer exits early (e.g. + `seq | head -1`) produces the right output and must not fail the job + (regression: the producer's rc -13 was treated as a failure).""" + out = tmp_path / "out.txt" + dag = DAG() + job = Job( + Pipeline( + Command("seq", "1000000"), + Command("head", "-1"), + file_output=out, + ), + "sigpipe", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job not in executor.jobs_with_errors + assert out.read_text().strip() == "1" + + +def test_nonzero_producer_still_fails_job(): + """SIGPIPE forgiveness must not mask a producer that exits non-zero + for another reason: `false | cat` still fails the job.""" + dag = DAG() + job = Job(Pipeline(Command("false"), Command("cat")), "real-fail") + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +class _FakeContext: + """A stand-in Context whose cleanup() can be told to raise.""" + + def __init__(self, cleanup_error=None): + self.commands = [] + self.cleaned = False + self._error = cleanup_error + + async def cleanup(self): + self.cleaned = True + if self._error is not None: + raise self._error + + +@pytest.mark.asyncio +async def test_cleanup_quietly_swallows_oserror(): + """A launch failure surfacing from cleanup() during teardown is logged, + not raised -- so it cannot escape execute() on the abort/interrupt + paths (symmetry with _drive's per-job handling).""" + context = _FakeContext(FileNotFoundError("no_such_inner")) + await _cleanup_quietly(context) # must not raise + assert context.cleaned + + +@pytest.mark.asyncio +async def test_cleanup_quietly_propagates_non_oserror(): + """A non-OSError from cleanup() is a bug and must still propagate.""" + with pytest.raises(RuntimeError): + await _cleanup_quietly(_FakeContext(RuntimeError("bug"))) + + +@pytest.mark.asyncio +async def test_shutdown_releases_all_contexts_despite_cleanup_error(): + """A cleanup() OSError during interrupt shutdown is swallowed and the + remaining contexts are still released, instead of the error escaping + _shutdown() and skipping their teardown.""" + executor = LocalExecutor(ThreadScheduler(DAG(), 1)) + bad = _FakeContext(FileNotFoundError("no_such_inner")) + good = _FakeContext() + + async def _done(): + return 0 + + executor.running = [ + ( + Job(Pipeline(Command("true")), "j1"), + bad, + asyncio.create_task(_done()), + 0, + ), + ( + Job(Pipeline(Command("false")), "j2"), + good, + asyncio.create_task(_done()), + 0, + ), + ] + await executor._shutdown() # must not raise + + assert bad.cleaned and good.cleaned diff --git a/tests/integration/test_signals.py b/tests/integration/test_signals.py new file mode 100644 index 0000000..4aeae88 --- /dev/null +++ b/tests/integration/test_signals.py @@ -0,0 +1,217 @@ +""" +Signal-handling tests for LocalExecutor. + +These deliver real signals to the current process. That is safe here because +the executor installs an asyncio loop signal handler that suppresses the +default disposition (e.g. SIGINT -> KeyboardInterrupt) for the duration of the +run, and the timer that raises the signal is always armed *after* the handler +is installed. If these ever prove flaky in a constrained CI, the same behavior +can be exercised by running the workflow in a subprocess and signalling the +child. +""" + +import os +import signal +import sys +import threading +import time + +# Add the parent directory to the path +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +) + +import sentieon_cli.executor as executor_mod # noqa: E402 +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.executor import LocalExecutor # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import ( # noqa: E402 + Command, + InputProcSub, + Pipeline, +) + + +def _echo_dag(out): + dag = DAG() + job = Job(Pipeline(Command("echo", "hi"), file_output=out), "echo") + dag.add_job(job) + return dag + + +def test_off_main_thread_does_not_crash(tmp_path): + """LocalExecutor with signal handlers runs fine off the main thread.""" + out = tmp_path / "o.txt" + holder = {} + + def run(): + try: + LocalExecutor( + ThreadScheduler(_echo_dag(out), 1), + install_signal_handlers=True, + ).execute() + except Exception as exc: # pragma: no cover - regression guard + holder["exc"] = exc + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=10) + + assert not thread.is_alive() + assert "exc" not in holder, holder.get("exc") + assert out.read_text().strip() == "hi" + + +def test_default_does_not_touch_signal_handlers(tmp_path): + """The default executor installs no handlers and leaves them intact.""" + before_int = signal.getsignal(signal.SIGINT) + before_term = signal.getsignal(signal.SIGTERM) + + out = tmp_path / "o.txt" + LocalExecutor(ThreadScheduler(_echo_dag(out), 1)).execute() + + assert signal.getsignal(signal.SIGINT) is before_int + assert signal.getsignal(signal.SIGTERM) is before_term + + +def test_opt_in_restores_signal_handlers(tmp_path): + """Opt-in handlers are restored to their prior values after a run.""" + before_int = signal.getsignal(signal.SIGINT) + before_term = signal.getsignal(signal.SIGTERM) + + out = tmp_path / "o.txt" + LocalExecutor( + ThreadScheduler(_echo_dag(out), 1), + install_signal_handlers=True, + ).execute() + + assert signal.getsignal(signal.SIGINT) is before_int + assert signal.getsignal(signal.SIGTERM) is before_term + + +def test_sigint_triggers_graceful_shutdown(monkeypatch): + """A delivered SIGINT terminates running jobs and cleans up.""" + temp_dirs = [] + real_context = executor_mod.Context + + class SpyContext(real_context): + async def cleanup(self): + temp_dirs.append(self.temp_dir.name) + await super().cleanup() + + monkeypatch.setattr(executor_mod, "Context", SpyContext) + + dag = DAG() + dag.add_job(Job(Pipeline(Command("sleep", "30")), "sleeper")) + executor = LocalExecutor( + ThreadScheduler(dag, 1), + install_signal_handlers=True, + shutdown_grace_period=1.0, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 # returned well before `sleep 30` would finish + assert executor.running == [] + assert temp_dirs # cleanup ran for the interrupted job + for temp_dir in temp_dirs: + assert not os.path.exists(temp_dir) # no leaked temp dir + + +def test_interrupted_job_outputs_are_removed(tmp_path): + """A job killed by an interrupt has its declared outputs deleted.""" + out = tmp_path / "out.txt" + out.write_text("partial") + dag = DAG() + job = Job(Pipeline(Command("sleep", "30")), "sleeper", outputs=[out]) + dag.add_job(job) + executor = LocalExecutor( + ThreadScheduler(dag, 1), + install_signal_handlers=True, + shutdown_grace_period=1.0, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + executor.execute() + finally: + timer.cancel() + + assert executor.running == [] + assert not out.exists() + + +def test_grace_period_escalates_to_sigkill(): + """A job that ignores SIGTERM is still killed after the grace period.""" + script = ( + "import signal, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "time.sleep(30)" + ) + dag = DAG() + job = Job(Pipeline(Command(sys.executable, "-c", script)), "stubborn") + dag.add_job(job) + executor = LocalExecutor( + ThreadScheduler(dag, 1), + install_signal_handlers=True, + shutdown_grace_period=0.5, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 # SIGKILL fired after grace, not after time.sleep(30) + assert executor.running == [] + + +def test_sigint_with_blocked_procsub_cleans_up(): + """SIGINT while a proc-sub FIFO open is still blocked must not hang + the shutdown path (regression: cleanup deadlock).""" + dag = DAG() + # sh never opens the FIFO in "$1", so the proc sub's open() stays + # blocked until cleanup unblocks it. + job = Job( + Pipeline( + Command( + "sh", + "-c", + "sleep 30", + "sh", + InputProcSub(Pipeline(Command("echo", "x"))), + ) + ), + "blocked-procsub", + ) + dag.add_job(job) + executor = LocalExecutor( + ThreadScheduler(dag, 2), + install_signal_handlers=True, + shutdown_grace_period=1.0, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 + assert executor.running == [] diff --git a/tests/unit/test_dag.py b/tests/unit/test_dag.py new file mode 100644 index 0000000..444d07e --- /dev/null +++ b/tests/unit/test_dag.py @@ -0,0 +1,279 @@ +""" +Unit tests for DAG traversal, duplicate detection, and the resume prune. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG, has_all_outputs # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +def _job(name, threads=1, resources=None): + """A trivial job whose identity is its (unique) command name.""" + return Job(Pipeline(Command(name)), name, threads, resources) + + +def test_job_identity_is_the_pipeline(): + # Two jobs with the same pipeline are equal even if named + # differently; a DAG deduplicates on this identity. + j1 = Job(Pipeline(Command("echo", "x")), "name-a") + j2 = Job(Pipeline(Command("echo", "x")), "name-b") + j3 = Job(Pipeline(Command("echo", "y")), "name-c") + assert j1 == j2 + assert hash(j1) == hash(j2) + assert j1 != j3 + + +class TestDAGTraversal: + """Test DAG bookkeeping through mark_finished""" + + def test_add_job_with_unknown_dependency_raises(self): + dag = DAG() + known = _job("known") + dag.add_job(known) + + unknown = _job("unknown") + with pytest.raises(ValueError): + dag.add_job(_job("downstream"), {unknown}) + + def test_finishing_an_unready_job_raises(self): + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, {a}) + + with pytest.raises(DagExecutionError): + dag.mark_finished(b) # b is not ready + + def test_diamond_dependencies(self): + dag = DAG() + top = _job("top") + left = _job("left") + right = _job("right") + bottom = _job("bottom") + + dag.add_job(top) + dag.add_job(left, {top}) + dag.add_job(right, {top}) + dag.add_job(bottom, {left, right}) + + assert set(dag.ready_jobs) == {top} + + assert set(dag.mark_finished(top)) == {left, right} + + # bottom needs BOTH left and right + assert set(dag.mark_finished(left)) == set() + assert set(dag.mark_finished(right)) == {bottom} + + def test_multilevel_chain(self): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + + assert set(dag.ready_jobs) == {a} + assert set(dag.mark_finished(a)) == {b} + assert set(dag.mark_finished(b)) == {c} + + # Everything has been consumed and marked finished. + dag.mark_finished(c) + assert a in dag.finished_jobs + assert b in dag.finished_jobs + assert c in dag.finished_jobs + assert len(dag.waiting_jobs) == 0 + assert len(dag.ready_jobs) == 0 + + def test_add_duplicate_pipeline_raises(self): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "step-1")) + # A second job with an identical pipeline collides on identity. + with pytest.raises(ValueError): + dag.add_job(Job(Pipeline(Command("echo", "x")), "step-2")) + + def test_jobs_differing_only_in_exec_kwargs_are_not_duplicates(self): + # A command's env/cwd is part of its identity, so two jobs that + # differ only there are distinct, not rejected as duplicates. + dag = DAG() + j1 = Job(Pipeline(Command("run", exec_kwargs={"cwd": "/a"})), "a") + j2 = Job(Pipeline(Command("run", exec_kwargs={"cwd": "/b"})), "b") + dag.add_job(j1) + dag.add_job(j2) # must not raise "already in the DAG" + assert j1 in dag.ready_jobs + assert j2 in dag.ready_jobs + + def test_add_job_accepts_list_dependencies(self): + # Regression: non-set dependencies used to be silently dropped, + # leaving the job immediately ready. + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, [a]) + + assert b not in dag.ready_jobs + assert dag.waiting_jobs[b] == {a} + assert set(dag.mark_finished(a)) == {b} + + def test_add_job_accepts_any_iterable_dependencies(self): + # Tuples, frozensets, and single-use generators all work; the + # generator case also guards against consuming the iterable twice. + for make in (tuple, frozenset, iter): + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, make([a])) + assert dag.waiting_jobs[b] == {a}, make + + def test_add_job_rejects_unknown_dependency_in_list(self): + dag = DAG() + dag.add_job(_job("known")) + with pytest.raises(ValueError): + dag.add_job(_job("downstream"), [_job("unknown")]) + + def test_add_job_with_empty_iterable_is_ready(self): + for deps in ([], iter([])): + dag = DAG() + job = _job("solo") + dag.add_job(job, deps) + assert job in dag.ready_jobs + + def test_stored_dependencies_are_independent_of_caller_set(self): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + dag.add_job(a) + dag.add_job(b) + + deps = {a} + dag.add_job(c, deps) + deps.add(b) # mutating the caller's set must not affect the DAG + + assert dag.waiting_jobs[c] == {a} + + +class TestSkipSatisfied: + """Test the pre-run resume prune""" + + def test_satisfied_root_is_marked_finished(self): + dag = DAG() + job = _job("a") + dag.add_job(job) + skipped = dag.skip_satisfied(lambda j: True) + assert skipped == [job] + assert job in dag.finished_jobs + assert len(dag.ready_jobs) == 0 + + def test_unsatisfied_root_stays_ready(self): + dag = DAG() + job = _job("a") + dag.add_job(job) + skipped = dag.skip_satisfied(lambda j: False) + assert skipped == [] + assert job in dag.ready_jobs + assert dag.finished_jobs == [] + + def test_cascades_down_a_chain_in_topological_order(self): + dag = DAG() + a, b, c = _job("a"), _job("b"), _job("c") + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + skipped = dag.skip_satisfied(lambda j: True) + assert skipped == [a, b, c] + assert dag.finished_jobs == [a, b, c] + assert len(dag.ready_jobs) == 0 + assert len(dag.waiting_jobs) == 0 + + def test_stops_at_the_first_unsatisfied_job(self): + dag = DAG() + a, b, c = _job("a"), _job("b"), _job("c") + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + skipped = dag.skip_satisfied(lambda j: j == a) + assert skipped == [a] + assert b in dag.ready_jobs + assert c in dag.waiting_jobs + + def test_blocked_jobs_are_not_offered_to_the_predicate(self): + dag = DAG() + a, b, c = _job("a"), _job("b"), _job("c") + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + seen = [] + + def predicate(job): + seen.append(job) + return job == a + + dag.skip_satisfied(predicate) + # b is offered (a was skipped, unblocking it); c never is -- the + # ancestor rule: a job behind an unsatisfied job cannot be skipped. + assert seen == [a, b] + + def test_diamond_with_partial_parents(self): + dag = DAG() + top = _job("top") + left, right = _job("left"), _job("right") + bottom = _job("bottom") + dag.add_job(top) + dag.add_job(left, {top}) + dag.add_job(right, {top}) + dag.add_job(bottom, {left, right}) + skipped = dag.skip_satisfied(lambda j: j in (top, left)) + assert skipped == [top, left] + assert right in dag.ready_jobs + assert dag.waiting_jobs[bottom] == {right} + + def test_empty_dag_returns_empty(self): + assert DAG().skip_satisfied(lambda j: True) == [] + + +class TestHasAllOutputs: + """Test the default resume predicate""" + + def test_no_declared_outputs_is_never_satisfied(self): + assert has_all_outputs(_job("a")) is False + + def test_true_when_all_outputs_exist(self, tmp_path): + full = tmp_path / "full.txt" + full.write_text("data") + empty = tmp_path / "empty.txt" + empty.touch() # zero-byte files count as present + job = Job(Pipeline(Command("true")), "j", outputs=[full, empty]) + assert has_all_outputs(job) is True + + def test_false_when_any_output_is_missing(self, tmp_path): + present = tmp_path / "present.txt" + present.touch() + missing = tmp_path / "missing.txt" + job = Job(Pipeline(Command("true")), "j", outputs=[present, missing]) + assert has_all_outputs(job) is False + + def test_dangling_symlink_counts_as_missing(self, tmp_path): + target = tmp_path / "target.txt" + target.touch() + link = tmp_path / "link.txt" + link.symlink_to(target) + target.unlink() # now dangling + job = Job(Pipeline(Command("true")), "j", outputs=[link]) + assert has_all_outputs(job) is False diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index c42159f..e9bacdf 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -426,8 +426,7 @@ def test_dag_execution_simulation(self): dag.add_job(job3, {job1, job2}) # Simulate execution - dag_gen = dag.update_dag() - ready_jobs = dag_gen.send(None) + ready_jobs = dag.ready_jobs # Initially job1 and job2 should be ready assert job1 in ready_jobs @@ -435,11 +434,11 @@ def test_dag_execution_simulation(self): assert job3 not in ready_jobs # Complete job1 - new_ready = dag_gen.send(job1) + new_ready = dag.mark_finished(job1) assert len(new_ready) == 0 # job3 still waiting for job2 # Complete job2 - new_ready = dag_gen.send(job2) + new_ready = dag.mark_finished(job2) assert job3 in new_ready # Now job3 is ready diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py new file mode 100644 index 0000000..4f46942 --- /dev/null +++ b/tests/unit/test_job.py @@ -0,0 +1,52 @@ +""" +Unit tests for the Job model. +""" + +import os +import pathlib +import sys + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +def _pipe(text): + return Pipeline(Command("echo", text)) + + +def test_new_fields_default_empty(): + job = Job(_pipe("a"), "a") + assert job.inputs == [] + assert job.outputs == [] + assert job.image is None + + +def test_new_fields_are_stored(): + ins = [pathlib.Path("in1"), pathlib.Path("in2")] + outs = [pathlib.Path("out")] + job = Job(_pipe("a"), "a", inputs=ins, outputs=outs, image="org/img:1") + assert job.inputs == ins + assert job.outputs == outs + assert job.image == "org/img:1" + + +def test_input_output_sequences_are_snapshotted(): + ins = [pathlib.Path("in1")] + job = Job(_pipe("a"), "a", inputs=ins) + ins.append(pathlib.Path("in2")) # mutating the caller's list... + assert job.inputs == [pathlib.Path("in1")] # ...doesn't change the job + + +def test_identity_ignores_metadata_fields(): + # Identity is keyed on the shell pipeline; the other fields don't + # affect it. + j1 = Job(_pipe("same"), "j1", inputs=[pathlib.Path("x")], image="a") + j2 = Job(_pipe("same"), "j2", outputs=[pathlib.Path("y")], image="b") + assert j1 == j2 + assert hash(j1) == hash(j2) diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py new file mode 100644 index 0000000..aaecf0e --- /dev/null +++ b/tests/unit/test_scheduler.py @@ -0,0 +1,104 @@ +""" +Unit tests for the ThreadScheduler. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +def _job(name, threads=1, resources=None): + return Job(Pipeline(Command(name)), name, threads, resources) + + +def test_schedules_independent_jobs_together(): + dag = DAG() + j1 = _job("j1", threads=1) + j2 = _job("j2", threads=1) + dag.add_job(j1) + dag.add_job(j2) + + scheduler = ThreadScheduler(dag, threads=4) + assert scheduler.start() == {j1, j2} + + +def test_thread_budget_gates_scheduling(): + dag = DAG() + j1 = _job("j1", threads=2) + j2 = _job("j2", threads=2) + dag.add_job(j1) + dag.add_job(j2) + + # A 2-thread budget only fits one 2-thread job at a time. + scheduler = ThreadScheduler(dag, threads=2) + + first = scheduler.start() + assert len(first) == 1 + running = next(iter(first)) + + # The second job starts only once the first frees its threads. + second = scheduler.job_finished(running) + assert len(second) == 1 + remaining = next(iter(second)) + + assert {running, remaining} == {j1, j2} + + +def test_named_resource_gates_scheduling(): + dag = DAG() + j1 = _job("j1", threads=1, resources={"node0": 1}) + j2 = _job("j2", threads=1, resources={"node0": 1}) + dag.add_job(j1) + dag.add_job(j2) + + # Plenty of threads, but only a single node0 token. + scheduler = ThreadScheduler(dag, threads=8, resources={"node0": 1}) + + first = scheduler.start() + assert len(first) == 1 + running = next(iter(first)) + + second = scheduler.job_finished(running) + assert len(second) == 1 + assert {running, next(iter(second))} == {j1, j2} + + +def test_resource_requirements_ignored_when_unmanaged(): + # Jobs may request a resource the scheduler does not track; those + # requirements are simply ignored. + dag = DAG() + j1 = _job("j1", threads=1, resources={"gpu": 1}) + j2 = _job("j2", threads=1, resources={"gpu": 1}) + dag.add_job(j1) + dag.add_job(j2) + + scheduler = ThreadScheduler(dag, threads=8) + assert scheduler.start() == {j1, j2} + + +def test_oversized_thread_request_is_rejected(): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + ThreadScheduler(dag, threads=2).start() + + +def test_oversized_resource_request_is_rejected(): + dag = DAG() + job = Job(Pipeline(Command("echo", "hi")), "greedy", 1, {"node0": 3}) + dag.add_job(job) + with pytest.raises(DagExecutionError): + ThreadScheduler(dag, threads=8, resources={"node0": 1}).start() diff --git a/tests/unit/test_scheduler_contract.py b/tests/unit/test_scheduler_contract.py new file mode 100644 index 0000000..0460403 --- /dev/null +++ b/tests/unit/test_scheduler_contract.py @@ -0,0 +1,109 @@ +""" +Conformance tests for the BaseScheduler contract. + +``drive`` mimics what an executor does: ask the scheduler for the initial +batch, then report each job finished (FIFO) and collect whatever that +unblocks, until the DAG drains. Any BaseScheduler implementation can be +dropped into ``drive`` to check that it honors the contract; the +dependency-order tests run against every scheduler in ``SCHEDULERS``. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + +SCHEDULERS = [ + pytest.param(lambda dag: ThreadScheduler(dag, threads=8), id="thread"), +] + + +def _job(name, threads=1, resources=None): + return Job(Pipeline(Command(name)), name, threads, resources) + + +def drive(scheduler): + """Drive a scheduler to completion; return the job finish order.""" + order = [] + queue = list(scheduler.start()) + while queue: + job = queue.pop(0) + order.append(job) + queue.extend(scheduler.job_finished(job)) + return order + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_drives_a_diamond_to_completion(make_scheduler): + dag = DAG() + top = _job("top") + left = _job("left") + right = _job("right") + bottom = _job("bottom") + dag.add_job(top) + dag.add_job(left, {top}) + dag.add_job(right, {top}) + dag.add_job(bottom, {left, right}) + + order = drive(make_scheduler(dag)) + + # Every job runs exactly once. + assert len(order) == len(set(order)) + assert sorted(j.name for j in order) == [ + "bottom", + "left", + "right", + "top", + ] + # Dependencies precede their dependents. + pos = {job: i for i, job in enumerate(order)} + assert pos[top] < pos[left] < pos[bottom] + assert pos[top] < pos[right] < pos[bottom] + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_drives_a_chain_to_completion(make_scheduler): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + + order = drive(make_scheduler(dag)) + assert [j.name for j in order] == ["a", "b", "c"] + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_job_finished_on_unknown_job_raises(make_scheduler): + dag = DAG() + dag.add_job(_job("a")) + scheduler = make_scheduler(dag) + scheduler.start() + + with pytest.raises(DagExecutionError): + scheduler.job_finished(_job("never-scheduled")) + + +def test_budget_constrained_run_still_completes(): + # ThreadScheduler-specific: a 1-thread budget serializes two independent + # jobs, but driving to completion still runs both exactly once. + dag = DAG() + dag.add_job(_job("a", threads=1)) + dag.add_job(_job("b", threads=1)) + + order = drive(ThreadScheduler(dag, threads=1)) + assert sorted(j.name for j in order) == ["a", "b"] diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py new file mode 100644 index 0000000..800623f --- /dev/null +++ b/tests/unit/test_storage.py @@ -0,0 +1,49 @@ +""" +Unit tests for the storage providers. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.storage import LocalStorageProvider # noqa: E402 + + +@pytest.mark.asyncio +async def test_stage_in_present_file_ok(tmp_path): + f = tmp_path / "in.txt" + f.write_text("data") + await LocalStorageProvider().stage_in(f) # no raise + + +@pytest.mark.asyncio +async def test_stage_in_empty_file_ok(tmp_path): + f = tmp_path / "in.txt" + f.touch() # a zero-byte file still counts as present + await LocalStorageProvider().stage_in(f) + + +@pytest.mark.asyncio +async def test_stage_in_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + await LocalStorageProvider().stage_in(tmp_path / "nope.txt") + + +@pytest.mark.asyncio +async def test_stage_out_present_file_ok(tmp_path): + f = tmp_path / "out.txt" + f.touch() + await LocalStorageProvider().stage_out(f) + + +@pytest.mark.asyncio +async def test_stage_out_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + await LocalStorageProvider().stage_out(tmp_path / "nope.txt") From faf6c9a2c2a341c5a70db138fe6fcb770dd46421 Mon Sep 17 00:00:00 2001 From: Don Freed Date: Wed, 12 Aug 2026 12:55:24 -0700 Subject: [PATCH 3/4] Fix BasePipeline lifecycle issues - main() created the temp dir and only removed it after build_dag() and run() returned, so any raise in between (an infeasible DAG, a launch failure, a KeyboardInterrupt) leaked the directory. Wrap build/run/check in try/finally so the temp dir is always removed, and honor retain_tmpdir on failure too so a failed run can be inspected. - check_execution raised a bare ValueError("Execution failed"); raise DagExecutionError naming the failed jobs instead, to aid post-mortems. The unexecuted-jobs check uses DagExecutionError as well. - setup_logging set the level on the logger's parent, which resolved to the root logger -- so -v/--debug turned on DEBUG for every third-party library. Add logging.set_level(), which scopes the level to the sentieon_cli package logger; the module loggers are at NOTSET and inherit from it, and the root logger is left untouched. - add_arguments mutated the shared class-level params specs (deleting "flags" and injecting inferred "type" entries), so registering a pipeline class on a second parser lost its short flags (e.g. -r, -t). Build a fresh kwargs dict per argument instead. Co-Authored-By: Claude Fable 5 --- sentieon_cli/logging.py | 14 ++- sentieon_cli/pipeline.py | 47 ++++---- tests/unit/test_pipeline_lifecycle.py | 147 ++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_pipeline_lifecycle.py diff --git a/sentieon_cli/logging.py b/sentieon_cli/logging.py index bd5d15a..128485d 100644 --- a/sentieon_cli/logging.py +++ b/sentieon_cli/logging.py @@ -4,9 +4,21 @@ handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s")) -def get_logger(name: str): +def get_logger(name: str) -> logging.Logger: """Return a logger with a StreamHandler.""" logger = logging.getLogger(name) logger.addHandler(handler) logger.propagate = False return logger + + +def set_level(level: int | str) -> None: + """Set the level of the package logger. + + The package's module loggers (``sentieon_cli.dag``, + ``sentieon_cli.executor``, ...) are all at NOTSET and inherit their + effective level from the ``sentieon_cli`` package logger, so setting it + here scopes verbosity to this package without touching the root logger + (and thus third-party libraries). + """ + logging.getLogger(__package__).setLevel(level) diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index ddb67e1..1433db2 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -15,9 +15,10 @@ from . import command_strings as cmds from .dag import DAG +from .exceptions import DagExecutionError from .executor import BaseExecutor, DryRunExecutor, LocalExecutor from .job import Job -from .logging import get_logger +from .logging import get_logger, set_level from .scheduler import ThreadScheduler from .util import __version__, check_version, path_arg, tmp @@ -71,17 +72,17 @@ class BasePipeline(ABC): @classmethod def add_arguments(cls, parser: argparse.ArgumentParser): - for k, kwargs in cls.params.items(): - flags = ["--" + k] + # Build a fresh kwargs dict per argument so repeated calls do not + # mutate the shared class-level params/positionals specs. + for k, spec in cls.params.items(): + kwargs = dict(spec) + flags = kwargs.pop("flags", ["--" + k]) if "default" in kwargs and "type" not in kwargs: kwargs["type"] = type(kwargs["default"]) - if "flags" in kwargs: - flags = kwargs["flags"] - del kwargs["flags"] parser.add_argument(*flags, **kwargs) - for k, kwargs in cls.positionals.items(): - parser.add_argument(k, **kwargs) + for k, spec in cls.positionals.items(): + parser.add_argument(k, **dict(spec)) def handle_arguments(self, args: argparse.Namespace): """Update self using the argparse object""" @@ -106,9 +107,10 @@ def handle_arguments(self, args: argparse.Namespace): def setup_logging(self, args: argparse.Namespace) -> None: self.logger = get_logger(__name__) - if not self.logger.parent: - raise RuntimeError("Logger has no parent logger") - self.logger.parent.setLevel(args.loglevel) + # Scope verbosity to this package's loggers rather than the root + # logger, so -v/--debug does not turn on DEBUG for every + # third-party library. + set_level(args.loglevel) self.logger.info("Starting sentieon-cli version: %s", __version__) def __init__(self) -> None: @@ -130,13 +132,17 @@ def main(self, args: argparse.Namespace) -> None: tmp_dir_str = tmp() self.tmp_dir = pathlib.Path(tmp_dir_str) - dag = self.build_dag() - executor = self.run(dag) - - if not self.retain_tmpdir: - shutil.rmtree(tmp_dir_str) - - self.check_execution(dag, executor) + # Always clean up the temp dir, even if build_dag()/run() raise (an + # infeasible DAG, a launch failure, or a KeyboardInterrupt) -- + # otherwise it would leak. retain_tmpdir is honored on failure too, + # so a failed run can still be inspected. + try: + dag = self.build_dag() + executor = self.run(dag) + self.check_execution(dag, executor) + finally: + if not self.retain_tmpdir: + shutil.rmtree(tmp_dir_str) def check_execution( self, @@ -145,10 +151,11 @@ def check_execution( ): """Check the DAG and executor after a run""" if executor.jobs_with_errors: - raise ValueError("Execution failed") + failed = ", ".join(str(job) for job in executor.jobs_with_errors) + raise DagExecutionError(f"Execution failed for jobs: {failed}") if len(dag.waiting_jobs) > 0 or len(dag.ready_jobs) > 0: - raise ValueError( + raise DagExecutionError( "The DAG has some unexecuted jobs\n" f"Waiting jobs: {dag.waiting_jobs}\n" f"Ready jobs: {dag.ready_jobs}\n" diff --git a/tests/unit/test_pipeline_lifecycle.py b/tests/unit/test_pipeline_lifecycle.py new file mode 100644 index 0000000..b9659c1 --- /dev/null +++ b/tests/unit/test_pipeline_lifecycle.py @@ -0,0 +1,147 @@ +""" +Unit tests for the BasePipeline lifecycle (argument registration, logging +scope, temp-dir cleanup, and post-run checks). +""" + +import argparse +import logging +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.pipeline import BasePipeline # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +class _DummyPipeline(BasePipeline): + """A minimal concrete pipeline for exercising BasePipeline plumbing.""" + + def validate(self) -> None: + pass + + def configure(self) -> None: + pass + + def build_dag(self) -> DAG: + return DAG() + + +class _FailingPipeline(_DummyPipeline): + """A pipeline whose DAG construction raises.""" + + def build_dag(self) -> DAG: + raise RuntimeError("boom") + + +def _option_strings(parser): + return [opt for action in parser._actions for opt in action.option_strings] + + +def test_add_arguments_does_not_mutate_the_class_specs(): + before_reference = dict(BasePipeline.params["reference"]) + before_cores = dict(BasePipeline.params["cores"]) + + parser = argparse.ArgumentParser() + BasePipeline.add_arguments(parser) + + assert BasePipeline.params["reference"] == before_reference + assert BasePipeline.params["cores"] == before_cores + # In particular the specs keep their flags and gain no inferred type. + assert "flags" in BasePipeline.params["reference"] + assert "type" not in BasePipeline.params["cores"] + + +def test_add_arguments_keeps_short_flags_across_parsers(): + # Registering the same pipeline class on a second parser must produce + # the same options; the short flags used to be lost because the first + # call popped them from the shared class-level spec. + p1 = argparse.ArgumentParser() + p2 = argparse.ArgumentParser() + BasePipeline.add_arguments(p1) + BasePipeline.add_arguments(p2) + + for parser in (p1, p2): + opts = _option_strings(parser) + assert "-r" in opts + assert "--reference" in opts + assert "-t" in opts + assert "--cores" in opts + + +def test_setup_logging_does_not_touch_the_root_logger(): + root_level_before = logging.getLogger().level + package_logger = logging.getLogger("sentieon_cli") + package_level_before = package_logger.level + try: + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel=logging.DEBUG)) + + assert logging.getLogger().level == root_level_before + assert package_logger.level == logging.DEBUG + finally: + package_logger.setLevel(package_level_before) + + +def test_main_cleans_up_tmpdir_when_build_dag_raises(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _FailingPipeline() + + with pytest.raises(RuntimeError, match="boom"): + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert list(tmp_path.iterdir()) == [] # no leaked temp dir + + +def test_main_retains_tmpdir_on_failure_when_requested(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _FailingPipeline() + pipeline.retain_tmpdir = True + + with pytest.raises(RuntimeError, match="boom"): + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert len(list(tmp_path.iterdir())) == 1 # kept for inspection + + +def test_main_cleans_up_tmpdir_on_success(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _DummyPipeline() + pipeline.dry_run = True + + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert list(tmp_path.iterdir()) == [] + + +class _StubExecutor: + def __init__(self, jobs_with_errors=None): + self.jobs_with_errors = jobs_with_errors or [] + + +def test_check_execution_names_the_failed_jobs(): + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel="WARNING")) + job = Job(Pipeline(Command("false")), "broken-step") + + with pytest.raises(DagExecutionError, match="broken-step"): + pipeline.check_execution(DAG(), _StubExecutor([job])) + + +def test_check_execution_flags_unexecuted_jobs(): + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel="WARNING")) + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "unexecuted")) + + with pytest.raises(DagExecutionError, match="unexecuted"): + pipeline.check_execution(dag, _StubExecutor()) From ffd4b603b325496306b9ac076bc4cc078124885b Mon Sep 17 00:00:00 2001 From: Don Freed Date: Thu, 13 Aug 2026 00:34:01 -0700 Subject: [PATCH 4/4] Trim unused engine functionality and condense comments Remove engine capabilities that nothing in the CLI uses: - DAG.skip_satisfied() and the has_all_outputs predicate; no pipeline offers a resume option, so the pre-run prune had no callers. - The storage module (StorageProvider/LocalStorageProvider) and the executor's input/output staging built on it, along with LocalExecutor._remove_outputs(); pipelines do not declare per-job files, so the staging and failed-output removal were dead paths. - Job.inputs/outputs/image; Job is back to (pipeline, name, threads, resources). Drop the tests that covered the removed functionality and condense some overly verbose code comments. Co-Authored-By: Claude Fable 5 --- sentieon_cli/dag.py | 41 +------ sentieon_cli/executor.py | 55 +-------- sentieon_cli/job.py | 18 +-- sentieon_cli/logging.py | 9 +- sentieon_cli/pipeline.py | 7 -- sentieon_cli/shell_pipeline.py | 20 +--- sentieon_cli/storage.py | 46 -------- tests/integration/test_executor.py | 174 +---------------------------- tests/integration/test_signals.py | 24 ---- tests/unit/test_dag.py | 114 +------------------ tests/unit/test_job.py | 52 --------- tests/unit/test_storage.py | 49 -------- 12 files changed, 10 insertions(+), 599 deletions(-) delete mode 100644 sentieon_cli/storage.py delete mode 100644 tests/unit/test_job.py delete mode 100644 tests/unit/test_storage.py diff --git a/sentieon_cli/dag.py b/sentieon_cli/dag.py index 13c3343..d40419d 100644 --- a/sentieon_cli/dag.py +++ b/sentieon_cli/dag.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import Callable, Dict, Iterable, List, Optional, Set +from typing import Dict, Iterable, List, Optional, Set from .exceptions import DagExecutionError from .job import Job @@ -83,42 +83,3 @@ def mark_finished(self, job: Job) -> Dict[Job, None]: new_ready_jobs[dependency] = None del self.waiting_jobs[dependency] return new_ready_jobs - - def skip_satisfied(self, is_satisfied: Callable[[Job], bool]) -> List[Job]: - """Mark already-satisfied jobs finished before a run (resume). - - Walks the ready frontier: a job is skipped iff ``is_satisfied(job)`` - returns True and every one of its dependencies was itself skipped. - A job behind an unsatisfied ancestor is never offered to the - predicate -- it only becomes ready once all its dependencies are - finished, and during this walk "finished" means "skipped". - - Skipped jobs land in :attr:`finished_jobs`, so an executor never - sees them; unsatisfied jobs are left in place untouched. Call this - before handing the DAG to a scheduler. - - Returns the skipped jobs in topological order. - """ - skipped: List[Job] = [] - frontier: List[Job] = list(self.ready_jobs) - i = 0 - while i < len(frontier): - job = frontier[i] - i += 1 - if is_satisfied(job): - skipped.append(job) - frontier.extend(self.mark_finished(job)) - return skipped - - -def has_all_outputs(job: Job) -> bool: - """The default resume predicate for :meth:`DAG.skip_satisfied`. - - True iff ``job`` declares at least one output and every declared - output exists on the local filesystem. Jobs with no declared outputs - always return False -- they cannot be verified, so they must rerun. - Zero-byte files count as present; a dangling symlink does not - (``Path.exists()`` follows links) -- matching - :class:`~sentieon_cli.storage.LocalStorageProvider` semantics. - """ - return bool(job.outputs) and all(path.exists() for path in job.outputs) diff --git a/sentieon_cli/executor.py b/sentieon_cli/executor.py index ad0e39d..66b1aaf 100644 --- a/sentieon_cli/executor.py +++ b/sentieon_cli/executor.py @@ -7,13 +7,12 @@ import threading import time from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Tuple from .job import Job from .logging import get_logger from .scheduler import BaseScheduler from .shell_pipeline import Context -from .storage import LocalStorageProvider, StorageProvider logger = get_logger(__name__) @@ -203,16 +202,12 @@ def __init__( *, install_signal_handlers: bool = False, shutdown_grace_period: float = 10.0, - storage: Optional[StorageProvider] = None, ) -> None: super().__init__( scheduler, install_signal_handlers=install_signal_handlers, ) self.shutdown_grace_period = shutdown_grace_period - self.storage: StorageProvider = ( - storage if storage is not None else LocalStorageProvider() - ) self.running: List[ Tuple[ Job, @@ -229,8 +224,6 @@ async def run_job(self, job: Job) -> None: context = Context() start_time = time.monotonic_ns() try: - for path in job.inputs: - await self.storage.stage_in(path) proc = await cmd.run( context, stderr=sys.stderr, @@ -258,7 +251,6 @@ async def run_job(self, job: Job) -> None: # race): the job is already recorded, so swallow it. await self._terminate_context(context) await _cleanup_quietly(context) - self._remove_outputs(job) async def _terminate_context(self, context: Context) -> None: """Tear down processes already spawned in an aborted context. @@ -282,30 +274,6 @@ async def _terminate_context(self, context: Context) -> None: _kill_survivors(context) await asyncio.wait(waits) - def _remove_outputs(self, job: Job) -> None: - """Delete a failed job's declared outputs. - - A partial or unverified output must not survive a failure: resume - (:meth:`~sentieon_cli.dag.DAG.skip_satisfied`) treats an existing - declared output as proof that its producer succeeded. Directories are - deliberately not removed (never recurse into user paths); a warning - is logged instead. - """ - for path in job.outputs: - try: - path.unlink() - except FileNotFoundError: - continue - except OSError as exc: - logger.warning( - "could not remove output %s of failed job %s: %s", - path, - job, - exc, - ) - else: - logger.info("Removed output of failed job %s: %s", job, path) - async def _drive(self, stop_event: asyncio.Event) -> None: """Schedule and run jobs until the DAG drains or we are stopped.""" ready_jobs = self.scheduler.start() @@ -375,19 +343,6 @@ async def _drive(self, stop_event: asyncio.Event) -> None: "Possible out-of-memory?" ) - if not cmd_failed: - for path in job.outputs: - try: - await self.storage.stage_out(path) - except OSError as exc: - logger.error( - "missing output %s for %s: %s", - path, - job.shell, - exc, - ) - cmd_failed = True - # Release resources before recording the verdict. A # proc-sub inner command that failed to launch surfaces # its OSError from the background task here; treat it as @@ -404,7 +359,6 @@ async def _drive(self, stop_event: asyncio.Event) -> None: logger.error("Command failure: %s", job.shell) self.jobs_with_errors.append(job) self.start_new_jobs = False - self._remove_outputs(job) else: logger.info( f"Finished command in " @@ -457,10 +411,3 @@ async def _shutdown(self) -> None: # tear down after an interrupt; log it rather than let it escape, # and keep releasing the remaining contexts. await _cleanup_quietly(context) - - # Interrupted jobs never had their outputs verified; a partial file - # must not survive to fool a later resume. Jobs that completed in the - # same wake-up as the stop signal are conservatively included -- they - # were never verified either. - for job, _context, _task, _start in running: - self._remove_outputs(job) diff --git a/sentieon_cli/job.py b/sentieon_cli/job.py index 9d7d4d2..3e36fa2 100644 --- a/sentieon_cli/job.py +++ b/sentieon_cli/job.py @@ -2,8 +2,7 @@ Job objects """ -import pathlib -from typing import Dict, Iterable, List, Optional +from typing import Dict, Optional from .shell_pipeline import Pipeline @@ -17,10 +16,6 @@ class Job: * ``name`` -- a human-readable label (not part of identity). * ``threads`` -- CPU threads the job needs (a local scheduling budget). * ``resources`` -- named resource counts (e.g. NUMA-node tokens). - * ``inputs`` / ``outputs`` -- local files the job reads / writes; when - declared, the executor verifies them around the run and removes the - outputs of a failed attempt. - * ``image`` -- container image (or environment) to run in on a backend. A job's identity is keyed only on ``shell``: two jobs with the same pipeline are equal (and collide in a DAG) regardless of the other fields. @@ -32,22 +27,11 @@ def __init__( name: str, threads: int = 1, resources: Optional[Dict[str, int]] = None, - *, - inputs: Optional[Iterable[pathlib.Path]] = None, - outputs: Optional[Iterable[pathlib.Path]] = None, - image: Optional[str] = None, ) -> None: self.shell = pipeline self.name = name self.threads = threads self.resources = {} if resources is None else resources - self.inputs: List[pathlib.Path] = ( - [] if inputs is None else list(inputs) - ) - self.outputs: List[pathlib.Path] = ( - [] if outputs is None else list(outputs) - ) - self.image = image def __hash__(self) -> int: return hash(self.shell) diff --git a/sentieon_cli/logging.py b/sentieon_cli/logging.py index 128485d..faca6d0 100644 --- a/sentieon_cli/logging.py +++ b/sentieon_cli/logging.py @@ -13,12 +13,5 @@ def get_logger(name: str) -> logging.Logger: def set_level(level: int | str) -> None: - """Set the level of the package logger. - - The package's module loggers (``sentieon_cli.dag``, - ``sentieon_cli.executor``, ...) are all at NOTSET and inherit their - effective level from the ``sentieon_cli`` package logger, so setting it - here scopes verbosity to this package without touching the root logger - (and thus third-party libraries). - """ + """Set the level of the package logger.""" logging.getLogger(__package__).setLevel(level) diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 1433db2..be82e0d 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -107,9 +107,6 @@ def handle_arguments(self, args: argparse.Namespace): def setup_logging(self, args: argparse.Namespace) -> None: self.logger = get_logger(__name__) - # Scope verbosity to this package's loggers rather than the root - # logger, so -v/--debug does not turn on DEBUG for every - # third-party library. set_level(args.loglevel) self.logger.info("Starting sentieon-cli version: %s", __version__) @@ -132,10 +129,6 @@ def main(self, args: argparse.Namespace) -> None: tmp_dir_str = tmp() self.tmp_dir = pathlib.Path(tmp_dir_str) - # Always clean up the temp dir, even if build_dag()/run() raise (an - # infeasible DAG, a launch failure, or a KeyboardInterrupt) -- - # otherwise it would leak. retain_tmpdir is honored on failure too, - # so a failed run can still be inspected. try: dag = self.build_dag() executor = self.run(dag) diff --git a/sentieon_cli/shell_pipeline.py b/sentieon_cli/shell_pipeline.py index 2469484..1d6d26d 100644 --- a/sentieon_cli/shell_pipeline.py +++ b/sentieon_cli/shell_pipeline.py @@ -22,13 +22,7 @@ def _freeze(value: Any) -> Any: - """Return an order-independent, hashable view of ``value``. - - ``exec_kwargs`` (and a nested ``env`` dict) have no stable iteration - order, so hashing their raw form would let two *equal* commands hash - differently, breaking the ``a == b implies hash(a) == hash(b)`` - contract the DAG relies on. Canonicalise dicts/sets recursively. - """ + """Return an order-independent, hashable view of ``value``.""" if isinstance(value, dict): return frozenset((k, _freeze(v)) for k, v in value.items()) if isinstance(value, (list, tuple)): @@ -42,12 +36,7 @@ def _set_pipe_size(fd: int, size: int) -> None: """Best-effort enlarge the OS pipe buffer backing ``fd``. Lets a throughput-bound pipeline hold more in-flight data between - stages. ``F_SETPIPE_SZ`` is Linux-only and, for unprivileged - processes, capped by ``/proc/sys/fs/pipe-max-size`` (exceeding it - raises EPERM) -- both deployment concerns, so a failure is logged and - the pipe keeps its default size rather than aborting the run. A no-op - where the constant is unavailable (non-Linux), preserving POSIX - portability. + stages. """ set_pipe_sz = getattr(fcntl, "F_SETPIPE_SZ", None) if set_pipe_sz is None: @@ -220,8 +209,6 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: if not isinstance(other, Command): return NotImplemented - # Identity is the *definition* of the command: not self.proc, which - # is runtime state that would mutate identity once the job runs. return ( self.executable == other.executable and self.args == other.args @@ -264,8 +251,6 @@ async def run( stderr: Union[IO[Any], int, None] = None, ) -> asyncio.subprocess.Process: # We cannot have handles from both files and to run(). - # Test `is not None`, not truthiness: an int fd of 0 is a valid - # stream but falsy. if self.file_input is not None and stdin is not None: raise ValueError( f"Pipeline {self} was given both a file_input and a stdin " @@ -389,7 +374,6 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: if not isinstance(other, ProcSub): return NotImplemented - # An input and an output proc sub are not the same: <(x) != >(x). return type(self) is type(other) and self.node == other.node def __hash__(self) -> int: diff --git a/sentieon_cli/storage.py b/sentieon_cli/storage.py deleted file mode 100644 index 5fb6263..0000000 --- a/sentieon_cli/storage.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Storage staging for job inputs and outputs.""" - -import pathlib -from abc import ABC, abstractmethod - - -class StorageProvider(ABC): - """Stage a job's declared files in before it runs and out after. - - On a shared filesystem the files are already local, so staging - degenerates to verifying presence (see :class:`LocalStorageProvider`). - A cloud provider would instead pull inputs from and push outputs to - object storage, deriving each object's URI from a configured prefix and - the local path. The methods are async so they compose with the async - executors. - """ - - @abstractmethod - async def stage_in(self, local: pathlib.Path) -> None: - """Make ``local`` available before a job runs. - - Pull it from storage, or verify it is present. Raise if it cannot be - made available. - """ - - @abstractmethod - async def stage_out(self, local: pathlib.Path) -> None: - """Persist ``local`` after a job runs. - - Push it to storage, or verify it was produced. Raise if it is missing. - """ - - -class LocalStorageProvider(StorageProvider): - """Shared-filesystem provider: no data movement, just verify presence. - - A file counts as present when it exists, even if it is empty (zero bytes). - """ - - async def stage_in(self, local: pathlib.Path) -> None: - if not local.exists(): - raise FileNotFoundError(f"missing input: {local}") - - async def stage_out(self, local: pathlib.Path) -> None: - if not local.exists(): - raise FileNotFoundError(f"missing output: {local}") diff --git a/tests/integration/test_executor.py b/tests/integration/test_executor.py index 4f0a7f4..63c3779 100644 --- a/tests/integration/test_executor.py +++ b/tests/integration/test_executor.py @@ -16,7 +16,7 @@ 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) ) -from sentieon_cli.dag import DAG, has_all_outputs # noqa: E402 +from sentieon_cli.dag import DAG # noqa: E402 from sentieon_cli.exceptions import DagExecutionError # noqa: E402 from sentieon_cli.executor import ( # noqa: E402 DryRunExecutor, @@ -284,140 +284,8 @@ async def boom(self, *args, **kwargs): LocalExecutor(ThreadScheduler(dag, 1)).execute() -def test_missing_input_fails_before_run(tmp_path): - """A declared input that is missing fails the job before it runs.""" - missing = tmp_path / "in.txt" - marker = tmp_path / "ran.txt" - dag = DAG() - job = Job( - Pipeline(Command("touch", str(marker))), - "j", - inputs=[missing], - ) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert not marker.exists() # the command never ran - - -def test_present_inputs_and_outputs_allow_success(tmp_path): - """Empty (zero-byte) declared inputs/outputs still count as present.""" - inp = tmp_path / "in.txt" - inp.touch() # empty input file - out = tmp_path / "out.txt" - dag = DAG() - job = Job( - Pipeline(Command("touch", str(out))), # produces an empty output - "j", - inputs=[inp], - outputs=[out], - ) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert executor.jobs_with_errors == [] - assert out.exists() - - -def test_missing_output_fails_after_run(tmp_path): - """A declared output the command does not produce fails the job.""" - missing = tmp_path / "never.txt" - dag = DAG() - job = Job(Pipeline(Command("true")), "j", outputs=[missing]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - - -def test_failed_job_outputs_are_removed(tmp_path): - """A failed job's declared outputs are deleted from disk.""" - out = tmp_path / "out.txt" - out.write_text("stale") - dag = DAG() - job = Job(Pipeline(Command("false")), "j", outputs=[out]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert not out.exists() - - -def test_missing_output_failure_removes_produced_siblings(tmp_path): - """A missing declared output deletes the outputs that were produced.""" - produced = tmp_path / "out1.txt" - never = tmp_path / "out2.txt" - dag = DAG() - job = Job( - Pipeline(Command("touch", str(produced))), - "j", - outputs=[produced, never], - ) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert not produced.exists() - - -def test_launch_failure_removes_outputs(tmp_path): - """A job that fails to launch has its declared outputs deleted.""" - out = tmp_path / "out.txt" - out.write_text("stale") - dag = DAG() - job = Job(Pipeline(Command("no_such_cmd_zzz")), "j", outputs=[out]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert not out.exists() - - -def test_missing_input_failure_removes_outputs(tmp_path): - """Any failed attempt clears the job's declared outputs -- even a - missing-input failure where the command never ran.""" - missing = tmp_path / "in.txt" - out = tmp_path / "out.txt" - out.write_text("stale") - dag = DAG() - job = Job(Pipeline(Command("true")), "j", inputs=[missing], outputs=[out]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert not out.exists() - - -def test_successful_job_keeps_outputs(tmp_path): - """A successful job's outputs are never deleted.""" - out = tmp_path / "out.txt" - dag = DAG() - job = Job(Pipeline(Command("touch", str(out))), "j", outputs=[out]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert executor.jobs_with_errors == [] - assert out.exists() - - -def test_failed_job_directory_output_is_left_in_place(tmp_path): - """Directories declared as outputs are never removed on failure.""" - out_dir = tmp_path / "out_dir" - out_dir.mkdir() - (out_dir / "keep.txt").write_text("keep") - dag = DAG() - job = Job(Pipeline(Command("false")), "j", outputs=[out_dir]) - dag.add_job(job) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert job in executor.jobs_with_errors - assert out_dir.is_dir() - assert (out_dir / "keep.txt").read_text() == "keep" - - def test_local_executor_empty_dag_completes(): - """An empty DAG (e.g. everything skipped by resume) runs cleanly.""" + """An empty DAG runs cleanly.""" executor = LocalExecutor(ThreadScheduler(DAG(), 1)) executor.execute() assert executor.jobs_with_errors == [] @@ -431,44 +299,6 @@ def test_dry_run_empty_dag_completes(capsys): assert executor.jobs_with_errors == [] -def test_resume_after_failure_reruns_only_failed_subtree(tmp_path): - """After a partial failure, skip_satisfied + has_all_outputs resumes - the run from the surviving intermediate files.""" - a_out = tmp_path / "a.txt" - b_out = tmp_path / "b.txt" - - def build_dag(b_cmd): - dag = DAG() - a = Job( - Pipeline(Command("echo", "a"), file_output=a_out), - "a", - outputs=[a_out], - ) - b = Job(Pipeline(b_cmd), "b", outputs=[b_out]) - dag.add_job(a) - dag.add_job(b, {a}) - return dag, a, b - - # First run: a succeeds, b fails (and b's partial output is deleted). - b_out.write_text("partial") - dag, _a, b = build_dag(Command("false")) - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert b in executor.jobs_with_errors - assert a_out.exists() - assert not b_out.exists() - - # Second run: rebuild the DAG with b fixed; resume skips a. - dag, a, b = build_dag(Command("touch", str(b_out))) - skipped = dag.skip_satisfied(has_all_outputs) - assert skipped == [a] - executor = LocalExecutor(ThreadScheduler(dag, 1)) - executor.execute() - assert executor.jobs_with_errors == [] - assert b_out.exists() - assert a_out.read_text().strip() == "a" # a was not rerun over - - def _execute_bounded(executor, timeout=15): """Run executor.execute() with a hang guard.""" holder = {} diff --git a/tests/integration/test_signals.py b/tests/integration/test_signals.py index 4aeae88..8aa2042 100644 --- a/tests/integration/test_signals.py +++ b/tests/integration/test_signals.py @@ -126,30 +126,6 @@ async def cleanup(self): assert not os.path.exists(temp_dir) # no leaked temp dir -def test_interrupted_job_outputs_are_removed(tmp_path): - """A job killed by an interrupt has its declared outputs deleted.""" - out = tmp_path / "out.txt" - out.write_text("partial") - dag = DAG() - job = Job(Pipeline(Command("sleep", "30")), "sleeper", outputs=[out]) - dag.add_job(job) - executor = LocalExecutor( - ThreadScheduler(dag, 1), - install_signal_handlers=True, - shutdown_grace_period=1.0, - ) - - timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) - timer.start() - try: - executor.execute() - finally: - timer.cancel() - - assert executor.running == [] - assert not out.exists() - - def test_grace_period_escalates_to_sigkill(): """A job that ignores SIGTERM is still killed after the grace period.""" script = ( diff --git a/tests/unit/test_dag.py b/tests/unit/test_dag.py index 444d07e..aa9d565 100644 --- a/tests/unit/test_dag.py +++ b/tests/unit/test_dag.py @@ -1,5 +1,5 @@ """ -Unit tests for DAG traversal, duplicate detection, and the resume prune. +Unit tests for DAG traversal and duplicate detection. """ import os @@ -13,7 +13,7 @@ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), ) -from sentieon_cli.dag import DAG, has_all_outputs # noqa: E402 +from sentieon_cli.dag import DAG # noqa: E402 from sentieon_cli.exceptions import DagExecutionError # noqa: E402 from sentieon_cli.job import Job # noqa: E402 from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 @@ -167,113 +167,3 @@ def test_stored_dependencies_are_independent_of_caller_set(self): deps.add(b) # mutating the caller's set must not affect the DAG assert dag.waiting_jobs[c] == {a} - - -class TestSkipSatisfied: - """Test the pre-run resume prune""" - - def test_satisfied_root_is_marked_finished(self): - dag = DAG() - job = _job("a") - dag.add_job(job) - skipped = dag.skip_satisfied(lambda j: True) - assert skipped == [job] - assert job in dag.finished_jobs - assert len(dag.ready_jobs) == 0 - - def test_unsatisfied_root_stays_ready(self): - dag = DAG() - job = _job("a") - dag.add_job(job) - skipped = dag.skip_satisfied(lambda j: False) - assert skipped == [] - assert job in dag.ready_jobs - assert dag.finished_jobs == [] - - def test_cascades_down_a_chain_in_topological_order(self): - dag = DAG() - a, b, c = _job("a"), _job("b"), _job("c") - dag.add_job(a) - dag.add_job(b, {a}) - dag.add_job(c, {b}) - skipped = dag.skip_satisfied(lambda j: True) - assert skipped == [a, b, c] - assert dag.finished_jobs == [a, b, c] - assert len(dag.ready_jobs) == 0 - assert len(dag.waiting_jobs) == 0 - - def test_stops_at_the_first_unsatisfied_job(self): - dag = DAG() - a, b, c = _job("a"), _job("b"), _job("c") - dag.add_job(a) - dag.add_job(b, {a}) - dag.add_job(c, {b}) - skipped = dag.skip_satisfied(lambda j: j == a) - assert skipped == [a] - assert b in dag.ready_jobs - assert c in dag.waiting_jobs - - def test_blocked_jobs_are_not_offered_to_the_predicate(self): - dag = DAG() - a, b, c = _job("a"), _job("b"), _job("c") - dag.add_job(a) - dag.add_job(b, {a}) - dag.add_job(c, {b}) - seen = [] - - def predicate(job): - seen.append(job) - return job == a - - dag.skip_satisfied(predicate) - # b is offered (a was skipped, unblocking it); c never is -- the - # ancestor rule: a job behind an unsatisfied job cannot be skipped. - assert seen == [a, b] - - def test_diamond_with_partial_parents(self): - dag = DAG() - top = _job("top") - left, right = _job("left"), _job("right") - bottom = _job("bottom") - dag.add_job(top) - dag.add_job(left, {top}) - dag.add_job(right, {top}) - dag.add_job(bottom, {left, right}) - skipped = dag.skip_satisfied(lambda j: j in (top, left)) - assert skipped == [top, left] - assert right in dag.ready_jobs - assert dag.waiting_jobs[bottom] == {right} - - def test_empty_dag_returns_empty(self): - assert DAG().skip_satisfied(lambda j: True) == [] - - -class TestHasAllOutputs: - """Test the default resume predicate""" - - def test_no_declared_outputs_is_never_satisfied(self): - assert has_all_outputs(_job("a")) is False - - def test_true_when_all_outputs_exist(self, tmp_path): - full = tmp_path / "full.txt" - full.write_text("data") - empty = tmp_path / "empty.txt" - empty.touch() # zero-byte files count as present - job = Job(Pipeline(Command("true")), "j", outputs=[full, empty]) - assert has_all_outputs(job) is True - - def test_false_when_any_output_is_missing(self, tmp_path): - present = tmp_path / "present.txt" - present.touch() - missing = tmp_path / "missing.txt" - job = Job(Pipeline(Command("true")), "j", outputs=[present, missing]) - assert has_all_outputs(job) is False - - def test_dangling_symlink_counts_as_missing(self, tmp_path): - target = tmp_path / "target.txt" - target.touch() - link = tmp_path / "link.txt" - link.symlink_to(target) - target.unlink() # now dangling - job = Job(Pipeline(Command("true")), "j", outputs=[link]) - assert has_all_outputs(job) is False diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py deleted file mode 100644 index 4f46942..0000000 --- a/tests/unit/test_job.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Unit tests for the Job model. -""" - -import os -import pathlib -import sys - -# Add the parent directory to the path so we can import sentieon_cli -sys.path.insert( - 0, - os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), -) - -from sentieon_cli.job import Job # noqa: E402 -from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 - - -def _pipe(text): - return Pipeline(Command("echo", text)) - - -def test_new_fields_default_empty(): - job = Job(_pipe("a"), "a") - assert job.inputs == [] - assert job.outputs == [] - assert job.image is None - - -def test_new_fields_are_stored(): - ins = [pathlib.Path("in1"), pathlib.Path("in2")] - outs = [pathlib.Path("out")] - job = Job(_pipe("a"), "a", inputs=ins, outputs=outs, image="org/img:1") - assert job.inputs == ins - assert job.outputs == outs - assert job.image == "org/img:1" - - -def test_input_output_sequences_are_snapshotted(): - ins = [pathlib.Path("in1")] - job = Job(_pipe("a"), "a", inputs=ins) - ins.append(pathlib.Path("in2")) # mutating the caller's list... - assert job.inputs == [pathlib.Path("in1")] # ...doesn't change the job - - -def test_identity_ignores_metadata_fields(): - # Identity is keyed on the shell pipeline; the other fields don't - # affect it. - j1 = Job(_pipe("same"), "j1", inputs=[pathlib.Path("x")], image="a") - j2 = Job(_pipe("same"), "j2", outputs=[pathlib.Path("y")], image="b") - assert j1 == j2 - assert hash(j1) == hash(j2) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py deleted file mode 100644 index 800623f..0000000 --- a/tests/unit/test_storage.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Unit tests for the storage providers. -""" - -import os -import sys - -import pytest - -# Add the parent directory to the path so we can import sentieon_cli -sys.path.insert( - 0, - os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), -) - -from sentieon_cli.storage import LocalStorageProvider # noqa: E402 - - -@pytest.mark.asyncio -async def test_stage_in_present_file_ok(tmp_path): - f = tmp_path / "in.txt" - f.write_text("data") - await LocalStorageProvider().stage_in(f) # no raise - - -@pytest.mark.asyncio -async def test_stage_in_empty_file_ok(tmp_path): - f = tmp_path / "in.txt" - f.touch() # a zero-byte file still counts as present - await LocalStorageProvider().stage_in(f) - - -@pytest.mark.asyncio -async def test_stage_in_missing_file_raises(tmp_path): - with pytest.raises(FileNotFoundError): - await LocalStorageProvider().stage_in(tmp_path / "nope.txt") - - -@pytest.mark.asyncio -async def test_stage_out_present_file_ok(tmp_path): - f = tmp_path / "out.txt" - f.touch() - await LocalStorageProvider().stage_out(f) - - -@pytest.mark.asyncio -async def test_stage_out_missing_file_raises(tmp_path): - with pytest.raises(FileNotFoundError): - await LocalStorageProvider().stage_out(tmp_path / "nope.txt")