diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index 4bdebde..2595fe1 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -18,6 +18,11 @@ logger = get_logger(__name__) +# Requested OS pipe-buffer size (bytes) for throughput-bound alignment +# pipelines. Applied best-effort: the kernel caps unprivileged requests at +# fs.pipe-max-size and non-Linux platforms keep their default size. +ALN_PIPE_SIZE = 268435456 + def cmd_fai_to_bed( fai: pathlib.Path, @@ -596,9 +601,6 @@ def cmd_samtools_fastq_bwa( if input_ref: ref_cmd = ["--reference", str(reference)] - pipebuf_cmd = Command( - "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" - ) if collate: collate_cmd = Command( "samtools", @@ -667,20 +669,18 @@ def cmd_samtools_fastq_bwa( ) if collate_cmd: return Pipeline( - pipebuf_cmd, collate_cmd, fastq_cmd, bwa_cmd, sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) else: return Pipeline( - pipebuf_cmd, fastq_cmd, bwa_cmd, sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) @@ -746,21 +746,10 @@ def cmd_fastq_bwa( split: Optional[str] = None, ) -> Pipeline: """Align an input fastq file with bwa""" - pipebuf_cmd = Command( - "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" - ) - r1_unzip = Pipeline( - pipebuf_cmd, - Command(str(unzip), "-dc", str(r1)), - skip_pipe=(0,), - ) + r1_unzip = Pipeline(Command(str(unzip), "-dc", str(r1))) r2_unzip = None if r2: - r2_unzip = Pipeline( - pipebuf_cmd, - Command(str(unzip), "-dc", str(r2)), - skip_pipe=(0,), - ) + r2_unzip = Pipeline(Command(str(unzip), "-dc", str(r2))) bwa_cmd_args: List[Union[str, InputProcSub]] = [ *(["taskset", "-c", numa] if numa else []), "sentieon", @@ -780,7 +769,6 @@ def cmd_fastq_bwa( ] if split: extract_cmd = Pipeline( - pipebuf_cmd, Command( "sentieon", "fqidx", @@ -792,7 +780,6 @@ def cmd_fastq_bwa( InputProcSub(r1_unzip), *([InputProcSub(r2_unzip)] if r2_unzip else []), ), - skip_pipe=(0,), ) bwa_cmd_args.append(InputProcSub(extract_cmd)) else: @@ -816,10 +803,9 @@ def cmd_fastq_bwa( *util_sort_args.split(), ) return Pipeline( - pipebuf_cmd, Command(str(bwa_cmd_args[0]), *bwa_cmd_args[1:]), sort_cmd, - skip_pipe=(0,), + pipe_size=ALN_PIPE_SIZE, ) diff --git a/sentieon_cli/dag.py b/sentieon_cli/dag.py index b3997b6..d40419d 100644 --- a/sentieon_cli/dag.py +++ b/sentieon_cli/dag.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import Dict, Generator, List, Optional, Set +from typing import Dict, Iterable, List, Optional, Set from .exceptions import DagExecutionError from .job import Job @@ -25,54 +25,61 @@ def __init__(self) -> None: self.ready_jobs: Dict[Job, None] = {} self.finished_jobs: List[Job] = [] - def add_job(self, job: Job, dependencies: Optional[Set[Job]] = None): - """Add a job to the DAG""" - if dependencies: - for dependency in dependencies: - if ( - dependency not in self.waiting_jobs - and dependency not in self.ready_jobs - ): - raise ValueError( - f"Dependency '{dependency}' is not in the DAG" - ) + def add_job( + self, job: Job, dependencies: Optional[Iterable[Job]] = None + ) -> None: + """Add a job to the DAG. - if isinstance(dependencies, set) and len(dependencies) > 0: - self.waiting_jobs[job] = dependencies.copy() - for dependency in dependencies: + ``dependencies`` may be any iterable of jobs already in the DAG; + it is consumed once and stored as a private set. + """ + deps: Set[Job] = ( + set(dependencies) if dependencies is not None else set() + ) + if job in self.waiting_jobs or job in self.ready_jobs: + raise ValueError( + f"{job} is already in the DAG; jobs are identified by their " + f"shell pipeline, so the same pipeline cannot be added twice." + ) + for dependency in deps: + if ( + dependency not in self.waiting_jobs + and dependency not in self.ready_jobs + ): + raise ValueError( + f"Dependency '{dependency}' is not in the DAG" + ) + + if deps: + self.waiting_jobs[job] = deps + for dependency in deps: downstream = self.planned_jobs.setdefault(dependency, []) downstream.append(job) else: self.ready_jobs[job] = None - def update_dag( - self, - ) -> Generator[Dict[Job, None], Optional[Job], None]: - """Update the DAG with a finished job""" - finished_job = yield self.ready_jobs.copy() + def mark_finished(self, job: Job) -> Dict[Job, None]: + """Record a finished job and return the jobs it unblocked. - while True: - logger.debug("Waiting jobs: %s", self.waiting_jobs) - logger.debug("Planned jobs: %s", self.planned_jobs) - logger.debug("Ready jobs: %s", self.ready_jobs) - logger.debug("Finished jobs: %s", self.finished_jobs) - logger.debug("Newly finished: %s", finished_job) + The returned mapping is used as an ordered set (values are ``None``). + Raises :class:`~sentieon_cli.exceptions.DagExecutionError` if ``job`` + was not currently ready. + """ + if job not in self.ready_jobs: + raise DagExecutionError( + f"Finished job '{job}' was not ready for execution" + ) - new_ready_jobs: Dict[Job, None] = {} - if isinstance(finished_job, Job): - if finished_job not in self.ready_jobs: - raise DagExecutionError( - f"Finished job '{finished_job}' was not ready for " - "execution" - ) + logger.debug("Marking finished: %s", job) + del self.ready_jobs[job] + self.finished_jobs.append(job) - del self.ready_jobs[finished_job] - self.finished_jobs.append(finished_job) - for dependency in self.planned_jobs.get(finished_job, []): - upstream = self.waiting_jobs[dependency] - upstream.remove(finished_job) - if len(upstream) < 1: - self.ready_jobs[dependency] = None - new_ready_jobs[dependency] = None - del self.waiting_jobs[dependency] - finished_job = yield new_ready_jobs + new_ready_jobs: Dict[Job, None] = {} + for dependency in self.planned_jobs.get(job, []): + upstream = self.waiting_jobs[dependency] + upstream.remove(job) + if len(upstream) < 1: + self.ready_jobs[dependency] = None + new_ready_jobs[dependency] = None + del self.waiting_jobs[dependency] + return new_ready_jobs diff --git a/sentieon_cli/executor.py b/sentieon_cli/executor.py index 4d60f3e..66b1aaf 100644 --- a/sentieon_cli/executor.py +++ b/sentieon_cli/executor.py @@ -1,30 +1,70 @@ """Execute jobs""" import asyncio -import asyncio.subprocess +import contextlib import signal import sys +import threading import time -from typing import Any, List, Tuple +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Tuple from .job import Job from .logging import get_logger -from .scheduler import ThreadScheduler +from .scheduler import BaseScheduler from .shell_pipeline import Context logger = get_logger(__name__) -class BaseExecutor: +def _signal_procs(context: Context, sig: int) -> None: + """Send a signal to every live sub-process in a context.""" + for subcommand in context.commands: + proc = subcommand.proc + if proc is not None and proc.returncode is None: + try: + proc.send_signal(sig) + except ProcessLookupError: + pass + + +def _kill_survivors(context: Context) -> None: + """SIGKILL any sub-process that is still running.""" + for subcommand in context.commands: + proc = subcommand.proc + if proc is not None and proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + + +async def _cleanup_quietly(context: Context) -> None: + """Release a context, logging -- not raising -- a surfaced launch failure. + + A proc-sub inner command that fails to spawn surfaces its OSError from + ``context.cleanup()``'s gather. On the abort and shutdown paths the owning + job is already recorded (or the whole run is being torn down), so a stored + OSError must not escape ``execute()``; log it and release the rest. A + non-OSError stays a bug and propagates, matching ``_drive``'s per-job + handling. + """ + try: + await context.cleanup() + except OSError as exc: + logger.error("error releasing resources during teardown: %s", exc) + + +class BaseExecutor(ABC): """Execute jobs""" - def __init__(self, scheduler: ThreadScheduler): + def __init__(self, scheduler: BaseScheduler): self.scheduler = scheduler self.jobs_with_errors: List[Job] = [] + @abstractmethod def execute(self) -> None: """Execute jobs from the DAG""" - raise NotImplementedError class DryRunExecutor(BaseExecutor): @@ -35,8 +75,7 @@ def run_job(self, job: Job) -> None: print(job.shell) def execute(self) -> None: - scheduler_gen = self.scheduler.schedule() - ready_jobs = scheduler_gen.send(None) + ready_jobs = self.scheduler.start() for job in ready_jobs: self.run_job(job) @@ -45,18 +84,130 @@ def execute(self) -> None: ready_jobs = { new_job for completed_job in finished_jobs - for new_job in scheduler_gen.send(completed_job) + for new_job in self.scheduler.job_finished(completed_job) } for job in ready_jobs: self.run_job(job) -class LocalExecutor(BaseExecutor): - """Run jobs locally""" +class AsyncExecutor(BaseExecutor, ABC): + """Base for executors that run an asyncio event loop. + + Provides the ``execute()`` entry point and opt-in SIGINT/SIGTERM handling + with graceful shutdown; subclasses implement :meth:`_drive` (the run loop) + and :meth:`_shutdown` (teardown after an interrupt). + + Set ``install_signal_handlers=True`` to install handlers (on the main + thread only) for the duration of a run; the previous handlers are + restored afterwards, so a caller's signal state is left untouched. + """ - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) + def __init__( + self, + scheduler: BaseScheduler, + *, + install_signal_handlers: bool = False, + ) -> None: + super().__init__(scheduler) + self.install_signal_handlers = install_signal_handlers self.start_new_jobs = True + + def _install_signal_handlers( + self, + loop: asyncio.AbstractEventLoop, + stop_event: asyncio.Event, + ) -> Callable[[], None]: + """Install SIGINT/SIGTERM handlers on the running loop. + + Returns a callable that removes the handlers and restores the + previous ones. A no-op is returned when handlers are disabled, or + when we are not on the main thread (Python can only manage signals + there, so we skip rather than crash). + """ + if not self.install_signal_handlers: + return lambda: None + if threading.current_thread() is not threading.main_thread(): + logger.debug("Signal handlers require the main thread; skipping.") + return lambda: None + + def on_signal(signum: int) -> None: + logger.error( + "Received %s; terminating jobs.", + signal.Signals(signum).name, + ) + self.start_new_jobs = False + stop_event.set() + + signums = (signal.SIGINT, signal.SIGTERM) + previous: Dict[int, Any] = { + signum: signal.getsignal(signum) for signum in signums + } + installed: List[int] = [] + for signum in signums: + try: + loop.add_signal_handler(signum, on_signal, signum) + installed.append(signum) + except (NotImplementedError, RuntimeError, ValueError) as exc: + logger.debug( + "Could not install handler for %s: %s", signum, exc + ) + + def restore() -> None: + for signum in installed: + try: + loop.remove_signal_handler(signum) + except (RuntimeError, ValueError): + pass + handler = previous[signum] + if handler is not None: + try: + signal.signal(signum, handler) + except (OSError, ValueError, TypeError): + pass + + return restore + + def execute(self) -> None: + """Execute jobs from the DAG""" + asyncio.run(self._execute()) + + async def _execute(self) -> None: + """Execute jobs from the DAG""" + self.jobs_with_errors = [] + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + restore = self._install_signal_handlers(loop, stop_event) + try: + await self._drive(stop_event) + if stop_event.is_set(): + await self._shutdown() + finally: + restore() + + @abstractmethod + async def _drive(self, stop_event: asyncio.Event) -> None: + """Submit/run jobs until the DAG drains or a stop is requested.""" + + @abstractmethod + async def _shutdown(self) -> None: + """Tear down whatever is still running after a stop request.""" + + +class LocalExecutor(AsyncExecutor): + """Run jobs locally as async subprocesses.""" + + def __init__( + self, + scheduler: BaseScheduler, + *, + install_signal_handlers: bool = False, + shutdown_grace_period: float = 10.0, + ) -> None: + super().__init__( + scheduler, + install_signal_handlers=install_signal_handlers, + ) + self.shutdown_grace_period = shutdown_grace_period self.running: List[ Tuple[ Job, @@ -65,26 +216,6 @@ def __init__(self, *args: Any, **kwargs: Any): int, ] ] = [] - signal.signal(signal.SIGINT, self.exit_gracefully) - signal.signal(signal.SIGTERM, self.exit_gracefully) - - def exit_gracefully(self, signum, frame) -> None: - logger.error("Termination signal detected. Terminating jobs.") - self.start_new_jobs = False - for running_job in self.running: - for subcommand in running_job[1].commands: - if not subcommand.proc: - continue - subcommand.proc.send_signal(signal=signal.SIGTERM) - time.sleep(10) - for running_job in self.running: - context = running_job[1] - for subcommand in context.commands: - if not subcommand.proc: - continue - if subcommand.proc.returncode is None: - subcommand.proc.kill() - asyncio.run(context.cleanup()) async def run_job(self, job: Job) -> None: """Run a job""" @@ -105,85 +236,178 @@ async def run_job(self, job: Job) -> None: start_time, ) ) - except Exception as e: + except OSError as e: + # OSError covers the expected launch failures (command not found, + # permission denied, pipe/FIFO errors). Anything else is a bug in + # the caller's pipeline and is left to propagate. logger.error("failed to start command: %s", job.shell) logger.error("Error: %s", str(e)) - await context.cleanup() + self.jobs_with_errors.append(job) + self.start_new_jobs = False + # A later pipeline stage can fail to spawn after earlier stages + # are already running; terminate them so they do not outlive the + # workflow, then release the remaining resources. A proc-sub inner + # launch failure can also surface from cleanup() here (narrow + # race): the job is already recorded, so swallow it. + await self._terminate_context(context) + await _cleanup_quietly(context) - def execute(self) -> None: - """Execute jobs from the DAG""" - asyncio.run(self._execute()) + async def _terminate_context(self, context: Context) -> None: + """Tear down processes already spawned in an aborted context. - async def _execute(self) -> None: - """Execute jobs from the DAG""" - self.jobs_with_errors: List[Job] = [] - scheduler_gen = self.scheduler.schedule() - ready_jobs = scheduler_gen.send(None) + Used when a job aborts mid-launch (a later pipeline stage failed to + spawn): earlier stages are already running and would otherwise be + leaked. SIGTERM them, allow a grace period, then SIGKILL any + survivors, waiting for each to be reaped. + """ + live = [ + subcommand.proc + for subcommand in context.commands + if subcommand.proc is not None + and subcommand.proc.returncode is None + ] + if not live: + return + _signal_procs(context, signal.SIGTERM) + waits = [asyncio.create_task(proc.wait()) for proc in live] + await asyncio.wait(waits, timeout=self.shutdown_grace_period) + _kill_survivors(context) + await asyncio.wait(waits) + + async def _drive(self, stop_event: asyncio.Event) -> None: + """Schedule and run jobs until the DAG drains or we are stopped.""" + ready_jobs = self.scheduler.start() for job in ready_jobs: + if not self.start_new_jobs: + # A job in this batch failed to launch; drain, don't start + # the rest -- matching the policy the main loop applies. + break await self.run_job(job) - while self.running: - done, _running = await asyncio.wait( - [job[2] for job in self.running], - return_when=asyncio.FIRST_COMPLETED, - ) + stop_task = asyncio.ensure_future(stop_event.wait()) + try: + while self.running and not stop_event.is_set(): + wait_on: List[Any] = [job[2] for job in self.running] + wait_on.append(stop_task) + done, _pending = await asyncio.wait( + wait_on, + return_when=asyncio.FIRST_COMPLETED, + ) - finished_jobs = [ - self.running.pop(i) - for i in reversed(range(len(self.running))) - if self.running[i][2] in done - ] + if stop_event.is_set(): + break - # Check job execution - for job, context, _task, start_time in finished_jobs: - end_time = time.monotonic_ns() - total_seconds = (end_time - start_time) / 1e9 - - # Check if the command failed - cmd_failed = False - for subcommand in context.commands: - if not subcommand.proc: - logger.error( - "Sub-command has no process: %s", - subcommand, - ) - cmd_failed = True - continue - ret = ( - await subcommand.proc.wait() - ) # Wait on all sub-commands - if ret != 0 and not subcommand.fail_ok: - logger.error( - f"Return code {ret} for sub-command: {subcommand}" - ) - cmd_failed = True - if ret == -9: + finished_jobs = [ + self.running.pop(i) + for i in reversed(range(len(self.running))) + if self.running[i][2] in done + ] + + # Check job execution + for job, context, _task, start_time in finished_jobs: + end_time = time.monotonic_ns() + total_seconds = (end_time - start_time) / 1e9 + + # Check if the command failed + cmd_failed = False + for subcommand in context.commands: + if not subcommand.proc: + logger.error( + "Sub-command has no process: %s", + subcommand, + ) + cmd_failed = True + continue + ret = ( + await subcommand.proc.wait() + ) # Wait on all sub-commands + # A subcommand killed by SIGPIPE is not a failure: + # it was writing to a pipe whose reader exited early + # (e.g. `... | head`), which is normal in default + # (non-pipefail) bash. Real crashes surface as other + # non-zero codes and still fail the job. + if ret == -signal.SIGPIPE: + logger.debug( + "Sub-command exited on SIGPIPE (ok): %s", + subcommand, + ) + elif ret != 0 and not subcommand.fail_ok: logger.error( - "Sub-command received SIGKILL. " - "Possible out-of-memory?" + f"Return code {ret} for sub-command: " + f"{subcommand}" ) + cmd_failed = True + if ret == -9: + logger.error( + "Sub-command received SIGKILL. " + "Possible out-of-memory?" + ) - if cmd_failed: - logger.error("Command failure: %s", job.shell) - self.jobs_with_errors.append(job) - self.start_new_jobs = False - else: - logger.info( - f"Finished command in " - f"{total_seconds:.2f}: {job.shell}" + # Release resources before recording the verdict. A + # proc-sub inner command that failed to launch surfaces + # its OSError from the background task here; treat it as + # a launch failure for this job rather than letting it + # abort the run while siblings are still mid-flight. + try: + await context.cleanup() + except OSError as exc: + logger.error("failed to start command: %s", job.shell) + logger.error("Error: %s", str(exc)) + cmd_failed = True + + if cmd_failed: + logger.error("Command failure: %s", job.shell) + self.jobs_with_errors.append(job) + self.start_new_jobs = False + else: + logger.info( + f"Finished command in " + f"{total_seconds:.2f}: {job.shell}" + ) + + if not self.start_new_jobs: + # Don't start new jobs + continue + + ready_jobs = { + new_job + for completed_job in finished_jobs + for new_job in self.scheduler.job_finished( + completed_job[0] ) - await context.cleanup() + } - if not self.start_new_jobs: - # Don't start new jobs - continue + # Run the ready jobs + for job in ready_jobs: + if not self.start_new_jobs: + # A job in this batch failed to launch; drain the + # rest of the run without starting more. + break + await self.run_job(job) + finally: + stop_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await stop_task - ready_jobs = { - new_job - for completed_job in finished_jobs - for new_job in scheduler_gen.send(completed_job[0]) - } + async def _shutdown(self) -> None: + """Terminate running jobs, escalate to SIGKILL, then clean up.""" + running = self.running + self.running = [] - # Run the ready jobs - for job in ready_jobs: - await self.run_job(job) + for _job, context, _task, _start in running: + _signal_procs(context, signal.SIGTERM) + + tasks = [tup[2] for tup in running] + if tasks: + await asyncio.wait(tasks, timeout=self.shutdown_grace_period) + + for _job, context, _task, _start in running: + _kill_survivors(context) + if tasks: + await asyncio.wait(tasks) + + for _job, context, _task, _start in running: + # A proc-sub inner launch failure can surface from cleanup() as we + # tear down after an interrupt; log it rather than let it escape, + # and keep releasing the remaining contexts. + await _cleanup_quietly(context) diff --git a/sentieon_cli/job.py b/sentieon_cli/job.py index dea8343..3e36fa2 100644 --- a/sentieon_cli/job.py +++ b/sentieon_cli/job.py @@ -2,19 +2,24 @@ Job objects """ -import asyncio -import sys -import time from typing import Dict, Optional -from .logging import get_logger -from .shell_pipeline import Context, Pipeline - -logger = get_logger(__name__) +from .shell_pipeline import Pipeline class Job: - """A job for execution""" + """A unit of work: a shell pipeline plus its execution metadata. + + Fields: + + * ``shell`` -- the pipeline to run. + * ``name`` -- a human-readable label (not part of identity). + * ``threads`` -- CPU threads the job needs (a local scheduling budget). + * ``resources`` -- named resource counts (e.g. NUMA-node tokens). + + A job's identity is keyed only on ``shell``: two jobs with the same + pipeline are equal (and collide in a DAG) regardless of the other fields. + """ def __init__( self, @@ -22,53 +27,25 @@ def __init__( name: str, threads: int = 1, resources: Optional[Dict[str, int]] = None, - ): + ) -> None: self.shell = pipeline self.name = name self.threads = threads self.resources = {} if resources is None else resources - def __hash__(self): + def __hash__(self) -> int: return hash(self.shell) - def __eq__(self, other: object): + def __eq__(self, other: object) -> bool: if isinstance(other, Job): return self.shell == other.shell return False - def __ne__(self, other: object): + def __ne__(self, other: object) -> bool: return not self == other - def __repr__(self): + def __repr__(self) -> str: return f"Job({self.name})" - def __str__(self): + def __str__(self) -> str: return f"Job({self.name})" - - def run(self, dry_run: bool = False): - """Run a command""" - if dry_run: - print(self.shell) - return - - logger.info("running command: %s", self.shell) - t0 = time.time() - context = Context() - try: - asyncio.run( - self.shell.run(context, stdout=sys.stdout, stderr=sys.stderr) - ) - for subcommand in context.commands: - if not subcommand.proc: - logger.error("subcommand has no process: %s", subcommand) - continue - ret = asyncio.run(subcommand.proc.wait()) - if ret != 0 and not subcommand.fail_ok: - logger.error( - f"subcommand failed with code {ret}: {subcommand}" - ) - except Exception as e: - logger.error(f"Failed to run command: {e}") - finally: - asyncio.run(context.cleanup()) - logger.info("finished in: %s seconds", f"{time.time() - t0:.1f}") diff --git a/sentieon_cli/logging.py b/sentieon_cli/logging.py index bd5d15a..faca6d0 100644 --- a/sentieon_cli/logging.py +++ b/sentieon_cli/logging.py @@ -4,9 +4,14 @@ handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s")) -def get_logger(name: str): +def get_logger(name: str) -> logging.Logger: """Return a logger with a StreamHandler.""" logger = logging.getLogger(name) logger.addHandler(handler) logger.propagate = False return logger + + +def set_level(level: int | str) -> None: + """Set the level of the package logger.""" + logging.getLogger(__package__).setLevel(level) diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 39672c1..be82e0d 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -15,9 +15,10 @@ from . import command_strings as cmds from .dag import DAG +from .exceptions import DagExecutionError from .executor import BaseExecutor, DryRunExecutor, LocalExecutor from .job import Job -from .logging import get_logger +from .logging import get_logger, set_level from .scheduler import ThreadScheduler from .util import __version__, check_version, path_arg, tmp @@ -71,17 +72,17 @@ class BasePipeline(ABC): @classmethod def add_arguments(cls, parser: argparse.ArgumentParser): - for k, kwargs in cls.params.items(): - flags = ["--" + k] + # Build a fresh kwargs dict per argument so repeated calls do not + # mutate the shared class-level params/positionals specs. + for k, spec in cls.params.items(): + kwargs = dict(spec) + flags = kwargs.pop("flags", ["--" + k]) if "default" in kwargs and "type" not in kwargs: kwargs["type"] = type(kwargs["default"]) - if "flags" in kwargs: - flags = kwargs["flags"] - del kwargs["flags"] parser.add_argument(*flags, **kwargs) - for k, kwargs in cls.positionals.items(): - parser.add_argument(k, **kwargs) + for k, spec in cls.positionals.items(): + parser.add_argument(k, **dict(spec)) def handle_arguments(self, args: argparse.Namespace): """Update self using the argparse object""" @@ -106,9 +107,7 @@ def handle_arguments(self, args: argparse.Namespace): def setup_logging(self, args: argparse.Namespace) -> None: self.logger = get_logger(__name__) - if not self.logger.parent: - raise RuntimeError("Logger has no parent logger") - self.logger.parent.setLevel(args.loglevel) + set_level(args.loglevel) self.logger.info("Starting sentieon-cli version: %s", __version__) def __init__(self) -> None: @@ -130,13 +129,13 @@ def main(self, args: argparse.Namespace) -> None: tmp_dir_str = tmp() self.tmp_dir = pathlib.Path(tmp_dir_str) - dag = self.build_dag() - executor = self.run(dag) - - if not self.retain_tmpdir: - shutil.rmtree(tmp_dir_str) - - self.check_execution(dag, executor) + try: + dag = self.build_dag() + executor = self.run(dag) + self.check_execution(dag, executor) + finally: + if not self.retain_tmpdir: + shutil.rmtree(tmp_dir_str) def check_execution( self, @@ -145,10 +144,11 @@ def check_execution( ): """Check the DAG and executor after a run""" if executor.jobs_with_errors: - raise ValueError("Execution failed") + failed = ", ".join(str(job) for job in executor.jobs_with_errors) + raise DagExecutionError(f"Execution failed for jobs: {failed}") if len(dag.waiting_jobs) > 0 or len(dag.ready_jobs) > 0: - raise ValueError( + raise DagExecutionError( "The DAG has some unexecuted jobs\n" f"Waiting jobs: {dag.waiting_jobs}\n" f"Ready jobs: {dag.ready_jobs}\n" @@ -250,8 +250,13 @@ def run(self, dag: DAG) -> BaseExecutor: ) self.logger.debug("Creating the executor") - Executor = DryRunExecutor if self.dry_run else LocalExecutor - executor = Executor(scheduler) + executor: BaseExecutor + if self.dry_run: + executor = DryRunExecutor(scheduler) + else: + # Handle Ctrl-C/SIGTERM by terminating running jobs gracefully; + # the handlers are installed only for the duration of the run. + executor = LocalExecutor(scheduler, install_signal_handlers=True) self.logger.info("Starting execution") executor.execute() diff --git a/sentieon_cli/scheduler.py b/sentieon_cli/scheduler.py index ffc0bb8..4ed5329 100644 --- a/sentieon_cli/scheduler.py +++ b/sentieon_cli/scheduler.py @@ -1,15 +1,48 @@ """Schedule jobs""" -from typing import Dict, Generator, Optional, Set +from abc import ABC, abstractmethod +from typing import Dict, Optional, Set from .dag import DAG +from .exceptions import DagExecutionError from .job import Job from .logging import get_logger logger = get_logger(__name__) -class ThreadScheduler: +class BaseScheduler(ABC): + """Selects which ready jobs an executor may run, subject to a policy. + + An executor drives a scheduler with two calls: + + 1. :meth:`start` once, to get the first batch of jobs to run. + 2. :meth:`job_finished` for each job as it completes, to get the jobs + that its completion unblocked (possibly an empty set). + + Contract / invariants: + + * :meth:`start` is called exactly once, before any :meth:`job_finished`. + * Every job passed to :meth:`job_finished` was previously returned by + :meth:`start` or :meth:`job_finished`, and is reported at most once. + * A scheduler instance drives a single run; do not reuse it. + * :meth:`start` may raise + :class:`~sentieon_cli.exceptions.DagExecutionError` if the DAG + contains a job that can never be scheduled. + * The executor stops when it has no running jobs and the scheduler + returns no new ones. + """ + + @abstractmethod + def start(self) -> Set[Job]: + """Return the first batch of jobs to run.""" + + @abstractmethod + def job_finished(self, job: Job) -> Set[Job]: + """Record a finished job and return the jobs it unblocked.""" + + +class ThreadScheduler(BaseScheduler): """Schedule jobs as threads are available""" def __init__( @@ -17,41 +50,65 @@ def __init__( dag: DAG, threads: int = 1, resources: Optional[Dict[str, int]] = None, - ): + ) -> None: self.dag = dag self.threads = threads self.available_threads = threads self.resources = {} if resources is None else resources + # Jobs that are ready to run but have not yet been scheduled. + self._ready: Dict[Job, None] = {} + + def _check_feasible(self) -> None: + """Ensure every job can eventually be scheduled. + + A job that requests more threads than the total budget, or more of a + managed resource than exists, can never be released and would + otherwise stall the run silently. + """ + jobs = list(self.dag.waiting_jobs) + list(self.dag.ready_jobs) + for job in jobs: + if job.threads > self.threads: + raise DagExecutionError( + f"{job} requests {job.threads} threads but the scheduler " + f"budget is only {self.threads}" + ) + for resource, needed in job.resources.items(): + capacity = self.resources.get(resource) + if capacity is not None and needed > capacity: + raise DagExecutionError( + f"{job} requests {needed} of resource '{resource}' " + f"but the scheduler capacity is only {capacity}" + ) + + def _fit(self) -> Set[Job]: + """Reserve budget for as many ready jobs as fit; return that set.""" + logger.debug("Ready pool: %s", self._ready) - def schedule(self) -> Generator[Set[Job], Optional[Job], None]: - """Schedule a job for execution""" - dag_gen = self.dag.update_dag() - ready_jobs = dag_gen.send(None) - - while True: - logger.debug("Ready jobs: %s", ready_jobs) - - scheduled_jobs: Set[Job] = set() - for ready_job in ready_jobs: - if self.available_threads - ready_job.threads >= 0 and all( - [ - available - ready_job.resources.get(resource, 0) >= 0 - for resource, available in self.resources.items() - ] - ): - self.available_threads -= ready_job.threads - for resource, used in ready_job.resources.items(): - if resource in self.resources: - self.resources[resource] -= used - scheduled_jobs.add(ready_job) - - for job in scheduled_jobs: - del ready_jobs[job] - - finished_job = yield scheduled_jobs - if isinstance(finished_job, Job): - self.available_threads += finished_job.threads - for resource, used in finished_job.resources.items(): + scheduled_jobs: Set[Job] = set() + for ready_job in self._ready: + if self.available_threads - ready_job.threads >= 0 and all( + available - ready_job.resources.get(resource, 0) >= 0 + for resource, available in self.resources.items() + ): + self.available_threads -= ready_job.threads + for resource, used in ready_job.resources.items(): if resource in self.resources: - self.resources[resource] += used - ready_jobs.update(dag_gen.send(finished_job)) + self.resources[resource] -= used + scheduled_jobs.add(ready_job) + + for job in scheduled_jobs: + del self._ready[job] + return scheduled_jobs + + def start(self) -> Set[Job]: + self._check_feasible() + self._ready = dict(self.dag.ready_jobs) + return self._fit() + + def job_finished(self, job: Job) -> Set[Job]: + self.available_threads += job.threads + for resource, used in job.resources.items(): + if resource in self.resources: + self.resources[resource] += used + self._ready.update(self.dag.mark_finished(job)) + return self._fit() diff --git a/sentieon_cli/shell_pipeline.py b/sentieon_cli/shell_pipeline.py index 02eb082..1d6d26d 100644 --- a/sentieon_cli/shell_pipeline.py +++ b/sentieon_cli/shell_pipeline.py @@ -1,21 +1,66 @@ -"""Complex shell pipelines with process substitution""" +"""Complex shell pipelines with process substitution. + +POSIX only: pipelines are wired together with OS pipes and FIFOs, and process +substitution relies on named pipes, so this module does not work on Windows. +""" from __future__ import annotations import asyncio +import dataclasses +import fcntl import io import os import pathlib import tempfile import shlex -import sys from abc import ABC, abstractmethod -from typing import Any, Dict, IO, Iterable, List, Optional, Union +from typing import Any, Dict, IO, List, Optional, Union from .logging import get_logger logger = get_logger(__name__) +def _freeze(value: Any) -> Any: + """Return an order-independent, hashable view of ``value``.""" + if isinstance(value, dict): + return frozenset((k, _freeze(v)) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return tuple(_freeze(v) for v in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(v) for v in value) + return value + + +def _set_pipe_size(fd: int, size: int) -> None: + """Best-effort enlarge the OS pipe buffer backing ``fd``. + + Lets a throughput-bound pipeline hold more in-flight data between + stages. + """ + set_pipe_sz = getattr(fcntl, "F_SETPIPE_SZ", None) + if set_pipe_sz is None: + return + try: + fcntl.fcntl(fd, set_pipe_sz, size) + except OSError as exc: + logger.warning( + "could not set pipe size to %d (%s); using the default. " + "Raise fs.pipe-max-size to allow larger buffers.", + size, + exc, + ) + + +@dataclasses.dataclass +class _ProcSubWait: + """A proc-sub FIFO whose blocking open may need unblocking.""" + + fifo_path: str + opened: bool = False # run_inner's open() returned (set pre-await) + unblocked: bool = False # cleanup already opened both ends + + class Context: """Holds shared resources.""" @@ -23,11 +68,15 @@ def __init__(self) -> None: # This directory is already unique per pipeline run self.temp_dir = tempfile.TemporaryDirectory() # Background tasks to wait for (e.g., proc sub processes) - self.tasks: List[asyncio.Task] = [] + self.tasks: List[asyncio.Task[Any]] = [] # Hold the commands run in this context self.commands: List[Command] = [] self._counter = 0 # Monotonic counter self.file_handles: List[io.IOBase] = [] + # Set by cleanup(); a proc sub whose FIFO open was unblocked by + # cleanup must not spawn its inner command. + self.closing: bool = False + self._procsub_waits: List[_ProcSubWait] = [] def get_new_fifo(self) -> str: """Creates a new unique FIFO and returns its path.""" @@ -38,11 +87,48 @@ def get_new_fifo(self) -> str: return path async def cleanup(self) -> None: - if self.tasks: - await asyncio.gather(*self.tasks) + """Wait for background tasks, then release all resources. + + A proc sub's blocking FIFO open() never returns if the outer + command exited without opening the other end, so cleanup first + unblocks those opens, then waits for the tasks. Task exceptions + are re-raised only after every resource has been released. + """ + self.closing = True + unblock_fds: List[int] = [] + try: + while True: + for wait in self._procsub_waits: + if wait.opened or wait.unblocked: + continue + wait.unblocked = True + # Hold both ends open: this unblocks a blocked open() + # of either end, and read-then-write cannot raise + # ENXIO. + unblock_fds.append( + os.open(wait.fifo_path, os.O_RDONLY | os.O_NONBLOCK) + ) + unblock_fds.append( + os.open(wait.fifo_path, os.O_WRONLY | os.O_NONBLOCK) + ) + # Tasks can register new proc subs (nested pipelines), so + # re-snapshot until everything is done. + pending = [t for t in self.tasks if not t.done()] + if not pending: + break + await asyncio.gather(*pending, return_exceptions=True) + finally: + for fd in unblock_fds: + os.close(fd) for fh in self.file_handles: fh.close() self.temp_dir.cleanup() + for task in self.tasks: + if task.cancelled(): + continue + exc = task.exception() + if exc is not None and not isinstance(exc, asyncio.CancelledError): + raise exc class ShellNode(ABC): @@ -56,8 +142,8 @@ def __init__( self, executable: str, *args: str | ProcSub, - fail_ok=False, - exec_kwargs: Optional[Dict] = None, + fail_ok: bool = False, + exec_kwargs: Optional[Dict[str, Any]] = None, ) -> None: self.executable = executable self.args = list(args) @@ -68,9 +154,9 @@ def __init__( async def run( self, context: Context, - stdin: Union[IO, int, None] = None, - stdout: Union[IO, int, None] = None, - stderr: Union[IO, int, None] = None, + stdin: Union[IO[Any], int, None] = None, + stdout: Union[IO[Any], int, None] = None, + stderr: Union[IO[Any], int, None] = None, ) -> asyncio.subprocess.Process: # 1. Resolve Arguments (Handle Process Substitutions) final_args = [self.executable] @@ -96,13 +182,28 @@ async def run( return self.proc def __str__(self) -> str: - return shlex.join([self.executable] + [str(x) for x in self.args]) + rendered = shlex.join([self.executable] + [str(x) for x in self.args]) + # Render env/cwd so the command is self-contained bash (backends + # submit str(job.shell)). Other exec_kwargs (close_fds, ...) have no + # shell rendering and are omitted. env is a conventional K=V prefix + # that merges into the inherited environment. + env = self.exec_kwargs.get("env") + if env: + assignments = " ".join( + f"{key}={shlex.quote(str(val))}" for key, val in env.items() + ) + rendered = f"{assignments} {rendered}" + cwd = self.exec_kwargs.get("cwd") + if cwd: + rendered = f"(cd {shlex.quote(str(cwd))} && {rendered})" + return rendered def __repr__(self) -> str: return ( f"{self.__class__.__name__}({self.executable}, " + ", ".join([repr(x) for x in self.args]) - + f", fail_ok={self.fail_ok})" + + f", fail_ok={self.fail_ok}, " + + f"exec_kwargs={self.exec_kwargs!r})" ) def __eq__(self, other: object) -> bool: @@ -112,11 +213,14 @@ def __eq__(self, other: object) -> bool: self.executable == other.executable and self.args == other.args and self.fail_ok == other.fail_ok - and self.proc == other.proc + and self.exec_kwargs == other.exec_kwargs ) def __hash__(self) -> int: - return hash(tuple([self.executable, self.fail_ok] + self.args)) + return hash( + tuple([self.executable, self.fail_ok] + self.args) + + (_freeze(self.exec_kwargs),) + ) class Pipeline(ShellNode): @@ -125,39 +229,38 @@ class Pipeline(ShellNode): def __init__( self, *nodes: Command, - skip_pipe: Optional[Iterable[int]] = None, + pipe_size: Optional[int] = None, file_input: Optional[pathlib.Path] = None, file_output: Optional[pathlib.Path] = None, ): self.nodes = list(nodes) if not self.nodes: raise ValueError("Pipeline nodes cannot be empty") - self.skip_pipe = set(skip_pipe) if skip_pipe else set() + # Best-effort OS pipe-buffer size (bytes) for every internal pipe; + # None keeps the kernel default. A local-execution tuning hint, so + # it is not part of pipeline identity (like Job.threads). + self.pipe_size = pipe_size self.file_input = file_input self.file_output = file_output async def run( self, context: Context, - stdin: Union[IO, int, None] = None, - stdout: Union[IO, int, None] = None, - stderr: Union[IO, int, None] = None, + stdin: Union[IO[Any], int, None] = None, + stdout: Union[IO[Any], int, None] = None, + stderr: Union[IO[Any], int, None] = None, ) -> asyncio.subprocess.Process: - # We cannot have handles from both files and to run() - if self.file_input and stdin: - logger.error( - "Pipeline %s with both file and stdin input. %s", - self, - str(stdin), + # We cannot have handles from both files and to run(). + if self.file_input is not None and stdin is not None: + raise ValueError( + f"Pipeline {self} was given both a file_input and a stdin " + f"stream ({stdin!r}); provide only one." ) - sys.exit(1) - if self.file_output and stdout: - logger.error( - "Pipeline %s with both file and stdout output. %s", - self, - str(stdout), + if self.file_output is not None and stdout is not None: + raise ValueError( + f"Pipeline {self} was given both a file_output and a stdout " + f"stream ({stdout!r}); provide only one." ) - sys.exit(1) processes = [] # We need to close these FDs in the parent after passing them to @@ -166,19 +269,18 @@ async def run( # pass the file handle as input to the pipeline current_stdin = stdin - if self.file_input: + if self.file_input is not None: current_stdin = open(self.file_input, "rb") context.file_handles.append(current_stdin) try: # Chain all commands except the last one for i in range(len(self.nodes) - 1): - read_fd = None - write_fd = None - if i not in self.skip_pipe: - # Create OS pipe - read_fd, write_fd = os.pipe() - fds_to_close.extend([read_fd, write_fd]) + # Create the OS pipe linking this command to the next. + read_fd, write_fd = os.pipe() + fds_to_close.extend([read_fd, write_fd]) + if self.pipe_size is not None: + _set_pipe_size(write_fd, self.pipe_size) # Run the node (Command or sub-Pipeline) # stdout goes to the write-end of the pipe @@ -190,16 +292,15 @@ async def run( ) processes.append(proc) - if write_fd: - # Close write_fd in parent (child has it now) - os.close(write_fd) - fds_to_close.remove(write_fd) + # Close write_fd in parent (child has it now) + os.close(write_fd) + fds_to_close.remove(write_fd) # Next command reads from the read-end current_stdin = read_fd # Send the pipeline output to a file - if self.file_output: + if self.file_output is not None: stdout = open(self.file_output, "wb") context.file_handles.append(stdout) @@ -226,13 +327,7 @@ def __str__(self) -> str: res = [] if self.file_input: res.append(f"<'{self.file_input}' ") - res.append(str(self.nodes[0])) - for i, node in enumerate(self.nodes[1:]): - if i in self.skip_pipe: - res.append("; ") - else: - res.append(" | ") - res.append(str(node)) + res.append(" | ".join(str(node) for node in self.nodes)) if self.file_output: res.append(f" >'{self.file_output}'") return "".join(res) @@ -241,26 +336,23 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" + ", ".join(repr(x) for x in self.nodes) - + f"skip_pipe={self.skip_pipe},file_input={self.file_input}," + + f", pipe_size={self.pipe_size}, " + + f"file_input={self.file_input}, " + f"file_output={self.file_output})" ) def __eq__(self, other: object) -> bool: if not isinstance(other, Pipeline): return NotImplemented + # pipe_size is a tuning hint, not part of identity (see __init__). return ( self.nodes == other.nodes - and self.skip_pipe == other.skip_pipe and self.file_input == other.file_input and self.file_output == other.file_output ) def __hash__(self) -> int: attrs: List[Any] = self.nodes + [self.file_input, self.file_output] - if self.skip_pipe is None: - attrs.append(None) - else: - attrs.append(tuple(self.skip_pipe)) return hash(tuple(attrs)) @@ -282,7 +374,7 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: if not isinstance(other, ProcSub): return NotImplemented - return self.node == other.node + return type(self) is type(other) and self.node == other.node def __hash__(self) -> int: return hash((self.__class__.__name__, self.node)) @@ -296,22 +388,26 @@ class InputProcSub(ProcSub): """ async def setup(self, context: Context) -> str: - self.fifo_path = context.get_new_fifo() + fifo_path = context.get_new_fifo() + self.fifo_path = fifo_path + wait = _ProcSubWait(fifo_path) # We must open the FIFO for writing. # WARNING: opening a FIFO blocks until the other end is opened. # We run the open() in a thread to prevent blocking the async event # loop. - def open_fifo_write(): - if not self.fifo_path: - raise RuntimeError("FIFO path not set") - return open(self.fifo_path, "w") + def open_fifo_write() -> IO[Any]: + return open(fifo_path, "w") # Start the task that waits for the open, then runs the process - async def run_inner(): - # This awaits until the outer command opens the file for reading! + async def run_inner() -> None: + # This awaits until the outer command opens the file for + # reading (or until cleanup() unblocks the open)! write_handle = await asyncio.to_thread(open_fifo_write) + wait.opened = True try: + if context.closing: + return # cleanup unblocked us; do not spawn proc = await self.node.run( context, stdout=write_handle, @@ -321,10 +417,10 @@ async def run_inner(): write_handle.close() # Schedule this task in the background - task = asyncio.create_task(run_inner()) - context.tasks.append(task) + context.tasks.append(asyncio.create_task(run_inner())) + context._procsub_waits.append(wait) - return self.fifo_path + return fifo_path def __str__(self) -> str: return "<(" + str(self.node) + ")" @@ -338,26 +434,30 @@ class OutputProcSub(ProcSub): """ async def setup(self, context: Context) -> str: - self.fifo_path = context.get_new_fifo() + fifo_path = context.get_new_fifo() + self.fifo_path = fifo_path + wait = _ProcSubWait(fifo_path) - def open_fifo_read(): - if not self.fifo_path: - raise RuntimeError("FIFO path not set") - return open(self.fifo_path, "r") + def open_fifo_read() -> IO[Any]: + return open(fifo_path, "r") - async def run_inner(): - # This awaits until the outer command opens the file for writing! + async def run_inner() -> None: + # This awaits until the outer command opens the file for + # writing (or until cleanup() unblocks the open)! read_handle = await asyncio.to_thread(open_fifo_read) + wait.opened = True try: + if context.closing: + return # cleanup unblocked us; do not spawn proc = await self.node.run(context, stdin=read_handle) await proc.wait() finally: read_handle.close() - task = asyncio.create_task(run_inner()) - context.tasks.append(task) + context.tasks.append(asyncio.create_task(run_inner())) + context._procsub_waits.append(wait) - return self.fifo_path + return fifo_path def __str__(self) -> str: return ">(" + str(self.node) + ")" diff --git a/tests/integration/test_executor.py b/tests/integration/test_executor.py index 7199ecc..63c3779 100644 --- a/tests/integration/test_executor.py +++ b/tests/integration/test_executor.py @@ -1,13 +1,13 @@ """ -Integration tests for the Executor +Integration tests for the executors. """ -import io +import asyncio import os import pathlib import sys import tempfile -from unittest.mock import patch +import threading import pytest @@ -17,7 +17,12 @@ ) from sentieon_cli.dag import DAG # noqa: E402 -from sentieon_cli.executor import LocalExecutor # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.executor import ( # noqa: E402 + DryRunExecutor, + LocalExecutor, + _cleanup_quietly, +) from sentieon_cli.job import Job # noqa: E402 from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 from sentieon_cli.shell_pipeline import ( # noqa: E402 @@ -30,64 +35,64 @@ def test_local_executor_simple_job(): """Test LocalExecutor with a single, simple job""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() - job = Job(Pipeline(Command("echo", "hello executor"), file_output=cmd_out), "echo-job") + job = Job( + Pipeline(Command("echo", "hello executor"), file_output=cmd_out), + "echo-job", + ) dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() - cmd_out_str = open(cmd_out).read() - assert "hello executor" in cmd_out_str + assert "hello executor" in cmd_out.read_text() assert len(executor.jobs_with_errors) == 0 def test_local_executor_pipeline_job(): """Test LocalExecutor with a job containing a pipeline""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() - pipeline = Pipeline(Command("echo", "hello pipeline"), Command("cat"), file_output=cmd_out) + pipeline = Pipeline( + Command("echo", "hello pipeline"), + Command("cat"), + file_output=cmd_out, + ) job = Job(pipeline, "pipeline-job") dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() - cmd_out_str = open(cmd_out).read() - assert "hello pipeline" in cmd_out_str + assert "hello pipeline" in cmd_out.read_text() assert len(executor.jobs_with_errors) == 0 + def test_local_executor_failing_job(): """Test LocalExecutor with a job that fails""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_in = tmp_dir / "test_in.txt" + cmd_in = pathlib.Path(tmp_dir_str) / "test_in.txt" dag = DAG() - # This command will fail + # This command will fail: the input file does not exist job = Job(Pipeline(Command("cat", str(cmd_in))), "failing-job") dag.add_job(job) scheduler = ThreadScheduler(dag, 1) executor = LocalExecutor(scheduler) - executor.execute() assert len(executor.jobs_with_errors) == 1 assert executor.jobs_with_errors[0] == job + def test_local_executor_proc_sub_job(): """Test LocalExecutor with a job using process substitution""" with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = pathlib.Path(tmp_dir_str) - cmd_out = tmp_dir / "test_out.txt" + cmd_out = pathlib.Path(tmp_dir_str) / "test_out.txt" dag = DAG() command = Command( "diff", @@ -100,18 +105,423 @@ def test_local_executor_proc_sub_job(): # Need at least 2 threads for the two proc subs scheduler = ThreadScheduler(dag, 2) executor = LocalExecutor(scheduler) - executor.execute() # diff will write to stdout and fail - output = open(cmd_out).read() + output = cmd_out.read_text() assert "1c1" in output assert "< a" in output assert "> b" in output - # The job should be marked as an error because the diff command fails + # The job is marked as an error because the diff command fails assert len(executor.jobs_with_errors) == 1 -if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file +def test_dry_run_executor_prints_without_running(capsys): + """DryRunExecutor prints the shell but does not execute it""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + cmd_out = pathlib.Path(tmp_dir_str) / "out.txt" + dag = DAG() + job = Job( + Pipeline(Command("echo", "dryrun"), file_output=cmd_out), + "dry-job", + ) + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = DryRunExecutor(scheduler) + executor.execute() + + printed = capsys.readouterr().out + assert "echo" in printed + assert "dryrun" in printed + assert not cmd_out.exists() + assert len(executor.jobs_with_errors) == 0 + + +def test_dry_run_executor_drains_dependencies(capsys): + """DryRunExecutor walks the whole DAG, including dependent jobs""" + dag = DAG() + a = Job(Pipeline(Command("echo", "a")), "a") + b = Job(Pipeline(Command("echo", "b")), "b") + dag.add_job(a) + dag.add_job(b, {a}) + + scheduler = ThreadScheduler(dag, 4) + executor = DryRunExecutor(scheduler) + executor.execute() + + printed = capsys.readouterr().out + assert "echo a" in printed + assert "echo b" in printed + + +def test_fail_ok_subcommand_does_not_fail_job(): + """A failing but fail_ok sub-command does not mark the job as errored""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + missing = tmp_dir / "missing.txt" + out = tmp_dir / "out.txt" + + # `cat missing | wc -c`; cat fails (no file) but is tolerated. + dag = DAG() + job = Job( + Pipeline( + Command("cat", str(missing), fail_ok=True), + Command("wc", "-c"), + file_output=out, + ), + "tolerant-job", + ) + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + executor.execute() + + assert len(executor.jobs_with_errors) == 0 + + +def test_independent_jobs_all_complete(): + """Two independent jobs both run and produce output""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + o1 = tmp_dir / "o1.txt" + o2 = tmp_dir / "o2.txt" + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "1"), file_output=o1), "j1")) + dag.add_job(Job(Pipeline(Command("echo", "2"), file_output=o2), "j2")) + + scheduler = ThreadScheduler(dag, 2) + executor = LocalExecutor(scheduler) + executor.execute() + + assert o1.read_text().strip() == "1" + assert o2.read_text().strip() == "2" + assert len(executor.jobs_with_errors) == 0 + + +def test_dependent_job_runs_after_dependency(): + """A dependent job runs and both jobs complete successfully""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + first = tmp_dir / "first.txt" + second = tmp_dir / "second.txt" + dag = DAG() + a = Job(Pipeline(Command("echo", "a"), file_output=first), "a") + b = Job(Pipeline(Command("echo", "b"), file_output=second), "b") + dag.add_job(a) + dag.add_job(b, {a}) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + executor.execute() + + assert first.read_text().strip() == "a" + assert second.read_text().strip() == "b" + assert len(executor.jobs_with_errors) == 0 + + +def test_job_that_fails_to_start_is_recorded(): + """A job whose command cannot be spawned is recorded as an error.""" + dag = DAG() + job = Job(Pipeline(Command("no_such_cmd_zzz")), "ghost") + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + executor.execute() + assert job in executor.jobs_with_errors + + +def test_launch_loop_stops_after_a_failed_launch(): + """Once a job in a ready batch fails to launch, the loop stops starting + the rest of the batch instead of launching all of it.""" + dag = DAG() + for i in range(5): + # Distinct args -> distinct pipeline identities (else the DAG + # rejects them as duplicates). + dag.add_job(Job(Pipeline(Command("true", str(i))), f"j{i}")) + executor = LocalExecutor(ThreadScheduler(dag, 5)) + + launched = [] + + async def fake_run_job(job): + launched.append(job) + executor.start_new_jobs = False # simulate a launch failure + + executor.run_job = fake_run_job + executor.execute() + + # The guard stopped the loop after the first job rather than starting + # all five. + assert len(launched) == 1 + + +def test_infeasible_job_raises_instead_of_deadlocking(): + """A job larger than the budget raises rather than stalling silently.""" + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + LocalExecutor(ThreadScheduler(dag, 2)).execute() + + +def test_dry_run_also_rejects_infeasible_job(): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + DryRunExecutor(ThreadScheduler(dag, 2)).execute() + + +def test_non_oserror_on_start_propagates(monkeypatch): + """A non-OSError while starting a job is a bug and must propagate.""" + + async def boom(self, *args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr("sentieon_cli.shell_pipeline.Pipeline.run", boom) + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "j")) + with pytest.raises(RuntimeError): + LocalExecutor(ThreadScheduler(dag, 1)).execute() + + +def test_local_executor_empty_dag_completes(): + """An empty DAG runs cleanly.""" + executor = LocalExecutor(ThreadScheduler(DAG(), 1)) + executor.execute() + assert executor.jobs_with_errors == [] + + +def test_dry_run_empty_dag_completes(capsys): + """DryRunExecutor handles an empty DAG without printing anything.""" + executor = DryRunExecutor(ThreadScheduler(DAG(), 1)) + executor.execute() + assert capsys.readouterr().out == "" + assert executor.jobs_with_errors == [] + + +def _execute_bounded(executor, timeout=15): + """Run executor.execute() with a hang guard.""" + holder = {} + + def run(): + try: + executor.execute() + except BaseException as exc: # pragma: no cover - regression guard + holder["exc"] = exc + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout=timeout) + assert not thread.is_alive(), "executor.execute() hung" + assert "exc" not in holder, holder.get("exc") + + +def test_procsub_job_with_early_exiting_outer_does_not_hang(): + """An outer command that exits without opening its proc-sub FIFO must + not hang the executor (regression: cleanup deadlock).""" + dag = DAG() + job = Job( + Pipeline( + Command("false", InputProcSub(Pipeline(Command("echo", "x")))) + ), + "early-exit", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_job_with_unspawnable_outer_does_not_hang(): + """A missing outer executable with a proc-sub argument must not hang + the executor's error path (regression: cleanup deadlock).""" + dag = DAG() + job = Job( + Pipeline( + Command( + "no_such_cmd_zzz", + InputProcSub(Pipeline(Command("echo", "x"))), + ) + ), + "ghost-with-procsub", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_inner_that_fails_to_launch_is_recorded(): + """An inner proc-sub command that cannot be spawned fails its job + instead of crashing the executor (regression: a raw FileNotFoundError + from the background task propagated out of execute()).""" + dag = DAG() + job = Job( + Pipeline( + Command( + "cat", + InputProcSub(Pipeline(Command("no_such_cmd_zzz"))), + ) + ), + "inner-fail", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +def test_procsub_inner_failure_does_not_skip_sibling_bookkeeping(): + """A job whose inner proc-sub fails to launch must not abort the run + for independent sibling jobs (regression: the exception escaped mid- + flight and skipped other jobs' bookkeeping).""" + dag = DAG() + bad = Job( + Pipeline( + Command( + "cat", + InputProcSub(Pipeline(Command("no_such_cmd_zzz"))), + ) + ), + "inner-fail", + ) + good = Job(Pipeline(Command("true")), "sibling") + dag.add_job(bad) + dag.add_job(good) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert bad in executor.jobs_with_errors + # The sibling ran to completion and was recorded as a success, not + # silently dropped when the bad job's failure surfaced. + assert good not in executor.jobs_with_errors + + +def test_mid_pipeline_spawn_failure_terminates_running_stages(monkeypatch): + """When a later pipeline stage fails to spawn, earlier stages that are + already running must be terminated, not leaked (regression: a + Pipeline(sleep 600, no_such_cmd) left sleep alive past the workflow).""" + import sentieon_cli.executor as executor_mod + + contexts = [] + real_context = executor_mod.Context + + class RecordingContext(real_context): # type: ignore[valid-type,misc] + def __init__(self) -> None: + super().__init__() + contexts.append(self) + + monkeypatch.setattr(executor_mod, "Context", RecordingContext) + + dag = DAG() + job = Job( + Pipeline( + Command("sleep", "600"), + Command("no_such_cmd_zzz"), + ), + "leak", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + + assert job in executor.jobs_with_errors + # The first stage spawned; every spawned proc must be reaped, not left + # running to outlive the workflow. + spawned = [sub.proc for c in contexts for sub in c.commands if sub.proc] + assert spawned, "expected the first stage to have spawned" + for proc in spawned: + assert proc.returncode is not None + + +def test_sigpipe_producer_does_not_fail_job(tmp_path): + """A producer killed by SIGPIPE when a consumer exits early (e.g. + `seq | head -1`) produces the right output and must not fail the job + (regression: the producer's rc -13 was treated as a failure).""" + out = tmp_path / "out.txt" + dag = DAG() + job = Job( + Pipeline( + Command("seq", "1000000"), + Command("head", "-1"), + file_output=out, + ), + "sigpipe", + ) + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job not in executor.jobs_with_errors + assert out.read_text().strip() == "1" + + +def test_nonzero_producer_still_fails_job(): + """SIGPIPE forgiveness must not mask a producer that exits non-zero + for another reason: `false | cat` still fails the job.""" + dag = DAG() + job = Job(Pipeline(Command("false"), Command("cat")), "real-fail") + dag.add_job(job) + executor = LocalExecutor(ThreadScheduler(dag, 2)) + _execute_bounded(executor) + assert job in executor.jobs_with_errors + + +class _FakeContext: + """A stand-in Context whose cleanup() can be told to raise.""" + + def __init__(self, cleanup_error=None): + self.commands = [] + self.cleaned = False + self._error = cleanup_error + + async def cleanup(self): + self.cleaned = True + if self._error is not None: + raise self._error + + +@pytest.mark.asyncio +async def test_cleanup_quietly_swallows_oserror(): + """A launch failure surfacing from cleanup() during teardown is logged, + not raised -- so it cannot escape execute() on the abort/interrupt + paths (symmetry with _drive's per-job handling).""" + context = _FakeContext(FileNotFoundError("no_such_inner")) + await _cleanup_quietly(context) # must not raise + assert context.cleaned + + +@pytest.mark.asyncio +async def test_cleanup_quietly_propagates_non_oserror(): + """A non-OSError from cleanup() is a bug and must still propagate.""" + with pytest.raises(RuntimeError): + await _cleanup_quietly(_FakeContext(RuntimeError("bug"))) + + +@pytest.mark.asyncio +async def test_shutdown_releases_all_contexts_despite_cleanup_error(): + """A cleanup() OSError during interrupt shutdown is swallowed and the + remaining contexts are still released, instead of the error escaping + _shutdown() and skipping their teardown.""" + executor = LocalExecutor(ThreadScheduler(DAG(), 1)) + bad = _FakeContext(FileNotFoundError("no_such_inner")) + good = _FakeContext() + + async def _done(): + return 0 + + executor.running = [ + ( + Job(Pipeline(Command("true")), "j1"), + bad, + asyncio.create_task(_done()), + 0, + ), + ( + Job(Pipeline(Command("false")), "j2"), + good, + asyncio.create_task(_done()), + 0, + ), + ] + await executor._shutdown() # must not raise + + assert bad.cleaned and good.cleaned diff --git a/tests/integration/test_signals.py b/tests/integration/test_signals.py new file mode 100644 index 0000000..8aa2042 --- /dev/null +++ b/tests/integration/test_signals.py @@ -0,0 +1,193 @@ +""" +Signal-handling tests for LocalExecutor. + +These deliver real signals to the current process. That is safe here because +the executor installs an asyncio loop signal handler that suppresses the +default disposition (e.g. SIGINT -> KeyboardInterrupt) for the duration of the +run, and the timer that raises the signal is always armed *after* the handler +is installed. If these ever prove flaky in a constrained CI, the same behavior +can be exercised by running the workflow in a subprocess and signalling the +child. +""" + +import os +import signal +import sys +import threading +import time + +# Add the parent directory to the path +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +) + +import sentieon_cli.executor as executor_mod # noqa: E402 +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.executor import LocalExecutor # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import ( # noqa: E402 + Command, + InputProcSub, + Pipeline, +) + + +def _echo_dag(out): + dag = DAG() + job = Job(Pipeline(Command("echo", "hi"), file_output=out), "echo") + dag.add_job(job) + return dag + + +def test_off_main_thread_does_not_crash(tmp_path): + """LocalExecutor with signal handlers runs fine off the main thread.""" + out = tmp_path / "o.txt" + holder = {} + + def run(): + try: + LocalExecutor( + ThreadScheduler(_echo_dag(out), 1), + install_signal_handlers=True, + ).execute() + except Exception as exc: # pragma: no cover - regression guard + holder["exc"] = exc + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=10) + + assert not thread.is_alive() + assert "exc" not in holder, holder.get("exc") + assert out.read_text().strip() == "hi" + + +def test_default_does_not_touch_signal_handlers(tmp_path): + """The default executor installs no handlers and leaves them intact.""" + before_int = signal.getsignal(signal.SIGINT) + before_term = signal.getsignal(signal.SIGTERM) + + out = tmp_path / "o.txt" + LocalExecutor(ThreadScheduler(_echo_dag(out), 1)).execute() + + assert signal.getsignal(signal.SIGINT) is before_int + assert signal.getsignal(signal.SIGTERM) is before_term + + +def test_opt_in_restores_signal_handlers(tmp_path): + """Opt-in handlers are restored to their prior values after a run.""" + before_int = signal.getsignal(signal.SIGINT) + before_term = signal.getsignal(signal.SIGTERM) + + out = tmp_path / "o.txt" + LocalExecutor( + ThreadScheduler(_echo_dag(out), 1), + install_signal_handlers=True, + ).execute() + + assert signal.getsignal(signal.SIGINT) is before_int + assert signal.getsignal(signal.SIGTERM) is before_term + + +def test_sigint_triggers_graceful_shutdown(monkeypatch): + """A delivered SIGINT terminates running jobs and cleans up.""" + temp_dirs = [] + real_context = executor_mod.Context + + class SpyContext(real_context): + async def cleanup(self): + temp_dirs.append(self.temp_dir.name) + await super().cleanup() + + monkeypatch.setattr(executor_mod, "Context", SpyContext) + + dag = DAG() + dag.add_job(Job(Pipeline(Command("sleep", "30")), "sleeper")) + executor = LocalExecutor( + ThreadScheduler(dag, 1), + install_signal_handlers=True, + shutdown_grace_period=1.0, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 # returned well before `sleep 30` would finish + assert executor.running == [] + assert temp_dirs # cleanup ran for the interrupted job + for temp_dir in temp_dirs: + assert not os.path.exists(temp_dir) # no leaked temp dir + + +def test_grace_period_escalates_to_sigkill(): + """A job that ignores SIGTERM is still killed after the grace period.""" + script = ( + "import signal, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "time.sleep(30)" + ) + dag = DAG() + job = Job(Pipeline(Command(sys.executable, "-c", script)), "stubborn") + dag.add_job(job) + executor = LocalExecutor( + ThreadScheduler(dag, 1), + install_signal_handlers=True, + shutdown_grace_period=0.5, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 # SIGKILL fired after grace, not after time.sleep(30) + assert executor.running == [] + + +def test_sigint_with_blocked_procsub_cleans_up(): + """SIGINT while a proc-sub FIFO open is still blocked must not hang + the shutdown path (regression: cleanup deadlock).""" + dag = DAG() + # sh never opens the FIFO in "$1", so the proc sub's open() stays + # blocked until cleanup unblocks it. + job = Job( + Pipeline( + Command( + "sh", + "-c", + "sleep 30", + "sh", + InputProcSub(Pipeline(Command("echo", "x"))), + ) + ), + "blocked-procsub", + ) + dag.add_job(job) + executor = LocalExecutor( + ThreadScheduler(dag, 2), + install_signal_handlers=True, + shutdown_grace_period=1.0, + ) + + timer = threading.Timer(0.5, lambda: os.kill(os.getpid(), signal.SIGINT)) + timer.start() + try: + start = time.monotonic() + executor.execute() + elapsed = time.monotonic() - start + finally: + timer.cancel() + + assert elapsed < 10 + assert executor.running == [] diff --git a/tests/unit/test_dag.py b/tests/unit/test_dag.py new file mode 100644 index 0000000..aa9d565 --- /dev/null +++ b/tests/unit/test_dag.py @@ -0,0 +1,169 @@ +""" +Unit tests for DAG traversal and duplicate detection. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +def _job(name, threads=1, resources=None): + """A trivial job whose identity is its (unique) command name.""" + return Job(Pipeline(Command(name)), name, threads, resources) + + +def test_job_identity_is_the_pipeline(): + # Two jobs with the same pipeline are equal even if named + # differently; a DAG deduplicates on this identity. + j1 = Job(Pipeline(Command("echo", "x")), "name-a") + j2 = Job(Pipeline(Command("echo", "x")), "name-b") + j3 = Job(Pipeline(Command("echo", "y")), "name-c") + assert j1 == j2 + assert hash(j1) == hash(j2) + assert j1 != j3 + + +class TestDAGTraversal: + """Test DAG bookkeeping through mark_finished""" + + def test_add_job_with_unknown_dependency_raises(self): + dag = DAG() + known = _job("known") + dag.add_job(known) + + unknown = _job("unknown") + with pytest.raises(ValueError): + dag.add_job(_job("downstream"), {unknown}) + + def test_finishing_an_unready_job_raises(self): + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, {a}) + + with pytest.raises(DagExecutionError): + dag.mark_finished(b) # b is not ready + + def test_diamond_dependencies(self): + dag = DAG() + top = _job("top") + left = _job("left") + right = _job("right") + bottom = _job("bottom") + + dag.add_job(top) + dag.add_job(left, {top}) + dag.add_job(right, {top}) + dag.add_job(bottom, {left, right}) + + assert set(dag.ready_jobs) == {top} + + assert set(dag.mark_finished(top)) == {left, right} + + # bottom needs BOTH left and right + assert set(dag.mark_finished(left)) == set() + assert set(dag.mark_finished(right)) == {bottom} + + def test_multilevel_chain(self): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + + assert set(dag.ready_jobs) == {a} + assert set(dag.mark_finished(a)) == {b} + assert set(dag.mark_finished(b)) == {c} + + # Everything has been consumed and marked finished. + dag.mark_finished(c) + assert a in dag.finished_jobs + assert b in dag.finished_jobs + assert c in dag.finished_jobs + assert len(dag.waiting_jobs) == 0 + assert len(dag.ready_jobs) == 0 + + def test_add_duplicate_pipeline_raises(self): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "step-1")) + # A second job with an identical pipeline collides on identity. + with pytest.raises(ValueError): + dag.add_job(Job(Pipeline(Command("echo", "x")), "step-2")) + + def test_jobs_differing_only_in_exec_kwargs_are_not_duplicates(self): + # A command's env/cwd is part of its identity, so two jobs that + # differ only there are distinct, not rejected as duplicates. + dag = DAG() + j1 = Job(Pipeline(Command("run", exec_kwargs={"cwd": "/a"})), "a") + j2 = Job(Pipeline(Command("run", exec_kwargs={"cwd": "/b"})), "b") + dag.add_job(j1) + dag.add_job(j2) # must not raise "already in the DAG" + assert j1 in dag.ready_jobs + assert j2 in dag.ready_jobs + + def test_add_job_accepts_list_dependencies(self): + # Regression: non-set dependencies used to be silently dropped, + # leaving the job immediately ready. + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, [a]) + + assert b not in dag.ready_jobs + assert dag.waiting_jobs[b] == {a} + assert set(dag.mark_finished(a)) == {b} + + def test_add_job_accepts_any_iterable_dependencies(self): + # Tuples, frozensets, and single-use generators all work; the + # generator case also guards against consuming the iterable twice. + for make in (tuple, frozenset, iter): + dag = DAG() + a = _job("a") + b = _job("b") + dag.add_job(a) + dag.add_job(b, make([a])) + assert dag.waiting_jobs[b] == {a}, make + + def test_add_job_rejects_unknown_dependency_in_list(self): + dag = DAG() + dag.add_job(_job("known")) + with pytest.raises(ValueError): + dag.add_job(_job("downstream"), [_job("unknown")]) + + def test_add_job_with_empty_iterable_is_ready(self): + for deps in ([], iter([])): + dag = DAG() + job = _job("solo") + dag.add_job(job, deps) + assert job in dag.ready_jobs + + def test_stored_dependencies_are_independent_of_caller_set(self): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + dag.add_job(a) + dag.add_job(b) + + deps = {a} + dag.add_job(c, deps) + deps.add(b) # mutating the caller's set must not affect the DAG + + assert dag.waiting_jobs[c] == {a} diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index c42159f..e9bacdf 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -426,8 +426,7 @@ def test_dag_execution_simulation(self): dag.add_job(job3, {job1, job2}) # Simulate execution - dag_gen = dag.update_dag() - ready_jobs = dag_gen.send(None) + ready_jobs = dag.ready_jobs # Initially job1 and job2 should be ready assert job1 in ready_jobs @@ -435,11 +434,11 @@ def test_dag_execution_simulation(self): assert job3 not in ready_jobs # Complete job1 - new_ready = dag_gen.send(job1) + new_ready = dag.mark_finished(job1) assert len(new_ready) == 0 # job3 still waiting for job2 # Complete job2 - new_ready = dag_gen.send(job2) + new_ready = dag.mark_finished(job2) assert job3 in new_ready # Now job3 is ready diff --git a/tests/unit/test_pipeline_lifecycle.py b/tests/unit/test_pipeline_lifecycle.py new file mode 100644 index 0000000..b9659c1 --- /dev/null +++ b/tests/unit/test_pipeline_lifecycle.py @@ -0,0 +1,147 @@ +""" +Unit tests for the BasePipeline lifecycle (argument registration, logging +scope, temp-dir cleanup, and post-run checks). +""" + +import argparse +import logging +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.pipeline import BasePipeline # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +class _DummyPipeline(BasePipeline): + """A minimal concrete pipeline for exercising BasePipeline plumbing.""" + + def validate(self) -> None: + pass + + def configure(self) -> None: + pass + + def build_dag(self) -> DAG: + return DAG() + + +class _FailingPipeline(_DummyPipeline): + """A pipeline whose DAG construction raises.""" + + def build_dag(self) -> DAG: + raise RuntimeError("boom") + + +def _option_strings(parser): + return [opt for action in parser._actions for opt in action.option_strings] + + +def test_add_arguments_does_not_mutate_the_class_specs(): + before_reference = dict(BasePipeline.params["reference"]) + before_cores = dict(BasePipeline.params["cores"]) + + parser = argparse.ArgumentParser() + BasePipeline.add_arguments(parser) + + assert BasePipeline.params["reference"] == before_reference + assert BasePipeline.params["cores"] == before_cores + # In particular the specs keep their flags and gain no inferred type. + assert "flags" in BasePipeline.params["reference"] + assert "type" not in BasePipeline.params["cores"] + + +def test_add_arguments_keeps_short_flags_across_parsers(): + # Registering the same pipeline class on a second parser must produce + # the same options; the short flags used to be lost because the first + # call popped them from the shared class-level spec. + p1 = argparse.ArgumentParser() + p2 = argparse.ArgumentParser() + BasePipeline.add_arguments(p1) + BasePipeline.add_arguments(p2) + + for parser in (p1, p2): + opts = _option_strings(parser) + assert "-r" in opts + assert "--reference" in opts + assert "-t" in opts + assert "--cores" in opts + + +def test_setup_logging_does_not_touch_the_root_logger(): + root_level_before = logging.getLogger().level + package_logger = logging.getLogger("sentieon_cli") + package_level_before = package_logger.level + try: + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel=logging.DEBUG)) + + assert logging.getLogger().level == root_level_before + assert package_logger.level == logging.DEBUG + finally: + package_logger.setLevel(package_level_before) + + +def test_main_cleans_up_tmpdir_when_build_dag_raises(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _FailingPipeline() + + with pytest.raises(RuntimeError, match="boom"): + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert list(tmp_path.iterdir()) == [] # no leaked temp dir + + +def test_main_retains_tmpdir_on_failure_when_requested(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _FailingPipeline() + pipeline.retain_tmpdir = True + + with pytest.raises(RuntimeError, match="boom"): + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert len(list(tmp_path.iterdir())) == 1 # kept for inspection + + +def test_main_cleans_up_tmpdir_on_success(tmp_path, monkeypatch): + monkeypatch.setenv("SENTIEON_TMPDIR", str(tmp_path)) + pipeline = _DummyPipeline() + pipeline.dry_run = True + + pipeline.main(argparse.Namespace(loglevel="WARNING")) + + assert list(tmp_path.iterdir()) == [] + + +class _StubExecutor: + def __init__(self, jobs_with_errors=None): + self.jobs_with_errors = jobs_with_errors or [] + + +def test_check_execution_names_the_failed_jobs(): + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel="WARNING")) + job = Job(Pipeline(Command("false")), "broken-step") + + with pytest.raises(DagExecutionError, match="broken-step"): + pipeline.check_execution(DAG(), _StubExecutor([job])) + + +def test_check_execution_flags_unexecuted_jobs(): + pipeline = _DummyPipeline() + pipeline.setup_logging(argparse.Namespace(loglevel="WARNING")) + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "x")), "unexecuted")) + + with pytest.raises(DagExecutionError, match="unexecuted"): + pipeline.check_execution(dag, _StubExecutor()) diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py new file mode 100644 index 0000000..aaecf0e --- /dev/null +++ b/tests/unit/test_scheduler.py @@ -0,0 +1,104 @@ +""" +Unit tests for the ThreadScheduler. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + + +def _job(name, threads=1, resources=None): + return Job(Pipeline(Command(name)), name, threads, resources) + + +def test_schedules_independent_jobs_together(): + dag = DAG() + j1 = _job("j1", threads=1) + j2 = _job("j2", threads=1) + dag.add_job(j1) + dag.add_job(j2) + + scheduler = ThreadScheduler(dag, threads=4) + assert scheduler.start() == {j1, j2} + + +def test_thread_budget_gates_scheduling(): + dag = DAG() + j1 = _job("j1", threads=2) + j2 = _job("j2", threads=2) + dag.add_job(j1) + dag.add_job(j2) + + # A 2-thread budget only fits one 2-thread job at a time. + scheduler = ThreadScheduler(dag, threads=2) + + first = scheduler.start() + assert len(first) == 1 + running = next(iter(first)) + + # The second job starts only once the first frees its threads. + second = scheduler.job_finished(running) + assert len(second) == 1 + remaining = next(iter(second)) + + assert {running, remaining} == {j1, j2} + + +def test_named_resource_gates_scheduling(): + dag = DAG() + j1 = _job("j1", threads=1, resources={"node0": 1}) + j2 = _job("j2", threads=1, resources={"node0": 1}) + dag.add_job(j1) + dag.add_job(j2) + + # Plenty of threads, but only a single node0 token. + scheduler = ThreadScheduler(dag, threads=8, resources={"node0": 1}) + + first = scheduler.start() + assert len(first) == 1 + running = next(iter(first)) + + second = scheduler.job_finished(running) + assert len(second) == 1 + assert {running, next(iter(second))} == {j1, j2} + + +def test_resource_requirements_ignored_when_unmanaged(): + # Jobs may request a resource the scheduler does not track; those + # requirements are simply ignored. + dag = DAG() + j1 = _job("j1", threads=1, resources={"gpu": 1}) + j2 = _job("j2", threads=1, resources={"gpu": 1}) + dag.add_job(j1) + dag.add_job(j2) + + scheduler = ThreadScheduler(dag, threads=8) + assert scheduler.start() == {j1, j2} + + +def test_oversized_thread_request_is_rejected(): + dag = DAG() + dag.add_job(Job(Pipeline(Command("echo", "hi")), "big", 100)) + with pytest.raises(DagExecutionError): + ThreadScheduler(dag, threads=2).start() + + +def test_oversized_resource_request_is_rejected(): + dag = DAG() + job = Job(Pipeline(Command("echo", "hi")), "greedy", 1, {"node0": 3}) + dag.add_job(job) + with pytest.raises(DagExecutionError): + ThreadScheduler(dag, threads=8, resources={"node0": 1}).start() diff --git a/tests/unit/test_scheduler_contract.py b/tests/unit/test_scheduler_contract.py new file mode 100644 index 0000000..0460403 --- /dev/null +++ b/tests/unit/test_scheduler_contract.py @@ -0,0 +1,109 @@ +""" +Conformance tests for the BaseScheduler contract. + +``drive`` mimics what an executor does: ask the scheduler for the initial +batch, then report each job finished (FIFO) and collect whatever that +unblocks, until the DAG drains. Any BaseScheduler implementation can be +dropped into ``drive`` to check that it honors the contract; the +dependency-order tests run against every scheduler in ``SCHEDULERS``. +""" + +import os +import sys + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.exceptions import DagExecutionError # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import Command, Pipeline # noqa: E402 + +SCHEDULERS = [ + pytest.param(lambda dag: ThreadScheduler(dag, threads=8), id="thread"), +] + + +def _job(name, threads=1, resources=None): + return Job(Pipeline(Command(name)), name, threads, resources) + + +def drive(scheduler): + """Drive a scheduler to completion; return the job finish order.""" + order = [] + queue = list(scheduler.start()) + while queue: + job = queue.pop(0) + order.append(job) + queue.extend(scheduler.job_finished(job)) + return order + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_drives_a_diamond_to_completion(make_scheduler): + dag = DAG() + top = _job("top") + left = _job("left") + right = _job("right") + bottom = _job("bottom") + dag.add_job(top) + dag.add_job(left, {top}) + dag.add_job(right, {top}) + dag.add_job(bottom, {left, right}) + + order = drive(make_scheduler(dag)) + + # Every job runs exactly once. + assert len(order) == len(set(order)) + assert sorted(j.name for j in order) == [ + "bottom", + "left", + "right", + "top", + ] + # Dependencies precede their dependents. + pos = {job: i for i, job in enumerate(order)} + assert pos[top] < pos[left] < pos[bottom] + assert pos[top] < pos[right] < pos[bottom] + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_drives_a_chain_to_completion(make_scheduler): + dag = DAG() + a = _job("a") + b = _job("b") + c = _job("c") + dag.add_job(a) + dag.add_job(b, {a}) + dag.add_job(c, {b}) + + order = drive(make_scheduler(dag)) + assert [j.name for j in order] == ["a", "b", "c"] + + +@pytest.mark.parametrize("make_scheduler", SCHEDULERS) +def test_job_finished_on_unknown_job_raises(make_scheduler): + dag = DAG() + dag.add_job(_job("a")) + scheduler = make_scheduler(dag) + scheduler.start() + + with pytest.raises(DagExecutionError): + scheduler.job_finished(_job("never-scheduled")) + + +def test_budget_constrained_run_still_completes(): + # ThreadScheduler-specific: a 1-thread budget serializes two independent + # jobs, but driving to completion still runs both exactly once. + dag = DAG() + dag.add_job(_job("a", threads=1)) + dag.add_job(_job("b", threads=1)) + + order = drive(ThreadScheduler(dag, threads=1)) + assert sorted(j.name for j in order) == ["a", "b"] diff --git a/tests/unit/test_shell_pipeline.py b/tests/unit/test_shell_pipeline.py index 943d5b9..38ec2bd 100644 --- a/tests/unit/test_shell_pipeline.py +++ b/tests/unit/test_shell_pipeline.py @@ -3,6 +3,7 @@ """ import asyncio +import fcntl import os import pathlib import sys @@ -31,7 +32,6 @@ async def test_simple_command(): cmd = Command("echo", "hello world") context = Context() - # Use a file for stdout with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: proc = await cmd.run(context, stdout=stdout_file) await proc.wait() @@ -73,7 +73,6 @@ async def test_pipeline_with_file_io(): """Test a pipeline with file input and output""" context = Context() - # Create input and output files with tempfile.NamedTemporaryFile(mode="w", delete=False) as infile: infile.write("hello from file") infile_path = pathlib.Path(infile.name) @@ -82,12 +81,16 @@ async def test_pipeline_with_file_io(): cmd1 = Command("cat") cmd2 = Command("wc", "-c") - pipeline = Pipeline(cmd1, cmd2, file_input=infile_path, file_output=outfile_path) + pipeline = Pipeline( + cmd1, + cmd2, + file_input=infile_path, + file_output=outfile_path, + ) proc = await pipeline.run(context) await proc.wait() - # Verify output file content with open(outfile_path, "r") as f: output = f.read().strip() @@ -132,18 +135,14 @@ async def test_output_process_substitution(): """Test output process substitution: tee >(wc -c)""" context = Context() - # We will write the output of wc -c to a file to check it outfile_path = pathlib.Path(context.temp_dir.name) / "wc_output.txt" - # The pipeline will be: echo "hello" | tee >(wc -c > outfile.txt) + # echo "hello" | tee >(wc -c > outfile.txt) # The main output of the pipeline should be "hello" - # The output of the process substitution should be the character count "6" - - # Inner pipeline for process substitution + # The output of the process substitution should be the char count. wc_cmd = Command("wc", "-c") wc_pipeline = Pipeline(wc_cmd, file_output=outfile_path) - # Main pipeline echo_cmd = Command("echo", "hello") tee_cmd = Command("tee", OutputProcSub(wc_pipeline)) main_pipeline = Pipeline(echo_cmd, tee_cmd) @@ -158,10 +157,8 @@ async def test_output_process_substitution(): stdout_file.seek(0) main_output = stdout_file.read().strip() - # Verify main output assert main_output == "hello" - # Verify process substitution output with open(outfile_path, "r") as f: wc_output = f.read().strip() @@ -171,3 +168,257 @@ async def test_output_process_substitution(): await context.cleanup() os.remove(stdout_file.name) assert proc.returncode == 0 + + +@pytest.mark.asyncio +async def test_pipe_size_enlarges_internal_pipes(tmp_path): + """pipe_size raises F_SETPIPE_SZ on the pipeline's internal pipes, + which the downstream child inherits.""" + if not hasattr(fcntl, "F_SETPIPE_SZ"): + pytest.skip("F_SETPIPE_SZ not available (non-Linux)") + try: + pipe_max = int(open("/proc/sys/fs/pipe-max-size").read()) + except OSError: + pytest.skip("cannot read fs.pipe-max-size") + target = 128 * 1024 # a power of two, so the kernel sets it exactly + if pipe_max < target: + pytest.skip("fs.pipe-max-size too small for this test") + + report = tmp_path / "size.txt" + # The first stage's stdout is the internal pipe to `cat`; report that + # pipe's buffer size (1032 == F_GETPIPE_SZ) rather than writing to it. + reporter = ( + "import fcntl, sys; " + "open(sys.argv[1], 'w').write(str(fcntl.fcntl(1, 1032)))" + ) + context = Context() + pipeline = Pipeline( + Command(sys.executable, "-c", reporter, str(report)), + Command("cat"), + pipe_size=target, + ) + proc = await pipeline.run(context) + await proc.wait() + for sub in context.commands: + if sub.proc: + await sub.proc.wait() + await context.cleanup() + + assert int(report.read_text()) == target + + +def test_command_and_pipeline_hash_identity(): + """Equal commands/pipelines hash equally (DAG relies on this).""" + c1 = Command("echo", "x") + c2 = Command("echo", "x") + c3 = Command("echo", "y") + assert c1 == c2 + assert hash(c1) == hash(c2) + assert c1 != c3 + + p1 = Pipeline(Command("echo", "x"), Command("cat")) + p2 = Pipeline(Command("echo", "x"), Command("cat")) + p3 = Pipeline(Command("echo", "x")) + assert p1 == p2 + assert hash(p1) == hash(p2) + assert p1 != p3 + + +def test_procsub_input_and_output_are_distinct(): + """<(x) and >(x) wrap the same pipeline but are not the same node: they + must be unequal and hash differently.""" + p = Pipeline(Command("echo", "x")) + assert InputProcSub(p) != OutputProcSub(p) + assert hash(InputProcSub(p)) != hash(OutputProcSub(p)) + # Each still equals another of its own kind wrapping an equal pipeline. + other = Pipeline(Command("echo", "x")) + assert InputProcSub(p) == InputProcSub(other) + assert hash(InputProcSub(p)) == hash(InputProcSub(other)) + + +def test_command_identity_is_stable_across_running(): + """A Command's identity must not change once it has started: __eq__ + used to mix in self.proc, so running a job mutated its identity.""" + c1 = Command("echo", "x") + c2 = Command("echo", "x") + assert c1 == c2 and hash(c1) == hash(c2) + c1.proc = object() # stand in for a started asyncio process + assert c1 == c2 + assert hash(c1) == hash(c2) + + +def test_command_identity_includes_exec_kwargs(): + """exec_kwargs participates in identity: commands differing only in + env/cwd are distinct, and equal env hashes the same regardless of the + dict's insertion order.""" + assert Command("run", exec_kwargs={"cwd": "/a"}) != Command( + "run", exec_kwargs={"cwd": "/b"} + ) + assert Command("run", exec_kwargs={"cwd": "/a"}) != Command("run") + e1 = Command("run", exec_kwargs={"env": {"A": "1", "B": "2"}}) + e2 = Command("run", exec_kwargs={"env": {"B": "2", "A": "1"}}) + assert e1 == e2 + assert hash(e1) == hash(e2) + + +def test_str_renders_env_and_cwd(): + """str() renders exec_kwargs env/cwd as a conventional bash prefix, + quoting values that need it.""" + env_cmd = Command("samtools", "sort", exec_kwargs={"env": {"REF": "hg"}}) + assert str(env_cmd) == "REF=hg samtools sort" + cwd_cmd = Command("samtools", "sort", exec_kwargs={"cwd": "/data"}) + assert str(cwd_cmd) == "(cd /data && samtools sort)" + both = Command( + "run", exec_kwargs={"cwd": "/data dir", "env": {"K": "v w"}} + ) + assert str(both) == "(cd '/data dir' && K='v w' run)" + + +def test_empty_pipeline_raises(): + with pytest.raises(ValueError): + Pipeline() + + +@pytest.mark.asyncio +async def test_file_output_and_stdout_conflict_raises(): + """A file/stdout conflict raises ValueError (not sys.exit).""" + context = Context() + out = pathlib.Path(context.temp_dir.name) / "out.txt" + pipeline = Pipeline(Command("echo", "x"), file_output=out) + with pytest.raises(ValueError): + await pipeline.run(context, stdout=1) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_file_input_and_stdin_fd0_conflict_raises(tmp_path): + """file_input plus a stdin of fd 0 conflicts: fd 0 is a valid stream but + falsy, so the guard must test `is not None`, not truthiness.""" + inp = tmp_path / "in.txt" + inp.write_text("data") + context = Context() + pipeline = Pipeline(Command("cat"), file_input=inp) + with pytest.raises(ValueError): + await pipeline.run(context, stdin=0) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_file_input_and_stdin_conflict_raises(): + """A file/stdin conflict raises ValueError (not sys.exit).""" + context = Context() + infile = pathlib.Path(context.temp_dir.name) / "in.txt" + pipeline = Pipeline(Command("cat"), file_input=infile) + with pytest.raises(ValueError): + await pipeline.run(context, stdin=1) + await context.cleanup() + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_unopened_input_procsub(): + """cleanup() must not hang when an <(...) FIFO was never opened.""" + context = Context() + proc_sub = InputProcSub(Pipeline(Command("echo", "x"))) + fifo_path = await proc_sub.setup(context) + assert os.path.exists(fifo_path) + + await asyncio.wait_for(context.cleanup(), timeout=10) + + # The inner command never spawned: nothing opened the FIFO. + assert context.commands == [] + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_unopened_output_procsub(): + """cleanup() must not hang when a >(...) FIFO was never opened.""" + context = Context() + proc_sub = OutputProcSub(Pipeline(Command("cat"))) + await proc_sub.setup(context) + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert context.commands == [] + + +@pytest.mark.asyncio +async def test_cleanup_after_outer_exits_without_opening(): + """An outer command that exits without opening its proc-sub FIFO + (here: false ignores its arguments) must not hang cleanup().""" + context = Context() + cmd = Command("false", InputProcSub(Pipeline(Command("echo", "x")))) + proc = await cmd.run(context) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert [c.executable for c in context.commands] == ["false"] + + +@pytest.mark.asyncio +async def test_cleanup_unblocks_multiple_procsubs(): + """Several never-opened proc subs are all unblocked by cleanup().""" + context = Context() + cmd = Command( + "false", + InputProcSub(Pipeline(Command("echo", "x"))), + OutputProcSub(Pipeline(Command("cat"))), + ) + proc = await cmd.run(context) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert all(task.done() for task in context.tasks) + + +@pytest.mark.asyncio +async def test_cleanup_still_raises_inner_launch_failure(): + """An inner command that cannot be spawned still propagates from + cleanup(), but only after all resources have been released.""" + context = Context() + cmd = Command( + "cat", InputProcSub(Pipeline(Command("no_such_cmd_zzz"))) + ) + proc = await cmd.run(context) + await proc.wait() + + with pytest.raises(FileNotFoundError): + await asyncio.wait_for(context.cleanup(), timeout=10) + + assert not os.path.exists(context.temp_dir.name) + + +@pytest.mark.asyncio +async def test_cleanup_does_not_stall_running_inner_writer(): + """cleanup() must not touch the FIFO of a proc sub whose inner command + is already running; holding a read end there would prevent the EPIPE + that lets the inner writer exit.""" + context = Context() + cmd = Command( + "sh", + "-c", + 'sleep 0.5 <"$1"', + "sh", + InputProcSub(Pipeline(Command("yes"))), + ) + proc = await cmd.run(context) + # Wait until the inner command has spawned (sh + yes). + async with asyncio.timeout(5): + while len(context.commands) < 2: + await asyncio.sleep(0.01) + + await asyncio.wait_for(context.cleanup(), timeout=10) + assert await proc.wait() == 0 + + +@pytest.mark.asyncio +async def test_cleanup_twice_is_safe(): + """A second cleanup() call is a harmless no-op.""" + context = Context() + cmd = Command("cat", InputProcSub(Pipeline(Command("echo", "hello")))) + proc = await cmd.run(context, stdout=asyncio.subprocess.DEVNULL) + await proc.wait() + + await asyncio.wait_for(context.cleanup(), timeout=10) + await asyncio.wait_for(context.cleanup(), timeout=10) + assert proc.returncode == 0