DockerStream

Class to stream the output of a docker run command via the subprocess module. Each Batch object will create its own DockerStream object to run the appropriate snakemake rule within a docker container. This will then stream the container's output back to the Batch.

Parameters:
  • name (str | None) –

    Name of the stream. If None, will just be given a random UUID

  • image (str) –

    Name or URL to docker image, already built

  • outdir (Path | str) –

    Directory for final generated reports

  • contdir (str) –

    Working directory in the container

  • files (list[Path]) –

    List of file paths, as generated by Batch's file pattern and ID data

  • print_quarto (bool) –

    Whether to run Quarto in verbose mode, including its printout of each chunk rendered

  • print_snakemake (bool) –

    Whether to run snakemake in verbose mode

Source code in src/eqcli/docker.py
class DockerStream:
    """Class to stream the output of a `docker run` command via the subprocess module. Each `Batch` object will create its own `DockerStream` object to run the appropriate snakemake rule within a docker container. This will then stream the container's output back to the `Batch`.

    Parameters
    ----------
    name : str | None
        Name of the stream. If None, will just be given a random UUID
    image : str
        Name or URL to docker image, already built
    outdir : Path | str
        Directory for final generated reports
    contdir : str
        Working directory in the container
    files : list[Path]
        List of file paths, as generated by `Batch`'s file pattern and ID data
    print_quarto : bool
        Whether to run Quarto in verbose mode, including its printout of each chunk rendered
    print_snakemake : bool
        Whether to run snakemake in verbose mode
    """

    def __init__(
        self,
        name: str | None,
        image: str,
        outdir: Path | str,
        contdir: str,
        files: list[Path],
        print_quarto: bool,
        print_snakemake: bool,
    ):
        if name is None:
            self.name = f"docker-{uuid.uuid4()}"
        else:
            self.name = name
        self.image = image
        # self.files = files
        self.files = [f"{contdir}/{fn}" for fn in files]
        self.quiet_quarto = not print_quarto
        self.snake_quiet_opt = self._make_snakeopt(print_snakemake)
        self.volume = self._make_volume(outdir, contdir)
        # self.stream = self.stream_docker()

    def _make_snakeopt(self, print_snakemake):
        if print_snakemake:
            return "host"
        else:
            return "all"

    def _make_volume(self, outdir, contdir):
        outdir = Path(outdir)
        # if contdir[0] != "/":
        #     contdir = f"/{contdir}"
        return f"{outdir.absolute()}:/project/{contdir}"

    def stream_docker(self):
        quiet_config = f"quiet={self.quiet_quarto}"
        process = subprocess.Popen(
            [
                "docker",
                "run",
                # "--rm",
                "-v",
                self.volume,
                self.image,
                *self.files,
                "--cores",
                "all",
                "--config",
                quiet_config,
                "--quiet",
                self.snake_quiet_opt,
                "--keep-going",
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            universal_newlines=True,
        )
        return process.stdout

    def __str__(self) -> str:
        return f"""
Docker container: {self.name}
Image: '{self.image}'
Volume: {self.volume}"""