Make output folder part of stage templates (#144)

* Do not add "output" directory when executing stage

* Update CHANGELOG

* Create output directory as part of template

* Add output directories to template

Author: grst

CHANGELOG.md

@@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning][].
 
 ### Fixes
 
+-  Fix how the `output` directory in stages is treated. Previously, the `output` directory was not
+   part of of the stage template. Instead it got added dynamically during `dso exec quarto` and removed
+   again in case it was empty. This could lead to failures of `dvc repro`, because `dvc` removed the `output`
+   directory on repro, expecting it to be recreated by the script. Now, the `output` directory is part of the
+   stage template, and a `mkdir -p output` is included in the stage command, ensuring that the directory will
+   always be created ([#144](https://github.com/Boehringer-Ingelheim/dso/pull/144)).
 -  Exit gracefully when aborting template selections ([#134](https://github.com/Boehringer-Ingelheim/dso/pull/134)).
 -  Make sure watermarking doesn't modify external images inplace ([#140](https://github.com/Boehringer-Ingelheim/dso/pull/140)).
 -  Do not exclude `meta.yaml` from templates. We leave it up to the templates to .gitignore them ([#142](https://github.com/Boehringer-Ingelheim/dso/pull/142)).
@@ -50,7 +56,7 @@ and this project adheres to [Semantic Versioning][].
 
 ### Template updates
 
--   Add `_quarto.yml` to "init" template ([#115](https://github.com/Boehringer-Ingelheim/dso/pull/115))
+-   Add `_quarto.yml` to "init" template `.gitignore` ([#115](https://github.com/Boehringer-Ingelheim/dso/pull/115))
 
 ### Chore
 

src/dso/_util.py

@@ -5,7 +5,6 @@
 import sys
 import tomllib
 from collections.abc import Sequence
-from contextlib import contextmanager
 from functools import cache
 from os import environ
 from pathlib import Path
@@ -160,20 +159,6 @@ def _update_dot_dso_json(dir: Path, update_dict: dict):
         json.dump(config, f)
 
 
-@contextmanager
-def add_directory(dir: Path):
-    """Context manager that temporarily creates a directory and removes it again if it's empty"""
-    dir.mkdir(exist_ok=True)
-    try:
-        yield
-    finally:
-        try:
-            dir.rmdir()
-        except OSError:
-            # directory not empty
-            pass
-
-
 def check_ask_pre_commit(dir: Path):
     """
     Check if pre-commit hooks are installed and asks to install them

src/dso/cli/_exec.py

@@ -5,7 +5,6 @@
 from ruamel.yaml import YAML
 
 from dso._logging import log
-from dso._util import add_directory
 
 
 @click.command("quarto")
@@ -74,15 +73,14 @@ def dso_exec_quarto(stage: str, skip_compile: bool = True):
         except KeyError:
             pass
 
-    with add_directory(stage_dir / "output"):
-        with quarto_config_yml(quarto_config, stage_dir / "src"):
-            render_quarto(
-                stage_dir / "src",
-                report_dir=stage_dir / "report",
-                before_script=before_script,
-                cwd=stage_dir,
-                with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config,
-            )
+    with quarto_config_yml(quarto_config, stage_dir / "src"):
+        render_quarto(
+            stage_dir / "src",
+            report_dir=stage_dir / "report",
+            before_script=before_script,
+            cwd=stage_dir,
+            with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config,
+        )
 
 
 @click.group(name="exec")

src/dso/templates/stage/quarto_ipynb/dvc.yaml

@@ -8,4 +8,5 @@ stages:
       - output
       - report/{{ name }}.html
     cmd:
+      - mkdir -p output
       - uv run dso exec quarto .

src/dso/templates/stage/quarto_ipynb/output/.gitkeeptemplate

@@ -8,4 +8,5 @@ stages:
       - output
       - report/{{ name }}.html
     cmd:
+      - mkdir -p output
       - uv run dso exec quarto .

src/dso/templates/stage/quarto_py/dvc.yaml

@@ -8,4 +8,5 @@ stages:
       - output
       - report/{{ name }}.html
     cmd:
+      - mkdir -p output
       - dso exec quarto .