Merge pull request #12 from htcondor/v0.2.0

v0.2.0

Author: JoshKarpel

.codecov.yml

@@ -0,0 +1,7 @@
+coverage:
+  range: 50..90
+  round: down
+  precision: 0
+  
+comment:
+  layout: "diff, files"

.dockerignore

@@ -105,6 +105,11 @@ venv.bak/
 .mypy_cache/
 
 .idea/
+.vscode/
 
 Dockerfile
 dr
+
+/examples/mandelbrot/mandelbrot-dag/
+/examples/mandelbrot/test.png
+/examples/mandelbrot/test.ppm

.gitignore

@@ -106,3 +106,7 @@ venv.bak/
 # Editors
 .idea/
 .vscode/
+
+/examples/mandelbrot/mandelbrot-dag/
+/examples/mandelbrot/test.png
+/examples/mandelbrot/test.ppm

docs/source/api.rst

@@ -3,6 +3,10 @@ API Reference
 
 .. py:currentmodule:: htcondor_dags
 
+.. attention::
+    This is not documentation for DAGMan itself! If you run into DAGMan jargon
+    that isn't explained here, see `The DAGMan Manual <https://htcondor.readthedocs.io/en/latest/users-manual/dagman-applications.html>`_.
+
 Creating DAGs
 -------------
 
@@ -11,6 +15,7 @@ Creating DAGs
 
 .. autoclass:: WalkOrder
 
+
 Nodes and Node-likes
 ++++++++++++++++++++
 
@@ -28,8 +33,15 @@ Nodes and Node-likes
 .. autoclass:: Nodes
    :members:
 
+
+Edges
++++++
+
 .. autoclass:: OneToOne
 .. autoclass:: ManyToMany
+.. autoclass:: Grouper
+.. autoclass:: Slicer
+
 
 Node Configuration
 ++++++++++++++++++
@@ -39,6 +51,12 @@ Node Configuration
 .. autoclass:: DAGAbortCondition
 
 
+Writing a DAG to Disk
++++++++++++++++++++++
+
+.. autofunction:: write_dag
+
+
 DAG Configuration
 -----------------
 
@@ -47,3 +65,9 @@ DAG Configuration
 .. autoclass:: NodeStatusFile
 
 
+Rescue DAGs
+-----------
+
+.. autofunction:: rescue
+
+.. autofunction:: find_rescue_file

docs/source/conf.py

@@ -43,6 +43,7 @@
     "sphinx.ext.mathjax",
     "sphinx.ext.ifconfig",
     "sphinx.ext.viewcode",
+    "sphinx.ext.graphviz",
     "nbsphinx",
 ]
 
@@ -80,8 +81,8 @@
 
 autodoc_member_order = "bysource"
 autoclass_content = "both"
-autodoc_default_flags = ["undoc-members"]
-
+autodoc_default_options = {"undoc-members": True}
+add_module_names = False
 napoleon_use_rtype = False
 
 

docs/source/versions/v0_2_0.rst

@@ -0,0 +1,36 @@
+v0.2.0
+======
+
+.. py:currentmodule:: htcondor_dags
+
+New Features
+------------
+
+* You can now walk over all of the ancestors of a node
+  (i.e., its parents, the parents of its parents, the parents of parents of
+  its parents, etc.) using :meth:`~BaseNode.walk_ancestors`,
+  and similarly for its descendants using :meth:`~BaseNode.walk_descendants`.
+* Implemented a new :class:`~Grouper` edge. It allows flexible "chunking" of the
+  parent and child layers. For example, every three nodes in the parent layer can
+  be connected to every two nodes in the child layer.
+* Custom edges can now be created by implementing the :class:`~BaseEdge` interface.
+* ``NOOP`` and ``DONE`` can now be set on individual nodes inside a layer.
+  In addition to being a single boolean value, the corresponding :class:`~NodeLayer`
+  attributes can now be set to a dictionary mapping node indices to a boolean.
+  Missing indices are assumed to be ``False``
+  For example, if node index 5 is not present in the ``noop`` dictionary, it is not ``NOOP``.
+  It is only ``NOOP`` if it is present and its value is ``True``.
+* The submit description for a :class:`~NodeLayer` can be given as a path to an
+  existing HTCondor submit file.
+* State information from previous DAGMan runs, encoded in a DAGMan "rescue" file,
+  can now be loaded and laid over an existing :class:`~DAG` using the new
+  :func:`~rescue` function.
+
+
+Bug Fixes
+---------
+
+
+Known Issues
+------------
+

dr.cmd

@@ -0,0 +1,17 @@
+@echo off
+
+SET CONTAINER_TAG=htcondor-dags-test-container
+
+docker build ^
+       --quiet ^
+       --tag %CONTAINER_TAG% ^
+       --file tests/_inf/Dockerfile ^
+       .
+
+docker run ^
+       -it ^
+       --rm ^
+       --mount type=bind,source="%CD%",target=/home/dagger/htcondor-dags ^
+       -p 8000:8000 ^
+       %CONTAINER_TAG% ^
+       %*

examples/basic_diamond/make_dag.py

@@ -38,7 +38,7 @@
 # to vars.
 # Because NUM_CHUNKS is 5, this will create 5 DAGMan nodes, one to process
 # each chunk created by the previous layer.
-count_words = split_words.child(
+count_words = split_words.child_layer(
     name="count_words",
     submit_description=htcondor.Submit(
         {
@@ -57,7 +57,7 @@
 # of the previous step.
 # It's input files are all of the output files from the previous step, which
 # is easy in this case because we know the naming scheme.
-combine_counts = count_words.child(
+combine_counts = count_words.child_layer(
     name="combine_counts",
     submit_description=htcondor.Submit(
         {
@@ -78,5 +78,5 @@
 # If you write it out to a different directory, you may need to be careful
 # about filepaths in your submit descriptions!
 this_dir = Path(__file__).parent
-diamond.write(this_dir)
+dags.write_dag(diamond, this_dir)
 print(f"Wrote DAG files to {this_dir}")

examples/delayed_diamond/README.md

@@ -16,7 +16,7 @@ This will necessarily change the structure of our workflow. In the Basic Diamond
 example, the `count_words` layer looked like this:
 
 ```python
-count_words = split_words.child(
+count_words = split_words.child_layer(
     name="count_words",
     submit_description=Submit(
         {

examples/delayed_diamond/make_analysis_dag.py

@@ -31,7 +31,7 @@
 )
 
 # This is the "combine the counts from each chunk" step.
-combine_counts = count_words.child(
+combine_counts = count_words.child_layer(
     name="combine_counts",
     submit_description=htcondor.Submit(
         {
@@ -52,5 +52,5 @@
 # If you write it out to a different directory, you may need to be careful
 # about filepaths in your submit descriptions!
 this_dir = Path(__file__).parent
-analysis_dag.write(this_dir, dag_file_name="analysis.dag")
+dags.write_dag(analysis_dag, this_dir, dag_file_name="analysis.dag")
 print(f"Wrote DAG files to {this_dir}")

examples/delayed_diamond/make_top_dag.py

@@ -29,5 +29,5 @@
 # Now that we're going to have two DAG input files in this directory, we need
 # to give them unique names.
 this_dir = Path(__file__).parent
-top_layer_dag.write(this_dir, dag_file_name="top_level.dag")
+dags.write_dag(top_layer_dag, this_dir, dag_file_name="top_level.dag")
 print(f"Wrote DAG files to {this_dir}")

examples/mandelbrot/Dockerfile

@@ -0,0 +1,17 @@
+FROM htcondor/htc-base-notebook:latest
+
+USER root
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends imagemagick-6.q16 graphviz \
+ && apt-get -y clean \
+ && rm -rf /var/lib/apt/lists/*
+
+USER jovyan
+
+RUN pip install --upgrade htcondor==8.9.4b1 graphviz
+
+COPY --chown=jovyan:0 . /home/jovyan/htcondor-dags
+RUN pip install -e /home/jovyan/htcondor-dags
+
+WORKDIR /home/jovyan/htcondor-dags/examples/mandelbrot

examples/mandelbrot/edit.cmd

@@ -0,0 +1,6 @@
+@echo off
+
+SET CONTAINER_TAG=htcondor-dags-mandelbrot-example
+
+docker build -t %CONTAINER_TAG% --file examples/mandelbrot/Dockerfile . || exit /b
+docker run -it --rm -p 8888:8888 --mount type=bind,source="%cd%",target=/home/jovyan/htcondor-dags %CONTAINER_TAG% %* || exit /b