Add Pipeline feature

.clippy.toml

@@ -1,3 +1,4 @@
 excessive-nesting-threshold = 4
 too-many-arguments-threshold = 10
 allowed-idents-below-min-chars = ["..", "k", "v", "f", "re", "id", "Ok", "'_"]
+large-error-threshold = 129

.devcontainer/Dockerfile

@@ -1,32 +1,40 @@
 # DevContainer image
-FROM rust:1.87-slim
+FROM ubuntu:24.04
 RUN \
-    adduser --system --disabled-password --shell /bin/bash --home /home/vscode vscode && \
     # install docker
     apt-get update && \
-    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    apt-get install ca-certificates curl -y && \
     mkdir -m 0755 -p /etc/apt/keyrings && \
-    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
-    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \
+    chmod a+r /etc/apt/keyrings/docker.asc && \
+    echo \
+        "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+        $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+        tee /etc/apt/sources.list.d/docker.list > /dev/null && \
     apt-get update && \
     apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
-    usermod -aG docker vscode && \
+    usermod -aG docker ubuntu && \
     apt-get clean
 RUN \
     # dev setup
     apt update && \
-    apt-get install sudo jq git bash-completion graphviz rsync -y && \
-    usermod -aG sudo vscode && \
+    apt-get install sudo jq bash-completion graphviz rsync software-properties-common gcc -y && \
+    add-apt-repository ppa:git-core/ppa && \
+    apt update && \
+    apt-get install git -y && \
+    usermod -aG sudo ubuntu && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
-    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
-    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
-    chown vscode:nogroup /home/vscode/.bashrc && \
+    echo '. /etc/bash_completion' >> /home/ubuntu/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/ubuntu/.bashrc && \
+    chown ubuntu:nogroup /home/ubuntu/.bashrc && \
     apt-get clean
 
-USER vscode
-ENV CARGO_HOME=/home/vscode/.cargo \
-    PATH=/home/vscode/.cargo/bin:/home/vscode/.local/bin:$PATH
+USER ubuntu
+ENV CARGO_HOME=/home/ubuntu/.cargo \
+    PATH=/home/ubuntu/.cargo/bin:/home/ubuntu/.local/bin:$PATH
 RUN \
+    # install rust
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=1.87.0 -y && \
     # rust auto formatting
     rustup component add rustfmt && \
     # rust style linter
@@ -47,5 +55,5 @@ RUN \
     # useful in examples
     uv pip install ipykernel eclipse-zenoh -p ~/.local/share/base && \
     echo '. ~/.local/share/base/bin/activate' >> ~/.bashrc
-ENV VIRTUAL_ENV=/home/vscode/.local/share/base
+ENV VIRTUAL_ENV=/home/ubuntu/.local/share/base
 CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]

.devcontainer/devcontainer.json

@@ -1,29 +1,15 @@
 {
 	"name": "Development",
-	"build": {
-		"dockerfile": "Dockerfile",
-		"context": ".."
-	},
-	"overrideCommand": false,
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "ide",
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	"mounts": [
-		"source=${localWorkspaceFolderBasename}_devcontainer_docker_data,target=/var/lib/docker,type=volume"
-	],
-	"remoteEnv": {
-		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}",
-		"RUST_BACKTRACE": "1"
-	},
+	"initializeCommand": ".devcontainer/make_env_file.py ${localWorkspaceFolder}",
 	"postStartCommand": "mkdir -p tests/.tmp && docker system prune -fa && docker volume prune -f",
 	"hostRequirements": {
 		"cpus": 2,
 		"memory": "8gb",
 		"storage": "32gb"
 	},
-	"runArgs": [
-		"--name=${localWorkspaceFolderBasename}_devcontainer",
-		"--privileged",
-		"--cgroupns=host"
-	],
 	"customizations": {
 		"vscode": {
 			"extensions": [

.devcontainer/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  ide:
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+    privileged: true   # Docker-in-Docker needs this
+    cgroup: host       # Allow setting resource constraints (e.g. memory, cpu) for containers
+    environment:
+    - RUST_BACKTRACE=1 # Display error trace on errors/panics
+    volumes:
+    - ${LOCAL_WORKSPACE_PATH}:${ENV_WORKSPACE_PATH} # source
+    - docker_data:/var/lib/docker                   # Docker image cache
+    - ${LOCAL_GIT_PATH}:${ENV_GIT_PATH}             # Base Git worktree history
+volumes:
+  docker_data:

.devcontainer/make_env_file.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+#
+# assumes git worktree was made using relative paths e.g.
+# git worktree add ../dir-name branch-name --relative-paths
+import sys
+from pathlib import Path
+from textwrap import dedent
+import subprocess
+from os.path import relpath
+
+
+def make_config(local_workspace_path: Path):
+    env_workspace_path = Path(f"/workspaces/{local_workspace_path.name}")
+    local_git_path = Path(
+        subprocess.run(
+            "git rev-parse --path-format=absolute --git-common-dir".split(" "),
+            capture_output=True,
+            text=True,
+        ).stdout.strip()
+    )
+    env_git_path = (
+        env_workspace_path / ".git"
+        if local_workspace_path / ".git" == local_git_path
+        else (
+            env_workspace_path
+            / relpath(local_git_path.parent, local_workspace_path)
+            / ".git"
+        ).resolve()
+    )
+
+    return dedent(
+        f"""
+        LOCAL_WORKSPACE_PATH={local_workspace_path}
+        ENV_WORKSPACE_PATH={env_workspace_path}
+        LOCAL_GIT_PATH={local_git_path}
+        ENV_GIT_PATH={env_git_path}
+        """
+    ).strip()
+
+
+if __name__ == "__main__":
+    local_workspace_path = Path(sys.argv[1])
+
+    with open(Path(__file__).absolute().parent / ".env", "w") as f:
+        f.write(make_config(local_workspace_path))

.github/actions/build-diagram/action.yaml

@@ -9,20 +9,25 @@ runs:
         sudo apt-get install graphviz -y
         cargo install cargo-modules
     - shell: bash
+      env:
+        DIAGRAM_SCOPE:
+          orcapod::uniffi::{
+            model::{Pod, PodJob, PodResult, Pipeline, PipelineJob, PipelineResult},
+            orchestrator::{
+              PodRun,
+              docker::LocalDockerOrchestrator,
+              agent::{AgentClient, Agent},
+            },
+            pipeline::PipelineRun,
+            store::filestore::LocalFileStore,
+          }
       run: |
         : Generate crate diagram
         mkdir -p docs/images
         set -o pipefail
-        cargo modules dependencies --lib --max-depth 0 \
-          --no-uses --no-fns \
-          --focus-on "
-            orcapod::uniffi::{
-              model::{Pod,PodJob,PodResult},
-              store::filestore::LocalFileStore,
-              orchestrator::{PodRun,docker::LocalDockerOrchestrator},
-            }
-          " \
-          --layout dot | \
+        cargo modules dependencies --lib \
+          --max-depth 0 --no-uses --no-fns \
+          --focus-on ${DIAGRAM_SCOPE// /} --layout dot | \
           dot -T svg > docs/images/orcapod_diagram.svg
     - uses: actions/upload-artifact@v4
       with:

.github/workflows/tests.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Install code coverage
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Run syntax and style tests
-        run: cargo clippy --all-targets -- -D warnings
+        run: cargo clippy --no-default-features --features=test --all-targets -- -D warnings
       - name: Run format test
         run: cargo fmt --check
       - name: Run integration tests w/ coverage report
@@ -31,6 +31,8 @@ jobs:
         run: |
           mkdir -p tests/.tmp
           cargo llvm-cov \
+            --no-default-features \
+            --features=test \
             --ignore-filename-regex "bin/.*|lib\.rs" \
             --cobertura \
             --output-path target/llvm-cov-target/cobertura.xml \

.gitignore

@@ -6,3 +6,4 @@ lcov.info
 *.tar.gz
 *.dot
 tests/.tmp
+*.pyc

.vscode/launch.json

@@ -8,6 +8,8 @@
             "cargo": {
                 "args": [
                     "test",
+                    "--no-default-features",
+                    "--features=test",
                     "--no-run",
                     "--package=orcapod",
                     "--test",
@@ -31,6 +33,8 @@
             "cargo": {
                 "args": [
                     "test",
+                    "--no-default-features",
+                    "--features=test",
                     "--no-run",
                     "--package=orcapod",
                     "--test",
@@ -53,6 +57,8 @@
             "cargo": {
                 "args": [
                     "test",
+                    "--no-default-features",
+                    "--features=test",
                     "--no-run",
                     "--lib",
                     "--package=orcapod"
@@ -86,6 +92,8 @@
             "cargo": {
                 "args": [
                     "test",
+                    "--no-default-features",
+                    "--features=test",
                     "--no-run",
                     "--package=orcapod",
                     "--bin=exe_file_stem"

.vscode/settings.json

@@ -13,10 +13,21 @@
         "max_width=100"
     ],
     "rust-analyzer.check.command": "clippy",
+    "rust-analyzer.cargo.noDefaultFeatures": true,
+    "rust-analyzer.cargo.features": [
+        "test"
+    ],
+    "rust-analyzer.runnables.extraTestBinaryArgs": [
+        "--nocapture"
+    ],
     "files.autoSave": "off",
     "gitlens.showWelcomeOnInstall": false,
     "gitlens.showWhatsNewAfterUpgrades": false,
     "lldb.consoleMode": "evaluate",
+    "jupyter.kernels.excludePythonEnvironments": [
+        "/bin/python3",
+        "/usr/bin/python3",
+    ],
     "python.defaultInterpreterPath": "~/.local/share/base/bin/python3",
     "python.terminal.activateEnvironment": false,
     "notebook.formatOnSave.enabled": true,

Cargo.toml

@@ -23,6 +23,10 @@ categories = [
 license = "MIT license"
 edition = "2024"
 
+[features]
+default = []
+test = []
+
 [lib]
 crate-type = ["rlib", "cdylib"]
 
@@ -37,6 +41,8 @@ chrono = "0.4.39"
 colored = "2.1.0"
 # derive utilities for new types
 derive_more = { version = "2.0.1", features = ["display"] }
+# traverse DOT graphs
+dot-parser = "0.5.1"
 # allow sync functions to be called from async using `block_on`
 futures-executor = "0.3.31"
 # chaining async calls and processing stream data in local docker orchestrator
@@ -47,10 +53,22 @@ getset = { version = "0.1.5", git = "https://github.com/guzman-raphael/getset.gi
 glob = "0.3.1"
 # strings to snake_case
 heck = "0.5.0"
+# todo: temporary dependency until hash implemented for PipelineJob
+hex = "0.4.3"
 # hashmaps that preserve insertion order
 indexmap = { version = "2.9.0", features = ["serde"] }
+# pretty multiline strings
+indoc = "2.0.5"
+# support for cartesian products
+itertools = "0.14.0"
+# minimal text template engine similar to Jinja2
+minijinja = "2.11.0"
 # random name generator
 names = "0.14.0"
+# graph algorithms
+petgraph = { version = "0.8.2", features = ["serde-1", "dot_parser"] }
+# todo: temporary dependency until hash implemented for PipelineJob
+rand = "0.9.1"
 # complex pattern matching in strings
 regex = "1.11.0"
 # serialization/deserialization to/from filestore
@@ -70,14 +88,12 @@ tokio-util = "0.7.13"
 # automated CFFI + bindings in other languages
 uniffi = { version = "0.29.1", features = ["cli", "tokio"] }
 # shared, distributed memory via communication
-zenoh = { version = "1.3.4" }
+zenoh = "1.3.4"
 
 [[bin]]
 name = "uniffi-bindgen"
 
 [dev-dependencies]
-# pretty multiline strings
-indoc = "2.0.5"
 # creating temp directories
 tempfile = "3.13.0"
 
@@ -113,6 +129,7 @@ impl_trait_in_params = { level = "allow", priority = 127 }             # impl in
 implicit_return = { level = "allow", priority = 127 }                  # missing return ok
 inline_asm_x86_intel_syntax = { level = "allow", priority = 127 }      # intel syntax ok
 integer_division = { level = "allow", priority = 127 }                 # allow discarding remainder
+iter_over_hash_type = { level = "allow", priority = 127 }              # allow iterating over unordered iterables like `HashMap`
 little_endian_bytes = { level = "allow", priority = 127 }              # allow to_le_bytes / from_le_bytes
 missing_asserts_for_indexing = { level = "allow", priority = 127 }     # missing assert before indexing ok
 missing_docs_in_private_items = { level = "allow", priority = 127 }    # missing docs on private ok

README.md

@@ -11,22 +11,23 @@
 ```bash
 #!/bin/bash
 set -e  # fail early on non-zero exit
-cargo clippy --all-targets -- -D warnings  # Rust syntax and style tests
+cargo clippy --no-default-features --features=test --all-targets -- -D warnings  # Rust syntax and style tests
 cargo fmt --check  # Rust formatting test
-cargo llvm-cov --ignore-filename-regex "bin/.*|lib\.rs" -- --nocapture  # Rust integration tests w/ stdout coverage summary
-cargo llvm-cov --ignore-filename-regex "bin/.*|lib\.rs" --html -- --nocapture  # Rust integration tests w/ HTML coverage report (target/llvm-cov/html/index.html)
-cargo llvm-cov --ignore-filename-regex "bin/.*|lib\.rs" --codecov --output-path target/llvm-cov-target/codecov.json -- --nocapture  # Rust integration tests w/ codecov coverage report
-cargo llvm-cov --ignore-filename-regex "bin/.*|lib\.rs" --cobertura --output-path target/llvm-cov-target/cobertura.xml -- --nocapture  # Rust integration tests w/ cobertura coverage report
+cargo llvm-cov --no-default-features --features=test --no-clean --ignore-filename-regex "bin/.*|lib\.rs" -- --nocapture  # Rust integration tests w/ stdout coverage summary
+cargo llvm-cov --no-default-features --features=test --no-clean --ignore-filename-regex "bin/.*|lib\.rs" --html -- --nocapture  # Rust integration tests w/ HTML coverage report (target/llvm-cov/html/index.html)
+cargo llvm-cov --no-default-features --features=test --no-clean --ignore-filename-regex "bin/.*|lib\.rs" --codecov --output-path target/llvm-cov-target/codecov.json -- --nocapture  # Rust integration tests w/ codecov coverage report
+cargo llvm-cov --no-default-features --features=test --no-clean --ignore-filename-regex "bin/.*|lib\.rs" --cobertura --output-path target/llvm-cov-target/cobertura.xml -- --nocapture  # Rust integration tests w/ cobertura coverage report
 . ~/.local/share/base/bin/activate && maturin develop --uv && export RUST_BACKTRACE=1 && python tests/extra/python/smoke_test.py -- tests/.tmp && python tests/extra/python/agent_test.py # Python integration tests
 ```
 
 ## Docs
 
 ```bash
 cargo doc --no-deps                       # gen api docs (target/doc/orcapod/index.html)
-cargo modules dependencies --lib --no-uses --no-fns --focus-on "orcapod::uniffi::{model::{Pod,PodJob,PodResult},store::filestore::LocalFileStore,orchestrator::{PodRun,docker::LocalDockerOrchestrator}}" --layout dot > docs/images/orcapod_diagram.dot # orcapod diagram as DOT
-cargo modules dependencies --lib --no-uses --no-fns --focus-on "orcapod::uniffi::{model::{Pod,PodJob,PodResult},store::filestore::LocalFileStore,orchestrator::{PodRun,docker::LocalDockerOrchestrator}}" --layout dot | dot -T png > docs/images/orcapod_diagram.png # orcapod diagram as PNG
-cargo modules dependencies --lib --no-uses --no-fns --focus-on "orcapod::uniffi::{model::{Pod,PodJob,PodResult},store::filestore::LocalFileStore,orchestrator::{PodRun,docker::LocalDockerOrchestrator}}" --layout dot | dot -T svg > docs/images/orcapod_diagram.svg # orcapod diagram as SVG
+DIAGRAM_SCOPE="orcapod::uniffi::{model::{Pod, PodJob, PodResult, Pipeline, PipelineJob, PipelineResult},orchestrator::{PodRun,docker::LocalDockerOrchestrator,agent::{AgentClient, Agent}},pipeline::PipelineRun,store::filestore::LocalFileStore}"
+cargo modules dependencies --lib --max-depth 0 --no-uses --no-fns --focus-on $DIAGRAM_SCOPE --layout dot > docs/images/orcapod_diagram.dot # orcapod diagram as DOT
+cargo modules dependencies --lib --max-depth 0 --no-uses --no-fns --focus-on $DIAGRAM_SCOPE --layout dot | dot -T png > docs/images/orcapod_diagram.png # orcapod diagram as PNG
+cargo modules dependencies --lib --max-depth 0 --no-uses --no-fns --focus-on $DIAGRAM_SCOPE --layout dot | dot -T svg > docs/images/orcapod_diagram.svg # orcapod diagram as SVG
 ```
 
 ## Project Management

cspell.json

@@ -18,7 +18,8 @@
         "cffi",
         "zenoh",
         "PodJob",
-        "stresser"
+        "stresser",
+        "petgraph"
     ],
     "ignoreWords": [
         "relpath",
@@ -77,7 +78,9 @@
         "strsim",
         "getrandom",
         "wasi",
-        "patchelf"
+        "patchelf",
+        "itertools",
+        "pipefail"
     ],
     "useGitignore": false,
     "ignorePaths": [