Skip to content

Collect and report all errors in dora check (#1359) #16

Collect and report all errors in dora check (#1359)

Collect and report all errors in dora check (#1359) #16

Workflow file for this run

name: CI
on:
push:
branches:
- main
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CI: true
jobs:
test:
name: "Test"
strategy:
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.platform }}
timeout-minutes: 90
steps:
- name: Print available space (Windows only)
run: Get-PSDrive
if: runner.os == 'Windows'
- name: Override cargo target dir (Windows only)
run: echo "CARGO_TARGET_DIR=C:\cargo-target" >> "$GITHUB_ENV"
shell: bash
if: runner.os == 'Windows'
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
if: runner.os == 'Linux'
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: false
- name: Free disk Space (Windows)
if: runner.os == 'Windows'
run: |
docker system prune --all -f
docker builder prune -af
Remove-Item "C:\Android" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\Ruby" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\go" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\Java*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\CodeQL" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files (x86)\Microsoft SDKs\Azure" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\Azure Cosmos DB Emulator" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\MongoDB" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\mysql*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\PostgreSQL" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Selenium" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\SeleniumWebDrivers" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\dotnet\sdk\*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\dotnet\shared" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files (x86)\Microsoft Visual Studio\2019" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\ProgramData\Chocolatey" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\mingw64" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Strawberry" -Force -Recurse -ErrorAction SilentlyContinue
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
cache-directories: ${{ env.CARGO_TARGET_DIR }}
- name: "Check"
run: cargo check --all
- name: "Build (Without Python dep as it is build with maturin)"
run: cargo build --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
- name: "Test"
run: cargo test --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
# Run examples as separate job because otherwise we will exhaust the disk
# space of the GitHub action runners.
examples:
name: "Examples"
strategy:
matrix:
platform: [ubuntu-22.04, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.platform }}
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
if: runner.os == 'Linux'
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: true
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- name: Free disk Space (Windows)
if: runner.os == 'Windows'
run: |
docker system prune --all -f
docker builder prune -af
Remove-Item "C:\Android" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\Ruby" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\go" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\Java*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\hostedtoolcache\windows\CodeQL" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files (x86)\Microsoft SDKs\Azure" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\Azure Cosmos DB Emulator" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\MongoDB" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\mysql*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\PostgreSQL" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Selenium" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\SeleniumWebDrivers" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\dotnet\sdk\*" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files\dotnet\shared" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Program Files (x86)\Microsoft Visual Studio\2019" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\ProgramData\Chocolatey" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\mingw64" -Force -Recurse -ErrorAction SilentlyContinue
Remove-Item "C:\Strawberry" -Force -Recurse -ErrorAction SilentlyContinue
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
# CLI tests
- name: "Build cli and binaries"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
run: |
cargo install --path binaries/cli --locked
# general examples
- name: "Build examples"
timeout-minutes: 30
run: cargo build --examples
- name: "Rust Dataflow example"
timeout-minutes: 30
run: cargo run --example rust-dataflow
- name: "Rust Git Dataflow example"
timeout-minutes: 30
run: cargo run --example rust-dataflow-git
- name: "Multiple Daemons example"
timeout-minutes: 30
run: cargo run --example multiple-daemons
- name: "C Dataflow example"
timeout-minutes: 15
run: cargo run --example c-dataflow
- name: "C++ Dataflow example"
timeout-minutes: 15
run: cargo run --example cxx-dataflow
- name: "Install Arrow C++ Library"
timeout-minutes: 10
shell: bash
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
# For Ubuntu
sudo apt-get update
sudo apt-get install -y -V ca-certificates lsb-release wget
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt-get update
sudo apt-get install -y -V libarrow-dev libarrow-glib-dev
elif [ "$RUNNER_OS" == "macOS" ]; then
# For macOS
brew update
brew install apache-arrow
fi
- name: "C++ Dataflow2 example"
timeout-minutes: 15
run: cargo run --example cxx-arrow-dataflow
- name: "Cmake example"
if: runner.os == 'Linux'
timeout-minutes: 30
run: cargo run --example cmake-dataflow
- name: "Unix Domain Socket example"
if: runner.os == 'Linux'
run: cargo run --example rust-dataflow -- dataflow_socket.yml
# ROS2 bridge examples
ros2-bridge-examples:
name: "ROS2 Bridge Examples"
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
if: runner.os == 'Linux'
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: false
- run: cargo --version --verbose
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: humble
- run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"'
- name: Install the latest version of uv
uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- name: Install pyarrow
run: pip install pyarrow
- name: "Test"
run: cargo test -p dora-ros2-bridge-python
- name: "Rust ROS2 Bridge example"
timeout-minutes: 30
env:
QT_QPA_PLATFORM: offscreen
run: |
source /opt/ros/humble/setup.bash &&
cargo run -p dora-ros2-bridge --example rust-ros2-dataflow
- uses: actions/setup-python@v5
if: runner.os != 'Windows'
with:
python-version: "3.8"
- uses: actions/setup-python@v5
if: runner.os == 'Windows'
with:
python-version: "3.10"
- name: "python-ros2-dataflow"
timeout-minutes: 30
env:
QT_QPA_PLATFORM: offscreen
run: |
source /opt/ros/humble/setup.bash &&
uv venv --seed -p 3.12
## If not Windows
if [ "$RUNNER_OS" != "Windows" ]; then
source .venv/bin/activate
else
source .venv/Scripts/activate
fi
uv pip install -e apis/python/node
uv pip install pyarrow
cargo run -p dora-ros2-bridge --example python-ros2-dataflow
- name: "c++-ros2-dataflow"
timeout-minutes: 30
env:
QT_QPA_PLATFORM: offscreen
run: |
source /opt/ros/humble/setup.bash &&
cargo run -p dora-ros2-bridge --example cxx-ros2-dataflow
bench:
name: "Bench"
strategy:
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.platform }}
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: "Benchmark example"
timeout-minutes: 30
run: cargo run --example benchmark --release
CLI:
name: "CLI Test"
strategy:
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.platform }}
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
if: runner.os == 'Linux'
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: true
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
# CLI tests
- name: "Build cli and binaries"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
run: |
cargo install --path binaries/cli --locked
- name: "Test Rust template Project"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
run: |
# Test Rust template Project
dora new test_rust_project --internal-create-with-path-dependencies
cd test_rust_project
cargo build --all
dora run dataflow.yml --stop-after 10s
cd ..
- name: "Test Dynamic Rust Dataflow example"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
run: |
dora up
dora list
dora build examples/rust-dataflow/dataflow_dynamic.yml
dora start examples/rust-dataflow/dataflow_dynamic.yml --name ci-rust-dynamic --detach
cargo run -p rust-dataflow-example-sink-dynamic
sleep 5
dora stop --name ci-rust-dynamic --grace-duration 5s
dora destroy
- uses: actions/setup-python@v5
with:
# TODO: Support Python 3.13 when https://github.com/pytorch/pytorch/issues/130249 is fixed
python-version: "3.12"
- name: Install the latest version of uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- name: "Set up venv"
shell: bash
run: |
uv venv --seed -p 3.12
# Set VIRTUAL_ENV for all following steps
echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
## If not Windows
if [ "$RUNNER_OS" != "Windows" ]; then
source .venv/bin/activate
else
source .venv/Scripts/activate
fi
uv pip install pyarrow
uv pip install -e apis/python/node
uv pip install ruff pytest
- name: "Test CLI (Python): Template Project"
timeout-minutes: 80
# fail-fast by using bash shell explicitly
shell: bash
run: |
# Test Python template Project
dora new test_python_project --lang python --internal-create-with-path-dependencies
cd test_python_project
echo "Running dora build"
dora build dataflow.yml --uv
# Check Compliancy
uv run ruff check .
uv run pytest
export OPERATING_MODE=SAVE
dora build dataflow.yml --uv
echo "Running CI Python Test"
dora run dataflow.yml --uv --stop-after 10s
cd ..
- name: "Test CLI (Python): Node example"
timeout-minutes: 20
# fail-fast by using bash shell explicitly
shell: bash
env:
RUST_LOG: info
run: |
echo "Running Python Node Example"
dora build examples/python-dataflow/dataflow.yml --uv
dora run examples/python-dataflow/dataflow.yml --uv --stop-after 10s
- name: "Test CLI (Python): Dynamic Node example"
timeout-minutes: 20
# fail-fast by using bash shell explicitly
shell: bash
run: |
echo "Running Python Node Dynamic Example"
dora build examples/python-dataflow/dataflow_dynamic.yml --uv
dora up
dora start examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach --uv
uv run opencv-plot
sleep 10
echo "Running dora stop"
dora stop --name ci-python-dynamic --grace-duration 30s
dora destroy
- name: "Test CLI (Python): Operator example"
timeout-minutes: 20
# fail-fast by using bash shell explicitly
shell: bash
run: |
echo "Running CI Operator Test"
dora build examples/python-operator-dataflow/dataflow.yml --uv
dora run examples/python-operator-dataflow/dataflow.yml --uv --stop-after 20s
- name: "Test Python Example: Multiple Arrays"
timeout-minutes: 30
run: |
dora build examples/python-multiple-arrays/dataflow.yml --uv
dora run examples/python-multiple-arrays/dataflow.yml --uv --stop-after 30s
- name: "Test CLI (Python): Async Example"
timeout-minutes: 30
## TODO: Fix async Windows support
if: runner.os != 'Windows'
env:
RUST_LOG: debug
run: dora run examples/python-async/dataflow.yaml --uv --stop-after 5m
- name: "Test Python Example: Drain"
timeout-minutes: 30
run: dora run examples/python-drain/dataflow.yaml --uv
- name: "Test Python Example: Dataflow Builder"
timeout-minutes: 30
run: uv run examples/python-dataflow-builder/simple_example.py
- name: "Python Queue Latency Test"
timeout-minutes: 30
run: dora run tests/queue_size_latest_data_python/dataflow.yaml --uv
- name: "Python Queue Latency + Timeout Test"
timeout-minutes: 30
run: dora run tests/queue_size_and_timeout_python/dataflow.yaml --uv
- name: "Rust Queue Latency Test"
timeout-minutes: 30
run: |
# Run Rust queue latency test
echo "Running CI Queue Size Latest Data Rust Test"
dora build tests/queue_size_latest_data_rust/dataflow.yaml --uv
dora run tests/queue_size_latest_data_rust/dataflow.yaml --uv
- name: "Test CLI (C)"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
if: runner.os == 'Linux'
run: |
# Test C template Project
dora new test_c_project --lang c --internal-create-with-path-dependencies
cd test_c_project
cmake -B build
cmake --build build
cmake --install build
dora run dataflow.yml --stop-after 10s
- name: "Test CLI (C++)"
timeout-minutes: 45
# fail-fast by using bash shell explicitly
shell: bash
if: runner.os == 'Linux'
run: |
# Test C++ template Project
dora new test_cxx_project --lang cxx --internal-create-with-path-dependencies
cd test_cxx_project
cmake -B build
cmake --build build
cmake --install build
dora run dataflow.yml --stop-after 10s
clippy:
name: "Clippy"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- name: "Clippy"
run: cargo clippy --all
- name: "Clippy (tracing feature)"
run: cargo clippy --all --features tracing
if: false # only the dora-runtime has this feature, but it is currently commented out
- name: "Clippy (metrics feature)"
run: cargo clippy --all --features metrics
if: false # only the dora-runtime has this feature, but it is currently commented out
rustfmt:
name: "Formatting"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- name: "rustfmt"
run: cargo fmt --all -- --check
check-license:
name: "License Checks"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- run: cargo --version --verbose
- uses: Swatinem/rust-cache@v2
with:
cache-provider: buildjet
cache-on-failure: true
# only save caches for `main` branch
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo install cargo-lichking
- name: "Check dependency licenses"
run: cargo lichking check
typos:
name: Typos
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Typos check with custom config file
uses: crate-ci/typos@master
cross-check:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: ubuntu-22.04
target: x86_64-unknown-linux-gnu
- runner: ubuntu-22.04
target: i686-unknown-linux-gnu
- runner: ubuntu-22.04
target: aarch64-unknown-linux-gnu
- runner: ubuntu-22.04
target: aarch64-unknown-linux-musl
- runner: ubuntu-22.04
target: armv7-unknown-linux-musleabihf
- runner: ubuntu-22.04
target: x86_64-pc-windows-gnu
- runner: macos-15
target: aarch64-apple-darwin
- runner: macos-15
target: x86_64-apple-darwin
fail-fast: false
steps:
- uses: actions/checkout@v3
- uses: r7kamura/rust-problem-matchers@v1.1.0
- name: "Add toolchains"
run: rustup target add ${{ matrix.platform.target }}
- name: Install system-level dependencies
if: runner.target == 'x86_64-pc-windows-gnu'
run: |
sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
- name: "Check"
uses: actions-rs/cargo@v1
with:
use-cross: true
command: check
args: --target ${{ matrix.platform.target }} --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
# copied from https://github.com/rust-lang/cargo/blob/6833aa715d724437dc1247d0166afe314ab6854e/.github/workflows/main.yml#L291
msrv:
name: "Check for specified `rust-version`"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: taiki-e/install-action@cargo-hack
- run: cargo hack check --all-targets --rust-version --workspace --ignore-private --locked