Merge pull request #390 from F1R3FLY-io/preston/rust/dev #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test, and Deploy | |
| on: | |
| push: | |
| branches: | |
| - staging | |
| - trying | |
| - rust/dev | |
| tags: "**" | |
| pull_request: | |
| branches: | |
| - rust/dev | |
| - "feature/**" | |
| env: | |
| # Enable AES/SSE2 CPU features for gxhash dependency (required by PathMap crate) | |
| # GitHub Actions runners support these features on x86_64 | |
| RUSTFLAGS: "-C target-feature=+aes,+sse2" | |
| jobs: | |
| # Build and save artifacts for next jobs. | |
| build_base: | |
| name: Build Base | |
| runs-on: ubuntu-latest | |
| outputs: | |
| VERSION: ${{ env.VERSION }} | |
| BRANCH: ${{ env.BRANCH }} | |
| DEV_LATEST_TAG: ${{ env.DEV_LATEST_TAG }} | |
| steps: | |
| - name: Clone Repository | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install APT Dependencies | |
| shell: bash -ex {0} | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y $(cat .github/apt-dependencies.txt) | |
| - name: Tool versions | |
| shell: bash -ex {0} | |
| run: | | |
| python --version | |
| pip --version | |
| rustc --version | |
| cargo --version | |
| - name: Initialize Environment | |
| shell: bash -exu -o pipefail {0} | |
| run: | | |
| git fetch origin | |
| git fetch origin --tags --force | |
| # Version from Git repository (tag-commit) | |
| VERSION="$(git describe --tags --always)" | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| # Find latest tag on rust/dev branch, with fallback | |
| if git ls-remote --heads origin rust/dev 2>/dev/null | grep -q "refs/heads/rust/dev"; then | |
| DEV_LATEST_TAG="$(git describe --tags --abbrev=0 origin/rust/dev 2>/dev/null || echo 'unknown')" | |
| else | |
| DEV_LATEST_TAG="unknown" | |
| fi | |
| echo "DEV_LATEST_TAG=$DEV_LATEST_TAG" >> $GITHUB_ENV | |
| # Find related HEAD branch | |
| BRANCH="" | |
| if [[ $GITHUB_REF =~ ^refs/tags/ ]]; then | |
| # Tag related to multiple branches leaves empty branch variable | |
| RAW_BRANCH=$(git branch -r --contains ${{ github.ref }}) | |
| if [[ $RAW_BRANCH =~ ^[\ ]*origin/([^ ]*)$ ]]; then | |
| BRANCH="${BASH_REMATCH[1]}" | |
| fi | |
| elif [[ $GITHUB_REF =~ ^refs/heads/ ]]; then | |
| BRANCH=${GITHUB_REF#refs/*/} | |
| else | |
| BRANCH=$GITHUB_HEAD_REF | |
| fi | |
| echo "BRANCH=$BRANCH" >> $GITHUB_ENV | |
| - name: Pack Working Tree | |
| shell: bash -ex {0} | |
| run: tar -H posix -czf /tmp/f1r3fly-worktree.tar.gz . | |
| - name: Save Working Tree | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: f1r3fly-worktree | |
| path: /tmp/f1r3fly-worktree.tar.gz | |
| required_rust_unit_tests: | |
| name: Required Unit Tests (Rust) | |
| needs: build_base | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tests: | |
| - crypto | |
| - rholang | |
| - rspace++ | |
| - shared | |
| - models | |
| - casper | |
| - node | |
| - block-storage | |
| - comm | |
| - graphz | |
| steps: | |
| - name: Clone Repository | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install APT Dependencies | |
| shell: bash -ex {0} | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y $(cat .github/apt-dependencies.txt) | |
| - name: Load Working Tree | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: f1r3fly-worktree | |
| path: /tmp | |
| - name: Restore Working Tree | |
| shell: bash -ex {0} | |
| run: tar -H posix -xzf /tmp/f1r3fly-worktree.tar.gz | |
| - name: Run Unit Tests | |
| shell: bash -ex {0} | |
| env: | |
| RUSTFLAGS: "-C target-feature=+aes,+sse2 -D warnings" | |
| RUST_BACKTRACE: "1" | |
| run: | | |
| mkdir -p /tmp/test-logs | |
| pushd ${{ matrix.tests }} | |
| cargo test --release 2>&1 | tee /tmp/test-logs/${{ matrix.tests }}-test-output.txt | |
| TEST_EXIT_CODE=${PIPESTATUS[0]} | |
| popd | |
| exit $TEST_EXIT_CODE | |
| - name: Upload Test Logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-logs-${{ matrix.tests }} | |
| path: /tmp/test-logs/ | |
| retention-days: 7 | |
| # Build pure Rust Docker image from node/Dockerfile | |
| # Multi-arch builds (amd64 + arm64) only for tags/releases to save CI time | |
| # Regular PRs and branches build only for amd64 (native, faster) | |
| build_rust_docker_image: | |
| name: Build Rust Docker Image | |
| needs: build_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone Repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Enable Containerd Image Store | |
| shell: bash -ex {0} | |
| run: | | |
| sudo mkdir -p /etc/docker | |
| echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json | jq | |
| sudo systemctl restart docker | |
| - name: Determine build platforms | |
| id: platforms | |
| shell: bash -ex {0} | |
| run: | | |
| # Build multi-arch for tags/releases and pull requests | |
| # For PRs: enable multiplatform builds for testing | |
| # For releases: multiplatform builds for production | |
| if [[ "${{ github.event_name }}" == "push" ]] && \ | |
| ([[ "${{ github.ref }}" =~ ^refs/tags/ ]] || \ | |
| [[ "${{ github.ref }}" == "refs/heads/trying" ]] || \ | |
| [[ "${{ github.ref }}" == "refs/heads/staging" ]] || \ | |
| [[ "${{ github.ref }}" == "refs/heads/rust/dev" ]]); then | |
| PLATFORMS="linux/amd64,linux/arm64" | |
| echo "Building for both platforms (tag/release detected)" | |
| else | |
| PLATFORMS="linux/amd64" | |
| echo "Building for amd64 only (non-release branch)" | |
| fi | |
| echo "platforms=$PLATFORMS" >> $GITHUB_OUTPUT | |
| - name: Build Docker Image | |
| shell: bash -ex {0} | |
| run: | | |
| if [[ "${{ steps.platforms.outputs.platforms }}" == "linux/amd64,linux/arm64" ]]; then | |
| docker buildx build \ | |
| --platform ${{ steps.platforms.outputs.platforms }} \ | |
| --file node/Dockerfile \ | |
| --tag f1r3flyindustries/f1r3fly-rust-node:latest \ | |
| --cache-from type=gha \ | |
| --cache-to type=gha,mode=max \ | |
| --output type=oci,dest=/tmp/rust-node-docker.tar \ | |
| . | |
| else | |
| docker buildx build \ | |
| --platform ${{ steps.platforms.outputs.platforms }} \ | |
| --file node/Dockerfile \ | |
| --tag f1r3flyindustries/f1r3fly-rust-node:latest \ | |
| --cache-from type=gha \ | |
| --cache-to type=gha,mode=max \ | |
| --load \ | |
| . | |
| fi | |
| - name: Export Docker Image | |
| shell: bash -ex {0} | |
| run: | | |
| mkdir -p /tmp/artifacts | |
| git describe --tags --always >/tmp/artifacts/version.txt | |
| if [[ "${{ steps.platforms.outputs.platforms }}" == "linux/amd64,linux/arm64" ]]; then | |
| # Multi-arch build: OCI file already created, just gzip it | |
| gzip -c /tmp/rust-node-docker.tar > /tmp/artifacts/rust-node-docker.tar.gz | |
| else | |
| # Single-platform build: Image is loaded, save it | |
| docker image save f1r3flyindustries/f1r3fly-rust-node \ | |
| | gzip > /tmp/artifacts/rust-node-docker.tar.gz | |
| fi | |
| - name: Save Packages Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifacts-packages | |
| path: /tmp/artifacts | |
| # Get pure Rust Docker image and run integration tests. | |
| # | |
| # These steps are run directly on runner's host (note there's no container key | |
| # in the job configuration). That is because bind mounting in container runner | |
| # is broken[1] and we need to mount host's /tmp onto container's /tmp (see | |
| # "Running from Docker" in integration-tests/README.md). The host doesn't have | |
| # everything we need (pipenv, pyenv), so we're going to run integration tests | |
| # in rchain/buildenv container started manually as the last step. | |
| # | |
| # The problem is that host's runner runs everything under a non-privileged | |
| # account, whereas the rchain/buildenv container runs as root by default. The | |
| # container image does not have an account corresponding to the host's | |
| # unprivileged account UID, so we're going to run it as root and do some | |
| # workarounds (see below). | |
| # | |
| # [1] https://github.community/t5/GitHub-Actions/Container-volumes-key-not-mounting-volume/m-p/34798 | |
| required_rust_integration_tests: | |
| name: Required Integration Tests | |
| needs: build_rust_docker_image | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # This runs integration tests in parallel. | |
| # | |
| # For each entry a runner node is spawned with entry value in | |
| # matrix.tests workflow variable, which is also put into TESTS | |
| # environment variable (see below) and used by last step, execution of | |
| # .github/run-integration-test-selection, which passes it verbatim | |
| # (except for REMAINDER) to Pytest's -k parameter. | |
| # | |
| # To learn about REMAINDER, see github/print-integration-test-selection. | |
| tests: | |
| - test_backward_compatible | |
| - test_genesis_ceremony | |
| - test_heartbeat | |
| - test_storage | |
| - test_wallets | |
| - test_web_api | |
| env: | |
| TESTS: ${{ matrix.tests }} | |
| steps: | |
| - name: Clone Repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14.0" | |
| cache: "pip" # caching pip dependencies | |
| - name: Install Python dependencies | |
| shell: bash -ex {0} | |
| run: | | |
| pip install pipenv | |
| pushd integration-tests | |
| pipenv sync | |
| popd | |
| - name: Enable Containerd Image Store | |
| shell: bash -ex {0} | |
| run: | | |
| sudo mkdir -p /etc/docker | |
| echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json | jq | |
| sudo systemctl restart docker | |
| - name: Load Docker Image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: artifacts-packages | |
| path: /tmp | |
| - name: Import Docker Image | |
| shell: bash -ex {0} | |
| run: | | |
| # Load Docker image (multi-arch or single-platform) | |
| zcat /tmp/rust-node-docker.tar.gz | docker image load | |
| - name: Run Integration Test | |
| shell: bash -ex {0} | |
| run: | | |
| mkdir -p /tmp/test-logs | |
| pushd integration-tests | |
| pipenv run ../.github/run-integration-test-selection 2>&1 | tee /tmp/test-logs/${{ matrix.tests }}-integration-output.txt | |
| TEST_EXIT_CODE=${PIPESTATUS[0]} | |
| popd | |
| exit $TEST_EXIT_CODE | |
| - name: Upload Integration Test Logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-logs-${{ matrix.tests }} | |
| path: /tmp/test-logs/ | |
| retention-days: 7 | |
| - name: Upload Docker Logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-logs-${{ matrix.tests }} | |
| path: /tmp/*.log | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| # release_* jobs make built artifacts available to public and run only on new | |
| # tags or pushes to "staging" or "trying" branches used by Bors (bors r+ and bors try). | |
| # These jobs require secrets! See "env" variables and "Secrets" page in GitHub repository | |
| # settings. Release destinations differ slightly depending on the event that | |
| # triggered the job (tag or branch push). See "Publish ..." steps for details. | |
| # VERSION and BRANCH are used from "build_base" job outputs as a new way to share | |
| # data between jobs and steps. Legacy "version" via artifact file is still used. | |
| # Upload pure Rust Docker image to Docker Hub. | |
| # Note: Docker Hub push is commented out - job runs for testing purposes | |
| release_rust_docker_image: | |
| name: Release Rust Docker Image | |
| if: github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/trying' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/rust/dev') | |
| needs: | |
| - required_rust_unit_tests | |
| - required_rust_integration_tests | |
| - build_rust_docker_image | |
| - build_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Enable Containerd Image Store | |
| shell: bash -ex {0} | |
| run: | | |
| sudo mkdir -p /etc/docker | |
| echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json | jq | |
| sudo systemctl restart docker | |
| - name: Load Docker Image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: artifacts-packages | |
| path: /tmp | |
| - name: Import Docker Image | |
| shell: bash -ex {0} | |
| run: zcat /tmp/rust-node-docker.tar.gz | docker image load | |
| # Docker Hub push disabled - commented out for testing purposes | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Publish Docker Image | |
| env: | |
| # Output from build_base job | |
| VERSION: ${{ needs.build_base.outputs.VERSION }} | |
| BRANCH: ${{ needs.build_base.outputs.BRANCH }} | |
| DEV_LATEST_TAG: ${{ needs.build_base.outputs.DEV_LATEST_TAG }} | |
| shell: bash -exu -o pipefail {0} | |
| run: | | |
| DOCKER_IMAGE_NAME="f1r3flyindustries/f1r3fly-rust-node" | |
| SUFFIX="" | |
| CI_RUN="" | |
| # Add suffix if trying or merging | |
| if [[ "$BRANCH" =~ ^(trying|staging)$ ]]; then | |
| SUFFIX="-$BRANCH" | |
| # Add CI run number if trying | |
| if [[ $BRANCH =~ ^trying$ ]]; then | |
| CI_RUN="-ci-$GITHUB_RUN_NUMBER" | |
| fi | |
| fi | |
| set -x | |
| IMAGE_NAME="$DOCKER_IMAGE_NAME$SUFFIX" | |
| IMG_VERSION_RAW="$IMAGE_NAME:$VERSION$CI_RUN" | |
| # Replace unsupported character `+` | |
| IMG_VERSION="${IMG_VERSION_RAW//[+]/__}" | |
| IMG_LATEST="$IMAGE_NAME:latest" | |
| # Release Docker image | |
| docker tag f1r3flyindustries/f1r3fly-rust-node:latest $IMG_VERSION | |
| docker push $IMG_VERSION | |
| echo "Docker image released: $IMG_VERSION" | |
| # Tag Docker image as latest if tag is the latest on rust/dev branch | |
| if [[ $GITHUB_REF =~ ^refs/tags/ ]] && [[ $DEV_LATEST_TAG =~ $VERSION ]]; then | |
| docker tag f1r3flyindustries/f1r3fly-rust-node:latest $IMG_LATEST | |
| docker push $IMG_LATEST | |
| echo "Docker image released: $IMG_LATEST" | |
| fi | |
| # GitHub (create) release and packages | |
| # Note: Job runs for testing purposes on pull requests | |
| release_packages: | |
| name: Release Packages | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| permissions: | |
| contents: write # Required for creating/updating releases | |
| needs: | |
| - required_rust_unit_tests | |
| - required_rust_integration_tests | |
| - build_rust_docker_image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Load Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: artifacts-packages | |
| path: /tmp/artifacts | |
| - name: Update release and release artifacts | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| artifacts: /tmp/artifacts/* | |
| prerelease: true | |
| draft: true | |
| allowUpdates: true | |
| omitBodyDuringUpdate: true | |
| token: ${{ secrets.GITHUB_TOKEN }} |