docker-build-test.yml

name: VM Build and Flow Test

on:
  push:
  pull_request:
    branches: [develop, stage-in]
  workflow_dispatch:

env:
  BUILD_DIR: /tmp/dc-build
  UTILS_BUILD_DIR: /tmp/dc-utils-build
  INSTALL_PREFIX: /tmp/dc-install
  INSTALLED_CONFIG_TEMPLATE_YAML: /tmp/dc-install/etc/datacrumbs/configs/docker.yaml
  CONFIG_TEMPLATE_YAML: ${{ github.workspace }}/datacrumbs-utils/docs/configs/docker.yaml
  CONFIG_YAML: /tmp/datacrumbs-ci-config.yaml
  PROBE_FILE: /tmp/datacrumbs-ci-probes.json.gz
  JOB_ID: "1"
  LOG_DIR: /tmp/datacrumbs-logs
  TRACE_DIR: /tmp/datacrumbs-traces
  OUT_FILE: /tmp/datacrumbs-ci-output.bin
  BPFTOOL_SOURCE_REF: v7.5.0
  BPFTOOLS_PREFIX: /tmp/dc-bpftools

jobs:
  vm-build-and-test:
    name: VM ${{ matrix.flow_name }} (${{ matrix.os_name }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_name: ubuntu-22.04
            runner: ubuntu-22.04
            flow_name: Build and Flow Test
            flow_id: signed
            disable_signing: "false"
          - os_name: ubuntu-22.04
            runner: ubuntu-22.04
            flow_name: Disabled Signing Flow Test
            flow_id: disabled-signing
            disable_signing: "true"
          - os_name: ubuntu-24.04
            runner: ubuntu-24.04
            flow_name: Build and Flow Test
            flow_id: signed
            disable_signing: "false"
          - os_name: ubuntu-24.04
            runner: ubuntu-24.04
            flow_name: Disabled Signing Flow Test
            flow_id: disabled-signing
            disable_signing: "true"
    runs-on: ${{ matrix.runner }}

    steps:
      - name: Checkout datacrumbs
        uses: actions/checkout@v4
        with:
          path: datacrumbs
          submodules: recursive

      - name: Resolve compatible datacrumbs-utils ref
        run: |
          set -euo pipefail
          cmake -DOUTPUT="${RUNNER_TEMP}/datacrumbs-utils-compatibility.env" \
            -P "${GITHUB_WORKSPACE}/datacrumbs/cmake/write_datacrumbs_utils_compatibility_env.cmake"
          cat "${RUNNER_TEMP}/datacrumbs-utils-compatibility.env" >> "${GITHUB_ENV}"

      - name: Checkout datacrumbs-utils
        uses: actions/checkout@v4
        with:
          repository: ${{ env.DATACRUMBS_UTILS_COMPATIBLE_REPOSITORY }}
          ref: ${{ env.DATACRUMBS_UTILS_COMPATIBLE_REF }}
          fetch-depth: 1
          path: datacrumbs-utils

      - name: Install base build and runtime dependencies
        run: |
          set -euo pipefail
          sudo apt-get update

          libclang_dev_pkg=$(apt-cache search '^libclang-[0-9]+-dev$' | awk '{print $1}' | sort -V | tail -n 1)
          if [[ -z "${libclang_dev_pkg}" ]]; then
            echo "Unable to determine a versioned libclang development package" >&2
            exit 1
          fi

          sudo apt-get install -y \
            build-essential \
            cmake \
            git \
            pkg-config \
            clang \
            llvm \
            llvm-dev \
            libclang-dev \
            "${libclang_dev_pkg}" \
            libelf-dev \
            libssl-dev \
            libcap-dev \
            libyaml-cpp-dev \
            libjson-c-dev \
            zlib1g-dev \
            jq \
            patchelf \
            python3 \
            openmpi-bin \
            libopenmpi-dev
          if [[ ! -d "/usr/src/linux-headers-$(uname -r)" ]]; then
            sudo apt-get install -y "linux-headers-$(uname -r)" || sudo apt-get install -y linux-headers-generic
          fi

      - name: Export LLVM toolchain paths
        run: |
          set -euo pipefail
          llvm_include_dir="$(llvm-config --includedir)"
          llvm_lib_dir="$(llvm-config --libdir)"
          echo "C_INCLUDE_PATH=${llvm_include_dir}:${C_INCLUDE_PATH:-}" >> "${GITHUB_ENV}"
          echo "CPLUS_INCLUDE_PATH=${llvm_include_dir}:${CPLUS_INCLUDE_PATH:-}" >> "${GITHUB_ENV}"
          echo "LIBRARY_PATH=${llvm_lib_dir}:${LIBRARY_PATH:-}" >> "${GITHUB_ENV}"
          echo "LD_LIBRARY_PATH=${llvm_lib_dir}:${LD_LIBRARY_PATH:-}" >> "${GITHUB_ENV}"

      - name: Build and stage libbpf and bpftool from source
        run: |
          set -euo pipefail
          workdir="${RUNNER_TEMP}/bpftool-src"
          prefix="${BPFTOOLS_PREFIX}"
          rm -rf "${workdir}" "${prefix}"
          mkdir -p "${prefix}"
          git clone https://github.com/libbpf/bpftool.git "${workdir}"
          pushd "${workdir}"
          git checkout tags/${BPFTOOL_SOURCE_REF} -b "${BPFTOOL_SOURCE_REF}"
          git submodule update --init --recursive

          pushd libbpf
          git checkout tags/v1.5.0 -b v1.5.0
          cd src
          DESTDIR="${prefix}" make install -j"$(nproc)"
          popd

          cd src
          DESTDIR="${prefix}" make install -j"$(nproc)"
          popd

          for staged_root in "${prefix}/usr/local" "${prefix}/usr"; do
            if [[ ! -d "${staged_root}" ]]; then
              continue
            fi
            for dir in include lib lib64 libexec bin sbin; do
              if [[ -d "${staged_root}/${dir}" ]]; then
                mkdir -p "${prefix}/${dir}"
                find "${staged_root}/${dir}" -mindepth 1 -maxdepth 1 -exec mv {} "${prefix}/${dir}" \;
              fi
            done
          done
          rm -rf "${prefix}/usr"

          bpftool_path=""
          if [[ -x "${prefix}/sbin/bpftool" ]]; then
            bpftool_path="${prefix}/sbin/bpftool"
          elif [[ -x "${prefix}/bin/bpftool" ]]; then
            bpftool_path="${prefix}/bin/bpftool"
          else
            echo "bpftool was not staged under ${prefix}" >&2
            find "${prefix}" -maxdepth 4 -type f -name bpftool -print >&2 || true
            exit 1
          fi

          libbpf_pc=$(find "${prefix}" -path '*/pkgconfig/libbpf.pc' -print -quit)
          if [[ -z "${libbpf_pc}" ]]; then
            echo "libbpf.pc was not staged under ${prefix}" >&2
            find "${prefix}" -maxdepth 5 -type f -name 'libbpf*' -print >&2 || true
            exit 1
          fi

          sed -i "s|^prefix=.*|prefix=${prefix}|" "${libbpf_pc}"
          echo "Using staged bpftool: ${bpftool_path}"
          echo "Using staged libbpf.pc: ${libbpf_pc}"
          cat "${libbpf_pc}"

      - name: Export source-built bpftool and libbpf environment
        run: |
          set -euo pipefail
          echo "PATH=${BPFTOOLS_PREFIX}/bin:${BPFTOOLS_PREFIX}/sbin:${PATH}" >> "${GITHUB_ENV}"
          echo "LD_LIBRARY_PATH=${BPFTOOLS_PREFIX}/lib:${BPFTOOLS_PREFIX}/lib64:${LD_LIBRARY_PATH:-}" >> "${GITHUB_ENV}"
          echo "CMAKE_PREFIX_PATH=${BPFTOOLS_PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "${GITHUB_ENV}"
          echo "PKG_CONFIG_PATH=${BPFTOOLS_PREFIX}/lib/pkgconfig:${BPFTOOLS_PREFIX}/lib64/pkgconfig:${PKG_CONFIG_PATH:-}" >> "${GITHUB_ENV}"

      - name: Verify runner supports systemd services
        run: |
          set -euo pipefail
          systemctl --version
          test -d /run/systemd/system

      - name: Configure datacrumbs
        run: |
          set -euo pipefail
          extra_cmake_args=()
          if [[ "${{ matrix.disable_signing }}" == "true" ]]; then
            extra_cmake_args+=("-DDATACRUMBS_DISABLE_PROBE_SIGNING=ON")
          fi

          cmake -S "${GITHUB_WORKSPACE}/datacrumbs" -B "${BUILD_DIR}" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
            -DBPFTOOL_EXECUTABLE="$(command -v bpftool)" \
            -DDATACRUMBS_HOST=docker \
            -DDATACRUMBS_SCHEDULER_TYPE=SLURM \
            -DDATACRUMBS_USER="${USER}" \
            -DDATACRUMBS_INSTALL_USER="${USER}" \
            -DDATACRUMBS_CONFIGURED_LOG_DIR="${LOG_DIR}" \
            -DDATACRUMBS_CONFIGURED_TRACE_DIR="${TRACE_DIR}" \
            -DDATACRUMBS_KERNEL_HEADERS_PATH="/usr/src/linux-headers-$(uname -r)" \
            "${extra_cmake_args[@]}"

      - name: Build datacrumbs
        run: |
          set -euo pipefail
          cmake --build "${BUILD_DIR}" -j"$(nproc)"

      - name: Install datacrumbs
        run: |
          set -euo pipefail
          sudo cmake --install "${BUILD_DIR}"

      - name: Configure compatible datacrumbs-utils
        run: |
          set -euo pipefail
          utils_extra_cmake_args=()
          if [[ "${{ matrix.disable_signing }}" == "true" ]]; then
            utils_extra_cmake_args+=("-DDATACRUMBS_DISABLE_PROBE_SIGNING=ON")
          fi

          cmake -S "${GITHUB_WORKSPACE}/datacrumbs-utils" -B "${UTILS_BUILD_DIR}" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
            -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX};${CMAKE_PREFIX_PATH:-}" \
            "${utils_extra_cmake_args[@]}"

      - name: Build datacrumbs-utils
        run: |
          set -euo pipefail
          cmake --build "${UTILS_BUILD_DIR}" -j"$(nproc)"

      - name: Install datacrumbs-utils
        run: |
          set -euo pipefail
          sudo cmake --install "${UTILS_BUILD_DIR}"

      - name: Verify probe configurator linkage
        run: |
          set -euo pipefail
          probe_exec="${INSTALL_PREFIX}/bin/datacrumbs_probe_configurator_exec"
          probe_wrapper="${INSTALL_PREFIX}/bin/datacrumbs_probe_configurator"

          test -x "${probe_exec}"
          test -x "${probe_wrapper}"

          echo "Wrapper LD_LIBRARY_PATH export:"
          grep -n "LD_LIBRARY_PATH" "${probe_wrapper}"

          echo "Probe configurator dynamic section:"
          readelf -d "${probe_exec}" | grep -E 'NEEDED|RPATH|RUNPATH'

          echo "Probe configurator shared library resolution:"
          ldd "${probe_exec}"

          ! ldd "${probe_exec}" | grep -q "not found"
          ldd "${probe_exec}" | grep -q "libclang"

      - name: Run system configurator explicitly
        run: |
          set -euo pipefail
          sudo "${INSTALL_PREFIX}/bin/datacrumbs_system_configurator"

      - name: Install and start signing-enabled systemd units
        if: matrix.disable_signing == 'false'
        run: |
          set -euo pipefail
          sudo ln -sf "${INSTALL_PREFIX}/etc/datacrumbs/systemd/datacrumbs_sign_probes.service" /etc/systemd/system/datacrumbs_sign_probes.service
          sudo ln -sf "${INSTALL_PREFIX}/etc/datacrumbs/systemd/datacrumbs@.service" /etc/systemd/system/datacrumbs@.service
          sudo systemctl daemon-reload
          sudo systemctl enable --now datacrumbs_sign_probes.service
          sudo systemctl status --no-pager datacrumbs_sign_probes.service

      - name: Install disabled-signing runtime systemd unit
        if: matrix.disable_signing == 'true'
        run: |
          set -euo pipefail
          sudo ln -sf "${INSTALL_PREFIX}/etc/datacrumbs/systemd/datacrumbs@.service" /etc/systemd/system/datacrumbs@.service
          sudo systemctl daemon-reload

      - name: Generate probes
        run: |
          set -euo pipefail
          set -x
          test -x "${INSTALL_PREFIX}/bin/datacrumbs_configure_template"
          config_template_yaml="${INSTALLED_CONFIG_TEMPLATE_YAML}"
          if [[ ! -f "${config_template_yaml}" ]]; then
            config_template_yaml="${CONFIG_TEMPLATE_YAML}"
          fi
          test -f "${config_template_yaml}"
          rm -f "${CONFIG_YAML}"
          "${INSTALL_PREFIX}/bin/datacrumbs_configure_template" "${config_template_yaml}" "${CONFIG_YAML}"
          test -f "${CONFIG_YAML}"
          rm -f "${PROBE_FILE}"
          probe_dir="$(dirname "${PROBE_FILE}")"
          fallback_probe_file=""

          set +e
          probe_output="$("${INSTALL_PREFIX}/bin/datacrumbs_probe_configurator" "${CONFIG_YAML}" "${PROBE_FILE}" 2>&1)"
          probe_status=$?
          set -e
          printf '%s\n' "${probe_output}"
          printf 'datacrumbs_probe_configurator exit status: %s\n' "${probe_status}"

          if [[ ! -s "${PROBE_FILE}" ]]; then
            set +e
            fallback_probe_file="$(find "${probe_dir}" -maxdepth 1 -name 'probes-*.json.gz' -size +0c -print | sort | tail -n 1)"
            set -e
            if [[ -n "${fallback_probe_file}" ]]; then
              cp "${fallback_probe_file}" "${PROBE_FILE}"
              echo "Using generated fallback probes file: ${fallback_probe_file}"
            fi
          fi

          if [[ "${probe_status}" != "0" ]]; then
            if [[ "${probe_status}" == "1" ]] && grep -Eq "One or more probes failed validation|Extraction summary: .*invalid=yes" <<<"${probe_output}"; then
              if [[ -s "${PROBE_FILE}" ]]; then
                echo "Continuing with generated probes file despite invalid candidate probes on this runner."
                ls -l "${probe_dir}"
                find "${probe_dir}" -maxdepth 1 \( -name 'probes-*.json' -o -name 'probes-*.json.gz' -o -name 'categories-*.json' \) -print || true
              else
                echo "Probe validation reported invalid candidates and no probes file was written."
                ls -l "${probe_dir}"
                find "${probe_dir}" -maxdepth 1 \( -name 'probes-*.json' -o -name 'probes-*.json.gz' -o -name 'categories-*.json' \) -print || true
                exit "${probe_status}"
              fi
            else
              echo "Probe generation failed unexpectedly."
              ls -l "${probe_dir}"
              find "${probe_dir}" -maxdepth 1 \( -name 'probes-*.json' -o -name 'probes-*.json.gz' -o -name 'categories-*.json' \) -print || true
              exit "${probe_status}"
            fi
          else
            test -s "${PROBE_FILE}"
          fi

          set +x

          if [[ "${{ matrix.disable_signing }}" == "true" ]]; then
            grep -q "Probe signing is disabled" <<<"${probe_output}"
            python3 - <<'PY'
            import gzip
            import json
            import os
            from pathlib import Path

            probe_file = Path(os.environ["PROBE_FILE"])
            with gzip.open(probe_file, "rt", encoding="utf-8") as handle:
                payload = json.load(handle)

            assert "checksum" not in payload, payload
            assert "checksum_algorithm" not in payload, payload
            assert isinstance(payload.get("categories"), list) and payload["categories"], payload
            PY
            fi

      - name: Run datacrumbs wrapper flow and verify logs
        run: |
          set -euo pipefail
          service_user="${USER}"
          log_file="${LOG_DIR}/datacrumbs_${service_user}_${JOB_ID}_$(hostname).log"

          cleanup() {
            "${INSTALL_PREFIX}/bin/datacrumbs_service_wrapper" stop "${JOB_ID}" "${service_user}" "${PROBE_FILE}" || true
            sudo systemctl status --no-pager "datacrumbs@${JOB_ID}.service" || true
            sudo journalctl -u "datacrumbs@${JOB_ID}.service" --no-pager -n 200 || true
            if [[ "${{ matrix.disable_signing }}" == "false" ]]; then
              sudo journalctl -u datacrumbs_sign_probes.service --no-pager -n 200 || true
            fi
          }
          trap cleanup EXIT

          rm -f "${OUT_FILE}"
          "${INSTALL_PREFIX}/bin/datacrumbs_service_wrapper" start "${JOB_ID}" "${service_user}" "${PROBE_FILE}"
          sudo systemctl is-active --quiet "datacrumbs@${JOB_ID}.service"

          "${INSTALL_PREFIX}/bin/datacrumbs_wrap" \
            dd if=/dev/zero of="${OUT_FILE}" bs=1M count=16 status=none

          test -f "${OUT_FILE}"
          test "$(wc -c <"${OUT_FILE}")" = "16777216"

          sleep 5
          "${INSTALL_PREFIX}/bin/datacrumbs_service_wrapper" stop "${JOB_ID}" "${service_user}" "${PROBE_FILE}"
          trap - EXIT

          ! sudo systemctl is-active --quiet "datacrumbs@${JOB_ID}.service"
          test -s "${log_file}"
          grep -q . "${log_file}"
          if [[ "${{ matrix.disable_signing }}" == "true" ]]; then
            grep -q "Probe signing is disabled" "${log_file}"
          fi
          tail -n 50 "${log_file}"

      - name: Upload datacrumbs logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: datacrumbs-${{ matrix.os_name }}-${{ matrix.flow_id }}-logs
          path: |
            /tmp/datacrumbs-logs
            /tmp/datacrumbs-traces
            /tmp/datacrumbs-ci-probes.json.gz
            /tmp/datacrumbs-ci-output.bin
          if-no-files-found: ignore