SNP CI on bigsku (#6989)

Author: achamayou

.github/workflows/README.md

@@ -5,15 +5,16 @@ Documents the various GitHub Actions workflows, the role they fulfil and 3rd par
 Builds and runs CCF performance tests, both end to end and micro-benchmarks. Results are posted to bencher.dev, and [plotted to make regressions obvious](https://bencher.dev/console/projects/ccf/plots).
 Triggered on every commit on `main`, but not on PR builds because the setup required to build from forks is complex and fragile in terms of security, and the increase in pool usage would be substantial.
 
+Tests are run and published on two different testbeds for comparison: gha-vmss-d16av5-ci (d16av5 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM), and are labeled accordingly in the bencher UI.
+
 File: `bencher.yml`
 3rd party dependencies:
 
 - `bencherdev/bencher@main`
 
 # Continuous Integration Containers GHCR
 
-Produces the build images used by nearly all other actions, particularly CI and release from 5.0.0-rc0 onwards. Complete images are attested and published to GHCR.
-Triggered on label creation (`build/*`).
+Produces the build images used by CI and release workflows between 5.0.0-rc0 and 6.0.0 (excluded). Complete images are attested and published to GHCR. Triggered on label creation (`build/*`).
 
 File: `ci-containers-ghcr.yml`
 3rd party dependencies:
@@ -22,18 +23,18 @@ File: `ci-containers-ghcr.yml`
 - `docker/metadata-action@v5`
 - `docker/build-push-action@v6`
 
-Note: This job will be removed with Ubuntu support, because installing dependencies on Azure Linux images is very fast, and producing CI-specific images is no longer necessary there.
+Note: This job is being kept until 5.0.x goes out of support.
 
 # Continuous Integration
 
-Main continuous integration job. Builds CCF for all target platforms, runs unit, end to end and partition tests Virtual. Run on every commit, including PRs from forks, gates merging. Also runs once a week, regardless of commits.
+Main continuous integration job. Builds CCF for all target platforms, runs unit, end to end and partition tests. Run on every commit, including PRs from forks, gates merging. Also runs once a week, regardless of commits.
 
 File: `ci.yml`
 3rd party dependencies: None
 
 # Long Tests
 
-Secondary continuous integration job. Runs more expensive, longer tests, such as tests against ASAN and TSAN builds, fuzzing etc.
+Secondary continuous integration job. Runs more expensive, longer tests, such as tests against ASAN and TSAN builds, extended fuzzing etc.
 
 - Runs daily on week days.
 - Can be manually run on a PR by setting `run-long-test` label, or via workflow dispatch.
@@ -70,14 +71,14 @@ File: `long-verification.yml`
 
 # Release
 
-Produces CCF release artefacts from 5.0.0-rc0 onwards, for all languages and platforms. Triggered on tags matching `ccf-[56].\*`. The output of the job is a draft release, which needs to be published manually. Publishing triggers the downstream jobs listed below.
+Produces CCF reference release artefacts from 5.0.0-rc0 onwards, for all languages and platforms. Triggered on tags matching `ccf-[56].\*`. The output of the job is a draft release, which needs to be published manually. Publishing triggers the downstream jobs listed below.
 
 File: `release.yml`
 3rd party dependencies: None
 
 # Containers GHCR
 
-Produces reference release images for 5.x release version. Not used from 6.0.0 onwards. Complete images are attested and published to GHCR. Triggered on release publishing.
+Produces reference release images for 5.x release versions. Not used from 6.0.0 onwards. Complete images are attested and published to GHCR. Triggered on release publishing.
 
 File: `containers-ghcr.yml`
 3rd party dependencies:
@@ -86,6 +87,8 @@ File: `containers-ghcr.yml`
 - `docker/metadata-action@v5`
 - `docker/build-push-action@v6`
 
+Note: This job is being kept until 5.0.x goes out of support.
+
 # NPM
 
 Publishes ccf-app TS package from a GitHub release to NPM. Triggered on release publishing.

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ permissions:
 jobs:
   checks:
     name: "Format and License Checks"
-    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    runs-on: [self-hosted, 1ES.Pool=gha-vmss-d16av5-ci]
     container:
       image: mcr.microsoft.com/azurelinux/base/core:3.0
       options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
@@ -47,7 +47,7 @@ jobs:
 
   build_with_tidy:
     name: "Build with clang-tidy"
-    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    runs-on: [self-hosted, 1ES.Pool=gha-vmss-d16av5-ci]
     container:
       image: mcr.microsoft.com/azurelinux/base/core:3.0
       options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
@@ -81,16 +81,10 @@ jobs:
           ninja
         shell: bash
 
-  build_and_test:
-    name: "CI"
+  build_and_test_virtual:
+    name: "Virtual CI"
     needs: checks
-    strategy:
-      matrix:
-        platform:
-          - name: virtual
-          - name: snp
-
-    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    runs-on: [self-hosted, 1ES.Pool=gha-vmss-d16av5-ci]
     container:
       image: mcr.microsoft.com/azurelinux/base/core:3.0
       options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
@@ -107,6 +101,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: "cpuinfo"
+        run: |
+          cat /proc/cpuinfo
+        shell: bash
+
       - name: "Install dependencies"
         shell: bash
         run: |
@@ -119,12 +118,11 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCMAKE_BUILD_TYPE=Debug ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug ..
           ninja
         shell: bash
 
-      - name: "Test ${{ matrix.platform.name }}"
-        if: "${{ matrix.platform.name != 'snp' }}" # Needs 1ES Pool support
+      - name: "Test virtual"
         run: |
           set -ex
           cd build
@@ -133,16 +131,16 @@ jobs:
           export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
-          # All other acceptably fast tests, which are now supported on Azure Linux.
+          # End to end tests
           ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit"
           # Partitions tests
           ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions
         shell: bash
 
-      - name: "Upload logs for ${{ matrix.platform.name }}"
+      - name: "Upload logs for virtual"
         uses: actions/upload-artifact@v4
         with:
-          name: logs-azurelinux-${{ matrix.platform.name }}
+          name: logs-azurelinux-virtual
           path: |
             build/workspace/*/*.config.json
             build/workspace/*/out
@@ -152,15 +150,29 @@ jobs:
         if: success() || failure()
 
   build_and_test_caci:
-    name: "Confidential Container (ACI) CI"
-    runs-on: [self-hosted, 1ES.Pool=gha-caci-ne]
+    name: "Confidential Container CI"
+    runs-on: [self-hosted, 1ES.Pool=gha-c-aci-ci]
     needs: checks
 
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
+      - name: "Dump environment"
+        run: |
+          set -ex
+          # Dump environment variables, extract Fabric_NodeIPOrFQDN
+          # and save it to a file for reconfiguration test using THIM.
+          cat /proc/*/environ | tr '\000' '\n' | sort -u | grep Fabric_NodeIPOrFQDN > /Fabric_NodeIPOrFQDN
+          echo "::group::Disk usage"
+          df -kh
+          echo "::endgroup::"
+          echo "::group::CPU Info"
+          cat /proc/cpuinfo
+          echo "::endgroup::"
+        shell: bash
+
       - name: "Build Debug"
         run: |
           set -ex
@@ -178,18 +190,42 @@ jobs:
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
           export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
-          # Unit tests, minus indexing that is sometimes timing out with this few cores
-          ./tests.sh --output-on-failure -L unit -j$(nproc --ignore=1) -E indexing
-          # Minimal end to end test that exercises SNP attestation verification
-          # but works within the current 4 core budget.
-          ./tests.sh --timeout 360 --output-on-failure -R code_update
+          # Unit tests
+          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
+
+          # End to end tests
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit|reconfiguration"
+
+          # DISABLED until Pool issue causing the CI to stop after 30 minutes is resolved
+          # Reconfiguration tests
+          # ./tests.sh --timeout 360 --output-on-failure -L reconfiguration -C reconfiguration
         shell: bash
 
+      - name: "Partition Tests"
+        run: |
+          set -ex
+          cd build
+          # DISABLED until Pool issue causing the CI to stop after 30 minutes is resolved
+          # Partitions tests
+          # ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions
+        shell: bash
+
+      - name: "Capture dmesg"
+        run: |
+          set -ex
+          echo "::group::Disk usage"
+          df -kh
+          echo "::endgroup::"
+          dmesg > dmesg.log
+        shell: bash
+        if: success() || failure()
+
       - name: "Upload logs"
         uses: actions/upload-artifact@v4
         with:
           name: logs-caci-snp
           path: |
+            dmesg.log
             build/workspace/*/*.config.json
             build/workspace/*/out
             build/workspace/*/err

CMakeLists.txt

@@ -1067,6 +1067,9 @@ if(BUILD_TESTS)
         --test-duration 300 --test-suite reconfiguration --jinja-templates-path
         ${CMAKE_SOURCE_DIR}/samples/templates
     )
+    set_property(
+      TEST reconfiguration_test_suite PROPERTY LABELS reconfiguration
+    )
 
     if(LONG_TESTS)
       add_e2e_test(
@@ -1368,6 +1371,7 @@ if(BUILD_TESTS)
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/reconfiguration.py
       ADDITIONAL_ARGS ${RECONFIG_TEST_ARGS}
     )
+    set_property(TEST reconfiguration_test_cft PROPERTY LABELS reconfiguration)
 
     add_e2e_test(
       NAME election_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py

tests/infra/remote.py

@@ -260,9 +260,6 @@ def setup(self, use_links=True):
         Empty the temporary directory if it exists,
         and populate it with the initial set of files.
         """
-        # SNP Testing currently runs on a fileshare which does not support symlinks
-        if snp.IS_SNP:
-            use_links = False
         self._setup_files(use_links)
 
     def get_cmd(self, include_dir=True):

tests/infra/snp.py

@@ -56,6 +56,9 @@ def get_aci_env():
     else:
         (security_context_dir,) = glob.glob("/security-context-*")
         env[ACI_SEV_SNP_ENVVAR_UVM_SECURITY_CONTEXT_DIR] = security_context_dir
+    # If Fabric_NodeIPOrFQDN is set, pick it up
+    if "Fabric_NodeIPOrFQDN" in os.environ:
+        env["Fabric_NodeIPOrFQDN"] = os.environ["Fabric_NodeIPOrFQDN"]
     return env