ci: Build ROM release from versioned git ref (revive)

.github/workflows/fpga.yml

@@ -4,7 +4,6 @@ name: FPGA Build
 on:
   push:
     branches: ["main"]
-  pull_request:
   workflow_call:
     inputs:
       artifact-suffix:
@@ -25,6 +24,12 @@ on:
       rom-version:
         default: "latest"
         type: string
+      rom-ref:
+        default: "main"
+        type: string
+      fw-ref:
+        default: "main"
+        type: string
       workflow_call:
         description: 'Set true for workflow_call'
         default: true
@@ -36,7 +41,6 @@ on:
         default: true
         type: boolean
 
-
 jobs:
   check_cache:
     runs-on: ubuntu-22.04
@@ -100,9 +104,21 @@ jobs:
           path: /tmp/caliptra-fpga-kmod/
           retention-days: 1
 
+  # this workflow uploads a file named caliptra-unified-<rom-ref>-<fw-ref>-<hw-version>-<artifact-suffix>
+  versioned-checkout-build:
+    uses: ./.github/workflows/versioned-checkout.yml
+    with:
+      rom-ref: "${{ inputs.rom-ref }}"
+      rom-logging: "${{ inputs.rom-logging }}"
+      fw-ref: "${{ inputs.fw-ref }}"
+      hw-version: "${{ inputs.hw-version }}"
+      workflow_call: "${{ inputs.workflow_call }}"
+      artifact-suffix: "${{ inputs.artifact-suffix }}"
+
   build_test_binaries:
     runs-on: [e2-standard-16]
     timeout-minutes: 60
+    needs: versioned-checkout-build
 
     env:
       # Change this to a new random value if you suspect the cache is corrupted
@@ -121,6 +137,12 @@ jobs:
           path: /tmp/caliptra-fpga-sysroot.tar
           key: sysroot-v9-${{ env.CACHE_BUSTER }}
 
+      - name: Extract Versioned ROM and FMC/RT
+        uses: actions/download-artifact@v4
+        with:
+          # defined in versioned-checkout.yml, files will be moved to /tmp/caliptra
+          name: caliptra-unified-${{ inputs.rom-ref }}-${{ inputs.fw-ref }}-${{ inputs.hw-version }}-${{ inputs.artifact-suffix }}
+
       - name: Extract sysroot
         if: "steps.restore_sysroot_cache.outputs.cache-hit"
         run: |
@@ -131,7 +153,7 @@ jobs:
         run: |
           sudo apt-get update -qy && sudo apt-get -y install debootstrap binfmt-support qemu-user-static u-boot-tools
 
-      - name: build sysroot
+      - name: Build sysroot
         # Note: This is the sysroot for the tiny debian installation we run on the FPGA;
         # it is missing xilinx-provided kernel headers needed to build kernel modules
         if: "!steps.restore_sysroot_cache.outputs.cache-hit"
@@ -140,7 +162,7 @@ jobs:
           sudo debootstrap --include linux-libc-dev --arch arm64 --foreign bookworm /tmp/caliptra-fpga-sysroot
           sudo chroot /tmp/caliptra-fpga-sysroot /debootstrap/debootstrap --second-stage
 
-          # Remove unnecesary files
+          # Remove unnecessary files
           sudo find /tmp/caliptra-fpga-sysroot/ \( -type d -and ! -perm -o=r \) -prune -exec rm -rf {} \;
           sudo find /tmp/caliptra-fpga-sysroot/ \( -type d -and ! -perm -o=x \) -prune -exec rm -rf {} \;
           sudo find /tmp/caliptra-fpga-sysroot/ \( ! -perm -o=r \) -exec rm -f {} \;
@@ -173,9 +195,10 @@ jobs:
           else
             FEATURES=fpga_realtime,itrng
           fi
-          if [[ "${{ inputs.workflow_call }}" && "${{ inputs.hw-version }}" != "latest" ]]; then
+          if [[ "${{ inputs.workflow_call }}" && -n "${{ inputs.hw-version }}" && "${{ inputs.hw-version }}" != "latest" ]]; then
             FEATURES=$FEATURES,hw-${{ inputs.hw-version }}
           fi
+
           cargo nextest archive \
             --features=${FEATURES} \
             --release \
@@ -193,22 +216,13 @@ jobs:
           path: /tmp/caliptra-test-binaries.sqsh
           retention-days: 1
 
-      - name: Build test firmware
-        run: |
-          mkdir /tmp/caliptra-test-firmware
-
-          FEATURES=""
-          if [[ "${{ inputs.workflow_call }}" && "${{ inputs.hw-version }}" != "latest" ]]; then
-            FEATURES=hw-${{ inputs.hw-version }}
-          fi
-          cargo run --release -p caliptra-builder --features=${FEATURES} -- --all_elfs /tmp/caliptra-test-firmware
-
-      - name: 'Upload test firmware artifact'
-        uses: actions/upload-artifact@v4
-        with:
-          name: caliptra-test-firmware${{ inputs.artifact-suffix }}
-          path: /tmp/caliptra-test-firmware
-          retention-days: 1
+      # This is replaced by claiptra-unified-${{ inputs.rom-ref }}-${{ inputs.fw-ref }}-${{ inputs.hw-version }}-${{ inputs.artifact-suffix }}
+      # - name: 'Upload test firmware artifact'
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: caliptra-test-firmware${{ inputs.artifact-suffix }}
+      #     path: /tmp/caliptra-test-firmware
+      #     retention-days: 1
 
   build_kernel_modules:
     runs-on: ubuntu-22.04
@@ -284,6 +298,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: 'true'
+
+      - name: Pull dpe submodule
+        run: |
+          git submodule update --init dpe
 
       - name: Mount FPGA tools
         run: |
@@ -296,14 +314,14 @@ jobs:
           cd hw/fpga
           mkdir caliptra_build
           if [ "${{ inputs.fpga-itrng }}" == "false" ]; then
-            ITRNG=FALSE
+            export ITRNG=FALSE
           else
-            ITRNG=TRUE
+            export ITRNG=TRUE
           fi
           if [ "${{ inputs.workflow_call }}" ]; then
-            RTL_VERSION="${{ inputs.hw-version }}"
+            export RTL_VERSION="${{ inputs.hw-version }}"
           else
-            RTL_VERSION="latest"
+            export RTL_VERSION="latest"
           fi
           /fpga-tools/Xilinx/Vivado/2022.2/bin/vivado -mode batch -source fpga_configuration.tcl -tclargs BUILD=TRUE ITRNG=${ITRNG} RTL_VERSION=${RTL_VERSION}
           if [ ! -f caliptra_build/caliptra_fpga.bin ]; then
@@ -375,11 +393,11 @@ jobs:
           name: caliptra-test-binaries${{ inputs.artifact-suffix }}
           path: /tmp/caliptra-test-binaries.sqsh
 
-      - name: 'Download Test Firmware Artifact'
-        uses: actions/download-artifact@v4
-        with:
-          name: caliptra-test-firmware${{ inputs.artifact-suffix }}
-          path: /tmp/caliptra-test-firmware
+      # - name: 'Download Test Firmware Artifact'
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: caliptra-test-firmware${{ inputs.artifact-suffix }}
+      #     path: /tmp/caliptra-test-firmware
 
       - name: Mount binaries
         run: |
@@ -419,12 +437,27 @@ jobs:
       - name: Set clock rate
         run: |
           sudo bash -c 'echo 20000000 > /sys/bus/platform/drivers/xilinx_fclk/fclk0/set_rate'
+
+      - name: Extract Versioned ROM and FMC/RT
+        uses: actions/download-artifact@v4
+        with:
+          # defined in versioned-checkout.yml, files will be moved to /tmp/caliptra
+          name: caliptra-unified-${{ inputs.rom-ref }}-${{ inputs.fw-ref }}-${{ inputs.hw-version }}-${{ inputs.artifact-suffix }}
+          path: /tmp/caliptra-unified
+
+      - name: Debug List Downloads
+        run: |
+          ls -laR /tmp/caliptra-*
 
       - name: Execute tests
         run: |
           export RUST_TEST_THREADS=1
+          CALIPTRA_ARTIFACTS_PATH=caliptra-unified-${{ inputs.rom-ref }}-${{ inputs.fw-ref }}-${{ inputs.hw-version }}-${{ inputs.artifact-suffix }}
           TEST_BIN=/tmp/caliptra-test-binaries
-          VARS="CPTRA_UIO_NUM=4 CALIPTRA_PREBUILT_FW_DIR=/tmp/caliptra-test-firmware CALIPTRA_IMAGE_NO_GIT_REVISION=1"
+          VARS="CPTRA_UIO_NUM=4 "
+          VARS+="CALIPTRA_PREBUILT_FW_DIR=/tmp/caliptra-unified/ "
+          VARS+="CALIPTRA_PREBUILT_ROM_BIN=/tmp/caliptra-unified/caliptra-rom.bin "
+          VARS+="CALIPTRA_IMAGE_NO_GIT_REVISION=1 "
 
           if [ "${{ inputs.rom-logging }}" == "true" ] || [ -z "${{ inputs.rom-logging }}" ]; then
             VARS+=" CPTRA_ROM_TYPE=ROM_WITH_UART"

.github/workflows/fw-test-emu.yml

@@ -10,16 +10,31 @@ on:
       extra-features:
         default:
         type: string
+      rom-ref:
+        default: "main"
+        type: string
       rom-logging:
         default: true
         type: boolean
       rom-version:
         default: "latest"
         type: string
+      fw-ref:
+        default: "main"
+        type: string
 
 jobs:
+  versioned-checkout-build:
+    uses: ./.github/workflows/versioned-checkout.yml
+    with:
+      rom-ref: "${{ inputs.rom-ref }}"
+      rom-logging: "${{ inputs.rom-logging }}"
+      fw-ref: "${{ inputs.fw-ref }}"
+      artifact-suffix: "${{ inputs.artifact-suffix }}"
+
   build_and_test:
     runs-on: ubuntu-22.04
+    needs: versioned-checkout-build
 
     env:
       NEXTEST_VERSION: 0.9.63
@@ -47,19 +62,41 @@ jobs:
 
       - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
 
       - name: Pull dpe submodule
         run: |
           git submodule update --init dpe
 
-      - name: Build firmware
-        run: |
-          mkdir /tmp/caliptra-test-firmware
-          cargo run -p caliptra-builder -- --all_elfs /tmp/caliptra-test-firmware
+      - name: Extract Versioned ROM and FMC/RT
+        uses: actions/download-artifact@v4
+        with:
+          # defined in versioned-checkout.yml, files will be moved to /tmp/caliptra
+          # there is no hw version definition here because we are running in the emulator, so assume latest
+          name: caliptra-unified-${{ inputs.rom-ref }}-${{ inputs.fw-ref }}-latest-${{ inputs.artifact-suffix }}
+          path: /tmp/caliptra-unified
 
+      - name: Update rustup
+        run: |
+          ls -la /tmp/
+          df -hi
+          rustup toolchain install nightly-2023-04-15-x86_64-unknown-linux-gnu
+          rustup target add riscv32imc-unknown-none-elf --toolchain nightly-2023-04-15-x86_64-unknown-linux-gnu
+          rustup component add rust-src --toolchain nightly-2023-04-15-x86_64-unknown-linux-gnu
+          ls -la /tmp/
+          df -hi
+
       - name: Run tests
         run: |
-          export CALIPTRA_PREBUILT_FW_DIR=/tmp/caliptra-test-firmware
+          export CALIPTRA_PREBUILT_FW_DIR=/tmp/caliptra-unified/
+          export CALIPTRA_PREBUILT_ROM_BIN=/tmp/caliptra-unified/caliptra-rom.bin
+          export RUST_BACKTRACE=full
+
+          rustup toolchain install nightly-2023-04-15-x86_64-unknown-linux-gnu
+          rustup target add riscv32imc-unknown-none-elf --toolchain nightly-2023-04-15-x86_64-unknown-linux-gnu
+          rustup component add rust-src --toolchain nightly-2023-04-15-x86_64-unknown-linux-gnu
+
           if [ "${{ inputs.rom-version }}" != "latest" ]; then
             export CPTRA_CI_ROM_VERSION="${{ inputs.rom-version }}"
           fi

.github/workflows/nightly-release.yml

@@ -2,6 +2,17 @@ name: Nightly Release
 
 on:
   workflow_dispatch:
+    inupts:
+      hw-version:
+        default: "latest"
+        type: string
+      rom-ref:
+        default: "main"
+        type: string
+      firmware-version:
+        default: "main"
+        type: string
+
   schedule:
   # 2:11 AM PST tuesday-saturday
   - cron: '11 10 * * 2-6'
@@ -373,6 +384,9 @@ jobs:
 
       - name: Generate release zip
         run: |
+          # argc == 1: <release_name>
+          # argc == 3: <release_name> <rom_ref> <firmware_ref> 
+
           ./ci-tools/release/build_release.sh ${{ needs.find-latest-release.outputs.new_release_tag }}
           mv ./release/release.zip ./release/caliptra_${{ needs.find-latest-release.outputs.new_release_tag }}.zip
 

.github/workflows/test-versioned-checkout.yml

@@ -0,0 +1,54 @@
+name: Test Versioned Workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      hw-version:
+        default: "latest"
+        type: string
+      rom-ref:
+        default: "main"
+        type: string
+      fw-ref:
+        default: "main"
+        type: string
+      artifact-suffix:
+        default: "test"
+        type: string
+  pull_request:
+    inputs:
+      hw-version:
+        default: "latest"
+        type: string
+      rom-ref:
+        default: "main"
+        type: string
+      fw-ref:
+        default: "main"
+        type: string
+      artifact-suffix:
+        default: "test"
+        type: string
+    branches:
+      - main
+
+jobs:
+  test-versioned-checkout:
+    uses: ./.github/workflows/versioned-checkout.yml
+    with:
+        rom-ref: "main"
+        rom-logging: false
+        fw-ref: "main"
+        hw-version: "latest"
+        workflow_call: true
+        artifact-suffix: "test"
+
+  get-artifacts:
+    runs-on: ubuntu-22.04
+    needs: test-versioned-checkout
+    steps:
+    - name: Download caliptra-unified artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: caliptra-unified-main-main-latest-test
+        path: /tmp/caliptra-unified