feat: unify export + consensus entropy under featureExportRNG #1393
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nix - GA Runner | |
| on: | |
| push: | |
| branches: ["dev", "candidate", "release"] | |
| pull_request: | |
| branches: ["**"] | |
| types: [opened, synchronize, reopened, labeled, unlabeled] | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| full_matrix: | |
| description: "Force full matrix (6 configs)" | |
| required: false | |
| default: "false" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| matrix-setup: | |
| if: > | |
| github.event_name != 'pull_request' || | |
| contains(fromJson('["dev","candidate","release"]'), github.base_ref) || | |
| contains(join(github.event.pull_request.labels.*.name, ','), 'ci-full-build') | |
| runs-on: [self-hosted, generic, 20.04] | |
| container: python:3-slim | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: escape double quotes | |
| id: escape | |
| shell: bash | |
| env: | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| run: | | |
| ESCAPED_PR_TITLE="${PR_TITLE//\"/\\\"}" | |
| echo "title=${ESCAPED_PR_TITLE}" >> "$GITHUB_OUTPUT" | |
| - name: Generate build matrix | |
| id: set-matrix | |
| shell: python | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| import json | |
| import os | |
| import urllib.request | |
| # Full matrix with all 6 compiler configurations | |
| # Each configuration includes all parameters needed by the build job | |
| full_matrix = [ | |
| { | |
| "compiler_id": "gcc-11-libstdcxx", | |
| "compiler": "gcc", | |
| "cc": "gcc-11", | |
| "cxx": "g++-11", | |
| "compiler_version": 11, | |
| "stdlib": "libstdcxx", | |
| "configuration": "Debug" | |
| }, | |
| { | |
| "compiler_id": "gcc-13-libstdcxx", | |
| "compiler": "gcc", | |
| "cc": "gcc-13", | |
| "cxx": "g++-13", | |
| "compiler_version": 13, | |
| "stdlib": "libstdcxx", | |
| "configuration": "Debug" | |
| }, | |
| { | |
| "compiler_id": "clang-14-libstdcxx-gcc11", | |
| "compiler": "clang", | |
| "cc": "clang-14", | |
| "cxx": "clang++-14", | |
| "compiler_version": 14, | |
| "stdlib": "libstdcxx", | |
| "clang_gcc_toolchain": 11, | |
| "configuration": "Debug" | |
| }, | |
| { | |
| "compiler_id": "clang-16-libstdcxx-gcc13", | |
| "compiler": "clang", | |
| "cc": "clang-16", | |
| "cxx": "clang++-16", | |
| "compiler_version": 16, | |
| "stdlib": "libstdcxx", | |
| "clang_gcc_toolchain": 13, | |
| "configuration": "Debug" | |
| }, | |
| { | |
| "compiler_id": "clang-17-libcxx", | |
| "compiler": "clang", | |
| "cc": "clang-17", | |
| "cxx": "clang++-17", | |
| "compiler_version": 17, | |
| "stdlib": "libcxx", | |
| "configuration": "Debug" | |
| }, | |
| { | |
| # Clang 18 - testing if it's faster than Clang 17 with libc++ | |
| # Requires patching Conan v1 settings.yml to add version 18 | |
| "compiler_id": "clang-18-libcxx", | |
| "compiler": "clang", | |
| "cc": "clang-18", | |
| "cxx": "clang++-18", | |
| "compiler_version": 18, | |
| "stdlib": "libcxx", | |
| "configuration": "Debug" | |
| } | |
| ] | |
| # Minimal matrix for PRs and feature branches | |
| minimal_matrix = [ | |
| full_matrix[1], # gcc-13 (middle-ground gcc) | |
| full_matrix[2] # clang-14 (mature, stable clang) | |
| ] | |
| # Determine which matrix to use based on the target branch | |
| ref = "${{ github.ref }}" | |
| base_ref = "${{ github.base_ref }}" # For PRs, this is the target branch | |
| event_name = "${{ github.event_name }}" | |
| pr_title = """${{ steps.escape.outputs.title }}""" | |
| pr_labels = """${{ join(github.event.pull_request.labels.*.name, ',') }}""" | |
| pr_head_sha = "${{ github.event.pull_request.head.sha }}" | |
| # Get commit message - for PRs, fetch via API since head_commit.message is empty | |
| if event_name == "pull_request" and pr_head_sha: | |
| try: | |
| url = f"https://api.github.com/repos/${{ github.repository }}/commits/{pr_head_sha}" | |
| req = urllib.request.Request(url, headers={ | |
| "Accept": "application/vnd.github.v3+json", | |
| "Authorization": f"Bearer {os.environ.get('GH_TOKEN', '')}" | |
| }) | |
| with urllib.request.urlopen(req) as response: | |
| data = json.load(response) | |
| commit_message = data["commit"]["message"] | |
| except Exception as e: | |
| print(f"Failed to fetch commit message: {e}") | |
| commit_message = "" | |
| else: | |
| commit_message = """${{ github.event.head_commit.message }}""" | |
| # Debug logging | |
| print(f"Event: {event_name}") | |
| print(f"Ref: {ref}") | |
| print(f"Base ref: {base_ref}") | |
| print(f"PR head SHA: {pr_head_sha}") | |
| print(f"PR title: {pr_title}") | |
| print(f"PR labels: {pr_labels}") | |
| print(f"Commit message: {commit_message}") | |
| # Manual trigger input to force full matrix. | |
| manual_full = "${{ github.event.inputs.full_matrix || 'false' }}" == "true" | |
| # Label/manual overrides, while preserving existing title/commit behavior. | |
| force_full = ( | |
| manual_full | |
| or "[ci-nix-full-matrix]" in commit_message | |
| or "[ci-nix-full-matrix]" in pr_title | |
| or ("ci-full-build" in pr_labels and "ci-nix-full-matrix" in pr_labels) | |
| ) | |
| force_min = ( | |
| "ci-full-build" in pr_labels | |
| ) | |
| print(f"Force full matrix: {force_full}") | |
| print(f"Force min matrix: {force_min}") | |
| # Check if this is targeting a main branch | |
| # For PRs: check base_ref (target branch) | |
| # For pushes: check ref (current branch) | |
| main_branches = ["refs/heads/dev", "refs/heads/release", "refs/heads/candidate"] | |
| if force_full: | |
| # Override: always use full matrix if forced by manual input or label. | |
| use_full = True | |
| elif force_min: | |
| # Override: always use minimal matrix if ci-full-build label is present. | |
| use_full = False | |
| elif event_name == "pull_request": | |
| # For PRs, base_ref is just the branch name (e.g., "dev", not "refs/heads/dev") | |
| # Check if the PR targets release or candidate (more critical branches) | |
| use_full = base_ref in ["release", "candidate"] | |
| else: | |
| # For pushes, ref is the full reference (e.g., "refs/heads/dev") | |
| use_full = ref in main_branches | |
| # Select the appropriate matrix | |
| if use_full: | |
| if force_full: | |
| print(f"Using FULL matrix (6 configs) - forced by [ci-nix-full-matrix] tag") | |
| else: | |
| print(f"Using FULL matrix (6 configs) - targeting main branch") | |
| matrix = full_matrix | |
| else: | |
| print(f"Using MINIMAL matrix (2 configs) - feature branch/PR") | |
| matrix = minimal_matrix | |
| # Output the matrix as JSON | |
| output = json.dumps({"include": matrix}) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f"matrix={output}\n") | |
| build: | |
| needs: matrix-setup | |
| runs-on: [self-hosted, generic, 20.04] | |
| container: | |
| image: ubuntu:24.04 | |
| volumes: | |
| - /home/runner/.conan-cache:/.conan-cache | |
| - /home/runner/.ccache-cache:/github/home/.ccache-cache | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| artifact_name: ${{ steps.set-artifact-name.outputs.artifact_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-setup.outputs.matrix) }} | |
| env: | |
| build_dir: .build | |
| # Bump this number to invalidate all caches globally. | |
| CACHE_VERSION: 3 | |
| MAIN_BRANCH_NAME: dev | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install build dependencies | |
| run: | | |
| apt-get update | |
| apt-get install -y software-properties-common | |
| add-apt-repository ppa:ubuntu-toolchain-r/test -y | |
| apt-get update | |
| apt-get install -y python3 python-is-python3 pipx | |
| pipx ensurepath | |
| apt-get install -y cmake ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache | |
| apt-get install -y perl # for openssl build | |
| apt-get install -y libsqlite3-dev # for xahaud build | |
| # Install the specific GCC version needed for Clang | |
| if [ -n "${{ matrix.clang_gcc_toolchain }}" ]; then | |
| echo "=== Installing GCC ${{ matrix.clang_gcc_toolchain }} for Clang ===" | |
| apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev | |
| echo "=== GCC versions available after installation ===" | |
| ls -la /usr/lib/gcc/x86_64-linux-gnu/ | grep -E "^d" | |
| fi | |
| # For Clang < 16 with --gcc-toolchain, hide newer GCC versions | |
| # This is needed because --gcc-toolchain still picks the highest version | |
| # | |
| # THE GREAT GCC HIDING TRICK (for Clang < 16): | |
| # Clang versions before 16 don't have --gcc-install-dir, only --gcc-toolchain | |
| # which is deprecated and still uses discovery heuristics that ALWAYS pick | |
| # the highest version number. So we play a sneaky game... | |
| # | |
| # We rename newer GCC versions to very low integers (1, 2, 3...) which makes | |
| # Clang think they're ancient GCC versions. Since 11 > 3 > 2 > 1, Clang will | |
| # pick GCC 11 over our renamed versions. It's dumb but it works! | |
| # | |
| # Example: GCC 12→1, GCC 13→2, GCC 14→3, so Clang picks 11 (highest number) | |
| if [ -n "${{ matrix.clang_gcc_toolchain }}" ] && [ "${{ matrix.compiler_version }}" -lt "16" ]; then | |
| echo "=== Hiding GCC versions newer than ${{ matrix.clang_gcc_toolchain }} for Clang < 16 ===" | |
| target_version=${{ matrix.clang_gcc_toolchain }} | |
| counter=1 # Start with 1 - these will be seen as "GCC version 1, 2, 3" etc | |
| for dir in /usr/lib/gcc/x86_64-linux-gnu/*/; do | |
| if [ -d "$dir" ]; then | |
| version=$(basename "$dir") | |
| # Check if version is numeric and greater than target | |
| if [[ "$version" =~ ^[0-9]+$ ]] && [ "$version" -gt "$target_version" ]; then | |
| echo "Hiding GCC $version -> renaming to $counter (will be seen as GCC version $counter)" | |
| # Safety check: ensure target doesn't already exist | |
| if [ ! -e "/usr/lib/gcc/x86_64-linux-gnu/$counter" ]; then | |
| mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter" | |
| else | |
| echo "ERROR: Cannot rename GCC $version - /usr/lib/gcc/x86_64-linux-gnu/$counter already exists" | |
| exit 1 | |
| fi | |
| counter=$((counter + 1)) | |
| fi | |
| fi | |
| done | |
| fi | |
| # Verify what Clang will use | |
| if [ -n "${{ matrix.clang_gcc_toolchain }}" ]; then | |
| echo "=== Verifying GCC toolchain selection ===" | |
| echo "Available GCC versions:" | |
| ls -la /usr/lib/gcc/x86_64-linux-gnu/ | grep -E "^d.*[0-9]+$" || true | |
| echo "" | |
| echo "Clang's detected GCC installation:" | |
| ${{ matrix.cxx }} -v -E -x c++ /dev/null -o /dev/null 2>&1 | grep "Found candidate GCC installation" || true | |
| fi | |
| # Install libc++ dev packages if using libc++ (not needed for libstdc++) | |
| if [ "${{ matrix.stdlib }}" = "libcxx" ]; then | |
| apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev | |
| fi | |
| # Install Conan 2 | |
| pipx install "conan>=2.0,<3" | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Check environment | |
| run: | | |
| echo "PATH:" | |
| echo "${PATH}" | tr ':' '\n' | |
| which conan && conan --version || echo "Conan not found" | |
| which cmake && cmake --version || echo "CMake not found" | |
| which ${{ matrix.cc }} && ${{ matrix.cc }} --version || echo "${{ matrix.cc }} not found" | |
| which ${{ matrix.cxx }} && ${{ matrix.cxx }} --version || echo "${{ matrix.cxx }} not found" | |
| which ccache && ccache --version || echo "ccache not found" | |
| echo "---- Full Environment ----" | |
| env | |
| - name: Get commit message | |
| id: get-commit-message | |
| uses: ./.github/actions/xahau-ga-get-commit-message | |
| with: | |
| event-name: ${{ github.event_name }} | |
| head-commit-message: ${{ github.event.head_commit.message }} | |
| pr-head-sha: ${{ github.event.pull_request.head.sha }} | |
| - name: Install dependencies | |
| uses: ./.github/actions/xahau-ga-dependencies | |
| with: | |
| configuration: ${{ matrix.configuration }} | |
| build_dir: ${{ env.build_dir }} | |
| compiler-id: ${{ matrix.compiler_id }} | |
| cache_version: ${{ env.CACHE_VERSION }} | |
| main_branch: ${{ env.MAIN_BRANCH_NAME }} | |
| compiler: ${{ matrix.compiler }} | |
| compiler_version: ${{ matrix.compiler_version }} | |
| cc: ${{ matrix.cc }} | |
| cxx: ${{ matrix.cxx }} | |
| stdlib: ${{ matrix.stdlib }} | |
| gha_cache_enabled: 'false' # Disable caching for self hosted runner | |
| - name: Build | |
| uses: ./.github/actions/xahau-ga-build | |
| with: | |
| generator: Ninja | |
| configuration: ${{ matrix.configuration }} | |
| build_dir: ${{ env.build_dir }} | |
| cc: ${{ matrix.cc }} | |
| cxx: ${{ matrix.cxx }} | |
| compiler-id: ${{ matrix.compiler_id }} | |
| cache_version: ${{ env.CACHE_VERSION }} | |
| main_branch: ${{ env.MAIN_BRANCH_NAME }} | |
| stdlib: ${{ matrix.stdlib }} | |
| clang_gcc_toolchain: ${{ matrix.clang_gcc_toolchain || '' }} | |
| ccache_max_size: '100G' | |
| - name: Set artifact name | |
| id: set-artifact-name | |
| run: | | |
| ARTIFACT_NAME="build-output-nix-${{ github.run_id }}-${{ matrix.compiler }}-${{ matrix.configuration }}" | |
| echo "artifact_name=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT" | |
| echo "Using artifact name: ${ARTIFACT_NAME}" | |
| - name: Debug build directory | |
| run: | | |
| echo "Checking build directory contents: ${{ env.build_dir }}" | |
| ls -la ${{ env.build_dir }} || echo "Build directory not found or empty" | |
| - name: Run tests | |
| run: | | |
| # Ensure the binary exists before trying to run | |
| if [ -f "${{ env.build_dir }}/rippled" ]; then | |
| ${{ env.build_dir }}/rippled --unittest --unittest-jobs $(nproc) | |
| else | |
| echo "Error: rippled executable not found in ${{ env.build_dir }}" | |
| exit 1 | |
| fi |