test.yml

name: BVT

on:
  workflow_dispatch:
    inputs:
      ref:
        required: false
        default: ''
        type: string
      filter:
        required: false
        default: ''
        type: string
      log_level:
        required: true
        default: 'Full.Light'
        type: choice
        options:
        - 'Basic.Light'
        - 'Basic.Verbose'
        - 'Datapath.Light'
        - 'Datapath.Verbose'
        - 'Full.Light'
        - 'Full.Verbose'
        - 'Performance.Light'
        - 'Performance.Verbose'
        - 'RPS.Light'
        - 'RPS.Verbose'
        - 'SpinQuic.Light'
        - 'SpinQuicWarnings.Light'
        - 'Stacks.Light'
        - 'Stacks.Verbose'
  push:
    branches:
    - main
    - release/*
  pull_request:
    branches:
    - main
    - release/*

concurrency:
  # Cancel any workflow currently in progress for the same PR.
  # Allow running concurrently with any other commits.
  group: bvt-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

permissions:
  contents: read
  issues: write

jobs:
  build-windows-kernel:
    name: Build WinKernel
    strategy:
      fail-fast: false
      matrix:
        vec: [ # Note that GitHub windows-2025 doesn't have the necessary bits to build kernel drivers currently.
          { config: "Debug", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Release", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" }
        ]
    uses: ./.github/workflows/build-reuse-winkernel.yml
    with:
      config: ${{ matrix.vec.config }}
      plat: ${{ matrix.vec.plat }}
      os: ${{ matrix.vec.os }}
      arch: ${{ matrix.vec.arch }}
      tls: ${{ matrix.vec.tls }}
      build: ${{ matrix.vec.build }}
      ref: ${{ inputs.ref || '' }}
      repo: ${{ github.repository }}

  build-windows:
    name: Build WinUser
    strategy:
      fail-fast: false
      matrix:
        vec: [
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "quictls", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "openssl", build: "-Test" },
          { config: "Release", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", build: "-Test" }
        ]
    uses: ./.github/workflows/build-reuse-win.yml
    with:
      config: ${{ matrix.vec.config }}
      plat: ${{ matrix.vec.plat }}
      os: ${{ matrix.vec.os }}
      arch: ${{ matrix.vec.arch }}
      tls: ${{ matrix.vec.tls }}
      sanitize: ${{ matrix.vec.sanitize }}
      build: ${{ matrix.vec.build }}
      ref: ${{ inputs.ref || '' }}
      repo: ${{ github.repository }}

  build-unix:
    name: Build Unix
    strategy:
      fail-fast: false
      matrix:
        vec: [
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Release", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test" },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test" },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring" },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", sanitize: "-SanitizeAddress", iouring: "-UseIoUring" },
          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring" },
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "openssl", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", sanitize: "-SanitizeAddress", build: "-Test" },
        ]
    uses: ./.github/workflows/build-reuse-unix.yml
    with:
      config: ${{ matrix.vec.config }}
      plat: ${{ matrix.vec.plat }}
      os: ${{ matrix.vec.os }}
      arch: ${{ matrix.vec.arch }}
      tls: ${{ matrix.vec.tls }}
      systemcrypto: ${{ matrix.vec.systemcrypto }}
      sanitize: ${{ matrix.vec.sanitize }}
      build: ${{ matrix.vec.build }}
      iouring: ${{ matrix.vec.iouring }}
      ref: ${{ inputs.ref || '' }}
      repo: ${{ github.repository }}

  bvt:
    name: BVT
    needs: [build-windows, build-unix]
    strategy:
      fail-fast: false
      matrix:
        vec: [
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", sanitize: "-SanitizeAddress", build: "-Test"  },
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", sanitize: "-SanitizeAddress", build: "-Test"  },
          { config: "Release", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test"  },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", sanitize: "-SanitizeAddress", build: "-Test"  },
          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test"  },
          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring"  },
          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring"  },
          { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "openssl", sanitize: "-SanitizeAddress", build: "-Test"  },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", sanitize: "-SanitizeAddress", build: "-Test", log: "Full.Verbose" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", xdp: "-UseXdp", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", xdp: "-UseXdp", qtip: "-UseQtip", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "quictls", build: "-Test", log: "Full.Verbose" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "quictls", xdp: "-UseXdp", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "quictls", xdp: "-UseXdp", qtip: "-UseQtip", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "openssl", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "openssl", xdp: "-UseXdp", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "openssl", xdp: "-UseXdp", qtip: "-UseQtip", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", xdp: "-UseXdp", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", xdp: "-UseXdp", qtip: "-UseQtip", sanitize: "-SanitizeAddress", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "quictls", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "quictls", xdp: "-UseXdp", build: "-Test" },
          { config: "Debug", plat: "windows", os: "windows-2025", arch: "x64", tls: "quictls", xdp: "-UseXdp", qtip: "-UseQtip", build: "-Test" },
          { config: "Debug", plat: "windows", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test", log: "Full.Verbose" },
          { config: "Release", plat: "windows", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test", log: "Full.Verbose" },
        ]
    runs-on: ${{ matrix.vec.plat == 'windows' && matrix.vec.os == 'WinServerPrerelease' && fromJson(format('[''self-hosted'', ''1ES.Pool=1es-msquic-pool'', ''1ES.ImageOverride=WinServerPrerelease-LatestPwsh'', ''JobId=bvt-{0}-{1}-{2}-{3}-{4}-{5}'']', matrix.vec.config, matrix.vec.arch, matrix.vec.tls, github.run_id, github.run_number, github.run_attempt)) || matrix.vec.os }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      with:
        fetch-depth: 0 # Don't need history.
        ref: ${{ inputs.ref || '' }}
    - name: Download Windows Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      if: matrix.vec.plat == 'windows'
      with: # note we always use binaries built on windows-2025.
        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-windows-2025-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.build }}
        path: artifacts
    - name: Download Linux Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      if: matrix.vec.plat == 'linux'
      with:
        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}${{ matrix.vec.iouring }}${{ matrix.vec.build }}
        path: artifacts
    - name: Fix permissions for Unix
      if: matrix.vec.plat == 'linux' || matrix.vec.plat == 'macos'
      run: |
        sudo chmod -R 777 artifacts
    - name: Prepare Machine
      run: scripts/prepare-machine.ps1 -Tls ${{ matrix.vec.tls }} -ForTest ${{ matrix.vec.xdp }}
      shell: pwsh
    - name: Install ETW Manifest
      if: matrix.vec.plat == 'windows'
      shell: pwsh
      run: |
        $MsQuicDll = ".\artifacts\bin\windows\${{ matrix.vec.arch }}_${{ matrix.vec.config }}_${{ matrix.vec.tls }}\msquic.dll"
        $ManifestPath = ".\src\manifest\MsQuicEtw.man"
        wevtutil.exe um $ManifestPath
        wevtutil.exe im $ManifestPath /rf:$($MsQuicDll) /mf:$($MsQuicDll)
    - name: Test
      if: matrix.vec.os == 'WinServerPrerelease'
      shell: pwsh
      timeout-minutes: 120
      run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ inputs.filter && '-Filter' }} ${{ inputs.filter || '' }}
    - name: Test
      if: matrix.vec.os != 'WinServerPrerelease'
      shell: pwsh
      timeout-minutes: 120
      run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ inputs.filter && '-Filter' }} ${{ inputs.filter || '' }}
    - name: Upload on Failure or Cancellation
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      if: failure() || cancelled()
      with:
        name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.iouring }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}
        path: artifacts

  bvt-kernel:
    name: BVT Kernel
    needs: [build-windows, build-windows-kernel]
    strategy:
      fail-fast: false
      matrix:
        vec: [
          { config: "Debug", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Release", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Debug", plat: "winkernel", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Release", plat: "winkernel", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test" },
        ]
    runs-on: ${{ matrix.vec.plat == 'winkernel' && matrix.vec.os == 'WinServerPrerelease' && fromJson(format('[''self-hosted'', ''1ES.Pool=1es-msquic-pool'', ''1ES.ImageOverride=WinServerPrerelease-LatestPwsh'', ''JobId=bvt-kernel-{0}-{1}-{2}-{3}-{4}-{5}'']', matrix.vec.config, matrix.vec.arch, matrix.vec.tls, github.run_id, github.run_number, github.run_attempt)) || matrix.vec.os }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      with:
        fetch-depth: 0 # Don't need history.
        ref: ${{ inputs.ref || '' }}
    - name: Download Kernel Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with: # note we always use kernel binaries built on windows-2022.
        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-windows-2022-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.build }}
        path: artifacts
    - name: Download Usermode Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with: # note we always use binaries built on windows-2025.
        name: ${{ matrix.vec.config }}-windows-windows-2025-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.build }}
        path: artifacts
    - name: Prepare Machine
      shell: pwsh
      run: scripts/prepare-machine.ps1 -ForTest -ForKernel
    - name: Install ETW Manifest
      shell: pwsh
      run: |
        $MsQuicDll = ".\artifacts\bin\windows\${{ matrix.vec.arch }}_${{ matrix.vec.config }}_${{ matrix.vec.tls }}\msquic.dll"
        $ManifestPath = ".\src\manifest\MsQuicEtw.man"
        wevtutil.exe um $ManifestPath
        wevtutil.exe im $ManifestPath /rf:$($MsQuicDll) /mf:$($MsQuicDll)
    - name: Test
      shell: pwsh
      timeout-minutes: 120
      run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults -Kernel -Filter -*ValidateConfiguration:*ValidAlpnLengths:*ResumeRejection*:*ClientCertificate*:*LoadBalanced*
    - name: Upload on Failure or Cancellation
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      if: failure() || cancelled()
      with:
        name: BVT-Kernel-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}
        path: artifacts

  interop:
    name: Interop
    needs: [build-windows, build-unix]
    strategy:
      fail-fast: false
      matrix:
        vec: [
          { config: "Release", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", build: "-Test" },
          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", build: "-Test" },
          { config: "Release", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Release", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test", xdp: "-UseXdp" },
          { config: "Release", plat: "windows", os: "windows-2025", arch: "x64", tls: "schannel", build: "-Test" },
          { config: "Release", plat: "windows", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test" },
        ]
    runs-on: ${{ matrix.vec.plat == 'windows' && matrix.vec.os == 'WinServerPrerelease' && fromJson(format('[''self-hosted'', ''1ES.Pool=1es-msquic-pool'', ''1ES.ImageOverride=WinServerPrerelease-LatestPwsh'', ''JobId=interop-{0}-{1}-{2}-{3}-{4}-{5}'']', matrix.vec.config, matrix.vec.arch, matrix.vec.tls, github.run_id, github.run_number, github.run_attempt)) || matrix.vec.os }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      with:
        fetch-depth: 0 # Don't need history.
        ref: ${{ inputs.ref || '' }}
    - name: Download Windows Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      if: matrix.vec.plat == 'windows'
      with: # note we always use binaries built on windows-2025.
        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-windows-2025-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.build }}
        path: artifacts
    - name: Download Linux Build Artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      if: matrix.vec.plat == 'linux'
      with:
        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}-UseSystemOpenSSLCrypto${{ matrix.vec.build }}
        path: artifacts
    - name: Fix permissions for Unix
      if: matrix.vec.plat == 'linux' || matrix.vec.plat == 'macos'
      run: |
        sudo chmod -R 777 artifacts
    - name: Prepare Machine
      run: scripts/prepare-machine.ps1 -ForTest ${{ matrix.vec.xdp }}
      shell: pwsh
    - name: Interop
      shell: pwsh
      run: scripts/interop.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -GHA -GenerateXmlResults -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }}
    - name: Upload results
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      if: failure() || cancelled()
      with:
        name: InteropWinlatest-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}
        path: artifacts

  Complete:
    name: BVT Complete
    if: always()
    needs: [bvt, bvt-kernel, interop]
    runs-on: ubuntu-latest
    permissions: {} # No need for any permissions.
    steps:
    - name: Decide whether the needed jobs succeeded or failed
      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
      with:
        jobs: ${{ toJSON(needs) }}

  Create-Or-Update-Issue:
    needs: [Complete]
    if: ${{ failure() && github.event_name == 'push' }}
    runs-on: ubuntu-latest
    steps:
    - name: Download BVT Failures
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        pattern: BVT-*
        path: artifacts
        merge-multiple: false

    - name: Create or Update Issue
      uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
      with:
        script: |
          const fs = require('fs');
          const path = require('path');
          const root = 'artifacts'
          const potential_issues = [];
          // If root does not exist, exit silently
          console.log("Let's start filing issues if any");
          if (!fs.existsSync(root)) {
              console.log('No artifacts found');
              process.exit(0);
          }
          class Issue {
              constructor(title, description) {
                  this.title = title;
                  this.description = description;
              }
          }
          let run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
          function find_issues(env, directory) {
              if (!fs.existsSync(path.join(root, env, 'logs', directory))) {
                  console.log(`${env} / ${directory} does not exist`);
                  return;
              }
              for (const log of fs.readdirSync(path.join(root, env, 'logs', directory))) {
                  // If log is not a directory, skip it.
                  if (!fs.statSync(path.join(root, env, 'logs', directory, log)).isDirectory()) {
                      console.log(`${log} is not a directory`);
                      continue;
                  }
                  for (const testcase of fs.readdirSync(path.join(root, env, 'logs', directory, log))) {
                      let title = `[CI - FAILURE]: ${env} / ${testcase} (${directory})`
                      let new_issue = new Issue(title, run_url);
                      potential_issues.push(new_issue);
                  }
              }
          }

          find_issues('', 'msquictest');
          find_issues('', 'msquicplatformtest');
          find_issues('', 'msquiccoretest');
          find_issues('', 'msquictest.exe');
          find_issues('', 'msquicplatformtest.exe');
          find_issues('', 'msquiccoretest.exe');

          for (const env of fs.readdirSync(root)) {
              // If env is not a directory, skip it.
              if (!fs.statSync(path.join(root, env)).isDirectory()) {
                  console.log(`ENV: ${env} is not a directory`);
                  continue;
              }
              find_issues(env, 'msquictest');
              find_issues(env, 'msquicplatformtest');
              find_issues(env, 'msquiccoretest');
              find_issues(env, 'msquictest.exe');
              find_issues(env, 'msquicplatformtest.exe');
              find_issues(env, 'msquiccoretest.exe');
          }
          for (let Issue of potential_issues) {
              console.log(Issue.title);
              console.log(Issue.description);
          }

          // --- Prepare to File Issues ---
          let owner = context.repo.owner;
          let repo = context.repo.repo;
          const dedup = new Map();
          for (const it of potential_issues) {
            if (!dedup.has(it.title)) dedup.set(it.title, it);
          }
          const issuesToFile = [...dedup.values()];

          if (issuesToFile.length > 10) {
            console.log('Too many CI failures: ' + issuesToFile.length);
            let body = run_url;
            for (const it of issuesToFile) {
              body += it.title + ", "
            }
            await github.rest.issues.create({
              owner, repo,
              title: 'Too many CI failures: ' + issuesToFile.length,
              body: body,
              labels: ['bvt', 'ci-failure'] // tweak/remove labels as you wish
            });
            process.exit(0);
          }

          // --- Fetch all open issues once (pagination safe) ---
          const openIssues = await github.paginate(
            github.rest.issues.listForRepo,
            { owner, repo, state: 'open', per_page: 100 }
          );
          // Build a quick lookup by title
          const openByTitle = new Map(openIssues.map(i => [i.title, i]));
          // --- Upsert: comment on existing (open) or create a new issue ---

          for (const it of issuesToFile) {
            const existing = openByTitle.get(it.title);
            if (existing) {
              await github.rest.issues.createComment({
                owner, repo,
                issue_number: existing.number,
                body: it.description
              });
              core.notice(`Commented on existing issue #${existing.number}: ${it.title}`);
            } else {
              const created = await github.rest.issues.create({
                owner, repo,
                title: it.title,
                body: it.description,
                labels: ['bvt', 'ci-failure'] // tweak/remove labels as you wish
              });
              core.notice(`Created issue #${created.data.number}: ${it.title}`);
            }
          }