stress-test-runtime.yml

name: Stress tests

on:
  schedule:
    - cron: "30 5 * * *"
  workflow_dispatch:
    inputs:
      sha:
        description: 'SHA to test'
        required: true
        type: string
        default: 'main'

permissions:
  packages: read

jobs:
  x86_64-linux:
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [release]
            target: test-stress-ubench-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [debug]
            target: test-stress-ubench-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [cd] [release]
            target: test-stress-ubench-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [cd] [debug]
            target: test-stress-ubench-with-cd-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [release]
            target: test-stress-ubench-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [debug]
            target: test-stress-ubench-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [cd] [release]
            target: test-stress-ubench-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [cd] [debug]
            target: test-stress-ubench-with-cd-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [release]
            target: test-stress-tcp-open-close-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [debug]
            target: test-stress-tcp-open-close-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [cd] [release]
            target: test-stress-tcp-open-close-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20250115
            name: x86-64-unknown-linux-ubuntu24.04 [cd] [debug]
            target: test-stress-tcp-open-close-with-cd-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [release]
            target: test-stress-tcp-open-close-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [debug]
            target: test-stress-tcp-open-close-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [cd] [release]
            target: test-stress-tcp-open-close-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203
            name: x86-64-unknown-linux-musl [cd] [debug]
            target: test-stress-tcp-open-close-with-cd-debug
            debugger: lldb

    name: ${{matrix.target}}:${{ matrix.name }}
    container:
      image: ${{ matrix.image }}
      options: --user pony --cap-add=SYS_PTRACE --security-opt seccomp=unconfined

    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1
        with:
          ref: ${{ github.event.inputs.sha || 'main' }}
      - name: Restore Libs Cache
        id: restore-libs
        uses: actions/cache/restore@v4
        with:
          path: build/libs
          key: libs-${{ matrix.image }}-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Libs
        if: steps.restore-libs.outputs.cache-hit != 'true'
        run: make libs build_flags=-j8
      - name: Save Libs Cache
        if: steps.restore-libs.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: build/libs
          key: libs-${{ matrix.image }}-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Debug Runtime
        run: |
          make configure config=debug
          make build config=debug
      - name: Run Stress Test
        run: make ${{ matrix.target }} config=debug usedebugger='${{ matrix.debugger }}'
      - name: Send alert on failure
        if: ${{ failure() }}
        uses: zulip/github-actions-zulip/send-message@e4c8f27c732ba9bd98ac6be0583096dea82feea5
        with:
          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
          organization-url: 'https://ponylang.zulipchat.com/'
          to: notifications
          type: stream
          topic: ${{ github.repository }} scheduled job failure
          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }}failed.

  aarch64-linux:
    runs-on: ubuntu-24.04-arm

    strategy:
      fail-fast: false
      matrix:
        include:
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [release]
            target: test-stress-ubench-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [debug]
            target: test-stress-ubench-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [cd] [release]
            target: test-stress-ubench-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [cd] [debug]
            target: test-stress-ubench-with-cd-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [release]
            target: test-stress-tcp-open-close-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [debug]
            target: test-stress-tcp-open-close-debug
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [cd] [release]
            target: test-stress-tcp-open-close-with-cd-release
            debugger: lldb
          - image: ghcr.io/ponylang/ponyc-ci-aarch64-unknown-linux-ubuntu24.04-builder:20250118
            name: aarch64-unknown-linux-ubuntu24.04 [cd] [debug]
            target: test-stress-tcp-open-close-with-cd-debug
            debugger: lldb

    name: ${{matrix.target}}:${{ matrix.name }}
    container:
      image: ${{ matrix.image }}
      options: --user pony --cap-add=SYS_PTRACE --security-opt seccomp=unconfined

    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1
        with:
          ref: ${{ github.event.inputs.sha || 'main' }}
      - name: Restore Libs Cache
        id: restore-libs
        uses: actions/cache/restore@v4
        with:
          path: build/libs
          key: libs-${{ matrix.image }}-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Libs
        if: steps.restore-libs.outputs.cache-hit != 'true'
        run: make libs build_flags=-j8
      - name: Save Libs Cache
        if: steps.restore-libs.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: build/libs
          key: libs-${{ matrix.image }}-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Debug Runtime
        run: |
          make configure arch=armv8-a config=debug
          make build config=debug
      - name: Run Stress Test
        run: make ${{ matrix.target }} config=debug usedebugger='${{ matrix.debugger }}'
      - name: Send alert on failure
        if: ${{ failure() }}
        uses: zulip/github-actions-zulip/send-message@e4c8f27c732ba9bd98ac6be0583096dea82feea5
        with:
          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
          organization-url: 'https://ponylang.zulipchat.com/'
          to: notifications
          type: stream
          topic: ${{ github.repository }} scheduled job failure
          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }}failed.

  x86_64-windows:
    runs-on: windows-2025
    defaults:
      run:
        shell: pwsh

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "x86-64 Windows [release]"
            target: stress-test-ubench-release
          - name: "x86-64 Windows [debug]"
            target: stress-test-ubench-debug
          - name: "x86-64 Windows [cd] [release]"
            target: stress-test-ubench-with-cd-release
          - name: "x86-64 Windows [cd] [debug]"
            target: stress-test-ubench-with-cd-debug
          - name: "x86-64 Windows [release]"
            target: stress-test-tcp-open-close-release
          - name: "x86-64 Windows [debug]"
            target: stress-test-tcp-open-close-debug
          - name: "x86-64 Windows [cd] [release]"
            target: stress-test-tcp-open-close-with-cd-release
          - name: "x86-64 Windows [cd] [debug]"
            target: stress-test-tcp-open-close-with-cd-debug

    name: ${{matrix.target}}:${{ matrix.name }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1
        with:
          ref: ${{ github.event.inputs.sha || 'main' }}
      - name: Install Dependencies
        run: |
          function msys() { C:\msys64\usr\bin\bash.exe @('-lc') + @Args }; msys ' '; msys 'pacman --noconfirm -Syuu'; msys 'pacman --noconfirm -Syuu'; msys 'pacman --noconfirm -S --needed base-devel mingw-w64-x86_64-lldb'; msys 'pacman --noconfirm -Scc'
          python.exe -m pip install --upgrade cloudsmith-cli
      - name: Restore Libs Cache
        id: restore-libs
        uses: actions/cache/restore@v4
        with:
          path: build/libs
          key: libs-windows-2025-${{ hashFiles('make.ps1', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Libs
        if: steps.restore-libs.outputs.cache-hit != 'true'
        run: .\make.ps1 -Command libs
      - name: Save Libs Cache
        if: steps.restore-libs.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: build/libs
          key: libs-windows-2025-${{ hashFiles('make.ps1', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Configure
        run: .\make.ps1 -Command configure -Config Debug
      - name: Build Debug Runtime
        run: .\make.ps1 -Command build -Config Debug
      - name: Run Stress Test
        run: .\make.ps1 -Command ${{ matrix.target }} -Config Debug
      - name: Send alert on failure
        if: ${{ failure() }}
        uses: zulip/github-actions-zulip/send-message@e4c8f27c732ba9bd98ac6be0583096dea82feea5
        with:
          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
          organization-url: 'https://ponylang.zulipchat.com/'
          to: notifications
          type: stream
          topic: ${{ github.repository }} scheduled job failure
          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed.

  x86_64-macos:
    runs-on: macos-13

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "x86-64 macOS [release]"
            target: test-stress-ubench-release
          - name: "x86-64 macOS [debug]"
            target: test-stress-ubench-debug
          - name: "x86-64 macOS [cd] [release]"
            target: test-stress-ubench-with-cd-release
          - name: "x86-64 macOS [cd] [debug]"
            target: test-stress-ubench-with-cd-debug
          - name: "x86-64 macOS [release]"
            target: test-stress-tcp-open-close-release
          - name: "x86-64 macOS [debug]"
            target: test-stress-tcp-open-close-debug
          - name: "x86-64 macOS [cd] [release]"
            target: test-stress-tcp-open-close-with-cd-release
          - name: "x86-64 macOS [cd] [debug]"
            target: test-stress-tcp-open-close-with-cd-debug

    name: ${{matrix.target}}:${{ matrix.name }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1
        with:
          ref: ${{ github.event.inputs.sha || 'main' }}
      - name: Restore Libs Cache
        id: restore-libs
        uses: actions/cache/restore@v4
        with:
          path: build/libs
          key: libs-x86-macos-13-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Libs
        if: steps.restore-libs.outputs.cache-hit != 'true'
        run: make libs build_flags=-j8
      - name: Save Libs Cache
        if: steps.restore-libs.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: build/libs
          key: libs-x86-macos-13-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Debug Runtime
        run: |
          make configure arch=x86-64 config=debug
          make build config=debug
      - name: Configure networking
        run: bash .ci-scripts/macOS-configure-networking.bash
      - name: Run Stress Test
        run: make ${{ matrix.target }} config=debug usedebugger=lldb
      - name: Send alert on failure
        if: ${{ failure() }}
        uses: zulip/github-actions-zulip/send-message@e4c8f27c732ba9bd98ac6be0583096dea82feea5
        with:
          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
          organization-url: 'https://ponylang.zulipchat.com/'
          to: notifications
          type: stream
          topic: ${{ github.repository }} scheduled job failure
          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed.

  arm64-macos:
    runs-on: macos-14

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "arm64 macOS [release]"
            target: test-stress-ubench-release
          - name: "arm64 macOS [debug]"
            target: test-stress-ubench-debug
          - name: "arm64 macOS [cd] [release]"
            target: test-stress-ubench-with-cd-release
          - name: "arm64 macOS [cd] [debug]"
            target: test-stress-ubench-with-cd-debug
          - name: "arm64 macOS [release]"
            target: test-stress-tcp-open-close-release
          - name: "arm64 macOS [debug]"
            target: test-stress-tcp-open-close-debug
          - name: "arm64 macOS [cd] [release]"
            target: test-stress-tcp-open-close-with-cd-release
          - name: "arm64 macOS [cd] [debug]"
            target: test-stress-tcp-open-close-with-cd-debug

    name: ${{matrix.target}}:${{ matrix.name }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1
        with:
          ref: ${{ github.event.inputs.sha || 'main' }}
      - name: Restore Libs Cache
        id: restore-libs
        uses: actions/cache/restore@v4
        with:
          path: build/libs
          key: libs-arm64-macos-14-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Libs
        if: steps.restore-libs.outputs.cache-hit != 'true'
        run: make libs build_flags=-j8
      - name: Save Libs Cache
        if: steps.restore-libs.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: build/libs
          key: libs-arm64-macos-14-${{ hashFiles('Makefile', 'CMakeLists.txt', 'lib/CMakeLists.txt') }}
      - name: Build Debug Runtime
        run: |
          make configure config=debug
          make build config=debug
      - name: Configure networking
        run: bash .ci-scripts/macOS-configure-networking.bash
      - name: Run Stress Test
        run: make ${{ matrix.target }} config=debug usedebugger=lldb
      - name: Send alert on failure
        if: ${{ failure() }}
        uses: zulip/github-actions-zulip/send-message@e4c8f27c732ba9bd98ac6be0583096dea82feea5
        with:
          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
          organization-url: 'https://ponylang.zulipchat.com/'
          to: notifications
          type: stream
          topic: ${{ github.repository }} scheduled job failure
          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed.