Merge pull request #2069 from confident-ai/tmp/maintainer-workflow

pushing maintainer only workflow to upstream

Author: trevor-cai

.github/workflows/full_test_core_for_pr.yml

@@ -0,0 +1,113 @@
+name: Full Tests (maintainer only)
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: "PR number"
+        required: true
+      ref_kind:
+        description: "Which ref to test (merge|head)"
+        required: false
+        default: "merge"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: full-tests-pr-${{ github.event.inputs.pr }}-${{ github.event.inputs.ref_kind }}
+  cancel-in-progress: true
+
+
+jobs:
+  full-tests:
+    if: ${{ github.repository_owner == 'confident-ai' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    environment: ci-secrets
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+    steps:
+      - name: Resolve ref
+        id: refsel
+        run: |
+          if [ "${{ github.event.inputs.ref_kind }}" = "head" ]; then
+            echo "ref=refs/pull/${{ github.event.inputs.pr }}/head" >> $GITHUB_OUTPUT
+          else
+            # test what would merge
+            echo "ref=refs/pull/${{ github.event.inputs.pr }}/merge" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout PR ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.refsel.outputs.ref }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Cache virtualenv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      # Core deps only (main)
+      - name: Install dependencies (main)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root --only main
+
+      - name: Install project (main)
+        run: poetry install --no-interaction --only main
+
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+
+      # Run Core tests
+      - name: Run core tests (with secrets)
+        if: ${{ env.OPENAI_API_KEY != '' }}
+        run: |
+          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys \
+            tests/test_core/ \
+            --ignore=tests/test_core/test_synthesizer/ \
+            --ignore=tests/test_core/test_datasets/
+
+      - name: Run core tests (no secrets)
+        if: ${{ env.OPENAI_API_KEY == '' }}
+        run: |
+          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys \
+            tests/test_core/ \
+            --ignore=tests/test_core/test_synthesizer/ \
+            --ignore=tests/test_core/test_datasets/ \
+            --ignore=tests/test_core/test_evaluation/test_end_to_end/test_configs.py \
+            --ignore=tests/test_core/test_tracing/test_dataset_iterator.py
+
+      # Install dev dependencies and run dev tests
+      - name: Install dev dependencies
+        run: poetry install --no-interaction --with dev
+
+      - name: Run dev tests (with secrets)
+        if: ${{ env.OPENAI_API_KEY != '' }}
+        run: |
+          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys -o faulthandler_timeout=300 \
+            tests/test_core/test_synthesizer/ tests/test_core/test_datasets/
+
+      - name: Run dev tests (no secrets)
+        if: ${{ env.OPENAI_API_KEY == '' }}
+        run: |
+          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys -o faulthandler_timeout=300 \
+            tests/test_core/test_datasets/