Refactor _get_class_balancing_data() (#191) #1065
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End2End Test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=4096 | |
| DATASET_PATH: $(pwd)/datasets | |
| E2E_BASE_URL: http://localhost:8001 | |
| LIGHTLY_STUDIO_LICENSE_KEY: ${{ secrets.MUNDIG_LICENSE_KEY }} | |
| jobs: | |
| end-to-end-test: | |
| name: End2End Test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| # Authenticate to download lightly-edge-sdk. | |
| - name: Authenticate with GCP | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: boris-250909 | |
| credentials_json: ${{ secrets.GCP_LIGHTLY_STUDIO_CI_READONLY }} | |
| ### Caching rules for the workflow ### | |
| - name: Cache UI app build | |
| id: cache-ui-build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| lightly_studio_view/build | |
| lightly_studio/src/lightly_studio/dist_lightly_studio_view_app | |
| key: ui-build-${{ hashFiles('lightly_studio_view/src/**', 'lightly_studio_view/public/**', 'lightly_studio_view/package.json', 'lightly_studio_view/package-lock.json') }} | |
| - name: Cache example dataset | |
| id: cache-example-dataset | |
| uses: actions/cache@v4 | |
| with: | |
| path: lightly_studio/datasets/coco-128 | |
| key: coco-example-dataset | |
| - name: Cache lightly_studio app build | |
| id: cache-python-build | |
| uses: actions/cache@v4 | |
| with: | |
| path: lightly_studio/dist | |
| key: python-build-${{ hashFiles('lightly_studio/src/**', 'lightly_studio/pyproject.toml', 'lightly_studio_view/src/**', 'lightly_studio_view/public/**', 'lightly_studio_view/package.json', 'lightly_studio_view/package-lock.json') }} | |
| - name: Cache e2e-tests | |
| id: cache-e2e-tests | |
| uses: actions/cache@v4 | |
| with: | |
| path: lightly_studio_view/playwright-report | |
| key: python-build-${{ hashFiles('lightly_studio/e2e-tests/index_dataset_for_end2end_ui_tests.py', 'lightly_studio/src/**', 'lightly_studio/pyproject.toml', 'lightly_studio_view/e2e/**', 'lightly_studio_view/src/**', 'lightly_studio_view/public/**', 'lightly_studio_view/package.json', 'lightly_studio_view/package-lock.json') }} | |
| ### Setup steps for the workflow ### | |
| - name: Set Up Python | |
| if: steps.cache-python-build.outputs.cache-hit != 'true' || steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Set Up uv | |
| if: steps.cache-python-build.outputs.cache-hit != 'true' || steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version: "0.8.17" | |
| enable-cache: true | |
| - name: Set Up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22.11.0' | |
| cache: 'npm' | |
| cache-dependency-path: lightly_studio_view/package-lock.json | |
| - name: Install dependencies | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' || steps.cache-python-build.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio_view | |
| run: npm ci | |
| # Cache Playwright browsers to avoid re-downloading each run | |
| - name: Cache Playwright browsers | |
| id: cache-playwright-browsers | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: playwright-${{ runner.os }}-${{ hashFiles('lightly_studio_view/package-lock.json') }} | |
| # Always ensure system packages for browsers are present on Ubuntu | |
| - name: Install Playwright system dependencies | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio_view | |
| run: npx playwright install-deps chromium | |
| # Only download browser binaries when cache misses; otherwise reuse cache | |
| - name: Install Playwright browsers | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' && steps.cache-playwright-browsers.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio_view | |
| run: npx playwright install chromium | |
| - name: Download example dataset | |
| if: steps.cache-example-dataset.outputs.cache-hit != 'true' | |
| run: make download-example-dataset | |
| - name: Install lightly_studio python dependencies | |
| if: steps.cache-python-build.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio | |
| run: make install-optional-deps | |
| - name: Export schema from backend | |
| if: steps.cache-python-build.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio | |
| run: make export-schema | |
| ### Buildging and steps for the workflow ### | |
| - name: Build the ui application | |
| if: steps.cache-ui-build.outputs.cache-hit != 'true' || steps.cache-python-build.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio_view | |
| run: | | |
| rm -rf build | |
| npm run build | |
| - name: Copy the built ui app to lightly_studio source folder | |
| if: steps.cache-ui-build.outputs.cache-hit != 'true' || steps.cache-python-build.outputs.cache-hit != 'true' | |
| run: | | |
| rm -rf lightly_studio/src/lightly_studio/dist_lightly_studio_view_app | |
| cp -r lightly_studio_view/build/ lightly_studio/src/lightly_studio/dist_lightly_studio_view_app | |
| - name: Index dataset with using lightly_studio pypi package | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio | |
| # TODO(Michal, 10/2025): Starting up the server redownloads the embedding model. Consider caching it. | |
| run: | | |
| uv run e2e-tests/index_dataset_for_end2end_ui_tests.py > server.log 2>&1 & | |
| echo "SERVER_PID=$!" >> $GITHUB_ENV | |
| id: server | |
| - name: Wait for the server to start | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| echo "Waiting for the server to become available..." | |
| timeout=360 | |
| elapsed=0 | |
| interval=5 | |
| while [ $elapsed -lt $timeout ]; do | |
| if curl -s -f ${E2E_BASE_URL}/healthz > /dev/null 2>&1; then | |
| echo "✅ Server is up and running after ${elapsed} seconds!" | |
| break | |
| fi | |
| echo "⏳ Server not ready yet, waiting... (${elapsed}/${timeout} seconds)" | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| if [ $elapsed -ge $timeout ]; then | |
| echo "❌ Timed out waiting for server to become available" | |
| exit 1 | |
| fi | |
| ### Running end-to-end tests ### | |
| - name: Run end-to-end tests for indexed dataset | |
| if: steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio_view | |
| run: npx playwright test --reporter=html --trace=on | |
| timeout-minutes: 10 | |
| ### Cleanup steps for the workflow ### | |
| - name: Show server logs after failure | |
| if: failure() && steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| working-directory: lightly_studio | |
| run: | | |
| echo "Server logs:" | |
| cat server.log | |
| - name: Upload Playwright report after failure | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: Upload Playwright Report | |
| path: lightly_studio_view/playwright-report | |
| retention-days: 5 | |
| - name: Stop lightly_studio application running in background | |
| if: always() && steps.cache-e2e-tests.outputs.cache-hit != 'true' | |
| run: | | |
| kill ${{ env.SERVER_PID }} || true |