add gpt5.2, gemeini 3 flash, deepseek v3.2 (#870) #170
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - legacy-lancedb | |
| push: | |
| branches: | |
| - main | |
| - legacy-lancedb | |
| tags: | |
| - "v*" | |
| jobs: | |
| check_changes: | |
| name: Check for changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| has-changes: ${{ steps.check.outputs.has-changes }} | |
| steps: | |
| - name: Check | |
| id: check | |
| uses: jiahuei/check-changes-action@v0 | |
| with: | |
| watch-dirs: "clients/python/ services/api/ docker/ .github/" | |
| sdk_tests_noop: | |
| # This job is needed so that status checks can still pass | |
| # This is because strategy matrix is evaluated after if condition | |
| name: SDK unit tests | |
| runs-on: ubuntu-latest | |
| needs: check_changes | |
| if: ${{ !(needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push') }} | |
| strategy: | |
| matrix: | |
| jamai-mode: ["oss"] | |
| test-group: [group1, group2, group3, group4] | |
| timeout-minutes: 2 | |
| steps: | |
| - name: No-op | |
| run: echo Tests skipped !!! | |
| sdk_tests: | |
| name: SDK unit tests | |
| runs-on: ubuntu-latest-l | |
| needs: check_changes | |
| if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' | |
| strategy: | |
| matrix: | |
| jamai-mode: ["oss"] | |
| test-group: [group1, group2, group3, group4] | |
| timeout-minutes: 90 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| lfs: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Inspect git version | |
| run: git --version | |
| - name: Check Docker Version | |
| run: docker version | |
| - name: Check Docker Compose Version | |
| run: docker compose version | |
| - name: Remove cloud-only modules | |
| if: matrix.jamai-mode == 'oss' | |
| run: bash scripts/remove_cloud_modules.sh | |
| - name: Inspect directory tree | |
| run: tree | |
| - name: Install jamaibase & owl | |
| run: | | |
| pushd clients/python | |
| uv pip install --system -e .[test] | |
| popd | |
| pushd services/api | |
| uv pip install --system -e .[test] | |
| - name: Inspect jamaibase environment | |
| run: | | |
| uv pip list | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Edit env file | |
| run: | | |
| mv .env.example .env | |
| ORGS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 1) | |
| KEYS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 2-) | |
| # Convert them into arrays | |
| ORG_ARRAY=($ORGS) | |
| KEY_ARRAY=($KEYS) | |
| # Loop through the ORG_ARRAY | |
| for i in "${!ORG_ARRAY[@]}"; do | |
| # Get the org and key | |
| org="${ORG_ARRAY[$i]}" | |
| key="${KEY_ARRAY[$i]}" | |
| # Replace the org with the key in the .env file | |
| sed -i "s/$org=.*/$org=$key/g" .env | |
| done | |
| echo "OWL_DB_INIT=0" >> .env | |
| echo "OWL_COMPUTE_STORAGE_PERIOD_SEC=15" >> .env | |
| echo "OWL_STRIPE_WEBHOOK_SECRET_TEST=${OWL_STRIPE_WEBHOOK_SECRET_TEST}" >> .env | |
| echo "OWL_STRIPE_PUBLISHABLE_KEY_TEST=${OWL_STRIPE_PUBLISHABLE_KEY_TEST}" >> .env | |
| echo 'OWL_SERVICE_KEY=lalala' >> .env | |
| echo 'JAMAI_TOKEN=lalala' >> .env | |
| echo 'JAMAI_API_BASE=http://localhost:6969/api' >> .env | |
| echo 'OWL_FLUSH_CLICKHOUSE_BUFFER_SEC=5' >> .env | |
| env: | |
| OWL_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OWL_COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| OWL_DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| OWL_ELLM_API_KEY: ${{ secrets.CUSTOM_API_KEY }} | |
| OWL_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| OWL_HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }} | |
| OWL_JINA_AI_API_KEY: ${{ secrets.JINA_AI_API_KEY }} | |
| OWL_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OWL_TOGETHER_AI_API_KEY: ${{ secrets.TOGETHER_AI_API_KEY }} | |
| OWL_STRIPE_WEBHOOK_SECRET_TEST: ${{ secrets.OWL_STRIPE_WEBHOOK_SECRET_TEST }} | |
| OWL_STRIPE_PUBLISHABLE_KEY_TEST: ${{ secrets.OWL_STRIPE_PUBLISHABLE_KEY_TEST }} | |
| - name: Configure Docker timeout | |
| run: | | |
| mkdir -p ~/.docker | |
| echo '{"max-concurrent-downloads": 3, "max-download-attempts": 5}' > ~/.docker/config.json | |
| - name: Launch services | |
| id: launch_services | |
| timeout-minutes: 20 | |
| if: always() | |
| run: | | |
| docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait | |
| env: | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_HTTP_TIMEOUT: 30 | |
| DOCKER_CLIENT_TIMEOUT: 30 | |
| - name: Inspect owl logs if failed to launch | |
| timeout-minutes: 1 | |
| if: failure() && steps.launch_services.outcome == 'failure' | |
| run: docker compose -p jm -f docker/compose.ci.yml logs owl | |
| - name: Inspect owl UV and Python version | |
| run: | | |
| docker exec jm-owl-1 uv -V | |
| docker exec jm-owl-1 $(docker exec jm-owl-1 uv python find) -V | |
| - name: Inspect owl environment | |
| if: always() | |
| run: docker exec jm-owl-1 uv pip list | |
| - name: Copy OpenAPI JSON | |
| id: copy_openapi | |
| if: always() && matrix.test-group == 'group1' | |
| run: | | |
| curl localhost:6969/api/public/openapi.json > openapi.json | |
| - name: Generate OpenAPI Redoc HTML page | |
| id: generate_redoc_html | |
| if: always() && matrix.test-group == 'group1' && steps.copy_openapi.outcome == 'success' | |
| run: | | |
| npx @redocly/cli@latest build-docs openapi.json | |
| mkdir openapi | |
| mv redoc-static.html openapi | |
| mv openapi.json openapi | |
| - name: Upload Redoc HTML | |
| id: upload_redoc_html | |
| uses: actions/upload-artifact@v4 | |
| if: always() && matrix.test-group == 'group1' && steps.generate_redoc_html.outcome == 'success' | |
| with: | |
| name: redoc-html-${{ matrix.jamai-mode }} | |
| path: openapi | |
| - name: Publish Redoc HTML link as PR comment | |
| uses: thollander/actions-comment-pull-request@v3 | |
| if: always() && matrix.test-group == 'group1' && github.event_name == 'pull_request' && steps.upload_redoc_html.outcome == 'success' | |
| with: | |
| message: | | |
| [Link to OpenAPI Redoc HTML (${{ matrix.jamai-mode }})](${{ steps.upload_redoc_html.outputs.artifact-url }}) | |
| comment-tag: redoc_html_comment_${{ matrix.jamai-mode }} | |
| - name: Python SDK tests | |
| id: python_sdk_test | |
| if: always() && steps.launch_services.outcome == 'success' | |
| run: | | |
| cp .env services/api/.env | |
| pushd services/api | |
| if [ "${{ matrix.test-group }}" = "group1" ]; then | |
| DIRS=(tests --ignore=tests/gen_table/test_row_ops.py --ignore=tests/gen_table/test_row_ops_v2.py --ignore=tests/routers --ignore=tests/utils) | |
| elif [ "${{ matrix.test-group }}" = "group2" ]; then | |
| DIRS=(tests/gen_table/test_row_ops.py) | |
| elif [ "${{ matrix.test-group }}" = "group3" ]; then | |
| DIRS=(tests/gen_table/test_row_ops_v2.py tests/utils) | |
| else | |
| DIRS=(tests/routers) | |
| fi | |
| coverage run --data-file=coverage/.coverage.${{ matrix.test-group }} --rcfile=pyproject.toml -m \ | |
| pytest \ | |
| --timeout 300 \ | |
| --no-flaky-report \ | |
| --junitxml=pytest_regular.xml \ | |
| -m "not (${{ matrix.jamai-mode == 'cloud' && 'oss' || 'cloud' }} or stripe)" \ | |
| "${DIRS[@]}" | |
| # Shut down owl to allow coverage data to be flushed | |
| popd | |
| docker compose -p jm -f docker/compose.ci.yml down | |
| # Copy Pytest coverage data | |
| sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} + | |
| env: | |
| OWL_DB_PATH: postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl | |
| OWL_CLICKHOUSE_HOST: localhost | |
| OWL_REDIS_HOST: localhost | |
| - name: Inspect owl logs | |
| if: always() && steps.launch_services.outcome == 'success' | |
| timeout-minutes: 1 | |
| run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs owl > logs/owl.log | |
| - name: Inspect starling logs | |
| if: always() && steps.launch_services.outcome == 'success' | |
| timeout-minutes: 1 | |
| run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs starling > logs/starling.log | |
| - name: Inspect test-llm logs | |
| timeout-minutes: 1 | |
| if: always() && steps.launch_services.outcome == 'success' | |
| run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs test-llm > logs/test-llm.log | |
| - name: Test Stripe integration (Cloud only) | |
| id: test_stripe | |
| if: matrix.jamai-mode == 'cloud' && matrix.test-group == 'group1' && steps.launch_services.outcome == 'success' | |
| run: | | |
| sudo rm -rf docker_data | |
| # Relaunch | |
| echo "OWL_STRIPE_API_KEY=${OWL_STRIPE_API_KEY}" >> .env | |
| docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait --force-recreate | |
| # Install Stripe CLI | |
| curl -L https://github.com/stripe/stripe-cli/releases/download/v1.27.0/stripe_1.27.0_linux_x86_64.tar.gz --output stripe.tar.gz | |
| tar -xvf stripe.tar.gz | |
| # Listen for Stripe events and forward them to local endpoint | |
| nohup ./stripe listen \ | |
| --forward-to http://localhost:6969/api/v2/organizations/webhooks/stripe & | |
| # --events customer.created,invoice.paid | |
| # Give stripe listen a moment to establish the tunnel | |
| sleep 5 | |
| # Run tests | |
| pushd services/api | |
| coverage run --data-file=coverage/.coverage.stripe --rcfile=pyproject.toml -m \ | |
| pytest \ | |
| --timeout 300 \ | |
| --no-flaky-report \ | |
| --junitxml=pytest_stripe.xml \ | |
| -m stripe \ | |
| tests | |
| # Shut down owl to allow coverage data to be flushed | |
| popd | |
| docker compose -p jm -f docker/compose.ci.yml down | |
| # Move existing coverage data | |
| sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} + | |
| env: | |
| STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }} | |
| OWL_STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }} | |
| OWL_CLICKHOUSE_HOST: localhost | |
| OWL_REDIS_HOST: localhost | |
| - name: Inspect owl logs if Stripe integration failed | |
| timeout-minutes: 1 | |
| if: failure() && matrix.jamai-mode == 'cloud' && steps.test_stripe.outcome == 'failure' | |
| run: mkdir -p logs && docker compose -p jm --env-file .env -f docker/compose.ci.yml logs owl > logs/owl_stripe.log | |
| # - name: TS/JS SDK tests | |
| # id: ts_sdk_test | |
| # if: always() && steps.launch_services.outcome == 'success' | |
| # run: | | |
| # cd clients/typescript | |
| # echo "BASEURL=http://localhost:6969" >> __tests__/.env | |
| # npm install | |
| # npm run test | |
| - name: Upload logs | |
| id: upload_logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.launch_services.outcome == 'success' | |
| with: | |
| name: logs-${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| path: logs | |
| - name: Publish logs link as PR comment | |
| uses: thollander/actions-comment-pull-request@v3 | |
| if: always() && github.event_name == 'pull_request' && steps.upload_logs.outcome == 'success' | |
| with: | |
| message: | | |
| [Link to logs (${{ matrix.jamai-mode }}, ${{ matrix.test-group }})](${{ steps.upload_logs.outputs.artifact-url }}) | |
| comment-tag: logs_comment_${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| - name: Merge JUnit XML and Coverage data | |
| id: merge_test_data | |
| if: always() && steps.launch_services.outcome == 'success' | |
| run: | | |
| # Shut down owl to allow coverage data to be flushed | |
| docker compose -p jm --env-file .env -f docker/compose.ci.yml down | |
| # Combine coverage data | |
| coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \ | |
| services/api/coverage | |
| # Merge JUnit XML files | |
| mkdir -p services/api/junit_xml | |
| junitparser merge --glob "services/api/pytest_*.xml" services/api/junit_xml/pytest-${{ matrix.jamai-mode }}-${{ matrix.test-group }}.xml | |
| - name: Upload coverage data | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.merge_test_data.outcome == 'success' | |
| with: | |
| name: owl-coverage-${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| path: services/api/coverage | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| - name: Upload JUnit XML file | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.merge_test_data.outcome == 'success' | |
| with: | |
| name: junit-xml-data-${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| path: services/api/junit_xml | |
| - name: Generate coverage reports | |
| id: generate_coverage_report | |
| if: always() && steps.merge_test_data.outcome == 'success' | |
| run: | | |
| cd services/api | |
| coverage html --data-file=coverage/.coverage -d coverage/html | |
| coverage xml --data-file=coverage/.coverage -o coverage/coverage.xml | |
| coverage report --data-file=coverage/.coverage | |
| - name: Upload coverage HTML report | |
| id: upload_coverage_html | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.merge_test_data.outcome == 'success' | |
| with: | |
| name: pytest-coverage-${{ matrix.jamai-mode }}-${{ matrix.test-group }} | |
| path: services/api/coverage/html | |
| merge_coverage: | |
| name: Merge Coverage Reports | |
| runs-on: ubuntu-latest | |
| needs: sdk_tests | |
| if: always() && (needs.sdk_tests.result == 'success' || needs.sdk_tests.result == 'skipped') | |
| strategy: | |
| matrix: | |
| jamai-mode: ["oss"] | |
| steps: | |
| - name: Skip coverage merge (no tests ran) | |
| if: needs.sdk_tests.result == 'skipped' | |
| run: echo "Skipping coverage merge - no tests were executed (no changes detected)" | |
| - name: Checkout code | |
| if: needs.sdk_tests.result == 'success' | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Install uv | |
| if: needs.sdk_tests.result == 'success' | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - name: Set up Python | |
| if: needs.sdk_tests.result == 'success' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install jamaibase & owl | |
| if: needs.sdk_tests.result == 'success' | |
| run: | | |
| pushd services/api | |
| uv pip install --system -e .[test] | |
| - name: Download coverage data artifacts | |
| uses: actions/download-artifact@v4 | |
| if: needs.sdk_tests.result == 'success' | |
| with: | |
| pattern: owl-coverage-* | |
| path: ./ | |
| - name: Download junit xml artifacts | |
| uses: actions/download-artifact@v4 | |
| if: needs.sdk_tests.result == 'success' | |
| with: | |
| pattern: junit-xml-data-* | |
| path: junit-xml-data | |
| - name: Log coverage data files | |
| if: needs.sdk_tests.result == 'success' | |
| run: | | |
| find owl-coverage-* -type f | head -50 | |
| find junit-xml-data -type f | head -50 | |
| - name: Merge JUnit XML and Coverage data (${{ matrix.jamai-mode }}) | |
| id: merge_coverage | |
| if: needs.sdk_tests.result == 'success' | |
| run: | | |
| coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \ | |
| ./owl-coverage-${{ matrix.jamai-mode }}-group[1-4] | |
| # Merge JUnit XML files | |
| junitparser merge --glob "junit-xml-data/junit-xml-data-${{ matrix.jamai-mode }}*/pytest-*.xml" \ | |
| junit-xml-data/pytest-${{ matrix.jamai-mode }}.xml | |
| - name: Generate coverage report | |
| if: always() && steps.merge_coverage.outcome == 'success' | |
| run: | | |
| cd services/api | |
| coverage html --data-file=coverage/.coverage -d coverage/html-${{ matrix.jamai-mode }} | |
| coverage xml --data-file=coverage/.coverage -o coverage/coverage-${{ matrix.jamai-mode }}.xml | |
| coverage report --data-file=coverage/.coverage | |
| - name: Upload ${{ matrix.jamai-mode }} coverage HTML report | |
| id: upload_coverage_html | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.merge_coverage.outcome == 'success' | |
| with: | |
| name: pytest-coverage-${{ matrix.jamai-mode }} | |
| path: services/api/coverage/html-${{ matrix.jamai-mode }} | |
| - name: Pytest coverage comment | |
| uses: MishaKav/pytest-coverage-comment@main | |
| if: always() && github.event_name == 'pull_request' && steps.merge_coverage.outcome == 'success' | |
| with: | |
| title: Coverage Report (${{ matrix.jamai-mode }}) | |
| pytest-xml-coverage-path: services/api/coverage/coverage-${{ matrix.jamai-mode }}.xml | |
| junitxml-path: junit-xml-data/pytest-${{ matrix.jamai-mode }}.xml | |
| unique-id-for-comment: coverage_report_comment_${{ matrix.jamai-mode }} | |
| report-only-changed-files: true | |
| - name: Merge All JUnit XML and Coverage data | |
| id: merge_all_test_data | |
| if: needs.sdk_tests.result == 'success' && matrix.jamai-mode == 'oss' | |
| run: | | |
| coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \ | |
| ./owl-coverage-oss-group[1-4] | |
| # Merge JUnit XML files | |
| junitparser merge --glob "junit-xml-data/junit-xml-data-*/pytest-*.xml" junit-xml-data/pytest.xml | |
| - name: Generate coverage reports | |
| id: generate_coverage_report | |
| if: always() && steps.merge_all_test_data.outcome == 'success' | |
| run: | | |
| cd services/api | |
| coverage html --data-file=coverage/.coverage -d coverage/html | |
| coverage xml --data-file=coverage/.coverage -o coverage/coverage.xml | |
| coverage report --data-file=coverage/.coverage | |
| - name: Pytest coverage comment | |
| uses: MishaKav/pytest-coverage-comment@main | |
| if: always() && github.event_name == 'pull_request' && steps.merge_all_test_data.outcome == 'success' | |
| with: | |
| title: Coverage Report (all) | |
| pytest-xml-coverage-path: services/api/coverage/coverage.xml | |
| unique-id-for-comment: coverage_report_comment_all | |
| report-only-changed-files: true | |
| - name: Upload all coverage HTML report | |
| id: upload_all_coverage_html | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.merge_all_test_data.outcome == 'success' | |
| with: | |
| name: pytest-coverage-all | |
| path: services/api/coverage/html | |
| - name: Publish coverage HTML report link as PR comment | |
| uses: thollander/actions-comment-pull-request@v3 | |
| if: always() && github.event_name == 'pull_request' && steps.upload_all_coverage_html.outcome == 'success' | |
| with: | |
| message: | | |
| [Link to coverage HTML report (All)](${{ steps.upload_all_coverage_html.outputs.artifact-url }}) | |
| comment-tag: coverage_html_report_comment_all |