Postgress connection standardization #1617
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| push: | |
| branches: [ main ] | |
| jobs: | |
| integration-test: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # Allow the workflow to commit the auto-generated | |
| # test/lyrics_expected_gte_512.json back to main on first run. | |
| contents: write | |
| services: | |
| postgres: | |
| image: postgres:15-alpine | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: audiomuse_migration_test | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Need a real (non-detached) checkout so we can push back the | |
| # auto-recorded test/lyrics_expected_gte_512.json. On pull_request events | |
| # we explicitly check out the PR head branch (not the merge ref) | |
| # so `git push` lands on the PR branch and the change becomes | |
| # visible in the PR before merge. | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref || github.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ffmpeg | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r test/requirements.txt | |
| - name: Download models | |
| run: | | |
| dclap_url="https://github.com/NeptuneHub/AudioMuse-AI-DCLAP/releases/download/v1" | |
| base_url="https://github.com/NeptuneHub/AudioMuse-AI/releases/download/v4.0.0-model" | |
| mkdir -p test/models | |
| echo "Downloading DCLAP audio model..." | |
| curl -L "${dclap_url}/model_epoch_36.onnx" -o "test/models/model_epoch_36.onnx" | |
| echo "Downloading DCLAP audio model data..." | |
| curl -L "${dclap_url}/model_epoch_36.onnx.data" -o "test/models/model_epoch_36.onnx.data" | |
| echo "Downloading CLAP text model..." | |
| curl -L "${base_url}/clap_text_model.onnx" -o "test/models/clap_text_model.onnx" | |
| echo "Downloading MusiCNN embedding model..." | |
| curl -L "${base_url}/musicnn_embedding.onnx" -o "test/models/musicnn_embedding.onnx" | |
| echo "Downloading MusiCNN prediction model..." | |
| curl -L "${base_url}/musicnn_prediction.onnx" -o "test/models/musicnn_prediction.onnx" | |
| echo "Models downloaded successfully" | |
| ls -lh test/models/ | |
| - name: Download HuggingFace tokenizer cache (release v4.0.0-model) | |
| run: | | |
| # The CLAP analyzer loads roberta-base via AutoTokenizer.from_pretrained(..., local_files_only=True). | |
| # Reuse the same huggingface_models.tar.gz bundled in the GitHub release that the | |
| # Dockerfile uses, and extract it into an HF cache directory under test/. | |
| base_url="https://github.com/NeptuneHub/AudioMuse-AI/releases/download/v4.0.0-model" | |
| hf_cache_dir="${GITHUB_WORKSPACE}/test/.hf_cache" | |
| mkdir -p "$hf_cache_dir" | |
| echo "Downloading huggingface_models.tar.gz..." | |
| curl -L "${base_url}/huggingface_models.tar.gz" -o /tmp/huggingface_models.tar.gz | |
| echo "Extracting into ${hf_cache_dir}..." | |
| tar -xzf /tmp/huggingface_models.tar.gz -C "$hf_cache_dir" | |
| rm -f /tmp/huggingface_models.tar.gz | |
| if [ ! -d "$hf_cache_dir/hub" ]; then | |
| echo "ERROR: extraction did not produce a hub/ directory" >&2 | |
| ls -lah "$hf_cache_dir" | |
| exit 1 | |
| fi | |
| echo "HF cache ready:" | |
| du -sh "$hf_cache_dir" | |
| # Expose HF_HOME to subsequent steps so transformers resolves the cache offline. | |
| echo "HF_HOME=$hf_cache_dir" >> "$GITHUB_ENV" | |
| - name: Download gte model bundle (release v4.0.0-model) | |
| run: | | |
| # The Dockerfile downloads lyrics_model_gte_vnni.tar.gz from the | |
| # v4.0.0-model release and extracts it flat into /app/model/. After | |
| # extraction the layout is: | |
| # <models>/gte-multilingual-base-int8.onnx (INT8 ONNX weights) | |
| # <models>/gte-multilingual-base/tokenizer.json (+ tokenizer files) | |
| # We mirror the same layout under test/models/ for the integration | |
| # test, which loads via lyrics/gte_onnx.py. | |
| base_url="https://github.com/NeptuneHub/AudioMuse-AI/releases/download/v4.0.0-model" | |
| mkdir -p test/models | |
| if [ ! -f test/models/gte-multilingual-base-int8.onnx ] || [ ! -f test/models/gte-multilingual-base/tokenizer.json ]; then | |
| echo "Downloading lyrics_model_gte_vnni.tar.gz..." | |
| curl -L "${base_url}/lyrics_model_gte_vnni.tar.gz" -o /tmp/lyrics_model_gte_vnni.tar.gz | |
| tar -xzf /tmp/lyrics_model_gte_vnni.tar.gz -C test/models | |
| rm -f /tmp/lyrics_model_gte_vnni.tar.gz | |
| fi | |
| echo "test/models/ after extraction:" | |
| ls -la test/models/ | |
| echo "gte-multilingual-base/ contents:" | |
| ls -la test/models/gte-multilingual-base/ || true | |
| echo "gte model checksums (sha256):" | |
| sha256sum test/models/gte-multilingual-base-int8.onnx test/models/gte-multilingual-base/tokenizer.json || true | |
| echo "LYRICS_MODEL_DIR=${GITHUB_WORKSPACE}/test/models" >> "$GITHUB_ENV" | |
| echo "LYRICS_GTE_ONNX_PATH=${GITHUB_WORKSPACE}/test/models/gte-multilingual-base-int8.onnx" >> "$GITHUB_ENV" | |
| echo "LYRICS_GTE_TOKENIZER_DIR=${GITHUB_WORKSPACE}/test/models/gte-multilingual-base" >> "$GITHUB_ENV" | |
| - name: Verify test files exist | |
| run: | | |
| echo "Checking for config.py..." | |
| ls -la config.py | |
| echo "" | |
| echo "Checking for ONNX models in test/models/..." | |
| ls -la test/models/ | |
| echo "" | |
| echo "Checking for audio files in test/songs/..." | |
| ls -la test/songs/ | |
| echo "" | |
| echo "Runner CPU INT8 path (vnni => int32 accumulation, none => int16/VPMADDUBSW):" | |
| grep -o 'avx512_vnni\|avx_vnni' /proc/cpuinfo | sort -u | head -1 || echo "none (non-VNNI)" | |
| - name: Run all integration tests | |
| env: | |
| AUDIOMUSE_TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/audiomuse_migration_test | |
| run: | | |
| # Run the WHOLE integration dir so any new test/integration/test_*.py is | |
| # picked up automatically (no per-file list to keep in sync). DB-backed | |
| # tests use AUDIOMUSE_TEST_DATABASE_URL; the MusiCNN/CLAP/lyrics model | |
| # tests read the model paths exported to GITHUB_ENV by the download steps | |
| # above. On first run the lyrics test auto-RECORDs | |
| # test/lyrics_expected_gte_512.json; the next step commits it back so | |
| # subsequent runs pin against it (cosine similarity >= 0.98). | |
| pytest test/integration/ -s -v --tb=short | |
| - name: Commit recorded lyrics expected vectors | |
| # Push back the auto-recorded vectors on: | |
| # * push to main | |
| # * pull_request originating from the SAME repository (forks can't | |
| # get write permission via GITHUB_TOKEN, so we skip them). | |
| if: | | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main') || | |
| (github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository) | |
| run: | | |
| if [ ! -f test/lyrics_expected_gte_512.json ]; then | |
| echo "No test/lyrics_expected_gte_512.json produced - nothing to commit." | |
| exit 0 | |
| fi | |
| # Only commit if the file is new or actually changed. | |
| if git diff --quiet -- test/lyrics_expected_gte_512.json && \ | |
| ! git status --porcelain test/lyrics_expected_gte_512.json | grep -q '^??'; then | |
| echo "test/lyrics_expected_gte_512.json is unchanged - skipping commit." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add test/lyrics_expected_gte_512.json | |
| git commit -m "test(lyrics): record expected vectors for integration test" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| target_branch="${{ github.event.pull_request.head.ref }}" | |
| else | |
| target_branch="main" | |
| fi | |
| echo "Pushing to branch: $target_branch" | |
| git push origin "HEAD:${target_branch}" | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-results | |
| path: | | |
| test/*.log | |
| test/temp/ | |
| retention-days: 7 |