chore: update pre-commit hooks #599
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CD | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| release: | |
| types: | |
| - published | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| FORCE_COLOR: 3 | |
| jobs: | |
| generate-indices: | |
| name: Generate index artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: requirements-cd.txt | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-cd.txt | |
| - name: Authorize Google Cloud | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| credentials_json: "${{ secrets.SERVICE_ACCOUNT_KEY }}" | |
| create_credentials_file: true | |
| export_environment_variables: true | |
| - name: Execute SQL Query and Generate Parquet Files | |
| run: | | |
| python scripts/python/idc_index_data_manager.py \ | |
| --generate-parquet \ | |
| --output-dir release_artifacts \ | |
| --cache-bucket idc-index-data-cache | |
| env: | |
| GCP_PROJECT: ${{ env.GCP_PROJECT }} | |
| - name: Generate GDC patient check parquet | |
| run: | | |
| python scripts/gdc/gdc_parquet_generator.py | |
| mv gdc_idc_mapping.parquet release_artifacts/ | |
| env: | |
| GCP_PROJECT: ${{ env.GCP_PROJECT }} | |
| - name: Generate TCIA subset parquet | |
| run: | | |
| python scripts/tcia/tcia_parquet_generator.py \ | |
| release_artifacts/idc_index.parquet \ | |
| -o release_artifacts/tcia_idc_subset.parquet | |
| - name: Add provenance metadata and SHA256 checksums | |
| run: | | |
| python scripts/python/add_parquet_provenance.py \ | |
| release_artifacts/ \ | |
| --version "$(git describe --tags --long)" | |
| - name: Report generated index sizes | |
| run: | |
| python scripts/python/report_index_sizes.py release_artifacts | tee -a | |
| $GITHUB_STEP_SUMMARY | |
| - name: Upload index artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: idc-index-artifacts | |
| path: release_artifacts/ | |
| retention-days: 14 | |
| if-no-files-found: error | |
| dist: | |
| name: Distribution build | |
| runs-on: ubuntu-latest | |
| needs: [generate-indices] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download pre-generated index artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: idc-index-artifacts | |
| path: release_artifacts/ | |
| - name: Move artifacts to project root for packaging | |
| run: | | |
| mv release_artifacts/*.parquet ./src/idc_index_data || true | |
| mv release_artifacts/*.json ./src/idc_index_data || true | |
| mv release_artifacts/*.sql ./src/idc_index_data || true | |
| mv release_artifacts/*.zip ./src/idc_index_data || true | |
| ls -la ./src/idc_index_data | |
| - name: Build distribution package | |
| run: | | |
| # Unset GCP_PROJECT to prevent build hook from regenerating files | |
| # Files already exist from the generate-indices job | |
| unset GCP_PROJECT | |
| python -m pip install --upgrade build | |
| python -m build | |
| env: | |
| # Explicitly unset GCP_PROJECT to skip data generation in build hook | |
| GCP_PROJECT: "" | |
| - name: Upload distribution artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: Packages | |
| path: dist/ | |
| retention-days: 14 | |
| - name: Inspect package contents | |
| run: | | |
| python -m pip install --upgrade check-wheel-contents | |
| ls -lh dist/ | |
| check-wheel-contents dist/*.whl | |
| attach-to-release: | |
| name: Attach artifacts to release | |
| needs: [generate-indices] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - name: Download index artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: idc-index-artifacts | |
| path: release_artifacts/ | |
| - name: Attach artifacts to release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| artifacts: "release_artifacts/*.parquet,release_artifacts/*.json,release_artifacts/*.sql,release_artifacts/*.sha256" | |
| allowUpdates: true | |
| omitBodyDuringUpdate: true | |
| upload-to-gcs: | |
| name: Upload artifacts to GCS | |
| needs: [generate-indices] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - name: Download index artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: idc-index-artifacts | |
| path: release_artifacts/ | |
| - name: Authorize Google Cloud | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" | |
| - name: Upload artifacts to GCS bucket | |
| uses: google-github-actions/upload-cloud-storage@v3 | |
| with: | |
| path: release_artifacts | |
| destination: | |
| idc-index-data-artifacts/${{ github.event.release.tag_name }} | |
| - name: Update current/ folder with latest release | |
| run: | | |
| gcloud storage rm -r gs://idc-index-data-artifacts/current/ || true | |
| gcloud storage cp -r \ | |
| gs://idc-index-data-artifacts/${{ github.event.release.tag_name }}/* \ | |
| gs://idc-index-data-artifacts/current/ | |
| - name: Ensure bucket is publicly readable | |
| run: | | |
| gcloud storage buckets add-iam-policy-binding gs://idc-index-data-artifacts \ | |
| --member=allUsers --role=roles/storage.objectViewer | |
| - name: Ensure CORS policy is set | |
| run: | | |
| cat > /tmp/cors.json << 'CORS' | |
| [ | |
| { | |
| "origin": ["*"], | |
| "method": ["GET", "HEAD"], | |
| "responseHeader": ["Content-Range", "Content-Length", "Content-Type", "Accept-Ranges", "Range"], | |
| "maxAgeSeconds": 86400 | |
| } | |
| ] | |
| CORS | |
| gcloud storage buckets update gs://idc-index-data-artifacts --cors-file=/tmp/cors.json | |
| gcs-cors: | |
| name: GCS CORS regression tests | |
| needs: [upload-to-gcs] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install test dependencies | |
| run: pip install pandas pyarrow pytest requests | |
| - name: Run GCS CORS tests | |
| run: pytest tests/test_gcs_cors.py -v --tb=short | |
| publish: | |
| needs: [dist] | |
| name: Publish to PyPI | |
| environment: pypi | |
| permissions: | |
| id-token: write | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: Packages | |
| path: dist | |
| - uses: pypa/gh-action-pypi-publish@release/v1 |