cd.yml

name: CD

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main
  release:
    types:
      - published

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  FORCE_COLOR: 3

jobs:
  generate-indices:
    name: Generate index artifacts
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install db-dtypes duckdb google-cloud-bigquery google-cloud-bigquery-storage pandas pyarrow requests

      - name: Authorize Google Cloud
        uses: google-github-actions/auth@v3
        with:
          credentials_json: "${{ secrets.SERVICE_ACCOUNT_KEY }}"
          create_credentials_file: true
          export_environment_variables: true

      - name: Execute SQL Query and Generate Parquet Files
        run: |
          python scripts/python/idc_index_data_manager.py \
            --generate-parquet \
            --output-dir release_artifacts
        env:
          GCP_PROJECT: ${{ env.GCP_PROJECT }}

      - name: Generate GDC patient check parquet
        run: |
          python scripts/gdc/gdc_parquet_generator.py
          mv gdc_idc_mapping.parquet release_artifacts/
        env:
          GCP_PROJECT: ${{ env.GCP_PROJECT }}

      - name: Generate TCIA subset parquet
        run: |
          python scripts/tcia/tcia_parquet_generator.py \
            release_artifacts/idc_index.parquet \
            -o release_artifacts/tcia_idc_subset.parquet

      - name: Report generated index sizes
        run:
          python scripts/python/report_index_sizes.py release_artifacts | tee -a
          $GITHUB_STEP_SUMMARY

      - name: Upload index artifacts
        uses: actions/upload-artifact@v7
        with:
          name: idc-index-artifacts
          path: release_artifacts/
          retention-days: 14
          if-no-files-found: error

  dist:
    name: Distribution build
    runs-on: ubuntu-latest
    needs: [generate-indices]

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download pre-generated index artifacts
        uses: actions/download-artifact@v8
        with:
          name: idc-index-artifacts
          path: release_artifacts/

      - name: Move artifacts to project root for packaging
        run: |
          mv release_artifacts/*.parquet ./src/idc_index_data || true
          mv release_artifacts/*.json ./src/idc_index_data || true
          mv release_artifacts/*.sql ./src/idc_index_data || true
          mv release_artifacts/*.zip ./src/idc_index_data || true
          ls -la ./src/idc_index_data

      - name: Build distribution package
        run: |
          # Unset GCP_PROJECT to prevent build hook from regenerating files
          # Files already exist from the generate-indices job
          unset GCP_PROJECT
          python -m pip install --upgrade build
          python -m build
        env:
          # Explicitly unset GCP_PROJECT to skip data generation in build hook
          GCP_PROJECT: ""

      - name: Upload distribution artifacts
        uses: actions/upload-artifact@v7
        with:
          name: Packages
          path: dist/
          retention-days: 14

      - name: Inspect package contents
        run: |
          python -m pip install --upgrade check-wheel-contents
          ls -lh dist/
          check-wheel-contents dist/*.whl

  attach-to-release:
    name: Attach artifacts to release
    needs: [generate-indices]
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'published'

    steps:
      - name: Download index artifacts
        uses: actions/download-artifact@v8
        with:
          name: idc-index-artifacts
          path: release_artifacts/

      - name: Attach artifacts to release
        uses: ncipollo/release-action@v1
        with:
          artifacts: "release_artifacts/*.parquet,release_artifacts/*.json,release_artifacts/*.sql"
          allowUpdates: true
          omitBodyDuringUpdate: true

  upload-to-gcs:
    name: Upload artifacts to GCS
    needs: [generate-indices]
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'published'

    steps:
      - name: Download index artifacts
        uses: actions/download-artifact@v8
        with:
          name: idc-index-artifacts
          path: release_artifacts/

      - name: Authorize Google Cloud
        uses: google-github-actions/auth@v3
        with:
          credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"

      - name: Upload artifacts to GCS bucket
        uses: google-github-actions/upload-cloud-storage@v3
        with:
          path: release_artifacts
          destination:
            idc-index-data-artifacts/${{ github.event.release.tag_name }}

      - name: Update current/ folder with latest release
        run: |
          gcloud storage rm -r gs://idc-index-data-artifacts/current/ || true
          gcloud storage cp -r \
            gs://idc-index-data-artifacts/${{ github.event.release.tag_name }}/* \
            gs://idc-index-data-artifacts/current/

      - name: Ensure bucket is publicly readable
        run: |
          gcloud storage buckets add-iam-policy-binding gs://idc-index-data-artifacts \
            --member=allUsers --role=roles/storage.objectViewer

      - name: Ensure CORS policy is set
        run: |
          cat > /tmp/cors.json << 'CORS'
          [
            {
              "origin": [
                "https://localhost.gdc.cancer.gov:3010",
                "https://portal.gdc.cancer.gov"
              ],
              "method": ["GET", "HEAD"],
              "responseHeader": ["Content-Type", "Content-Length", "Content-Range"],
              "maxAgeSeconds": 3600
            }
          ]
          CORS
          gcloud storage buckets update gs://idc-index-data-artifacts --cors-file=/tmp/cors.json

  publish:
    needs: [dist]
    name: Publish to PyPI
    environment: pypi
    permissions:
      id-token: write
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'published'

    steps:
      - uses: actions/download-artifact@v8
        with:
          name: Packages
          path: dist

      - uses: pypa/gh-action-pypi-publish@release/v1