upload-to-s3.yml

name: Upload to S3

on:
  workflow_dispatch:
    inputs:
      release_tag:
        description: "Tag of the draft release holding the file to upload"
        required: true
        type: string
      s3_destination:
        description: "S3 path within the stdpopsim bucket (e.g. annotations/HomSap/file.tar.gz)"
        required: true
        type: string
      expected_sha256:
        description: "Expected SHA256 checksum of the file"
        required: true
        type: string
      species_id:
        description: "Species ID (e.g. HomSap)"
        required: true
        type: string
      resource_type:
        description: "Resource type: genetic_map or annotation"
        required: true
        type: string
      resource_id:
        description: "Resource ID as defined in the catalog (e.g. HapMapII_GRCh38)"
        required: true
        type: string
      dry_run:
        description: "If true, run all validation but skip the actual S3 upload"
        required: false
        type: boolean
        default: false

concurrency:
  group: s3-upload
  cancel-in-progress: false

jobs:
  upload:
    runs-on: ubuntu-latest
    environment: s3-upload
    permissions:
      contents: write  # needed to download release assets and delete releases
    steps:
      - name: Validate inputs
        run: |
          DEST="${{ inputs.s3_destination }}"
          TYPE="${{ inputs.resource_type }}"

          # Resource type must be genetic_map or annotation
          if [[ "$TYPE" != "genetic_map" && "$TYPE" != "annotation" ]]; then
            echo "ERROR: resource_type must be 'genetic_map' or 'annotation', got: $TYPE"
            exit 1
          fi

          # S3 destination must start with genetic_maps/ or annotations/
          if [[ ! "$DEST" =~ ^(genetic_maps|annotations)/ ]]; then
            echo "ERROR: S3 destination must start with 'genetic_maps/' or 'annotations/'"
            exit 1
          fi

          # S3 destination must end with .tar.gz or .tgz
          if [[ ! "$DEST" =~ \.(tar\.gz|tgz)$ ]]; then
            echo "ERROR: S3 destination must end with .tar.gz or .tgz"
            exit 1
          fi

          echo "Input validation passed."
          echo "  Species:     ${{ inputs.species_id }}"
          echo "  Type:        ${{ inputs.resource_type }}"
          echo "  Resource:    ${{ inputs.resource_id }}"
          echo "  S3 dest:     $DEST"

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install stdpopsim
        run: |
          pip install -e .

      - name: Validate against catalog
        run: |
          python3 << 'PYEOF'
          import stdpopsim
          import sys
          import re

          species_id = "${{ inputs.species_id }}"
          resource_type = "${{ inputs.resource_type }}"
          resource_id = "${{ inputs.resource_id }}"
          s3_dest = "${{ inputs.s3_destination }}"
          expected_sha256 = "${{ inputs.expected_sha256 }}"

          # Verify species exists
          try:
              species = stdpopsim.get_species(species_id)
          except (ValueError, KeyError):
              available = [s.id for s in stdpopsim.all_species()]
              print(f"ERROR: Species '{species_id}' not found in catalog.")
              print(f"Available species: {available}")
              sys.exit(1)

          # Verify resource exists and extract expected URL/SHA256
          if resource_type == "genetic_map":
              resources = {gm.id: gm for gm in species.genetic_maps}
              if resource_id not in resources:
                  print(f"ERROR: Genetic map '{resource_id}' not found for {species_id}.")
                  print(f"Available genetic maps: {list(resources.keys())}")
                  sys.exit(1)
              resource = resources[resource_id]
              catalog_url = resource.url
              catalog_sha256 = resource.sha256

          elif resource_type == "annotation":
              resources = {a.id: a for a in species.annotations}
              if resource_id not in resources:
                  print(f"ERROR: Annotation '{resource_id}' not found for {species_id}.")
                  print(f"Available annotations: {list(resources.keys())}")
                  sys.exit(1)
              resource = resources[resource_id]
              catalog_url = resource.intervals_url
              catalog_sha256 = resource.intervals_sha256

          # Verify the S3 destination matches what the catalog expects
          m = re.match(r"https://stdpopsim\.s3[.-]us-west-2\.amazonaws\.com/(.*)", catalog_url)
          if not m:
              print(f"ERROR: Could not parse S3 URL from catalog: {catalog_url}")
              sys.exit(1)

          catalog_s3_dest = m.group(1)
          if catalog_s3_dest != s3_dest:
              print(f"ERROR: S3 destination mismatch!")
              print(f"  Provided:        {s3_dest}")
              print(f"  Catalog expects: {catalog_s3_dest}")
              sys.exit(1)

          # Verify SHA256 matches what the catalog expects
          if catalog_sha256 != expected_sha256:
              print(f"ERROR: SHA256 mismatch with catalog!")
              print(f"  Provided:        {expected_sha256}")
              print(f"  Catalog expects: {catalog_sha256}")
              sys.exit(1)

          print("Catalog validation passed!")
          print(f"  Species:  {species_id} ({species.name})")
          print(f"  Type:     {resource_type}")
          print(f"  Resource: {resource_id}")
          print(f"  URL:      {catalog_url}")
          print(f"  SHA256:   {catalog_sha256}")
          PYEOF

      - name: Download asset from draft release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          echo "Downloading asset from release: ${{ inputs.release_tag }}"
          gh release download "${{ inputs.release_tag }}" \
            --repo "${{ github.repository }}" \
            --dir ./download

          # Expect exactly one file
          FILE_COUNT=$(find ./download -type f | wc -l)
          if [ "$FILE_COUNT" -ne 1 ]; then
            echo "ERROR: Expected exactly 1 asset, found $FILE_COUNT"
            exit 1
          fi

          DOWNLOADED_FILE=$(find ./download -type f)
          echo "Downloaded: $DOWNLOADED_FILE"
          echo "DOWNLOADED_FILE=$DOWNLOADED_FILE" >> "$GITHUB_ENV"

      - name: Verify SHA256
        run: |
          ACTUAL_SHA256=$(sha256sum "$DOWNLOADED_FILE" | awk '{print $1}')
          EXPECTED="${{ inputs.expected_sha256 }}"

          echo "Expected SHA256: $EXPECTED"
          echo "Actual SHA256:   $ACTUAL_SHA256"

          if [ "$ACTUAL_SHA256" != "$EXPECTED" ]; then
            echo "ERROR: SHA256 mismatch!"
            exit 1
          fi

          echo "SHA256 verified."

      - name: Check if S3 object already exists
        if: ${{ inputs.dry_run != true }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2
        run: |
          DEST="${{ inputs.s3_destination }}"
          if aws s3api head-object --bucket stdpopsim --key "$DEST" 2>/dev/null; then
            echo "ERROR: s3://stdpopsim/$DEST already exists. Refusing to overwrite."
            echo "If you need to replace this file, delete it from S3 first."
            exit 1
          fi
          echo "Confirmed: s3://stdpopsim/$DEST does not exist yet."

      - name: Upload to S3
        if: ${{ inputs.dry_run != true }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2
        run: |
          DEST="${{ inputs.s3_destination }}"
          echo "Uploading to s3://stdpopsim/$DEST"
          aws s3 cp "$DOWNLOADED_FILE" "s3://stdpopsim/$DEST"
          echo ""
          echo "Upload complete."
          echo "S3 URL: https://stdpopsim.s3-us-west-2.amazonaws.com/$DEST"
          echo "SHA256: ${{ inputs.expected_sha256 }}"

      - name: Dry run summary
        if: ${{ inputs.dry_run == true }}
        run: |
          echo "=== DRY RUN COMPLETE ==="
          echo "All validation passed. Skipped S3 upload."
          echo ""
          echo "  Species:  ${{ inputs.species_id }}"
          echo "  Type:     ${{ inputs.resource_type }}"
          echo "  Resource: ${{ inputs.resource_id }}"
          echo "  S3 dest:  s3://stdpopsim/${{ inputs.s3_destination }}"
          echo "  SHA256:   ${{ inputs.expected_sha256 }}"
          echo ""
          echo "To perform the actual upload, re-run without --dry-run."

      - name: Clean up draft release
        if: always()
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          TAG="${{ inputs.release_tag }}"
          echo "Cleaning up draft release: $TAG"

          # Delete the release
          gh release delete "$TAG" \
            --repo "${{ github.repository }}" \
            --yes \
            --cleanup-tag 2>/dev/null || echo "Warning: could not delete release $TAG"

          echo "Cleanup complete."