refresh.yml

name: refresh

# Sonda FTP DATASUS, arquiva delta em Parquet, emite provenance,
# sincroniza com S3, invalida CloudFront, publica GitHub Release
# (que dispara o webhook Zenodo → DOI).
#
# Cadência: segunda-feira 06:00 UTC. DATASUS publica SIA-PA com 45-90
# dias de lag após fim da competência; sonda semanal cobre bem.
#
# Secrets necessários (configurados uma vez no repo):
#   AWS_ROLE_ARN                — IAM role com acesso ao bucket + distribution
#   S3_BUCKET                   — nome do bucket S3 (sa-east-1)
#   CLOUDFRONT_DISTRIBUTION_ID  — opcional, para invalidation

on:
  schedule:
    - cron: '0 6 * * 1'
  workflow_dispatch:
    inputs:
      dryRun:
        description: 'Dry run — detect only, skip archive/upload'
        type: boolean
        default: false

permissions:
  contents: write
  id-token: write

jobs:
  detect:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      hasNew: ${{ steps.detect.outputs.hasNew }}
      pendingCount: ${{ steps.detect.outputs.pendingCount }}
      latestCompetencia: ${{ steps.detect.outputs.latestCompetencia }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 22
          cache: pnpm
      - run: pnpm install --frozen-lockfile
      - id: detect
        run: pnpm detect-new
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: pending
          path: state/pending.json
          retention-days: 7

  archive:
    needs: detect
    if: needs.detect.outputs.hasNew == 'true' && inputs.dryRun != true
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Generate release-bot token
        id: app-token
        uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
        with:
          app-id: ${{ secrets.RELEASE_APP_ID }}
          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}

      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          token: ${{ steps.app-token.outputs.token }}
      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 22
          cache: pnpm
      - run: pnpm install --frozen-lockfile

      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: pending
          path: state/

      - name: Archive SIA-PA delta (DBC → Parquet raw)
        run: pnpm archive-sia-pa

      - name: Emit provenance (SHA256 + schema + metadata)
        run: pnpm emit-provenance

      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: sa-east-1

      # Sobe primeiro os arquivos do delta (sem manifest.json — esse é
      # reconstruído logo abaixo a partir do estado real do bucket).
      - name: Upload delta para S3
        run: |
          aws s3 sync build/ "s3://${{ secrets.S3_BUCKET }}/" \
            --exclude '_archive-run.json' \
            --exclude '**/*.ndjson' \
            --exclude 'manifest.json' \
            --cache-control 'public, max-age=3600'

      # build-manifest reconstrói o manifest a partir do bucket — varrer
      # `build/` só veria o delta e regrediria o catálogo histórico.
      - name: Lista partições no bucket
        run: |
          aws s3 ls "s3://${{ secrets.S3_BUCKET }}/" --recursive > /tmp/s3-listing.txt
          wc -l /tmp/s3-listing.txt

      - name: Build manifest (a partir do bucket)
        run: pnpm build-manifest -- --s3-listing /tmp/s3-listing.txt

      # Guard contra regressão: se o novo manifest cobrir menos partições
      # do que o publicado, aborta sem subir. Bug histórico (run
      # 26020087919) publicou manifest com 12 partições sobrescrevendo o
      # de 5843 — versioning + esse guard impedem que se repita.
      - name: Verifica que o manifest novo não regride o publicado
        run: |
          REMOTE=$(aws s3 cp "s3://${{ secrets.S3_BUCKET }}/manifest.json" - 2>/dev/null || echo '{}')
          REMOTE_PARTS=$(echo "$REMOTE" | jq '[.datasets[].partitions] | add // 0')
          LOCAL_PARTS=$(jq '[.datasets[].partitions] | add // 0' build/manifest.json)
          echo "remote=$REMOTE_PARTS local=$LOCAL_PARTS"
          if [ "$LOCAL_PARTS" -lt "$REMOTE_PARTS" ]; then
            echo "::error::manifest novo ($LOCAL_PARTS partições) regride o publicado ($REMOTE_PARTS) — abortando upload"
            exit 1
          fi

      - name: Upload manifest.json
        run: |
          aws s3 cp build/manifest.json "s3://${{ secrets.S3_BUCKET }}/manifest.json" \
            --content-type 'application/json' \
            --cache-control 'public, max-age=3600'

      - name: Invalidate CloudFront
        env:
          DIST_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}
        if: ${{ env.DIST_ID != '' }}
        run: |
          aws cloudfront create-invalidation \
            --distribution-id "$DIST_ID" \
            --paths '/manifest.json' '/*/provenance/*'

      - name: Atualizar state (merge pending → dataset state)
        run: pnpm detect-new -- --mark-processed

      - name: Commit state atualizado
        run: |
          git config user.name 'precisa-saude-release-bot[bot]'
          git config user.email '${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com'
          git add state/
          if git diff --cached --quiet; then
            echo 'state inalterado — skip commit'
            exit 0
          fi
          git commit -m "chore(data): refresh state ${{ needs.detect.outputs.latestCompetencia }}"
          git pull --rebase origin main
          git push

      - name: Criar GitHub Release (dispara webhook Zenodo → DOI)
        uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2
        with:
          tag_name: dataset-${{ needs.detect.outputs.latestCompetencia }}
          name: Dataset ${{ needs.detect.outputs.latestCompetencia }}
          body: |
            Refresh automático: **${{ needs.detect.outputs.pendingCount }}** novas competências
            (mais recente: **${{ needs.detect.outputs.latestCompetencia }}**).

            Assets incluem `manifest.json`. Parquet e provenance por partição estão no CDN.
            Para validar byte-a-byte contra o FTP oficial, veja `docs/provenance.md`.

            DOI Zenodo emitido automaticamente via webhook — veja
            https://zenodo.org/search?q=Precisa-Saude/datasus-parquet.
          files: |
            build/manifest.json
          generate_release_notes: true