refresh #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: refresh | |
| # Sonda FTP DATASUS, arquiva delta em Parquet, emite provenance, | |
| # sincroniza com S3, invalida CloudFront, publica GitHub Release | |
| # (que dispara o webhook Zenodo → DOI). | |
| # | |
| # Cadência: segunda-feira 06:00 UTC. DATASUS publica SIA-PA com 45-90 | |
| # dias de lag após fim da competência; sonda semanal cobre bem. | |
| # | |
| # Secrets necessários (configurados uma vez no repo): | |
| # AWS_ROLE_ARN — IAM role com acesso ao bucket + distribution | |
| # S3_BUCKET — nome do bucket S3 (sa-east-1) | |
| # CLOUDFRONT_DISTRIBUTION_ID — opcional, para invalidation | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' | |
| workflow_dispatch: | |
| inputs: | |
| dryRun: | |
| description: 'Dry run — detect only, skip archive/upload' | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| id-token: write | |
| jobs: | |
| detect: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| hasNew: ${{ steps.detect.outputs.hasNew }} | |
| pendingCount: ${{ steps.detect.outputs.pendingCount }} | |
| latestCompetencia: ${{ steps.detect.outputs.latestCompetencia }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - id: detect | |
| run: pnpm detect-new | |
| - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: pending | |
| path: state/pending.json | |
| retention-days: 7 | |
| archive: | |
| needs: detect | |
| if: needs.detect.outputs.hasNew == 'true' && inputs.dryRun != true | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: pending | |
| path: state/ | |
| - name: Archive SIA-PA delta (DBC → Parquet raw) | |
| run: pnpm archive-sia-pa | |
| - name: Emit provenance (SHA256 + schema + metadata) | |
| run: pnpm emit-provenance | |
| - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: sa-east-1 | |
| # Sobe primeiro os arquivos do delta (sem manifest.json — esse é | |
| # reconstruído logo abaixo a partir do estado real do bucket). | |
| - name: Upload delta para S3 | |
| run: | | |
| aws s3 sync build/ "s3://${{ secrets.S3_BUCKET }}/" \ | |
| --exclude '_archive-run.json' \ | |
| --exclude '**/*.ndjson' \ | |
| --exclude 'manifest.json' \ | |
| --cache-control 'public, max-age=3600' | |
| # build-manifest reconstrói o manifest a partir do bucket — varrer | |
| # `build/` só veria o delta e regrediria o catálogo histórico. | |
| - name: Lista partições no bucket | |
| run: | | |
| aws s3 ls "s3://${{ secrets.S3_BUCKET }}/" --recursive > /tmp/s3-listing.txt | |
| wc -l /tmp/s3-listing.txt | |
| - name: Build manifest (a partir do bucket) | |
| run: pnpm build-manifest -- --s3-listing /tmp/s3-listing.txt | |
| # Guard contra regressão: se o novo manifest cobrir menos partições | |
| # do que o publicado, aborta sem subir. Bug histórico (run | |
| # 26020087919) publicou manifest com 12 partições sobrescrevendo o | |
| # de 5843 — versioning + esse guard impedem que se repita. | |
| - name: Verifica que o manifest novo não regride o publicado | |
| run: | | |
| REMOTE=$(aws s3 cp "s3://${{ secrets.S3_BUCKET }}/manifest.json" - 2>/dev/null || echo '{}') | |
| REMOTE_PARTS=$(echo "$REMOTE" | jq '[.datasets[].partitions] | add // 0') | |
| LOCAL_PARTS=$(jq '[.datasets[].partitions] | add // 0' build/manifest.json) | |
| echo "remote=$REMOTE_PARTS local=$LOCAL_PARTS" | |
| if [ "$LOCAL_PARTS" -lt "$REMOTE_PARTS" ]; then | |
| echo "::error::manifest novo ($LOCAL_PARTS partições) regride o publicado ($REMOTE_PARTS) — abortando upload" | |
| exit 1 | |
| fi | |
| - name: Upload manifest.json | |
| run: | | |
| aws s3 cp build/manifest.json "s3://${{ secrets.S3_BUCKET }}/manifest.json" \ | |
| --content-type 'application/json' \ | |
| --cache-control 'public, max-age=3600' | |
| - name: Invalidate CloudFront | |
| env: | |
| DIST_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} | |
| if: ${{ env.DIST_ID != '' }} | |
| run: | | |
| aws cloudfront create-invalidation \ | |
| --distribution-id "$DIST_ID" \ | |
| --paths '/manifest.json' '/*/provenance/*' | |
| - name: Atualizar state (merge pending → dataset state) | |
| run: pnpm detect-new -- --mark-processed | |
| - name: Commit state atualizado | |
| run: | | |
| git config user.name 'datasus-parquet-bot' | |
| git config user.email 'noreply@precisa-saude.com.br' | |
| git add state/ | |
| if git diff --cached --quiet; then | |
| echo 'state inalterado — skip commit' | |
| exit 0 | |
| fi | |
| git commit -m "chore(data): refresh state ${{ needs.detect.outputs.latestCompetencia }}" | |
| git push | |
| - name: Criar GitHub Release (dispara webhook Zenodo → DOI) | |
| uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 | |
| with: | |
| tag_name: dataset-${{ needs.detect.outputs.latestCompetencia }} | |
| name: Dataset ${{ needs.detect.outputs.latestCompetencia }} | |
| body: | | |
| Refresh automático: **${{ needs.detect.outputs.pendingCount }}** novas competências | |
| (mais recente: **${{ needs.detect.outputs.latestCompetencia }}**). | |
| Assets incluem `manifest.json`, Parquet por mês e provenance por partição. | |
| Para validar byte-a-byte contra o FTP oficial, veja `docs/provenance.md`. | |
| DOI Zenodo emitido automaticamente via webhook — veja | |
| https://zenodo.org/search?q=Precisa-Saude/datasus-parquet. | |
| files: | | |
| build/manifest.json | |
| build/**/provenance/**/*.json | |
| generate_release_notes: true |