Check Nightly Build Status #756
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check Nightly Build Status | |
| on: | |
| workflow_call: | |
| inputs: | |
| event: | |
| type: string | |
| should_publish: | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| event: | |
| description: event triggered the run (`workflow_dispatch`) | |
| type: string | |
| default: 'workflow_dispatch' | |
| should_publish: | |
| description: true - to publish the report | |
| type: string | |
| default: 'false' | |
| permissions: | |
| contents: write | |
| issues: write | |
| concurrency: | |
| group: status-${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: duckdb/duckdb | |
| GH_ISSUE_REPO: ${{ github.repository }} | |
| jobs: | |
| get-run-info: | |
| name: Generate nightly build artifact json file | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| outputs: | |
| CURR_DATE: ${{ steps.curr-date.outputs.CURR_DATE }} | |
| matrix: ${{ steps.set-outputs.outputs.matrix }} | |
| run_id: ${{ steps.get_run_id.outputs.run_id }} | |
| previous_tag: ${{ steps.get_run_id.outputs.previous_tag }} | |
| previous_sha: ${{ steps.get_run_id.outputs.previous_sha }} | |
| steps: | |
| - id: curr-date | |
| run: echo "CURR_DATE=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT | |
| - name: Checkout repo with the script | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository_owner }}/duckdb-build-status | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install DuckDB for Python | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install duckdb | |
| - name: Create Tables and Generate inputs.json | |
| shell: bash | |
| id: get_run_id | |
| env: | |
| CURR_DATE: ${{ steps.curr-date.outputs.CURR_DATE }} | |
| run: | | |
| python scripts/create_tables_and_inputs.py --branch "${{ github.ref_name }}" --event "${{ inputs.event }}" | |
| # run_id=$(python scripts/create_tables_and_inputs.py --branch "${{ github.ref_name }}" --event "${{ inputs.event }}") | |
| # echo "run_id=$run_id" >> $GITHUB_OUTPUT | |
| - name: Upload DuckDB file | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.ref_name }}_run_info_tables.duckdb | |
| path: ${{ github.ref_name }}_run_info_tables.duckdb | |
| - name: Upload inputs_${{ github.ref_name }}.json | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: inputs_${{ github.ref_name }}.json | |
| path: inputs_${{ github.ref_name }}.json | |
| if-no-files-found: ignore | |
| - name: Read JSON and create matrix | |
| id: set-outputs | |
| run: | | |
| matrix=$(cat inputs_${{ github.ref_name }}.json | jq -c '.') | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| echo "***" | |
| cat inputs_${{ github.ref_name }}.json | |
| run-tests: | |
| name: ${{ matrix.config.branch }} - ${{ matrix.config.nightly_build }} ${{ matrix.config.duckdb_arch }} (${{ matrix.config.runs_on }}) | |
| needs: | |
| - get-run-info | |
| if: ${{ fromJson(needs.get-run-info.outputs.matrix) != '[]' }} | |
| uses: ./.github/workflows/VerifyBuild.yml | |
| strategy: | |
| matrix: | |
| config: ${{ fromJson(needs.get-run-info.outputs.matrix) }} | |
| with: | |
| runs_on: ${{ matrix.config.runs_on }} | |
| duckdb_arch: ${{ matrix.config.duckdb_arch }} | |
| duckdb_binary: ${{ matrix.config.duckdb_binary }} | |
| nightly_build: ${{ matrix.config.nightly_build }} | |
| branch: ${{ github.ref_name }} | |
| run_id: '0' | |
| report: | |
| name: Create complete report | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - get-run-info | |
| - run-tests | |
| env: | |
| AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository_owner }}/duckdb-build-status | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install DuckDB for Python | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install duckdb pandas tabulate requests | |
| - name: Download inputs_${{ github.ref_name }}.json | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: inputs_${{ github.ref_name }}.json | |
| path: . | |
| - name: Download extensions artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ${{ github.ref_name }}_ext* | |
| path: ${{ github.ref_name }}_failed_ext | |
| - name: Download duckdb file | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ github.ref_name }}_run_info_tables.duckdb | |
| path: ${{ github.ref_name }}_tables | |
| - name: Generate report | |
| shell: bash | |
| env: | |
| CURR_DATE: ${{ needs.get-run-info.outputs.CURR_DATE }} | |
| run: | | |
| python scripts/create_build_report.py --branch ${{ github.ref_name }} | |
| - name: Upload REPORT | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: REPORT-${{ github.ref_name }} | |
| path: ${{ needs.get-run-info.outputs.CURR_DATE }}-${{ github.ref_name }}.md | |
| - name: Generate digests | |
| shell: bash | |
| run: | | |
| # get a run_id | |
| run_id=${{ needs.get-run-info.outputs.run_id }} | |
| # download nightly release files from GH and generate digests | |
| aws s3 ls --recursive s3://duckdb-staging/${{ needs.get-run-info.outputs.previous_sha }}/${{ needs.get-run-info.outputs.previous_tag }}/${{ env.GH_REPO }}/github_release | awk '{print $4}' > release-assets.txt | |
| cat release-assets.txt | |
| for file_path in $(cat release-assets.txt); do | |
| aws s3 cp s3://duckdb-staging/$file_path .; | |
| file=$(basename $file_path) | |
| shasum -a 256 $file >> ${{ needs.get-run-info.outputs.CURR_DATE }}_checksum_${{ github.ref_name }}.txt; | |
| rm $file; | |
| done | |
| - name: Upload checksum file | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: CHECKSUM_${{ github.ref_name }} | |
| path: ${{ needs.get-run-info.outputs.CURR_DATE }}_checksum_${{ github.ref_name }}.txt | |
| push-reports: | |
| name: Push Reports | |
| if: ${{ inputs.should_publish == 'true' }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-run-info | |
| - report | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository_owner }}/duckdb-build-status | |
| - name: Download Reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: REPORT* | |
| path: . | |
| - name: Download checksum file | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: CHECKSUM* | |
| path: . | |
| - name: Upload REPORT to the docs directory | |
| shell: bash | |
| env: | |
| CI_COMMIT_MESSAGE: "Add ${{ needs.get-run-info.outputs.CURR_DATE }} Reports" | |
| CI_COMMIT_AUTHOR: "hmeriann" | |
| CI_COMMIT_EMAIL: "zuleykha.pavlichenkova@gmail.com" | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| git config --global user.name ${{ env.CI_COMMIT_AUTHOR }} | |
| git config --global user.email ${{ env.CI_COMMIT_EMAIL }} | |
| git pull --rebase origin ${{ github.ref_name }} | |
| mkdir -p docs/ | |
| mkdir -p docs/${{ github.ref_name }} | |
| mkdir -p docs/${{ github.ref_name }}/checksum | |
| mv REPORT*/${{ needs.get-run-info.outputs.CURR_DATE }}-${{ github.ref_name }}.md docs/${{ github.ref_name }}/ | |
| mv CHECKSUM*/${{ needs.get-run-info.outputs.CURR_DATE }}_checksum_${{ github.ref_name }}.txt docs/${{ github.ref_name }}/checksum | |
| ls -lah docs/${{ github.ref_name }} | |
| ls -lah docs/${{ github.ref_name }}/checksum | |
| git add docs/${{ github.ref_name }}/${{ needs.get-run-info.outputs.CURR_DATE }}-${{ github.ref_name }}.md | |
| git add docs/${{ github.ref_name }}/checksum/${{ needs.get-run-info.outputs.CURR_DATE }}_checksum_${{ github.ref_name }}.txt | |
| git commit -m "${{ env.CI_COMMIT_MESSAGE }}" | |
| git push origin ${{ github.ref_name }} | |
| - name: Checkout main | |
| if: ${{ github.ref_name != 'main'}} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository_owner }}/duckdb-build-status | |
| ref: main | |
| - name: Push new report to main | |
| if: ${{ github.ref_name != 'main'}} | |
| shell: bash | |
| run: | | |
| report_file="docs/${{ github.ref_name }}/${{ needs.get-run-info.outputs.CURR_DATE }}-${{ github.ref_name }}.md" | |
| checksum_file="docs/${{ github.ref_name }}/checksum/${{ needs.get-run-info.outputs.CURR_DATE }}_checksum_${{ github.ref_name }}.txt" | |
| git fetch origin ${{ github.ref_name }}:${{ github.ref_name }} | |
| git checkout ${{ github.ref_name }} -- $report_file | |
| git checkout ${{ github.ref_name }} -- $checksum_file | |
| git add $report_file $checksum_file | |
| git commit -m "Adding $report_file and $checksum_file to main" | |
| git push -f origin main |