Check Nightly Build Status #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check Nightly Build Status | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 9 * * 1-5' # runs at 10am CET on working days | |
| permissions: | |
| contents: write | |
| issues: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: duckdb/duckdb | |
| GH_ISSUE_REPO: ${{ github.repository }} | |
| jobs: | |
| # check-wget: | |
| # name: Check in advance | |
| # runs-on: macos-13 | |
| # steps: | |
| # - name: Checkout repo with the scripts | |
| # uses: actions/checkout@v4 | |
| # - shell: bash | |
| # run: chmod +x scripts/check_ext.sh && ./scripts/check_ext.sh | |
| # - uses: actions/upload-artifact@v4 | |
| # with: | |
| # path: log.csv | |
| get-run-info: | |
| name: Generate nightly build artifact json file | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-outputs.outputs.matrix }} | |
| CURR_DATE: ${{ steps.curr-date.outputs.CURR_DATE }} | |
| steps: | |
| - id: curr-date | |
| run: echo "CURR_DATE=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT | |
| - name: Checkout repo with the script | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install DuckDB for Python | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install duckdb | |
| - name: Create Tables and Generate inputs.json | |
| shell: bash | |
| env: | |
| CURR_DATE: ${{ steps.curr-date.outputs.CURR_DATE }} | |
| run: | | |
| python scripts/create_tables_and_inputs.py | |
| - name: Upload DuckDB file | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: run_info_tables.duckdb | |
| path: run_info_tables.duckdb | |
| - name: Read JSON and create matrix | |
| id: set-outputs | |
| run: | | |
| matrix=$(cat inputs.json | jq -c '.') | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| echo "***" | |
| cat inputs.json | |
| - name: Upload inputs.json | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: inputs.json | |
| path: inputs.json | |
| if-no-files-found: ignore | |
| run-tests: | |
| name: ${{ matrix.inputs.nightly_build }} - ${{ matrix.inputs.duckdb_arch }} (${{ matrix.inputs.runs_on }}) | |
| needs: get-run-info | |
| if: ${{ needs.get-run-info.outputs.matrix != '[]' }} | |
| strategy: | |
| matrix: | |
| inputs: ${{ fromJson(needs.get-run-info.outputs.matrix) }} | |
| continue-on-error: true | |
| runs-on: ${{ matrix.inputs.runs_on }} | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: pip install duckdb docker | |
| - name: Checkout repo with the scripts | |
| uses: actions/checkout@v4 | |
| - name: Checkout repo with extensions config file | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.GH_REPO }} | |
| sparse-checkout: | | |
| .github/config | |
| path: ext | |
| - name: Verify ${{ matrix.inputs.nightly_build }} build version and Test extensions | |
| id: verify-build | |
| shell: bash | |
| env: | |
| CURR_DATE: ${{ needs.get-run-info.outputs.CURR_DATE }} | |
| run: | | |
| uname -m | |
| if [[ ${{ matrix.inputs.nightly_build }} != 'python' ]]; then | |
| echo "Downloading duckdb-binaries-${{ matrix.inputs.duckdb_binary }} artifact..." | |
| if gh run download ${{ matrix.inputs.run_id }} --repo ${{ env.GH_REPO }} -n duckdb-binaries-${{ matrix.inputs.duckdb_binary }}; then | |
| echo "Artifact duckdb-binaries-${{ matrix.inputs.duckdb_binary }} is successfuly downloaded." | |
| if [[ ${{ matrix.inputs.nightly_build }} == 'osx' ]]; then | |
| unzip duckdb_cli-*.zip -d duckdb_path | |
| else | |
| unzip duckdb_cli-${{ matrix.inputs.duckdb_binary }}.zip -d duckdb_path | |
| fi | |
| fi | |
| fi | |
| echo "Verifying version and testing extensions..." | |
| python scripts/verify_and_test.py \ | |
| --nightly_build ${{ matrix.inputs.nightly_build }} \ | |
| --architecture ${{ matrix.inputs.duckdb_arch }} \ | |
| --run_id ${{ matrix.inputs.run_id }} \ | |
| --runs_on ${{ matrix.inputs.runs_on }} | |
| - name: Upload actions for extensions | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ext_${{ matrix.inputs.nightly_build }}_${{ matrix.inputs.duckdb_arch }} | |
| path: | | |
| list_failed_ext_${{ matrix.inputs.nightly_build }}_${{ matrix.inputs.duckdb_arch }}.csv | |
| non_matching_sha_${{ matrix.inputs.nightly_build }}_${{ matrix.inputs.duckdb_arch }}.txt | |
| if-no-files-found: ignore | |
| report: | |
| name: Create complete report | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - get-run-info | |
| - run-tests | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install DuckDB for Python | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install duckdb pandas tabulate requests | |
| - name: Download inputs.json | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: inputs.json | |
| path: . | |
| - name: Download extensions artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ext_* | |
| path: failed_ext | |
| - name: Download duckdb file | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: run_info_tables.duckdb | |
| path: tables | |
| - name: Generate report | |
| shell: bash | |
| env: | |
| CURR_DATE: ${{ needs.get-run-info.outputs.CURR_DATE }} | |
| run: | | |
| python scripts/create_build_report.py | |
| - name: Upload REPORT | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: REPORT | |
| path: ${{ needs.get-run-info.outputs.CURR_DATE }}-report.md | |
| - name: Upload REPORT to the _posts directory | |
| shell: bash | |
| env: | |
| CI_COMMIT_MESSAGE: "Add ${{ needs.get-run-info.outputs.CURR_DATE }}-report.md" | |
| CI_COMMIT_AUTHOR: "hmeriann" | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| git config --global user.name ${{ env.CI_COMMIT_AUTHOR }} | |
| git config --global user.email "zuleykha.pavlichenkova@gmail.com" | |
| git config pull.rebase false | |
| git pull origin gh-pages --allow-unrelated-histories | |
| mv ${{ needs.get-run-info.outputs.CURR_DATE }}-report.md _posts | |
| ls -lah _posts | |
| git add _posts/${{ needs.get-run-info.outputs.CURR_DATE }}-report.md | |
| git commit -m "${{ env.CI_COMMIT_MESSAGE }}" | |
| git push origin main:gh-pages |