Generate and Commit FNL Model Updates Changelogs #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate and Commit FNL Model Updates Changelogs | |
| permissions: | |
| contents: read | |
| actions: write | |
| on: | |
| workflow_run: | |
| workflows: ["Check New MDFs"] | |
| types: [completed] | |
| workflow_dispatch: # update an mdb with a certain model version | |
| inputs: | |
| models_spec_path: | |
| description: "Path to YAML files with models & refs their MDFs." | |
| required: false | |
| default: "config/mdb_models.yml" | |
| model: | |
| description: "Specific model handle to process" | |
| required: false | |
| default: "" | |
| version: | |
| description: "Specific version to process" | |
| required: false | |
| default: "" | |
| mdf_files: | |
| description: "Path or URL to specific MDF files to process" | |
| required: false | |
| default: "" | |
| datahub_only: | |
| description: "Only include datahub models" | |
| required: false | |
| type: boolean | |
| default: false | |
| mdb_id: | |
| description: "ID of MDB to update" | |
| required: false | |
| type: string | |
| default: "fnl-mdb-dev" | |
| log_level: | |
| description: "Log level" | |
| required: false | |
| type: string | |
| default: "info" | |
| dry_run: | |
| description: "Dry run flag" | |
| required: false | |
| type: boolean | |
| default: false | |
| no_commit: | |
| description: "Don't commit changes to GitHub." | |
| required: false | |
| type: boolean | |
| default: false | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| mdb_models_changed: ${{ steps.filter.outputs.mdb_models }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| mdb_models: | |
| - 'config/mdb_models.yml' | |
| determine-models: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.mdb_models_changed == 'true' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| affected_models: ${{ steps.set-matrix.outputs.model_matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Install Python | |
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c | |
| with: | |
| python-version-file: ".python-version" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 | |
| with: | |
| version: "0.5.10" | |
| - name: Install dependencies and project | |
| run: | | |
| uv pip install --system -e . | |
| uv sync --all-extras --dev | |
| - name: Generate Model Version Matrix | |
| id: set-matrix | |
| env: | |
| PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} | |
| PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }} | |
| run: | | |
| set -xeuo pipefail | |
| trap 'echo "Error in $BASH_COMMAND at line $LINENO (exit code $?)"' ERR | |
| # Use provided MDB ID or default to fnl-mdb-dev | |
| MDB_ID="${{ github.event.inputs.mdb_id || 'fnl-mdb-dev' }}" | |
| # If specific model/version/mdfs provided, use them | |
| if [ "${{ github.event.inputs.model }}" != "" ] && [ "${{ github.event.inputs.version }}" != "" ]; then | |
| MODEL_INFO="{\"model\": \"${{ github.event.inputs.model }}\", \"version\": \"${{ github.event.inputs.version }}\", \"mdf_files\": [${{ github.event.inputs.mdf_files || '[]' }}]}" | |
| echo "{\"include\": [$MODEL_INFO]}" > matrix.json | |
| else | |
| # Generate matrix for FNL database (focuses on lower/prerelease models) | |
| FNL_PARAMS=$(jq -n \ | |
| --arg mdb_id "$MDB_ID" \ | |
| --arg model_specs_yaml "${{ github.event.inputs.models_spec_path || 'config/mdb_models.yml' }}" \ | |
| --argjson datahub_only "${{ github.event.inputs.datahub_only || false }}" \ | |
| '{ | |
| "mdb_id": $mdb_id, | |
| "model_specs_yaml": $model_specs_yaml, | |
| "datahub_only": $datahub_only | |
| }') | |
| echo "Running Prefect deployment for FNL with params: $FNL_PARAMS" | |
| watch_out=$( | |
| prefect deployment run generate-model-version-matrix/generate-model-version-matrix \ | |
| --params "$FNL_PARAMS" \ | |
| --watch \ | |
| --watch-interval 30 2>&1 | tee watch.log | |
| ) | |
| echo "watch_out=$watch_out" | |
| RUN_ID=$( | |
| grep -oP "(?<=UUID:\s)[0-9a-fA-F-]+" watch.log \ | |
| | head -n1 | |
| ) | |
| echo "FNL run ID: $RUN_ID" | |
| echo "Waiting for logs to be available..." | |
| sleep 10 | |
| LOG_FILE="${RUN_ID}_run.log" | |
| prefect flow-run logs "$RUN_ID" > "$LOG_FILE" | |
| echo "Checking for MATRIX_JSON in FNL logs:" | |
| grep -a "MATRIX_JSON" "$LOG_FILE" || echo "No MATRIX_JSON found in FNL logs" | |
| MATRIX_OUTPUT=$(grep -a -m1 "MATRIX_JSON:" "$LOG_FILE" \ | |
| | grep -oP 'MATRIX_JSON:\K\{.*\}' \ | |
| || echo "") | |
| if [ -n "$MATRIX_OUTPUT" ]; then | |
| echo "$MATRIX_OUTPUT" > matrix.json | |
| echo "FNL matrix extracted successfully" | |
| else | |
| echo "Warning: Could not extract matrix from FNL deployment output" | |
| echo "{\"include\": []}" > matrix.json | |
| fi | |
| fi | |
| # validate json | |
| if ! jq . matrix.json > /dev/null 2>&1; then | |
| echo "Error: Invalid JSON in matrix.json" | |
| exit 1 | |
| fi | |
| MATRIX=$(cat matrix.json) | |
| MATRIX_COMPACT=$(echo "$MATRIX" | jq -c .) | |
| echo "model_matrix=$MATRIX_COMPACT" >> $GITHUB_OUTPUT | |
| echo "FNL Matrix: $MATRIX" | |
| generate-changelogs: | |
| needs: determine-models | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.determine-models.outputs.affected_models) }} | |
| fail-fast: false | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Install Python | |
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c | |
| with: | |
| python-version-file: ".python-version" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 | |
| with: | |
| version: "0.5.10" | |
| - name: Install dependencies and project | |
| run: | | |
| uv pip install --system -e . | |
| uv sync --all-extras --dev | |
| - name: Make Model Changelog | |
| run: | | |
| set -e | |
| MDF_FILES_ARRAY=() | |
| if [ -n "${{ toJson(matrix.mdf_files) }}" ] && [ "${{ toJson(matrix.mdf_files) }}" != "null" ] && [ "${{ toJson(matrix.mdf_files) }}" != "[]" ]; then | |
| for file in $(echo '${{ toJson(matrix.mdf_files) }}' | jq -r '.[]'); do | |
| MDF_FILES_ARRAY+=("--mdf_files" "$file") | |
| done | |
| fi | |
| mkdir -p data/output/model_changelogs/${{ matrix.model }}/ | |
| make_model_changelog \ | |
| --model_handle "${{ matrix.model }}" \ | |
| --model_version "${{ matrix.version }}" \ | |
| "${MDF_FILES_ARRAY[@]}" \ | |
| --output_file_path "data/output/model_changelogs/${{ matrix.model }}/${{ matrix.model }}_changelog_${{ matrix.version }}.xml" \ | |
| --author "GitHub Actions" \ | |
| --_commit "${{ github.sha }}" \ | |
| --latest_version ${{ fromJson('true') }} \ | |
| --add_rollback ${{ fromJson('false') }} | |
| - name: Get Model PVs and Synonyms | |
| run: | | |
| set -e | |
| MDF_FILES_ARRAY=() | |
| if [ -n "${{ toJson(matrix.mdf_files) }}" ] && [ "${{ toJson(matrix.mdf_files) }}" != "null" ] && [ "${{ toJson(matrix.mdf_files) }}" != "[]" ]; then | |
| for file in $(echo '${{ toJson(matrix.mdf_files) }}' | jq -r '.[]'); do | |
| MDF_FILES_ARRAY+=("--mdf_files" "$file") | |
| done | |
| fi | |
| mkdir -p data/output/model_cde_pvs/${{ matrix.model }}/ | |
| get_pvs_and_synonyms \ | |
| --model_handle "${{ matrix.model }}" \ | |
| --model_version "${{ matrix.version }}" \ | |
| "${MDF_FILES_ARRAY[@]}" | |
| - name: Make Model CDE Changelog | |
| run: | | |
| set -e | |
| make_model_cde_changelog \ | |
| --model_handle "${{ matrix.model }}" \ | |
| --model_version "${{ matrix.version }}" \ | |
| --author "GitHub Actions" \ | |
| --_commit "${{ github.sha }}" | |
| - name: Upload Generated Files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: changelog-${{ matrix.model }}-${{ matrix.version }} | |
| path: | | |
| data/output/model_changelogs/${{ matrix.model }}/${{ matrix.model }}_changelog_${{ matrix.version }}.xml | |
| data/output/model_cde_pvs/${{ matrix.model }}/${{ matrix.model }}_${{ matrix.version }}_cdes.yml | |
| data/output/model_changelogs/${{ matrix.model }}/${{ matrix.model }}_${{ matrix.version }}_cde_changelog.xml | |
| retention-days: 1 | |
| - name: Report Status | |
| if: always() | |
| run: | | |
| echo "## Changelog Generation Status for ${{ matrix.model }} v${{ matrix.version }}" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "✅ **Success**: Generated all changelog files" >> $GITHUB_STEP_SUMMARY | |
| echo "- Model changelog: ${{ matrix.model }}_changelog_${{ matrix.version }}.xml" >> $GITHUB_STEP_SUMMARY | |
| echo "- CDE PVs: ${{ matrix.model }}_${{ matrix.version }}_cdes.yml" >> $GITHUB_STEP_SUMMARY | |
| echo "- CDE changelog: ${{ matrix.model }}_${{ matrix.version }}_cde_changelog.xml" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Failed**: Error generating changelog files" >> $GITHUB_STEP_SUMMARY | |
| echo "Please check the logs for model ${{ matrix.model }} version ${{ matrix.version }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| commit-all-changelogs: | |
| needs: [determine-models, generate-changelogs] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| if: ${{ !fromJson(github.event.inputs.no_commit || 'false') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Download changelog artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: changelog-* | |
| continue-on-error: true | |
| - name: Move files to proper structure | |
| run: | | |
| for artifact_dir in changelog-*; do | |
| if [ -d "$artifact_dir" ]; then | |
| echo "Processing $artifact_dir" | |
| if [ -d "$artifact_dir/model_changelogs" ]; then | |
| cp -rn "$artifact_dir/model_changelogs" data/output/ | |
| fi | |
| if [ -d "$artifact_dir/model_cde_pvs" ]; then | |
| cp -rn "$artifact_dir/model_cde_pvs" data/output/ | |
| fi | |
| rm -rf "$artifact_dir" | |
| fi | |
| done | |
| echo "Final structure:" | |
| find data/output -type f \( -name "*.xml" -o -name "*.yml" \) | head -20 | |
| - name: Commit Generated Files | |
| run: | | |
| git config --global user.name "GitHub Actions Bot" | |
| git config --global user.email "[email protected]" | |
| git add data/output/model_changelogs/ | |
| git add data/output/model_cde_pvs/ | |
| if [ -n "$(git status --porcelain)" ]; then | |
| MODEL_COUNT=$(echo '${{ needs.determine-models.outputs.affected_models }}' | jq -r '.include | length') | |
| MODELS_LIST=$(echo '${{ needs.determine-models.outputs.affected_models }}' | jq -r '.include[] | "- \(.model) v\(.version)"' | sort) | |
| COMMIT_MSG=$(printf "Add FNL changelogs for %s model versions\n\nGenerated FNL changelogs for:\n%s\n\nGenerated by workflow run: %s/%s/actions/runs/%s" \ | |
| "$MODEL_COUNT" \ | |
| "$MODELS_LIST" \ | |
| "${{ github.server_url }}" \ | |
| "${{ github.repository }}" \ | |
| "${{ github.run_id }}") | |
| git commit -m "$COMMIT_MSG" | |
| git push origin ${{ github.ref_name }} | |
| else | |
| echo "No changes to commit" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| collect-changelogs: | |
| needs: [determine-models, generate-changelogs] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| changelog_files: ${{ steps.collect-changelogs.outputs.changelog_files }} | |
| steps: | |
| - name: Collect All Changelog Files | |
| id: collect-changelogs | |
| run: | | |
| MATRIX='${{ needs.determine-models.outputs.affected_models }}' | |
| CHANGELOGS="[]" | |
| if [ "$MATRIX" != "" ] && [ "$MATRIX" != "null" ] && [ "$MATRIX" != "{\"include\": []}" ]; then | |
| MODEL_COUNT=$(echo "$MATRIX" | jq '.include | length') | |
| if [ "$MODEL_COUNT" -gt 0 ]; then | |
| while IFS= read -r model_version; do | |
| IFS=':' read -r model version <<< "$model_version" | |
| CHANGELOG_FILE="data/output/model_changelogs/${model}/${model}_changelog_${version}.xml" | |
| CDE_CHANGELOG_FILE="data/output/model_changelogs/${model}/${model}_${version}_cde_changelog.xml" | |
| CHANGELOGS=$(echo "$CHANGELOGS" | jq --arg cf "$CHANGELOG_FILE" --arg cdf "$CDE_CHANGELOG_FILE" '. + [$cf, $cdf]') | |
| done < <(echo "$MATRIX" | jq -r '.include[] | "\(.model):\(.version)"') | |
| fi | |
| fi | |
| echo "changelog_files=$(echo $CHANGELOGS | jq -c .)" >> $GITHUB_OUTPUT | |
| echo "Changelog files: $(echo $CHANGELOGS | jq .)" | |
| update-fnl-mdb-dev: | |
| needs: [determine-models, collect-changelogs, commit-all-changelogs] | |
| if: ${{ needs.collect-changelogs.outputs.changelog_files != '' && needs.collect-changelogs.outputs.changelog_files != '[]' && !fromJson(github.event.inputs.no_commit || 'false') }} | |
| uses: ./.github/workflows/update_mdb.yml | |
| secrets: inherit | |
| with: | |
| mdb_id: fnl-mdb-dev | |
| changelog_files: ${{ needs.collect-changelogs.outputs.changelog_files }} | |
| dry_run: ${{ fromJson(github.event.inputs.dry_run || 'false') }} | |
| log_level: ${{ github.event.inputs.log_level || 'info' }} |