Feature/improve gh actions #10
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "10: 🎯 Airflow Validations" | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| branches: | |
| - 'release/**/*' | |
| paths: | |
| - orchestrate/**/* | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| # This cancels a run if another change is pushed to the same branch | |
| concurrency: | |
| group: orchestrate-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| validate-branch: | |
| runs-on: ubuntu-latest | |
| name: Validate Branch Names and Merge Rules and assure branch has changes from main | |
| env: | |
| SOURCE_BRANCH: ${{ github.head_ref }} | |
| TARGET_BRANCH: ${{ github.base_ref }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Validate current branch | |
| run: automate/git/branch_validator.py | |
| airflow: | |
| name: Pull Request Airflow Tests | |
| runs-on: ubuntu-latest | |
| container: datacoves/ci-airflow-dbt-snowflake:3.4 | |
| env: | |
| AIRFLOW__CORE__DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate/dags | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300 | |
| AIRFLOW__ARTIFACTS_PATH: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate | |
| DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| DATACOVES__REPO_PATH_RO: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| PYTHONPATH: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| FORCE_COLOR: 1 | |
| OUTPUT_FILE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/test_output.md | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Test DAGs Load time and variable usage at top level | |
| id: test_load_time_variables | |
| run: python /usr/app/test_dags.py --dag-loadtime-threshold 1 --check-variable-usage --write-output --filename "$OUTPUT_FILE" | |
| # if write-output is set in the prior step, the following step will run | |
| - name: Add PR comment of results of test_load_time_variables tests | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| filePath: ${{ env.OUTPUT_FILE }} | |
| comment_tag: Test DAGs Load time and variable usage at top level | |
| - name: Custom Airflow Validation Tests | |
| env: | |
| NO_COLOR: 1 | |
| run: pytest $AIRFLOW__ARTIFACTS_PATH/test_dags/validate_dags.py --output-file "$OUTPUT_FILE" | |
| - name: Add PR comment of results of custom Airflow validation tests | |
| if: always() | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| # filePath: formatted_output.md | |
| filePath: ${{ env.OUTPUT_FILE }} | |
| comment_tag: Custom Tests | |
| GITHUB_TOKEN: ${{ github.token }} | |
| airflow-job-status: | |
| runs-on: ubuntu-latest | |
| needs: [airflow] | |
| if: always() | |
| steps: | |
| - name: Comment PR with Airflow status | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| message: | | |
| ## 🎯 Airflow Workflow Status | |
| - **Job**: Pull Request Airflow Tests → **${{ needs.airflow.result }}** | |
| ${{ needs.airflow.result == 'skipped' && '⏭️ _Skipped because no orchestrate/ files were changed_' || '' }} | |
| ${{ needs.airflow.result == 'success' && '✅ _All orchestrate validations passed_' || '' }} | |
| ${{ needs.airflow.result == 'failure' && '❌ _Orchestrate validations failed_' || '' }} | |
| comment_tag: airflow-workflow-status |