veda-data-airflow/.github/workflows/cicd.yml at ea20dd786431dc948993271c4f05fd8f29b8f632 · NASA-IMPACT/veda-data-airflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
name: CICD 🚀

permissions:
  contents: read

on:
  push:
    branches:
      - dev
  pull_request:
    branches:
      - main
      - dev
      - production
    types: [ opened, reopened, edited, synchronize ]

jobs:
  gitflow-enforcer:
    name: GitFlow Enforcer 👮‍
    runs-on: ubuntu-latest
    steps:
      - name: Check branch
        if: github.base_ref == 'main' && github.head_ref != 'dev' || github.base_ref == 'production' && github.head_ref != 'main'
        run: |
          echo "ERROR: You can only merge to main from dev and to production from main"
          exit 1
  test-dags:
    name: Test Airflow DAGs 🧪
    needs: gitflow-enforcer
    runs-on: ubuntu-latest
    steps:
      - name: Check out Git repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0

      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 #v8.3.2
        name: Setup uv
        with:
          python-version: 3.11

      - name: Run uv sync
        run: uv sync --all-groups

      - name: Test Airflow DAGs
        run: |
          export AIRFLOW_HOME=/tmp
          uv run airflow db migrate
          uv run pytest

  define-environment:
    name: Set ✨ environment ✨
    needs: [gitflow-enforcer, test-dags]
    runs-on: ubuntu-latest
    steps:
      - name: Set the environment based on the branch
        id: define_environment
        run: |
          if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
            echo "env_name=development" >> $GITHUB_OUTPUT
          fi
      - name: Print the environment
        run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}"

    outputs:
      env_name: ${{ steps.define_environment.outputs.env_name }}

  plan:
    name: 'Run `terraform plan` against development environment'
    runs-on: ubuntu-latest
    if: ${{ !needs.define-environment.outputs.env_name }}
    needs: [gitflow-enforcer, define-environment]
    environment: development
    concurrency: development
    permissions:
      id-token: write # Required to request OIDC token for use with AWS

    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0
        with:
          lfs: "true"
          submodules: "recursive"

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d #v6.2.2
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "veda-airflow-github-development-planner"
          aws-region: us-west-2

      - name: Plan Changes
        uses: "./.github/actions/terraform-deploy-sm2a"
        with:
          dir: .
          env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }}
          env-file: .env
          aws-region: us-west-2
          run_command: plan

  deploy:
    name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀
    runs-on: ubuntu-latest
    if: ${{ needs.define-environment.outputs.env_name }}
    needs: [gitflow-enforcer, define-environment]
    environment: ${{ needs.define-environment.outputs.env_name }}
    concurrency: ${{ needs.define-environment.outputs.env_name }}
    permissions:
      id-token: write # Required to request OIDC token for use with AWS

    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0
        with:
          lfs: "true"
          submodules: "recursive"

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d #v6.2.2
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "veda-airflow-github-${{ needs.define-environment.outputs.env_name }}-deployment"
          aws-region: us-west-2

      - name: Run SM2A deployment
        # Flag to deploy SM2A
        if: ${{ vars.DEPLOY_SM2A }} = "true"
        uses: "./.github/actions/terraform-deploy-sm2a"
        with:
          dir: .
          env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }}
          env-file: .env
          aws-region: us-west-2