feat(ci): db migrations

andhreljaKern · andhreljaKern · commit 46030642898b · 2024-07-24T12:59:08.000+02:00
diff --git a/.github/workflows/k8s_db_migrate.yml b/.github/workflows/k8s_db_migrate.yml
@@ -0,0 +1,78 @@
+name: 'K8: Database Migrate'
+
+on:
+  workflow_call:
+    inputs:
+      alembic_upgrade_rev:
+        description: 'Rev of migration to migrate to'
+        required: false
+        default: 'head'
+        type: string
+      docker_image_tag:
+        description: 'Docker Image Tag'
+        required: false
+        default: 'latest'
+        type: string
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+jobs:
+  k8-db-migrate:
+    name: 'K8: Database Migrate'
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    environment: ${{ github.ref_name }}
+    env:
+      KUBELOGIN_VERSION: "v0.0.25"
+      KUBERNETES_CLUSTER_REPO_NAME: "${{ vars.KUBERNETES_CLUSTER_REPO_NAME }}"
+      KUBERNETES_CLUSTER_NAME: "${{ vars.KUBERNETES_CLUSTER_NAME }}"
+      KUBERNETES_NAMESPACE: "${{ vars.KUBERNETES_NAMESPACE }}"
+      KUBERNETES_MANIFEST_PATH: "${{ vars.KUBERNETES_MANIFEST_PATH }}"
+      AZURE_RESOURCE_GROUP: "${{ vars.AZURE_RESOURCE_GROUP }}"
+      AZURE_CONTAINER_REGISTRY: "${{ vars.AZURE_CONTAINER_REGISTRY }}"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: GitHub Configuration
+      run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+    - name: Clone cicd-deployment-scripts
+      run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git
+    
+    - name: Clone ${{ env.KUBERNETES_CLUSTER_REPO_NAME }}
+      run: git clone https://github.com/code-kern-ai/${{ env.KUBERNETES_CLUSTER_REPO_NAME }}.git
+
+    - name: Azure Cloud Login
+      uses: azure/login@v2
+      with:
+        client-id: ${{ secrets.AZURE_CLIENT_ID }}
+        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+    - name: Set up kubelogin for non-interactive login
+      uses: azure/use-kubelogin@v1
+      with:
+        kubelogin-version: ${{ env.KUBELOGIN_VERSION }}
+
+    - uses: azure/aks-set-context@v3
+      with:
+        resource-group: ${{ env.AZURE_RESOURCE_GROUP }}
+        cluster-name: ${{ env.KUBERNETES_CLUSTER_NAME }}
+        admin: 'false'
+        use-kubelogin: 'true'
+
+    - name: Apply Alembic Migrate
+      shell: bash
+      run: |
+        bash cicd-deployment-scripts/k8s/db_migrate.sh \
+          -e ${{ github.ref_name }} \
+          -d ${{ github.event.repository.name }} \
+          -p ${{ github.workspace }}/${{ env.KUBERNETES_CLUSTER_REPO_NAME }} \
+          -n ${{ env.KUBERNETES_NAMESPACE }} \
+          -r ${{ env.AZURE_CONTAINER_REGISTRY }} \
+          -t ${{ inputs.docker_image_tag }} \
+          -a ${{ inputs.alembic_upgrade_rev }}
diff --git a/.github/workflows/k8s_db_rollback.yml b/.github/workflows/k8s_db_rollback.yml
@@ -8,6 +8,11 @@ on:
         required: false
         default: 'base'
         type: string
+      docker_image_tag:
+        description: 'Docker Image Tag'
+        required: false
+        default: 'latest'
+        type: string
 
 # Special permissions required for OIDC authentication
 permissions:
@@ -37,6 +42,9 @@ jobs:
 
     - name: Clone cicd-deployment-scripts
       run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git
+    
+    - name: Clone ${{ env.KUBERNETES_CLUSTER_REPO_NAME }}
+      run: git clone https://github.com/code-kern-ai/${{ env.KUBERNETES_CLUSTER_REPO_NAME }}.git
 
     - name: Azure Cloud Login
       uses: azure/login@v2
@@ -63,7 +71,8 @@ jobs:
         bash cicd-deployment-scripts/k8s/db_rollback.sh \
           -e ${{ github.ref_name }} \
           -d ${{ github.event.repository.name }} \
+          -p ${{ github.workspace }}/${{ env.KUBERNETES_CLUSTER_REPO_NAME }} \
           -n ${{ env.KUBERNETES_NAMESPACE }} \
           -r ${{ env.AZURE_CONTAINER_REGISTRY }} \
-          -t latest \
+          -t ${{ inputs.docker_image_tag }} \
           -a ${{ inputs.alembic_downgrade_rev }}
diff --git a/.github/workflows/k8s_test.yml b/.github/workflows/k8s_test.yml
@@ -15,14 +15,21 @@ permissions:
   actions: read
 
 jobs:
-  call-az-acr-push:
+  call-az-acr-push-test:
     uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/az_acr_test.yml@dev
     secrets: inherit
   
+  call-k8-db-migrate:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_db_migrate.yml@dev
+    needs: [call-az-acr-push-test]
+    secrets: inherit
+    with:
+      docker_image_tag: ${{ needs.call-az-acr-push-test.outputs.GH_REF_NAME }}
+
   k8-test:
     name: 'K8: Test'
     runs-on: [self-hosted, dev]
-    needs: [call-az-acr-push]
+    needs: [call-az-acr-push-test, call-k8-db-migrate]
     environment: dev
     env:
       KUBELOGIN_VERSION: "v0.0.25"
@@ -31,16 +38,10 @@ jobs:
       KUBERNETES_MANIFEST_PATH: "${{ vars.KUBERNETES_MANIFEST_PATH }}"
       AZURE_RESOURCE_GROUP: "${{ vars.AZURE_RESOURCE_GROUP }}"
       AZURE_CONTAINER_REGISTRY: "${{ vars.AZURE_CONTAINER_REGISTRY }}"
-    outputs:
-      GH_REF_NAME: ${{ steps.branch_name.outputs.GH_REF_NAME }}
     steps:
     # Checkout the repository to the GitHub Actions runner
     - name: Checkout
       uses: actions/checkout@v4
-    
-    - name: Configure branch name
-      id: branch_name
-      run: echo "GH_REF_NAME=$(echo ${{ github.event.pull_request.head.ref }} | sed 's|/|-|g')" >> $GITHUB_OUTPUT
 
     - name: GitHub Configuration
       run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
@@ -75,6 +76,6 @@ jobs:
         bash cicd-deployment-scripts/k8s/test.sh \
           -n ${{ env.KUBERNETES_NAMESPACE }} \
           -d ${{ github.event.repository.name }} \
-          -t test-${{ steps.branch_name.outputs.GH_REF_NAME }} \
+          -t ${{ needs.call-az-acr-push-test.outputs.GH_REF_NAME }} \
           -r ${{ env.AZURE_CONTAINER_REGISTRY }} \
           -c "${{ inputs.test_cmd }}"
diff --git a/k8s/db_migrate.sh b/k8s/db_migrate.sh
@@ -0,0 +1,55 @@
+# !/bin/bash
+set -e
+
+ENVIRONMENT_NAME=""
+KUBERNETES_DEPLOYMENT_NAME=""
+KUBERNETES_DEPLOYMENT_REPO_PATH=""
+KUBERNETES_NAMESPACE=""
+AZURE_CONTAINER_REGISTRY=""
+IMAGE_TAG=""
+alembic_upgrade_rev=""
+
+while getopts e:d:p:n:r:t:a: flag
+do
+    case "${flag}" in
+        e) ENVIRONMENT_NAME=${OPTARG};;
+        d) KUBERNETES_DEPLOYMENT_NAME=${OPTARG};;
+        p) KUBERNETES_DEPLOYMENT_REPO_PATH=${OPTARG};;
+        n) KUBERNETES_NAMESPACE=${OPTARG};;
+        r) AZURE_CONTAINER_REGISTRY=${OPTARG};;
+        t) IMAGE_TAG=${OPTARG};;
+        a) alembic_upgrade_rev=${OPTARG};;
+    esac
+done
+
+kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
+echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
+
+echo "::group::Migrating to revision: $alembic_upgrade_rev"
+
+sed 's|${ALEMBIC_COMMAND}|upgrade|g' \
+    $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.tmpl \
+    > $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+sed -i.bak 's|${ALEMBIC_ARGS}|'${alembic_upgrade_rev}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+sed -i.bak 's|${IMAGE_TAG}|'${IMAGE_TAG}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+
+rm $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml.bak
+cat $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+echo "::endgroup::"
+
+echo "::group::Apply Kubernetes Job"
+kubectl apply --filename $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+
+echo "Waiting for migration job to complete ..."
+kubectl wait --for=condition=complete --timeout 60s job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+echo "::endgroup::"
+
+echo "::group::Rolling out deployment"
+kubectl rollout restart deployment/$KUBERNETES_DEPLOYMENT_NAME
+kubectl rollout status deployment/$KUBERNETES_DEPLOYMENT_NAME
+
+kubectl rollout restart deployment/cognition-gateway
+kubectl rollout status deployment/cognition-gateway
+
+kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+echo "::endgroup::"
diff --git a/k8s/db_rollback.sh b/k8s/db_rollback.sh
@@ -3,16 +3,18 @@ set -e
 
 ENVIRONMENT_NAME=""
 KUBERNETES_DEPLOYMENT_NAME=""
+KUBERNETES_DEPLOYMENT_REPO_PATH=""
 KUBERNETES_NAMESPACE=""
 AZURE_CONTAINER_REGISTRY=""
 IMAGE_TAG=""
 alembic_downgrade_rev=""
 
-while getopts e:d:r:t:n:a: flag
+while getopts e:d:p:n:r:t:a: flag
 do
     case "${flag}" in
         e) ENVIRONMENT_NAME=${OPTARG};;
         d) KUBERNETES_DEPLOYMENT_NAME=${OPTARG};;
+        p) KUBERNETES_DEPLOYMENT_REPO_PATH=${OPTARG};;
         n) KUBERNETES_NAMESPACE=${OPTARG};;
         r) AZURE_CONTAINER_REGISTRY=${OPTARG};;
         t) IMAGE_TAG=${OPTARG};;
@@ -23,6 +25,31 @@ done
 kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
 
-echo "Rolling back to migration revision: $rev_rollback_migrations"
-kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- alembic downgrade ${rev_rollback_migrations}
-kubectl rollout status deployment/${KUBERNETES_DEPLOYMENT_NAME}
+echo "::group::Migrating to revision: $alembic_downgrade_rev"
+
+sed 's|${ALEMBIC_COMMAND}|downgrade|g' \
+    $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.tmpl \
+    > $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+sed -i.bak 's|${ALEMBIC_ARGS}|'${alembic_downgrade_rev}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+sed -i.bak 's|${IMAGE_TAG}|'${IMAGE_TAG}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+
+rm $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml.bak
+cat $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+echo "::endgroup::"
+
+echo "::group::Apply Kubernetes Job"
+kubectl apply --filename $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
+
+echo "Waiting for migration job to complete ..."
+kubectl wait --for=condition=complete --timeout 60s job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+echo "::endgroup::"
+
+echo "::group::Rolling out deployment"
+kubectl rollout restart deployment/$KUBERNETES_DEPLOYMENT_NAME
+kubectl rollout status deployment/$KUBERNETES_DEPLOYMENT_NAME
+
+kubectl rollout restart deployment/cognition-gateway
+kubectl rollout status deployment/cognition-gateway
+
+kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+echo "::endgroup::"
diff --git a/k8s/test.sh b/k8s/test.sh
@@ -18,17 +18,14 @@ do
     esac
 done
 
+echo "::group::Kubernetes Context"
 kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
-
-set +e
-alembic_exitcode=0
-ALEMBIC_CURRENT_REVISION=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- alembic current 2> /dev/null)
-alembic_exitcode=$?
-set -e
+echo "::endgroup::"
 
 echo "::notice::running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
 
+echo "::group::Upgrade deployment image"
 KUBERNETES_POD_EXISTING_IMAGE=$(kubectl get pod --output json \
     --selector app=${KUBERNETES_DEPLOYMENT_NAME} \
     | jq -r '.items[0] | .spec.containers[0].image')
@@ -37,33 +34,18 @@ kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYME
 echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
 
 kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
+echo "::endgroup::"
 
-if [ $alembic_exitcode -eq 0 ]; then
-    kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- alembic upgrade head
-fi
-
+echo "::group::Running test command"
 set +e
 exitcode=0
 echo "::warning::running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
-kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- "$TEST_CMD"
+kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- ''$TEST_CMD''
 exitcode=$?
 set -e
+echo "::endgroup::"
 
 kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${KUBERNETES_POD_EXISTING_IMAGE}
 echo "::notice::using ${KUBERNETES_POD_EXISTING_IMAGE}"
 
-if [ $alembic_exitcode -eq 0 ] && [ $exitcode -ne 0 ]; then
-    ALEMBIC_HEAD=${ALEMBIC_CURRENT_REVISION:0:12}
-    
-    ALEMBIC_UPDATED_REVISION=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- alembic current)
-    ALEMBIC_UPDATED_HEAD=${ALEMBIC_UPDATED_REVISION:0:12}
-
-    if [ $ALEMBIC_HEAD = $ALEMBIC_UPDATED_HEAD ]; then
-        echo "::notice::skipping alembic downgrade"
-    else
-        echo "::notice::downgrading to alembic revision: $ALEMBIC_HEAD"
-        kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- alembic downgrade $ALEMBIC_HEAD
-    fi
-fi
-
 exit $exitcode
