perf: alembic migrations

Author: andhreljaKern

.github/workflows/k8s_db_rollback.yml renamed to .github/workflows/k8s_db_downgrade.yml

@@ -1,4 +1,4 @@
-name: 'K8: Database Rollback'
+name: 'K8: Database Downgrade'
 
 on:
   workflow_call:
@@ -21,8 +21,8 @@ permissions:
   actions: read
 
 jobs:
-  k8-db-rollback:
-    name: 'K8: Database Rollback'
+  k8-db-downgrade:
+    name: 'K8: Database Downgrade'
     runs-on: [self-hosted, "${{ github.ref_name }}"]
     environment: ${{ github.ref_name }}
     env:
@@ -68,7 +68,7 @@ jobs:
     - name: Apply Alembic Rollback
       shell: bash
       run: |
-        bash cicd-deployment-scripts/k8s/db_rollback.sh \
+        bash cicd-deployment-scripts/k8s/db_downgrade.sh \
           -e ${{ github.ref_name }} \
           -d ${{ github.event.repository.name }} \
           -p ${{ github.workspace }}/${{ env.KUBERNETES_CLUSTER_REPO_NAME }} \

.github/workflows/k8s_db_migrate.yml renamed to .github/workflows/k8s_db_upgrade.yml

@@ -1,4 +1,4 @@
-name: 'K8: Database Migrate'
+name: 'K8: Database Upgrade'
 
 on:
   workflow_call:
@@ -13,6 +13,13 @@ on:
         required: false
         default: 'latest'
         type: string
+    outputs:
+      alembic_current_rev:
+        description: 'Current Alembic Rev'
+        value: ${{ jobs.k8-db-upgrade.outputs.ALEMBIC_CURRENT_REV }}
+      alembic_upgraded_rev:
+        description: 'Upgraded Alembic Rev'
+        value: ${{ jobs.k8-db-upgrade.outputs.ALEMBIC_UPGRADED_REV }}
 
 # Special permissions required for OIDC authentication
 permissions:
@@ -21,8 +28,8 @@ permissions:
   actions: read
 
 jobs:
-  k8-db-migrate:
-    name: 'K8: Database Migrate'
+  k8-db-upgrade:
+    name: 'K8: Database Upgrade'
     runs-on: [self-hosted, "${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.ref_name }}"]
     environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.ref_name }}
     env:
@@ -34,6 +41,9 @@ jobs:
       KUBERNETES_MANIFEST_PATH: "${{ vars.KUBERNETES_MANIFEST_PATH }}"
       AZURE_RESOURCE_GROUP: "${{ vars.AZURE_RESOURCE_GROUP }}"
       AZURE_CONTAINER_REGISTRY: "${{ vars.AZURE_CONTAINER_REGISTRY }}"
+    outputs:
+      ALEMBIC_CURRENT_REV: ${{ steps.alembic_command.outputs.ALEMBIC_CURRENT_REV }}
+      ALEMBIC_UPGRADED_REV: ${{ steps.alembic_command.outputs.ALEMBIC_UPGRADED_REV }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -68,8 +78,9 @@ jobs:
 
     - name: Apply Alembic Migrate
       shell: bash
+      id: alembic_command
       run: |
-        bash cicd-deployment-scripts/k8s/db_migrate.sh \
+        bash cicd-deployment-scripts/k8s/db_upgrade.sh \
           -e ${{ env.ENVIRONMENT_NAME }} \
           -d ${{ github.event.repository.name }} \
           -p ${{ github.workspace }}/${{ env.KUBERNETES_CLUSTER_REPO_NAME }} \

.github/workflows/k8s_test.yml

@@ -19,17 +19,20 @@ jobs:
     uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/az_acr_test.yml@dev
     secrets: inherit
 
-  call-k8-db-migrate:
-    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_db_migrate.yml@dev
+  call-k8-db-upgrade:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_db_upgrade.yml@dev
     needs: [call-az-acr-push-test]
+    if: ${{ github.event.repository.name == 'refinery-gateway' || github.event.repository.name == 'gates-gateway'  || github.event.repository.name == 'cognition-gateway' }}
     secrets: inherit
     with:
+      alembic_downgrade_rev: 'head'
       docker_image_tag: ${{ needs.call-az-acr-push-test.outputs.GH_REF_NAME }}
 
   k8-test:
     name: 'K8: Test'
     runs-on: [self-hosted, dev]
-    needs: [call-az-acr-push-test, call-k8-db-migrate]
+    needs: [call-az-acr-push-test, call-k8-db-upgrade]
+    if: !failure()
     environment: dev
     env:
       KUBELOGIN_VERSION: "v0.0.25"
@@ -79,3 +82,12 @@ jobs:
           -t ${{ needs.call-az-acr-push-test.outputs.GH_REF_NAME }} \
           -r ${{ env.AZURE_CONTAINER_REGISTRY }} \
           -c "${{ inputs.test_cmd }}"
+
+  call-k8-db-downgrade:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_db_downgrade.yml@dev
+    needs: [call-az-acr-push-test, call-k8-db-upgrade, k8-test]
+    if: ${{ failure() && (github.event.repository.name == 'refinery-gateway' || github.event.repository.name == 'gates-gateway'  || github.event.repository.name == 'cognition-gateway') }}
+    secrets: inherit
+    with:
+      alembic_downgrade_rev: ${{ needs.call-k8-db-upgrade.outputs.ALEMBIC_CURRENT_REV }}
+      docker_image_tag: ${{ needs.call-az-acr-push-test.outputs.GH_REF_NAME }}

k8s/db_rollback.sh renamed to k8s/db_downgrade.sh

@@ -8,6 +8,7 @@ KUBERNETES_NAMESPACE=""
 AZURE_CONTAINER_REGISTRY=""
 IMAGE_TAG=""
 alembic_downgrade_rev=""
+alembic_command="downgrade"
 
 while getopts e:d:p:n:r:t:a: flag
 do
@@ -22,37 +23,39 @@ do
     esac
 done
 
+if [ $KUBERNETES_DEPLOYMENT_NAME = "cognition-gateway" ]; then
+    KUBERNETES_DEPLOYMENT_NAME="refinery-gateway"
+fi
+
 echo "::group::Kubernetes Context"
 kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
 echo "::endgroup::"
 
-echo "::group::Migrating to revision: $alembic_downgrade_rev"
 
-sed 's|${ALEMBIC_COMMAND}|downgrade|g' \
+echo "::group::Migrating to revision: $alembic_downgrade_rev"
+sed 's|${ALEMBIC_COMMAND}|'$alembic_command'|g' \
     $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.tmpl \
     > $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 sed -i.bak 's|${ALEMBIC_ARGS}|'${alembic_downgrade_rev}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 sed -i.bak 's|${IMAGE_TAG}|'${IMAGE_TAG}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
-
 rm $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml.bak
 cat $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 echo "::endgroup::"
 
+
 echo "::group::Apply Kubernetes Job"
 kubectl apply --filename $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
-
 echo "Waiting for migration job to complete ..."
 kubectl wait --for=condition=complete --timeout 60s job/$KUBERNETES_DEPLOYMENT_NAME-migrate
 kubectl logs job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
 echo "::endgroup::"
 
-echo "::group::Rolling out deployment"
+
+echo "::group::Rolling out deployments"
 kubectl rollout restart deployment/$KUBERNETES_DEPLOYMENT_NAME
 kubectl rollout status deployment/$KUBERNETES_DEPLOYMENT_NAME
-
 kubectl rollout restart deployment/cognition-gateway
 kubectl rollout status deployment/cognition-gateway
-
-kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
 echo "::endgroup::"

k8s/db_migrate.sh renamed to k8s/db_upgrade.sh

@@ -8,6 +8,7 @@ KUBERNETES_NAMESPACE=""
 AZURE_CONTAINER_REGISTRY=""
 IMAGE_TAG=""
 alembic_upgrade_rev=""
+alembic_command="upgrade"
 
 while getopts e:d:p:n:r:t:a: flag
 do
@@ -22,38 +23,53 @@ do
     esac
 done
 
+if [ $KUBERNETES_DEPLOYMENT_NAME = "cognition-gateway" ]; then
+    KUBERNETES_DEPLOYMENT_NAME="refinery-gateway"
+fi
 
 echo "::group::Kubernetes Context"
 kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
 echo "::endgroup::"
 
-echo "::group::Migrating to revision: $alembic_upgrade_rev"
 
-sed 's|${ALEMBIC_COMMAND}|upgrade|g' \
+echo "::group::Get Alembic current revision"
+ALEMBIC_CURRENT_REV=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -- alembic current || true)
+echo "ALEMBIC_CURRENT_REV=${ALEMBIC_CURRENT_REV:0:12}" >> $GITHUB_OUTPUT
+echo "Alembic current revision: $ALEMBIC_CURRENT_REV"
+echo "::endgroup::"
+
+
+echo "::group::Migrating to revision: $alembic_upgrade_rev"
+sed 's|${ALEMBIC_COMMAND}|'$alembic_command'|g' \
     $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.tmpl \
     > $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 sed -i.bak 's|${ALEMBIC_ARGS}|'${alembic_upgrade_rev}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 sed -i.bak 's|${IMAGE_TAG}|'${IMAGE_TAG}'|g' $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
-
 rm $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml.bak
 cat $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
 echo "::endgroup::"
 
+
 echo "::group::Apply Kubernetes Job"
 kubectl apply --filename $KUBERNETES_DEPLOYMENT_REPO_PATH/infrastructure/$ENVIRONMENT_NAME/job/$KUBERNETES_DEPLOYMENT_NAME-migrate.yml
-
 echo "Waiting for migration job to complete ..."
 kubectl wait --for=condition=complete --timeout 60s job/$KUBERNETES_DEPLOYMENT_NAME-migrate
 kubectl logs job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
+echo "::endgroup::"
+
+
+echo "::group::Get Alembic upgraded revision"
+ALEMBIC_UPGRADED_REV=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -- alembic current 2>/dev/null || true)
+echo "ALEMBIC_UPGRADED_REV=${ALEMBIC_UPGRADED_REV:0:12}" >> $GITHUB_OUTPUT
+echo "Alembic upgraded revision: $ALEMBIC_UPGRADED_REV"
 echo "::endgroup::"
 
-echo "::group::Rolling out deployment"
+
+echo "::group::Rolling out deployments"
 kubectl rollout restart deployment/$KUBERNETES_DEPLOYMENT_NAME
 kubectl rollout status deployment/$KUBERNETES_DEPLOYMENT_NAME
-
 kubectl rollout restart deployment/cognition-gateway
 kubectl rollout status deployment/cognition-gateway
-
-kubectl delete job/$KUBERNETES_DEPLOYMENT_NAME-migrate
 echo "::endgroup::"

k8s/test.sh

@@ -23,29 +23,30 @@ kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
 echo "::endgroup::"
 
-echo "::notice::running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
 
 echo "::group::Upgrade deployment image"
 KUBERNETES_POD_EXISTING_IMAGE=$(kubectl get pod --output json \
     --selector app=${KUBERNETES_DEPLOYMENT_NAME} \
     | jq -r '.items[0] | .spec.containers[0].image')
 
 kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}
-echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
-
 kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
 echo "::endgroup::"
 
-echo "::group::Running test command"
+
+echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
+
+
+echo "::group::Running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
 set +e
 exitcode=0
-echo "::warning::running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
 kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- ''$TEST_CMD''
 exitcode=$?
 set -e
 echo "::endgroup::"
 
 kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${KUBERNETES_POD_EXISTING_IMAGE}
+
 echo "::notice::using ${KUBERNETES_POD_EXISTING_IMAGE}"
 
 exit $exitcode