Skip to content

Commit ba57e98

Browse files
Prajwal-MicrosoftPrajwal-Microsoft
authored
Merge pull request #19 from microsoft/PSL-US-16294
feat: Automate Deployment Lifecycle with Azure Bicep & Resource Cleanup
2 parents 032a46c + 67fa36c commit ba57e98

File tree

2 files changed

+445
-0
lines changed

2 files changed

+445
-0
lines changed

‎.github/workflows/deploy.yml‎

Lines changed: 350 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,350 @@
1+
name: Deployment Lifecycle Automation
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
schedule:
8+
- cron: "0 9,21 * * *" # Runs at 9:00 AM and 9:00 PM GMT
9+
workflow_dispatch:
10+
11+
jobs:
12+
deploy:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout Code
16+
uses: actions/checkout@v3
17+
18+
- name: Setup Azure CLI
19+
run: |
20+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
21+
az --version # Verify installation
22+
23+
- name: Login to Azure
24+
run: |
25+
az login --service-principal -u ${{ secrets.AZURE_MAINTENANCE_CLIENT_ID }} -p ${{ secrets.AZURE_MAINTENANCE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
26+
27+
- name: Run Quota Check
28+
id: quota-check
29+
run: |
30+
export AZURE_MAINTENANCE_CLIENT_ID=${{ secrets.AZURE_MAINTENANCE_CLIENT_ID }}
31+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
32+
export AZURE_MAINTENANCE_CLIENT_SECRET=${{ secrets.AZURE_MAINTENANCE_CLIENT_SECRET }}
33+
export AZURE_MAINTENANCE_SUBSCRIPTION_ID="${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}"
34+
export GPT_MIN_CAPACITY="100"
35+
export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
36+
37+
chmod +x infra/scripts/checkquota.sh
38+
if ! infra/scripts/checkquota.sh; then
39+
# If quota check fails due to insufficient quota, set the flag
40+
if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then
41+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
42+
fi
43+
exit 1 # Fail the pipeline if any other failure occurs
44+
fi
45+
46+
- name: Send Notification on Quota Failure
47+
if: env.QUOTA_FAILED == 'true'
48+
run: |
49+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
50+
EMAIL_BODY=$(cat <<EOF
51+
{
52+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
53+
}
54+
EOF
55+
)
56+
57+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
58+
-H "Content-Type: application/json" \
59+
-d "$EMAIL_BODY" || echo "Failed to send notification"
60+
61+
- name: Fail Pipeline if Quota Check Fails
62+
if: env.QUOTA_FAILED == 'true'
63+
run: exit 1
64+
65+
- name: Install Bicep CLI
66+
run: az bicep install
67+
68+
- name: Set Deployment Region
69+
run: |
70+
echo "Selected Region: $VALID_REGION"
71+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
72+
73+
- name: Generate Resource Group Name
74+
id: generate_rg_name
75+
run: |
76+
echo "Generating a unique resource group name..."
77+
TIMESTAMP=$(date +%Y%m%d%H%M)
78+
# Define the common part and add a "cps-" prefix
79+
COMMON_PART="pslautomation"
80+
UNIQUE_RG_NAME="cps-${COMMON_PART}${TIMESTAMP}"
81+
echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
82+
echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}"
83+
84+
- name: Check and Create Resource Group
85+
id: check_create_rg
86+
run: |
87+
set -e
88+
echo "Checking if resource group exists..."
89+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
90+
if [ "$rg_exists" = "false" ]; then
91+
echo "Resource group does not exist. Creating..."
92+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} \
93+
--location ${{ env.AZURE_LOCATION }} \
94+
--tags "CreatedBy=Deployment Lifecycle Automation Pipeline" \
95+
"Purpose=Deploying and Cleaning Up Resources for Validation" \
96+
"ApplicationName=Content Processing Accelerator" \
97+
|| { echo "Error creating resource group"; exit 1; }
98+
else
99+
echo "Resource group already exists."
100+
fi
101+
102+
- name: Generate Unique Solution Prefix
103+
id: generate_solution_prefix
104+
run: |
105+
set -e
106+
COMMON_PART="pslr"
107+
TIMESTAMP=$(date +%s)
108+
UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3)
109+
UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
110+
echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
111+
echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}"
112+
113+
- name: Deploy Bicep Template
114+
id: deploy
115+
run: |
116+
set -e
117+
az deployment group create \
118+
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
119+
--template-file infra/main.json \
120+
--parameters \
121+
environmentName="${{ env.SOLUTION_PREFIX }}" \
122+
secondaryLocation="EastUs2" \
123+
contentUnderstandingLocation="WestUS" \
124+
deploymentType="GlobalStandard" \
125+
gptModelName="gpt-4o" \
126+
gptModelVersion="2024-08-06" \
127+
gptDeploymentCapacity="30" \
128+
minReplicaContainerApp="1" \
129+
maxReplicaContainerApp="1" \
130+
minReplicaContainerApi="1" \
131+
maxReplicaContainerApi="1" \
132+
minReplicaContainerWeb="1" \
133+
maxReplicaContainerWeb="1" \
134+
useLocalBuild="false"
135+
136+
- name: Delete Bicep Deployment
137+
if: success()
138+
run: |
139+
set -e
140+
echo "Checking if resource group exists..."
141+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
142+
if [ "$rg_exists" = "true" ]; then
143+
echo "Resource group exists. Cleaning..."
144+
az group delete \
145+
--name ${{ env.RESOURCE_GROUP_NAME }} \
146+
--yes \
147+
--no-wait
148+
echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}"
149+
else
150+
echo "Resource group does not exist."
151+
fi
152+
153+
- name: Wait for Resource Deletion to Complete
154+
run: |
155+
echo "Fetching resources in the resource group: ${{ env.RESOURCE_GROUP_NAME }}"
156+
157+
# Ensure correct subscription is set
158+
az account set --subscription "${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}"
159+
160+
# Fetch all resource IDs dynamically (instead of names)
161+
resources_to_check=($(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[].id" -o tsv))
162+
163+
# Exit early if no resources found
164+
if [ ${#resources_to_check[@]} -eq 0 ]; then
165+
echo "No resources found in the resource group. Skipping deletion check."
166+
exit 0
167+
fi
168+
169+
echo "Resources to check: ${resources_to_check[@]}"
170+
171+
# Extract only resource names and store them in a space-separated string
172+
resources_to_purge=""
173+
for resource_id in "${resources_to_check[@]}"; do
174+
resource_name=$(basename "$resource_id") # Extract the last part of the ID as the name
175+
resources_to_purge+="$resource_name "
176+
done
177+
178+
# Save the list for later use
179+
echo "RESOURCES_TO_PURGE=$resources_to_purge" >> "$GITHUB_ENV"
180+
181+
echo "Waiting for resources to be fully deleted..."
182+
183+
# Maximum retries & retry intervals
184+
max_retries=10
185+
retry_intervals=(150 180 210 240 270 300) # increased intervals for each retry for potentially long deletion times
186+
retries=0
187+
188+
while true; do
189+
all_deleted=true
190+
191+
for resource_id in "${resources_to_check[@]}"; do
192+
echo "Checking if resource '$resource_id' is deleted..."
193+
194+
# Check resource existence using full ID
195+
resource_status=$(az resource show --ids "$resource_id" --query "id" -o tsv 2>/dev/null || echo "NotFound")
196+
197+
if [[ "$resource_status" != "NotFound" ]]; then
198+
echo "Resource '$resource_id' is still present."
199+
all_deleted=false
200+
else
201+
echo "Resource '$resource_id' is fully deleted."
202+
fi
203+
done
204+
205+
# Break loop if all resources are deleted
206+
if [ "$all_deleted" = true ]; then
207+
echo "All resources are fully deleted. Proceeding with purging..."
208+
break
209+
fi
210+
211+
# Stop retrying if max retries are reached
212+
if [ $retries -ge $max_retries ]; then
213+
echo "Some resources were not deleted after $max_retries retries. Failing the pipeline."
214+
exit 1
215+
fi
216+
217+
echo "Some resources are still present. Retrying in ${retry_intervals[$retries]} seconds..."
218+
sleep ${retry_intervals[$retries]}
219+
retries=$((retries + 1))
220+
done
221+
222+
- name: Purging the Resources
223+
if: success()
224+
run: |
225+
set -e
226+
227+
echo "Using saved list of deleted resources from previous step..."
228+
229+
# Ensure the correct subscription is set
230+
az account set --subscription "${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}"
231+
232+
# Iterate over each deleted resource
233+
for resource_name in $RESOURCES_TO_PURGE; do
234+
echo "Checking for deleted resource: $resource_name"
235+
236+
# Query Azure for deleted resources based on type
237+
case "$resource_name" in
238+
*"kv-cps"*)
239+
deleted_resource=$(az keyvault list-deleted --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json)
240+
;;
241+
*"stcps"*)
242+
deleted_resource=$(az storage account list --query "[?name=='$resource_name']" -o json || echo "{}")
243+
;;
244+
*"cosmos-cps"*)
245+
deleted_resource=$(az cosmosdb show --name "$resource_name" --query "{name:name, type:type, id:id}" -o json 2>/dev/null || echo "{}")
246+
;;
247+
*"aisa-cps"*)
248+
deleted_resource=$(az cognitiveservices account list-deleted --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json)
249+
;;
250+
*"appcs-cps"*)
251+
deleted_resource=$(az resource list --query "[?starts_with(name, 'appcs') && type=='Microsoft.Insights/components'].{name:name, type:type, id:id}" -o json)
252+
;;
253+
*"appi-cps"*)
254+
deleted_resource=$(az resource list --query "[?starts_with(name, 'appi') && type=='Microsoft.Insights/components'].{name:name, type:type, id:id}" -o json)
255+
;;
256+
*"ca-cps"*)
257+
deleted_resource=$(az resource list --query "[?starts_with(name, 'ca') && type=='Microsoft.Web/containerApps'].{name:name, type:type, id:id}" -o json)
258+
;;
259+
*)
260+
deleted_resource=$(az resource list --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json)
261+
;;
262+
esac
263+
264+
if [[ -z "$deleted_resource" || "$deleted_resource" == "[]" || "$deleted_resource" == "{}" ]]; then
265+
echo "Resource $resource_name not found in deleted list. Skipping..."
266+
continue
267+
fi
268+
269+
# Extract name, type, and ID from the JSON response
270+
name=$(echo "$deleted_resource" | jq -r '.[0].name')
271+
type=$(echo "$deleted_resource" | jq -r '.[0].type')
272+
id=$(echo "$deleted_resource" | jq -r '.[0].id')
273+
274+
echo "Purging resource: $name (Type: $type)"
275+
276+
case "$type" in
277+
"Microsoft.KeyVault/deletedVaults")
278+
echo "Purging Key Vault: $name"
279+
purge_output=$(az keyvault purge --name "$name" 2>&1 || true)
280+
281+
if echo "$purge_output" | grep -q "MethodNotAllowed"; then
282+
echo "WARNING: Soft Delete Protection is enabled for $name. Purge is not allowed. Skipping..."
283+
else
284+
echo "Key Vault $name purged successfully."
285+
fi
286+
;;
287+
288+
"Microsoft.ContainerRegistry/registries")
289+
echo "Deleting Azure Container Registry (ACR): $name"
290+
az acr delete --name "$name" --yes || echo "Failed to delete Azure Container Registry: $name"
291+
;;
292+
293+
"Microsoft.Storage/storageAccounts")
294+
echo "Purging Storage Account: $name"
295+
az storage account delete --name "$name" --yes || echo "Failed to delete Storage Account: $name"
296+
;;
297+
298+
"Microsoft.DocumentDB/databaseAccounts")
299+
echo "Purging Cosmos DB: $name"
300+
az cosmosdb delete --name "$name" --yes || echo "Failed to delete Cosmos DB Account: $name"
301+
;;
302+
303+
"Microsoft.CognitiveServices/deletedAccounts")
304+
echo "Purging Cognitive Services Account: $name"
305+
az cognitiveservices account purge --location "${{ env.AZURE_LOCATION }}" --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --name "$name" || echo "Failed to purge Cognitive Services Account: $name"
306+
;;
307+
308+
"Microsoft.AppConfiguration/configurationStores")
309+
echo "Deleting App Configuration: $name"
310+
az appconfig delete --name "$name" --yes || echo "Failed to delete App Configuration: $name"
311+
;;
312+
313+
"Microsoft.Insights/components")
314+
echo "Deleting Application Insights: $name"
315+
az monitor app-insights component delete --ids "$id" || echo "Failed to delete Application Insights: $name"
316+
;;
317+
318+
"Microsoft.Web/containerApps")
319+
echo "Deleting Container App: $name"
320+
az containerapp delete --name "$name" --yes || echo "Failed to delete Container App: $name"
321+
;;
322+
323+
*)
324+
echo "Purging General Resource: $name"
325+
if [[ -n "$id" && "$id" != "null" ]]; then
326+
az resource delete --ids "$id" --verbose || echo "Failed to delete $name"
327+
else
328+
echo "Resource ID not found for $name. Skipping purge."
329+
fi
330+
;;
331+
esac
332+
done
333+
334+
echo "Resource purging completed successfully"
335+
336+
- name: Send Notification on Failure
337+
if: failure()
338+
run: |
339+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
340+
341+
EMAIL_BODY=$(cat <<EOF
342+
{
343+
"body": "<p>Dear Team,</p><p>We would like to inform you that the Content Processing Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
344+
}
345+
EOF
346+
)
347+
348+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
349+
-H "Content-Type: application/json" \
350+
-d "$EMAIL_BODY" || echo "Failed to send notification"

0 commit comments

Comments
 (0)