-
Notifications
You must be signed in to change notification settings - Fork 113
/
Copy pathAzureAppServiceZipDeployBehaviour.cs
421 lines (359 loc) · 26.5 KB
/
AzureAppServiceZipDeployBehaviour.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.ResourceManager;
using Azure.ResourceManager.AppService;
using Azure.ResourceManager.Resources;
using Calamari.Azure;
using Calamari.Azure.AppServices;
using Calamari.AzureAppService.Azure;
using Calamari.CloudAccounts;
using Calamari.Common.Commands;
using Calamari.Common.FeatureToggles;
using Calamari.Common.Plumbing.Extensions;
using Calamari.Common.Plumbing.FileSystem;
using Calamari.Common.Plumbing.Logging;
using Calamari.Common.Plumbing.Pipeline;
using Calamari.Common.Plumbing.Variables;
using Octopus.CoreUtilities.Extensions;
using Polly;
using Polly.Timeout;
using AccountVariables = Calamari.AzureAppService.Azure.AccountVariables;
namespace Calamari.AzureAppService.Behaviors
{
internal class AzureAppServiceZipDeployBehaviour : IDeployBehaviour
{
readonly ICalamariFileSystem fileSystem;
public AzureAppServiceZipDeployBehaviour(ILog log, ICalamariFileSystem fileSystem)
{
this.fileSystem = fileSystem;
Log = log;
}
private ILog Log { get; }
public bool IsEnabled(RunningDeployment context) => true;
public async Task Execute(RunningDeployment context)
{
Log.Verbose("Starting Azure App Service deployment.");
var variables = context.Variables;
var hasJwt = !variables.Get(AccountVariables.Jwt).IsNullOrEmpty();
var account = hasJwt ? (IAzureAccount)new AzureOidcAccount(variables) : new AzureServicePrincipalAccount(variables);
Log.Verbose($"Using Azure Tenant '{account.TenantId}'");
Log.Verbose($"Using Azure Subscription '{account.SubscriptionNumber}'");
Log.Verbose($"Using Azure ServicePrincipal AppId/ClientId '{account.ClientId}'");
Log.Verbose($"Using Azure Cloud '{account.AzureEnvironment}'");
var pollingTimeoutVariableValue = variables.Get(SpecialVariables.Action.Azure.AsyncZipDeploymentTimeout, "5");
int.TryParse(pollingTimeoutVariableValue, out var pollingTimeoutValue);
var pollingTimeout = TimeSpan.FromMinutes(pollingTimeoutValue);
var asyncZipDeployTimeoutPolicy = Policy.TimeoutAsync<HttpResponseMessage>(pollingTimeout, TimeoutStrategy.Optimistic);
string? resourceGroupName = variables.Get(SpecialVariables.Action.Azure.ResourceGroupName);
if (resourceGroupName == null)
throw new Exception("resource group name must be specified");
Log.Verbose($"Using Azure Resource Group '{resourceGroupName}'.");
string? webAppName = variables.Get(SpecialVariables.Action.Azure.WebAppName);
if (webAppName == null)
throw new Exception("Web App Name must be specified");
Log.Verbose($"Using App Service Name '{webAppName}'.");
string? slotName = variables.Get(SpecialVariables.Action.Azure.WebAppSlot);
Log.Verbose(slotName == null
? "No Deployment Slot specified"
: $"Using Deployment Slot '{slotName}'");
var armClient = account.CreateArmClient();
var targetSite = new AzureTargetSite(account.SubscriptionNumber, resourceGroupName, webAppName, slotName);
var resourceGroups = armClient
.GetSubscriptionResource(SubscriptionResource.CreateResourceIdentifier(targetSite.SubscriptionId))
.GetResourceGroups();
Log.Verbose($"Checking existence of Resource Group '{resourceGroupName}'.");
if (!await resourceGroups.ExistsAsync(resourceGroupName))
{
Log.Error($"Resource Group '{resourceGroupName}' could not be found. Either it does not exist, or the Azure Account in use may not have permissions to access it.");
throw new Exception("Resource Group not found.");
}
//get a reference to the resource group resource
//this does not actually load the resource group, but we can use it later
var resourceGroupResource = armClient.GetResourceGroupResource(ResourceGroupResource.CreateResourceIdentifier(targetSite.SubscriptionId, resourceGroupName));
Log.Verbose($"Resource Group '{resourceGroupName}' found.");
Log.Verbose($"Checking existence of App Service '{targetSite.Site}'.");
if (!await resourceGroupResource.GetWebSites().ExistsAsync(targetSite.Site))
{
Log.Error($"Azure App Service '{targetSite.Site}' could not be found in resource group '{resourceGroupName}'. Either it does not exist, or the Azure Account in use may not have permissions to access it.");
throw new Exception($"App Service not found.");
}
var webSiteResource = armClient.GetWebSiteResource(targetSite.CreateWebSiteResourceIdentifier());
Log.Verbose($"App Service '{targetSite.Site}' found, with Azure Resource Manager Id '{webSiteResource.Id.ToString()}'.");
var packageFileInfo = new FileInfo(variables.Get(TentacleVariables.CurrentDeployment.PackageFilePath)!);
IPackageProvider packageProvider = packageFileInfo.Extension switch
{
".zip" => new ZipPackageProvider(),
".nupkg" => new NugetPackageProvider(),
".war" => new JavaPackageProvider(Log, fileSystem, variables, context, "/api/wardeploy"),
".jar" => new JavaPackageProvider(Log, fileSystem, variables, context, "/api/publish?type=jar"),
_ => throw new Exception("Unsupported archive type")
};
// Let's process our archive while the slot is spun up. We will await it later before we try to upload to it.
Task<WebSiteSlotResource>? slotCreateTask = null;
if (targetSite.HasSlot)
slotCreateTask = FindOrCreateSlot(armClient, webSiteResource, targetSite);
string[]? substitutionFeatures =
{
KnownVariables.Features.ConfigurationTransforms,
KnownVariables.Features.StructuredConfigurationVariables,
KnownVariables.Features.SubstituteInFiles
};
/*
* Calamari default behaviors
* https://github.com/OctopusDeploy/Calamari/tree/main/source/Calamari.Common/Features/Behaviours
*/
var uploadPath = string.Empty;
bool uploadFileNeedsCleaning = false;
try
{
FileInfo? uploadFile;
if (substitutionFeatures.Any(featureName => context.Variables.IsFeatureEnabled(featureName)))
{
uploadFile = (await packageProvider.PackageArchive(context.StagingDirectory, context.CurrentDirectory));
}
else
{
uploadFile = await packageProvider.ConvertToAzureSupportedFile(packageFileInfo);
}
uploadPath = uploadFile.FullName;
uploadFileNeedsCleaning = packageFileInfo.Extension != uploadFile.Extension;
if (uploadPath == null)
{
throw new Exception("Package File Path must be specified");
}
// need to ensure slot is created as slot creds may be used
if (targetSite.HasSlot && slotCreateTask != null)
{
await slotCreateTask;
}
Log.Verbose($"Retrieving publishing profile for App Service to determine correct deployment endpoint.");
using var publishingProfileXmlStream = await armClient.GetPublishingProfileXmlWithSecrets(targetSite);
var publishingProfile = await PublishingProfile.ParseXml(publishingProfileXmlStream);
Log.Verbose($"Using deployment endpoint '{publishingProfile.PublishUrl}' from publishing profile.");
Log.Info($"Uploading package to {targetSite.SiteAndSlot}");
//Need to check if site turn off
var scmPublishEnabled = await armClient.IsScmPublishEnabled(targetSite);
if (packageProvider.SupportsAsynchronousDeployment && FeatureToggle.AsynchronousAzureZipDeployFeatureToggle.IsEnabled(context.Variables))
{
await UploadZipAndPollAsync(account, publishingProfile, scmPublishEnabled, uploadPath, targetSite.ScmSiteAndSlot, packageProvider, pollingTimeout, asyncZipDeployTimeoutPolicy);
}
else
{
await UploadZipAsync(account, publishingProfile, scmPublishEnabled, uploadPath, targetSite.ScmSiteAndSlot, packageProvider);
}
}
finally
{
if (uploadFileNeedsCleaning)
{
CleanupUploadFile(uploadPath);
}
}
}
private async Task<WebSiteSlotResource> FindOrCreateSlot(ArmClient armClient, WebSiteResource webSiteResource, AzureTargetSite site)
{
Log.Verbose($"Checking if deployment slot '{site.Slot}' exists.");
var slots = webSiteResource.GetWebSiteSlots();
if (await slots.ExistsAsync(site.Slot))
{
Log.Verbose($"Found existing slot {site.Slot}");
return armClient.GetWebSiteSlotResource(site.CreateResourceIdentifier());
}
Log.Verbose($"Slot '{site.Slot}' not found.");
Log.Info($"Creating slot '{site.Slot}'.");
var webSiteResourceData = (await webSiteResource.GetAsync()).Value.Data;
var operation = await slots.CreateOrUpdateAsync(WaitUntil.Completed,
site.Slot,
webSiteResourceData);
return operation.Value;
}
private async Task UploadZipAsync(IAzureAccount azureAccount,
PublishingProfile publishingProfile,
bool scmPublishEnabled,
string uploadZipPath,
string targetSite,
IPackageProvider packageProvider)
{
Log.Verbose($"Path to upload: {uploadZipPath}");
Log.Verbose($"Target Site: {targetSite}");
if (!new FileInfo(uploadZipPath).Exists)
throw new FileNotFoundException(uploadZipPath);
var zipUploadUrl = $"{publishingProfile.PublishUrl}{packageProvider.UploadUrlPath}";
Log.Verbose($@"Publishing {uploadZipPath} to {zipUploadUrl}");
var authenticationHeader = await GetAuthenticationHeaderValue(azureAccount, publishingProfile, scmPublishEnabled);
using var httpClient = new HttpClient(new HttpClientHandler
{
#pragma warning disable DE0003
Proxy = WebRequest.DefaultWebProxy
#pragma warning restore DE0003
})
{
// The HttpClient default timeout is 100 seconds: https://docs.microsoft.com/en-us/dotnet/api/system.net.http.httpclient.timeout?view=net-5.0#remarks
// This timeouts with even relatively small packages: https://octopus.zendesk.com/agent/tickets/69928
// We'll set this to an hour for now, but we should probably implement some more advanced retry logic, similar to https://github.com/OctopusDeploy/Sashimi.AzureWebApp/blob/bbea36152b2fb531c2893efedf0330a06ae0cef0/source/Calamari/AzureWebAppBehaviour.cs#L70
Timeout = TimeSpan.FromHours(1)
};
//we add some retry just in case the web app's Kudu/SCM is not running just yet
var response = await RetryPolicies.TransientHttpErrorsPolicy.ExecuteAsync(async () =>
{
#if NETFRAMEWORK
using var fileStream = new FileStream(uploadZipPath, FileMode.Open, FileAccess.Read, FileShare.Read);
#else
await using var fileStream = new FileStream(uploadZipPath, FileMode.Open, FileAccess.Read, FileShare.Read);
#endif
using var streamContent = new StreamContent(fileStream);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
//we have to create a new request message each time
var request = new HttpRequestMessage(HttpMethod.Post, zipUploadUrl)
{
Headers =
{
Authorization = authenticationHeader
},
Content = streamContent
};
var r = await httpClient.SendAsync(request);
r.EnsureSuccessStatusCode();
return r;
});
if (!response.IsSuccessStatusCode)
throw new Exception($"Zip upload to {zipUploadUrl} failed with HTTP Status {(int)response.StatusCode} '{response.ReasonPhrase}'.");
Log.Verbose("Finished deploying");
}
static async Task<AuthenticationHeaderValue> GetAuthenticationHeaderValue(IAzureAccount azureAccount, PublishingProfile publishingProfile, bool scmPublishEnabled)
{
AuthenticationHeaderValue authenticationHeader;
if (scmPublishEnabled)
{
authenticationHeader = new AuthenticationHeaderValue("Basic", publishingProfile.GetBasicAuthCredentials());
}
else
{
var accessToken = await azureAccount.GetAccessTokenAsync();
authenticationHeader = new AuthenticationHeaderValue("Bearer", accessToken);
}
return authenticationHeader;
}
private async Task UploadZipAndPollAsync(IAzureAccount azureAccount,
PublishingProfile publishingProfile,
bool scmPublishEnabled,
string uploadZipPath,
string targetSite,
IPackageProvider packageProvider,
TimeSpan pollingTimeout,
AsyncTimeoutPolicy<HttpResponseMessage> asyncZipDeployTimeoutPolicy)
{
Log.Verbose($"Path to upload: {uploadZipPath}");
Log.Verbose($"Target Site: {targetSite}");
if (!new FileInfo(uploadZipPath).Exists)
throw new FileNotFoundException(uploadZipPath);
var zipUploadUrl = $"{publishingProfile.PublishUrl}{packageProvider.UploadUrlPath}?isAsync=true";
Log.Verbose($"Publishing {uploadZipPath} to {zipUploadUrl} and checking for deployment");
using var httpClient = new HttpClient(new HttpClientHandler
{
#pragma warning disable DE0003
Proxy = WebRequest.DefaultWebProxy
#pragma warning restore DE0003
})
{
// Similar to the above increased timeout for sync uploads, we're increasing the timeout.
// In this case 10 minutes is selected over an hour as we're not waiting for the single request to complete.
// This is likely only relevant for the original upload request, the individual poll requests should always be quick.
Timeout = TimeSpan.FromMinutes(10)
};
var authenticationHeader = await GetAuthenticationHeaderValue(azureAccount, publishingProfile, scmPublishEnabled);
//we add some retry just in case the web app's Kudu/SCM is not running just yet
var uploadResponse = await RetryPolicies.TransientHttpErrorsPolicy.ExecuteAsync(async () =>
{
//we have to create a new request message each time
#if NETFRAMEWORK
using var fileStream = new FileStream(uploadZipPath, FileMode.Open, FileAccess.Read, FileShare.Read);
#else
await using var fileStream = new FileStream(uploadZipPath, FileMode.Open, FileAccess.Read, FileShare.Read);
#endif
using var streamContent = new StreamContent(fileStream);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var uploadRequest = new HttpRequestMessage(HttpMethod.Post, zipUploadUrl)
{
Headers =
{
Authorization = authenticationHeader
},
Content = streamContent
};
var r = await httpClient.SendAsync(uploadRequest);
r.EnsureSuccessStatusCode();
return r;
});
if (!uploadResponse.IsSuccessStatusCode)
throw new Exception($"Zip upload to {zipUploadUrl} failed with HTTP Status {(int)uploadResponse.StatusCode} '{uploadResponse.ReasonPhrase}'.");
Log.Verbose("Zip upload succeeded. Monitoring for deployment completion");
var location = uploadResponse.Headers.Location;
//wrap the entire thing in a Polly Timeout policy which uses the cancellation token to raise the timout
var result = await asyncZipDeployTimeoutPolicy.ExecuteAndCaptureAsync(async timeoutCancellationToken =>
{
//the outer policy should only retry when the response is a 202
return await RetryPolicies.AsynchronousZipDeploymentOperationPolicy
.ExecuteAsync(async (_, ct1) =>
//we nest this policy so any transient errors are handled and retried. If it just keeps falling over, then we want it to bail out of the outer operation
await RetryPolicies.TransientHttpErrorsPolicy
.ExecuteAsync(async ct2 =>
{
//we have to create a new request message each time
var checkRequest = new HttpRequestMessage(HttpMethod.Get, location)
{
Headers =
{
Authorization = authenticationHeader
}
};
var r = await httpClient.SendAsync(checkRequest, ct2);
r.EnsureSuccessStatusCode();
return r;
},
ct1),
//pass the logger so we can log the retries
new Dictionary<string, object>
{
[nameof(RetryPolicies.ContextKeys.Log)] = Log
},
timeoutCancellationToken);
},
CancellationToken.None);
if (result.Outcome == OutcomeType.Failure)
{
throw result.FinalException switch
{
OperationCanceledException oce => new Exception($"Zip deployment failed to complete after {pollingTimeout}.", oce),
_ => new Exception($"Zip deployment failed to complete.", result.FinalException)
};
}
if (!result.Result.IsSuccessStatusCode)
throw new Exception($"Zip deployment check failed with HTTP Status {(int)result.FinalHandledResult.StatusCode} '{result.FinalHandledResult.ReasonPhrase}'.");
Log.Verbose("Finished zip deployment");
}
static void CleanupUploadFile(string? uploadPath)
{
Policy.Handle<IOException>()
.WaitAndRetry(
5,
i => TimeSpan.FromMilliseconds(200))
.Execute(() =>
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath!);
}
});
}
}
}