Flex Deployment (#3404)

* Flex Deployment.

* Skipping the test as it is failing because of PPE.

* type

* Changed back the API verison.

* Not need to skip

* Removing the extra params which I added for private stamp testing.

* Removing the mitigation.

* logs

* More changes for the deployment.

* Minor changes

* Changes for 502 issue.

* 90 sec to 60 secs wait.

* logs url for flex

* Removed the comment.

* Removed the second check

* Removed WebsiteRunFromPackage validation.

Author: khkh-ms

src/Azure.Functions.Cli/Actions/AzureActions/PublishFunctionAppAction.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -10,6 +11,7 @@
 using Azure.Functions.Cli.Actions.LocalActions;
 using Azure.Functions.Cli.Arm.Models;
 using Azure.Functions.Cli.Common;
+using Azure.Functions.Cli.Diagnostics;
 using Azure.Functions.Cli.Extensions;
 using Azure.Functions.Cli.Helpers;
 using Azure.Functions.Cli.Interfaces;
@@ -18,6 +20,7 @@
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Newtonsoft.Json;
+using NuGet.Common;
 using static Azure.Functions.Cli.Common.OutputTheme;
 
 namespace Azure.Functions.Cli.Actions.AzureActions
@@ -105,6 +108,7 @@ public override ICommandLineParserResult ParseArgs(string[] args)
                 .Setup<string>("additional-packages")
                 .WithDescription("List of packages to install when building native dependencies. For example: \"python3-dev libevent-dev\"")
                 .Callback(p => AdditionalPackages = p);
+
             Parser
                 .Setup<bool>("force")
                 .WithDescription("Depending on the publish scenario, this will ignore pre-publish checks")
@@ -429,7 +433,7 @@ private async Task PublishFunctionApp(Site functionApp, GitIgnoreParser ignorePa
             var functionAppRoot = ScriptHostHelpers.GetFunctionAppRootDirectory(Environment.CurrentDirectory);
 
             // For dedicated linux apps, we do not support run from package right now
-            var isFunctionAppDedicatedLinux = functionApp.IsLinux && !functionApp.IsDynamic && !functionApp.IsElasticPremium;
+            var isFunctionAppDedicatedLinux = functionApp.IsLinux && !functionApp.IsDynamic && !functionApp.IsElasticPremium && !functionApp.IsFlex;
 
             if (GlobalCoreToolsSettings.CurrentWorkerRuntime == WorkerRuntime.python && !functionApp.IsLinux)
             {
@@ -442,6 +446,7 @@ private async Task PublishFunctionApp(Site functionApp, GitIgnoreParser ignorePa
                 ColoredConsole.WriteLine(WarningColor("Recommend using '--build remote' to resolve project dependencies remotely on Azure"));
             }
 
+            ColoredConsole.WriteLine(GetLogMessage("Starting the function app deployment..."));
             Func<Task<Stream>> zipStreamFactory = () => ZipHelper.GetAppZipFile(functionAppRoot, BuildNativeDeps, PublishBuildOption,
                 NoBuild, ignoreParser, AdditionalPackages, ignoreDotNetCheck: true);
 
@@ -457,6 +462,11 @@ private async Task PublishFunctionApp(Site functionApp, GitIgnoreParser ignorePa
                 // Consumption Linux
                 shouldSyncTriggers = await HandleLinuxConsumptionPublish(functionApp, zipStreamFactory);
             }
+            else if (functionApp.IsFlex)
+            {
+                // Flex
+                shouldSyncTriggers = await HandleFlexConsumptionPublish(functionApp, zipStreamFactory);
+            }
             else if (functionApp.IsLinux && functionApp.IsElasticPremium)
             {
                 // Elastic Premium Linux
@@ -496,8 +506,8 @@ private async Task PublishFunctionApp(Site functionApp, GitIgnoreParser ignorePa
             {
                 await PublishZipDeploy(functionApp, zipStreamFactory);
             }
-
-            if (shouldSyncTriggers)
+            
+            if (shouldSyncTriggers && !functionApp.IsFlex)
             {
                 await Task.Delay(TimeSpan.FromSeconds(5));
                 await SyncTriggers(functionApp);
@@ -518,7 +528,7 @@ private async Task SyncTriggers(Site functionApp)
         {
             await RetryHelper.Retry(async () =>
             {
-                ColoredConsole.WriteLine("Syncing triggers...");
+                ColoredConsole.WriteLine(GetLogMessage("Syncing triggers..."));
                 HttpResponseMessage response = null;
                 // This SyncTriggers function calls the endpoint for linux syncTriggers
                 response = await AzureHelper.SyncTriggers(functionApp, AccessToken, ManagementURL);
@@ -585,6 +595,7 @@ private async Task PerformAppServiceRemoteBuild(Func<Task<Stream>> zipStreamFact
                 functionApp.AzureAppSettings.Remove("WEBSITE_RUN_FROM_PACKAGE");
                 appSettingsUpdated = true;
             }
+            
             appSettingsUpdated = functionApp.AzureAppSettings.SafeLeftMerge(Constants.KuduLiteDeploymentConstants.LinuxDedicatedBuildSettings) || appSettingsUpdated;
             if (appSettingsUpdated)
             {
@@ -602,6 +613,85 @@ await WaitForAppSettingUpdateSCM(functionApp, shouldHaveSettings: functionApp.Az
             await PerformServerSideBuild(functionApp, zipStreamFactory, pollDedicatedBuild);
         }
 
+        /// <summary>
+        /// Handler for Linux Consumption publish event
+        /// </summary>
+        /// <param name="functionApp">Function App in Azure</param>
+        /// <param name="zipFileFactory">Factory for local project zipper</param>
+        /// <returns>ShouldSyncTrigger value</returns>
+        private async Task<bool> HandleFlexConsumptionPublish(Site functionApp, Func<Task<Stream>> zipFileFactory)
+        {
+            Task<DeployStatus> pollDeploymentStatusTask(HttpClient client) => KuduLiteDeploymentHelpers.WaitForFlexDeployment(client, functionApp);
+            var deploymentParameters = new Dictionary<string, string>();
+            
+            if (PublishBuildOption == BuildOption.Remote)
+            {
+                deploymentParameters.Add("RemoteBuild", true.ToString());
+            }
+
+            var deploymentStatus = await PerformFlexDeployment(functionApp, zipFileFactory, pollDeploymentStatusTask, deploymentParameters);
+
+            return deploymentStatus == DeployStatus.Success;
+        }
+
+        public async Task<DeployStatus> PerformFlexDeployment(Site functionApp, Func<Task<Stream>> zipFileFactory, Func<HttpClient, Task<DeployStatus>> deploymentStatusPollTask, IDictionary<string, string> deploymentParameters)
+        {
+            using (var handler = new ProgressMessageHandler(new HttpClientHandler()))
+            using (var client = GetRemoteZipClient(functionApp, handler))
+            using (var request = new HttpRequestMessage(HttpMethod.Post, new Uri(
+                $"api/Deploy/Zip?isAsync=true&author={Environment.MachineName}&Deployer=core_tools&{string.Join("&", deploymentParameters?.Select(kvp => $"{kvp.Key}={kvp.Value}")) ?? string.Empty}", UriKind.Relative)))
+            {
+                ColoredConsole.WriteLine(GetLogMessage("Creating archive for current directory..."));
+
+                request.Headers.IfMatch.Add(EntityTagHeaderValue.Any);
+
+                (var content, var length) = CreateStreamContentZip(await zipFileFactory());
+                request.Content = content;
+                HttpResponseMessage response = await PublishHelper.InvokeLongRunningRequest(client, handler, request, length, "Uploading");
+                await PublishHelper.CheckResponseStatusAsync(response, GetLogMessage("Uploading archive..."));
+
+                // Streaming deployment status for Linux Server Side Build
+                DeployStatus status = await deploymentStatusPollTask(client);
+
+                if (status == DeployStatus.Success)
+                {
+                    // the deployment was successful. Waiting for 60 seconds so that Kudu finishes the sync trigger.
+                    await Task.Delay(TimeSpan.FromSeconds(60));
+
+                    // Checking the function app host status
+                    try
+                    {
+                        await AzureHelper.CheckFunctionHostStatusForFlex(functionApp, AccessToken, ManagementURL);
+                    }
+                    catch (Exception)
+                    {
+                        throw new CliException("Deployment was successful but the app appears to be unhealthy, please check the app logs.");
+                    }
+                    
+                    ColoredConsole.WriteLine(VerboseColor(GetLogMessage("The deployment was successful!")));
+                }
+                else if (status == DeployStatus.Failed)
+                {
+                    throw new CliException("The deployment failed, Please check the printed logs.");
+                }
+                else if (status == DeployStatus.Conflict)
+                {
+                    throw new CliException("Deployment was cancelled, another deployment in progress.");
+                }
+                else if (status == DeployStatus.PartialSuccess)
+                {
+                    ColoredConsole.WriteLine(WarningColor(GetLogMessage("\"Deployment was partially successful, Please check the printed logs.")));
+                }
+                else if (status == DeployStatus.Unknown)
+                {
+                    ColoredConsole.WriteLine(WarningColor(GetLogMessage($"Failed to retrieve deployment status, please visit https://{functionApp.ScmUri}/api/deployments")));
+                }
+                
+                return status;
+            }
+        }
+
+
         /// <summary>
         /// Handler for Linux Consumption publish event
         /// </summary>
@@ -1069,6 +1159,7 @@ private static (StreamContent, long) CreateStreamContentZip(Stream zipFile)
         private HttpClient GetRemoteZipClient(Site functionApp, HttpMessageHandler handler = null)
         {
             handler = handler ?? new HttpClientHandler();
+
             var client = new HttpClient(handler)
             {
                 BaseAddress = new Uri($"https://{functionApp.ScmUri}"),
@@ -1109,6 +1200,16 @@ private static string NormalizeDotnetFrameworkVersion(string version)
             return $"{parsedVersion.Major}.{parsedVersion.Minor}";
         }
 
+        private string  GetLogMessage(string message)
+        {
+            return GetLogPrefix() + message;
+        }
+
+        private string GetLogPrefix()
+        {
+            return $"[{DateTime.UtcNow.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffZ", CultureInfo.InvariantCulture)}] ".ToString();
+        }
+
         // For testing
         internal class AzureHelperService
         {

src/Azure.Functions.Cli/Arm/Models/Site.cs

@@ -40,6 +40,9 @@ public bool IsLinux
         public bool IsDynamic
             => Sku?.Equals("dynamic", StringComparison.OrdinalIgnoreCase) == true;
 
+        public bool IsFlex
+            => Sku?.Equals("flexconsumption", StringComparison.OrdinalIgnoreCase) == true;
+
         public bool IsElasticPremium
             => Sku?.Equals("elasticpremium", StringComparison.OrdinalIgnoreCase) == true;
 

src/Azure.Functions.Cli/Common/DeploymentStatus.cs

@@ -11,6 +11,8 @@ public enum DeployStatus
         Building = 1,
         Deploying = 2,
         Failed = 3,
-        Success = 4
+        Success = 4,
+        Conflict = 5,
+        PartialSuccess = 6
     }
 }

src/Azure.Functions.Cli/Helpers/AzureHelper.cs

@@ -66,6 +66,7 @@ private static async Task<Site> TryGetFunctionAppFromArg(string name, IEnumerabl
             try
             {
                 string siteId = await GetResourceIDFromArg(subscriptions, query, accessToken, managementURL);
+
                 var app = new Site(siteId);
 
                 // The implementation of the load function is different for certain function apps like function apps based on Container Apps. 
@@ -122,6 +123,7 @@ internal static async Task<string> GetResourceIDFromArg(IEnumerable<string> subI
                 ?? throw new CliException("Error finding the Azure Resource information.");
         }
 
+
         internal static async Task<bool> IsBasicAuthAllowedForSCM(Site functionApp, string accessToken, string managementURL)
         {
             var url = new Uri($"{managementURL}{functionApp.SiteId}/basicPublishingCredentialsPolicies/scm?api-version={ArmUriTemplates.BasicAuthCheckApiVersion}");
@@ -362,6 +364,53 @@ public static Task<HttpResponseMessage> SyncTriggers(Site functionApp, string ac
             return ArmClient.HttpInvoke(HttpMethod.Post, url, accessToken);
         }
 
+        internal static async Task CheckFunctionHostStatusForFlex(Site functionApp, string accessToken, string managementURL,
+            HttpMessageHandler messageHandler = null)
+        {
+            ColoredConsole.Write("Checking the app health...");
+            var masterKey = await GetMasterKeyAsync(functionApp.SiteId, accessToken, managementURL);
+
+            if (masterKey is null)
+            {
+                throw new CliException($"The masterKey is null. hostname: {functionApp.HostName}.");
+            }
+
+            HttpMessageHandler handler = messageHandler ?? new HttpClientHandler();
+            if (StaticSettings.IsDebug)
+            {
+                handler = new LoggingHandler(handler);
+            }
+
+            var functionAppReadyClient = new HttpClient(handler);
+            const string jsonContentType = "application/json";
+            functionAppReadyClient.DefaultRequestHeaders.Add("User-Agent", Constants.CliUserAgent);
+            functionAppReadyClient.DefaultRequestHeaders.Add("Accept", jsonContentType);
+            
+            await RetryHelper.Retry(async () =>
+            {
+                functionAppReadyClient.DefaultRequestHeaders.Add("x-ms-request-id", Guid.NewGuid().ToString());
+                var uri = new Uri($"https://{functionApp.HostName}/admin/host/status?code={masterKey}");
+                var request = new HttpRequestMessage()
+                {
+                    RequestUri = uri,
+                    Method = HttpMethod.Get
+                };
+
+                var response = await functionAppReadyClient.SendAsync(request);
+                ColoredConsole.Write(".");
+
+                if (response.IsSuccessStatusCode)
+                {
+                    ColoredConsole.WriteLine(" done");
+                }
+                else
+                {
+                    throw new CliException($"The host didn't return success status. Returned: {response.StatusCode}");
+                }
+
+            }, 15, TimeSpan.FromSeconds(3));
+        }
+
         public static async Task<Site> LoadSiteObjectAsync(Site site, string accessToken, string managementURL)
         {
             var url = new Uri($"{managementURL}{site.SiteId}?api-version={ArmUriTemplates.WebsitesApiVersion}");
@@ -558,10 +607,10 @@ public static async Task PrintFunctionsInfo(Site functionApp, string accessToken
             }
 
             ArmArrayWrapper<FunctionInfo> functions = await GetFunctions(functionApp, accessToken, managementURL);
-
             functions = await GetFunctions(functionApp, accessToken, managementURL);
 
             ColoredConsole.WriteLine(TitleColor($"Functions in {functionApp.SiteName}:"));
+            
             if (functionApp.IsKubeApp && !functions.value.Any())
             {
                 ColoredConsole.WriteLine(WarningColor(