[release/8.0.4xx] Containers - Retry on download blob

src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs

@@ -215,6 +215,11 @@ private static async Task<int> PushToLocalRegistryAsync(ILogger logger, BuiltIma
             await containerRegistry.LoadAsync(builtImage, sourceImageReference, destinationImageReference, cancellationToken).ConfigureAwait(false);
             logger.LogInformation(Strings.ContainerBuilder_ImageUploadedToLocalDaemon, destinationImageReference, containerRegistry);
         }
+        catch (UnableToDownloadFromRepositoryException)
+        {
+            logger.LogError(Resource.FormatString(nameof(Strings.UnableToDownloadFromRepository)), sourceImageReference);
+            return 1;
+        }
         catch (Exception ex)
         {
             logger.LogError(Resource.FormatString(nameof(Strings.RegistryOutputPushFailed), ex.Message));

src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.NET.Build.Containers;
+
+internal sealed class UnableToDownloadFromRepositoryException : Exception
+{
+    public UnableToDownloadFromRepositoryException(string repository)
+        : base($"The download of the image from repository { repository } has failed.")
+    {
+    }
+}

src/Containers/Microsoft.NET.Build.Containers/ImagePublisher.cs

@@ -107,6 +107,10 @@ private static async Task PushToLocalRegistryAsync<T>(
             await loadFunc(image, sourceImageReference, destinationImageReference, cancellationToken).ConfigureAwait(false);
             Log.LogMessage(MessageImportance.High, Strings.ContainerBuilder_ImageUploadedToLocalDaemon, destinationImageReference, localRegistry);
         }
+        catch (UnableToDownloadFromRepositoryException)
+        {
+            Log.LogErrorWithCodeFromResources(nameof(Strings.UnableToDownloadFromRepository), sourceImageReference);
+        }
         catch (ContainerHttpException e)
         {
             Log.LogErrorFromException(e, true);

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -74,6 +74,8 @@ internal sealed class Registry
     private const string DockerHubRegistry1 = "registry-1.docker.io";
     private const string DockerHubRegistry2 = "registry.hub.docker.com";
     private static readonly int s_defaultChunkSizeBytes = 1024 * 64;
+    private const int MaxDownloadRetries = 5;
+    private readonly Func<TimeSpan> _retryDelayProvider;
 
     private readonly ILogger _logger;
     private readonly IRegistryAPI _registryAPI;
@@ -86,7 +88,7 @@ internal sealed class Registry
     /// </summary>
     public string RegistryName { get; }
 
-    internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) :
+    internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null) :
         this(new Uri($"https://{registryName}"), logger, registryAPI, settings)
     { }
 
@@ -95,15 +97,15 @@ internal Registry(string registryName, ILogger logger, RegistryMode mode, Regist
     { }
 
 
-    internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) :
+    internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null) :
         this(baseUri, logger, new RegistryApiFactory(registryAPI), settings)
     { }
 
     internal Registry(Uri baseUri, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) :
         this(baseUri, logger, new RegistryApiFactory(mode), settings)
     { }
 
-    private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null)
+    private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null)
     {
         RegistryName = DeriveRegistryName(baseUri);
 
@@ -117,6 +119,8 @@ private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, Regist
         _logger = logger;
         _settings = settings ?? new RegistrySettings(RegistryName);
         _registryAPI = factory.Create(RegistryName, BaseUri, logger, _settings.IsInsecure);
+
+        _retryDelayProvider = retryDelayProvider ?? (() => TimeSpan.FromSeconds(1));
     }
 
     private static string DeriveRegistryName(Uri baseUri)
@@ -401,26 +405,48 @@ public async Task<string> DownloadBlobAsync(string repository, Descriptor descri
     {
         cancellationToken.ThrowIfCancellationRequested();
         string localPath = ContentStore.PathForDescriptor(descriptor);
-
+    
         if (File.Exists(localPath))
         {
             // Assume file is up to date and just return it
             return localPath;
         }
-
-        // No local copy, so download one
-        using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false);
-
+
         string tempTarballPath = ContentStore.GetTempFile();
-        using (FileStream fs = File.Create(tempTarballPath))
+
+        int retryCount = 0;
+        while (retryCount < MaxDownloadRetries)
         {
-            await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+            try
+            {
+                // No local copy, so download one
+                using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false);
+
+                using (FileStream fs = File.Create(tempTarballPath))
+                {
+                    await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+                }
+
+                // Break the loop if successful
+                break;
+            }
+            catch (Exception ex)
+            {
+                retryCount++;
+                if (retryCount >= MaxDownloadRetries)
+                {
+                    throw new UnableToDownloadFromRepositoryException(repository);
+                }
+
+                _logger.LogTrace("Download attempt {0}/{1} for repository '{2}' failed. Error: {3}", retryCount, MaxDownloadRetries, repository, ex.ToString());
+
+                // Wait before retrying
+                await Task.Delay(_retryDelayProvider(), cancellationToken).ConfigureAwait(false);
+            }
         }
-
-        cancellationToken.ThrowIfCancellationRequested();
-
+
         File.Move(tempTarballPath, localPath, overwrite: true);
-
+    
         return localPath;
     }
 

src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.resx

@@ -461,6 +461,10 @@
     <value>CONTAINER1015: Unable to access the repository '{0}' at tag '{1}' in the registry '{2}'. Please confirm that this name and tag are present in the registry.</value>
     <comment>{StrBegin="CONTAINER1015: "}</comment>
   </data>
+  <data name="UnableToDownloadFromRepository" xml:space="preserve">
+    <value>CONTAINER1018: Unable to download image from the repository '{0}'.</value>
+    <comment>{StrBegins="CONTAINER1018:" }</comment>
+  </data>
   <data name="UnableToAccessRepository" xml:space="preserve">
     <value>CONTAINER1016: Unable to access the repository '{0}' in the registry '{1}'. Please confirm your credentials are correct and that you have access to this repository and registry.</value>
     <comment>{StrBegin="CONTAINER1016:" }</comment>