chunk level retry for file container upload. (#509)

Author: TingluoHuang

src/Agent.Listener/_project.json

@@ -34,7 +34,7 @@
             "version": "1.0.0-*"
         },
         "Microsoft.Win32.Registry": "4.0.0",
-        "vss-api-netcore": "0.5.24-private",
+        "vss-api-netcore": "0.5.25-private",
         "System.ServiceProcess.ServiceController": "4.1.0",
         "System.IO.Compression.ZipFile": "4.0.1",
         "System.IO.FileSystem.AccessControl": "4.0.0",

src/Agent.Worker/Build/FileContainerServer.cs

@@ -10,6 +10,7 @@
 using System.Diagnostics;
 using Microsoft.VisualStudio.Services.WebApi;
 using System.Net.Http;
+using System.Net;
 
 namespace Microsoft.VisualStudio.Services.Agent.Worker.Build
 {
@@ -183,8 +184,6 @@ private async Task<List<string>> UploadAsync(IAsyncCommandContext context, int u
             while (_fileUploadQueue.TryDequeue(out fileToUpload))
             {
                 token.ThrowIfCancellationRequested();
-                Interlocked.Increment(ref _filesProcessed);
-
                 using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read))
                 {
                     string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
@@ -198,6 +197,12 @@ private async Task<List<string>> UploadAsync(IAsyncCommandContext context, int u
                     catch (OperationCanceledException) when (token.IsCancellationRequested)
                     {
                         context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload));
+                        if (response != null)
+                        {
+                            response.Dispose();
+                            response = null;
+                        }
+
                         throw;
                     }
                     catch (Exception ex)
@@ -208,7 +213,7 @@ private async Task<List<string>> UploadAsync(IAsyncCommandContext context, int u
                     }
 
                     uploadTimer.Stop();
-                    if (catchExceptionDuringUpload || response.StatusCode != System.Net.HttpStatusCode.Created)
+                    if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created))
                     {
                         if (response != null)
                         {
@@ -246,7 +251,15 @@ private async Task<List<string>> UploadAsync(IAsyncCommandContext context, int u
                             }
                         }
                     }
+
+                    if (response != null)
+                    {
+                        response.Dispose();
+                        response = null;
+                    }
                 }
+
+                Interlocked.Increment(ref _filesProcessed);
             }
 
             return failedFiles;

src/Agent.Worker/_project.json

@@ -33,7 +33,7 @@
             "target": "project",
             "version": "1.0.0-*"
         },
-        "vss-api-netcore": "0.5.24-private",
+        "vss-api-netcore": "0.5.25-private",
         "System.Collections.NonGeneric": "4.0.1",
         "System.Diagnostics.TraceSource": "4.0.0",
         "System.Linq": "4.1.0",

src/Microsoft.VisualStudio.Services.Agent/Constants.cs

@@ -38,7 +38,7 @@ public enum OSPlatform
 
         public static class Agent
         {
-            public static readonly string Version = "2.104.2";
+            public static readonly string Version = "2.105.0";
 
 #if OS_LINUX
             public static readonly OSPlatform Platform = OSPlatform.Linux;

src/Microsoft.VisualStudio.Services.Agent/_project.json

@@ -26,7 +26,7 @@
     },
     "dependencies": {
         "NETStandard.Library": "1.6.0",
-        "vss-api-netcore": "0.5.24-private",
+        "vss-api-netcore": "0.5.25-private",
         "Microsoft.Win32.Registry": "4.0.0",
         "Newtonsoft.Json": "8.0.3",
         "System.Diagnostics.TraceSource": "4.0.0",

src/Test/_project.json

@@ -30,7 +30,7 @@
         "System.Buffers": "4.0.0",
         "System.Reflection.TypeExtensions": "4.1.0",
         "System.Threading.ThreadPool": "4.0.10",
-        "vss-api-netcore": "0.5.24-private",
+        "vss-api-netcore": "0.5.25-private",
         "moq.netcore": "4.4.0-beta8",
         "xunit": "2.1.0",
         "xunit.console.netcore": "1.0.3-prerelease-00520-02",