Fixes processing 429s from SPs (microsoft#2165)

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs

@@ -307,7 +307,7 @@ public async Task HardDeleteAsync(ResourceKey key, bool keepCurrentVersion, Canc
             }
             catch (CosmosException exception)
             {
-                if (exception.IsRequestEntityTooLarge())
+                if (exception.IsRequestRateExceeded())
                 {
                     throw;
                 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs

@@ -59,7 +59,8 @@ public Task ProcessErrorResponse(ResponseMessage response)
 
         public void ProcessErrorResponse(HttpStatusCode statusCode, Headers headers, string errorMessage)
         {
-            if (statusCode == HttpStatusCode.TooManyRequests)
+            // Stored procedure statuses will be in the substatuscode
+            if (statusCode == HttpStatusCode.TooManyRequests || headers.GetSubStatusValue() == (int)HttpStatusCode.TooManyRequests)
             {
                 string retryHeader = headers["x-ms-retry-after-ms"];
                 throw new RequestRateExceededException(int.TryParse(retryHeader, out int milliseconds) ? TimeSpan.FromMilliseconds(milliseconds) : null);

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js

@@ -1,6 +1,6 @@
 /**
-* This stored procedure acquires list of available export jobs. 
-* 
+* This stored procedure acquires list of available export jobs.
+*
 * @constructor
 * @param {string} maximumNumberOfConcurrentJobsAllowedInString - The maximum number of concurrent jobs allowed in string.
 * @param {string} jobHeartbeatTimeoutThresholdInSecondsInString - The number of seconds allowed before the job is considered to be stale in string.
@@ -145,6 +145,6 @@ function acquireExportJobs(maximumNumberOfConcurrentJobsAllowedInString, jobHear
     }
 
     function throwTooManyRequestsError() {
-        throw new Error(ErrorCodes.RequestEntityTooLarge, `The request could not be completed.`);
+        throw new Error(429, `The request could not be completed.`);
     }
 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js

@@ -1,6 +1,6 @@
 /**
-* This stored procedure acquires list of available reindex jobs. 
-* 
+* This stored procedure acquires list of available reindex jobs.
+*
 * @constructor
 * @param {string} maximumNumberOfConcurrentJobsAllowedInString - The maximum number of concurrent jobs allowed in string.
 * @param {string} jobHeartbeatTimeoutThresholdInSecondsInString - The number of seconds allowed before the job is considered to be stale in string.
@@ -145,6 +145,6 @@ function acquireReindexJobs(maximumNumberOfConcurrentJobsAllowedInString, jobHea
     }
 
     function throwTooManyRequestsError() {
-        throw new Error(ErrorCodes.RequestEntityTooLarge, `The request could not be completed.`);
+        throw new Error(429, `The request could not be completed.`);
     }
 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/hardDelete.js

@@ -96,6 +96,6 @@ function hardDelete(resourceTypeName, resourceId, keepCurrentVersion) {
     }
 
     function throwTooManyRequestsError() {
-        throw new Error(ErrorCodes.RequestEntityTooLarge, `The request could not be completed.`);
+        throw new Error(429, `The request could not be completed.`);
     }
 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js

@@ -1,6 +1,6 @@
 /**
 * This stored procedure can be used to replace an existing document
-* 
+*
 * @constructor
 * @param {any} doc - The CosmosResourceWrapper to save
 * @param {string} matchVersionId - required etag to match against when replacing an existing document
@@ -81,7 +81,7 @@ function replaceSingleResource(doc, matchVersionId) {
     }
 
     function throwRequestNotQueuedError() {
-        throw new Error(503, "Request could not be queued.");
+        throw new Error(429, "Request could not be queued.");
     }
 
     function throwPreconditionFailedError() {