[Response Ops][Reporting] Fixing error in calculating delay value between retries (elastic#245431)

## Summary

It was pointed out that there's an error in calculating the delay
between retries for scheduled reporting tasks.

Author: ymao1

x-pack/platform/plugins/private/reporting/server/lib/retry_on_error.test.ts

@@ -29,14 +29,16 @@ const report = new SavedReport({
 
 describe('retryOnError', () => {
   beforeEach(() => {
-    jest.resetAllMocks();
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+    jest.spyOn(global, 'setTimeout');
     jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier);
   });
 
   afterEach(() => {
-    jest.restoreAllMocks();
+    jest.clearAllTimers();
+    jest.useRealTimers();
   });
-
   it(`doesn't retry if operation is successful`, async () => {
     const operationMock = jest.fn().mockResolvedValue('success');
     expect(await retryOnError({ operation: operationMock, retries: 3, report, logger })).toEqual(
@@ -47,13 +49,24 @@ describe('retryOnError', () => {
     expect(logger.info).not.toHaveBeenCalled();
     expect(logger.warn).not.toHaveBeenCalled();
     expect(logger.error).not.toHaveBeenCalled();
+    expect(setTimeout).not.toHaveBeenCalled();
   });
 
   it('logs an error message on retry', async () => {
     const error = new Error('fail');
     const operationMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success');
 
-    await retryOnError({ operation: operationMock, retries: 3, report, logger });
+    const retryPromise = retryOnError({ operation: operationMock, retries: 3, report, logger });
+    await Promise.resolve();
+
+    jest.runAllTimers();
+    await retryPromise;
+
+    expect(setTimeout).toHaveBeenCalledTimes(1);
+    expect(setTimeout).toHaveBeenCalledWith(
+      expect.any(Function),
+      2000 * (1 + randomDelayMultiplier)
+    );
     expect(logger.error).toHaveBeenCalledTimes(1);
     expect(logger.error.mock.calls[0][0]).toEqual(
       `Retrying report generation for report[${
@@ -78,6 +91,7 @@ describe('retryOnError', () => {
     expect(logger.info).not.toHaveBeenCalled();
     expect(logger.warn).not.toHaveBeenCalled();
     expect(logger.error).not.toHaveBeenCalled();
+    expect(setTimeout).not.toHaveBeenCalled();
     expect(operationMock).toHaveBeenCalledTimes(1);
   });
 
@@ -93,6 +107,7 @@ describe('retryOnError', () => {
     expect(logger.info).not.toHaveBeenCalled();
     expect(logger.warn).not.toHaveBeenCalled();
     expect(logger.error).not.toHaveBeenCalled();
+    expect(setTimeout).not.toHaveBeenCalled();
     expect(operationMock).toHaveBeenCalledTimes(1);
   });
 
@@ -105,10 +120,28 @@ describe('retryOnError', () => {
       .mockRejectedValueOnce(error)
       .mockResolvedValue('success');
 
-    expect(await retryOnError({ operation: operationMock, retries: 3, report, logger })).toEqual(
-      'success'
-    );
+    const retryPromise = retryOnError({ operation: operationMock, retries: 3, report, logger });
+    await Promise.resolve();
+
+    jest.runAllTimersAsync().catch(() => {});
+    expect(await retryPromise).toEqual('success');
     // initial attempt + 3 retries
+    expect(setTimeout).toHaveBeenCalledTimes(3);
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      1,
+      expect.any(Function),
+      2000 * (1 + randomDelayMultiplier)
+    );
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      2,
+      expect.any(Function),
+      4000 * (1 + randomDelayMultiplier)
+    );
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      3,
+      expect.any(Function),
+      8000 * (1 + randomDelayMultiplier)
+    );
     expect(operationMock).toHaveBeenCalledTimes(4);
     expect(logger.error).toHaveBeenCalledTimes(3);
     expect(logger.error.mock.calls[0][0]).toEqual(
@@ -136,14 +169,30 @@ describe('retryOnError', () => {
     const error = new Error('fail');
     const operationMock = jest.fn().mockRejectedValue(error);
 
-    await expect(
-      retryOnError({ operation: operationMock, retries: 3, report, logger })
-    ).rejects.toThrowErrorMatchingInlineSnapshot(`"fail"`);
-    expect(logger.error).toHaveBeenCalledTimes(3);
+    const retryPromise = retryOnError({ operation: operationMock, retries: 3, report, logger });
+    await Promise.resolve();
+
+    jest.runAllTimersAsync().catch(() => {});
+    await expect(retryPromise).rejects.toThrowErrorMatchingInlineSnapshot(`"fail"`);
 
     // initial attempt + 3 retries
     expect(operationMock).toHaveBeenCalledTimes(4);
-
+    expect(setTimeout).toHaveBeenCalledTimes(3);
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      1,
+      expect.any(Function),
+      2000 * (1 + randomDelayMultiplier)
+    );
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      2,
+      expect.any(Function),
+      4000 * (1 + randomDelayMultiplier)
+    );
+    expect(setTimeout).toHaveBeenNthCalledWith(
+      3,
+      expect.any(Function),
+      8000 * (1 + randomDelayMultiplier)
+    );
     expect(logger.error.mock.calls[0][0]).toEqual(
       `Retrying report generation for report[${
         report._id

x-pack/platform/plugins/private/reporting/server/lib/retry_on_error.ts

@@ -58,7 +58,8 @@ export const retryOnError = async ({
       );
 
       // delay with some randomness
-      await delay(retryDelaySec + 1000 * Math.random());
+      // convert delay to milliseconds and multiply by random number between 1.0 and 2.0
+      await delay(retryDelaySec * 1000 * (1 + Math.random()));
       return retryOnError({ operation, logger, report, retries, attempt: retryCount });
     }