Merge pull request #1195 from ReliefApplications/beta

Beta

Author: AntoineRelief

__tests__/unit-tests/routes/activity/activity.service.spec.ts

@@ -216,13 +216,25 @@ describe('Activity Service', () => {
       const query = {};
       const aggregationResult = [];
       const listAggregationMock = jest.spyOn(service as any, 'listAggregation');
-      listAggregationMock.mockResolvedValueOnce(aggregationResult);
+      // Count
+      listAggregationMock.mockReturnValueOnce({
+        count: () => [{ count: 20000 }],
+      });
+      // 2 pages
+      listAggregationMock.mockReturnValueOnce({
+        skip: jest.fn().mockReturnThis(),
+        limit: jest.fn().mockResolvedValueOnce(aggregationResult),
+      });
+      listAggregationMock.mockReturnValueOnce({
+        skip: jest.fn().mockReturnThis(),
+        limit: jest.fn().mockResolvedValueOnce(aggregationResult),
+      });
 
       const mockFile = Buffer.from('Test content');
       jest.spyOn(XLSBuilder, 'default').mockResolvedValueOnce(mockFile as any);
 
       const result = await service.downloadList(query);
-      expect(listAggregationMock).toHaveBeenCalled();
+      expect(listAggregationMock).toHaveBeenCalledTimes(3); // 1 for count, 2 for pagination
       expect(result).toEqual({
         fileName: (service as any).listExportFileName,
         file: mockFile,
@@ -235,7 +247,11 @@ describe('Activity Service', () => {
 
       const listAggregationMock = jest
         .spyOn(service as any, 'listAggregation')
-        .mockRejectedValueOnce(error);
+        .mockReturnValueOnce({
+          count: () => {
+            throw error;
+          },
+        });
 
       await expect(service.downloadList(query)).rejects.toThrow(error);
       expect(listAggregationMock).toHaveBeenCalled();

package-lock.json

@@ -45,6 +45,7 @@
     "@ucast/mongo2js": "^1.3.3",
     "apollo-server-cache-redis": "^3.3.1",
     "axios": "^1.4.0",
+    "axios-cache-interceptor": "^1.6.2",
     "body-parser": "^1.20.1",
     "bson": "^4.2.3",
     "config": "^3.3.9",

package.json

@@ -352,18 +352,33 @@ export class ActivityService {
         filters.push(queryFilters);
       }
 
-      const aggregation = await this.listAggregation(
+      const totalCountAggregation = await this.listAggregation(
         sortField,
         sortOrder,
         filters
-      );
-      const formattedData = aggregation.map((activity) => ({
-        timestamp: this.formatDate(activity.createdAt, timeZone),
-        userId: activity.userId?.toString(),
-        page: activity.metadata?.title || '',
-        username: activity.username,
-        attributes: activity.attributes,
-      }));
+      ).count('count');
+
+      const formattedData = [];
+
+      for (let skip = 0; skip < totalCountAggregation[0].count; skip += 10000) {
+        const aggregation = await this.listAggregation(
+          sortField,
+          sortOrder,
+          filters
+        )
+          .skip(skip)
+          .limit(1000);
+        formattedData.push(
+          ...aggregation.map((activity) => ({
+            timestamp: this.formatDate(activity.createdAt, timeZone),
+            userId: activity.userId?.toString(),
+            page: activity.metadata?.title || '',
+            username: activity.username,
+            attributes: activity.attributes,
+          }))
+        );
+      }
+
       const file = await xlsBuilder(
         this.listExportFileName,
         this.listExportColumns,

src/routes/activity/activity.service.ts

@@ -0,0 +1,37 @@
+import Axios from 'axios';
+import {
+  AxiosCacheInstance,
+  setupCache,
+  buildKeyGenerator,
+} from 'axios-cache-interceptor/dev';
+
+let axios: AxiosCacheInstance;
+
+/**
+ * Create a dedicated axios instance for Common Services
+ * Add cache mechanism
+ * Cached requests expire after 5 minutes.
+ *
+ * @returns Common Services axios instance
+ */
+export default () => {
+  if (!axios) {
+    const instance = Axios.create();
+    axios = setupCache(instance, {
+      methods: ['get', 'post'],
+      // Avoid using cache control set to false
+      interpretHeader: false,
+      debug: console.log,
+      // Generate an unique key, based on all parameters listed
+      generateKey: buildKeyGenerator((request) => ({
+        method: request.method,
+        baseURL: request.baseURL,
+        params: request.params,
+        url: request.url,
+        data: request.data,
+        custom: request.headers.Authorization,
+      })),
+    });
+  }
+  return axios;
+};

src/server/common-services.ts

@@ -2,9 +2,11 @@ import { Context } from '../../server/apollo/context';
 import { CustomAPI } from '../../server/apollo/dataSources';
 import config from 'config';
 import { logger } from '@services/logger.service';
-import axios, { AxiosHeaders } from 'axios';
+import axios, { AxiosHeaders, AxiosStatic } from 'axios';
 import get from 'lodash/get';
 import jsonpath from 'jsonpath';
+import commonServices from '@server/common-services';
+import { AxiosCacheInstance } from 'axios-cache-interceptor';
 
 /**
  * Gets display text from choice value.
@@ -114,6 +116,7 @@ export const getFullChoices = async (
         }
       }
     } else if (field.choicesByGraphQL) {
+      let sender: AxiosCacheInstance | AxiosStatic = axios;
       const url: string = field.choicesByGraphQL.url;
       let choices: any[] = [];
       const valueField = get(field, 'choicesByGraphQL.value', null);
@@ -126,13 +129,14 @@ export const getFullChoices = async (
         url.includes(config.get('commonServices.url'))
       ) {
         headers.setAuthorization(`Bearer ${context.accesstoken}`);
+        sender = commonServices();
       } else {
         headers.setAuthorization(context.token);
         if (context.accesstoken) {
           headers.set('accesstoken', context.accesstoken);
         }
       }
-      await axios({
+      await sender({
         url,
         method: 'post',
         headers,

src/utils/form/getDisplayText.ts

@@ -1,9 +1,12 @@
 import fetch from 'node-fetch';
 import get from 'lodash/get';
 import jsonpath from 'jsonpath';
-import axios, { AxiosHeaders } from 'axios';
+import { AxiosHeaders, AxiosStatic } from 'axios';
 import config from 'config';
 import { Request } from 'express';
+import commonServices from '@server/common-services';
+import { AxiosCacheInstance } from 'axios-cache-interceptor';
+import axios from 'axios';
 
 /**
  * Fetches the choices for a question field by URL
@@ -63,11 +66,13 @@ export const getChoices = async (req: Request, field: any): Promise<any[]> => {
     const url = get(field, 'choicesByGraphQL.url', null);
     const valueField = get(field, 'choicesByGraphQL.value', null);
     const textField = get(field, 'choicesByGraphQL.text', null);
+    let sender: AxiosCacheInstance | AxiosStatic = axios;
     if (
       config.get('commonServices.url') &&
       url.includes(config.get('commonServices.url'))
     ) {
       headers.setAuthorization(`Bearer ${req.headers.accesstoken}`);
+      sender = commonServices();
     } else {
       headers.setAuthorization(req.headers.authorization);
       if (req.headers.accesstoken) {
@@ -76,7 +81,7 @@ export const getChoices = async (req: Request, field: any): Promise<any[]> => {
     }
     try {
       let choices: any[] = [];
-      await axios({
+      await sender({
         url,
         method: 'post',
         headers,