Feature/parallel testing

components/api-server/src/routes/events.js

@@ -80,7 +80,10 @@ module.exports = async function (expressApp, app) {
         if (!hmacValid) { return next(errors.invalidAccessToken('Invalid read token.')); }
         next();
       })
-      .catch((err) => next(errors.unexpectedError(err)));
+      .catch((err) => {
+        if (err.id === 'invalid-access-token') return next(errors.invalidAccessToken('Invalid read token.'));
+        next(errors.unexpectedError(err));
+      });
     // The promise chain above calls next on all branches.
   }
   // Create an event.

components/api-server/test/accesses-personal.test.js

@@ -96,7 +96,7 @@ describe('accesses (personal)', function () {
             integrity.accesses.set(e);
           }
           for (const e of res.body.accesses) {
-            if (e.id === 'a_0') { e.lastUsed = 0; }
+            if (e.id === testData.accesses[0].id) { e.lastUsed = 0; }
           }
           validation.check(res, {
             status: 200,

components/api-server/test/events.test.js

@@ -526,7 +526,6 @@ describe('events', function () {
     it('[F29M] must return the attached file with the correct headers', function (done) {
       const event = testData.events[0];
       const attachment = event.attachments[0];
-
       request.get(path(event.id) + '/' + attachment.id).end(function (res) {
         res.statusCode.should.eql(200);
 
@@ -1326,7 +1325,7 @@ describe('events', function () {
 
     it('[8GSS] allows access at level=read', async () => {
       const request = supertest(server.url);
-      const access = _.find(testData.accesses, (v) => v.id === 'a_2');
+      const access = testData.accesses[2];
       const event = testData.events[0];
 
       const response = await request.get(path(event.id))

components/api-server/test/hooks.js

@@ -4,6 +4,7 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited
  * Proprietary and confidential
  */
+require('test-helpers/src/api-server-tests-config');
 const fs = require('fs');
 const { getConfig } = require('@pryv/boiler');
 const util = require('util');
@@ -21,7 +22,6 @@ async function initIndexPlatform () {
 exports.mochaHooks = {
   async beforeAll () {
     const config = await getConfig();
-
     // create preview directories that would normally be created in normal setup
     const attachmentsDirPath = config.get('eventFiles:attachmentsDirPath');
     const previewsDirPath = config.get('eventFiles:previewsDirPath');

components/api-server/test/permissions.test.js

@@ -186,10 +186,11 @@ describe('[ACCP] Access permissions', function () {
     // note: personal (i.e. full) access is implicitly covered by streams/events tests
 
     it('[BSFP] `get` must only return streams for which permissions are defined', function (done) {
+      const { runId } = require('test-helpers/src/runid');
       request.get(basePath, token(1)).query({ state: 'all' }).end(async function (res) {
         const expectedStreamids = [testData.streams[0].id, testData.streams[1].id, testData.streams[2].children[0].id];
         if (isAuditActive) {
-          expectedStreamids.push(':_audit:access-a_1');
+          expectedStreamids.push(':_audit:access-a_1-' + runId);
         }
         assert.exists(res.body.streams);
         res.body.streams.length.should.eql(expectedStreamids.length);

components/api-server/test/sockets.test.js

@@ -247,7 +247,7 @@ describe('Socket.IO', function () {
 
     it('[TO6Z] must accept streamQuery as Javascript Object', function (done) {
       ioCons.con = connect(namespace, { auth: token });
-      ioCons.con.emit('events.get', { streams: { any: ['s_0_1'], all: ['s_8'] } }, function (err, res) {
+      ioCons.con.emit('events.get', { streams: { any: [testData.streams[0].children[1].id], all: [testData.streams[8].id] } }, function (err, res) {
         should(err).be.null();
         should(res.events).not.be.null();
         done();

components/api-server/test/system.test.js

@@ -149,6 +149,7 @@ describe('[SYER] system (ex-register)', function () {
       return basePath() + '/create-user';
     }
     function post (data, callback) {
+      testData.addUserToBeErased(data.username);
       return request.post(path())
         .set('authorization', helpers.dependencies.settings.auth.adminAccessKey)
         .send(data)

components/business/src/users/repository.js

@@ -340,7 +340,7 @@ class UsersRepository {
    * @param {boolean | null} skipFowardToRegister
    * @returns {Promise<number>}
    */
-  async deleteOne (userId, username, skipFowardToRegister) {
+  async deleteOne (userId, username = null, skipFowardToRegister = false) {
     const user = await this.getUserById(userId);
     if (username == null) {
       username = user?.username;

components/platform/src/DB.js

@@ -21,9 +21,12 @@ class DB {
     mkdirp.sync(basePath);
 
     this.db = new SQLite3(basePath + '/platform-wide.db');
-    this.db.pragma('journal_mode = WAL');
-
-    this.db.prepare('CREATE TABLE IF NOT EXISTS keyValue (key TEXT PRIMARY KEY, value TEXT NOT NULL);').run();
+    await concurentSafeWriteStatement(() => {
+      this.db.pragma('journal_mode = WAL');
+    });
+    await concurentSafeWriteStatement(() => {
+      this.db.prepare('CREATE TABLE IF NOT EXISTS keyValue (key TEXT PRIMARY KEY, value TEXT NOT NULL);').run();
+    });
 
     this.queries = {};
     this.queries.getValueWithKey = this.db.prepare('SELECT key, value FROM keyValue WHERE key = ?');
@@ -109,6 +112,28 @@ class DB {
   }
 }
 
+const WAIT_LIST_MS = [1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100];
+/**
+   * Will look "retries" times, in case of "SQLITE_BUSY".
+   * This is CPU intensive, but tests have shown this solution to be efficient
+   */
+async function concurentSafeWriteStatement (statement, retries = 100) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      statement();
+      return;
+    } catch (error) {
+      if (error.code !== 'SQLITE_BUSY') { // ignore
+        throw error;
+      }
+      const waitTime = i > (WAIT_LIST_MS.length - 1) ? 100 : WAIT_LIST_MS[i];
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+      this.logger.debug('SQLITE_BUSY, retrying in ' + waitTime + 'ms');
+    }
+  }
+  throw new Error('Failed write action on Audit after ' + retries + ' retries');
+}
+
 /**
  * Return an object from an entry in the table
  * @param {Entry} entry

components/test-helpers/src/data.js

@@ -25,18 +25,33 @@ const { getConfigUnsafe, getConfig, getLogger } = require('@pryv/boiler');
 const { getMall } = require('mall');
 const logger = getLogger('test-helpers:data');
 
+const { runId } = require('./runid');
+
 // users
 
 const users = (exports.users = require('./data/users'));
 const defaultUser = users[0];
 
+const userNamesToErase = [];
+exports.addUserToBeErased = function (userId) {
+  userNamesToErase.push(userId);
+};
+
 exports.resetUsers = async () => {
   logger.debug('resetUsers');
   await getConfig(); // lock up to the time config is ready
   await SystemStreamsSerializer.init();
   const customAccountProperties = buildCustomAccountProperties();
   const usersRepository = await getUsersRepository();
-  await usersRepository.deleteAll();
+  for (const user of users) {
+    await usersRepository.deleteOne(user.id);
+  }
+  while (userNamesToErase.length > 0) {
+    const userName = userNamesToErase.pop();
+    const userId = await usersRepository.getUserIdForUsername(userName);
+    if (userId != null) await usersRepository.deleteOne(userId);
+  }
+  // await usersRepository.deleteAll();
   for (const user of users) {
     const userObj = new User(_.merge(customAccountProperties, user)); // might alter storage "dump data" script
     await usersRepository.insertOne(userObj, false, true);
@@ -160,12 +175,12 @@ function resetData (storage, user, items, done) {
 const attachmentsDirPath = (exports.attachmentsDirPath = path.join(__dirname, '/data/attachments/'));
 
 const attachments = (exports.attachments = {
-  animatedGif: getAttachmentInfo('animatedGif', 'animated.gif', 'image/gif'),
-  document: getAttachmentInfo('document', 'document.pdf', 'application/pdf'),
-  document_modified: getAttachmentInfo('document', 'document.modified.pdf', 'application/pdf'),
-  image: getAttachmentInfo('image', 'image (space and special chars)é__.png', 'image/png'),
-  imageBigger: getAttachmentInfo('imageBigger', 'image-bigger.jpg', 'image/jpeg'),
-  text: getAttachmentInfo('text', 'text.txt', 'text/plain')
+  animatedGif: getAttachmentInfo('animatedGif-' + runId, 'animated.gif', 'image/gif'),
+  document: getAttachmentInfo('document-' + runId, 'document.pdf', 'application/pdf'),
+  document_modified: getAttachmentInfo('document-' + runId, 'document.modified.pdf', 'application/pdf'),
+  image: getAttachmentInfo('image-' + runId, 'image (space and special chars)é__.png', 'image/png'),
+  imageBigger: getAttachmentInfo('imageBigger-' + runId, 'image-bigger.jpg', 'image/jpeg'),
+  text: getAttachmentInfo('text-' + runId, 'text.txt', 'text/plain')
 });
 
 // following https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity

components/test-helpers/src/data/accesses.js

@@ -6,8 +6,8 @@
  */
 const streams = require('./streams');
 const timestamp = require('unix-timestamp');
-
-module.exports = [
+const { runIdMap } = require('../runid');
+module.exports = runIdMap([
   {
     id: 'a_0',
     token: 'a_0_token',
@@ -164,4 +164,4 @@ module.exports = [
     calls: {},
     deleted: timestamp.now('-1m')
   } */
-];
+]);

components/test-helpers/src/data/events.js

@@ -8,7 +8,7 @@ const streams = require('./streams');
 const timestamp = require('unix-timestamp');
 const { TAG_PREFIX } = require('api-server/src/methods/helpers/backwardCompatibility');
 const { integrity } = require('business');
-
+const { runId } = require('../runid');
 const events = [
   {
     id: getTestEventId(0),
@@ -20,13 +20,13 @@ const events = [
     description: 'First period event, with attachments',
     attachments: [
       {
-        id: 'document',
+        id: 'document-' + runId,
         fileName: 'document.pdf',
         type: 'application/pdf',
         size: 6701
       },
       {
-        id: 'image',
+        id: 'image-' + runId,
         fileName: 'image (space and special chars)é__.png',
         type: 'image/png',
         size: 2765
@@ -64,7 +64,7 @@ const events = [
     description: '陳容龍',
     attachments: [
       {
-        id: 'imageBigger',
+        id: 'imageBigger-' + runId,
         fileName: 'image-bigger.jpg',
         type: 'image/jpeg',
         size: 177476
@@ -196,7 +196,7 @@ const events = [
     tags: [],
     attachments: [
       {
-        id: 'animatedGif',
+        id: 'animatedGif-' + runId,
         fileName: 'animated.gif',
         type: 'image/gif',
         size: 88134
@@ -429,5 +429,5 @@ module.exports = events;
  */
 function getTestEventId (n) {
   n = n + '';
-  return 'cthisistesteventno' + (n.length >= 7 ? n : new Array(7 - n.length + 1).join('0') + n);
+  return 'cth' + runId + 'testeventno' + (n.length >= 7 ? n : new Array(7 - n.length + 1).join('0') + n);
 }

components/test-helpers/src/data/followedSlices.js

@@ -5,9 +5,9 @@
  * Proprietary and confidential
  */
 const accesses = require('./accesses');
-
+const { runIdMap } = require('../runid');
 module.exports = function (url) {
-  return [
+  return runIdMap([
     {
       id: 'b_0',
       name: 'Zero\'s First Access',
@@ -26,5 +26,5 @@ module.exports = function (url) {
       url,
       accessToken: accesses[3].token
     }
-  ];
+  ]);
 };

components/test-helpers/src/data/streams.js

@@ -6,8 +6,8 @@
  */
 const timestamp = require('unix-timestamp');
 const { TAG_ROOT_STREAMID, TAG_PREFIX } = require('api-server/src/methods/helpers/backwardCompatibility');
-
-module.exports = [
+const { runIdStreamTree } = require('../runid');
+module.exports = runIdStreamTree([
   {
     id: 's_0',
     name: 'Root Stream 0',
@@ -248,4 +248,4 @@ module.exports = [
       }
     ]
   }
-];
+]);

components/test-helpers/src/data/users.js

@@ -4,8 +4,9 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited
  * Proprietary and confidential
  */
+const { runIdMap } = require('../runid');
 
-module.exports = [
+const events = [
   {
     id: 'u_0',
     username: 'userzero',
@@ -57,3 +58,7 @@ module.exports = [
     language: 'en',
   }, */
 ];
+
+runIdMap(events);
+runIdMap(events, 'email', '-', true);
+module.exports = events;

components/test-helpers/src/runid.js

@@ -0,0 +1,52 @@
+/**
+ * @license
+ * Copyright (C) 2012–2023 Pryv S.A. https://pryv.com - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited
+ * Proprietary and confidential
+ */
+
+// 4 letters code to happend to all ids to run in parallel
+const runId = (Math.random() + 1).toString(36).substring(8);
+exports.runId = runId;
+
+/**
+ * Transform an array by adding runId to items[property]
+ * @param {Array} array
+ * @param {string} [property='id'] the property to change (default: 'id')
+ * @param {string} [space='-'] spacer between original property value and runId (default '-')
+ * @param {boolean} [head=false] if true happend at the begging, false at the end (default false)
+ * @returns array
+ */
+exports.runIdMap = function runIdMap (array, property = 'id', space = '-', head = false) {
+  for (const item of array) {
+    if (head) {
+      item[property] = runId + space + item[property];
+    } else {
+      item[property] += space + runId;
+    }
+  }
+  return array;
+};
+
+/**
+ * Transform a tree by adding runId to items[property]
+ * @param {Array} array
+ * @param {string} [property='id'] the property to change (default: 'id')
+ * @param {string} [childrenProperty='children'] the property to find children of item (default: 'children')
+ * @param {string} [space='-'] spacer between original property value and runId (default '-')
+ * @returns array
+ */
+function runIdTree (array, property = 'id', childrenProperty = 'children', space = '-') {
+  for (const item of array) {
+    if (item[property] != null) item[property] += space + runId;
+    if (item[childrenProperty] != null) runIdTree(item[childrenProperty], property, childrenProperty, space);
+  }
+  return array;
+}
+exports.runIdTree = runIdTree;
+
+exports.runIdStreamTree = function runIdStreamTree (array) {
+  runIdTree(array);
+  runIdTree(array, 'parentId');
+  return array;
+};

components/utils/src/encryption.js

@@ -49,7 +49,7 @@ exports.fileReadToken = function (fileId, accessId, accessToken, secret) {
  * @returns {Object} Contains `accessId` and `hmac` parts if successful; empty otherwise.
  */
 exports.parseFileReadToken = function (fileReadToken) {
-  const sepIndex = fileReadToken.indexOf('-');
+  const sepIndex = fileReadToken.lastIndexOf('-'); // take the lastIndexOf as "-" might appear un the accesId.
   if (sepIndex <= 0) {
     return {};
   }
@@ -74,6 +74,6 @@ function getFileHMAC (fileId, token, secret) {
   return base64HMAC
     .toString() // function signature says we might have a buffer here.
     .replace(/\//g, '_')
-    .replace(/\+/g, '-')
+    .replace(/\+/g, '_') // don't use '-' as it will be the seprator with the accesId
     .replace(/=/g, '');
 }