diff --git a/components/api-server/src/routes/events.js b/components/api-server/src/routes/events.js index 11620eebf..c3cc996ca 100644 --- a/components/api-server/src/routes/events.js +++ b/components/api-server/src/routes/events.js @@ -80,7 +80,10 @@ module.exports = async function (expressApp, app) { if (!hmacValid) { return next(errors.invalidAccessToken('Invalid read token.')); } next(); }) - .catch((err) => next(errors.unexpectedError(err))); + .catch((err) => { + if (err.id === 'invalid-access-token') return next(errors.invalidAccessToken('Invalid read token.')); + next(errors.unexpectedError(err)); + }); // The promise chain above calls next on all branches. } // Create an event. diff --git a/components/api-server/test/accesses-personal.test.js b/components/api-server/test/accesses-personal.test.js index 7bc86542e..f6224d7e5 100644 --- a/components/api-server/test/accesses-personal.test.js +++ b/components/api-server/test/accesses-personal.test.js @@ -96,7 +96,7 @@ describe('accesses (personal)', function () { integrity.accesses.set(e); } for (const e of res.body.accesses) { - if (e.id === 'a_0') { e.lastUsed = 0; } + if (e.id === testData.accesses[0].id) { e.lastUsed = 0; } } validation.check(res, { status: 200, diff --git a/components/api-server/test/events.test.js b/components/api-server/test/events.test.js index 74079fe36..649772db7 100644 --- a/components/api-server/test/events.test.js +++ b/components/api-server/test/events.test.js @@ -526,7 +526,6 @@ describe('events', function () { it('[F29M] must return the attached file with the correct headers', function (done) { const event = testData.events[0]; const attachment = event.attachments[0]; - request.get(path(event.id) + '/' + attachment.id).end(function (res) { res.statusCode.should.eql(200); @@ -1326,7 +1325,7 @@ describe('events', function () { it('[8GSS] allows access at level=read', async () => { const request = supertest(server.url); - const access = _.find(testData.accesses, (v) => v.id === 'a_2'); + const access = testData.accesses[2]; const event = testData.events[0]; const response = await request.get(path(event.id)) diff --git a/components/api-server/test/hooks.js b/components/api-server/test/hooks.js index 7ce1c7fcf..9c026b3a0 100644 --- a/components/api-server/test/hooks.js +++ b/components/api-server/test/hooks.js @@ -4,6 +4,7 @@ * Unauthorized copying of this file, via any medium is strictly prohibited * Proprietary and confidential */ +require('test-helpers/src/api-server-tests-config'); const fs = require('fs'); const { getConfig } = require('@pryv/boiler'); const util = require('util'); @@ -21,7 +22,6 @@ async function initIndexPlatform () { exports.mochaHooks = { async beforeAll () { const config = await getConfig(); - // create preview directories that would normally be created in normal setup const attachmentsDirPath = config.get('eventFiles:attachmentsDirPath'); const previewsDirPath = config.get('eventFiles:previewsDirPath'); diff --git a/components/api-server/test/permissions.test.js b/components/api-server/test/permissions.test.js index 4955d760d..87bd9da99 100644 --- a/components/api-server/test/permissions.test.js +++ b/components/api-server/test/permissions.test.js @@ -186,10 +186,11 @@ describe('[ACCP] Access permissions', function () { // note: personal (i.e. full) access is implicitly covered by streams/events tests it('[BSFP] `get` must only return streams for which permissions are defined', function (done) { + const { runId } = require('test-helpers/src/runid'); request.get(basePath, token(1)).query({ state: 'all' }).end(async function (res) { const expectedStreamids = [testData.streams[0].id, testData.streams[1].id, testData.streams[2].children[0].id]; if (isAuditActive) { - expectedStreamids.push(':_audit:access-a_1'); + expectedStreamids.push(':_audit:access-a_1-' + runId); } assert.exists(res.body.streams); res.body.streams.length.should.eql(expectedStreamids.length); diff --git a/components/api-server/test/sockets.test.js b/components/api-server/test/sockets.test.js index 9e42bb592..c2a7301df 100644 --- a/components/api-server/test/sockets.test.js +++ b/components/api-server/test/sockets.test.js @@ -247,7 +247,7 @@ describe('Socket.IO', function () { it('[TO6Z] must accept streamQuery as Javascript Object', function (done) { ioCons.con = connect(namespace, { auth: token }); - ioCons.con.emit('events.get', { streams: { any: ['s_0_1'], all: ['s_8'] } }, function (err, res) { + ioCons.con.emit('events.get', { streams: { any: [testData.streams[0].children[1].id], all: [testData.streams[8].id] } }, function (err, res) { should(err).be.null(); should(res.events).not.be.null(); done(); diff --git a/components/api-server/test/system.test.js b/components/api-server/test/system.test.js index 759e453c6..41c308a2c 100644 --- a/components/api-server/test/system.test.js +++ b/components/api-server/test/system.test.js @@ -149,6 +149,7 @@ describe('[SYER] system (ex-register)', function () { return basePath() + '/create-user'; } function post (data, callback) { + testData.addUserToBeErased(data.username); return request.post(path()) .set('authorization', helpers.dependencies.settings.auth.adminAccessKey) .send(data) diff --git a/components/business/src/users/repository.js b/components/business/src/users/repository.js index 65ed073c2..db92c07c3 100644 --- a/components/business/src/users/repository.js +++ b/components/business/src/users/repository.js @@ -340,7 +340,7 @@ class UsersRepository { * @param {boolean | null} skipFowardToRegister * @returns {Promise} */ - async deleteOne (userId, username, skipFowardToRegister) { + async deleteOne (userId, username = null, skipFowardToRegister = false) { const user = await this.getUserById(userId); if (username == null) { username = user?.username; diff --git a/components/platform/src/DB.js b/components/platform/src/DB.js index 42ebd4dc1..40252635d 100644 --- a/components/platform/src/DB.js +++ b/components/platform/src/DB.js @@ -21,9 +21,12 @@ class DB { mkdirp.sync(basePath); this.db = new SQLite3(basePath + '/platform-wide.db'); - this.db.pragma('journal_mode = WAL'); - - this.db.prepare('CREATE TABLE IF NOT EXISTS keyValue (key TEXT PRIMARY KEY, value TEXT NOT NULL);').run(); + await concurentSafeWriteStatement(() => { + this.db.pragma('journal_mode = WAL'); + }); + await concurentSafeWriteStatement(() => { + this.db.prepare('CREATE TABLE IF NOT EXISTS keyValue (key TEXT PRIMARY KEY, value TEXT NOT NULL);').run(); + }); this.queries = {}; this.queries.getValueWithKey = this.db.prepare('SELECT key, value FROM keyValue WHERE key = ?'); @@ -109,6 +112,28 @@ class DB { } } +const WAIT_LIST_MS = [1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100]; +/** + * Will look "retries" times, in case of "SQLITE_BUSY". + * This is CPU intensive, but tests have shown this solution to be efficient + */ +async function concurentSafeWriteStatement (statement, retries = 100) { + for (let i = 0; i < retries; i++) { + try { + statement(); + return; + } catch (error) { + if (error.code !== 'SQLITE_BUSY') { // ignore + throw error; + } + const waitTime = i > (WAIT_LIST_MS.length - 1) ? 100 : WAIT_LIST_MS[i]; + await new Promise((resolve) => setTimeout(resolve, waitTime)); + this.logger.debug('SQLITE_BUSY, retrying in ' + waitTime + 'ms'); + } + } + throw new Error('Failed write action on Audit after ' + retries + ' retries'); +} + /** * Return an object from an entry in the table * @param {Entry} entry diff --git a/components/test-helpers/src/data.js b/components/test-helpers/src/data.js index 315e8e825..ae6e79359 100644 --- a/components/test-helpers/src/data.js +++ b/components/test-helpers/src/data.js @@ -25,18 +25,33 @@ const { getConfigUnsafe, getConfig, getLogger } = require('@pryv/boiler'); const { getMall } = require('mall'); const logger = getLogger('test-helpers:data'); +const { runId } = require('./runid'); + // users const users = (exports.users = require('./data/users')); const defaultUser = users[0]; +const userNamesToErase = []; +exports.addUserToBeErased = function (userId) { + userNamesToErase.push(userId); +}; + exports.resetUsers = async () => { logger.debug('resetUsers'); await getConfig(); // lock up to the time config is ready await SystemStreamsSerializer.init(); const customAccountProperties = buildCustomAccountProperties(); const usersRepository = await getUsersRepository(); - await usersRepository.deleteAll(); + for (const user of users) { + await usersRepository.deleteOne(user.id); + } + while (userNamesToErase.length > 0) { + const userName = userNamesToErase.pop(); + const userId = await usersRepository.getUserIdForUsername(userName); + if (userId != null) await usersRepository.deleteOne(userId); + } + // await usersRepository.deleteAll(); for (const user of users) { const userObj = new User(_.merge(customAccountProperties, user)); // might alter storage "dump data" script await usersRepository.insertOne(userObj, false, true); @@ -160,12 +175,12 @@ function resetData (storage, user, items, done) { const attachmentsDirPath = (exports.attachmentsDirPath = path.join(__dirname, '/data/attachments/')); const attachments = (exports.attachments = { - animatedGif: getAttachmentInfo('animatedGif', 'animated.gif', 'image/gif'), - document: getAttachmentInfo('document', 'document.pdf', 'application/pdf'), - document_modified: getAttachmentInfo('document', 'document.modified.pdf', 'application/pdf'), - image: getAttachmentInfo('image', 'image (space and special chars)é__.png', 'image/png'), - imageBigger: getAttachmentInfo('imageBigger', 'image-bigger.jpg', 'image/jpeg'), - text: getAttachmentInfo('text', 'text.txt', 'text/plain') + animatedGif: getAttachmentInfo('animatedGif-' + runId, 'animated.gif', 'image/gif'), + document: getAttachmentInfo('document-' + runId, 'document.pdf', 'application/pdf'), + document_modified: getAttachmentInfo('document-' + runId, 'document.modified.pdf', 'application/pdf'), + image: getAttachmentInfo('image-' + runId, 'image (space and special chars)é__.png', 'image/png'), + imageBigger: getAttachmentInfo('imageBigger-' + runId, 'image-bigger.jpg', 'image/jpeg'), + text: getAttachmentInfo('text-' + runId, 'text.txt', 'text/plain') }); // following https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity diff --git a/components/test-helpers/src/data/accesses.js b/components/test-helpers/src/data/accesses.js index 9e736afb2..28c9c085b 100644 --- a/components/test-helpers/src/data/accesses.js +++ b/components/test-helpers/src/data/accesses.js @@ -6,8 +6,8 @@ */ const streams = require('./streams'); const timestamp = require('unix-timestamp'); - -module.exports = [ +const { runIdMap } = require('../runid'); +module.exports = runIdMap([ { id: 'a_0', token: 'a_0_token', @@ -164,4 +164,4 @@ module.exports = [ calls: {}, deleted: timestamp.now('-1m') } */ -]; +]); diff --git a/components/test-helpers/src/data/events.js b/components/test-helpers/src/data/events.js index db11c9f16..11b83b4d2 100644 --- a/components/test-helpers/src/data/events.js +++ b/components/test-helpers/src/data/events.js @@ -8,7 +8,7 @@ const streams = require('./streams'); const timestamp = require('unix-timestamp'); const { TAG_PREFIX } = require('api-server/src/methods/helpers/backwardCompatibility'); const { integrity } = require('business'); - +const { runId } = require('../runid'); const events = [ { id: getTestEventId(0), @@ -20,13 +20,13 @@ const events = [ description: 'First period event, with attachments', attachments: [ { - id: 'document', + id: 'document-' + runId, fileName: 'document.pdf', type: 'application/pdf', size: 6701 }, { - id: 'image', + id: 'image-' + runId, fileName: 'image (space and special chars)é__.png', type: 'image/png', size: 2765 @@ -64,7 +64,7 @@ const events = [ description: '陳容龍', attachments: [ { - id: 'imageBigger', + id: 'imageBigger-' + runId, fileName: 'image-bigger.jpg', type: 'image/jpeg', size: 177476 @@ -196,7 +196,7 @@ const events = [ tags: [], attachments: [ { - id: 'animatedGif', + id: 'animatedGif-' + runId, fileName: 'animated.gif', type: 'image/gif', size: 88134 @@ -429,5 +429,5 @@ module.exports = events; */ function getTestEventId (n) { n = n + ''; - return 'cthisistesteventno' + (n.length >= 7 ? n : new Array(7 - n.length + 1).join('0') + n); + return 'cth' + runId + 'testeventno' + (n.length >= 7 ? n : new Array(7 - n.length + 1).join('0') + n); } diff --git a/components/test-helpers/src/data/followedSlices.js b/components/test-helpers/src/data/followedSlices.js index 95acd866f..c0b27eaf5 100644 --- a/components/test-helpers/src/data/followedSlices.js +++ b/components/test-helpers/src/data/followedSlices.js @@ -5,9 +5,9 @@ * Proprietary and confidential */ const accesses = require('./accesses'); - +const { runIdMap } = require('../runid'); module.exports = function (url) { - return [ + return runIdMap([ { id: 'b_0', name: 'Zero\'s First Access', @@ -26,5 +26,5 @@ module.exports = function (url) { url, accessToken: accesses[3].token } - ]; + ]); }; diff --git a/components/test-helpers/src/data/streams.js b/components/test-helpers/src/data/streams.js index 2c0641a98..a509e0603 100644 --- a/components/test-helpers/src/data/streams.js +++ b/components/test-helpers/src/data/streams.js @@ -6,8 +6,8 @@ */ const timestamp = require('unix-timestamp'); const { TAG_ROOT_STREAMID, TAG_PREFIX } = require('api-server/src/methods/helpers/backwardCompatibility'); - -module.exports = [ +const { runIdStreamTree } = require('../runid'); +module.exports = runIdStreamTree([ { id: 's_0', name: 'Root Stream 0', @@ -248,4 +248,4 @@ module.exports = [ } ] } -]; +]); diff --git a/components/test-helpers/src/data/users.js b/components/test-helpers/src/data/users.js index 7b01a61d9..d53cfc2b6 100644 --- a/components/test-helpers/src/data/users.js +++ b/components/test-helpers/src/data/users.js @@ -4,8 +4,9 @@ * Unauthorized copying of this file, via any medium is strictly prohibited * Proprietary and confidential */ +const { runIdMap } = require('../runid'); -module.exports = [ +const events = [ { id: 'u_0', username: 'userzero', @@ -57,3 +58,7 @@ module.exports = [ language: 'en', }, */ ]; + +runIdMap(events); +runIdMap(events, 'email', '-', true); +module.exports = events; diff --git a/components/test-helpers/src/runid.js b/components/test-helpers/src/runid.js new file mode 100644 index 000000000..adc2f2559 --- /dev/null +++ b/components/test-helpers/src/runid.js @@ -0,0 +1,52 @@ +/** + * @license + * Copyright (C) 2012–2023 Pryv S.A. https://pryv.com - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited + * Proprietary and confidential + */ + +// 4 letters code to happend to all ids to run in parallel +const runId = (Math.random() + 1).toString(36).substring(8); +exports.runId = runId; + +/** + * Transform an array by adding runId to items[property] + * @param {Array} array + * @param {string} [property='id'] the property to change (default: 'id') + * @param {string} [space='-'] spacer between original property value and runId (default '-') + * @param {boolean} [head=false] if true happend at the begging, false at the end (default false) + * @returns array + */ +exports.runIdMap = function runIdMap (array, property = 'id', space = '-', head = false) { + for (const item of array) { + if (head) { + item[property] = runId + space + item[property]; + } else { + item[property] += space + runId; + } + } + return array; +}; + +/** + * Transform a tree by adding runId to items[property] + * @param {Array} array + * @param {string} [property='id'] the property to change (default: 'id') + * @param {string} [childrenProperty='children'] the property to find children of item (default: 'children') + * @param {string} [space='-'] spacer between original property value and runId (default '-') + * @returns array + */ +function runIdTree (array, property = 'id', childrenProperty = 'children', space = '-') { + for (const item of array) { + if (item[property] != null) item[property] += space + runId; + if (item[childrenProperty] != null) runIdTree(item[childrenProperty], property, childrenProperty, space); + } + return array; +} +exports.runIdTree = runIdTree; + +exports.runIdStreamTree = function runIdStreamTree (array) { + runIdTree(array); + runIdTree(array, 'parentId'); + return array; +}; diff --git a/components/utils/src/encryption.js b/components/utils/src/encryption.js index c99f5cdb0..6dae740e2 100644 --- a/components/utils/src/encryption.js +++ b/components/utils/src/encryption.js @@ -49,7 +49,7 @@ exports.fileReadToken = function (fileId, accessId, accessToken, secret) { * @returns {Object} Contains `accessId` and `hmac` parts if successful; empty otherwise. */ exports.parseFileReadToken = function (fileReadToken) { - const sepIndex = fileReadToken.indexOf('-'); + const sepIndex = fileReadToken.lastIndexOf('-'); // take the lastIndexOf as "-" might appear un the accesId. if (sepIndex <= 0) { return {}; } @@ -74,6 +74,6 @@ function getFileHMAC (fileId, token, secret) { return base64HMAC .toString() // function signature says we might have a buffer here. .replace(/\//g, '_') - .replace(/\+/g, '-') + .replace(/\+/g, '_') // don't use '-' as it will be the seprator with the accesId .replace(/=/g, ''); }