From a73acb66ef7de6149dabe89ed4dd381074aefea1 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Tue, 25 Apr 2023 18:05:35 +0200
Subject: [PATCH 1/9] initial testing for a rest-based datastore

---
 components/api-server/config/test-config.yml |  2 +
 components/api-server/test/hooks.js          |  4 +-
 components/mall/src/MallTransaction.js       |  5 ++
 components/mall/src/index.js                 |  6 ++
 startRestServer.js                           | 64 ++++++++++++++++++++
 5 files changed, 79 insertions(+), 2 deletions(-)
 create mode 100644 startRestServer.js

diff --git a/components/api-server/config/test-config.yml b/components/api-server/config/test-config.yml
index ac20c3be1..c7e8fe7a3 100644
--- a/components/api-server/config/test-config.yml
+++ b/components/api-server/config/test-config.yml
@@ -31,6 +31,8 @@ reporting:
   licenseName: pryv.io-test-license
   templateVersion: 1.0.0
 database:
+  engine: rest
+  baseURL: 'http://localhost:6789/{userId}'
   host: 127.0.0.1
   port: 27017
   name: pryv-node-test
diff --git a/components/api-server/test/hooks.js b/components/api-server/test/hooks.js
index 7ce1c7fcf..9d4ef960b 100644
--- a/components/api-server/test/hooks.js
+++ b/components/api-server/test/hooks.js
@@ -34,10 +34,10 @@ exports.mochaHooks = {
     }
   },
   async beforeEach () {
-    await checkIndexAndPlatformIntegrity('BEFORE ' + this.currentTest.title);
+    // await checkIndexAndPlatformIntegrity('BEFORE ' + this.currentTest.title);
   },
   async afterEach () {
-    await checkIndexAndPlatformIntegrity('AFTER ' + this.currentTest.title);
+    // await checkIndexAndPlatformIntegrity('AFTER ' + this.currentTest.title);
   }
 };
 
diff --git a/components/mall/src/MallTransaction.js b/components/mall/src/MallTransaction.js
index 42cb89462..7c070599e 100644
--- a/components/mall/src/MallTransaction.js
+++ b/components/mall/src/MallTransaction.js
@@ -26,6 +26,11 @@ class MallTransaction {
       return this.storeTransactions.get(storeId);
     }
     const store = this.mall.storesById.get(storeId);
+    if (store.newTransaction == null) {
+      return {
+        exec: async function (func) { return await func(); }
+      };
+    }
     const transaction = await store.newTransaction();
     this.storeTransactions.set(storeId, transaction);
     return transaction;
diff --git a/components/mall/src/index.js b/components/mall/src/index.js
index 6a8bbb423..e92af9935 100644
--- a/components/mall/src/index.js
+++ b/components/mall/src/index.js
@@ -62,6 +62,12 @@ async function getMall () {
     logger.info('Using PoC SQLite data store');
     const sqlite = require('storage/src/localDataStoreSQLite');
     mall.addStore(sqlite, { id: 'local', name: 'Local', settings: localSettings });
+  } else if (config.get('database:engine') === 'rest') {
+    logger.info('Using PoC SQLite data store');
+    const rest = require('@pryv/datastore/examples/rest');
+    const settings = Object.assign({}, localSettings);
+    settings.baseURL = config.get('database:baseURL');
+    mall.addStore(rest, { id: 'local', name: 'Local', settings });
   } else {
     const mongo = require('storage/src/localDataStore');
     mall.addStore(mongo, { id: 'local', name: 'Local', settings: localSettings });
diff --git a/startRestServer.js b/startRestServer.js
new file mode 100644
index 000000000..b322b2517
--- /dev/null
+++ b/startRestServer.js
@@ -0,0 +1,64 @@
+const path = require('path');
+
+const { getConfig } = require('@pryv/boiler').init({
+  appName: 'rest',
+  baseFilesDir: path.resolve(__dirname, './'),
+  baseConfigDir: path.resolve(__dirname, './components/api-server/config/'),
+  extraConfigs: [
+    {
+      scope: 'serviceInfo',
+      key: 'service',
+      urlFromKey: 'serviceInfoUrl'
+    },
+    {
+      scope: 'default-paths',
+      file: path.resolve(__dirname, './components/api-server/config/paths-config.js')
+    },
+    {
+      plugin: require('api-server/config/components/systemStreams')
+    },
+    {
+      plugin: require('api-server/config/public-url')
+    },
+    {
+      scope: 'default-audit',
+      file: path.resolve(__dirname, 'audit/config/default-config.yml')
+    },
+    {
+      scope: 'default-audit-path',
+      file: path.resolve(__dirname, 'audit/config/default-path.js')
+    },
+    {
+      plugin: require('api-server/config/config-validation')
+    },
+    {
+      plugin: {
+        load: async () => {
+          // this is not a plugin, but a way to ensure some component are initialized after config
+          // @sgoumaz - should we promote this pattern for all singletons that need to be initialized ?
+          const SystemStreamsSerializer = require('business/src/system-streams/serializer');
+          await SystemStreamsSerializer.init();
+        }
+      }
+    }
+  ]
+});
+
+const ds = require('storage/src/localDataStoreSQLite/');
+
+const server = require('@pryv/datastore/examples/rest/server');
+
+function debugMiddleware (req, res, next) {
+  console.log({ method: req.method, url: req.url, body: req.body });
+  next();
+}
+
+(async function () {
+  const config = await getConfig();
+  const localSettings = {
+    attachments: { setFileReadToken: true },
+    versioning: config.get('versioning')
+  };
+  await ds.init({ id: 'local', name: 'Local', settings: localSettings });
+  await server(ds, 6789, { middleware: debugMiddleware });
+})();

From 891a6d829c5698b4539b37012d7798fc443417ba Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Tue, 25 Apr 2023 20:02:05 +0200
Subject: [PATCH 2/9] added some comments

---
 startRestServer.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/startRestServer.js b/startRestServer.js
index b322b2517..8853a8d31 100644
--- a/startRestServer.js
+++ b/startRestServer.js
@@ -1,3 +1,7 @@
+
+// npx link ../pryv-datastore
+// launch with NODE_ENV=test LOGS=info node startRestServer.js
+
 const path = require('path');
 
 const { getConfig } = require('@pryv/boiler').init({
@@ -49,7 +53,7 @@ const ds = require('storage/src/localDataStoreSQLite/');
 const server = require('@pryv/datastore/examples/rest/server');
 
 function debugMiddleware (req, res, next) {
-  console.log({ method: req.method, url: req.url, body: req.body });
+  //console.log({ method: req.method, url: req.url, body: req.body });
   next();
 }
 

From 5226e5da7b6878c9431a2378fdd27c34dce3360b Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Thu, 27 Apr 2023 17:26:39 +0200
Subject: [PATCH 3/9] parly fixing attachments

---
 components/storage/src/localDataStore/localUserStreams.js | 4 ++++
 startRestServer.js                                        | 6 +-----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/components/storage/src/localDataStore/localUserStreams.js b/components/storage/src/localDataStore/localUserStreams.js
index 560a4a597..c25f2c2db 100644
--- a/components/storage/src/localDataStore/localUserStreams.js
+++ b/components/storage/src/localDataStore/localUserStreams.js
@@ -56,6 +56,10 @@ module.exports = ds.createUserStreams({
     if (!query.includeTrashed) {
       if (stream.trashed) return null;
       // i.e. default behavior (return non-trashed items)
+      if (! stream.children ) {
+        $$('checkthis', {stream});
+        stream.children = [];
+      }
       stream.children = treeUtils.filterTree(stream.children, false /* no orphans */, (stream) => !stream.trashed);
     }
 
diff --git a/startRestServer.js b/startRestServer.js
index 8853a8d31..4626911bd 100644
--- a/startRestServer.js
+++ b/startRestServer.js
@@ -9,11 +9,7 @@ const { getConfig } = require('@pryv/boiler').init({
   baseFilesDir: path.resolve(__dirname, './'),
   baseConfigDir: path.resolve(__dirname, './components/api-server/config/'),
   extraConfigs: [
-    {
-      scope: 'serviceInfo',
-      key: 'service',
-      urlFromKey: 'serviceInfoUrl'
-    },
+
     {
       scope: 'default-paths',
       file: path.resolve(__dirname, './components/api-server/config/paths-config.js')

From 2c22173009db9eef98d2d3ff3026ca4ef036a805 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Sun, 18 Jun 2023 18:18:17 +0200
Subject: [PATCH 4/9] updated to match latest changes on datastore api

---
 .../api-server/src/middleware/attachment-access.js  |  2 +-
 components/mall/src/MallUserEvents.js               |  4 ++--
 components/mall/src/index.js                        |  2 +-
 .../storage/src/localDataStore/localUserEvents.js   |  2 +-
 .../storage/src/localDataStore/localUserStreams.js  |  4 ++--
 .../localDataStoreSQLite/localUserEventsSQLite.js   |  2 +-
 startRestServer.js                                  | 13 +++++++++++--
 7 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/components/api-server/src/middleware/attachment-access.js b/components/api-server/src/middleware/attachment-access.js
index 5ebaa5ffd..d33e08468 100644
--- a/components/api-server/src/middleware/attachment-access.js
+++ b/components/api-server/src/middleware/attachment-access.js
@@ -72,7 +72,7 @@ async function attachmentsAccessMiddleware (req, res, next) {
       res.header('Digest', digest);
     }
   }
-  const fileReadStream = await mall.events.getAttachedFile(req.context.user.id, event, req.params.fileId);
+  const fileReadStream = await mall.events.getAttachment(req.context.user.id, event, req.params.fileId);
   // for Audit
   req.context.originalQuery = req.params;
   const pipedStream = fileReadStream.pipe(res);
diff --git a/components/mall/src/MallUserEvents.js b/components/mall/src/MallUserEvents.js
index 9158647d3..5790d8f8e 100644
--- a/components/mall/src/MallUserEvents.js
+++ b/components/mall/src/MallUserEvents.js
@@ -252,13 +252,13 @@ class MallUserEvents {
    * @param {string} fileId
    * @returns {Promise<any>}
    */
-  async getAttachedFile (userId, eventData, fileId) {
+  async getAttachment (userId, eventData, fileId) {
     const [storeId, storeEventId] = storeDataUtils.parseStoreIdAndStoreItemId(eventData.id);
     const eventsStore = this.eventsStores.get(storeId);
     if (!eventsStore) {
       throw errorFactory.unknownResource(`Unknown store "${storeId}"`, storeId);
     }
-    return await eventsStore.getAttachedFile(userId, storeEventId, fileId);
+    return await eventsStore.getAttachment(userId, storeEventId, fileId);
   }
 
   /**
diff --git a/components/mall/src/index.js b/components/mall/src/index.js
index e92af9935..71c63a568 100644
--- a/components/mall/src/index.js
+++ b/components/mall/src/index.js
@@ -63,7 +63,7 @@ async function getMall () {
     const sqlite = require('storage/src/localDataStoreSQLite');
     mall.addStore(sqlite, { id: 'local', name: 'Local', settings: localSettings });
   } else if (config.get('database:engine') === 'rest') {
-    logger.info('Using PoC SQLite data store');
+    logger.info('Using Rest SQLite data store');
     const rest = require('@pryv/datastore/examples/rest');
     const settings = Object.assign({}, localSettings);
     settings.baseURL = config.get('database:baseURL');
diff --git a/components/storage/src/localDataStore/localUserEvents.js b/components/storage/src/localDataStore/localUserEvents.js
index ac7e1096b..435fb27b5 100644
--- a/components/storage/src/localDataStore/localUserEvents.js
+++ b/components/storage/src/localDataStore/localUserEvents.js
@@ -116,7 +116,7 @@ module.exports = ds.createUserEvents({
     return event;
   },
 
-  async getAttachedFile (userId, eventId, fileId) {
+  async getAttachment (userId, eventId, fileId) {
     return this.eventsFileStorage.getAttachmentStream(userId, eventId, fileId);
   },
 
diff --git a/components/storage/src/localDataStore/localUserStreams.js b/components/storage/src/localDataStore/localUserStreams.js
index c25f2c2db..70c170ea9 100644
--- a/components/storage/src/localDataStore/localUserStreams.js
+++ b/components/storage/src/localDataStore/localUserStreams.js
@@ -56,8 +56,8 @@ module.exports = ds.createUserStreams({
     if (!query.includeTrashed) {
       if (stream.trashed) return null;
       // i.e. default behavior (return non-trashed items)
-      if (! stream.children ) {
-        $$('checkthis', {stream});
+      if (!stream.children) {
+        $$('checkthis', { stream });
         stream.children = [];
       }
       stream.children = treeUtils.filterTree(stream.children, false /* no orphans */, (stream) => !stream.trashed);
diff --git a/components/storage/src/localDataStoreSQLite/localUserEventsSQLite.js b/components/storage/src/localDataStoreSQLite/localUserEventsSQLite.js
index bf543658b..f1100e829 100644
--- a/components/storage/src/localDataStoreSQLite/localUserEventsSQLite.js
+++ b/components/storage/src/localDataStoreSQLite/localUserEventsSQLite.js
@@ -109,7 +109,7 @@ module.exports = ds.createUserEvents({
    * @param {string} fileId
    * @returns {Promise<any>}
    */
-  async getAttachedFile (userId, eventId, fileId) {
+  async getAttachment (userId, eventId, fileId) {
     return this.eventsFileStorage.getAttachmentStream(userId, eventId, fileId);
   },
 
diff --git a/startRestServer.js b/startRestServer.js
index 4626911bd..de17a84c2 100644
--- a/startRestServer.js
+++ b/startRestServer.js
@@ -3,6 +3,8 @@
 // launch with NODE_ENV=test LOGS=info node startRestServer.js
 
 const path = require('path');
+const eventsUtils = require('mall/src/helpers/eventsUtils');
+const stableRepresentation = require('@pryv/stable-object-representation');
 
 const { getConfig } = require('@pryv/boiler').init({
   appName: 'rest',
@@ -49,16 +51,23 @@ const ds = require('storage/src/localDataStoreSQLite/');
 const server = require('@pryv/datastore/examples/rest/server');
 
 function debugMiddleware (req, res, next) {
-  //console.log({ method: req.method, url: req.url, body: req.body });
+  console.log({ method: req.method, url: req.url, body: req.body });
   next();
 }
 
 (async function () {
   const config = await getConfig();
+  const algorithm = config.get('integrity:algorithm');
+  function setIntegrityForEvent (storeEventData) {
+    const event = eventsUtils.convertEventFromStore('local', storeEventData);
+    storeEventData.integrity = stableRepresentation.event.compute(event, algorithm).integrity;
+  }
+
   const localSettings = {
     attachments: { setFileReadToken: true },
     versioning: config.get('versioning')
   };
-  await ds.init({ id: 'local', name: 'Local', settings: localSettings });
+
+  await ds.init({ id: 'local', name: 'Local', settings: localSettings, integrity: { setOnEvent: setIntegrityForEvent } });
   await server(ds, 6789, { middleware: debugMiddleware });
 })();

From eb8a41561f30fda7446d877884ae657c1e009765 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Thu, 22 Jun 2023 18:16:49 +0200
Subject: [PATCH 5/9] resseting back originalErrors

---
 startRestServer.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/startRestServer.js b/startRestServer.js
index de17a84c2..77acd15de 100644
--- a/startRestServer.js
+++ b/startRestServer.js
@@ -2,6 +2,10 @@
 // npx link ../pryv-datastore
 // launch with NODE_ENV=test LOGS=info node startRestServer.js
 
+// keep a copy of error functions to reaasign back
+const { errors: dataStoreErrors } = require('@pryv/datastore');
+const originalErrors = Object.assign({}, dataStoreErrors);
+
 const path = require('path');
 const eventsUtils = require('mall/src/helpers/eventsUtils');
 const stableRepresentation = require('@pryv/stable-object-representation');
@@ -56,6 +60,9 @@ function debugMiddleware (req, res, next) {
 }
 
 (async function () {
+  // data store errors have been assigned by storeDataUtils assign them back to PryvDataStoreError
+  Object.assign(dataStoreErrors, originalErrors);
+
   const config = await getConfig();
   const algorithm = config.get('integrity:algorithm');
   function setIntegrityForEvent (storeEventData) {

From cfaedd4964cf498ca4cc1915846957badb8f167c Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Fri, 23 Jun 2023 10:04:24 +0200
Subject: [PATCH 6/9] some cleanup

---
 components/mall/src/index.js                              | 2 +-
 components/storage/src/localDataStore/localUserStreams.js | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/components/mall/src/index.js b/components/mall/src/index.js
index 71c63a568..9caaef9c4 100644
--- a/components/mall/src/index.js
+++ b/components/mall/src/index.js
@@ -65,7 +65,7 @@ async function getMall () {
   } else if (config.get('database:engine') === 'rest') {
     logger.info('Using Rest SQLite data store');
     const rest = require('@pryv/datastore/examples/rest');
-    const settings = Object.assign({}, localSettings);
+    const settings = structuredClone(localSettings);
     settings.baseURL = config.get('database:baseURL');
     mall.addStore(rest, { id: 'local', name: 'Local', settings });
   } else {
diff --git a/components/storage/src/localDataStore/localUserStreams.js b/components/storage/src/localDataStore/localUserStreams.js
index 70c170ea9..560a4a597 100644
--- a/components/storage/src/localDataStore/localUserStreams.js
+++ b/components/storage/src/localDataStore/localUserStreams.js
@@ -56,10 +56,6 @@ module.exports = ds.createUserStreams({
     if (!query.includeTrashed) {
       if (stream.trashed) return null;
       // i.e. default behavior (return non-trashed items)
-      if (!stream.children) {
-        $$('checkthis', { stream });
-        stream.children = [];
-      }
       stream.children = treeUtils.filterTree(stream.children, false /* no orphans */, (stream) => !stream.trashed);
     }
 

From 80c26644cc0f406a3903c260a407145fcedd2ca1 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Fri, 23 Jun 2023 10:07:40 +0200
Subject: [PATCH 7/9] Putting back checkThis debug

---
 components/storage/src/localDataStore/localUserStreams.js | 4 ++++
 startRestServer.js                                        | 8 +++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/components/storage/src/localDataStore/localUserStreams.js b/components/storage/src/localDataStore/localUserStreams.js
index 560a4a597..70c170ea9 100644
--- a/components/storage/src/localDataStore/localUserStreams.js
+++ b/components/storage/src/localDataStore/localUserStreams.js
@@ -56,6 +56,10 @@ module.exports = ds.createUserStreams({
     if (!query.includeTrashed) {
       if (stream.trashed) return null;
       // i.e. default behavior (return non-trashed items)
+      if (!stream.children) {
+        $$('checkthis', { stream });
+        stream.children = [];
+      }
       stream.children = treeUtils.filterTree(stream.children, false /* no orphans */, (stream) => !stream.trashed);
     }
 
diff --git a/startRestServer.js b/startRestServer.js
index 77acd15de..42d991e32 100644
--- a/startRestServer.js
+++ b/startRestServer.js
@@ -1,3 +1,9 @@
+/**
+ * @license
+ * Copyright (C) 2012–2023 Pryv S.A. https://pryv.com - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited
+ * Proprietary and confidential
+ */
 
 // npx link ../pryv-datastore
 // launch with NODE_ENV=test LOGS=info node startRestServer.js
@@ -55,7 +61,7 @@ const ds = require('storage/src/localDataStoreSQLite/');
 const server = require('@pryv/datastore/examples/rest/server');
 
 function debugMiddleware (req, res, next) {
-  console.log({ method: req.method, url: req.url, body: req.body });
+  // console.log({ method: req.method, url: req.url, body: req.body });
   next();
 }
 

From d3265c5aea26a19e58c76d23c2090b0c93c2c747 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Fri, 23 Jun 2023 16:41:01 +0200
Subject: [PATCH 8/9] Removing unecessary check in treeutils

---
 components/storage/src/localDataStore/localUserStreams.js | 4 ----
 components/utils/src/treeUtils.js                         | 4 +---
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/components/storage/src/localDataStore/localUserStreams.js b/components/storage/src/localDataStore/localUserStreams.js
index 70c170ea9..560a4a597 100644
--- a/components/storage/src/localDataStore/localUserStreams.js
+++ b/components/storage/src/localDataStore/localUserStreams.js
@@ -56,10 +56,6 @@ module.exports = ds.createUserStreams({
     if (!query.includeTrashed) {
       if (stream.trashed) return null;
       // i.e. default behavior (return non-trashed items)
-      if (!stream.children) {
-        $$('checkthis', { stream });
-        stream.children = [];
-      }
       stream.children = treeUtils.filterTree(stream.children, false /* no orphans */, (stream) => !stream.trashed);
     }
 
diff --git a/components/utils/src/treeUtils.js b/components/utils/src/treeUtils.js
index 28856e64d..36c269e4c 100644
--- a/components/utils/src/treeUtils.js
+++ b/components/utils/src/treeUtils.js
@@ -26,9 +26,7 @@ exports.buildTree = function (array, stripParentIds) {
   array.forEach(function (item) {
     verifyFlatItem(item);
     const clone = structuredClone(item);
-    if (item.hasOwnProperty('parentId')) {
-      clone.children = [];
-    }
+    clone.children = [];
     map[item.id] = clone;
   });
   const result = [];

From abf55de8110c963070d76eb828534d3c3184c504 Mon Sep 17 00:00:00 2001
From: Perki <perki@pryv.com>
Date: Fri, 23 Jun 2023 17:35:33 +0200
Subject: [PATCH 9/9] Fixing deleted non needing children

---
 components/utils/src/treeUtils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/components/utils/src/treeUtils.js b/components/utils/src/treeUtils.js
index 36c269e4c..4ffe6eeae 100644
--- a/components/utils/src/treeUtils.js
+++ b/components/utils/src/treeUtils.js
@@ -26,7 +26,7 @@ exports.buildTree = function (array, stripParentIds) {
   array.forEach(function (item) {
     verifyFlatItem(item);
     const clone = structuredClone(item);
-    clone.children = [];
+    if (clone.deleted == null) clone.children = [];
     map[item.id] = clone;
   });
   const result = [];