fix(domaincache): ZMS-247 implement domain cache to store all used domains of users (#845)

* implement domain cache to store all used domains of users

* add missing await before promise

* ignore duplicate key error, fix bug (use correct addressData object)

* fix comment

* settings-handler domain cache improve name

Domain cache will be used primarily to filter inbound emails as sort of a bloom filter

Author: NickOvt

indexes.yaml

@@ -745,6 +745,15 @@ indexes:
               enumerable: 1
               key: 1
 
+    # Indexes for the domaincache collection
+    - collection: domaincache
+      type: users
+      index:
+          name: by_domain
+          unique: true
+          key:
+              domain: 1
+
 deleteindexes:
     - collection: settings
       index: key_enumerable

lib/api/addresses.js

@@ -532,6 +532,11 @@ module.exports = (db, server, userHandler, settingsHandler) => {
             // insert alias address to email address registry
             try {
                 r = await db.users.collection('addresses').insertOne(addressData);
+                try {
+                    await db.database.collection('domaincache').insertOne({ domain: addressData.address.split('@')[1] });
+                } catch {
+                    // ignore
+                }
             } catch (err) {
                 res.status(500);
                 return res.json({
@@ -1724,6 +1729,12 @@ module.exports = (db, server, userHandler, settingsHandler) => {
 
             try {
                 r = await db.users.collection('addresses').insertOne(addressData);
+
+                try {
+                    await db.database.collection('domaincache').insertOne({ domain: addressData.address.split('@')[1] });
+                } catch {
+                    // ignore
+                }
             } catch (err) {
                 res.status(500);
                 return res.json({

lib/settings-handler.js

@@ -147,6 +147,15 @@ const SETTING_KEYS = [
         type: 'number',
         constKey: 'BULK_BATCH_SIZE',
         schema: Joi.number()
+    },
+
+    {
+        key: 'const:domaincache:enabled',
+        name: 'Domain cache enabled status',
+        description: 'A true/false boolean value indicating whether domains are stored in a separate cache. Only these domains are allowed for inbound emails',
+        type: 'boolean',
+        confValue: false,
+        schema: Joi.boolean()
     }
 ];
 

lib/user-handler.js

@@ -1609,6 +1609,12 @@ class UserHandler {
             try {
                 // insert alias address to email address registry
                 await this.users.collection('addresses').insertOne(addressData, { writeConcern: 'majority' });
+
+                try {
+                    await this.users.collection('domaincache').insertOne({ domain: addressData.address.split('@')[1] });
+                } catch {
+                    // ignore
+                }
             } catch (err) {
                 try {
                     // try to rollback
@@ -3676,6 +3682,13 @@ class UserHandler {
                 if (!r || !r.insertedId) {
                     throw new Error('Failed to insert');
                 }
+
+                try {
+                    await this.users.collection('domaincache').insertOne({ domain: address.address.split('@')[1] });
+                } catch {
+                    // ignore
+                }
+
                 recoveredAddresses.push(address);
                 log.info('Restore', 'ADDRRESTORE user=%s address=%s email=%s', user, address._id, address.address);
             } catch (err) {