Introduce app definition sync + new definitions (#432)

* Introduce app definition sync + new definitions

* Add api to allowed app categories

* Add portfolio-manager to allow app categories

Author: cbovis

src/app/initialise.js

@@ -4,13 +4,18 @@ const { QUEUE } = require('../constants');
 const { initQueues } = require('../queues');
 const configure = require('./configure');
 const populateRelayersCollection = require('../relayers/populate-relayers-collection');
+const syncAppDefinitions = require('../apps/sync-app-definitions');
 const tokenCache = require('../tokens/token-cache');
 
 const initialise = async () => {
   await configure();
   initQueues(Object.values(QUEUE), config.get('queues'));
 
-  await Promise.all([tokenCache.initialise(), populateRelayersCollection()]);
+  await Promise.all([
+    populateRelayersCollection(),
+    syncAppDefinitions(),
+    tokenCache.initialise(),
+  ]);
 };
 
 module.exports = initialise;

src/apps/definitions/0x-api.json

@@ -0,0 +1,16 @@
+{
+  "id": "052b4862-2142-4532-bdc0-416814b0a5fe",
+  "name": "0x API",
+  "logoUrl": "https://resources.0xtracker.com/logos/0x.png",
+  "urlSlug": "0x-api",
+  "websiteUrl": "https://0x.org/api",
+  "mappings": [
+    {
+      "type": "relayer",
+      "feeRecipientAddress": "0x1000000000000000000000000000000000000011"
+    }
+  ],
+  "categories": [
+    "api"
+  ]
+}

src/apps/definitions/defi-saver.json

@@ -0,0 +1,16 @@
+{
+  "id": "b986d2ba-77bf-420b-99e3-28e592d476e2",
+  "name": "DeFi Saver",
+  "logoUrl": "https://resources.0xtracker.com/logos/defi-saver.png",
+  "urlSlug": "defi-saver",
+  "websiteUrl": "https://defisaver.com/",
+  "mappings": [
+    {
+      "affiliateAddress": "0x322d58b9e75a6918f7e7849aee0ff09369977e08",
+      "type": "consumer"
+    }
+  ],
+  "categories": [
+    "portfolio-manager"
+  ]
+}

src/apps/definitions/paraswap.json

@@ -0,0 +1,24 @@
+{
+  "id": "4c3854b7-95c3-4714-83ab-a8ed53d2967e",
+  "name": "Paraswap",
+  "logoUrl": "https://resources.0xtracker.com/logos/paraswap.png",
+  "urlSlug": "paraswap",
+  "websiteUrl": "https://paraswap.io",
+  "mappings": [
+    {
+      "type": "consumer",
+      "feeRecipientAddress": "0x72338b82800400f5488eca2b5a37270ba3b7a111"
+    },
+    {
+      "type": "consumer",
+      "feeRecipientAddress": "0xf92c1ad75005e6436b4ee84e88cb23ed8a290988"
+    },
+    {
+      "type": "consumer",
+      "feeRecipientAddress": "0xffef2d75b3d266f90108e82b3b476a7f08fe5385"
+    }
+  ],
+  "categories": [
+    "dex-aggregator"
+  ]
+}

src/apps/index.js

@@ -1,3 +1,4 @@
 const resolveApps = require('./resolve-apps');
+const syncAppDefinitions = require('./sync-app-definitions');
 
-module.exports = { resolveApps };
+module.exports = { resolveApps, syncAppDefinitions };

src/apps/sync-app-definitions.js

@@ -0,0 +1,116 @@
+const _ = require('lodash');
+const Bluebird = require('bluebird');
+
+const { APP_TYPE } = require('../constants');
+const { getModel } = require('../model');
+// const { JOB, QUEUE } = require('../constants');
+// const { publishJob } = require('../queues');
+const getAppDefinitions = require('./get-app-definitions');
+
+// TODO: Introduce and test in future PR
+// const scheduleBackfill = async appId => {
+//   await publishJob(
+//     QUEUE.APP_PROCESSING,
+//     JOB.BACKFILL_APP,
+//     {
+//       id: appId,
+//     },
+//     {
+//       jobId: `backfill-app-${appId}`,
+//       removeOnComplete: false,
+//     },
+//   );
+// };
+
+const transformMappings = mappings =>
+  mappings.map(m => ({
+    ...m,
+    type: APP_TYPE[m.type.toUpperCase()],
+  }));
+
+const createApp = async definition => {
+  const App = getModel('App');
+  const app = {
+    ..._.omit(definition, 'id', 'mappings'),
+    _id: definition.id,
+    mappings: transformMappings(definition.mappings),
+  };
+
+  await App.create(app);
+  // await scheduleBackfill(definition.id);
+};
+
+const compareMappings = (currentMappings, definitionMappings) => {
+  const newMappings = _.differenceWith(
+    definitionMappings,
+    currentMappings,
+    _.isEqual,
+  );
+
+  const mappingsModified = newMappings.length > 0;
+
+  return mappingsModified;
+};
+
+const updateApp = async (app, definition) => {
+  const metadata = _.omit(definition, 'id', 'mappings');
+
+  Object.keys(metadata).forEach(metadataKey => {
+    app.set(metadataKey, metadata[metadataKey]);
+  });
+
+  const currentMappings = app.mappings.map(m =>
+    _.pickBy(
+      _.pick(
+        m,
+        'affiliateAddress',
+        'feeRecipientAddress',
+        'takerAddress',
+        'type',
+      ),
+      value => value !== undefined,
+    ),
+  );
+
+  const nextMappings = transformMappings(definition.mappings);
+  const mappingsModified = compareMappings(currentMappings, nextMappings);
+
+  if (mappingsModified) {
+    app.set('mappings', nextMappings);
+  }
+
+  if (app.isModified()) {
+    await app.save();
+  }
+
+  // if (mappingsModified) {
+  //   await scheduleBackfill(definition.id);
+  // }
+};
+
+/**
+ * Sync current app definitions with MongoDB making sure that any differences
+ * in metadata or mappings are reflected in MongoDB.
+ *
+ * Any changes to definition mappings will trigger an attributions backfill
+ * for the app in which the mappings changed.
+ *
+ * NOTE: Removal of apps or mappings must be handled manually. The sync process
+ * is not currently built to automate this since it's unlikely to occur.
+ */
+const syncAppDefinitions = async () => {
+  const App = getModel('App');
+  const definitions = getAppDefinitions();
+
+  await Bluebird.each(definitions, async definition => {
+    const app = await App.findById(definition.id);
+
+    if (app === null) {
+      await createApp(definition);
+    } else {
+      await updateApp(app, definition);
+    }
+  });
+};
+
+module.exports = syncAppDefinitions;