feat: transactions count on token table (#273)

* chore: empty commit (that will be rebased)

* feat: added transactions column to tokens table and tests to the method

* feat: adding hathor token to tokens table

* docs: added docs to getTokenListFromInputsAndOutputs

* refactor: using inner join instead of left outer join

* refactor: changed method name

* feat: added index on voided on address_tx_history

* feat: handling token transactions on reorg

* fix: wrong type

* tests: testing for hathor increment

* feat: add voided address_tx_history index to models

* refactor: improved checkTokenTable

Author: andreabadesso

db/migrations/20220811192312-add-transaction-count-to-token.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('token', 'transactions', {
+      type: Sequelize.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+    });
+  },
+
+  down: async (queryInterface) => {
+    await queryInterface.removeColumn('token', 'transactions');
+  },
+};

db/migrations/20220811212756-add-hathor-to-token-table.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.bulkInsert('token', [{
+      id: '00',
+      name: 'Hathor',
+      symbol: 'HTR',
+      transactions: 0,
+    }]);
+  },
+
+  down: async (queryInterface) => {
+    await queryInterface.bulkDelete('token', [{
+      id: '00',
+      name: 'Hathor',
+      symbol: 'HTR',
+    }]);
+  },
+};

db/migrations/20220811222729-add-voided-index-to-address-tx-history.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface) => queryInterface.addIndex(
+    'address_tx_history',
+    ['voided'], {
+      name: 'address_tx_history_voided_idx',
+      fields: 'voided',
+    },
+  ),
+  down: async (queryInterface) => queryInterface.removeIndex(
+    'address_tx_history',
+    'address_tx_history_voided_idx',
+  ),
+};

db/models/addresstxhistory.js

@@ -56,6 +56,9 @@ module.exports = (sequelize, DataTypes) => {
     }, {
       name: 'address_tx_history_timestamp_idx',
       fields: ['timestamp'],
+    }, {
+      name: 'address_tx_history_voided_idx',
+      fields: ['voided'],
     }],
   });
   return AddressTxHistory;

db/models/token.js

@@ -40,6 +40,10 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: DataTypes.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
     },
+    transactions: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+    },
   }, {
     sequelize,
     modelName: 'Token',

src/commons.ts

@@ -191,6 +191,27 @@ export const getAddressBalanceMap = (
   return addressBalanceMap;
 };
 
+/**
+ * Gets a list of tokens from a list of inputs and outputs
+ *
+ * @param inputs - The transaction inputs
+ * @param outputs - The transaction outputs
+ * @returns A list of tokens present in the inputs and outputs
+ */
+export const getTokenListFromInputsAndOutputs = (inputs: TxInput[], outputs: TxOutputWithIndex[]): string[] => {
+  const tokenIds = new Set<string>([]);
+
+  for (const input of inputs) {
+    tokenIds.add(input.token);
+  }
+
+  for (const output of outputs) {
+    tokenIds.add(output.token);
+  }
+
+  return [...tokenIds];
+};
+
 /**
  * Get the map of token balances for each wallet.
  *

src/db/index.ts

@@ -1255,8 +1255,20 @@ export const getWalletBalances = async (mysql: ServerlessMysql, walletId: string
     params.push(tokenIds);
   }
 
-  // use LEFT JOIN as HTR token ('00') won't be on the token table, so INNER JOIN would never match it
-  const query = `SELECT * FROM (${subquery}) w LEFT JOIN token ON w.token_id = token.id;`;
+  const query = `
+    SELECT NULL AS total_received,
+           w.unlocked_balance AS unlocked_balance,
+           w.locked_balance AS locked_balance,
+           w.unlocked_authorities AS unlocked_authorities,
+           w.locked_authorities AS locked_authorities,
+           w.timelock_expires AS timelock_expires,
+           w.transactions AS transactions,
+           w.token_id AS token_id,
+           token.name AS name,
+           token.symbol AS symbol
+      FROM (${subquery}) w
+INNER JOIN token ON w.token_id = token.id
+  `;
 
   const results: DbSelectResult = await mysql.query(query, params);
   for (const result of results) {
@@ -2109,6 +2121,7 @@ export const rebuildAddressBalancesFromUtxos = async (
 
   const addressTransactionCount: StringMap<number> = await getAffectedAddressTxCountFromTxList(mysql, txList);
   const addressTotalReceived: StringMap<number> = await getAffectedAddressTotalReceivedFromTxList(mysql, txList);
+  const tokenTransactionCount: StringMap<number> = await getAffectedTokenTxCountFromTxList(mysql, txList);
 
   const finalValues = oldAddressTokenTransactions.map(({ address, tokenId, transactions, totalReceived }) => {
     const diffTransactions = addressTransactionCount[`${address}_${tokenId}`] || 0;
@@ -2129,6 +2142,15 @@ export const rebuildAddressBalancesFromUtxos = async (
          AND \`token_id\` = ?
     `, item);
   }
+
+  // update token table with the correct amount of transactions
+  for (const token of Object.keys(tokenTransactionCount)) {
+    await mysql.query(`
+      UPDATE \`token\`
+        SET \`transactions\` = \`transactions\` - ?
+       WHERE \`id\` = ?
+    `, [tokenTransactionCount[token], token]);
+  }
 };
 
 /**
@@ -2546,6 +2568,39 @@ export const getAffectedAddressTxCountFromTxList = async (
   return addressTransactions as StringMap<number>;
 };
 
+/**
+ * Get the number of affected transactions for each token from the address_tx_history table
+ * given a list of transactions
+ *
+ * @param mysql - Database connection
+ * @param txList - A list of affected transactions to get the token tx count
+
+ * @returns A Map with tokenId as key and the transaction count as values
+ */
+export const getAffectedTokenTxCountFromTxList = async (
+  mysql: ServerlessMysql,
+  txList: string[],
+): Promise<StringMap<number>> => {
+  const results: DbSelectResult = await mysql.query(`
+    SELECT token_id AS tokenId, COUNT(DISTINCT(tx_id)) AS txCount
+      FROM address_tx_history
+     WHERE tx_id IN (?)
+       AND voided = TRUE
+  GROUP BY token_id
+  `, [txList]);
+
+  const tokenTransactions = results.reduce((acc, result) => {
+    const tokenId = result.tokenId as string;
+    const txCount = result.txCount as number;
+
+    acc[tokenId] = txCount;
+
+    return acc;
+  }, {});
+
+  return tokenTransactions as StringMap<number>;
+};
+
 /**
  * Get the affected total_received for each address/token pair given a list of transactions
  *
@@ -2560,9 +2615,10 @@ export const getAffectedAddressTotalReceivedFromTxList = async (
 ): Promise<StringMap<number>> => {
   const results: DbSelectResult = await mysql.query(`
     SELECT address, token_id as tokenId, SUM(value) as total
-     FROM tx_output
-     WHERE tx_id IN (?) AND voided = TRUE
-     GROUP BY address, token_id
+      FROM tx_output
+     WHERE tx_id IN (?)
+       AND voided = TRUE
+  GROUP BY address, token_id
   `, [txList]);
 
   const addressTotalReceivedMap = results.reduce((acc, result) => {
@@ -2577,3 +2633,20 @@ export const getAffectedAddressTotalReceivedFromTxList = async (
 
   return addressTotalReceivedMap as StringMap<number>;
 };
+
+/**
+ * Increment a list of tokens transactions count
+ *
+ * @param mysql - Database connection
+ * @param tokenList - The list of tokens to increment
+ */
+export const incrementTokensTxCount = async (
+  mysql: ServerlessMysql,
+  tokenList: string[],
+): Promise<void> => {
+  await mysql.query(`
+    UPDATE \`token\`
+       SET \`transactions\` = \`transactions\` + 1
+     WHERE \`id\` IN (?)
+  `, [tokenList]);
+};

src/txProcessor.ts

@@ -16,6 +16,7 @@ import {
   unlockUtxos,
   unlockTimelockedUtxos,
   searchForLatestValidBlock,
+  getTokenListFromInputsAndOutputs,
   handleReorg,
   handleVoided,
   prepareOutputs,
@@ -34,6 +35,7 @@ import {
   storeTokenInformation,
   updateAddressTablesWithTx,
   updateWalletTablesWithTx,
+  incrementTokensTxCount,
   fetchTx,
   addMiner,
 } from '@src/db';
@@ -377,6 +379,11 @@ const _unsafeAddNewTx = async (_logger: Logger, tx: Transaction, now: number, bl
     addressBalanceMap,
   });
 
+  const tokenList: string[] = getTokenListFromInputsAndOutputs(tx.inputs, outputs);
+
+  // Update transaction count with the new tx
+  await incrementTokensTxCount(mysql, tokenList);
+
   // update address tables (address, address_balance, address_tx_history)
   await updateAddressTablesWithTx(mysql, txId, tx.timestamp, addressBalanceMap);
 

src/types.ts

@@ -102,10 +102,13 @@ export class TokenInfo {
 
   symbol: string;
 
-  constructor(id: string, name: string, symbol: string) {
+  transactions: number;
+
+  constructor(id: string, name: string, symbol: string, transactions?: number) {
     this.id = id;
     this.name = name;
     this.symbol = symbol;
+    this.transactions = transactions || 0;
 
     const hathorConfig = hathorLib.constants.HATHOR_TOKEN_CONFIG;
 

tests/api.test.ts

@@ -261,16 +261,19 @@ test('GET /balances', async () => {
     readyAt: 10001,
   }]);
 
+  // add the hathor token as it will be deleted by the beforeAll
+  const htrToken = { id: '00', name: 'Hathor', symbol: 'HTR' };
   // add tokens
   const token1 = { id: 'token1', name: 'MyToken1', symbol: 'MT1' };
   const token2 = { id: 'token2', name: 'MyToken2', symbol: 'MT2' };
   const token3 = { id: 'token3', name: 'MyToken3', symbol: 'MT3' };
   const token4 = { id: 'token4', name: 'MyToken4', symbol: 'MT4' };
   await addToTokenTable(mysql, [
-    { id: token1.id, name: token1.name, symbol: token1.symbol },
-    { id: token2.id, name: token2.name, symbol: token2.symbol },
-    { id: token3.id, name: token3.name, symbol: token3.symbol },
-    { id: token4.id, name: token4.name, symbol: token4.symbol },
+    { ...htrToken, transactions: 0 },
+    { id: token1.id, name: token1.name, symbol: token1.symbol, transactions: 0 },
+    { id: token2.id, name: token2.name, symbol: token2.symbol, transactions: 0 },
+    { id: token3.id, name: token3.name, symbol: token3.symbol, transactions: 0 },
+    { id: token4.id, name: token4.name, symbol: token4.symbol, transactions: 0 },
   ]);
 
   // missing wallet
@@ -1249,8 +1252,8 @@ test('GET /wallet/tokens/token_id/details', async () => {
   const token2 = { id: 'token2', name: 'MyToken2', symbol: 'MT2' };
 
   await addToTokenTable(mysql, [
-    { id: token1.id, name: token1.name, symbol: token1.symbol },
-    { id: token2.id, name: token2.name, symbol: token2.symbol },
+    { id: token1.id, name: token1.name, symbol: token1.symbol, transactions: 0 },
+    { id: token2.id, name: token2.name, symbol: token2.symbol, transactions: 0 },
   ]);
 
   await addToUtxoTable(mysql, [

tests/db.test.ts

@@ -63,6 +63,7 @@ import {
   getTotalTransactions,
   getAvailableAuthorities,
   getAffectedAddressTxCountFromTxList,
+  incrementTokensTxCount,
 } from '@src/db';
 import {
   beginTransaction,
@@ -109,6 +110,7 @@ import {
   createOutput,
   createInput,
   countTxOutputTable,
+  checkTokenTable,
 } from '@tests/utils';
 import { AddressTxHistoryTableEntry } from '@tests/types';
 
@@ -917,8 +919,8 @@ test('getWalletBalances', async () => {
   }]);
 
   await addToTokenTable(mysql, [
-    { id: token1.id, name: token1.name, symbol: token1.symbol },
-    { id: token2.id, name: token2.name, symbol: token2.symbol },
+    { id: token1.id, name: token1.name, symbol: token1.symbol, transactions: 0 },
+    { id: token2.id, name: token2.name, symbol: token2.symbol, transactions: 0 },
   ]);
 
   // first test fetching all tokens
@@ -1633,6 +1635,18 @@ test('rebuildAddressBalancesFromUtxos', async () => {
 
   await addToAddressTxHistoryTable(mysql, txHistory);
 
+  // add to the token table
+  await addToTokenTable(mysql, [
+    { id: token1, name: 'token1', symbol: 'TKN1', transactions: 2 },
+  ]);
+
+  await expect(checkTokenTable(mysql, 1, [{
+    tokenId: token1,
+    tokenSymbol: 'TKN1',
+    tokenName: 'token1',
+    transactions: 2,
+  }])).resolves.toBe(true);
+
   // We are only using the txList parameter on `transactions` recalculation, so our balance
   // checks should include txId3 and txId4, but the transaction count should not.
   await rebuildAddressBalancesFromUtxos(mysql, [addr1, addr2], [txId3, txId4]);
@@ -1655,6 +1669,13 @@ test('rebuildAddressBalancesFromUtxos', async () => {
   expect(addressBalances[2].address).toStrictEqual(addr2);
   expect(addressBalances[2].transactions).toStrictEqual(1);
   expect(addressBalances[2].tokenId).toStrictEqual('token2');
+
+  await expect(checkTokenTable(mysql, 1, [{
+    tokenId: token1,
+    tokenSymbol: 'TKN1',
+    tokenName: 'token1',
+    transactions: 0,
+  }])).resolves.toBe(true);
 });
 
 test('markAddressTxHistoryAsVoided', async () => {
@@ -2121,3 +2142,36 @@ test('getAffectedAddressTxCountFromTxList', async () => {
   // We should get an empty object if no addresses have been affected:
   expect(await getAffectedAddressTxCountFromTxList(mysql, [txId2])).toStrictEqual({});
 });
+
+test('incrementTokensTxCount', async () => {
+  expect.hasAssertions();
+
+  const htr = new TokenInfo('00', 'Hathor', 'HTR', 5);
+  const token1 = new TokenInfo('token1', 'MyToken1', 'MT1', 10);
+  const token2 = new TokenInfo('token2', 'MyToken2', 'MT2', 15);
+
+  await addToTokenTable(mysql, [
+    { id: htr.id, name: htr.name, symbol: htr.symbol, transactions: htr.transactions },
+    { id: token1.id, name: token1.name, symbol: token1.symbol, transactions: token1.transactions },
+    { id: token2.id, name: token2.name, symbol: token2.symbol, transactions: token2.transactions },
+  ]);
+
+  await incrementTokensTxCount(mysql, ['token1', '00', 'token2']);
+
+  await expect(checkTokenTable(mysql, 3, [{
+    tokenId: token1.id,
+    tokenSymbol: token1.symbol,
+    tokenName: token1.name,
+    transactions: token1.transactions + 1,
+  }, {
+    tokenId: token2.id,
+    tokenSymbol: token2.symbol,
+    tokenName: token2.name,
+    transactions: token2.transactions + 1,
+  }, {
+    tokenId: htr.id,
+    tokenSymbol: htr.symbol,
+    tokenName: htr.name,
+    transactions: htr.transactions + 1,
+  }])).resolves.toBe(true);
+});