Merge pull request #272 from HathorNetwork/dev

Release v1.16.0-alpha

Author: andreabadesso

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "hathor-wallet-service",
-  "version": "1.15.2-alpha",
+  "version": "1.16.0-alpha",
   "description": "",
   "scripts": {
     "postinstall": "npm dedupe",

package.json

@@ -132,6 +132,7 @@ functions:
         enabled: false
   onNewTxRequest:
     handler: src/txProcessor.onNewTxRequest
+    timeout: 12 # seconds
     warmup:
       walletWarmer:
         enabled: false

serverless.yml

@@ -429,15 +429,27 @@ export const handleVoided = async (mysql: ServerlessMysql, logger: Logger, tx: T
     affectedUtxoList = [...affectedUtxoList, ...newAffectedUtxoList];
   }
 
-  // fetch all addresses affected by the voided tx
-  const affectedAddresses = affectedUtxoList.reduce((acc: Set<string>, utxo: DbTxOutput) => acc.add(utxo.address), new Set<string>());
+  // fetch all addresses and transactions affected by the voided transaction
+  const [affectedAddresses, affectedTxIds] = affectedUtxoList.reduce(
+    (acc: [Set<string>, Set<string>], utxo: DbTxOutput) => {
+      acc[0].add(utxo.address);
+      acc[1].add(utxo.txId);
+
+      return acc;
+    },
+    [new Set<string>(), new Set<string>()],
+  );
 
   if (affectedAddresses.size > 0) {
     const addresses = [...affectedAddresses];
+
     logger.debug(`Rebuilding balances for ${addresses.length} addresses.`, {
       addresses,
     });
-    await rebuildAddressBalancesFromUtxos(mysql, addresses);
+    logger.debug(`Rebuilding tx count from ${affectedTxIds.size} transactions`, {
+      affectedTxIds,
+    });
+    await rebuildAddressBalancesFromUtxos(mysql, addresses, [...affectedTxIds]);
     await validateAddressBalances(mysql, addresses);
   }
 
@@ -448,9 +460,6 @@ export const validateAddressBalances = async (mysql: ServerlessMysql, addresses:
   const addressBalances: AddressBalance[] = await fetchAddressBalance(mysql, addresses);
   const addressTxHistorySums: AddressTotalBalance[] = await fetchAddressTxHistorySum(mysql, addresses);
 
-  // if we have the full history of transactions, the number of rows must match:
-  assert.strictEqual(addressBalances.length, addressTxHistorySums.length);
-
   for (let i = 0; i < addressTxHistorySums.length; i++) {
     const addressBalance: AddressBalance = addressBalances[i];
     const addressTxHistorySum: AddressTotalBalance = addressTxHistorySums[i];
@@ -520,15 +529,26 @@ export const handleReorg = async (mysql: ServerlessMysql, logger: Logger): Promi
     await removeTxsHeight(mysql, remainingTxs);
   }
 
-  // fetch all addresses affected by the reorg
-  const affectedAddresses = affectedUtxoList.reduce((acc: Set<string>, utxo: DbTxOutput) => acc.add(utxo.address), new Set<string>());
+  // fetch all addresses and transactions affected by the reorg
+  const [affectedAddresses, affectedTxIds] = affectedUtxoList.reduce(
+    (acc: [Set<string>, Set<string>], utxo: DbTxOutput) => {
+      acc[0].add(utxo.address);
+      acc[1].add(utxo.txId);
+
+      return acc;
+    },
+    [new Set<string>(), new Set<string>()],
+  );
 
   if (affectedAddresses.size > 0) {
     const addresses = [...affectedAddresses];
     logger.debug(`Rebuilding balances for ${addresses.length} addresses.`, {
       addresses,
     });
-    await rebuildAddressBalancesFromUtxos(mysql, addresses);
+    logger.debug(`Rebuilding tx count from ${affectedTxIds.size} transactions`, {
+      affectedTxIds,
+    });
+    await rebuildAddressBalancesFromUtxos(mysql, addresses, [...affectedTxIds]);
     await validateAddressBalances(mysql, addresses);
   }
 

src/commons.ts

@@ -2008,15 +2008,35 @@ export const markWalletTxHistoryAsVoided = async (
 
  * @param mysql - Database connection
  * @param addresses - The list of addresses to rebuild
+ * @param txList - The list of affected transactions, to rebuild the transaction count
  */
 export const rebuildAddressBalancesFromUtxos = async (
   mysql: ServerlessMysql,
   addresses: string[],
+  txList: string[],
 ): Promise<void> => {
+  if (txList.length === 0) {
+    // This should never happen, we should throw so the re-org is rolled back
+    // and an error is triggered for manual inspection
+    throw new Error('Attempted to rebuild address balances but no transactions were affected');
+  }
+  // first we need to store the transactions count before deleting
+  const oldAddressTokenTransactions: DbSelectResult = await mysql.query(
+    `SELECT \`address\`, \`token_id\` AS tokenId, \`transactions\`
+       FROM \`address_balance\`
+      WHERE \`address\` IN (?)`,
+    [addresses],
+  );
+
   // delete affected address_balances
   await mysql.query(
-    `DELETE
-       FROM \`address_balance\`
+    `UPDATE \`address_balance\`
+        SET \`unlocked_balance\` = 0,
+            \`locked_balance\` = 0,
+            \`locked_authorities\` = 0,
+            \`unlocked_authorities\` = 0,
+            \`timelock_expires\` = NULL,
+            \`transactions\` = 0
       WHERE \`address\` IN (?)`,
     [addresses],
   );
@@ -2033,20 +2053,23 @@ export const rebuildAddressBalancesFromUtxos = async (
       \`timelock_expires\`,
       \`transactions\`
     )
-     SELECT address,
-            token_id,
-            SUM(\`value\`), -- unlocked_balance
-            0,
-            BIT_OR(\`authorities\`), -- unlocked_authorities
-            0, -- locked_authorities
-            0, -- timelock_expires
-            COUNT(DISTINCT \`tx_id\`) -- transactions
-       FROM \`tx_output\`
-      WHERE spent_by IS NULL
-        AND voided = FALSE
-        AND locked = FALSE
-        AND address IN (?)
-   GROUP BY address, token_id
+        SELECT address,
+                token_id,
+                SUM(\`value\`), -- unlocked_balance
+                0,
+                BIT_OR(\`authorities\`), -- unlocked_authorities
+                0, -- locked_authorities
+                NULL, -- timelock_expires
+                0 -- transactions
+          FROM \`tx_output\`
+         WHERE spent_by IS NULL
+           AND voided = FALSE
+           AND locked = FALSE
+           AND address IN (?)
+      GROUP BY address, token_id
+   ON DUPLICATE KEY UPDATE
+    unlocked_balance = VALUES(unlocked_balance),
+    unlocked_authorities = VALUES(unlocked_authorities)
   `, [addresses]);
 
   // update address balances with locked utxos
@@ -2066,7 +2089,7 @@ export const rebuildAddressBalancesFromUtxos = async (
               SUM(\`value\`) AS locked_balance,
               BIT_OR(\`authorities\`) AS locked_authorities,
               MIN(\`timelock\`) AS timelock_expires,
-              COUNT(DISTINCT \`tx_id\`) -- transactions
+              0 -- transactions
          FROM \`tx_output\`
         WHERE spent_by IS NULL
           AND voided = FALSE
@@ -2076,9 +2099,31 @@ export const rebuildAddressBalancesFromUtxos = async (
    ON DUPLICATE KEY UPDATE
     locked_balance = VALUES(locked_balance),
     locked_authorities = VALUES(locked_authorities),
-    timelock_expires = VALUES(timelock_expires),
-    transactions = transactions + VALUES(\`transactions\`)
+    timelock_expires = VALUES(timelock_expires)
    `, [addresses]);
+
+  const addressTransactionCount: StringMap<number> = await getAffectedAddressTxCountFromTxList(mysql, txList);
+  const finalTxCount = oldAddressTokenTransactions.map(({ address, tokenId, transactions }) => {
+    const diff = addressTransactionCount[`${address}_${tokenId}`] || 0;
+
+    return [address, tokenId, transactions as number - diff];
+  });
+
+  // update address balances with the correct amount of transactions
+  // We have to run multiple updates because we don't want to insert new rows to the table (which would be done
+  // if we used the INSERT ... ON CONFLICT syntax)
+  for (const addressTokenTx of finalTxCount) {
+    await mysql.query(`
+      UPDATE \`address_balance\` 
+         SET \`transactions\` = ?
+       WHERE \`address\` = ?
+         AND \`token_id\` = ?
+    `, [
+      addressTokenTx[2],
+      addressTokenTx[0],
+      addressTokenTx[1],
+    ]);
+  }
 };
 
 /**
@@ -2461,3 +2506,37 @@ export const getAvailableAuthorities = async (
 
   return utxos;
 };
+
+/**
+ * Get the number of transactions for each token from the address_tx_history table
+ * given a list of transactions
+ *
+ * @param mysql - Database connection
+ * @param txList - A list of affected transactions to get the addresses token transaction count
+
+ * @returns A Map with address_tokenId as key and the transaction count as values
+ */
+export const getAffectedAddressTxCountFromTxList = async (
+  mysql: ServerlessMysql,
+  txList: string[],
+): Promise<StringMap<number>> => {
+  const results: DbSelectResult = await mysql.query(`
+    SELECT address, COUNT(DISTINCT(tx_id)) AS txCount, token_id as tokenId
+      FROM address_tx_history 
+     WHERE tx_id IN (?)
+       AND voided = TRUE
+  GROUP BY address, token_id
+  `, [txList]);
+
+  const addressTransactions = results.reduce((acc, result) => {
+    const address = result.address as string;
+    const txCount = result.txCount as number;
+    const tokenId = result.tokenId as string;
+
+    acc[`${address}_${tokenId}`] = txCount;
+
+    return acc;
+  }, {});
+
+  return addressTransactions as StringMap<number>;
+};

src/db/index.ts

@@ -1265,9 +1265,9 @@ test('GET /wallet/tokens/token_id/details', async () => {
   ]);
 
   await addToAddressTxHistoryTable(mysql, [
-    [ADDRESSES[0], 'txId', token1.id, 100, 0],
-    [ADDRESSES[0], 'txId2', token2.id, 250, 0],
-    [ADDRESSES[0], 'txId3', token2.id, 0, 0],
+    { address: ADDRESSES[0], txId: 'txId', tokenId: token1.id, balance: 100, timestamp: 0 },
+    { address: ADDRESSES[0], txId: 'txId2', tokenId: token2.id, balance: 250, timestamp: 0 },
+    { address: ADDRESSES[0], txId: 'txId3', tokenId: token2.id, balance: 0, timestamp: 0 },
   ]);
 
   event = makeGatewayEventWithAuthorizer('my-wallet', { token_id: token1.id });