Merge pull request #477 from hack-a-chain-software/blocks-from-height

refactor: improved blocks-from-height query

Author: mschead

indexer/src/kadena-server/repository/infra/repository/block-db-repository.ts

@@ -193,11 +193,18 @@ export default class BlockDbRepository implements BlockRepository {
   }
 
   /**
-   * Retrieves blocks within a specific height range
+   * Retrieves blocks within a specific height range using a sliding window approach
    *
    * This method fetches blocks between the specified start and end heights,
    * supporting cursor-based pagination and chain filtering for efficient
-   * navigation through large result sets.
+   * navigation through large result sets. To prevent large database queries,
+   * it implements a sliding window approach with a default window size of 50.
+   *
+   * Sliding window behavior:
+   * - If endHeight is not provided or is too far from the current cursor position,
+   *   the query is limited to a window of 50 heights from the current position
+   * - As pagination progresses (via 'after' cursor), the window slides forward
+   * - This prevents unnecessary large queries when users request broad ranges
    *
    * Uses keyset pagination with compound cursors (height:id) for better performance
    * when navigating through large datasets.
@@ -228,8 +235,70 @@ export default class BlockDbRepository implements BlockRepository {
       last,
     });
 
+    const heightQuery = `SELECT max("height") FROM "Blocks"`;
+
+    const { rows: maxHeightRows } = await rootPgPool.query(heightQuery);
+
+    const maxHeight = maxHeightRows[0].max;
+    // Default window size to prevent large database queries
+    const DEFAULT_WINDOW_SIZE = 50;
+
+    // Calculate effective window range
+    let effectiveStartHeight = startHeight;
+    let effectiveEndHeight: number;
+
+    // Determine current position from cursor for window calculation
+    let currentHeight = startHeight;
+    if (after) {
+      const [height] = after.split(':');
+      const afterHeight = parseInt(height, 10);
+      if (!isNaN(afterHeight)) {
+        currentHeight = afterHeight;
+      }
+    } else if (before) {
+      const [height] = before.split(':');
+      const beforeHeight = parseInt(height, 10);
+      if (!isNaN(beforeHeight)) {
+        currentHeight = beforeHeight;
+      }
+    }
+
+    // If no endHeight provided or endHeight is too far from current position,
+    // use sliding window approach
+    if (!endHeight || endHeight - currentHeight > DEFAULT_WINDOW_SIZE) {
+      if (after) {
+        // Forward pagination: window starts from cursor position
+        effectiveStartHeight = currentHeight;
+        effectiveEndHeight = effectiveStartHeight + DEFAULT_WINDOW_SIZE;
+      } else if (before) {
+        // Backward pagination: window ends at cursor position
+        effectiveEndHeight = currentHeight;
+        effectiveStartHeight = Math.max(startHeight, effectiveEndHeight - DEFAULT_WINDOW_SIZE);
+      } else {
+        // Initial request: check if it's 'last' pagination (backward from end)
+        if (last) {
+          // For 'last' without cursor, use the maximum height across all chains
+          const actualEndHeight = endHeight ? Math.min(endHeight, maxHeight) : maxHeight;
+          effectiveEndHeight = actualEndHeight;
+          effectiveStartHeight = Math.max(startHeight, actualEndHeight - DEFAULT_WINDOW_SIZE);
+        } else {
+          // Forward pagination: window starts from startHeight
+          effectiveStartHeight = startHeight;
+          effectiveEndHeight = effectiveStartHeight + DEFAULT_WINDOW_SIZE;
+        }
+      }
+
+      // If original endHeight is provided and within window, respect it (except for 'last' mode)
+      if (endHeight && endHeight <= effectiveEndHeight && !last) {
+        effectiveEndHeight = endHeight;
+      }
+    } else {
+      // Use the provided endHeight when it's within reasonable range
+      effectiveEndHeight = endHeight;
+    }
+
     const order = orderParam === 'ASC' ? 'DESC' : 'ASC';
-    const queryParams: (string | number | string[])[] = [limit, startHeight];
+    const queryParams: (string | number | string[])[] = [limit];
     let conditions = '';
 
     if (before) {
@@ -243,7 +312,7 @@ export default class BlockDbRepository implements BlockRepository {
       }
 
       queryParams.push(beforeHeight, beforeId);
-      conditions += `\nAND (b.height, b.id) > ($${queryParams.length - 1}, $${queryParams.length})`;
+      conditions += `\nAND (b.height, b.id) < ($${queryParams.length - 1}, $${queryParams.length})`;
     }
 
     if (after) {
@@ -256,19 +325,26 @@ export default class BlockDbRepository implements BlockRepository {
         throw new Error('Invalid after cursor');
       }
       queryParams.push(afterHeight, afterId);
-      conditions += `\nAND (b.height, b.id) < ($${queryParams.length - 1}, $${queryParams.length})`;
+      conditions += `\nAND (b.height, b.id) > ($${queryParams.length - 1}, $${queryParams.length})`;
     }
 
     if (chainIds?.length) {
       queryParams.push(chainIds);
       conditions += `\nAND b."chainId" = ANY($${queryParams.length})`;
     }
 
-    if (endHeight) {
-      queryParams.push(endHeight);
-      conditions += `\nAND b."height" <= $${queryParams.length}`;
+    // Apply height range constraints
+    // Only add start height constraint if we don't have an 'after' cursor
+    // (cursor condition already handles the lower bound for 'after')
+    // For 'before' cursor, we still need the lower bound
+    if (!after) {
+      queryParams.push(effectiveStartHeight);
+      conditions += `\nAND b."height" >= $${queryParams.length}`;
     }
 
+    queryParams.push(effectiveEndHeight);
+    conditions += `\nAND b."height" <= $${queryParams.length}`;
+
     const query = `
       SELECT b.id,
         b.hash,
@@ -288,7 +364,7 @@ export default class BlockDbRepository implements BlockRepository {
         b."transactionsCount" as "transactionsCount",
         b."totalGasUsed" as "totalGasUsed"
       FROM "Blocks" b
-      WHERE b.height ${before ? '<=' : '>='} $2
+      WHERE 1=1
       ${conditions}
       ORDER BY b.height ${order}, b.id ${order}
       LIMIT $1;
@@ -301,7 +377,7 @@ export default class BlockDbRepository implements BlockRepository {
       node: blockValidator.validate(row),
     }));
 
-    const pageInfo = getPageInfo({ edges, order, limit, after, before });
+    const pageInfo = getPageInfo({ edges, order: orderParam, limit, after, before });
     return pageInfo;
   }
 

indexer/src/kadena-server/repository/infra/repository/network-db-repository.ts

@@ -224,7 +224,7 @@ export default class NetworkDbRepository implements NetworkRepository {
 
   async getCurrentChainHeights(): Promise<CurrentChainHeights> {
     const heightsQuery = `
-      SELECT "chainId", "canonicalBlocks" FROM "Counters" ORDER BY "canonicalBlocks"
+      SELECT "chainId", "canonicalBlocks" FROM "Counters"
     `;
 
     const { rows } = await rootPgPool.query(heightsQuery);

indexer/src/kadena-server/resolvers/fields/query-blocks-from-height-connection/total-count-query-blocks-from-height-connection-resolver.ts

@@ -1,15 +1,14 @@
 /**
- * Resolver for the totalCount field of the QueryEventsConnection type.
- * This module counts blockchain events based on various filter criteria.
+ * Resolver for the totalCount field of the QueryBlocksFromHeightConnection type.
+ * This module counts blockchain blocks based on various filter criteria.
  */
 import { ResolverContext } from '../../../config/apollo-server-config';
 import { QueryBlocksFromHeightConnectionResolvers } from '../../../config/graphql-types';
 import zod from 'zod';
 
 /**
- * Zod schema for validating event query parameters.
- * Defines the structure and types of filter parameters used to count events.
- * Note that qualifiedEventName is required, while other filters are optional.
+ * Zod schema for validating block query parameters.
+ * Defines the structure and types of filter parameters used to count blocks.
  */
 const schema = zod.object({
   chainIds: zod.array(zod.string()).optional(),
@@ -18,13 +17,83 @@ const schema = zod.object({
 });
 
 /**
- * Resolver function for the totalCount field of the QueryEventsConnection type.
- * Retrieves the total count of events matching the provided filter criteria.
+ * Interface for genesis height information
+ */
+interface GenesisHeight {
+  chainId: string;
+  height: number;
+}
+
+/**
+ * Interface for current chain heights
+ */
+interface CurrentChainHeights {
+  [chainId: string]: number;
+}
+
+/**
+ * Calculates the total count of blocks based on chain IDs and height ranges.
+ *
+ * Logic:
+ * - Chains 0-9: Start from genesis height (typically 0) to current/end height
+ * - Chains 10-19: Start from their specific genesis height to current/end height
+ *
+ * @param startHeight - The starting height for the query
+ * @param endHeight - The ending height for the query (null means use current heights)
+ * @param chainIds - Array of chain IDs to include (empty means all chains)
+ * @param genesisHeights - Array of genesis height information for each chain
+ * @param currentChainHeights - Current maximum height for each chain
+ * @returns The total count of blocks across all specified chains
+ */
+function calculateTotalBlockCount(
+  startHeight: number,
+  endHeight: number | null,
+  chainIds: string[],
+  genesisHeights: GenesisHeight[],
+  currentChainHeights: CurrentChainHeights,
+): number {
+  // Create a map for quick lookup of genesis heights
+  const genesisHeightMap = genesisHeights.reduce(
+    (map, { chainId, height }) => {
+      map[chainId] = height;
+      return map;
+    },
+    {} as Record<string, number>,
+  );
+
+  return chainIds.reduce((totalCount, chainId) => {
+    const genesisHeight = genesisHeightMap[chainId] ?? 0;
+    const currentMaxHeight = currentChainHeights[chainId] ?? 0;
+
+    // Use the minimum of provided endHeight and actual max height in database
+    const endHeightToUse =
+      endHeight === null || endHeight === undefined
+        ? currentMaxHeight
+        : Math.min(endHeight, currentMaxHeight);
+
+    // Ensure we don't go below the genesis height for any chain
+    const effectiveStartHeight = Math.max(startHeight, genesisHeight);
+
+    // If the effective start height is greater than the end height, no blocks exist in this range
+    if (effectiveStartHeight > endHeightToUse) {
+      return totalCount; // No blocks to add for this chain
+    }
+
+    // Calculate the count for this chain
+    const chainBlockCount = endHeightToUse - effectiveStartHeight + 1;
+
+    return totalCount + chainBlockCount;
+  }, 0);
+}
+
+/**
+ * Resolver function for the totalCount field of the QueryBlocksFromHeightConnection type.
+ * Retrieves the total count of blocks matching the provided filter criteria.
  *
  * @param parent - The parent object containing filter parameters
  * @param _args - GraphQL arguments (unused in this resolver)
  * @param context - The resolver context containing repositories and services
- * @returns The total count of events matching the criteria
+ * @returns The total count of blocks matching the criteria
  */
 export const totalCountQueryBlocksFromHeightConnectionResolver: QueryBlocksFromHeightConnectionResolvers<ResolverContext>['totalCount'] =
   async (parent, _args, context) => {
@@ -42,22 +111,16 @@ export const totalCountQueryBlocksFromHeightConnectionResolver: QueryBlocksFromH
       chainIds = [...networkInfo.nodeChains];
     }
 
+    const networkInfo = await context.networkRepository.getAllInfo();
     const currentChainHeights = await context.networkRepository.getCurrentChainHeights();
-    const startHeightOfChainsFrom9to19 = 852054;
-    const totalCount = chainIds.reduce((acum, chainId) => {
-      const endHeightToUse = endHeight ?? currentChainHeights[chainId];
-      let totalOfChain = endHeightToUse - startHeight + 1;
-      if (Number(chainId) > 9) {
-        if (startHeight >= startHeightOfChainsFrom9to19) {
-          totalOfChain = endHeightToUse - startHeight + 1;
-        } else if (endHeightToUse < startHeightOfChainsFrom9to19) {
-          totalOfChain = 0;
-        } else {
-          totalOfChain = endHeightToUse - startHeightOfChainsFrom9to19 + 1;
-        }
-      }
-      return totalOfChain + acum;
-    }, 0);
+
+    const totalCount = calculateTotalBlockCount(
+      startHeight,
+      endHeight ?? null,
+      chainIds,
+      networkInfo.genesisHeights,
+      currentChainHeights,
+    );
 
     return totalCount;
   };

indexer/tests/integration/builders/blocks-from-height.builder.ts

@@ -1,8 +1,8 @@
 import { gql } from 'graphql-request';
 
-export const getBlocksFromHeightQuery = (params: any): string => {
+const buildBlocksFromHeightQuery = (params: any, includeTotalCount: boolean = true): string => {
   if (Object.keys(params).length === 0) {
-    throw new Error('No parameters provided to getBlocksFromHeightQuery.');
+    throw new Error('No parameters provided to buildBlocksFromHeightQuery.');
   }
 
   const query = Object.entries(params)
@@ -14,9 +14,12 @@ export const getBlocksFromHeightQuery = (params: any): string => {
     })
     .join(', ');
 
+  const totalCountField = includeTotalCount ? 'totalCount' : '';
+
   const queryGql = gql`
     query {
       blocksFromHeight(${query}) {
+        ${totalCountField}
         edges {
           cursor
           node {
@@ -54,7 +57,6 @@ export const getBlocksFromHeightQuery = (params: any): string => {
             id
             minerAccount {
               accountName
-              balance
               chainId
               fungibleName
               guard {
@@ -101,3 +103,11 @@ export const getBlocksFromHeightQuery = (params: any): string => {
 
   return queryGql;
 };
+
+export const getBlocksFromHeightQuery = (params: any): string => {
+  return buildBlocksFromHeightQuery(params, true);
+};
+
+export const getBlocksFromHeightWithoutTotalCountQuery = (params: any): string => {
+  return buildBlocksFromHeightQuery(params, false);
+};