Merge pull request #504 from hack-a-chain-software/memory-leak

fix: removed memory leak by adding a correct configuration to pg pool

Author: mschead

Dockerfile

@@ -1,18 +1,24 @@
 FROM node:18-alpine as builder
 WORKDIR /app
 COPY indexer yarn.lock ./
+# Install build dependencies for native modules like heapdump
+RUN apk add --no-cache python3 make g++
 RUN rm -rf node_modules && yarn install --frozen-lockfile
 RUN npx graphql-codegen
 RUN yarn build
 
 FROM node:18-alpine
 WORKDIR /app
 COPY indexer/package.json indexer/tsconfig.json yarn.lock ./
+# Install build dependencies for native modules like heapdump
+RUN apk add --no-cache python3 make g++
 RUN yarn install --frozen-lockfile
 COPY --from=builder /app/dist ./dist
 COPY --from=builder /app/src/config/global-bundle.pem ./dist/config/global-bundle.pem
 COPY --from=builder /app/src/kadena-server/config/schema.graphql ./dist/kadena-server/config/schema.graphql
 COPY --from=builder /app/src/circulating-coins/ ./dist/circulating-coins/
+# Create snapshots directory for heap dumps
+RUN mkdir -p /snapshots
 EXPOSE 3001
 
 ARG INDEXER_MODE_PARAM

indexer/package.json

@@ -51,6 +51,7 @@
     "@types/decimal.js": "^7.4.3",
     "@types/eventsource": "^1.1.15",
     "@types/express": "^4.17.21",
+    "@types/heapdump": "^0.3.4",
     "@types/jest": "^29.5.12",
     "@types/mocha": "^10.0.7",
     "@types/node": "^20.11.17",
@@ -61,6 +62,7 @@
     "dotenv": "^16.4.4",
     "dotenv-cli": "^8.0.0",
     "eslint": "^8.56.0",
+    "heapdump": "^0.3.15",
     "jest": "^29.7.0",
     "mocha": "^10.7.3",
     "module-alias": "^2.2.3",

indexer/src/config/database.ts

@@ -44,12 +44,14 @@ if (isSslEnabled) {
  * This provides a lower-level database access mechanism than Sequelize.
  *
  * When SSL is enabled, it uses the global certificate bundle for secure connections.
- *
- * TODO: [OPTIMIZATION] Consider implementing connection pooling metrics to monitor performance
- * and adjust pool settings accordingly.
  */
 export const rootPgPool = new Pool({
   connectionString: DB_CONNECTION,
+  max: 20, // Max connections
+  idleTimeoutMillis: 30000, // Close idle connections after 30s
+  connectionTimeoutMillis: 10000, // Fail if can't get connection in 10s
+  statement_timeout: 30000, // Cancel queries after 30s (PostgreSQL)
+  query_timeout: 30000, // Client-side query timeout
   ...(isSslEnabled && {
     ssl: {
       rejectUnauthorized: rejectUnauthorized,

indexer/src/kadena-server/server.ts

@@ -19,6 +19,7 @@
  */
 
 import './plugins/instrument';
+import heapdump from 'heapdump';
 import { ApolloServer, ApolloServerPlugin } from '@apollo/server';
 import { expressMiddleware } from '@apollo/server/express4';
 import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer';
@@ -510,6 +511,23 @@ export async function startGraphqlServer() {
     res.status(404).end();
   });
 
+  const logMemoryUsage = () => {
+    const mem = process.memoryUsage();
+    console.info(
+      `[INFO][MEMORY] Heap: ${(mem.heapUsed / 1024 / 1024).toFixed(2)}MB / ${(mem.heapTotal / 1024 / 1024).toFixed(2)}MB | External: ${(mem.external / 1024 / 1024).toFixed(2)}MB | RSS: ${(mem.rss / 1024 / 1024).toFixed(2)}MB`,
+    );
+
+    // Capture heap snapshot at 1GB memory threshold
+    if (mem.heapUsed > 1 * 1024 * 1024 * 1024) {
+      const filename = `/snapshots/indexer-heap-1GB-${Date.now()}.heapsnapshot`;
+      console.warn(`[WARN][MEMORY] 1GB threshold reached! Capturing heap snapshot: ${filename}`);
+      heapdump.writeSnapshot(filename);
+    }
+  };
+
+  // Start periodic memory monitoring every one hour
+  setInterval(logMemoryUsage, 1000 * 60 * 60);
+
   // Initialize cache and start the server
   await initCache(context);
   await new Promise<void>(resolve => httpServer.listen({ port: KADENA_GRAPHQL_API_PORT }, resolve));

yarn.lock

@@ -2663,6 +2663,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/heapdump@^0.3.4":
+  version "0.3.4"
+  resolved "https://registry.yarnpkg.com/@types/heapdump/-/heapdump-0.3.4.tgz#59645b30165d832ea654449c8ffbed0c156ad60c"
+  integrity sha512-gTGn7W7i58F1yzvLZ5+hRE7M/VqgEzgccERJ9ZThC6qor5R8wo3oP7iiHvwgHIZzoqPF22ngu8mBagcqPWMVTg==
+
 "@types/http-errors@*":
   version "2.0.4"
   resolved "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz"
@@ -5345,6 +5350,13 @@ header-case@^2.0.4:
     capital-case "^1.0.4"
     tslib "^2.0.3"
 
+heapdump@^0.3.15:
+  version "0.3.15"
+  resolved "https://registry.yarnpkg.com/heapdump/-/heapdump-0.3.15.tgz#631a8a2585588ea64778d8ec80a64c6c025f6a08"
+  integrity sha512-n8aSFscI9r3gfhOcAECAtXFaQ1uy4QSke6bnaL+iymYZ/dWs9cqDqHM+rALfsHUwukUbxsdlECZ0pKmJdQ/4OA==
+  dependencies:
+    nan "^2.13.2"
+
 html-escaper@^2.0.0:
   version "2.0.2"
   resolved "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz"
@@ -6695,6 +6707,11 @@ [email protected]:
   resolved "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz"
   integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
 
+nan@^2.13.2:
+  version "2.23.0"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.23.0.tgz#24aa4ddffcc37613a2d2935b97683c1ec96093c6"
+  integrity sha512-1UxuyYGdoQHcGg87Lkqm3FzefucTa0NAiOcuRsDmysep3c1LVCRK2krrUDafMWtjSG04htvAmvg96+SDknOmgQ==
+
 napi-wasm@^1.1.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/napi-wasm/-/napi-wasm-1.1.0.tgz"