Correlating logs to traces (cadence-workflow#980)

* logs correlation

* update comment

Author: Assem-Uber

next.config.mjs

@@ -19,7 +19,12 @@ const nextConfig = {
       '@': path.resolve(__dirname, './src'),
     };
     if (options.isServer) {
-      config.externals.push('@grpc/grpc-js', 'require-in-the-middle', '@opentelemetry/exporter-jaeger');
+      config.externals.push(
+        '@grpc/grpc-js',
+        'require-in-the-middle',
+        'pino',
+        /^@opentelemetry\//
+      );
     }
     return config;
   },

package-lock.json

@@ -33,6 +33,7 @@
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.200.0",
     "@opentelemetry/instrumentation-grpc": "^0.200.0",
     "@opentelemetry/instrumentation-http": "^0.200.0",
+    "@opentelemetry/instrumentation-pino": "^0.50.0",
     "@opentelemetry/instrumentation-undici": "^0.11.0",
     "@opentelemetry/propagator-jaeger": "^2.0.0",
     "@opentelemetry/resources": "^2.0.0",

package.json

@@ -1,16 +1,32 @@
-import logger, { registerLoggers } from '@/utils/logger';
+import { type Instrumentation } from '@opentelemetry/instrumentation';
 
 import getTransformedConfigs from './utils/config/get-transformed-configs';
 import { setLoadedGlobalConfigs } from './utils/config/global-configs-ref';
 
 export async function register() {
+  let instrumentations: (Instrumentation | Instrumentation[])[] = [];
+  // register instrumentations before any other code is executed
+  if (
+    process.env.NEXT_RUNTIME === 'nodejs' &&
+    process.env.OTEL_SDK_DISABLED === 'false'
+  ) {
+    instrumentations = (
+      await import('@/utils/otel/otel-register-instrumentations')
+    ).otelRegisterInstrumentations({});
+  }
+
+  // import/register loggers after instrumentations are registered to have instrumented pino along with logs correlation
+  const { registerLoggers, default: logger } = await import('@/utils/logger');
   registerLoggers();
 
   if (process.env.NEXT_RUNTIME === 'nodejs') {
     if (process.env.OTEL_SDK_DISABLED === 'false') {
-      (await import('@/utils/otel/otel-register')).register();
+      await (
+        await import('@/utils/otel/otel-register')
+      ).otelRegister({
+        instrumentations,
+      });
     }
-
     try {
       const configs = await getTransformedConfigs();
       setLoadedGlobalConfigs(configs);

src/instrumentation.ts

@@ -0,0 +1,35 @@
+import {
+  GrpcInstrumentation,
+  type GrpcInstrumentationConfig,
+} from '@opentelemetry/instrumentation-grpc';
+import {
+  HttpInstrumentation,
+  type HttpInstrumentationConfig,
+} from '@opentelemetry/instrumentation-http';
+import {
+  PinoInstrumentation,
+  type PinoInstrumentationConfig,
+} from '@opentelemetry/instrumentation-pino';
+import {
+  UndiciInstrumentation,
+  type UndiciInstrumentationConfig,
+} from '@opentelemetry/instrumentation-undici';
+
+export function otelRegisterInstrumentations({
+  grpcInstrumentationConfig,
+  httpInstrumentationConfig,
+  undiciInstrumentationConfig,
+  pinoInstrumentationConfig,
+}: {
+  grpcInstrumentationConfig?: GrpcInstrumentationConfig;
+  httpInstrumentationConfig?: HttpInstrumentationConfig;
+  undiciInstrumentationConfig?: UndiciInstrumentationConfig;
+  pinoInstrumentationConfig?: PinoInstrumentationConfig;
+} = {}) {
+  return [
+    new GrpcInstrumentation(grpcInstrumentationConfig),
+    new HttpInstrumentation(httpInstrumentationConfig),
+    new UndiciInstrumentation(undiciInstrumentationConfig),
+    new PinoInstrumentation(pinoInstrumentationConfig),
+  ];
+}

src/utils/otel/otel-register-instrumentations.ts

@@ -1,7 +1,4 @@
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
-import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';
 import { JaegerPropagator } from '@opentelemetry/propagator-jaeger';
 import { resourceFromAttributes } from '@opentelemetry/resources';
 import { NodeSDK } from '@opentelemetry/sdk-node';
@@ -11,20 +8,14 @@ import logger from '@/utils/logger';
 
 import { type OtelRegisterConfig } from './otel.types';
 
-export async function register(config?: OtelRegisterConfig) {
-  const { grpcInstrumentationConfig, ...sdkConfig } = config || {};
+export async function otelRegister(config?: OtelRegisterConfig) {
   const sdk = new NodeSDK({
     resource: resourceFromAttributes({
       [ATTR_SERVICE_NAME]: 'cadence-web',
     }),
-    instrumentations: [
-      new GrpcInstrumentation(grpcInstrumentationConfig),
-      new HttpInstrumentation(),
-      new UndiciInstrumentation(),
-    ],
     textMapPropagator: new JaegerPropagator(),
     traceExporter: new OTLPTraceExporter(),
-    ...sdkConfig,
+    ...config,
   });
   try {
     await sdk.start();

src/utils/otel/otel-register.ts

@@ -1,6 +1,3 @@
-import { type GrpcInstrumentationConfig } from '@opentelemetry/instrumentation-grpc';
 import { type NodeSDKConfiguration } from '@opentelemetry/sdk-node';
 
-export type OtelRegisterConfig = Partial<NodeSDKConfiguration> & {
-  grpcInstrumentationConfig?: GrpcInstrumentationConfig;
-};
+export type OtelRegisterConfig = Partial<NodeSDKConfiguration>;