Reduce frequency of temporal telemetry (#2914)

Author: joanagmaia

services/archetypes/worker/src/activities/activityInterceptor.ts

@@ -18,7 +18,7 @@ export class ActivityMonitoringInterceptor implements ActivityInboundCallsInterc
       activity_type: this.ctx.info.activityType,
     }
 
-    const timer = telemetry.timer('temporal.activity_execution_duration', tags)
+    const start = new Date()
 
     try {
       const res = await next(input)
@@ -36,7 +36,13 @@ export class ActivityMonitoringInterceptor implements ActivityInboundCallsInterc
       log.error(err, 'Error while processing an activity!')
       throw err
     } finally {
-      timer.stop()
+      const end = new Date()
+      const duration = end.getTime() - start.getTime()
+
+      // Only send telemetry if duration is more than 2 hours
+      if (duration > 2 * 60 * 60 * 1000) {
+        telemetry.distribution('temporal.activity_execution_duration', duration, tags)
+      }
     }
   }
 }

services/archetypes/worker/src/interceptors.ts

@@ -34,7 +34,11 @@ export class WorkflowMonitoringInterceptor implements WorkflowInboundCallsInterc
     } finally {
       const end = new Date()
       const duration = end.getTime() - start.getTime()
-      await activity.telemetryDistribution('temporal.workflow_execution_duration', duration, tags)
+
+      // Only send telemetry if duration is more than 2 hours
+      if (duration > 3 * 60 * 60 * 1000) {
+        await activity.telemetryDistribution('temporal.workflow_execution_duration', duration, tags)
+      }
     }
   }
 }