Add support for waf_timeout tag in telemetry

dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java

@@ -429,6 +429,7 @@ public void onDataAvailable(
         resultWithData = doRunPowerwaf(reqCtx, newData, ctxAndAddr, gwCtx);
       } catch (TimeoutPowerwafException tpe) {
         reqCtx.increaseTimeouts();
+        WafMetricCollector.get().wafRequestTimeout();
         log.debug(LogCollector.EXCLUDE_TELEMETRY, "Timeout calling the WAF", tpe);
         if (gwCtx.isRasp) {
           WafMetricCollector.get().raspTimeout(gwCtx.raspRuleType);

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy

@@ -24,6 +24,7 @@ import datadog.trace.api.ConfigDefaults
 import datadog.trace.api.internal.TraceSegment
 import datadog.appsec.api.blocking.BlockingContentType
 import datadog.trace.api.gateway.Flow
+import datadog.trace.api.telemetry.WafMetricCollector
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer
 import datadog.trace.test.util.DDSpecification
@@ -44,6 +45,9 @@ class PowerWAFModuleSpecification extends DDSpecification {
   @Shared
   protected static final AgentTracer.TracerAPI ORIGINAL_TRACER = AgentTracer.get()
 
+  @Shared
+  protected static final ORIGINAL_METRIC_COLLECTOR = WafMetricCollector.get()
+
   private static final DataBundle ATTACK_BUNDLE = MapDataBundle.of(KnownAddresses.HEADERS_NO_COOKIES,
   new CaseInsensitiveMap<List<String>>(['user-agent': 'Arachni/v0']))
 
@@ -70,6 +74,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
   }
 
   void cleanup() {
+    WafMetricCollector.INSTANCE  = ORIGINAL_METRIC_COLLECTOR
     AgentTracer.forceRegister(ORIGINAL_TRACER)
     pwafAdditive?.close()
     release pwafModule
@@ -966,6 +971,9 @@ class PowerWAFModuleSpecification extends DDSpecification {
     TraceSegment segment = Mock()
     TraceSegmentPostProcessor pp = service.traceSegmentPostProcessors.last()
 
+    def mockWafMetricCollector = Mock(WafMetricCollector)
+    WafMetricCollector.INSTANCE = mockWafMetricCollector
+
     when:
     dataListener.onDataAvailable(flow, ctx, db, gwCtx)
 
@@ -974,6 +982,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
       pwafAdditive = it[0].openAdditive() }
     assert !flow.blocking
     1 * ctx.increaseTimeouts()
+    1 * mockWafMetricCollector.get().wafRequestTimeout()
 
     when:
     pp.processTraceSegment(segment, ctx, [])

internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java

@@ -12,12 +12,16 @@
 
 public class WafMetricCollector implements MetricCollector<WafMetricCollector.WafMetric> {
 
-  public static final WafMetricCollector INSTANCE = new WafMetricCollector();
+  public static WafMetricCollector INSTANCE = new WafMetricCollector();
 
   public static WafMetricCollector get() {
     return WafMetricCollector.INSTANCE;
   }
 
+  private WafMetricCollector() {
+    // Prevent external instantiation
+  }
+
   private static final String NAMESPACE = "appsec";
 
   private static final BlockingQueue<WafMetric> rawMetricsQueue =
@@ -29,11 +33,12 @@ public static WafMetricCollector get() {
   private static final AtomicRequestCounter wafRequestCounter = new AtomicRequestCounter();
   private static final AtomicRequestCounter wafTriggeredRequestCounter = new AtomicRequestCounter();
   private static final AtomicRequestCounter wafBlockedRequestCounter = new AtomicRequestCounter();
+  private static final AtomicRequestCounter wafTimeoutRequestCounter = new AtomicRequestCounter();
   private static final AtomicLongArray raspRuleEvalCounter =
       new AtomicLongArray(RuleType.getNumValues());
   private static final AtomicLongArray raspRuleMatchCounter =
       new AtomicLongArray(RuleType.getNumValues());
-  private static final AtomicLongArray respTimeoutCounter =
+  private static final AtomicLongArray raspTimeoutCounter =
       new AtomicLongArray(RuleType.getNumValues());
   private static final AtomicRequestCounter missingUserIdCounter = new AtomicRequestCounter();
 
@@ -78,6 +83,10 @@ public void wafRequestBlocked() {
     wafBlockedRequestCounter.increment();
   }
 
+  public void wafRequestTimeout() {
+    wafTimeoutRequestCounter.increment();
+  }
+
   public void raspRuleEval(final RuleType ruleType) {
     raspRuleEvalCounter.incrementAndGet(ruleType.ordinal());
   }
@@ -87,7 +96,7 @@ public void raspRuleMatch(final RuleType ruleType) {
   }
 
   public void raspTimeout(final RuleType ruleType) {
-    respTimeoutCounter.incrementAndGet(ruleType.ordinal());
+    raspTimeoutCounter.incrementAndGet(ruleType.ordinal());
   }
 
   public void missingUserId() {
@@ -116,6 +125,7 @@ public void prepareMetrics() {
               WafMetricCollector.wafVersion,
               WafMetricCollector.rulesVersion,
               false,
+              false,
               false))) {
         return;
       }
@@ -129,6 +139,7 @@ public void prepareMetrics() {
               WafMetricCollector.wafVersion,
               WafMetricCollector.rulesVersion,
               true,
+              false,
               false))) {
         return;
       }
@@ -142,6 +153,21 @@ public void prepareMetrics() {
               WafMetricCollector.wafVersion,
               WafMetricCollector.rulesVersion,
               true,
+              true,
+              false))) {
+        return;
+      }
+    }
+
+    // Timeout requests
+    if (wafTimeoutRequestCounter.get() > 0) {
+      if (!rawMetricsQueue.offer(
+          new WafRequestsRawMetric(
+              wafTimeoutRequestCounter.getAndReset(),
+              WafMetricCollector.wafVersion,
+              WafMetricCollector.rulesVersion,
+              false,
+              false,
               true))) {
         return;
       }
@@ -171,7 +197,7 @@ public void prepareMetrics() {
 
     // RASP timeout per rule type
     for (RuleType ruleType : RuleType.values()) {
-      long counter = respTimeoutCounter.getAndSet(ruleType.ordinal(), 0);
+      long counter = raspTimeoutCounter.getAndSet(ruleType.ordinal(), 0);
       if (counter > 0) {
         if (!rawMetricsQueue.offer(
             new RaspTimeout(counter, ruleType, WafMetricCollector.wafVersion))) {
@@ -228,14 +254,16 @@ public WafRequestsRawMetric(
         final String wafVersion,
         final String rulesVersion,
         final boolean triggered,
-        final boolean blocked) {
+        final boolean blocked,
+        final boolean wafTimeout) {
       super(
           "waf.requests",
           counter,
           "waf_version:" + wafVersion,
           "event_rules_version:" + rulesVersion,
           "rule_triggered:" + triggered,
-          "request_blocked:" + blocked);
+          "request_blocked:" + blocked,
+          "waf_timeout:" + wafTimeout);
     }
   }
 

internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy

@@ -22,12 +22,15 @@ class WafMetricCollectorTest extends DDSpecification {
     WafMetricCollector.get().wafRequest()
     WafMetricCollector.get().wafRequestTriggered()
     WafMetricCollector.get().wafRequestBlocked()
+    WafMetricCollector.get().wafRequestTimeout()
     WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
     WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
     WafMetricCollector.get().raspRuleMatch(RuleType.SQL_INJECTION)
     WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
     WafMetricCollector.get().raspTimeout(RuleType.SQL_INJECTION)
 
+
+
     WafMetricCollector.get().prepareMetrics()
 
     then:
@@ -63,7 +66,8 @@ class WafMetricCollectorTest extends DDSpecification {
       'waf_version:waf_ver1',
       'event_rules_version:rules.3',
       'rule_triggered:false',
-      'request_blocked:false'
+      'request_blocked:false',
+      'waf_timeout:false'
     ].toSet()
 
     def requestTriggeredMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[4]
@@ -74,9 +78,11 @@ class WafMetricCollectorTest extends DDSpecification {
       'waf_version:waf_ver1',
       'event_rules_version:rules.3',
       'rule_triggered:true',
-      'request_blocked:false'
+      'request_blocked:false',
+      'waf_timeout:false'
     ].toSet()
 
+
     def requestBlockedMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[5]
     requestBlockedMetric.namespace == 'appsec'
     requestBlockedMetric.metricName == 'waf.requests'
@@ -86,24 +92,38 @@ class WafMetricCollectorTest extends DDSpecification {
       'waf_version:waf_ver1',
       'event_rules_version:rules.3',
       'rule_triggered:true',
-      'request_blocked:true'
+      'request_blocked:true',
+      'waf_timeout:false'
+    ].toSet()
+
+    def requestTimeoutMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[6]
+    requestTimeoutMetric.namespace == 'appsec'
+    requestTimeoutMetric.metricName == 'waf.requests'
+    requestTimeoutMetric.type == 'count'
+    requestTimeoutMetric.value == 1
+    requestTimeoutMetric.tags.toSet() == [
+      'waf_version:waf_ver1',
+      'event_rules_version:rules.3',
+      'rule_triggered:false',
+      'request_blocked:false',
+      'waf_timeout:true'
     ].toSet()
 
-    def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[6]
+    def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[7]
     raspRuleEvalSqli.type == 'count'
     raspRuleEvalSqli.value == 3
     raspRuleEvalSqli.namespace == 'appsec'
     raspRuleEvalSqli.metricName == 'rasp.rule.eval'
     raspRuleEvalSqli.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet()
 
-    def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[7]
+    def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[8]
     raspRuleMatch.type == 'count'
     raspRuleMatch.value == 1
     raspRuleMatch.namespace == 'appsec'
     raspRuleMatch.metricName == 'rasp.rule.match'
     raspRuleMatch.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet()
 
-    def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[8]
+    def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[9]
     raspTimeout.type == 'count'
     raspTimeout.value == 1
     raspTimeout.namespace == 'appsec'

telemetry/src/test/groovy/datadog/telemetry/metric/WafMetricPeriodicActionSpecification.groovy

@@ -48,6 +48,7 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
     WafMetricCollector.get().wafRequest()
     WafMetricCollector.get().wafRequestBlocked()
     WafMetricCollector.get().wafRequest()
+    WafMetricCollector.get().wafRequestTimeout()
     WafMetricCollector.get().prepareMetrics()
     periodicAction.doIteration(telemetryService)
 
@@ -64,7 +65,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_1',
           'rule_triggered:false',
-          'request_blocked:false'
+          'request_blocked:false',
+          'waf_timeout:false'
         ]
     } )
     1 * telemetryService.addMetric( { Metric metric ->
@@ -75,7 +77,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_1',
           'rule_triggered:true',
-          'request_blocked:false'
+          'request_blocked:false',
+          'waf_timeout:false'
         ]
     } )
     1 * telemetryService.addMetric( { Metric metric ->
@@ -86,7 +89,20 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_1',
           'rule_triggered:true',
-          'request_blocked:true'
+          'request_blocked:true',
+          'waf_timeout:false'
+        ]
+    } )
+    1 * telemetryService.addMetric( { Metric metric ->
+      metric.namespace == 'appsec' &&
+        metric.metric == 'waf.requests' &&
+        metric.points[0][1] == 1 &&
+        metric.tags == [
+          'waf_version:0.0.0',
+          'event_rules_version:rules_ver_1',
+          'rule_triggered:false',
+          'request_blocked:false',
+          'waf_timeout:true'
         ]
     } )
     0 * _._
@@ -96,6 +112,7 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
     WafMetricCollector.get().wafRequest()
     WafMetricCollector.get().wafRequestTriggered()
     WafMetricCollector.get().wafRequestBlocked()
+    WafMetricCollector.get().wafRequestTimeout()
     WafMetricCollector.get().prepareMetrics()
     periodicAction.doIteration(telemetryService)
 
@@ -112,7 +129,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_2',
           'rule_triggered:false',
-          'request_blocked:false'
+          'request_blocked:false',
+          'waf_timeout:false'
         ]
     } )
     1 * telemetryService.addMetric( { Metric metric ->
@@ -123,7 +141,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_2',
           'rule_triggered:true',
-          'request_blocked:false'
+          'request_blocked:false',
+          'waf_timeout:false'
         ]
     } )
     1 * telemetryService.addMetric( { Metric metric ->
@@ -134,7 +153,20 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
           'waf_version:0.0.0',
           'event_rules_version:rules_ver_2',
           'rule_triggered:true',
-          'request_blocked:true'
+          'request_blocked:true',
+          'waf_timeout:false'
+        ]
+    } )
+    1 * telemetryService.addMetric( { Metric metric ->
+      metric.namespace == 'appsec' &&
+        metric.metric == 'waf.requests' &&
+        metric.points[0][1] == 1 &&
+        metric.tags == [
+          'waf_version:0.0.0',
+          'event_rules_version:rules_ver_2',
+          'rule_triggered:false',
+          'request_blocked:false',
+          'waf_timeout:true'
         ]
     } )
     0 * _._