Create metric: appsec.waf.config_errors (#8394)

Author: sezen-datadog

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java

@@ -48,6 +48,7 @@
 import datadog.trace.api.ProductActivation;
 import datadog.trace.api.UserIdCollectionMode;
 import datadog.trace.api.telemetry.LogCollector;
+import datadog.trace.api.telemetry.WafMetricCollector;
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -263,7 +264,9 @@ private void handleWafUpdateResultReport(String configKey, Map<String, Object> r
           "Invalid rule during waf config update for config key {}: {}",
           configKey,
           e.wafDiagnostics);
-
+      if (e.wafDiagnostics.getNumConfigError() > 0) {
+        WafMetricCollector.get().addWafConfigError(e.wafDiagnostics.getNumConfigError());
+      }
       // TODO: Propagate diagostics back to remote config apply_error
 
       initReporter.setReportForPublication(e.wafDiagnostics);

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy

@@ -13,6 +13,7 @@ import datadog.remoteconfig.state.ProductListener
 import datadog.trace.api.Config
 import datadog.trace.api.ProductActivation
 import datadog.trace.api.UserIdCollectionMode
+import datadog.trace.api.telemetry.WafMetricCollector
 import datadog.trace.test.util.DDSpecification
 
 import java.nio.file.Files
@@ -50,6 +51,7 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
   AppSecModuleConfigurer.Reconfiguration reconf = Stub()
   AppSecConfigServiceImpl appSecConfigService
   SavedListeners listeners
+  protected static final ORIGINAL_METRIC_COLLECTOR = WafMetricCollector.get()
 
   void cleanup() {
     appSecConfigService?.close()
@@ -712,6 +714,51 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
     AppSecSystem.active = true
   }
 
+  void 'InvalidRuleSetException is thrown when rules are not configured correctly' () {
+    setup:
+    // Mock WafMetricCollector
+    WafMetricCollector wafMetricCollector = Mock(WafMetricCollector)
+    WafMetricCollector.INSTANCE = wafMetricCollector
+
+    // Create a temporary file with invalid WAF configuration
+    Path p = Files.createTempFile('appsec', '.json')
+    p.toFile() << '''{
+      "version": "2.2",
+      "rules": [
+        {
+          "id": "invalid-rule",
+          "name": "Invalid Rule",
+          "tags": {
+            "type": "invalid_type",
+            "category": "invalid_category"
+          },
+          "conditions": [
+            {
+              "operator": "invalid_operator",
+              "parameters": {
+                "invalid_param": "invalid_value"
+              }
+            }
+          ],
+          "type": "invalid_type",
+          "data": []
+        }
+      ]
+    }'''
+
+    when:
+    appSecConfigService.init()
+
+    then:
+    1 * config.getAppSecRulesFile() >> (p as String)
+    1 * wafMetricCollector.addWafConfigError(_ as Integer)
+    thrown RuntimeException
+
+    cleanup:
+    WafMetricCollector.INSTANCE = ORIGINAL_METRIC_COLLECTOR
+    p.toFile().delete()
+  }
+
   private static AppSecFeatures autoUserInstrum(String mode) {
     return new AppSecFeatures().tap { features ->
       features.autoUserInstrum = new AppSecFeatures.AutoUserInstrum().tap { instrum ->

internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java

@@ -59,6 +59,7 @@ private WafMetricCollector() {
       new AtomicLongArray(LoginFramework.getNumValues());
   private static final AtomicLongArray appSecSdkEventQueue =
       new AtomicLongArray(LoginEvent.getNumValues() * LoginVersion.getNumValues());
+  private static final AtomicInteger wafConfigErrorCounter = new AtomicInteger();
 
   /** WAF version that will be initialized with wafInit and reused for all metrics. */
   private static String wafVersion = "";
@@ -353,6 +354,16 @@ public void prepareMetrics() {
         }
       }
     }
+
+    // WAF config errors
+    int configErrors = wafConfigErrorCounter.getAndSet(0);
+    if (configErrors > 0) {
+      if (!rawMetricsQueue.offer(
+          new WafConfigError(
+              configErrors, WafMetricCollector.wafVersion, WafMetricCollector.rulesVersion))) {
+        return;
+      }
+    }
   }
 
   public abstract static class WafMetric extends MetricCollector.Metric {
@@ -448,6 +459,20 @@ public WafRequestsRawMetric(
     }
   }
 
+  public void addWafConfigError(int nbErrors) {
+    wafConfigErrorCounter.addAndGet(nbErrors);
+  }
+
+  public static class WafConfigError extends WafMetric {
+    public WafConfigError(final long counter, final String wafVersion, final String rulesVersion) {
+      super(
+          "waf.config_errors",
+          counter,
+          "waf_version:" + wafVersion,
+          "event_rules_version:" + rulesVersion);
+    }
+  }
+
   public static class RaspRuleEval extends WafMetric {
     public RaspRuleEval(final long counter, final RuleType ruleType, final String wafVersion) {
       super(

internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy

@@ -485,6 +485,28 @@ class WafMetricCollectorTest extends DDSpecification {
     [stringTooLong, listMapTooLarge, objectTooDeep] << allBooleanCombinations(3)
   }
 
+  void 'test waf config error metrics'() {
+    given:
+    def collector = WafMetricCollector.get()
+
+    when:
+    collector.wafInit('waf_ver1', 'rules.1', true)
+    collector.addWafConfigError(5)
+    collector.addWafConfigError(3)
+    collector.prepareMetrics()
+
+    then:
+    def metrics = collector.drain()
+    def configErrorMetrics = metrics.findAll { it.metricName == 'waf.config_errors' }
+
+    final metric = configErrorMetrics[0]
+    metric.type == 'count'
+    metric.metricName == 'waf.config_errors'
+    metric.namespace == 'appsec'
+    metric.value == 8
+    metric.tags.toSet() == ['waf_version:waf_ver1', 'event_rules_version:rules.1'].toSet()
+  }
+
   /**
    * Helper method to generate all combinations of n boolean values.
    */