config to throttle all Scan endpoints, all IPs (#2646)

* all-paths, all-ips config

* remove declarations we don't want

* more precise match exprs
- see https://cloud.google.com/armor/docs/rules-language-reference

* use default allow in preview

* config and versionedExpr must be specified at same time

* Rules deletedWith the containing SecurityPolicy; much faster deletion

---------

Signed-off-by: Stephen Compall <stephen.compall@digitalasset.com>

Author: stephencompall-DA

cluster/deployment/config.yaml

@@ -88,6 +88,13 @@ monitoring:
 cloudArmor:
   enabled: false
   allRulesPreviewOnly: true
+  publicEndpoints:
+    publicScan:
+      hostname: scan.sv-2.scratcha.global.canton.network.digitalasset.com
+      pathPrefix: /api/scan
+      throttleAcrossAllEndpointsAllIps:
+        withinIntervalSeconds: 60
+        maxRequestsBeforeHttp429: 0
 multiValidator:
   postgresPvcSize: '100Gi'
   resources:

cluster/pulumi/infra/src/cloudArmor.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import * as gcp from '@pulumi/gcp';
 import * as pulumi from '@pulumi/pulumi';
+import * as _ from 'lodash';
 import { CLUSTER_BASENAME } from '@lfdecentralizedtrust/splice-pulumi-common';
 
 import * as config from './config';
@@ -19,17 +20,12 @@ export interface ApiEndpoint {
   hostname: string;
 }
 
-export interface ThrottleConfig {
-  perIp: boolean;
-  rate: number; // Requests per minute
-  interval: number; // Interval in seconds
-}
-
-export type CloudArmorConfig = Pick<config.CloudArmorConfig, 'enabled' | 'allRulesPreviewOnly'> & {
+export type CloudArmorConfig = config.CloudArmorConfig & {
   predefinedWafRules?: PredefinedWafRule[];
-  apiThrottles?: ApiThrottleConfig[];
 };
 
+type ThrottleConfig = CloudArmorConfig['publicEndpoints'];
+
 export interface PredefinedWafRule {
   name: string;
   action: 'allow' | 'deny' | 'throttle';
@@ -38,14 +34,6 @@ export interface PredefinedWafRule {
   sensitivityLevel?: 'off' | 'low' | 'medium' | 'high';
 }
 
-// TODO (DACH-NY/canton-network-internal#2115) replace this placeholder config
-// with the real yaml structure we want to use
-export interface ApiThrottleConfig {
-  endpoint: ApiEndpoint;
-  throttle: ThrottleConfig;
-  action: 'throttle' | 'ban';
-}
-
 /**
  * Creates a Cloud Armor security policy
  * @param cac loaded configuration
@@ -75,7 +63,7 @@ export function configureCloudArmorPolicy(
     opts
   );
 
-  const ruleOpts = { ...opts, parent: securityPolicy };
+  const ruleOpts = { ...opts, parent: securityPolicy, deletedWith: securityPolicy };
 
   // Step 2: Add predefined WAF rules
   if (cac.predefinedWafRules && cac.predefinedWafRules.length > 0) {
@@ -86,8 +74,8 @@ export function configureCloudArmorPolicy(
   addIpWhitelistRules(/*securityPolicy, cac.allRulesPreviewOnly, ruleOpts*/);
 
   // Step 4: Add throttling/banning rules for specific API endpoints
-  if (cac.apiThrottles && cac.apiThrottles.length > 0) {
-    addThrottleAndBanRules(securityPolicy, cac.apiThrottles, cac.allRulesPreviewOnly, ruleOpts);
+  if (cac.publicEndpoints && !_.isEmpty(cac.publicEndpoints)) {
+    addThrottleAndBanRules(securityPolicy, cac.publicEndpoints, cac.allRulesPreviewOnly, ruleOpts);
   }
 
   // Step 5: Add default deny rule
@@ -126,62 +114,59 @@ function addIpWhitelistRules(): void {
  */
 function addThrottleAndBanRules(
   securityPolicy: gcp.compute.SecurityPolicy,
-  apiThrottles: ApiThrottleConfig[],
+  throttles: ThrottleConfig,
   preview: boolean,
   opts: pulumi.ResourceOptions
 ): void {
-  apiThrottles.reduce((priority, apiConfig) => {
-    if (priority >= THROTTLE_BAN_RULE_MAX) {
-      throw new Error(
-        `Throttle rule priority ${priority} exceeds maximum ${THROTTLE_BAN_RULE_MAX}`
-      );
-    }
-
-    const { endpoint, throttle, action } = apiConfig;
-    const ruleName = `${action}${throttle.perIp ? '-per-ip' : ''}-${endpoint.name}`;
-
-    // Build the expression for path and hostname matching
-    const pathExpr = `request.path.matches('${endpoint.path}')`;
-    const hostExpr = `request.headers['host'].matches('${endpoint.hostname}')`;
-    const matchExpr = `${pathExpr} && ${hostExpr}`;
-
-    new gcp.compute.SecurityPolicyRule(
-      ruleName,
-      {
-        securityPolicy: securityPolicy.name,
-        description: `${action === 'throttle' ? 'Throttle' : 'Ban'} rule${throttle.perIp ? ' per-IP' : ''} for ${endpoint.name} API endpoint`,
-        priority,
-        preview,
-        action: action === 'ban' ? 'rate_based_ban' : 'throttle',
-        match: {
-          expr: {
-            expression: matchExpr,
-          },
-        },
-        rateLimitOptions: {
-          // ban point is banThreshold + ratelimit count; consider splitting up rather than doubling
-          ...(action === 'ban'
-            ? {
-                banDurationSec: 600,
-                banThreshold: {
-                  count: throttle.rate,
-                  intervalSec: throttle.interval,
-                },
-              }
-            : {}),
-          enforceOnKey: throttle.perIp ? 'IP' : 'ALL',
-          rateLimitThreshold: {
-            count: throttle.rate,
-            intervalSec: throttle.interval,
+  _.sortBy(Object.entries(throttles), e => e[0]).reduce(
+    (priority, [confEntryHead, singleServiceThrottle]) => {
+      if (priority >= THROTTLE_BAN_RULE_MAX) {
+        throw new Error(
+          `Throttle rule priority ${priority} exceeds maximum ${THROTTLE_BAN_RULE_MAX}`
+        );
+      }
+
+      const { hostname, pathPrefix, throttleAcrossAllEndpointsAllIps } = singleServiceThrottle;
+      // leave out the rule but consume the priority number if max is 0
+      // this makes the pulumi update cleaner if toggling just one service
+      if (throttleAcrossAllEndpointsAllIps.maxRequestsBeforeHttp429 > 0) {
+        const ruleName = `throttle-all-endpoints-all-ips-${confEntryHead}`;
+
+        // Build the expression for path and hostname matching
+        const pathExpr = `request.path.startsWith(R"${pathPrefix}")`;
+        const hostExpr = `request.headers['host'].matches(R"^${_.escapeRegExp(hostname)}(?::[0-9]+)?$")`;
+        const matchExpr = `${pathExpr} && ${hostExpr}`;
+
+        new gcp.compute.SecurityPolicyRule(
+          ruleName,
+          {
+            securityPolicy: securityPolicy.name,
+            description: `Throttle rule for all ${confEntryHead} API endpoints`,
+            priority,
+            preview: preview || singleServiceThrottle.rulePreviewOnly,
+            action: 'throttle',
+            match: {
+              expr: {
+                expression: matchExpr,
+              },
+            },
+            rateLimitOptions: {
+              enforceOnKey: 'ALL',
+              rateLimitThreshold: {
+                count: throttleAcrossAllEndpointsAllIps.maxRequestsBeforeHttp429,
+                intervalSec: throttleAcrossAllEndpointsAllIps.withinIntervalSeconds,
+              },
+              conformAction: 'allow',
+              exceedAction: 'deny(429)', // 429 Too Many Requests
+            },
           },
-          conformAction: 'allow',
-          exceedAction: 'deny(429)', // 429 Too Many Requests
-        },
-      },
-      opts
-    );
-    return priority + RULE_SPACING;
-  }, THROTTLE_BAN_RULE_MIN);
+          opts
+        );
+      }
+      return priority + RULE_SPACING;
+    },
+    THROTTLE_BAN_RULE_MIN
+  );
 }
 
 /**
@@ -192,15 +177,23 @@ function addDefaultDenyRule(
   preview: boolean,
   opts: pulumi.ResourceOptions
 ): void {
+  // when you create a SecurityPolicy it has a default allow rule; we assume
+  // that if you want all rules in preview, you *also* still want to allow
+  // all traffic
+  if (preview) {
+    return;
+  }
   new gcp.compute.SecurityPolicyRule(
     'default-deny',
     {
       securityPolicy: securityPolicy.name,
       description: 'Default rule to deny all other traffic',
       priority: DEFAULT_DENY_RULE_NUMBER,
-      preview,
+      // default rule cannot be in preview mode; google API gives 400 if you try
+      preview: false,
       action: 'deny',
       match: {
+        versionedExpr: 'SRC_IPS_V1',
         config: {
           srcIpRanges: ['*'],
         },

cluster/pulumi/infra/src/config.ts

@@ -65,8 +65,15 @@ const CloudArmorConfigSchema = z.object({
     .object({})
     .catchall(
       z.object({
-        domain: z.string(),
-        // TODO (DACH-NY/canton-network-internal#2115) more config
+        rulePreviewOnly: z.boolean().default(false),
+        hostname: z.string().regex(/^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/, 'valid DNS hostname'),
+        pathPrefix: z.string().regex(/^\/[^"]*$/, 'HTTP request path starting with /'),
+        throttleAcrossAllEndpointsAllIps: z.object({
+          withinIntervalSeconds: z.number().positive(),
+          maxRequestsBeforeHttp429: z
+            .number()
+            .min(0, '0 to disallow requests or positive to allow'),
+        }),
       })
     )
     .default({}),