HPCC-33922 Extend excludeKeys mechanism to support regex

Change and extend excludeKeys to support regex patterns.
e.g. "~.*::replicas".
A "~" prefix denoes component is a regex.

Also:
1) remove superfluous from eclagent.yaml
2) fix globalExcludeSectionRegexList to correctly exclude *-job.yaml
   sections.
3) Exclude all replicas via globalExcludeList with "~.*::replicas"

Signed-off-by: Jake Smith <[email protected]>

Author: jakesmith

helm/hpcc/templates/_helpers.tpl

@@ -2530,7 +2530,9 @@ This means pods only auto-restart if the non-excluded parts change.
 
 Pass in root, me, configMapHelper, component, excludeSectionRegexList and excludeKeyList
 excludeSectionRegexList is a list of regexp's that filter out top-level sections, e.g. [".*spec.yaml$" ]
-excludeKeyList is a list of key values to exclude from each section, e.g. [ "global", "esp.services" "esp.queues"]
+excludeKeyList is a list of key values with optional regex support (denoted by leading '~') to exclude from
+each section, each component of the key must be seperated by "::" due to the regex support.
+e.g. [ "global", "esp::services" "esp::queues", "~.*::loggging"]
 
 The configMap data section is reconstructed based on filtering out matches.
 
@@ -2543,32 +2545,53 @@ e.g. a cache of secrets, with an auto reload/refresh mechanism, or 'replicas'.
 {{- $excludeSectionRegexList := .excludeSectionRegexList -}}
 {{- $excludeKeyList := .excludeKeyList -}}
 {{- range $configElementName, $configElementDict := $config.data -}}
-  {{- $_ := set $configCtx "excludeSection" false -}}
-  {{- range $regex := $excludeSectionRegexList -}}
-    {{- if (regexMatch $regex $configElementName) -}}
-      {{- $_ := set $configCtx "excludeSection" true -}}
-    {{- end -}}
+ {{- $_ := set $configCtx "excludeSection" false -}}
+ {{- range $regex := $excludeSectionRegexList -}}
+  {{- if (regexMatch $regex $configElementName) -}}
+   {{- $_ := set $configCtx "excludeSection" true -}}
   {{- end -}}
-  {{- if not $configCtx.excludeSection -}}
-    {{- $configDictCtx := dict -}}
-    {{- range $key := $excludeKeyList -}}
-      {{- $_ := set $configDictCtx "keyDictStr" (regexReplaceAll "(.*)\\..*$" $key "${1}") -}}
-      {{- if eq $configDictCtx.keyDictStr $key -}}{{/* single component key, e.g. "global"*/}}
-        {{- $configElementDict := (unset $configElementDict $key) -}}
-      {{- else -}}{{/* scopes component key, e.g. "eclccserver.queue"*/}}
-        {{- $_ := set $configDictCtx "keyKeyStr" (regexReplaceAll ".*\\.(.*)$" $key "${1}") -}}
-        {{- $subDict := get $configElementDict $configDictCtx.keyDictStr -}}
-        {{- if $subDict -}}
-          {{- $_ := set $configElementDict $configDictCtx.keyDictStr (unset $subDict $configDictCtx.keyKeyStr) -}}
+ {{- end -}}
+ {{- if not $configCtx.excludeSection -}}
+  {{- $configDictCtx := dict -}}
+  {{- range $key := $excludeKeyList -}}
+   {{- $keyParts := splitList "::" $key -}}
+   {{- $outerRaw := index $keyParts 0 -}}
+   {{- $outerIsWild := hasPrefix "~" $outerRaw -}}
+   {{- $outerKey := trimPrefix "~" $outerRaw -}}
+   {{- $hasSubKey := eq (len $keyParts) 2 -}}
+   {{- $innerKey := "" -}}
+   {{- $innerIsWild := false -}}
+   {{- if $hasSubKey -}}
+    {{- $innerRaw := index $keyParts 1 -}}
+    {{- $innerIsWild = hasPrefix "~" $innerRaw -}}
+    {{- $innerKey = trimPrefix "~" $innerRaw -}}
+   {{- end -}}
+   {{- range $topKey, $topVal := $configElementDict -}}
+    {{- if or (and (not $outerIsWild) (eq $outerKey $topKey)) (and $outerIsWild (regexMatch $outerKey $topKey)) -}}
+     {{- if not $hasSubKey -}}
+      {{- $configElementDict = unset $configElementDict $topKey -}}
+     {{- else -}}
+      {{- $innerDict := get $configElementDict $topKey | default dict -}}
+      {{- if (kindIs "map" $innerDict) -}}
+       {{- if $innerIsWild -}}
+        {{- range $k, $_ := $innerDict -}}
+         {{- if regexMatch $innerKey $k -}}
+          {{- $innerDict = unset $innerDict $k -}}
+         {{- end -}}
         {{- end -}}
+       {{- else -}}
+        {{- $innerDict = unset $innerDict $innerKey -}}
+       {{- end -}}
       {{- end -}}
-    {{- end -}}{{/*range $key*/}}
-    {{- $configYaml := toYaml $configElementDict -}}
-    {{- $_ := set $config.data $configElementName $configYaml -}}
-  {{- else -}}
-    {{- $configData := (unset $config.data $configElementName) -}}
-    {{- $_ := set $config "data" $configData -}}
+      {{- $_ := set $configElementDict $topKey $innerDict -}}
+     {{- end -}}
+    {{- end -}}
+   {{- end -}}
   {{- end -}}
+ {{- else -}}
+  {{- $configData := (unset $config.data $configElementName) -}}
+  {{- $_ := set $config "data" $configData -}}
+ {{- end -}}
 {{- end -}}{{/*range $configElementName*/}}
 {{ toYaml $config }}
 {{- end -}}
@@ -2598,9 +2621,10 @@ globalExcludeList below is a hard-coded list of global keys to exclude.
 
 */}}
 {{- define "hpcc.getConfigSHA" }}
-{{- $globalExcludeList := list (printf "%s.replicas" .component) -}}
-{{- $globalExcludeSectionRegexList := list ".*spec.yaml$" -}}
-{{- $combinedExcludeKeyList := concat (splitList "," (.excludeKeys | default "")) $globalExcludeList -}}
+{{- $globalExcludeList := list "~.*::replicas" -}}
+{{- $globalExcludeSectionRegexList := list ".*-job.yaml$" -}}
+{{- $componentExcludeList := ternary (splitList "," (.excludeKeys | default "")) list (hasKey . "excludeKeys") -}}
+{{- $combinedExcludeKeyList := concat $globalExcludeList $componentExcludeList -}}
 {{- $ctx := merge (omit . "excludeKeys") (dict "excludeSectionRegexList" $globalExcludeSectionRegexList "excludeKeyList" $combinedExcludeKeyList) -}}
 {{- include "hpcc.filterConfig" $ctx | sha256sum }}
 {{- end -}}

helm/hpcc/templates/eclagent.yaml

@@ -124,7 +124,7 @@ data:
 {{- $enginePlaneDetails := dict -}}
 {{- $_ := include "hpcc.getEnginePlanes" (dict "root" $ "me" . "result" $enginePlaneDetails) -}}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories "includeCategories" $enginePlaneDetails.planeCategories "includeNames" $enginePlaneDetails.namedPlanes "env" $env }}
-{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.agentConfigMap" "component" "eclagent" "excludeKeys" (print "global," $apptype ".replicas") "lifeCycleCtx" (dict "containers" list))) }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.agentConfigMap" "component" "eclagent" "excludeKeys" "global" "lifeCycleCtx" (dict "containers" list))) }}
 {{- $_ := set $commonCtx "lifeCycleCtx" (dict "containers" list) -}}
 {{- include "hpcc.checkDefaultStoragePlane" $commonCtx }}
 apiVersion: apps/v1

helm/hpcc/templates/eclccserver.yaml

@@ -132,7 +132,7 @@ data:
 {{- $env := concat ($.Values.global.env | default list) (.env | default list) $gitenv -}}
 {{- $secretsCategories := list "system" "codeVerify" "git" "storage" "jfrog" }}
 {{- $commonCtx := dict "root" $ "me" . "includeCategories" (list "dll" "git" "debug") "secretsCategories" $secretsCategories "env" $env }}
-{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap" "component" "eclccserver" "excludeKeys" "global,eclccserver.queues" "lifeCycleCtx" (dict "containers" list))) }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap" "component" "eclccserver" "excludeKeys" "global,eclccserver::queues" "lifeCycleCtx" (dict "containers" list))) }}
 {{- $_ := set $commonCtx "lifeCycleCtx" (dict "containers" list) -}}
 apiVersion: apps/v1
 kind: Deployment

helm/hpcc/templates/esp.yaml

@@ -97,7 +97,7 @@ data:
 {{- $includeStorageCategories := ternary (list "lz" "data" "dll" "debug") (list "data" "dll" "debug") (eq $application "eclwatch") -}}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories  "includeCategories" $includeStorageCategories "env" $env -}}
 {{- $_ := set $commonCtx "lifeCycleCtx" (dict "containers" list) -}}
-{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap" "component" "esp" "excludeKeys" "global,esp.queues")) -}}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap" "component" "esp" "excludeKeys" "global,esp::queues")) -}}
 {{- if (ne (include "hpcc.isVisibilityPublic" (dict "root" $ "visibility" .service.visibility)) "") }}
  {{- /* If esp is using a public cert, this flag is used later to add the local MTLS mount and cert */ -}}
  {{- $_ := set $commonCtx "externalCert" true -}}