HPCC-33885 WIP

Signed-off-by: Dave Streeter <[email protected]>

Author: streeterd

helm/hpcc/templates/_helpers.tpl

@@ -1,7 +1,7 @@
 {{/*
 
 ---  DO NOT EDIT THIS FILE - all configuration of HPCC platform should be done via values.yaml ----
-  
+
 ##############################################################################
 
     HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
@@ -17,7 +17,7 @@
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
-    
+
 ##############################################################################
 
 */}}
@@ -64,7 +64,7 @@ Translate a port list to a comma-separated list
   {{- $local := dict "first" true -}}
   {{- range $key, $value := . -}}{{- if not $local.first -}},{{- end -}}{{- $value -}}{{- $_ := set $local "first" false -}}{{- end -}}
  {{- else -}}
-  {{- . -}} 
+  {{- . -}}
  {{- end -}}
 {{- end -}}
 
@@ -877,7 +877,7 @@ Generates image information into env. variables used at runtime for runtime plat
 {{- end -}}
 
 {{/*
-Add image attributes for a component 
+Add image attributes for a component
 Pass in a dictionary with root, me and imagename defined
 */}}
 {{- define "hpcc.addImageAttrs" -}}
@@ -890,7 +890,7 @@ imagePullPolicy: {{ .root.Values.global.image.pullPolicy | default "IfNotPresent
 {{- end -}}
 
 {{/*
-Add image pull credentials for a component 
+Add image pull credentials for a component
 Pass in a dictionary with root and me defined
 */}}
 {{- define "hpcc.addImagePullSecrets" -}}
@@ -1237,7 +1237,7 @@ Generate instance queue names
 {{- range $.Values.roxie -}}
  {{- if not .disabled -}}
 - name: {{ .name }}
-  type: roxie 
+  type: roxie
   {{- if hasKey . "prefix" }}
   prefix: {{ .prefix }}
   {{- end }}
@@ -1660,12 +1660,12 @@ spec:
   loadBalancerSourceRanges:
   {{- if ne $lvars.type "LoadBalancer" -}}
    {{- $_ := fail (printf "loadBalanceSourceRanges invalid unless service type is LoadBalancer" ) -}}
-  {{- end -}} 
+  {{- end -}}
   {{- range $cidr := $lvars.loadBalancerSourceRanges }}
   - {{ $cidr }}
   {{- end }}
 {{ end }}
-{{- if $lvars.ingress }} 
+{{- if $lvars.ingress }}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -1727,7 +1727,7 @@ dali data
 A template to generate the standard app.kubernetes.io labels and standard HPCC labels
 
 Pass in root, name, component, instance
-root name(k8s application name) component(component within the application, can be same as app) instance 
+root name(k8s application name) component(component within the application, can be same as app) instance
 
 https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
 ----------------------------+-------------------------------------------------------------+----------
@@ -2571,7 +2571,7 @@ e.g. a cache of secrets, with an auto reload/refresh mechanism, or 'replicas'.
    {{- range $topKey, $topVal := $configElementDict -}}
     {{- if or (and (not $outerIsWild) (eq $outerKey $topKey)) (and $outerIsWild (regexMatch $outerKey $topKey)) -}}
      {{- if not $hasSubKey -}}
-      {{- $configElementDict = unset $configElementDict $topKey -}}
+      {{- unset $configElementDict $topKey -}}
      {{- else -}}
       {{- $innerDict := get $configElementDict $topKey | default dict -}}
       {{- if (kindIs "map" $innerDict) -}}
@@ -2623,7 +2623,7 @@ globalExcludeList below is a hard-coded list of global keys to exclude.
 
 */}}
 {{- define "hpcc.getConfigSHA" }}
-{{- $globalExcludeList := list "~.*::replicas" "~.*::vaults" -}}
+{{- $globalExcludeList := list "~.*::logging" "~.*::replicas" "~.*::vaults" -}}
 {{- $globalExcludeSectionRegexList := list ".*-job.yaml$" -}}
 {{- $componentExcludeList := ternary (splitList "," (.excludeKeys | default "")) list (hasKey . "excludeKeys") -}}
 {{- $combinedExcludeKeyList := concat $globalExcludeList $componentExcludeList -}}
@@ -2803,7 +2803,7 @@ Pass in dict with .root, .me and .labels
   {{- $_ := set $lvars "egress" .me.egress -}}
  {{- end -}}
 {{- end -}}
-{{- if $lvars.egress }} 
+{{- if $lvars.egress }}
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -2921,7 +2921,7 @@ spec:
   minReplicas: {{ (hasKey .hpa "minReplicas") | ternary .hpa.minReplicas 1 }}
   maxReplicas: {{ .hpa.maxReplicas }}
    {{- if hasKey .hpa "behavior" }}
-  behavior: 
+  behavior:
     {{- if hasKey .hpa.behavior "scaleDown" }}
     scaleDown:
       {{- include "hpcc.addHPABehaviorScale" (dict "behaviorScale" .hpa.behavior.scaleDown) | nindent 6 }}

system/jlib/jlog.cpp

@@ -814,9 +814,9 @@ void SessionLogMsgFilter::addToPTree(IPropertyTree * tree) const
     tree->addPropTree("filter", filterTree);
 }
 
-bool RegexLogMsgFilter::includeMessage(const LogMsg & msg) const 
-{ 
-    if(localFlag && msg.queryRemoteFlag()) return false; 
+bool RegexLogMsgFilter::includeMessage(const LogMsg & msg) const
+{
+    if(localFlag && msg.queryRemoteFlag()) return false;
     SpinBlock b(lock);
     return const_cast<RegExpr &>(regex).find(msg.queryText()) != NULL;
 }
@@ -1303,7 +1303,7 @@ void RollingFileLogMsgHandler::doRollover(bool daily, const char *forceName)
             }
         }
     }
-    if(!handle) 
+    if(!handle)
     {
         handle = getNullHandle();
         OWARNLOG("RollingFileLogMsgHandler::doRollover : could not open log file %s for output", filename.str());
@@ -1420,9 +1420,9 @@ void LogMsgMonitor::addToPTree(IPropertyTree * tree) const
 void CLogMsgManager::MsgProcessor::push(LogMsg * msg)
 {
     //assertex(more); an assertex will just recurse here
-    if (!more) // we are effective stopped so don't bother even dropping (and leak parameter) as drop will involve 
-               // interaction with the base class which is stopped and could easily crash (as this condition 
-               // is expected not to occur - typically occurs if the user has incorrectly called exit on one thread 
+    if (!more) // we are effective stopped so don't bother even dropping (and leak parameter) as drop will involve
+               // interaction with the base class which is stopped and could easily crash (as this condition
+               // is expected not to occur - typically occurs if the user has incorrectly called exit on one thread
                // while still in the process of logging on another)
                // cf Bug #53695 for more discussion of the issue
         return;
@@ -1578,14 +1578,14 @@ void CLogMsgManager::enterQueueingMode()
 
 void CLogMsgManager::setQueueBlockingLimit(unsigned lim)
 {
-    CriticalBlock crit(modeLock); 
+    CriticalBlock crit(modeLock);
     if(processor)
         processor->setBlockingLimit(lim);
 }
 
 void CLogMsgManager::setQueueDroppingLimit(unsigned lim, unsigned numToDrop)
 {
-    CriticalBlock crit(modeLock); 
+    CriticalBlock crit(modeLock);
     if(processor)
         processor->setDroppingLimit(lim, numToDrop);
 }
@@ -1925,8 +1925,8 @@ void CLogMsgManager::addAllMonitorsToPTree(IPropertyTree * tree) const
         monitors.item(i).addToPTree(tree);
 }
 
-bool CLogMsgManager::rejectsCategory(const LogMsgCategory & cat) const 
-{ 
+bool CLogMsgManager::rejectsCategory(const LogMsgCategory & cat) const
+{
     if (!prefilter.includeCategory(cat))
         return true;
 
@@ -2329,7 +2329,6 @@ class CNullManager : implements ILogMsgManager
 
 static CNullManager nullManager;
 static Singleton<IRemoteLogAccess> logAccessor;
-static CriticalSection logAccessCrit;
 
 MODULE_INIT(INIT_PRIORITY_JLOG)
 {
@@ -2377,6 +2376,7 @@ static constexpr unsigned queueLenDefault = 512;
 static constexpr unsigned queueDropDefault = 0; // disabled by default
 static constexpr bool useSysLogDefault = false;
 
+static CConfigUpdateHook configUpdateHook;
 void setupContainerizedLogMsgHandler()
 {
     Owned<IPropertyTree> logConfig = getComponentConfigSP()->getPropTree("logging");
@@ -2391,22 +2391,25 @@ void setupContainerizedLogMsgHandler()
         if (logConfig->hasProp(logFormatAtt))
         {
             const char *logFormat = logConfig->queryProp(logFormatAtt);
-            bool newFormatDetected = false;
+            HandleLogMsgHandler *previousStderrHandler{nullptr};
             if (streq(logFormat, "xml"))
             {
+                previousStderrHandler = theStderrHandler;
                 theStderrHandler = new HandleLogMsgHandlerXML(stderr, MSGFIELD_STANDARD);
-                newFormatDetected = true;
             }
             else if (streq(logFormat, "json"))
             {
+                previousStderrHandler = theStderrHandler;
                 theStderrHandler = new HandleLogMsgHandlerJSON(stderr, MSGFIELD_STANDARD);
-                newFormatDetected = true;
             }
             else if (!streq(logFormat, "table"))
                 LOG(MCoperatorWarning, "JLog: Invalid log format configuration detected '%s'!", logFormat);
 
-            if (newFormatDetected)
+            if (previousStderrHandler)
+            {
                 theManager->resetMonitors();
+                delete previousStderrHandler;
+            }
         }
 
         if (logConfig->hasProp(logFieldsAtt))
@@ -2464,15 +2467,30 @@ void setupContainerizedLogMsgHandler()
         unsigned postMortemLines = logConfig->getPropInt(capturePostMortemAtt, 0);
         if (postMortemLines)
         {
+            PostMortemLogMsgHandler *previousPostMortemHandler{nullptr};
+            if (thePostMortemHandler)
+            {
+                previousPostMortemHandler = thePostMortemHandler;
+                queryLogMsgManager()->removeMonitor(thePostMortemHandler);
+            }
+
             // augment postmortem files with <pid> to avoid clashes where multiple processes are running within
             // same process space, e.g. hthor processes running in same k8s container
             unsigned pid = GetCurrentProcessId();
             VStringBuffer portMortemFileBase("/tmp/postmortem.%u.log", pid);
 
             thePostMortemHandler = new PostMortemLogMsgHandler(portMortemFileBase, postMortemLines, MSGFIELD_STANDARD);
             queryLogMsgManager()->addMonitor(thePostMortemHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, TopDetail));
+
+            delete previousPostMortemHandler;
         }
     }
+
+    auto updateConfigFunc = [](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration)
+    {
+        setupContainerizedLogMsgHandler();
+    };
+    configUpdateHook.installOnce(updateConfigFunc, false);
 }
 
 ILogMsgManager * queryLogMsgManager()
@@ -2504,7 +2522,7 @@ ISysLogEventLogger * querySysLogEventLogger()
 
 ILogMsgHandler * getSysLogMsgHandler(unsigned fields)
 {
-    return new SysLogMsgHandler(theSysLogEventLogger, fields); 
+    return new SysLogMsgHandler(theSysLogEventLogger, fields);
 }
 
 #ifdef _WIN32
@@ -2555,7 +2573,7 @@ bool CSysLogEventLogger::win32Report(unsigned eventtype, unsigned category, unsi
                 char src[_MAX_PATH+1];
                 LPTSTR tail;
                 DWORD res = SearchPath(NULL,"JELOG.DLL",NULL,sizeof(src),src,&tail);
-                if (res>0) 
+                if (res>0)
                     copyFile(path,src);
                 else
                     throw makeOsException(GetLastError());
@@ -2570,20 +2588,20 @@ bool CSysLogEventLogger::win32Report(unsigned eventtype, unsigned category, unsi
 
         }
         HKEY hk;
-        if (RegCreateKeyEx(HKEY_LOCAL_MACHINE,"SYSTEM\\CurrentControlSet\\Services\\EventLog\\Application\\Seisint", 
+        if (RegCreateKeyEx(HKEY_LOCAL_MACHINE,"SYSTEM\\CurrentControlSet\\Services\\EventLog\\Application\\Seisint",
                            NULL, NULL, 0, KEY_ALL_ACCESS, NULL, &hk, NULL)==0) {
             DWORD sizedata = 0;
             DWORD type = REG_EXPAND_SZ;
             if ((RegQueryValueEx(hk,"EventMessageFile",NULL, &type, NULL, &sizedata)!=0)||!sizedata) {
-                StringAttr str("%SystemRoot%\\System32\\JELOG.dll"); 
+                StringAttr str("%SystemRoot%\\System32\\JELOG.dll");
                 RegSetValueEx(hk,"EventMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), (DWORD)str.length() + 1);
                 RegSetValueEx(hk,"CategoryMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), (DWORD)str.length() + 1);
-                DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE | EVENTLOG_AUDIT_SUCCESS | EVENTLOG_AUDIT_FAILURE; 
+                DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE | EVENTLOG_AUDIT_SUCCESS | EVENTLOG_AUDIT_FAILURE;
                 RegSetValueEx(hk, "TypesSupported", 0, REG_DWORD,  (LPBYTE) &dwData,  sizeof(DWORD));
                 dwData = 16;
                 RegSetValueEx(hk, "CategoryCount", 0, REG_DWORD,  (LPBYTE) &dwData,  sizeof(DWORD));
             }
-            RegCloseKey(hk); 
+            RegCloseKey(hk);
         }
         hEventLog = RegisterEventSource(NULL,"Seisint");
         if (!hEventLog) {
@@ -2621,8 +2639,8 @@ bool CSysLogEventLogger::win32Report(unsigned eventtype, unsigned category, unsi
 
 CSysLogEventLogger::~CSysLogEventLogger()
 {
-    if (hEventLog!=0) 
-        DeregisterEventSource(hEventLog); 
+    if (hEventLog!=0)
+        DeregisterEventSource(hEventLog);
 }
 
 #else
@@ -2631,7 +2649,7 @@ CSysLogEventLogger::~CSysLogEventLogger()
 
 #define CATEGORY_AUDIT_FUNCTION_REQUIRED
 #define AUDIT_TYPES_BEGIN int auditTypeDataMap[NUM_AUDIT_TYPES+1] = {
-#define MAKE_AUDIT_TYPE(name, type, categoryid, eventid, level) level, 
+#define MAKE_AUDIT_TYPE(name, type, categoryid, eventid, level) level,
 #define AUDIT_TYPES_END 0 };
 #include "jelogtype.hpp"
 #undef CATEGORY_AUDIT_FUNCTION_REQUIRED
@@ -2932,7 +2950,7 @@ extern jlib_decl void UseSysLogForOperatorMessages(bool use)
         return;
     if (use) {
         msgHandler = getSysLogMsgHandler();
-        ILogMsgFilter * operatorFilter = getCategoryLogMsgFilter(MSGAUD_operator, MSGCLS_all, DefaultDetail, true);  
+        ILogMsgFilter * operatorFilter = getCategoryLogMsgFilter(MSGAUD_operator, MSGCLS_all, DefaultDetail, true);
         queryLogMsgManager()->addMonitorOwn(msgHandler, operatorFilter);
     }
     else {
@@ -3435,15 +3453,15 @@ TraceFlags loadTraceFlags(const IPropertyTree *ptree, const std::initializer_lis
     return dft;
 }
 
-void ctxlogReport(const LogMsgCategory & cat, const char * format, ...) 
+void ctxlogReport(const LogMsgCategory & cat, const char * format, ...)
 {
     va_list args;
     va_start(args, format);
     ctxlogReportVA(cat, NoLogMsgCode, format, args);
     va_end(args);
 }
 
-void ctxlogReportVA(const LogMsgCategory & cat, const char * format, va_list args) 
+void ctxlogReportVA(const LogMsgCategory & cat, const char * format, va_list args)
 {
     ctxlogReportVA(cat, NoLogMsgCode, format, args);
 }