Merge remote-tracking branch 'origin/candidate-9.10.x' into candidate-9.12.x

Signed-off-by: Gavin Halliday <[email protected]>

Author: ghalliday

.github/workflows/test-smoke-gh_runner.yml

@@ -77,6 +77,13 @@ jobs:
           sudo apt-get install -y gdb
           echo 'core_%e.%p' | sudo tee /proc/sys/kernel/core_pattern
 
+          # Install, configure and start TinyProxy
+          sudo apt-get install -y tinyproxy
+          echo "Port 8888"    > tinyproxy.conf
+          echo "Timeout 600" >> tinyproxy.conf
+          tinyproxy -c tinyproxy.conf
+          pgrep tinyproxy
+
       - name: Install Package
         shell: "bash"
         run: |

common/thorhelper/roxiehelper.cpp

@@ -1537,7 +1537,6 @@ bool CSafeSocket::readBlocktms(StringBuffer &ret, unsigned timeoutms, HttpHelper
         bool sockClosed = false;
         if (pHttpHelper && pHttpHelper->isHttp())
         {
-#define MAX_HTTP_HEADERSIZE 16000 //arbitrary per line limit, most web servers are lower, but REST queries can be complex..
             char header[MAX_HTTP_HEADERSIZE + 1]; // allow room for \0
             CTimeMon tm(timeoutms);
             unsigned totalRead = 0;
@@ -1585,6 +1584,8 @@ bool CSafeSocket::readBlocktms(StringBuffer &ret, unsigned timeoutms, HttpHelper
                     }
                     break;
                 }
+                if (totalRead >= MAX_HTTP_HEADERSIZE)
+                    throw makeStringException(THORHELPER_DATA_ERROR, "Badly formed HTTP header");
             }
 
             if (tm.timedout(&remaining))

common/thorhelper/thorhelper.hpp

@@ -32,6 +32,8 @@
 #define THORMASTERLOGSEARCHSTR "thormaster."
 #define THORSLAVELOGSEARCHSTR "thorslave."
 
+#define MAX_HTTP_HEADERSIZE 16000 //arbitrary per line limit, most web servers are lower, but REST queries can be complex..
+
 interface IXmlToRawTransformer : extends IInterface
 {
     virtual IDataVal & transform(IDataVal & result, size32_t len, const void * text, bool isDataset) = 0;

common/thorhelper/thorsoapcall.cpp

@@ -1992,6 +1992,8 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
     void createHttpRequest(StringBuffer &request, const Url &url, const IProperties * traceHeaders)
     {
         // Create the HTTP POST request
+        // NOTE: if using an http proxy and method is not https then path should be prepended with url host:port
+        //       but this would break regression tests that use esp as a proxy
         if (master->wscType == STsoap)
             request.clear().append("POST ").append(url.path).append(" HTTP/1.1\r\n");
         else
@@ -2517,6 +2519,10 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                     checkTimeLimitExceeded(&remainingMS);
                     Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0);
 
+                    bool useProxy = false;
+                    if (!master->proxyUrlArray.empty())
+                        useProxy = true;
+
                     CCycleTimer dnsTimer;
 
                     // TODO: for DNS, do we use timeoutMS or remainingMS or remainingMS / maxRetries+1 or ?
@@ -2554,7 +2560,54 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                         if (proto == PersistentProtocol::ProtoTLS)
                         {
 #ifdef _USE_OPENSSL
-                            Owned<ISecureSocket> ssock = master->createSecureSocket(socket.getClear(), connUrl.host);
+                            if (useProxy && strieq(connUrl.method.str(), "http"))
+                            {
+                                StringBuffer urlHost;
+                                if (streq(url.host.str(), "."))
+                                    urlHost.append(GetCachedHostName());
+                                else
+                                    urlHost.append(url.host.str());
+                                StringBuffer proxyText;
+                                proxyText.appendf("CONNECT %s:%d HTTP/1.1\r\n", urlHost.str(), url.port);
+                                proxyText.append("Proxy-Connection: Keep-Alive\r\n\r\n");
+                                socket->write(proxyText.str(),proxyText.length());
+
+                                checkTimeLimitExceeded(&remainingMS);
+                                unsigned proxyTimeout = MIN(remainingMS, master->timeoutMS);
+                                char proxyResponse[MAX_HTTP_HEADERSIZE + 1];
+                                unsigned proxyRemaining = proxyTimeout;
+                                CTimeMon tm(proxyTimeout);
+                                unsigned totalRead = 0;
+                                unsigned bytesRead = 0;
+                                bool sockClosed = false;
+                                while (!sockClosed && !tm.timedout(&proxyRemaining))
+                                {
+                                    unsigned maxPoss = MAX_HTTP_HEADERSIZE - totalRead;
+                                    sockClosed = readtmsAllowClose(socket, &proxyResponse[totalRead], 1, maxPoss, bytesRead, proxyRemaining);
+                                    totalRead += bytesRead;
+                                    proxyResponse[totalRead] = 0;
+
+                                    if (strstr(proxyResponse, "\r\n\r\n"))
+                                        break;
+                                    if (totalRead >= MAX_HTTP_HEADERSIZE)
+                                        break;
+                                }
+
+                                if (tm.timedout(&proxyRemaining))
+                                    throw makeStringException(-1, "Timed out waiting for proxy response");
+
+                                bool proxyTunnelOK = false;
+                                const char *okResp = strstr(proxyResponse, "HTTP/");
+                                if (okResp)
+                                {
+                                    if (strstr(okResp, " 200 "))
+                                        proxyTunnelOK = true;
+                                }
+                                if (!proxyTunnelOK)
+                                    throw makeStringException(-1, "Invalid response from proxy");
+                            }
+
+                            Owned<ISecureSocket> ssock = master->createSecureSocket(socket.getClear(), url.host);
                             if (ssock)
                             {
                                 checkTimeLimitExceeded(&remainingMS);

common/workunit/workunit.cpp

@@ -6157,6 +6157,11 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
         query.append('"').append(value).append("\"]");
     };
 
+    static void appendMinCostToQueryString(StringBuffer& query, WUSortField filter, const char* value)
+    {
+        query.append('[').append(queryFilterXPath(filter)).append(">=\"").append(money2cost_type(atof(value))).append("\"]");
+    };
+
     IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by (and flags for desc sort etc)
                                                 WUSortField *filters,   // NULL or list of fields to filter on (terminated by WUSFterm)
                                                 const void *filterbuf,  // (appended) string values for filters
@@ -6369,6 +6374,10 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
                     else
                         query.append("[@protected!=\"1\"]"); //The @protected is set to '0' or not set.
                 }
+                else if (subfmt==WUSFcostcompile || subfmt==WUSFcostexecute || subfmt==WUSFcostfileaccess )
+                {
+                    appendMinCostToQueryString(query, (WUSortField)subfmt, fv);
+                }
                 else if (!*fv)
                 {
                     unknownAttributes.append(getEnumText(subfmt,workunitSortFields));

common/workunit/workunit.hpp

@@ -1855,4 +1855,14 @@ class WORKUNIT_API WuidPattern
     inline operator StringBuffer& () { return pattern; }
 };
 
+inline cost_type getGuillotineCost(IConstWorkUnit *wu)
+{
+    cost_type wuMaxCost = money2cost_type(wu->getDebugValueReal("maxCost", 0));
+    cost_type guillotineCost = wuMaxCost ? wuMaxCost : money2cost_type(getConfigReal("cost/@limit"));
+    cost_type hardCostLimit = money2cost_type(getConfigReal("cost/@hardlimit"));
+    if (hardCostLimit && ((guillotineCost == 0) || (guillotineCost > hardCostLimit)))
+        guillotineCost = hardCostLimit;
+    return guillotineCost;
+}
+
 #endif

dali/base/dadfs.cpp

@@ -5100,9 +5100,10 @@ protected: friend class CDistributedFilePart;
         double sizeGB = getDiskSize(true, false) / ((double)1024 * 1024 * 1024);
         const IPropertyTree *attrs = root->queryPropTree("Attr");
 
+        //This array must still exist when getReadCost() etc. is called, otherwise cluster will have been freed.
+        StringArray clusterNames;
         if (isEmptyString(cluster))
         {
-            StringArray clusterNames;
             unsigned countClusters = getClusterNames(clusterNames);
             for (unsigned i = 0; i < countClusters; i++)
                 atRestCost += calcFileAtRestCost(clusterNames[i], sizeGB, fileAgeDays);

dali/base/dasds.cpp

@@ -2796,7 +2796,7 @@ class CServerRemoteTree : public CRemoteTreeBase
         if (SDSManager->ignoreExternals) return COMPRESS_METHOD_NONE;
         if (!hasProp(EXT_ATTR))
             return COMPRESS_METHOD_NONE;
-        return (CompressionMethod)getPropInt(COMPRESS_ATTR, COMPRESS_METHOD_LZW);
+        return (CompressionMethod)getPropInt(COMPRESS_ATTR, COMPRESS_METHOD_LZW_LITTLE_ENDIAN);
     }
 
     bool getProp(const char *xpath, StringBuffer &ret) const override

dockerfiles/platform-core-ubuntu-20.04.dockerfile

@@ -68,9 +68,9 @@ RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lf
 
 # Set the locale
 RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8     
+ENV LANG=en_US.UTF-8  
+ENV LANGUAGE=en_US:en  
+ENV LC_ALL=en_US.UTF-8     
 
 RUN groupadd -g 10001 hpcc
 RUN useradd -s /bin/bash -m -r -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc

dockerfiles/platform-core-ubuntu-22.04/Dockerfile

@@ -68,9 +68,9 @@ RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lf
 
 # Set the locale
 RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8     
+ENV LANG=en_US.UTF-8  
+ENV LANGUAGE=en_US:en  
+ENV LC_ALL=en_US.UTF-8     
 
 RUN groupadd -g 10001 hpcc
 RUN useradd -s /bin/bash -m -r -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc