added startup ready check for each extension and server, fixed autopilot error in pattern matching, fixed incorrect byRef function return

Author: SilenusTA

backend/data_center/modules/server_socket.js

@@ -126,6 +126,11 @@ const localConfig =
 
     // callback to main server to save the updated server config
     saveServerConfigFunc: null,
+    // startup extension check for readiness interval
+    StartupIntervalHandle: null,
+    StartupInterval: 200,
+    StartupMaxTime: 100, //give extension 10seconds (StartupInterval*StartupMaxTime)  to startup
+    StartupMaxTimeCounter: 0,
 }
 const triggersandactions =
 {
@@ -240,6 +245,48 @@ const triggersandactions =
             }
         ],
 }
+
+
+// check for extension status so we can send a message out when all have reported read
+/**
+ * checks for extensions sending 'ready' message and send out a 'StreamRollerReady' when all extensions have responded
+ */
+function extensionReadinessCheckScheduler ()
+{
+    clearTimeout(localConfig.StartupIntervalHandle)
+    localConfig.StartupIntervalHandle = setInterval(() =>
+    {
+        extensionReadinessCheck()
+    }, localConfig.StartupInterval);
+}
+/**
+ * Extension readiness check
+ */
+function extensionReadinessCheck ()
+{
+    let AllExtensionsReady = true; // negative check
+    for (var key in localConfig.extensions) 
+    {
+        if (localConfig.extensions[key].state != "ready")
+            AllExtensionsReady = false;
+    }
+    //are we out of time or extensions finished loading
+    if (localConfig.StartupMaxTimeCounter++ > localConfig.StartupMaxTime
+        || (AllExtensionsReady && localConfig.extensions != {} && localConfig.server_socket != null))
+    {
+        // list extensions that failed to load in time
+        if (localConfig.StartupMaxTimeCounter++ > localConfig.StartupMaxTime)
+        {
+            for (var ext in localConfig.extensions)
+            {
+                if (localConfig.extensions[ext].state != "ready")
+                    console.log(`server startup time out for ${ext} `)
+            }
+        }
+        mh.broadcastMessage(localConfig.server_socket, sr_api.ServerPacket("StreamRollerReady", localConfig.extensionname, {}))
+        clearInterval(localConfig.StartupIntervalHandle)
+    }
+}
 // ============================================================================
 //                           FUNCTION: start
 // ============================================================================
@@ -268,7 +315,15 @@ function start (app, server, exts, serverConfig, saveServerConfigFunc)
     // create our extension array
     exts.forEach((elem, i) =>
     {
-        localConfig.extensions[elem] = {};
+        // extensions without backend servers may not send a running message via the socket
+        // and will set it via a function callback from the init function.
+        // That will happen before we get here so we need to make sure we don't overwrite 
+        // the running flags for these extensions
+        if (!localConfig.extensions[elem])
+        {
+            localConfig.extensions[elem] = {}
+            localConfig.extensions[elem].state = "loading";
+        }
     });
     localConfig.backend_server = server;
     cm.initcrypto();
@@ -317,6 +372,7 @@ function onConnect (socket)
 {
     socket.emit("connected", socket.id);
     socket.join(localConfig.channel);
+    extensionReadinessCheckScheduler();
     sendAddressTrigger();
 }
 // ============================================================================
@@ -373,9 +429,14 @@ function onMessage (socket, server_packet)
         mh.errorMessage(socket, "Missing type/from field", server_packet);
         return;
     }
-    // add this socket to the extension if it doesn't already exist
+    // add this socket to the extension list if it doesn't already exist
+    // these will be extension not started by the server
     if (typeof (localConfig.extensions[server_packet.from]) === "undefined" || !localConfig.extensions[server_packet.from])
+    {
         localConfig.extensions[server_packet.from] = {};
+        // added new extension so restart the check
+        extensionReadinessCheckScheduler();
+    }
     // check we have a valid socket for this extension, don't need to check if the extension exists as it will have been added above
     if (typeof (localConfig.extensions[server_packet.from].socket) === "undefined" || !localConfig.extensions[server_packet.from].socket)
     {
@@ -388,7 +449,7 @@ function onMessage (socket, server_packet)
     else
     {
         // note that we currently only have one slot per connection. this works for extensions that are only loaded once
-        // but for webpage stuff we will need to allow more than one sockect for that extension name
+        // but for webpage stuff we will need to allow more than one socket for that extension name
         // we need to append the socket id to the extension name to fix this!!!
         if (localConfig.extensions[server_packet.from].socket.id != socket.id)
         {
@@ -403,7 +464,11 @@ function onMessage (socket, server_packet)
         localConfig.connected_extensionlist[server_packet.from] = localConfig.extensions[server_packet.from].socket.connected
     }
     // process the clients request
-    if (server_packet.type === "RequestSoftwareVersion")
+    if (server_packet.type === "ExtensionConnected")
+        localConfig.extensions[server_packet.from] = { state: "connected" };
+    else if (server_packet.type === "ExtensionReady")
+        localConfig.extensions[server_packet.from] = { state: "ready" };
+    else if (server_packet.type === "RequestSoftwareVersion")
         mh.sendSoftwareVersion(socket, server_packet.from);
     else if (server_packet.type === "RequestConfig")
         mh.sendConfig(socket, server_packet.from);
@@ -771,7 +836,7 @@ function dataMonitorScheduler ()
 //                           FUNCTION: sendAddressTrigger
 // ============================================================================
 /**
- * Finds the trigger using the passed messagetype
+ * sends "trigger_StreamRollerIPChanged" message
  * @returns trigger
  */
 function sendAddressTrigger ()
@@ -815,7 +880,7 @@ function findTriggerByMessageType (messagetype)
     for (let i = 0; i < triggersandactions.triggers.length; i++)
     {
         if (triggersandactions.triggers[i].messagetype.toLowerCase() == messagetype.toLowerCase())
-            return triggersandactions.triggers[i];
+            return structuredClone(triggersandactions.triggers[i]);
     }
     logger.err(localConfig.SYSTEM_LOGGING_TAG + localConfig.extensionname +
         ".findTriggerByMessageType", "failed to find trigger", messagetype);
@@ -845,8 +910,21 @@ function sendTrigger (data)
         )
     );
 }
+/**
+ * Used by locally started extensions that have client webpages with the socket connection
+ * these might not always be loading in a browser so we fake the ready flag on startup.
+ * @param {string} ext 
+ */
+function readyMessage (ext)
+{
+    if (!localConfig.extensions[ext])
+    {
+        localConfig.extensions[ext] = {}
+        localConfig.extensions[ext].state = "ready"
+    }
+}
 // ============================================================================
 //                           EXPORTS:
 // ============================================================================
-export { start, triggersandactions };
+export { start, triggersandactions, readyMessage };
 

backend/data_center/server.js

@@ -44,6 +44,14 @@ import * as logger from "./modules/logger.js";
 import * as ServerSocket from "./modules/server_socket.js";
 import sr_api from "./public/streamroller-message-api.cjs";
 
+/*
+process.on('unhandledRejection', (reason, promise) =>
+{
+    console.log('Unhandled Promise Rejection:');
+    console.log('Reason:', reason);
+    console.log('Stack Trace:', reason?.stack || 'No stack trace available');
+});
+*/
 // testing startup time
 let DEBUG_TIMING = false;
 let debugStartTime = performance.now()
@@ -313,6 +321,10 @@ async function loadExtensions (extensionFolder)
                                 );
                             return x;
                         })
+                        .catch((err) =>
+                        {
+                            console.log("error loading module", err)
+                        })
                     return x;
                 }
                 else
@@ -345,7 +357,8 @@ async function loadExtensions (extensionFolder)
                             app, "http://" + serverConfig.HOST,
                             serverConfig.PORT,
                             // add a slight offset to the heartbeat so they don't all end up synced
-                            serverConfig.heartbeat + (Math.floor(Math.random() * 100)));
+                            serverConfig.heartbeat + (Math.floor(Math.random() * 100)),
+                            ServerSocket.readyMessage);
                     else
                         logger.err("[" + serverConfig.SYSTEM_LOGGING_TAG + "]server.js", "Error: Extension module " + files[index] + " did not export an initialise function");
                     if (DEBUG_TIMING)

extensions/autopilot/autopilot.js

@@ -51,7 +51,7 @@ import * as server from "./server/server.js"
  * @param {number} port 
  * @param {number} heartbeat 
  */
-function initialise (app, host, port, heartbeat)
+function initialise (app, host, port, heartbeat, readyMessagefn)
 {
     config.heartBeatTimeout = heartbeat;
     app.use("/autopilot/", express.static(__dirname + "/public"));
@@ -72,6 +72,7 @@ function initialise (app, host, port, heartbeat)
                 }
             );
         });
+        readyMessagefn(config.EXTENSION_NAME)
     } catch (err)
     {
         logger.err(config.SYSTEM_LOGGING_TAG + config.EXTENSION_NAME + ".initialise", "initialise failed:", err);

extensions/autopilot/server/server.js

@@ -177,9 +177,7 @@ function onDataCenterConnect (socket)
  */
 function onDataCenterMessage (server_packet)
 {
-    // -------------------------------------------------------------------------------------------------
-    //                  RECEIVED CONFIG
-    // -------------------------------------------------------------------------------------------------
+
     if (server_packet.type === "ConfigFile")
     {
         // check it is our config
@@ -587,54 +585,63 @@ function ProcessReceivedTrigger (pairing, receivedTrigger)
     let match = true
     // we have the correct extension, channel and message type
     // lets check the variables to see if those are a match
+
+    // this just covers { MATCHER_sender: '1', sender: '' } part of teh parameter.
     pairing.trigger.data.forEach((param) =>
     {
         for (var i in param)
         {
+            let receivedParam = ""
+            if (typeof receivedTrigger.data.parameters[i] !== "undefined" && receivedTrigger.data.parameters[i] !== null)
+                receivedParam = receivedTrigger.data.parameters[i].toString().toLowerCase()
+            let checkParam = ""
+            if (typeof param[i] !== "undefined" && param[i] !== null)
+                checkParam = param[i].toString().toLowerCase()
             try
             {
                 // don't check the MATCHER variables as these are used to determine how to perform the match (Start of line etc)
                 if (i.indexOf("MATCHER_") != 0 && i != "cooldown" && i != "lastrun")
                 {
                     // get the relevant matcher for this value
                     let searchtype = param["MATCHER_" + i]
-                    if (typeof (receivedTrigger.data.parameters[i]) == "string")// && typeof param[i] === "string")
+
+                    if (receivedParam)// && typeof checkParam === "string")
                     {
                         switch (searchtype)
                         {
                             case "2"://match anywhere
-                                if (param[i] != "" && receivedTrigger.data.parameters[i].toLowerCase().indexOf(param[i].toLowerCase()) == -1)
+                                if (checkParam != "" && receivedParam.indexOf(checkParam) == -1)
                                     match = false;
                                 break;
                             case "3"://match start of line only
-                                if (param[i] != "" && receivedTrigger.data.parameters[i].toLowerCase().indexOf(param[i].toLowerCase()) != 0)
+                                if (checkParam != "" && receivedParam.indexOf(checkParam) != 0)
                                     match = false;
                                 break;
                             case "4"://doesn't match
-                                if (param[i] != "" && receivedTrigger.data.parameters[i].toLowerCase().indexOf(param[i].toLowerCase()) == 0)
+                                if (checkParam != "" && receivedParam.indexOf(checkParam) == 0)
                                     match = false;
                                 break;
                             case "5"://match complete word only
-                                if (param[i] != "" && receivedTrigger.data.parameters[i].toLowerCase().indexOf(param[i].toLowerCase()) == -1)
+                                if (checkParam != "" && receivedParam.indexOf(checkParam) == -1)
                                 {
                                     match = false;
                                 }
                                 else
                                 {
-                                    let wordArray = receivedTrigger.data.parameters[i].split(" ")
+                                    let wordArray = receivedParam.split(" ")
                                     match = false;
-                                    if (wordArray.includes(param[i]))
+                                    if (wordArray.includes(checkParam))
                                         match = true;
                                 }
                                 break;
                             default:
                                 // check for exact match
-                                if (param[i] != "" && receivedTrigger.data.parameters[i].toLowerCase() != param[i].toLowerCase())
+                                if (checkParam != "" && receivedParam != checkParam)
                                     match = false;
                         }
                     }
                     //check non string types for not matching
-                    else if (param[i] != "" && receivedTrigger.data.parameters[i] != param[i])
+                    else if (checkParam != "" && receivedParam != checkParam)
                         match = false;
                 }
             }
@@ -738,7 +745,7 @@ function TriggerAction (action, triggerParams)
                             // remove the first number of words the user specified
                             for (var x = 0; x < wordNumber; x++)
                                 sourceArray.splice(0, 1)
-                            tempAction = sourceArray.join(" ").trim()
+                            tempAction = tempAction.replace("%%" + sourceVar + "%%", sourceArray.join(" ").trim())
                         }
                         else
                         {
@@ -747,7 +754,7 @@ function TriggerAction (action, triggerParams)
                             sourceData.replaceAll("%%", "")
                             // split the data into an array so we can index the work the user wants
                             const sourceArray = sourceData.split(" ");
-                            tempAction = sourceArray[wordNumber]
+                            tempAction = tempAction.replace("%%" + sourceVar + "%%", sourceArray[wordNumber])
                         }
                     }
                     else