move module installer startup to background process, fixes : 69

Author: sdetweil

module_installer/formatter.js

@@ -0,0 +1,61 @@
+const { exec, execSync } = require("child_process");
+const path=require('path')
+const os= require('os')
+const fs = require('fs')
+const formatter=require(__dirname+'/'+'formatModuleInfo.js')
+const modules_url ="https://modules.magicmirror.builders/data/modules.json"
+const module_form_template=__dirname+"/module_schema_template.json"
+const module_selector_form=__dirname+"/module_selector_schema.json"
+const module_url_hash=__dirname+"/../module_url_hash.json"
+const modules_location=__dirname+"/../../../modules"
+const formTail = ", \"installable\":[]\n}}"
+
+async function buildFormData(/*NodeHelper,*/ sortOrder, debug){
+  // get the latest data from the 3rd party repo
+  //console.log("entered buildformdata")
+  const response = await fetch(modules_url);
+  if (!response.ok) {
+    const message = `An error occured: ${response.status}`;
+    throw new Error(message);
+  }
+  //console.log("fetch conmpleted")
+  // we need the json
+  const responseData = await response.json();
+  //console.log("have form data l="+responseData.length)
+
+  // Extract modules array from the new API structure
+  const modulesArray = responseData.modules || responseData;
+
+  // format in category and in category sorted as requested (date or time)
+  let data = await formatter(modulesArray, sortOrder, debug)
+  //console.log("back from formatter")
+  // make the form for the installer page
+  let newformdata=fs.readFileSync(module_form_template)+'"categories":'+JSON.stringify(data.categories,null,2)+formTail
+  // save it for page load
+  formdata=newformdata
+  // write it out for next time start
+  // write it out to be loaded by installer page
+  fs.writeFileSync(module_selector_form, newformdata)
+  // write out the updated url hash (adds/deletes done twice a day) 
+  fs.writeFileSync(module_url_hash, JSON.stringify(data.hash, null, 2))
+  // async, return something
+  return true
+}
+
+let sortOrder=process.argv[2];
+let debug= false
+//console.log("args=",process.argv)
+if(process.argv.length>2){
+	if(process.argv[3]==='debug')
+		debug = true
+}
+// check to see if url hash exists, and is complete
+fs.stat(module_url_hash, (err, stats) => {
+  //console.log("module hash stat rc="+err)
+  if (!err){  // found it, background process not processing
+    if(stats.size>200000){  // if large, should be done processing
+      //console.log("module hash stat size="+stats.size)
+      buildFormData(sortOrder, debug) // so we can check and update
+    }
+  }
+});

module_installer/geturlcontents.sh

@@ -1,13 +1,13 @@
-#!/bin/bash 
+#!/bin/bash
 
 curl_found=$(which curl | wc -l)
 wget_found=$(which wget | wc -l)
 
 url=$1
 
-if [ $curl_found -gt 0 ]; then
-   curl -sL $url stdout 2>/dev/null
-elif [ $wget_found -gt 0 ]; then
+if [ $wget_found -gt 0 ]; then
   wget -O - $url 2>/dev/null
+elif [ $curl_found -gt 0 ]; then
+  curl -sL $url stdout 2>/dev/null
 fi
 

module_installer/runserver.js

@@ -34,7 +34,7 @@ var processList=[]
 let in_docker_container = false
 
 // the generated module list form
-let formdata
+let formdata=null
 
 const Configurator_in_config_file = false
 
@@ -151,34 +151,15 @@ async function setupServer(expressApp, sortOrder){
 }
 
 async function buildFormData(/*NodeHelper,*/ sortOrder){
+  // used to do this inline, but it blocked MM startup,
+  // takes minutes to sync the initial module list
   // get the latest data from the 3rd party repo
-  //console.log("entered buildformdata")
-  const response = await fetch(modules_url);
-  if (!response.ok) {
-    const message = `An error occured: ${response.status}`;
-    throw new Error(message);
-  }
-  //console.log("fetch conmpleted")
-  // we need the json
-  const responseData = await response.json();
-  //console.log("have form data l="+responseData.length)
-
-  // Extract modules array from the new API structure
-  const modulesArray = responseData.modules || responseData;
-
-  // format in category and in category sorted as requested (date or time)
-  let data = await formatter(modulesArray, sortOrder, true /*local_debug*/)
-  //console.log("back from formatter")
-  // make the form for the installer page
-  let newformdata=fs.readFileSync(module_form_template)+'"categories":'+JSON.stringify(data.categories,null,2)+formTail
-  // save it for page load
-  formdata=newformdata
-  // write it out for next time start
-  // write it out to be loaded by installer page
-  fs.writeFileSync(module_selector_form, newformdata)
-  // write out the updated url hash (adds/deletes done twice a day) 
-  fs.writeFileSync(module_url_hash, JSON.stringify(data.hash, null, 2))
-  // async, return something
+  // execute the 3rd party module jsonform creator in the background
+  const cmdstring="node "+__dirname+"/formatter.js "+sortOrder
+  //console.log("cmdstring="+cmdstring+" cwd="+process.cwd())
+  exec(cmdstring, { cwd: __dirname })
+  // indicate we are working on new data
+  formdata=null;
   return true
 }
 
@@ -222,14 +203,24 @@ function handleConnection(socket, type) {
         console.log("cancel requested")
       console.log("cancel received, closing installer")
       socket.emit("close")
-     // setTimeout(()=>{process.exit(1)}, 3000)
     }
   });
 
   // when the page askes for form
   socket.on("getForm", () => {
     if(local_debug)
-  	  console.log("form requested")
+  	  console.log("installer form requested")
+    // if we haven't loaded the 3rd party modules form for installer
+    if(!formdata){
+      // if the file exists
+       if(fs.statSync(module_selector_form))
+         // load it as text
+	       formdata=fs.readFileSync(module_selector_form, "utf8");
+       else
+         // for didn't exist, don't send junk
+         // user will have to hit refresh to load the page html which will request again
+         return
+    }
     // send it via socket io, string data
     socket.emit("json", "'" + formdata + "'");
   });