Bugfix: switching brokers with self signed certs. (#1007)

* Fixes BLE scanning being disabled after switching server fails.

* Fixes crash caused by BLE scan results arriving while switching servers.

* Adds documentation to use the server switching command.

Author: h2zero

docs/use/gateway.md

@@ -57,6 +57,27 @@ Server, port, and secure_flag are only required if changing connection to anothe
 If the new connection fails the gateway will fallback to the previous connection.
 :::
 
+## Switching brokers and using self signed and client certificates
+
+In the `user_config.h` file it is possible to specify multiple MQTT brokers and client certificates. These are commonly self signed and are supported by defining `MQTT_SECURE_SELF_SIGNED` as true or 1.  
+Additonally, support for multiple brokers and client certificates has been added. To use this, it is required that the server certificate, client certificate, and client key are provided as their own constatnt string value as demonstrated in the file.  
+To add more than one broker and switch between them it is necessary to provide all of the relevant certificates/keys and add their respective variable names in the `certs_array` structure, as shown in `user_config.h`, and changing the array size to the number of different connections -1.  
+
+To switch between these servers with an MQTT command message, the format is as follows:
+```
+mosquitto_pub -t "home/OpenMQTTGateway/commands/MQTTtoSYS/config" -m
+'{
+  "mqtt_user": "user",
+  "mqtt_pass": "password",
+  "mqtt_server": "host",
+  "mqtt_port": "port",
+  "mqtt_secure": "true",
+  "mqtt_cert_index":0
+ }'
+ ```
+::: tip
+The `mqtt_cert_index` value corresponds to the 0 to X index of the `certs_array` in `user_config.h`.
+:::
 
 # Firmware update from MQTT (ESP only)
 

main/User_config.h

@@ -198,7 +198,7 @@ vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
 #  endif
 
 #  ifndef MQTT_SECURE_SELF_SIGNED_CLIENT
-#    define MQTT_SECURE_SELF_SIGNED_CLIENT 1
+#    define MQTT_SECURE_SELF_SIGNED_CLIENT 1 // If using a self signed certificate for the broker and not using client certificates set this to false or 0
 #  endif
 
 #  ifndef MQTT_SECURE_SELF_SIGNED_INDEX_DEFAULT
@@ -219,9 +219,9 @@ const char* ss_client_cert PROGMEM = R"EOF("
 ")EOF";
 
 const char* ss_client_key PROGMEM = R"EOF("
------BEGIN CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
 ...
------END CERTIFICATE-----
+-----END RSA PRIVATE KEY-----
 ")EOF";
 
 struct ss_certs {

main/ZgatewayBT.ino

@@ -623,85 +623,87 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks {
   }
 
   void onResult(BLEAdvertisedDevice* advertisedDevice) {
-    Log.trace(F("Creating BLE buffer" CR));
-    JsonObject& BLEdata = getBTJsonObject();
-    String mac_adress = advertisedDevice->getAddress().toString().c_str();
-    mac_adress.toUpperCase();
-    BLEdata.set("id", (char*)mac_adress.c_str());
-    Log.notice(F("Device detected: %s" CR), (char*)mac_adress.c_str());
-    BLEdevice* device = getDeviceByMac(BLEdata["id"].as<const char*>());
+    if (!ProcessLock) {
+      Log.trace(F("Creating BLE buffer" CR));
+      JsonObject& BLEdata = getBTJsonObject();
+      String mac_adress = advertisedDevice->getAddress().toString().c_str();
+      mac_adress.toUpperCase();
+      BLEdata.set("id", (char*)mac_adress.c_str());
+      Log.notice(F("Device detected: %s" CR), (char*)mac_adress.c_str());
+      BLEdevice* device = getDeviceByMac(BLEdata["id"].as<const char*>());
 
 #    if BLE_FILTER_CONNECTABLE
-    if (device->connect) {
-      Log.notice(F("Filtered connectable device" CR));
-      return;
-    }
+      if (device->connect) {
+        Log.notice(F("Filtered connectable device" CR));
+        return;
+      }
 #    endif
 
-    if ((!oneWhite || isWhite(device)) && !isBlack(device)) { //if not black listed mac we go AND if we have no white mac or this mac is  white we go out
-      if (advertisedDevice->haveName())
-        BLEdata.set("name", (char*)advertisedDevice->getName().c_str());
-      if (advertisedDevice->haveManufacturerData()) {
-        char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice->getManufacturerData().data(), advertisedDevice->getManufacturerData().length());
-        Log.trace(F("Manufacturer Data: %s" CR), manufacturerdata);
-        BLEdata.set("manufacturerdata", manufacturerdata);
-        free(manufacturerdata);
-      }
-      if (advertisedDevice->haveRSSI())
-        BLEdata.set("rssi", (int)advertisedDevice->getRSSI());
-      if (advertisedDevice->haveTXPower())
-        BLEdata.set("txpower", (int8_t)advertisedDevice->getTXPower());
-      if (advertisedDevice->haveRSSI() && !publishOnlySensors && hassPresence) {
-        hass_presence(BLEdata); // this device has an rssi and we don't want only sensors so in consequence we can use it for home assistant room presence component
-      }
-      if (advertisedDevice->haveServiceData()) {
-        int serviceDataCount = advertisedDevice->getServiceDataCount();
-        Log.trace(F("Get services data number: %d" CR), serviceDataCount);
-        for (int j = 0; j < serviceDataCount; j++) {
-          std::string service_data = convertServiceData(advertisedDevice->getServiceData(j));
-          Log.trace(F("Service data: %s" CR), service_data.c_str());
-          BLEdata.set("servicedata", (char*)service_data.c_str());
-          std::string serviceDatauuid = advertisedDevice->getServiceDataUUID(j).toString();
-          Log.trace(F("Service data UUID: %s" CR), (char*)serviceDatauuid.c_str());
-          BLEdata.set("servicedatauuid", (char*)serviceDatauuid.c_str());
-          process_bledata(BLEdata); // this will force to resolve all the service data
+      if ((!oneWhite || isWhite(device)) && !isBlack(device)) { //if not black listed mac we go AND if we have no white mac or this mac is  white we go out
+        if (advertisedDevice->haveName())
+          BLEdata.set("name", (char*)advertisedDevice->getName().c_str());
+        if (advertisedDevice->haveManufacturerData()) {
+          char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice->getManufacturerData().data(), advertisedDevice->getManufacturerData().length());
+          Log.trace(F("Manufacturer Data: %s" CR), manufacturerdata);
+          BLEdata.set("manufacturerdata", manufacturerdata);
+          free(manufacturerdata);
         }
-
-        if (serviceDataCount > 1) {
-          BLEdata.remove("servicedata");
-          BLEdata.remove("servicedatauuid");
-
-          int msglen = BLEdata.measureLength() + 1;
-          char jsonmsg[msglen];
-          char jsonmsgb[msglen];
-          BLEdata.printTo(jsonmsgb, sizeof(jsonmsgb));
+        if (advertisedDevice->haveRSSI())
+          BLEdata.set("rssi", (int)advertisedDevice->getRSSI());
+        if (advertisedDevice->haveTXPower())
+          BLEdata.set("txpower", (int8_t)advertisedDevice->getTXPower());
+        if (advertisedDevice->haveRSSI() && !publishOnlySensors && hassPresence) {
+          hass_presence(BLEdata); // this device has an rssi and we don't want only sensors so in consequence we can use it for home assistant room presence component
+        }
+        if (advertisedDevice->haveServiceData()) {
+          int serviceDataCount = advertisedDevice->getServiceDataCount();
+          Log.trace(F("Get services data number: %d" CR), serviceDataCount);
           for (int j = 0; j < serviceDataCount; j++) {
-            strcpy(jsonmsg, jsonmsgb); // the parse _destroys_ the message buffer
-            JsonObject& BLEdataLocal = getBTJsonObject(jsonmsg, j == 0); // note, that first time we will get here the BLEdata itself; haPresence for the first msg
-            if (!BLEdataLocal.containsKey("id")) { // would crash without id
-              Log.trace("Json parsing error for %s" CR, jsonmsgb);
-              break;
-            }
             std::string service_data = convertServiceData(advertisedDevice->getServiceData(j));
+            Log.trace(F("Service data: %s" CR), service_data.c_str());
+            BLEdata.set("servicedata", (char*)service_data.c_str());
             std::string serviceDatauuid = advertisedDevice->getServiceDataUUID(j).toString();
+            Log.trace(F("Service data UUID: %s" CR), (char*)serviceDatauuid.c_str());
+            BLEdata.set("servicedatauuid", (char*)serviceDatauuid.c_str());
+            process_bledata(BLEdata); // this will force to resolve all the service data
+          }
 
-            int last = atomic_load_explicit(&jsonBTBufferQueueLast, ::memory_order_seq_cst) % BTQueueSize;
-            int size1 = jsonBTBufferQueue[last].buffer.size();
-            BLEdataLocal.set("servicedata", (char*)service_data.c_str());
-            int size2 = jsonBTBufferQueue[last].buffer.size();
-            BLEdataLocal.set("servicedatauuid", (char*)serviceDatauuid.c_str());
-            int size3 = jsonBTBufferQueue[last].buffer.size();
-            Log.trace("Buffersize for %d : %d -> %d -> %d" CR, j, size1, size2, size3);
-            PublishDeviceData(BLEdataLocal);
+          if (serviceDataCount > 1) {
+            BLEdata.remove("servicedata");
+            BLEdata.remove("servicedatauuid");
+
+            int msglen = BLEdata.measureLength() + 1;
+            char jsonmsg[msglen];
+            char jsonmsgb[msglen];
+            BLEdata.printTo(jsonmsgb, sizeof(jsonmsgb));
+            for (int j = 0; j < serviceDataCount; j++) {
+              strcpy(jsonmsg, jsonmsgb); // the parse _destroys_ the message buffer
+              JsonObject& BLEdataLocal = getBTJsonObject(jsonmsg, j == 0); // note, that first time we will get here the BLEdata itself; haPresence for the first msg
+              if (!BLEdataLocal.containsKey("id")) { // would crash without id
+                Log.trace("Json parsing error for %s" CR, jsonmsgb);
+                break;
+              }
+              std::string service_data = convertServiceData(advertisedDevice->getServiceData(j));
+              std::string serviceDatauuid = advertisedDevice->getServiceDataUUID(j).toString();
+
+              int last = atomic_load_explicit(&jsonBTBufferQueueLast, ::memory_order_seq_cst) % BTQueueSize;
+              int size1 = jsonBTBufferQueue[last].buffer.size();
+              BLEdataLocal.set("servicedata", (char*)service_data.c_str());
+              int size2 = jsonBTBufferQueue[last].buffer.size();
+              BLEdataLocal.set("servicedatauuid", (char*)serviceDatauuid.c_str());
+              int size3 = jsonBTBufferQueue[last].buffer.size();
+              Log.trace("Buffersize for %d : %d -> %d -> %d" CR, j, size1, size2, size3);
+              PublishDeviceData(BLEdataLocal);
+            }
+          } else {
+            PublishDeviceData(BLEdata, false); // easy case
           }
         } else {
-          PublishDeviceData(BLEdata, false); // easy case
+          PublishDeviceData(BLEdata); // PublishDeviceData has its own logic whether it needs to publish the json or not
         }
       } else {
-        PublishDeviceData(BLEdata); // PublishDeviceData has its own logic whether it needs to publish the json or not
+        Log.trace(F("Filtered mac device" CR));
       }
-    } else {
-      Log.trace(F("Filtered mac device" CR));
     }
   }
 };

main/main.ino

@@ -868,6 +868,7 @@ void setupTLS(bool self_signed, uint8_t index) {
   WiFiClientSecure* sClient = (WiFiClientSecure*)eClient;
 #  if MQTT_SECURE_SELF_SIGNED
   if (self_signed) {
+    Log.notice(F("Using self signed cert index %u" CR), index);
 #    if defined(ESP32)
     sClient->setCACert(certs_array[index].server_cert);
 #      if MQTT_SECURE_SELF_SIGNED_CLIENT
@@ -1881,33 +1882,32 @@ void MQTTtoSYS(char* topicOri, JsonObject& SYSdata) { // json object decoding
     }
 
     if (SYSdata.containsKey("mqtt_user") && SYSdata.containsKey("mqtt_pass")) {
-#  if defined(ZgatewayBT) && defined(ESP32)
-      stopProcessing();
-#  endif
-      client.disconnect();
       bool update_server = false;
       bool secure_connect = SYSdata.get<bool>("mqtt_secure");
       void* prev_client = nullptr;
-      bool use_ss_cert = SYSdata.containsKey("mqtt_ss_cert");
+      bool use_ss_cert = SYSdata.containsKey("mqtt_cert_index");
       uint8_t cert_index = mqtt_ss_index;
 
       if (SYSdata.containsKey("mqtt_server") && SYSdata.containsKey("mqtt_port")) {
         if (!SYSdata.containsKey("mqtt_secure")) {
-          Log.warning(F("mqtt_server provided without mqtt_secure defined - ignoring command" CR));
+          Log.error(F("mqtt_server provided without mqtt_secure defined - ignoring command" CR));
           return;
         }
 #  if MQTT_SECURE_SELF_SIGNED
         if (use_ss_cert) {
-          cert_index = SYSdata.get<uint8_t>("mqtt_ss_cert");
+          cert_index = SYSdata.get<uint8_t>("mqtt_cert_index");
           if (cert_index >= sizeof(certs_array) / sizeof(ss_certs)) {
-            Log.warning(F("mqtt_ss_cert index invalid - ignoring command" CR));
+            Log.error(F("mqtt_cert_index invalid - ignoring command" CR));
             return;
           }
         }
 #  endif
 
+#  if defined(ZgatewayBT) && defined(ESP32)
+        stopProcessing();
+#  endif
+        client.disconnect();
         update_server = true;
-
         if (secure_connect != mqtt_secure) {
           prev_client = eClient;
           if (!mqtt_secure) {
@@ -1925,6 +1925,11 @@ void MQTTtoSYS(char* topicOri, JsonObject& SYSdata) { // json object decoding
         }
 
         client.setServer(SYSdata.get<const char*>("mqtt_server"), SYSdata.get<unsigned int>("mqtt_port"));
+      } else {
+#  if defined(ZgatewayBT) && defined(ESP32)
+        stopProcessing();
+#  endif
+        client.disconnect();
       }
 
       String prev_user = mqtt_user;