From 3a6b4b25b1322a42517324fc4857702555a0f8f9 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Tue, 18 Jan 2022 14:48:43 +0100 Subject: [PATCH 01/19] manually define the littlefs global object --- OTGW-firmware.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/OTGW-firmware.h b/OTGW-firmware.h index 6484f608..6c54a872 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -22,6 +22,10 @@ #include "safeTimers.h" #include "OTGWSerial.h" // Bron Schelte's Serial class - it upgrades and more #include "OTGW-Core.h" // Core code for this firmware +#include "LittleFS.h" + +#define NO_GLOBAL_LITTLEFS // prevent the libary from creating a filesystem +FS LittleFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, FS_MAX_OPEN_FILES))); //OTGW Nodoshop hardware definitions #define I2CSCL D1 From 46f7c6997bc94462f3d11b7179395125f43d3aaa Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Tue, 18 Jan 2022 15:01:56 +0100 Subject: [PATCH 02/19] rename the littlefs object to systemfs and add externs --- FSexplorer.ino | 58 +++++++++++++++++++++++------------------------ MQTTstuff.ino | 6 ++--- OTGW-Core.ino | 8 +++---- OTGW-firmware.h | 2 +- OTGW-firmware.ino | 6 +++-- OTGWSerial.cpp | 6 +++-- helperStuff.ino | 22 +++++++++--------- restAPI.ino | 2 +- settingStuff.ino | 6 ++--- versionStuff.ino | 2 +- 10 files changed, 61 insertions(+), 57 deletions(-) diff --git a/FSexplorer.ino b/FSexplorer.ino index aa6f6854..e9a580d1 100644 --- a/FSexplorer.ino +++ b/FSexplorer.ino @@ -24,7 +24,7 @@ ** setup() ** { ** setupFSexplorer(); -** httpServer.serveStatic("/FSexplorer.png", LittleFS, "/FSexplorer.png"); +** httpServer.serveStatic("/FSexplorer.png", SystemFS, "/FSexplorer.png"); ** httpServer.on("/", sendIndexPage); ** httpServer.on("/index", sendIndexPage); ** httpServer.on("/index.html",sendIndexPage); @@ -61,18 +61,18 @@ const char Header[] = "HTTP/1.1 303 OK\r\nLocation:FSexplorer.html\r\nCache-Cont //===================================================================================== void startWebserver(){ - if (!LittleFS.exists("/index.html")) { - httpServer.serveStatic("/", LittleFS, "/FSexplorer.html"); - httpServer.serveStatic("/index", LittleFS, "/FSexplorer.html"); - httpServer.serveStatic("/index.html", LittleFS, "/FSexplorer.html"); + if (!SystemFS.exists("/index.html")) { + httpServer.serveStatic("/", SystemFS, "/FSexplorer.html"); + httpServer.serveStatic("/index", SystemFS, "/FSexplorer.html"); + httpServer.serveStatic("/index.html", SystemFS, "/FSexplorer.html"); } else{ - httpServer.serveStatic("/", LittleFS, "/index.html"); - httpServer.serveStatic("/index", LittleFS, "/index.html"); - httpServer.serveStatic("/index.html", LittleFS, "/index.html"); + httpServer.serveStatic("/", SystemFS, "/index.html"); + httpServer.serveStatic("/index", SystemFS, "/index.html"); + httpServer.serveStatic("/index.html", SystemFS, "/index.html"); } - httpServer.serveStatic("/FSexplorer.png", LittleFS, "/FSexplorer.png"); - httpServer.serveStatic("/index.css", LittleFS, "/index.css"); - httpServer.serveStatic("/index.js", LittleFS, "/index.js"); + httpServer.serveStatic("/FSexplorer.png", SystemFS, "/FSexplorer.png"); + httpServer.serveStatic("/index.css", SystemFS, "/index.css"); + httpServer.serveStatic("/index.js", SystemFS, "/index.js"); //otgw pic functions httpServer.on("/pic", upgradepic); // all other api calls are catched in FSexplorer onNotFounD! @@ -86,11 +86,11 @@ void startWebserver(){ } //===================================================================================== void setupFSexplorer(){ - LittleFS.begin(); - if (LittleFS.exists("/FSexplorer.html")) + SystemFS.begin(); + if (SystemFS.exists("/FSexplorer.html")) { - httpServer.serveStatic("/FSexplorer.html", LittleFS, "/FSexplorer.html"); - httpServer.serveStatic("/FSexplorer", LittleFS, "/FSexplorer.html"); + httpServer.serveStatic("/FSexplorer.html", SystemFS, "/FSexplorer.html"); + httpServer.serveStatic("/FSexplorer", SystemFS, "/FSexplorer.html"); } else { @@ -139,14 +139,14 @@ void apifirmwarefilelist() { s = buffer; s += sprintf(buffer, "["); - dir = LittleFS.openDir("/"); + dir = SystemFS.openDir("/"); while (dir.next()) { if (dir.fileName().endsWith(".hex")) { version=""; fwversion=""; String verfile = "/" + dir.fileName(); verfile.replace(".hex", ".ver"); - f = LittleFS.open(verfile, "r"); + f = SystemFS.open(verfile, "r"); if (f) { version = f.readStringUntil('\n'); version.trim(); @@ -156,7 +156,7 @@ void apifirmwarefilelist() { DebugTf("GetVersion(%s) returned %s\n", dir.fileName().c_str(), fwversion.c_str()); if (fwversion.length() && strcmp(fwversion.c_str(),version.c_str())) { // versions do not match version=fwversion; // assign hex file version to version - if (f = LittleFS.open(verfile, "w")) { // write to .ver file + if (f = SystemFS.open(verfile, "w")) { // write to .ver file DebugTf("writing %s to %s\n",version.c_str(),verfile.c_str()); f.print(version + "\n"); f.close(); @@ -187,7 +187,7 @@ void apilistfiles() // Senden aller Daten an den Client _fileMeta dirMap[MAX_FILES_IN_LIST+1]; int fileNr = 0; - Dir dir = LittleFS.openDir("/"); // List files on LittleFS + Dir dir = SystemFS.openDir("/"); // List files on SystemFS while (dir.next() && (fileNr < MAX_FILES_IN_LIST)) { dirMap[fileNr].Name[0] = '\0'; @@ -231,7 +231,7 @@ void apilistfiles() // Senden aller Daten an den Client temp += R"({"name":")" + String(dirMap[f].Name) + R"(","size":")" + formatBytes(dirMap[f].Size) + R"("},)"; } - LittleFS.info(LittleFSinfo); + SystemFS.info(LittleFSinfo); temp += R"({"usedBytes":")" + formatBytes(LittleFSinfo.usedBytes * 1.05) + R"(",)" + // Berechnet den verwendeten Speicherplatz + 5% Sicherheitsaufschlag R"("totalBytes":")" + formatBytes(LittleFSinfo.totalBytes) + R"(","freeBytes":")" + // Zeigt die Größe des Speichers (LittleFSinfo.totalBytes - (LittleFSinfo.usedBytes * 1.05)) + R"("}])"; // Berechnet den freien Speicherplatz + 5% Sicherheitsaufschlag @@ -247,13 +247,13 @@ bool handleFile(String&& path) if (httpServer.hasArg("delete")) { DebugTf("Delete -> [%s]\n\r", httpServer.arg("delete").c_str()); - LittleFS.remove(httpServer.arg("delete")); // Datei löschen + SystemFS.remove(httpServer.arg("delete")); // Datei löschen httpServer.sendContent(Header); return true; } - if (!LittleFS.exists("/FSexplorer.html")) httpServer.send(200, "text/html", Helper); //Upload the FSexplorer.html + if (!SystemFS.exists("/FSexplorer.html")) httpServer.send(200, "text/html", Helper); //Upload the FSexplorer.html if (path.endsWith("/")) path += "index.html"; - return LittleFS.exists(path) ? ({File f = LittleFS.open(path, "r"); httpServer.streamFile(f, contentType(path)); f.close(); true;}) : false; + return SystemFS.exists(path) ? ({File f = SystemFS.open(path, "r"); httpServer.streamFile(f, contentType(path)); f.close(); true;}) : false; } // handleFile() @@ -270,7 +270,7 @@ void handleFileUpload() upload.filename = upload.filename.substring(upload.filename.length() - 30, upload.filename.length()); // Dateinamen auf 30 Zeichen kürzen } Debugln("FileUpload Name: " + upload.filename); - fsUploadFile = LittleFS.open("/" + httpServer.urlDecode(upload.filename), "w"); + fsUploadFile = SystemFS.open("/" + httpServer.urlDecode(upload.filename), "w"); } else if (upload.status == UPLOAD_FILE_WRITE) { @@ -292,9 +292,9 @@ void handleFileUpload() //===================================================================================== void formatLittleFS() { //Formatiert den Speicher - if (!LittleFS.exists("/!format")) return; - DebugTln(F("Format LittleFS")); - LittleFS.format(); + if (!SystemFS.exists("/!format")) return; + DebugTln(F("Format SystemFS")); + SystemFS.format(); httpServer.sendContent(Header); } // formatLittleFS() @@ -330,8 +330,8 @@ const String &contentType(String& filename) bool freeSpace(uint16_t const& printsize) { FSInfo LittleFSinfo; - LittleFS.info(LittleFSinfo); - Debugln(formatBytes(LittleFSinfo.totalBytes - (LittleFSinfo.usedBytes * 1.05)) + " im LittleFS frei"); + SystemFS.info(LittleFSinfo); + Debugln(formatBytes(LittleFSinfo.totalBytes - (LittleFSinfo.usedBytes * 1.05)) + " im SystemFS frei"); return (LittleFSinfo.totalBytes - (LittleFSinfo.usedBytes * 1.05) > printsize) ? true : false; } // freeSpace() diff --git a/MQTTstuff.ino b/MQTTstuff.ino index 75887a40..22d6e8f9 100644 --- a/MQTTstuff.ino +++ b/MQTTstuff.ino @@ -550,14 +550,14 @@ bool doAutoConfigureMsgid(byte OTid) //Let's open the MQTT autoconfig file File fh; //filehandle const char *cfgFilename = "/mqttha.cfg"; - LittleFS.begin(); + SystemFS.begin(); - if (!LittleFS.exists(cfgFilename)) { + if (!SystemFS.exists(cfgFilename)) { DebugTln(F("Error: confuration file not found.")); return _result; } - fh = LittleFS.open(cfgFilename, "r"); + fh = SystemFS.open(cfgFilename, "r"); if (!fh) { DebugTln(F("Error: could not open confuration file.")); diff --git a/OTGW-Core.ino b/OTGW-Core.ino index 6411e71e..051004e5 100644 --- a/OTGW-Core.ino +++ b/OTGW-Core.ino @@ -2007,13 +2007,13 @@ void refreshpic(String filename, String version) { http.begin(client, "http://otgw.tclcode.com/download/" + filename); code = http.GET(); if (code == HTTP_CODE_OK) { - File f = LittleFS.open("/" + filename, "w"); + File f = SystemFS.open("/" + filename, "w"); if (f) { http.writeToStream(&f); f.close(); String verfile = "/" + filename; verfile.replace(".hex", ".ver"); - f = LittleFS.open(verfile, "w"); + f = SystemFS.open(verfile, "w"); if (f) { f.print(latest + "\n"); f.close(); @@ -2037,9 +2037,9 @@ void upgradepic() { refreshpic(filename, version); } else if (action == "delete") { String path = "/" + filename; - LittleFS.remove(path); + SystemFS.remove(path); path.replace(".hex", ".ver"); - LittleFS.remove(path); + SystemFS.remove(path); } httpServer.sendHeader("Location", "index.html#tabPICflash", true); httpServer.send(303, "text/html", "Return"); diff --git a/OTGW-firmware.h b/OTGW-firmware.h index 6c54a872..10116b24 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -25,7 +25,7 @@ #include "LittleFS.h" #define NO_GLOBAL_LITTLEFS // prevent the libary from creating a filesystem -FS LittleFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, FS_MAX_OPEN_FILES))); +FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, FS_MAX_OPEN_FILES))); //OTGW Nodoshop hardware definitions #define I2CSCL D1 diff --git a/OTGW-firmware.ino b/OTGW-firmware.ino index 8aeeb618..0ef01502 100644 --- a/OTGW-firmware.ino +++ b/OTGW-firmware.ino @@ -34,7 +34,9 @@ #define OFF HIGH DECLARE_TIMER_SEC(timerpollsensor, settingGPIOSENSORSinterval, CATCH_UP_MISSED_TICKS); - + +extern FS SystemFS; //the global fs + //===================================================================== void setup() { // Serial is initialized by OTGWSerial. It resets the pic and opens serialdevice. @@ -52,7 +54,7 @@ void setup() { setLed(LED1, ON); setLed(LED2, ON); - LittleFS.begin(); + SystemFS.begin(); readSettings(true); //start with setting wifi hostname diff --git a/OTGWSerial.cpp b/OTGWSerial.cpp index b9c918f0..8617bb9b 100644 --- a/OTGWSerial.cpp +++ b/OTGWSerial.cpp @@ -27,6 +27,8 @@ #include #include "OTGWSerial.h" +extern FS SystemFS; + #define STX ((uint8_t)0x0F) #define ETX ((uint8_t)0x04) #define DLE ((uint8_t)0x05) @@ -204,7 +206,7 @@ OTGWError OTGWSerial::readHexFile(const char *hexfile, int *total) { OTGWError rc = OTGW_ERROR_HEX_FORMAT; File f; - f = LittleFS.open(hexfile, "r"); + f = SystemFS.open(hexfile, "r"); if (!f) return OTGW_ERROR_HEX_ACCESS; memset(_upgrade_data->codemem, -1, 4096 * sizeof(short)); memset(_upgrade_data->datamem, -1, 256 * sizeof(char)); @@ -311,7 +313,7 @@ int OTGWSerial::eepromSettings(const char *version, OTGWTransferData *xfer) { int last = 0, len, id, p1, p2, addr, size, mask, n; File f; - f = LittleFS.open("/transfer.dat", "r"); + f = SystemFS.open("/transfer.dat", "r"); if (!f) return last; while (f.available()) { len = f.readBytesUntil('\n', buffer, sizeof(buffer) - 1); diff --git a/helperStuff.ino b/helperStuff.ino index 0919a7bc..53748545 100644 --- a/helperStuff.ino +++ b/helperStuff.ino @@ -162,9 +162,9 @@ uint32_t updateRebootCount() //return: number of reboots (if it goes as planned) uint32_t _reboot = 0; #define REBOOTCNT_FILE "/reboot_count.txt" - if (LittleFS.begin()) { + if (SystemFS.begin()) { //start with opening the file - File fh = LittleFS.open(REBOOTCNT_FILE, "r"); + File fh = SystemFS.open(REBOOTCNT_FILE, "r"); if (fh) { //read from file if (fh.available()){ @@ -176,7 +176,7 @@ uint32_t updateRebootCount() //increment reboot counter _reboot++; //write back the reboot counter - fh = LittleFS.open(REBOOTCNT_FILE, "w"); + fh = SystemFS.open(REBOOTCNT_FILE, "w"); if (fh) { //write to _reboot to file fh.println(_reboot); @@ -264,9 +264,9 @@ bool updateRebootLog(String text) snprintf(log_line, LOG_LINE_LENGTH, "%d-%02d-%02d %02d:%02d:%02d - reboot cause: %s (%x) %s\r\n", year(), month(), day(), hour(), minute(), second(), CSTR(text), errorCode, log_line_excpt); - if (LittleFS.begin()) { + if (SystemFS.begin()) { //start with opening the file - File outfh = LittleFS.open(TEMPLOG_FILE, "w"); + File outfh = SystemFS.open(TEMPLOG_FILE, "w"); if (outfh) { //write to _reboot to file @@ -276,7 +276,7 @@ bool updateRebootLog(String text) outfh.print(log_line_regs); } - File infh = LittleFS.open(REBOOTLOG_FILE, "r"); + File infh = SystemFS.open(REBOOTLOG_FILE, "r"); int i = 1; if (infh) { @@ -293,11 +293,11 @@ bool updateRebootLog(String text) } outfh.close(); - if (LittleFS.exists(REBOOTLOG_FILE)) { - LittleFS.remove(REBOOTLOG_FILE); + if (SystemFS.exists(REBOOTLOG_FILE)) { + SystemFS.remove(REBOOTLOG_FILE); } - LittleFS.rename(TEMPLOG_FILE, REBOOTLOG_FILE); + SystemFS.rename(TEMPLOG_FILE, REBOOTLOG_FILE); return true; // succesfully logged } @@ -351,9 +351,9 @@ bool hourChanged(){ bool checklittlefshash(){ #define GITHASH_FILE "/version.hash" String _githash=""; - if (LittleFS.begin()) { + if (SystemFS.begin()) { //start with opening the file - File fh = LittleFS.open(GITHASH_FILE, "r"); + File fh = SystemFS.open(GITHASH_FILE, "r"); if (fh) { //read from file if (fh.available()){ diff --git a/restAPI.ino b/restAPI.ino index eaea6916..2233d259 100644 --- a/restAPI.ino +++ b/restAPI.ino @@ -332,7 +332,7 @@ void sendDeviceInfo() sendNestedJsonObj("flashchipsize", formatFloat((ESP.getFlashChipSize() / 1024.0 / 1024.0), 3)); sendNestedJsonObj("flashchiprealsize", formatFloat((ESP.getFlashChipRealSize() / 1024.0 / 1024.0), 3)); - LittleFS.info(LittleFSinfo); + SystemFS.info(LittleFSinfo); sendNestedJsonObj("LittleFSsize", formatFloat( (LittleFSinfo.totalBytes / (1024.0 * 1024.0)), 0)); sendNestedJsonObj("flashchipspeed", formatFloat((ESP.getFlashChipSpeed() / 1000.0 / 1000.0), 0)); diff --git a/settingStuff.ino b/settingStuff.ino index 0474f82a..916a4eda 100644 --- a/settingStuff.ino +++ b/settingStuff.ino @@ -16,7 +16,7 @@ void writeSettings(bool show) //let's use JSON to write the setting file DebugTf("Writing to [%s] ..\r\n", SETTINGS_FILE); - File file = LittleFS.open(SETTINGS_FILE, "w"); // open for reading and writing + File file = SystemFS.open(SETTINGS_FILE, "w"); // open for reading and writing if (!file) { DebugTf("open(%s, 'w') FAILED!!! --> Bailout\r\n", SETTINGS_FILE); @@ -65,10 +65,10 @@ void readSettings(bool show) { // Open file for reading - File file = LittleFS.open(SETTINGS_FILE, "r"); + File file = SystemFS.open(SETTINGS_FILE, "r"); DebugTf(" %s ..\r\n", SETTINGS_FILE); - if (!LittleFS.exists(SETTINGS_FILE)) + if (!SystemFS.exists(SETTINGS_FILE)) { //create settings file if it does not exist yet. DebugTln(F(" .. file not found! --> created file!")); writeSettings(show); diff --git a/versionStuff.ino b/versionStuff.ino index afea43d1..db369179 100644 --- a/versionStuff.ino +++ b/versionStuff.ino @@ -14,7 +14,7 @@ String GetVersion(const String hexfile){ unsigned short ptr; File f; //DebugTf("GetVersion opening %s\n",hexfile.c_str()); - f = LittleFS.open(hexfile, "r"); + f = SystemFS.open(hexfile, "r"); if (f) // only proceed if file exists { while (f.readBytesUntil('\n', hexbuf, sizeof(hexbuf)) != 0) { From f90a5b84fef4389ae63686ce70ca7ec0b5f25076 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 16:42:15 +0100 Subject: [PATCH 03/19] introduce second filesystem and logic to detect proper usage --- OTGW-firmware.h | 18 ++++- OTGW-firmware.ino | 168 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 180 insertions(+), 6 deletions(-) diff --git a/OTGW-firmware.h b/OTGW-firmware.h index 10116b24..d0f29eff 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -24,8 +24,22 @@ #include "OTGW-Core.h" // Core code for this firmware #include "LittleFS.h" -#define NO_GLOBAL_LITTLEFS // prevent the libary from creating a filesystem -FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, FS_MAX_OPEN_FILES))); +#define NO_GLOBAL_LITTLEFS // prevent the LittleFS libary from creating a filesystem +#define sFS_PHYS_ADDR 0x200000 // FS_PHYS_ADDR //use the external value generated by Arduino IDE. Consider hard coding at 0x200000 +#define sFS_PHYS_PAGE 512// FS_PHYS_PAGE //use the external value generated by Arduino IDE. Consider hard coding at 512 +#define sFS_PHYS_BLOCK 4096 // ESP8266 flash uses 4k blocks. Anything other than 4k multiples gives an exception. Arduino IDE uses 8k +#define sFS_PHYS_SIZE 1024 * 1024 //FS_PHYS_SIZE - OTGW_USER_FS_SIZE +#define sFS_MAX_OPEN_FILES 5 //FS_MAX_OPEN_FILES //hard coded since there is no external value generated by Arduino IDE + +#define uFS_PHYS_ADDR sFS_PHYS_ADDR + sFS_PHYS_SIZE + 4096 +#define uFS_PHYS_SIZE 512 * 1024 +#define uFS_PHYS_PAGE sFS_PHYS_PAGE +#define uFS_PHYS_BLOCK sFS_PHYS_BLOCK +#define uFS_MAX_OPEN_FILES sFS_MAX_OPEN_FILES + +extern FS SystemFS; // these FS are defined in OTGW-firmware.ino +extern FS UserFS; +bool bUserFSpresent = false; //OTGW Nodoshop hardware definitions #define I2CSCL D1 diff --git a/OTGW-firmware.ino b/OTGW-firmware.ino index 0ef01502..36b35dd2 100644 --- a/OTGW-firmware.ino +++ b/OTGW-firmware.ino @@ -35,7 +35,9 @@ DECLARE_TIMER_SEC(timerpollsensor, settingGPIOSENSORSinterval, CATCH_UP_MISSED_TICKS); -extern FS SystemFS; //the global fs +//FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES)));; //the global fs +FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); //the global fs +FS UserFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(uFS_PHYS_ADDR, uFS_PHYS_SIZE, uFS_PHYS_PAGE, uFS_PHYS_BLOCK, uFS_MAX_OPEN_FILES))); //the global fs //===================================================================== void setup() { @@ -54,9 +56,6 @@ void setup() { setLed(LED1, ON); setLed(LED2, ON); - SystemFS.begin(); - readSettings(true); - //start with setting wifi hostname WiFi.hostname(String(settingHostname)); @@ -68,6 +67,15 @@ void setup() { setLed(LED1, OFF); startTelnet(); // start the debug port 23 + + bool fsready = setupFilesystems(); + if (fsready) { + SystemFS.begin(); + readSettings(true); + } else { + blinkLED(LED1, 6, 500); + } + startNTP(); startMDNS(CSTR(settingHostname)); startLLMNR(CSTR(settingHostname)); @@ -104,6 +112,158 @@ void setup() { //===================================================================== +void printFSinfo(Stream &out, FS &aFS) { + + FSInfo fs_info; + aFS.info(fs_info); + + out.print("totalBytes: "); out.println(fs_info.totalBytes); + out.print("usedBytes: "); out.println(fs_info.usedBytes); + out.print("blockSize: "); out.println(fs_info.blockSize); + out.print("pageSize: "); out.println(fs_info.pageSize); + out.print("maxOpenFiles: "); out.println(fs_info.maxOpenFiles); + out.print("maxPathLength: "); out.println(fs_info.maxPathLength); +} + +void printFScontents(Stream &out, FS &aFS) { + if (aFS.begin()) { + out.println("Filesystem contents:"); + Dir dir = aFS.openDir("/"); + while (dir.next()) { + out.println(dir.fileName().c_str()); + } + SystemFS.end(); + } else { + out.println("Error: cannot print filesystem contents"); + } +} + +bool testFileSystem(FS &aFS, char *testFilename) { + + bool mounted = aFS.begin(); + if (!mounted) { return false; } + + printFSinfo(OTGWSerial, aFS); + + OTGWSerial.println("Succesfully mounted the filesystem, now testing file access"); + bool _success = aFS.exists(testFilename); + if (!_success) { + OTGWSerial.println("Test file does not exist on the filesystem"); + return false; + } + + OTGWSerial.println("Test file is reported to exist by filesystem"); + File fh = aFS.open(testFilename, "r"); + + if(!fh) { + OTGWSerial.println("Cannot open test file"); + return false; + } + + String _line = fh.readStringUntil('\n'); + if(_line.isEmpty()) { + OTGWSerial.printf("%s file cannot be read. Unmounting\n", testFilename); + _success = false; + } else { + OTGWSerial.printf("Check file %s contains: %s; and: \n %s\n", testFilename, _line.c_str(), fh.readStringUntil('\n').c_str()); + } + + fh.close(); + aFS.end(); + + return _success; +} + +//===================================================================== +bool setupFilesystems() { + bool s_mounted, u_mounted = false; + + // try the smaller implementation first + // SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); // moved to header + +// SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); +// SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); +// LittleFSinfo myFSinfo; +// if (SystemFS.info(myFSinfo)) { +// OTGWSerial.printf("LittleFS params initial filesystem: FS_PHYS_ADDR %x, FS_PHYS_SIZE %x, FS_PHYS_PAGE %x, FS_PHYS_BLOCK %x, sFS_MAX_OPEN_FILES %x\n", myFSinfo. FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES); +// } else { +// OTGWSerial.println("Failed to get info for the intially configured filesystem"); +// } + +// OTGWSerial.printf("LittleFS params: FS_PHYS_ADDR %x, FS_PHYS_SIZE %x, FS_PHYS_PAGE %x, FS_PHYS_BLOCK %x, sFS_MAX_OPEN_FILES %x\n", FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES); + + #define CHECK_FILE "/index.js" // a file over 8k in size, containing clear text, to be used to test for successful mounting of the filesystem + + LittleFSConfig cfg; + cfg.setAutoFormat(false); // prevent the fs from being formatted if it is actually the larger size + SystemFS.setConfig(cfg); + OTGWSerial.println("Attempting to mount the system partition"); + + s_mounted = testFileSystem(SystemFS, CHECK_FILE); + + if(s_mounted) { + OTGWSerial.println("Attempting to mount the user partition"); + u_mounted = UserFS.begin(); + if (UserFS.exists("/")) { //since we are creating the user partition, there is no reason to test further than this + OTGWSerial.println("User partition is accessable"); + bUserFSpresent = true; + } else { + OTGWSerial.printf("Formatting the user partition: %s\n", UserFS.format() ? "success" : "failed"); + } + } + + if (!s_mounted) { // smaller fs not mounted succesfully, try the original size with the generated parameters FS_* (i.e. no u or s prefix) + OTGWSerial.println("Failed to mount the 1M filesystem, fall back to the original 2M sized one"); + SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); + SystemFS.setConfig(cfg); + + s_mounted = testFileSystem(SystemFS, CHECK_FILE); + + if (!s_mounted) { + OTGWSerial.println("Failed to mount the original 2M filesystem, please reflash the firmware over usb"); + return false; + } + + OTGWSerial.println("Succesfully mounted the original 2M filesystem"); + OTGWSerial.println("Redirecting user partition to the system partion"); + UserFS = SystemFS; + bUserFSpresent = false; //technically not necessary since default state is false; + u_mounted = true; + } + + // From boards.txt + // nodemcuv2.menu.eesz.4M2M=4MB (FS:2MB OTA:~1019KB) + // nodemcuv2.menu.eesz.4M2M.build.flash_size=4M + // nodemcuv2.menu.eesz.4M2M.build.flash_size_bytes=0x400000 + // nodemcuv2.menu.eesz.4M2M.build.flash_ld=eagle.flash.4m2m.ld + // nodemcuv2.menu.eesz.4M2M.build.spiffs_pagesize=256 + // nodemcuv2.menu.eesz.4M2M.upload.maximum_size=1044464 + // nodemcuv2.menu.eesz.4M2M.build.rfcal_addr=0x3FC000 + // nodemcuv2.menu.eesz.4M2M.build.spiffs_start=0x200000 + // nodemcuv2.menu.eesz.4M2M.build.spiffs_end=0x3FA000 + // nodemcuv2.menu.eesz.4M2M.build.spiffs_blocksize=8192 + + + // FS_PHYS_ADDR, + // FS_PHYS_SIZE, + // FS_PHYS_PAGE, + // FS_PHYS_BLOCK, + // FS_MAX_OPEN_FILES + + + + // FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); + // FS UserFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(uFS_PHYS_ADDR, uFS_PHYS_SIZE, uFS_PHYS_PAGE, uFS_PHYS_BLOCK, uFS_MAX_OPEN_FILES))); + // + // FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK are declared by arduino IDE. Default arduino IDE parameters (for flash = 4MB) has the following mapping: + // 0..1MB: sketch + // 1..2MB: empty, usable for a temporary FS, used for OTA + // 2..4MB: default area for FS + // 3.99..4MB: eeprom + firmware area + // Robbert: get from boards.txt + return s_mounted && u_mounted; +} + //====[ restartWifi ]=== /* The restartWifi function takes tries to just reconnect to the wifi. When the wifi is restated, it then waits for maximum of 30 seconds (timeout). From b7a131f060bf7522e0e413ca5856ce20c8e67cc5 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 16:43:07 +0100 Subject: [PATCH 04/19] include build scripts for the littlefs filesystem --- build_littlefs.bat | 14 ++++++++++++++ build_littlefs_2M.bat | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 build_littlefs.bat create mode 100644 build_littlefs_2M.bat diff --git a/build_littlefs.bat b/build_littlefs.bat new file mode 100644 index 00000000..4a14b1b3 --- /dev/null +++ b/build_littlefs.bat @@ -0,0 +1,14 @@ +@echo off +set "mklittlefs_exec=%homedrive%%homepath%\AppData\Local\Arduino15\packages\esp8266\tools\mklittlefs\2.5.0-4-fe5bb56\mklittlefs.exe" +set /A block_size = 4096 +set /A page_size = 512 +set /A fs_size = 1024 * 1024 +set "source_dir=data" +set "img_name=OTGW-Firmware.littlefs.1M.bin" +set "build_command=%mklittlefs_exec% -c %source_dir% -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %build_command% +pause +set "check_command=%mklittlefs_exec% -l -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %check_command% + + diff --git a/build_littlefs_2M.bat b/build_littlefs_2M.bat new file mode 100644 index 00000000..b842f97e --- /dev/null +++ b/build_littlefs_2M.bat @@ -0,0 +1,14 @@ +@echo off +set "mklittlefs_exec=%homedrive%%homepath%\AppData\Local\Arduino15\packages\esp8266\tools\mklittlefs\2.5.0-4-fe5bb56\mklittlefs.exe" +set /A block_size = 8192 +set /A page_size = 512 +set /A fs_size = (2 * 1024 * 1024) - (24 * 1024) +set "source_dir=data" +set "img_name=OTGW-Firmware.littlefs.2M.bin" +set "build_command=%mklittlefs_exec% -c %source_dir% -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %build_command% +pause +set "check_command=%mklittlefs_exec% -l -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %check_command% + + From 69e5d0b67d5aa336936188e28cf2198a06c5f0f1 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 16:43:58 +0100 Subject: [PATCH 05/19] introduce debug command for file system parameters --- handleDebug.ino | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/handleDebug.ino b/handleDebug.ino index 393ec76b..981db003 100644 --- a/handleDebug.ino +++ b/handleDebug.ino @@ -4,6 +4,19 @@ void handleDebug(){ char c; c = TelnetStream.read(); switch (c){ + case 'f': + DebugTln(F("Filesystem information")); + DebugTln(F("System partition:")); + Debugf("physical address: %x\r\n", sFS_PHYS_ADDR); + + printFSinfo(TelnetStream, SystemFS); + Debugf("User partition internal setting: %s\n", bUserFSpresent ? "TRUE" : "FALSE"); //F() macro not working? + if(bUserFSpresent) { + DebugTln(F("User partition:")); + Debugf("physical address: %x\r\n", uFS_PHYS_ADDR); + printFSinfo(TelnetStream, UserFS); + } + break; case 'h': Debugln(); Debugln(F("---===[ Debug Help Menu ]===---")); @@ -78,7 +91,7 @@ void handleDebug(){ DebugTln("MyDEBUG =true"); settingMyDEBUG = true; break; - case 'f': + case 'c': if(settingMyDEBUG) { DebugTln(F("MyDEBUG = true")); From 01ee6a9b5e6e9bd906b63adefb7efcef2c631871 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 16:53:21 +0100 Subject: [PATCH 06/19] change settings logic to use userFS --- settingStuff.ino | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/settingStuff.ino b/settingStuff.ino index 916a4eda..7ea1281a 100644 --- a/settingStuff.ino +++ b/settingStuff.ino @@ -16,7 +16,7 @@ void writeSettings(bool show) //let's use JSON to write the setting file DebugTf("Writing to [%s] ..\r\n", SETTINGS_FILE); - File file = SystemFS.open(SETTINGS_FILE, "w"); // open for reading and writing + File file = UserFS.open(SETTINGS_FILE, "w"); // open for reading and writing if (!file) { DebugTf("open(%s, 'w') FAILED!!! --> Bailout\r\n", SETTINGS_FILE); @@ -65,10 +65,11 @@ void readSettings(bool show) { // Open file for reading - File file = SystemFS.open(SETTINGS_FILE, "r"); + File file = UserFS.open(SETTINGS_FILE, "r"); + + if (!UserFS.exists(SETTINGS_FILE)) DebugTf(" %s ..\r\n", SETTINGS_FILE); - if (!SystemFS.exists(SETTINGS_FILE)) { //create settings file if it does not exist yet. DebugTln(F(" .. file not found! --> created file!")); writeSettings(show); From bf7fe4b6e9007233e8f01c754334639ae95e7f94 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 17:41:29 +0100 Subject: [PATCH 07/19] move logs to use userFS" --- helperStuff.ino | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helperStuff.ino b/helperStuff.ino index 53748545..8ad20e2a 100644 --- a/helperStuff.ino +++ b/helperStuff.ino @@ -162,9 +162,9 @@ uint32_t updateRebootCount() //return: number of reboots (if it goes as planned) uint32_t _reboot = 0; #define REBOOTCNT_FILE "/reboot_count.txt" - if (SystemFS.begin()) { + if (UserFS.begin()) { //start with opening the file - File fh = SystemFS.open(REBOOTCNT_FILE, "r"); + File fh = UserFS.open(REBOOTCNT_FILE, "r"); if (fh) { //read from file if (fh.available()){ @@ -176,7 +176,7 @@ uint32_t updateRebootCount() //increment reboot counter _reboot++; //write back the reboot counter - fh = SystemFS.open(REBOOTCNT_FILE, "w"); + fh = UserFS.open(REBOOTCNT_FILE, "w"); if (fh) { //write to _reboot to file fh.println(_reboot); @@ -264,9 +264,9 @@ bool updateRebootLog(String text) snprintf(log_line, LOG_LINE_LENGTH, "%d-%02d-%02d %02d:%02d:%02d - reboot cause: %s (%x) %s\r\n", year(), month(), day(), hour(), minute(), second(), CSTR(text), errorCode, log_line_excpt); - if (SystemFS.begin()) { + if (UserFS.begin()) { //start with opening the file - File outfh = SystemFS.open(TEMPLOG_FILE, "w"); + File outfh = UserFS.open(TEMPLOG_FILE, "w"); if (outfh) { //write to _reboot to file @@ -276,7 +276,7 @@ bool updateRebootLog(String text) outfh.print(log_line_regs); } - File infh = SystemFS.open(REBOOTLOG_FILE, "r"); + File infh = UserFS.open(REBOOTLOG_FILE, "r"); int i = 1; if (infh) { @@ -293,11 +293,11 @@ bool updateRebootLog(String text) } outfh.close(); - if (SystemFS.exists(REBOOTLOG_FILE)) { - SystemFS.remove(REBOOTLOG_FILE); + if (UserFS.exists(REBOOTLOG_FILE)) { + UserFS.remove(REBOOTLOG_FILE); } - SystemFS.rename(TEMPLOG_FILE, REBOOTLOG_FILE); + UserFS.rename(TEMPLOG_FILE, REBOOTLOG_FILE); return true; // succesfully logged } From bdeecd989b8f199b46e032e74d8366493ededc72 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 17:42:21 +0100 Subject: [PATCH 08/19] move mqtt config to userFS --- MQTTstuff.ino | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/MQTTstuff.ino b/MQTTstuff.ino index 22d6e8f9..17770575 100644 --- a/MQTTstuff.ino +++ b/MQTTstuff.ino @@ -526,6 +526,36 @@ void doAutoConfigure(bool bForcaAll = false){ } // bool success = doAutoConfigure("config"); // the string "config" should match every line non-comment in mqttha.cfg } +//=========================================================================================== + +bool copyMQTTcfg() { + + DebugTln(F("Error: mqttha.cfg not present in user filesystem. Copying!")); + + String sLine = ""; + + const char *cfgFilename = "/mqttha.cfg"; + SystemFS.begin(); + UserFS.begin(); + + File fin = SystemFS.open(cfgFilename, "r"); + File fout = UserFS.open(cfgFilename, "w"); + + if ((!fin) || (!fout)) { + DebugTln(F("Error: couldn't open source or destination for copying!")); + return false; + } + + while (fin.available()) { + sLine = fin.readStringUntil('\n'); + fout.println(sLine.c_str()); + } + + fin.close(); + fout.close(); + return true; +} + //=========================================================================================== bool doAutoConfigureMsgid(byte OTid) { @@ -550,11 +580,11 @@ bool doAutoConfigureMsgid(byte OTid) //Let's open the MQTT autoconfig file File fh; //filehandle const char *cfgFilename = "/mqttha.cfg"; - SystemFS.begin(); + UserFS.begin(); - if (!SystemFS.exists(cfgFilename)) { + if (!UserFS.exists(cfgFilename)) { DebugTln(F("Error: confuration file not found.")); - return _result; + if(!copyMQTTcfg()) { return _result; } } fh = SystemFS.open(cfgFilename, "r"); From fd1101150e039147f647b8bbd90e08c3b0ee31a3 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Wed, 9 Feb 2022 17:53:42 +0100 Subject: [PATCH 09/19] update FSexplorer for use with the second filesystem --- FSexplorer.ino | 71 +++++++++-- data/FSexplorer.html | 284 ++++++++++++++++++++++++------------------- 2 files changed, 220 insertions(+), 135 deletions(-) diff --git a/FSexplorer.ino b/FSexplorer.ino index e9a580d1..b79c8562 100644 --- a/FSexplorer.ino +++ b/FSexplorer.ino @@ -98,8 +98,10 @@ void setupFSexplorer(){ } httpServer.on("/api/firmwarefilelist", apifirmwarefilelist); httpServer.on("/api/listfiles", apilistfiles); + httpServer.on("/api/listuserfiles", apilistuserfiles); httpServer.on("/LittleFSformat", formatLittleFS); - httpServer.on("/upload", HTTP_POST, []() {}, handleFileUpload); + httpServer.on("/upload", HTTP_POST, []() {}, handleSysFileUpload); + httpServer.on("/uploadusr", HTTP_POST, []() {}, handleUserFileUpload); httpServer.on("/ReBoot", reBootESP); httpServer.on("/ResetWireless", resetWirelessButton); @@ -120,6 +122,7 @@ void setupFSexplorer(){ { if (bDebugRestAPI) DebugTf("next: handleFile(%s)\r\n" , String(httpServer.urlDecode(httpServer.uri())).c_str()); + if (!handleFile(httpServer.urlDecode(httpServer.uri()))) { httpServer.send(404, "text/plain", "FileNotFound\r\n"); @@ -173,8 +176,20 @@ void apifirmwarefilelist() { //===================================================================================== +void apilistfiles() { + apilistFSfiles(SystemFS); +} + +void apilistuserfiles() { + if(bUserFSpresent) { + apilistFSfiles(UserFS); + } else { + httpServer.send(200, "application/json", ""); +// httpServer.send(404, "text/plain", ""); + } +} -void apilistfiles() // Senden aller Daten an den Client +void apilistFSfiles(FS thisFS) // Senden aller Daten an den Client { FSInfo LittleFSinfo; @@ -187,7 +202,7 @@ void apilistfiles() // Senden aller Daten an den Client _fileMeta dirMap[MAX_FILES_IN_LIST+1]; int fileNr = 0; - Dir dir = SystemFS.openDir("/"); // List files on SystemFS + Dir dir = thisFS.openDir("/"); // List files on SystemFS while (dir.next() && (fileNr < MAX_FILES_IN_LIST)) { dirMap[fileNr].Name[0] = '\0'; @@ -231,7 +246,7 @@ void apilistfiles() // Senden aller Daten an den Client temp += R"({"name":")" + String(dirMap[f].Name) + R"(","size":")" + formatBytes(dirMap[f].Size) + R"("},)"; } - SystemFS.info(LittleFSinfo); + thisFS.info(LittleFSinfo); temp += R"({"usedBytes":")" + formatBytes(LittleFSinfo.usedBytes * 1.05) + R"(",)" + // Berechnet den verwendeten Speicherplatz + 5% Sicherheitsaufschlag R"("totalBytes":")" + formatBytes(LittleFSinfo.totalBytes) + R"(","freeBytes":")" + // Zeigt die Größe des Speichers (LittleFSinfo.totalBytes - (LittleFSinfo.usedBytes * 1.05)) + R"("}])"; // Berechnet den freien Speicherplatz + 5% Sicherheitsaufschlag @@ -243,23 +258,57 @@ void apilistfiles() // Senden aller Daten an den Client //===================================================================================== bool handleFile(String&& path) +{ + bool success = false; + + if (!SystemFS.exists("/FSexplorer.html")) httpServer.send(200, "text/html", Helper); //Upload the FSexplorer.html + if (path.endsWith("/")) path += "index.html"; + + success = httpServer.hasArg("user") ? handleFSfile(UserFS, path) : handleFSfile(SystemFS, path); + + return success; + +} // handleFile() + +bool handleFSfile(FS sourceFS, String& path) { if (httpServer.hasArg("delete")) { DebugTf("Delete -> [%s]\n\r", httpServer.arg("delete").c_str()); - SystemFS.remove(httpServer.arg("delete")); // Datei löschen + sourceFS.remove(httpServer.arg("delete")); // Datei löschen httpServer.sendContent(Header); return true; } - if (!SystemFS.exists("/FSexplorer.html")) httpServer.send(200, "text/html", Helper); //Upload the FSexplorer.html - if (path.endsWith("/")) path += "index.html"; - return SystemFS.exists(path) ? ({File f = SystemFS.open(path, "r"); httpServer.streamFile(f, contentType(path)); f.close(); true;}) : false; + + if(!sourceFS.exists(path)) { return false;} -} // handleFile() + DebugTf("Serving file [%s] from -> %s partition \n\r", path.c_str(), httpServer.hasArg("user") ? "user" : "system"); + + File f = sourceFS.open(path, "r"); + httpServer.streamFile(f, contentType(path)); + f.close(); + + return true; + +} // handleFSfile() + + +//===================================================================================== +void handleSysFileUpload() +{ + handleFSFileUpload(SystemFS); +} + +//===================================================================================== +void handleUserFileUpload() +{ + handleFSFileUpload(UserFS); +} //===================================================================================== -void handleFileUpload() + +void handleFSFileUpload(FS destinationFS) { static File fsUploadFile; HTTPUpload& upload = httpServer.upload(); @@ -270,7 +319,7 @@ void handleFileUpload() upload.filename = upload.filename.substring(upload.filename.length() - 30, upload.filename.length()); // Dateinamen auf 30 Zeichen kürzen } Debugln("FileUpload Name: " + upload.filename); - fsUploadFile = SystemFS.open("/" + httpServer.urlDecode(upload.filename), "w"); + fsUploadFile = destinationFS.open("/" + httpServer.urlDecode(upload.filename), "w"); } else if (upload.status == UPLOAD_FILE_WRITE) { diff --git a/data/FSexplorer.html b/data/FSexplorer.html index eb25f07c..fe0786e5 100644 --- a/data/FSexplorer.html +++ b/data/FSexplorer.html @@ -10,128 +10,164 @@ --> - - - - - - FSexplorer ESP - - - -

FSexplorer ESP

-
-
-
-
- -
- - -
-
- -
- -
-
- -
- -
-
- -
- -
-
- -
-
- Bailout\r\n", SETTINGS_FILE); - return; + DebugTf("open(%s, 'w') for writing FAILED!!! --> Bailout\r\n", SETTINGS_FILE); + return succeeded; } yield(); - DebugT(F("Start writing setting data ")); - //const size_t capacity = JSON_OBJECT_SIZE(6); // save more setting, grow # of objects accordingly DynamicJsonDocument doc(1024); JsonObject root = doc.to(); @@ -52,11 +51,20 @@ void writeSettings(bool show) root["GPIOOUTPUTSpin"] = settingGPIOOUTPUTSpin; root["GPIOOUTPUTStriggerBit"] = settingGPIOOUTPUTStriggerBit; - serializeJsonPretty(root, file); - Debugln(F("... done!")); - if (show) serializeJsonPretty(root, TelnetStream); //Debug stream ;-) + DebugT(F("Start writing setting data ")); + + if(serializeJsonPretty(root, file) > 0) { //more than 0 bytes written + Debugln(F("... done!")); + succeeded = true; + } else { + Debugln(F("... failed!")); + } + file.close(); + if (succeeded && show) serializeJsonPretty(root, TelnetStream); //Debug stream ;-) + + return succeeded; } // writeSettings() From 6303b6b38cb24d938ee70524267c00d35c1b6420 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Fri, 11 Feb 2022 16:36:58 +0100 Subject: [PATCH 15/19] change some logic in readsettings to prevent recusion the readsettings function would recurse indefinately if writesettings would not generate a file. However, default settings are included in the main firmware header so there is no real need to write them if settings.ini does not exist. We can just write the settings when they change. --- settingStuff.ino | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/settingStuff.ino b/settingStuff.ino index f4abed32..89503bef 100644 --- a/settingStuff.ino +++ b/settingStuff.ino @@ -74,14 +74,12 @@ void readSettings(bool show) // Open file for reading + DebugTf("Open file for reading %s ..\r\n", SETTINGS_FILE); File file = UserFS.open(SETTINGS_FILE, "r"); - if (!UserFS.exists(SETTINGS_FILE)) - DebugTf(" %s ..\r\n", SETTINGS_FILE); - { //create settings file if it does not exist yet. - DebugTln(F(" .. file not found! --> created file!")); - writeSettings(show); - readSettings(false); //now it should work... + if (!file) + { // settings file if it does not exist yet. + DebugTf("%s file not found! --> file will be created next time you save your settings!\r\n", SETTINGS_FILE); return; } @@ -112,7 +110,7 @@ void readSettings(bool show) if (settingMQTThaprefix=="null") settingMQTThaprefix = HOME_ASSISTANT_DISCOVERY_PREFIX; settingMQTTharebootdetection = doc["MQTTharebootdetection"]|settingMQTTharebootdetection; settingMQTTuniqueid = doc["MQTTuniqueid"].as(); - if (settingMQTTuniqueid=="null") settingMQTTuniqueid = getUniqueId(); + if ((settingMQTTuniqueid=="null") || (settingMQTTuniqueid=="")) settingMQTTuniqueid = getUniqueId(); settingMQTTOTmessage = doc["MQTTOTmessage"]|settingMQTTOTmessage; settingNTPenable = doc["NTPenable"]; From 5343b310428e7c7dd4e37ea3e9c189580e958c57 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Fri, 11 Feb 2022 16:46:39 +0100 Subject: [PATCH 16/19] reshuffle global variables to allow initializing uniqueID from the start the fucntion getUniqueID() depends on networkstuff.h so the definition of MQTT variables is moved below this include, to allow settingMQTTuniqueid to be set from the start --- OTGW-firmware.h | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/OTGW-firmware.h b/OTGW-firmware.h index b07efb96..b6c99fa1 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -114,24 +114,6 @@ bool bPrintSummarymode = false; //default to PS=0 mode //All things that are settings String settingHostname = _HOSTNAME; -//MQTT settings -bool statusMQTTconnection = false; -bool settingMQTTenable = true; -bool settingMQTTsecure = false; -String settingMQTTbroker= "192.168.88.254"; -int16_t settingMQTTbrokerPort = 1883; -String settingMQTTuser = ""; -String settingMQTTpasswd = ""; -String settingMQTThaprefix = HOME_ASSISTANT_DISCOVERY_PREFIX; -bool settingMQTTharebootdetection = true; -String settingMQTTtopTopic = "otgw"; -String settingMQTTuniqueid = ""; // Intialized in readsettings -bool settingMQTTOTmessage = false; -bool settingNTPenable = true; -String settingNTPtimezone = NTP_DEFAULT_TIMEZONE; -String settingNTPhostname = NTP_HOST_DEFAULT; -bool settingLEDblink = true; - // GPIO Sensor Settings bool settingGPIOSENSORSenabled = false; int8_t settingGPIOSENSORSpin = 10; @@ -152,10 +134,29 @@ bool settingGPIOOUTPUTSenabled = false; int8_t settingGPIOOUTPUTSpin = 16; int8_t settingGPIOOUTPUTStriggerBit = 0; +bool settingNTPenable = true; +String settingNTPtimezone = NTP_DEFAULT_TIMEZONE; +String settingNTPhostname = NTP_HOST_DEFAULT; +bool settingLEDblink = true; + //Now load network suff #include "networkStuff.h" - // That's all folks... +//MQTT settings +bool statusMQTTconnection = false; +bool settingMQTTenable = true; +bool settingMQTTsecure = false; +String settingMQTTbroker= "192.168.88.254"; +int16_t settingMQTTbrokerPort = 1883; +String settingMQTTuser = ""; +String settingMQTTpasswd = ""; +String settingMQTThaprefix = HOME_ASSISTANT_DISCOVERY_PREFIX; +bool settingMQTTharebootdetection = true; +String settingMQTTtopTopic = "otgw"; +String settingMQTTuniqueid = getUniqueId(); // Intialization moved here from readsettings +bool settingMQTTOTmessage = false; + +// That's all folks... /*************************************************************************** * From 00038678788838fbfc0d42b1073ddb718e643a76 Mon Sep 17 00:00:00 2001 From: Robbert Lagreweij Date: Fri, 11 Feb 2022 17:17:28 +0100 Subject: [PATCH 17/19] clean up comments --- OTGW-firmware.h | 8 ++++---- OTGW-firmware.ino | 44 -------------------------------------------- settingStuff.ino | 2 +- 3 files changed, 5 insertions(+), 49 deletions(-) diff --git a/OTGW-firmware.h b/OTGW-firmware.h index b6c99fa1..7d4632a6 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -25,13 +25,13 @@ #include "LittleFS.h" #define NO_GLOBAL_LITTLEFS // prevent the LittleFS libary from creating a filesystem -#define sFS_PHYS_ADDR 0x200000 // FS_PHYS_ADDR //use the external value generated by Arduino IDE. Consider hard coding at 0x200000 -#define sFS_PHYS_PAGE 256// FS_PHYS_PAGE //use the external value generated by Arduino IDE. Consider hard coding at 512 +#define sFS_PHYS_ADDR 0x200000 // FS_PHYS_ADDR //use the external value generated by Arduino IDE in boards.txt. Consider hard coding at 0x200000 +#define sFS_PHYS_PAGE 256// FS_PHYS_PAGE //use the external value generated by Arduino IDE in boards.txt. Consider hard coding at 512 #define sFS_PHYS_BLOCK 4096 // ESP8266 flash uses 4k blocks. Anything other than 4k multiples gives an exception. Arduino IDE uses 8k #define sFS_PHYS_SIZE 1024 * 1024 //FS_PHYS_SIZE - OTGW_USER_FS_SIZE -#define sFS_MAX_OPEN_FILES 5 //FS_MAX_OPEN_FILES //hard coded since there is no external value generated by Arduino IDE +#define sFS_MAX_OPEN_FILES 5 //FS_MAX_OPEN_FILES //hard coded since there is no external value generated by Arduino IDE in boards.txt -#define uFS_PHYS_ADDR 0x37A000 // 0x3FA000 - uFS_PHYS_SIZE //0x3FA000 is provided by the ESP8266 arduino package in nodemcuv2.menu.eesz.4M2M.build.spiffs_end +#define uFS_PHYS_ADDR 0x37A000 // 0x3FA000 - uFS_PHYS_SIZE //0x3FA000 is provided by the ESP8266 arduino package in boards.txt -> nodemcuv2.menu.eesz.4M2M.build.spiffs_end #define uFS_PHYS_SIZE 512 * 1024 #define uFS_PHYS_PAGE sFS_PHYS_PAGE #define uFS_PHYS_BLOCK sFS_PHYS_BLOCK diff --git a/OTGW-firmware.ino b/OTGW-firmware.ino index 2916950e..0c92255a 100644 --- a/OTGW-firmware.ino +++ b/OTGW-firmware.ino @@ -178,20 +178,6 @@ bool testFileSystem(FS &aFS, char *testFilename) { bool setupFilesystems() { bool s_mounted, u_mounted = false; - // try the smaller implementation first - // SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); // moved to header - -// SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); -// SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); -// LittleFSinfo myFSinfo; -// if (SystemFS.info(myFSinfo)) { -// OTGWSerial.printf("LittleFS params initial filesystem: FS_PHYS_ADDR %x, FS_PHYS_SIZE %x, FS_PHYS_PAGE %x, FS_PHYS_BLOCK %x, sFS_MAX_OPEN_FILES %x\n", myFSinfo. FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES); -// } else { -// OTGWSerial.println("Failed to get info for the intially configured filesystem"); -// } - -// OTGWSerial.printf("LittleFS params: FS_PHYS_ADDR %x, FS_PHYS_SIZE %x, FS_PHYS_PAGE %x, FS_PHYS_BLOCK %x, sFS_MAX_OPEN_FILES %x\n", FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK, sFS_MAX_OPEN_FILES); - #define CHECK_FILE "/index.js" // a file over 8k in size, containing clear text, to be used to test for successful mounting of the filesystem LittleFSConfig cfg; @@ -232,36 +218,6 @@ bool setupFilesystems() { u_mounted = true; } - // From boards.txt - // nodemcuv2.menu.eesz.4M2M=4MB (FS:2MB OTA:~1019KB) - // nodemcuv2.menu.eesz.4M2M.build.flash_size=4M - // nodemcuv2.menu.eesz.4M2M.build.flash_size_bytes=0x400000 - // nodemcuv2.menu.eesz.4M2M.build.flash_ld=eagle.flash.4m2m.ld - // nodemcuv2.menu.eesz.4M2M.build.spiffs_pagesize=256 - // nodemcuv2.menu.eesz.4M2M.upload.maximum_size=1044464 - // nodemcuv2.menu.eesz.4M2M.build.rfcal_addr=0x3FC000 - // nodemcuv2.menu.eesz.4M2M.build.spiffs_start=0x200000 - // nodemcuv2.menu.eesz.4M2M.build.spiffs_end=0x3FA000 - // nodemcuv2.menu.eesz.4M2M.build.spiffs_blocksize=8192 - - - // FS_PHYS_ADDR, - // FS_PHYS_SIZE, - // FS_PHYS_PAGE, - // FS_PHYS_BLOCK, - // FS_MAX_OPEN_FILES - - - - // FS SystemFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(sFS_PHYS_ADDR, sFS_PHYS_SIZE, sFS_PHYS_PAGE, sFS_PHYS_BLOCK, sFS_MAX_OPEN_FILES))); - // FS UserFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(uFS_PHYS_ADDR, uFS_PHYS_SIZE, uFS_PHYS_PAGE, uFS_PHYS_BLOCK, uFS_MAX_OPEN_FILES))); - // - // FS_PHYS_ADDR, FS_PHYS_SIZE, FS_PHYS_PAGE, FS_PHYS_BLOCK are declared by arduino IDE. Default arduino IDE parameters (for flash = 4MB) has the following mapping: - // 0..1MB: sketch - // 1..2MB: empty, usable for a temporary FS, used for OTA - // 2..4MB: default area for FS - // 3.99..4MB: eeprom + firmware area - // Robbert: get from boards.txt return s_mounted && u_mounted; } diff --git a/settingStuff.ino b/settingStuff.ino index 89503bef..02aa4a5e 100644 --- a/settingStuff.ino +++ b/settingStuff.ino @@ -167,7 +167,7 @@ void readSettings(bool show) //======================================================================= -void updateSetting(const char *field, const char *newValue) +void updateSetting(const char *field, const char *newValue) //used by RestAPI { //do not just trust the caller to do the right thing, server side validation is here! DebugTf("-> field[%s], newValue[%s]\r\n", field, newValue); From 46782991e5b6edae0c24eddd4f370abf805314d6 Mon Sep 17 00:00:00 2001 From: Robert van den Breemen Date: Wed, 16 Feb 2022 20:13:12 +0100 Subject: [PATCH 18/19] adding 1.5MB version of SystemFS --- OTGW-firmware.h | 8 +++++--- build_littlefs.bat | 2 ++ build_littlefs1.5M.bat | 16 ++++++++++++++++ version.h | 26 +++++++++++++------------- 4 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 build_littlefs1.5M.bat diff --git a/OTGW-firmware.h b/OTGW-firmware.h index 7d4632a6..39b49161 100644 --- a/OTGW-firmware.h +++ b/OTGW-firmware.h @@ -28,11 +28,13 @@ #define sFS_PHYS_ADDR 0x200000 // FS_PHYS_ADDR //use the external value generated by Arduino IDE in boards.txt. Consider hard coding at 0x200000 #define sFS_PHYS_PAGE 256// FS_PHYS_PAGE //use the external value generated by Arduino IDE in boards.txt. Consider hard coding at 512 #define sFS_PHYS_BLOCK 4096 // ESP8266 flash uses 4k blocks. Anything other than 4k multiples gives an exception. Arduino IDE uses 8k -#define sFS_PHYS_SIZE 1024 * 1024 //FS_PHYS_SIZE - OTGW_USER_FS_SIZE +//#define sFS_PHYS_SIZE 1024 * 1024 //FS_PHYS_SIZE - OTGW_USER_FS_SIZE +#define sFS_PHYS_SIZE 1572864 #define sFS_MAX_OPEN_FILES 5 //FS_MAX_OPEN_FILES //hard coded since there is no external value generated by Arduino IDE in boards.txt -#define uFS_PHYS_ADDR 0x37A000 // 0x3FA000 - uFS_PHYS_SIZE //0x3FA000 is provided by the ESP8266 arduino package in boards.txt -> nodemcuv2.menu.eesz.4M2M.build.spiffs_end -#define uFS_PHYS_SIZE 512 * 1024 +//#define uFS_PHYS_ADDR 0x37A000 // 0x3FA000 - uFS_PHYS_SIZE //0x3FA000 is provided by the ESP8266 arduino package in boards.txt -> nodemcuv2.menu.eesz.4M2M.build.spiffs_end +#define uFS_PHYS_ADDR 0x380000 // 0x3FA000 - uFS_PHYS_SIZE //0x3FA000 is provided by the ESP8266 arduino package in boards.txt -> nodemcuv2.menu.eesz.4M2M.build.spiffs_end +#define uFS_PHYS_SIZE 512 * 1024 - 0x6000 #define uFS_PHYS_PAGE sFS_PHYS_PAGE #define uFS_PHYS_BLOCK sFS_PHYS_BLOCK #define uFS_MAX_OPEN_FILES sFS_MAX_OPEN_FILES diff --git a/build_littlefs.bat b/build_littlefs.bat index f5a9e52f..5b7ec930 100644 --- a/build_littlefs.bat +++ b/build_littlefs.bat @@ -5,9 +5,11 @@ set /A page_size = 256 set /A fs_size = 1024 * 1024 set "source_dir=data" set "img_name=OTGW-Firmware.littlefs.1M.bin" +echo Building filesystem (%source_dir%): set "build_command=%mklittlefs_exec% -c %source_dir% -b %block_size% -p %page_size% -s %fs_size% %img_name%" call %build_command% pause +echo Check content of filesystem: set "check_command=%mklittlefs_exec% -l -b %block_size% -p %page_size% -s %fs_size% %img_name%" call %check_command% diff --git a/build_littlefs1.5M.bat b/build_littlefs1.5M.bat new file mode 100644 index 00000000..90ec81c8 --- /dev/null +++ b/build_littlefs1.5M.bat @@ -0,0 +1,16 @@ +@echo off +set "mklittlefs_exec=%homedrive%%homepath%\AppData\Local\Arduino15\packages\esp8266\tools\mklittlefs\2.5.0-4-fe5bb56\mklittlefs.exe" +set /A block_size = 4096 +set /A page_size = 256 +set /A fs_size = 1572864 +set "source_dir=data" +set "img_name=OTGW-Firmware.littlefs.1.5M.bin" +echo Building filesystem (%source_dir%): +set "build_command=%mklittlefs_exec% -c %source_dir% -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %build_command% +pause +echo Check content of filesystem: +set "check_command=%mklittlefs_exec% -l -b %block_size% -p %page_size% -s %fs_size% %img_name%" +call %check_command% + + diff --git a/version.h b/version.h index 37dd4398..02b69181 100644 --- a/version.h +++ b/version.h @@ -1,16 +1,16 @@ //The version number conforms to semver.org format -#define _VERSION_MAJOR 0 -#define _VERSION_MINOR 9 -#define _VERSION_PATCH 3 -#define _VERSION_BUILD 1638 -#define _VERSION_GITHASH "b4b8026" +#define _VERSION_MAJOR 1 +#define _VERSION_MINOR 0 +#define _VERSION_PATCH 0 +#define _VERSION_BUILD 1642 +#define _VERSION_GITHASH "0003867" //#define _VERSION_PRERELEASE beta //uncomment to define prerelease labels: alpha - beta - rc -#define _VERSION_DATE "16-01-2022" -#define _VERSION_TIME "21:09:00" -#define _SEMVER_CORE "0.9.3" -#define _SEMVER_BUILD "0.9.3+1638" -#define _SEMVER_GITHASH "0.9.3+b4b8026" -#define _SEMVER_FULL "0.9.3+b4b8026" -#define _SEMVER_NOBUILD "0.9.3 (16-01-2022)" -#define _VERSION "0.9.3+b4b8026 (16-01-2022)" +#define _VERSION_DATE "15-02-2022" +#define _VERSION_TIME "21:01:54" +#define _SEMVER_CORE "1.0.0" +#define _SEMVER_BUILD "1.0.0+1642" +#define _SEMVER_GITHASH "1.0.0+0003867" +#define _SEMVER_FULL "1.0.0+0003867" +#define _SEMVER_NOBUILD "1.0.0 (15-02-2022)" +#define _VERSION "1.0.0+0003867 (15-02-2022)" //The version information is created automatically, more information here: https://github.com/rvdbreemen/autoinc-semver From 97ca755bfa90788b1785413ecab9aa247623435f Mon Sep 17 00:00:00 2001 From: Robert van den Breemen Date: Wed, 16 Feb 2022 21:20:46 +0100 Subject: [PATCH 19/19] fxing generating settings.ini on empty filesystem --- OTGW-firmware.ino | 1 + settingStuff.ino | 11 ++++++----- version.h | 18 +++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/OTGW-firmware.ino b/OTGW-firmware.ino index 0c92255a..83bd27e6 100644 --- a/OTGW-firmware.ino +++ b/OTGW-firmware.ino @@ -194,6 +194,7 @@ bool setupFilesystems() { OTGWSerial.println("User partition is accessable"); bUserFSpresent = true; printFSinfo(OTGWSerial, UserFS); + printFScontents(OTGWSerial, UserFS); } else { OTGWSerial.printf("Formatting the user partition: %s\n", UserFS.format() ? "success" : "failed"); } diff --git a/settingStuff.ino b/settingStuff.ino index 02aa4a5e..aaee23e1 100644 --- a/settingStuff.ino +++ b/settingStuff.ino @@ -73,13 +73,14 @@ void readSettings(bool show) { // Open file for reading - - DebugTf("Open file for reading %s ..\r\n", SETTINGS_FILE); File file = UserFS.open(SETTINGS_FILE, "r"); - if (!file) - { // settings file if it does not exist yet. - DebugTf("%s file not found! --> file will be created next time you save your settings!\r\n", SETTINGS_FILE); + DebugTf(" %s ..\r\n", SETTINGS_FILE); + if (!UserFS.exists(SETTINGS_FILE)) + { //create settings file if it does not exist yet. + DebugTln(F(" .. file not found! --> created file!")); + writeSettings(show); + readSettings(false); //now it should work... return; } diff --git a/version.h b/version.h index 02b69181..7f7799c9 100644 --- a/version.h +++ b/version.h @@ -2,15 +2,15 @@ #define _VERSION_MAJOR 1 #define _VERSION_MINOR 0 #define _VERSION_PATCH 0 -#define _VERSION_BUILD 1642 -#define _VERSION_GITHASH "0003867" +#define _VERSION_BUILD 1647 +#define _VERSION_GITHASH "4678299" //#define _VERSION_PRERELEASE beta //uncomment to define prerelease labels: alpha - beta - rc -#define _VERSION_DATE "15-02-2022" -#define _VERSION_TIME "21:01:54" +#define _VERSION_DATE "16-02-2022" +#define _VERSION_TIME "21:10:02" #define _SEMVER_CORE "1.0.0" -#define _SEMVER_BUILD "1.0.0+1642" -#define _SEMVER_GITHASH "1.0.0+0003867" -#define _SEMVER_FULL "1.0.0+0003867" -#define _SEMVER_NOBUILD "1.0.0 (15-02-2022)" -#define _VERSION "1.0.0+0003867 (15-02-2022)" +#define _SEMVER_BUILD "1.0.0+1647" +#define _SEMVER_GITHASH "1.0.0+4678299" +#define _SEMVER_FULL "1.0.0+4678299" +#define _SEMVER_NOBUILD "1.0.0 (16-02-2022)" +#define _VERSION "1.0.0+4678299 (16-02-2022)" //The version information is created automatically, more information here: https://github.com/rvdbreemen/autoinc-semver