Migrate to LittleFS under ESP8266

Make ESP8266 use LittleFS instead of deprecated SPIFFS
Make framework use the correct filesystem automatically and handle the call the FS.begin()
Change default MQTT keepalive to 60 seconds
Fix lodash security issue

Author: rjwats

README.md

@@ -76,13 +76,13 @@ platformio run -t upload
 
 The interface has been configured with create-react-app and react-app-rewired so the build can customized for the target device. The large artefacts are gzipped and source maps and service worker are excluded from the production build. This reduces the production build to around ~150k, which easily fits on the device.
 
-The interface will be automatically built by PlatformIO before it builds the firmware. The project can be configured to serve the interface from either PROGMEM or SPIFFS as your project requires. The default configuration is to serve the content from PROGMEM, serving from SPIFFS requires an additional upload step which is documented below.
+The interface will be automatically built by PlatformIO before it builds the firmware. The project can be configured to serve the interface from either PROGMEM or the filesystem as your project requires. The default configuration is to serve the content from PROGMEM, serving from the filesystem requires an additional upload step which is [documented below](#serving-the-interface-from-the-filesystem).
 
 #### Serving the interface from PROGMEM
 
 By default, the project is configured to serve the interface from PROGMEM. 
 
-> **Tip**: You do not need to upload a file system image unless you configure the framework to [serve the interface from SPIFFS](#serving-the-interface-from-spiffs).
+> **Tip**: You do not need to upload a file system image unless you configure the framework to [serve the interface from the filesystem](#serving-the-interface-from-the-filesystem).
 
 The interface will consume ~150k of program space which can be problematic if you already have a large binary artefact or if you have added large dependencies to the interface. The ESP32 binaries are fairly large in there simplest form so the addition of the interface resources requires us to use special partitioning for the ESP32.
 
@@ -96,9 +96,9 @@ platform = espressif32
 board = node32s
 ```
 
-#### Serving the interface from SPIFFS
+#### Serving the interface from the filesystem
 
-If you choose to serve the interface from SPIFFS you will need to change the default configuration and upload the file system image manually. 
+If you choose to serve the interface from the filesystem you will need to change the default configuration and upload the file system image manually. 
 
 Disable `-D PROGMEM_WWW build` flag in ['platformio.ini'](platformio.ini) and re-build the firmware. The build process will now copy the compiled interface to the `data/` directory and it may be uploaded to the device by pressing the "Upload File System image" button:
 
@@ -340,7 +340,7 @@ The following code creates the web server and esp8266React framework:
 
 ```cpp
 AsyncWebServer server(80);
-ESP8266React esp8266React(&server, &SPIFFS);
+ESP8266React esp8266React(&server);
 ```
 
 Now in the `setup()` function the initialization is performed:
@@ -350,13 +350,6 @@ void setup() {
   // start serial and filesystem
   Serial.begin(SERIAL_BAUD_RATE);
 
-  // start the file system (must be done before starting the framework)
-#ifdef ESP32
-  SPIFFS.begin(true);
-#elif defined(ESP8266)
-  SPIFFS.begin();
-#endif
-
   // start the framework and demo project
   esp8266React.begin();
 
@@ -615,6 +608,7 @@ The framework supplies access to various features via getter functions:
 
 SettingsService              | Description
 ---------------------------- | ----------------------------------------------
+getFS()                      | The filesystem used by the framework
 getSecurityManager()         | The security manager - detailed above
 getSecuritySettingsService() | Configures the users and other security settings
 getWiFiSettingsService()     | Configures and manages the WiFi network connection

factory_settings.ini

@@ -38,7 +38,7 @@ build_flags =
     -D FACTORY_MQTT_PASSWORD=\"\"
     ; if unspecified the devices hardware ID will be used
     ;-D FACTORY_MQTT_CLIENT_ID=\"esp-react\"
-    -D FACTORY_MQTT_KEEP_ALIVE=16
+    -D FACTORY_MQTT_KEEP_ALIVE=60
     -D FACTORY_MQTT_CLEAN_SESSION=true
     -D FACTORY_MQTT_MAX_TOPIC_LENGTH=128
 

interface/config-overrides.js

@@ -9,7 +9,7 @@ const fs = require('fs');
 
 module.exports = function override(config, env) {
   if (env === "production") {
-    // rename the ouput file, we need it's path to be short, for SPIFFS
+    // rename the ouput file, we need it's path to be short, for embedded FS
     config.output.filename = 'js/[id].[chunkhash:4].js';
     config.output.chunkFilename = 'js/[id].[chunkhash:4].js';
 

interface/package-lock.json

@@ -6,7 +6,7 @@
     "@material-ui/core": "^4.9.8",
     "@material-ui/icons": "^4.9.1",
     "@types/jwt-decode": "^2.2.1",
-    "@types/lodash": "^4.14.149",
+    "@types/lodash": "^4.14.157",
     "@types/node": "^12.12.32",
     "@types/react": "^16.9.27",
     "@types/react-dom": "^16.9.5",
@@ -15,7 +15,7 @@
     "@types/react-router-dom": "^5.1.3",
     "compression-webpack-plugin": "^4.0.0",
     "jwt-decode": "^2.2.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.19",
     "mime-types": "^2.1.25",
     "moment": "^2.26.0",
     "notistack": "^0.9.17",

interface/package.json

@@ -1,32 +1,32 @@
 #include <ESP8266React.h>
 
-ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) :
+ESP8266React::ESP8266React(AsyncWebServer* server) :
     _featureService(server),
-    _securitySettingsService(server, fs),
-    _wifiSettingsService(server, fs, &_securitySettingsService),
+    _securitySettingsService(server, &ESPFS),
+    _wifiSettingsService(server, &ESPFS, &_securitySettingsService),
     _wifiScanner(server, &_securitySettingsService),
     _wifiStatus(server, &_securitySettingsService),
-    _apSettingsService(server, fs, &_securitySettingsService),
+    _apSettingsService(server, &ESPFS, &_securitySettingsService),
     _apStatus(server, &_securitySettingsService, &_apSettingsService),
 #if FT_ENABLED(FT_NTP)
-    _ntpSettingsService(server, fs, &_securitySettingsService),
+    _ntpSettingsService(server, &ESPFS, &_securitySettingsService),
     _ntpStatus(server, &_securitySettingsService),
 #endif
 #if FT_ENABLED(FT_OTA)
-    _otaSettingsService(server, fs, &_securitySettingsService),
+    _otaSettingsService(server, &ESPFS, &_securitySettingsService),
 #endif
 #if FT_ENABLED(FT_UPLOAD_FIRMWARE)
     _uploadFirmwareService(server, &_securitySettingsService),
 #endif
 #if FT_ENABLED(FT_MQTT)
-    _mqttSettingsService(server, fs, &_securitySettingsService),
+    _mqttSettingsService(server, &ESPFS, &_securitySettingsService),
     _mqttStatus(server, &_mqttSettingsService, &_securitySettingsService),
 #endif
 #if FT_ENABLED(FT_SECURITY)
     _authenticationService(server, &_securitySettingsService),
 #endif
     _restartService(server, &_securitySettingsService),
-    _factoryResetService(server, fs, &_securitySettingsService),
+    _factoryResetService(server, &ESPFS, &_securitySettingsService),
     _systemStatus(server, &_securitySettingsService) {
 #ifdef PROGMEM_WWW
   // Serve static resources from PROGMEM
@@ -63,7 +63,7 @@ ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) :
   // OPTIONS get a straight up 200 response
   server->onNotFound([](AsyncWebServerRequest* request) {
     if (request->method() == HTTP_GET) {
-      request->send(SPIFFS, "/www/index.html");
+      request->send(ESPFS, "/www/index.html");
     } else if (request->method() == HTTP_OPTIONS) {
       request->send(200);
     } else {
@@ -81,6 +81,11 @@ ESP8266React::ESP8266React(AsyncWebServer* server, FS* fs) :
 }
 
 void ESP8266React::begin() {
+#ifdef ESP32
+  ESPFS.begin(true);
+#elif defined(ESP8266)
+  ESPFS.begin();
+#endif
   _wifiSettingsService.begin();
   _apSettingsService.begin();
 #if FT_ENABLED(FT_NTP)

lib/framework/ESP8266React.cpp

@@ -28,18 +28,23 @@
 #include <WiFiScanner.h>
 #include <WiFiSettingsService.h>
 #include <WiFiStatus.h>
+#include <ESPFS.h>
 
 #ifdef PROGMEM_WWW
 #include <WWWData.h>
 #endif
 
 class ESP8266React {
  public:
-  ESP8266React(AsyncWebServer* server, FS* fs);
+  ESP8266React(AsyncWebServer* server);
 
   void begin();
   void loop();
 
+  FS* getFS() {
+    return &ESPFS;
+  }
+
   SecurityManager* getSecurityManager() {
     return &_securitySettingsService;
   }

lib/framework/ESP8266React.h

@@ -0,0 +1,7 @@
+#ifdef ESP32
+#include <SPIFFS.h>
+#define ESPFS SPIFFS
+#elif defined(ESP8266)
+#include <LittleFS.h>
+#define ESPFS LittleFS
+#endif

lib/framework/ESPFS.h

@@ -15,7 +15,7 @@ void FactoryResetService::handleRequest(AsyncWebServerRequest* request) {
 }
 
 /**
- * Delete function assumes that all files are stored flat, within the config directory
+ * Delete function assumes that all files are stored flat, within the config directory.
  */
 void FactoryResetService::factoryReset() {
 #ifdef ESP32
@@ -27,7 +27,10 @@ void FactoryResetService::factoryReset() {
 #elif defined(ESP8266)
   Dir configDirectory = fs->openDir(FS_CONFIG_DIRECTORY);
   while (configDirectory.next()) {
-    fs->remove(configDirectory.fileName());
+    String path = FS_CONFIG_DIRECTORY;
+    path.concat("/");
+    path.concat(configDirectory.fileName());
+    fs->remove(path);
   }
 #endif
   RestartService::restartNow();

lib/framework/FactoryResetService.cpp

@@ -31,11 +31,11 @@ void SystemStatus::systemStatus(AsyncWebServerRequest* request) {
 // TODO - Ideally this class will take an *FS and extract the file system information from there.
 // ESP8266 and ESP32 do not have feature parity in FS.h which currently makes that difficult.
 #ifdef ESP32
-  root["fs_total"] = SPIFFS.totalBytes();
-  root["fs_used"] = SPIFFS.usedBytes();
+  root["fs_total"] = ESPFS.totalBytes();
+  root["fs_used"] = ESPFS.usedBytes();
 #elif defined(ESP8266)
   FSInfo fs_info;
-  SPIFFS.info(fs_info);
+  ESPFS.info(fs_info);
   root["fs_total"] = fs_info.totalBytes;
   root["fs_used"] = fs_info.usedBytes;
 #endif