Made path store segment size/count configurable

Author: attermann

README.md

@@ -29,6 +29,8 @@ Each class of allocator can be configured to use a separate type of allocator. A
 - `RNS_PSRAM_ALLOCATOR`: Basic allocator that uses PSRAM memory (if PSRAM is available)
 - `RNS_PSRAM_POOL_ALLOCATOR`: Optimized TLSF managed PSRAM buffer (if PSRAM is available)
 
+NOTE: When utilizing PSRAM on ESP32 boards, preprocessor directive `-DBOARD_HAS_PSRAM=1` must be included in build flags in order to enable on-board PSRAM.
+
 Each class of allocator pool can have a separate buffer size:
 - `RNS_HEAP_POOL_BUFFER_SIZE=N` Defines the HEAP TLSF memory pool buffer size (N bytes, default 0 which means dynamic)
 - `RNS_PSRAM_POOL_BUFFER_SIZE=N` Defines the PSRAM TLSF memory pool buffer size (N bytes, default 0 which means dynamic)
@@ -44,7 +46,25 @@ build_flags =
   -DRNS_CONTAINER_ALLOCATOR=RNS_PSRAM_POOL_ALLOCATOR
 ```
 
-NOTE: Currently `-DRNS_DEFAULT_ALLOCATOR=RNS_HEAP_POOL_ALLOCATOR` is only required on NRF52 boards since ESP32 already uses TLSF internally.
+NOTE: Currently "POOL" allocators are only required on NRF52 boards since ESP32 already uses TLSF internally for both SRAM and PSRAM resources.
+
+## microStore Options
+
+When using microReticulum to implement a transpport node, storage for the path table and other persistent data must be configured. This is accomplished using the File and FileSystem abstrctions provided in `microStore`. Several pre-build adapters are provided with the library, and optionally custom storage (e.g., external flash and SD card) can be used by deriving custom implementations from `microStore::File` and `microStore::FileSystem` and passing them to microReticulum for internal use.
+There are several pre-built filesystem "adapters" available in the `microStore` library that can be used without modification for the most common supported boards. These can be inluded using the following preprocessor directives:
+  `-DUSTORE_USE_POSIXFS`: Native as well as any ESP-IDF supported POSIX file system
+  `-DUSTORE_USE_STDIOFS`: Native STDIO file system
+  `-DUSTORE_USE_INTERNALFS`: InternalFileSystem for adafruit nrf52
+  `-DUSTORE_USE_LITTLEFS`: LittleFS file system
+  `-DUSTORE_USE_SPIFFS`: SPIFFS file system
+  `-DUSTORE_USE_FLASHFS`: Adafruit FlashFS file system
+  `-DUSTORE_USE_UNIVERSALFS`: Best supported file system for any platform
+
+Also note the following preprocessor directives used to tune `microStore::FileStore` which is the Bitcask-style storage engine used by microReticulum to persist the path table and other internal tables:
+  `-DRNS_PATH_TABLE_SEGMENT_SIZE`
+  `-DRNS_PATH_TABLE_SEGMENT_COUNT`
+Appropriate settings should be selected to match the storage and memory resources available on the target platform.
+Note that `RNS_PATH_TABLE_SEGMENT_COUNT` must be at least 3, and `RNS_PATH_TABLE_SEGMENT_COUNT` x `RNS_PATH_TABLE_SEGMENT_SIZE` represents the total approximate storage space available to the the path table.
 
 ## Building
 

platformio.ini

@@ -31,8 +31,6 @@ lib_deps =
 	ArduinoJson@^7.4.2
 	MsgPack@^0.4.2
 	https://github.com/attermann/Crypto.git
-	https://github.com/attermann/microStore.git
-	;microStore=symlink://../microStore
 test_framework = unity
 ; Following property enables Unity integration testing
 test_build_src = true
@@ -58,6 +56,7 @@ build_flags =
 	-DNATIVE
 lib_deps = 
 	${env.lib_deps}
+	https://github.com/attermann/microStore.git
 lib_compat_mode = off
 
 [env:native17]
@@ -71,6 +70,7 @@ build_flags =
 	-DNATIVE
 lib_deps = 
 	${env.lib_deps}
+	https://github.com/attermann/microStore.git
 lib_compat_mode = off
 
 [env:native20]
@@ -84,6 +84,7 @@ build_flags =
 	-DNATIVE
 lib_deps = 
 	${env.lib_deps}
+	https://github.com/attermann/microStore.git
 lib_compat_mode = off
 
 [env:native_debug]
@@ -119,6 +120,7 @@ build_flags =
 	;-DRNS_MEM_LOG
 lib_deps =
 	${env.lib_deps}
+	https://github.com/attermann/microStore.git
 lib_compat_mode = off
 
 [env:embedded]
@@ -133,6 +135,7 @@ build_flags =
 	-std=gnu++17
 lib_deps = 
 	${env.lib_deps}
+	https://github.com/attermann/microStore.git
 
 [env:ttgo-lora32-v21]
 extends = env:embedded
@@ -149,6 +152,7 @@ build_flags =
 	-DMSGPACK_USE_BOOST=OFF
 lib_deps =
 	${env:embedded.lib_deps}
+	https://github.com/attermann/microStore.git
 monitor_filters = esp32_exception_decoder
 
 [env:ttgo-t-beam]
@@ -163,6 +167,7 @@ build_flags =
 	-DMSGPACK_USE_BOOST=OFF
 lib_deps =
 	${env:embedded.lib_deps}
+	https://github.com/attermann/microStore.git
 monitor_filters = esp32_exception_decoder
 
 [env:lilygo_tbeam_supreme]
@@ -177,6 +182,7 @@ build_flags =
 	-DMSGPACK_USE_BOOST=OFF
 lib_deps =
 	${env:embedded.lib_deps}
+	https://github.com/attermann/microStore.git
 monitor_filters = esp32_exception_decoder
 
 ; also Heltec Wireless Tracker
@@ -200,6 +206,7 @@ build_flags =
 	-DRNS_PSRAM_POOL_BUFFER_SIZE=2048000
 lib_deps =
 	${env:embedded.lib_deps}
+	https://github.com/attermann/microStore.git
 monitor_filters = esp32_exception_decoder
 upload_speed = 921600
 
@@ -226,11 +233,18 @@ build_flags =
 	-DMSGPACK_USE_BOOST=OFF
 lib_deps =
 	${env:embedded.lib_deps}
+	https://github.com/attermann/microStore.git
 	adafruit/Adafruit TinyUSB Library
 
-; Overrides for specific tests (not working!)
-[test_env:test_objects]
+
+; Local build targets
+[env:native_local]
+platform = native
 build_flags =
-	${env:embedded.build_flags}
-	;-DTEST_OBJECT_DATA=1
-	-DTEST_OBJECT_IMPL=1
+	${env.build_flags}
+	-std=gnu++11
+	-DNATIVE
+lib_deps =
+	${env.lib_deps}
+	microStore=symlink://../microStore
+lib_compat_mode = off

src/Transport.cpp

@@ -38,6 +38,14 @@ using namespace RNS::Persistence;
 #define RNS_PATH_TABLE_MAX 100
 #endif
 
+#ifndef RNS_PATH_TABLE_SEGMENT_SIZE
+#define RNS_PATH_TABLE_SEGMENT_SIZE 65536
+#endif
+
+#ifndef RNS_PATH_TABLE_SEGMENT_COUNT
+#define RNS_PATH_TABLE_SEGMENT_COUNT 8
+#endif
+
 #ifndef RNS_ANNOUNCE_TABLE_MAX
 #define RNS_ANNOUNCE_TABLE_MAX 100
 #endif
@@ -133,7 +141,7 @@ using namespace RNS::Persistence;
 /*static*/ bool Transport::cleaning_caches = false;
 
 // CBA microStore
-/*static*/ PathStore Transport::_path_store;
+/*static*/ PathStore Transport::_path_store(RNS_PATH_TABLE_SEGMENT_SIZE, RNS_PATH_TABLE_SEGMENT_COUNT);
 /*static*/ NewPathTable Transport::_new_path_table(Transport::_path_store);
 
 DestinationEntry empty_destination_entry;

src/Transport.h

@@ -382,10 +382,6 @@ namespace RNS {
 		static std::set<Bytes> _packet_hashlist;	// A list of packet hashes for duplicate detection
 		static std::list<PacketReceipt> _receipts;	// Receipts of all outgoing packets for proof processing
 
-		// TODO: "destination_table" should really be renamed to "path_table"
-		// Notes on memory usage: 1 megabyte of memory can store approximately
-		// 55.100 path table entries or approximately 22.300 link table entries.
-
 		static AnnounceTable _announce_table;	// A table for storing announces currently waiting to be retransmitted
 		static PathTable _path_table;			// A lookup table containing the next hop to a given destination
 		static ReverseTable _reverse_table;		// A lookup table for storing packet hashes used to return proofs and replies