Merge pull request #69 from torlando-tech/fix/fresh-install-littlefs

fix: safely initialize erased LittleFS partitions

lib/storage/LittleFSInitializationPolicy.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Storage {
+
+constexpr std::size_t LITTLEFS_BLANK_SCAN_CHUNK_SIZE = 4096;
+
+template <typename Mount, typename ResolvePartition, typename ReadPartition,
+          typename FormatAndMount>
+bool mount_or_initialize_erased_littlefs(
+    Mount mount,
+    ResolvePartition resolve_partition,
+    ReadPartition read_partition,
+    FormatAndMount format_and_mount) {
+    if (mount()) return true;
+
+    std::size_t partition_size = 0;
+    if (!resolve_partition(partition_size) || partition_size == 0) return false;
+
+    uint8_t buffer[LITTLEFS_BLANK_SCAN_CHUNK_SIZE];
+    std::size_t offset = 0;
+    while (offset < partition_size) {
+        const std::size_t remaining = partition_size - offset;
+        const std::size_t chunk =
+            remaining < sizeof(buffer) ? remaining : sizeof(buffer);
+        if (!read_partition(offset, buffer, chunk)) return false;
+        for (std::size_t i = 0; i < chunk; ++i) {
+            if (buffer[i] != 0xFF) return false;
+        }
+        offset += chunk;
+    }
+
+    return format_and_mount();
+}
+
+}  // namespace Storage

src/main.cpp

@@ -25,6 +25,8 @@
 #include <new>  // placement new
 #include <soc/rtc_cntl_reg.h>
 
+#include "storage/LittleFSInitializationPolicy.h"
+
 // Reticulum
 #include <microReticulum/Reticulum.h>
 #include <microReticulum/Utilities/OS.h>
@@ -964,6 +966,8 @@ static void pump_ntp_sync_if_pending() {
     }
 }
 
+static constexpr const char* LITTLEFS_PARTITION_LABEL = "spiffs";
+
 void setup_hardware() {
     INFO("\n=== Hardware Initialization ===");
 
@@ -981,7 +985,44 @@ void setup_hardware() {
     // Pyxis's lib/universal_filesystem/ is now dead code on this build path and
     // can be deleted once the graft lands.
     static microStore::Adapters::LittleFSFileSystem fs("/littlefs");
-    persistent_storage_ready = fs.init(false);
+
+    // Mount non-destructively first: never format on a mount failure by itself,
+    // because a corrupt filesystem holds user conversations that must be preserved.
+    //
+    // An erased or uninitialized LittleFS partition is all 0xFF. Its mount fails,
+    // but formatting is safe because the partition contains no programmed bytes.
+    // Any nonblank or unreadable partition remains untouched and falls through to
+    // recovery mode. The policy is portable so every destructive branch is covered
+    // by host tests while these callbacks retain the real ESP32 operations.
+    const esp_partition_t* littlefs_partition = nullptr;
+    persistent_storage_ready = Storage::mount_or_initialize_erased_littlefs(
+        []() { return fs.init(false); },
+        [&littlefs_partition](size_t& size) {
+            littlefs_partition = esp_partition_find_first(
+                ESP_PARTITION_TYPE_DATA,
+                ESP_PARTITION_SUBTYPE_DATA_SPIFFS,
+                LITTLEFS_PARTITION_LABEL);
+            if (!littlefs_partition) {
+                ERROR("Unable to locate LittleFS partition for blank check");
+                return false;
+            }
+            size = littlefs_partition->size;
+            return true;
+        },
+        [&littlefs_partition](size_t offset, uint8_t* buffer, size_t size) {
+            if (esp_partition_read(littlefs_partition, offset, buffer, size) != ESP_OK) {
+                ERROR("Failed to read LittleFS partition for blank check");
+                return false;
+            }
+            return true;
+        },
+        []() {
+            // Use LittleFS.begin() directly because microStore's adapter returns
+            // immediately when its non-destructive mount fails.
+            INFO("Blank LittleFS partition; formatting once");
+            return LittleFS.begin(
+                true, "/littlefs", 10, LITTLEFS_PARTITION_LABEL);
+        });
     location_filesystem_available = persistent_storage_ready;
     if (!persistent_storage_ready) {
         ERROR("FileSystem mount failed; preserving persistent data");

tests/build_scripts/test_message_persistence_contract.py

@@ -147,7 +147,14 @@ def test_failed_littlefs_mount_enters_stable_recovery_mode_before_reticulum():
     setup = function_body(source, "void setup()", "void loop()")
     loop = source[source.index("void loop()"):]
 
-    assert "persistent_storage_ready = fs.init(false);" in source
+    assert "Storage::mount_or_initialize_erased_littlefs(" in source
+    assert "[]() { return fs.init(false); }" in source
+    assert """littlefs_partition = esp_partition_find_first(
+                ESP_PARTITION_TYPE_DATA,
+                ESP_PARTITION_SUBTYPE_DATA_SPIFFS,
+                LITTLEFS_PARTITION_LABEL);""" in source
+    assert 'static constexpr const char* LITTLEFS_PARTITION_LABEL = "spiffs";' in source
+    assert 'true, "/littlefs", 10, LITTLEFS_PARTITION_LABEL' in source
     assert "Persistent storage unavailable" in source
     assert "USB serial recovery remains available." in source
     assert setup.index("if (!persistent_storage_ready)") < setup.index("setup_reticulum();")

tests/build_scripts/test_patch_littlefs_paths.py

@@ -132,11 +132,23 @@ def test_persistent_partitions_do_not_overlap_app_slots():
         if not line or line.startswith("#"):
             continue
         fields = [field.strip() for field in line.split(",")]
-        rows.append((fields[0], int(fields[3], 0), int(fields[4], 0)))
+        rows.append(
+            (
+                fields[0],
+                fields[1],
+                fields[2],
+                int(fields[3], 0),
+                int(fields[4], 0),
+            )
+        )
 
-    by_name = {name: (offset, offset + size) for name, offset, size in rows}
-    persistent = [by_name["nvs"], by_name["spiffs"]]
-    applications = [by_name["app0"], by_name["app1"]]
+    by_name = {
+        name: (partition_type, subtype, offset, offset + size)
+        for name, partition_type, subtype, offset, size in rows
+    }
+    assert by_name["spiffs"] == ("data", "spiffs", 0x610000, 0x7F0000)
+    persistent = [by_name["nvs"][2:], by_name["spiffs"][2:]]
+    applications = [by_name["app0"][2:], by_name["app1"][2:]]
 
     for data_start, data_end in persistent:
         for app_start, app_end in applications:

tests/build_scripts/test_web_flasher_release_safety.py

@@ -67,6 +67,16 @@ def test_custom_firmware_upload_is_explicit_validated_and_update_only():
     assert "const useFullInstall = customFirmwareBytes ? false : eraseCheckbox.checked;" in source
 
 
+def test_full_install_erases_flash_while_updates_preserve_persistent_partitions():
+    source = FLASHER.read_text()
+    flash = source[source.index("async function flash()") :]
+
+    assert "const useFullInstall = customFirmwareBytes ? false : eraseCheckbox.checked;" in flash
+    assert "eraseAll: useFullInstall," in flash
+    assert "useFullInstall ? fw.full : fw.update" in flash
+    assert "{ offset: 0x610000" not in source
+
+
 def test_custom_firmware_selection_invalidates_stale_async_results():
     source = FLASHER.read_text()
     handler = source[source.index("customFirmwareInput.addEventListener('change'"):source.index("function selectPublishedRelease")]
@@ -163,6 +173,7 @@ def test_release_audit_rejects_stale_version_and_destructive_storage_firmware():
 
     assert 'b"Firmware: v1.0.0"' in audit
     assert 'b"FileSystem mount failed; preserving persistent data"' in audit
+    assert 'b"Blank LittleFS partition; formatting once"' in audit
     assert '"git", "describe", "--tags", "--always", "--dirty"' in audit
     assert 'os.environ.get("PYXIS_VERSION_OVERRIDE")' in audit
     assert 'f"Firmware: {expected_version}".encode()' in audit

tests/native/test_littlefs_initialization_policy.cpp

@@ -0,0 +1,116 @@
+#include <cstddef>
+#include <cstdint>
+#include <iostream>
+#include <vector>
+
+#include "LittleFSInitializationPolicy.h"
+
+using Storage::mount_or_initialize_erased_littlefs;
+
+static int passed = 0;
+static int failed = 0;
+#define CHECK(expr) do { if (expr) { ++passed; } else { ++failed; std::cerr << "FAIL line " << __LINE__ << ": " #expr "\n"; } } while (0)
+
+struct Harness {
+    std::vector<uint8_t> partition;
+    bool mount_result = false;
+    bool resolve_result = true;
+    bool read_result = true;
+    std::size_t fail_read_at = SIZE_MAX;
+    bool format_mount_result = true;
+    int mount_calls = 0;
+    int resolve_calls = 0;
+    int read_calls = 0;
+    int format_mount_calls = 0;
+
+    bool run() {
+        return mount_or_initialize_erased_littlefs(
+            [this]() {
+                ++mount_calls;
+                return mount_result;
+            },
+            [this](std::size_t& size) {
+                ++resolve_calls;
+                if (!resolve_result) return false;
+                size = partition.size();
+                return true;
+            },
+            [this](std::size_t offset, uint8_t* output, std::size_t size) {
+                ++read_calls;
+                if (!read_result || offset >= fail_read_at ||
+                    offset + size > partition.size()) return false;
+                for (std::size_t i = 0; i < size; ++i) output[i] = partition[offset + i];
+                return true;
+            },
+            [this]() {
+                ++format_mount_calls;
+                return format_mount_result;
+            });
+    }
+};
+
+int main() {
+    {
+        Harness harness;
+        harness.mount_result = true;
+        CHECK(harness.run());
+        CHECK(harness.mount_calls == 1);
+        CHECK(harness.resolve_calls == 0);
+        CHECK(harness.read_calls == 0);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        harness.partition.assign(8193, 0xFF);
+        CHECK(harness.run());
+        CHECK(harness.read_calls == 3);
+        CHECK(harness.format_mount_calls == 1);
+    }
+    {
+        Harness harness;
+        harness.partition.assign(8193, 0xFF);
+        harness.partition.back() = 0x00;
+        CHECK(!harness.run());
+        CHECK(harness.read_calls == 3);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        harness.resolve_result = false;
+        CHECK(!harness.run());
+        CHECK(harness.read_calls == 0);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        CHECK(!harness.run());
+        CHECK(harness.read_calls == 0);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        harness.partition.assign(4096, 0xFF);
+        harness.read_result = false;
+        CHECK(!harness.run());
+        CHECK(harness.read_calls == 1);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        harness.partition.assign(8192, 0xFF);
+        harness.fail_read_at = 4096;
+        CHECK(!harness.run());
+        CHECK(harness.read_calls == 2);
+        CHECK(harness.format_mount_calls == 0);
+    }
+    {
+        Harness harness;
+        harness.partition.assign(4096, 0xFF);
+        harness.format_mount_result = false;
+        CHECK(!harness.run());
+        CHECK(harness.format_mount_calls == 1);
+    }
+
+    std::cout << passed << " passed, " << failed << " failed\n";
+    return failed == 0 ? 0 : 1;
+}

tests/native/test_littlefs_initialization_policy.py

@@ -0,0 +1,19 @@
+"""Compile and execute the portable LittleFS initialization policy regression."""
+
+from pathlib import Path
+
+from native_test import compile_and_run
+
+
+HERE = Path(__file__).resolve().parent
+PYXIS_ROOT = HERE.parent.parent
+
+
+def test_littlefs_initialization_policy(tmp_path):
+    ran = compile_and_run(
+        tmp_path,
+        name="test_littlefs_initialization_policy",
+        sources=[HERE / "test_littlefs_initialization_policy.cpp"],
+        include_dirs=[PYXIS_ROOT / "lib" / "storage"],
+    )
+    assert "25 passed, 0 failed" in ran.stdout

tools/audit_release_build.py

@@ -34,6 +34,7 @@ EXCLUDED_STRINGS = (
 )
 REQUIRED_STRINGS = (
     b"FileSystem mount failed; preserving persistent data",
+    b"Blank LittleFS partition; formatting once",
 )
 EXCLUDED_SYMBOLS = (
     "test_call_",