From ef529a2255fb28fd90b54e37ac5b5dcb9ad6a05e Mon Sep 17 00:00:00 2001 From: Trail Mate Dev Date: Tue, 28 Jul 2026 19:26:40 +0800 Subject: [PATCH] refactor: consolidate storage maintenance ownership --- AGENTS.md | 18 +- apps/esp32_lvgl/CMakeLists.txt | 8 + .../esp32_lvgl_arduino_app_runtime_access.cpp | 9 - .../src/esp32_lvgl_idf_app_facade_runtime.cpp | 323 +- ...esp32_lvgl_ui_lifecycle_contract_smoke.cpp | 124 + apps/linux_sim_shell/CMakeLists.txt | 37 +- .../esp_idf/ESP_IDF_COMPONENT_SOURCES.cmake | 9 + docs/config_persistence_architecture.md | 116 +- .../specification/RUNTIME_CONCURRENCY_SPEC.md | 149 +- .../UI_STORAGE_EVENT_RUNTIME_DESIGN_SPEC.md | 220 +- docs/spi_bus_architecture.md | 43 +- .../chat/infra/mesh_peer_directory_core.h | 19 + .../include/chat/ports/i_chat_store.h | 11 + .../chat/ports/i_mesh_peer_directory.h | 2 + .../ports/i_mesh_peer_directory_blob_store.h | 12 + .../include/chat/usecase/chat_service.h | 9 + .../src/infra/mesh_peer_directory_core.cpp | 179 +- .../core_chat/src/usecase/contact_service.cpp | 20 +- .../test_esp_sd_store_read_state_contract.cpp | 339 +- .../test_mesh_peer_directory_contract.cpp | 138 + .../include/app/app_config_save_plan.h | 42 - .../include/app/config_persistence_runtime.h | 256 ++ .../ui/reticulum_group_config_runtime.h | 5 + .../include/sys/persistence_contracts.h | 22 + .../include/sys/persistence_runtime.h | 381 --- .../core_sys/include/sys/runtime_harness.h | 147 - .../include/sys/storage_event_runtime.h | 145 - .../test_app_config_change_detection.cpp | 46 - .../tests/test_config_persistence_runtime.cpp | 193 ++ .../tests/test_storage_event_runtime.cpp | 164 - ...test_ui_storage_event_runtime_contract.cpp | 24 - modules/ui_mono/include/ui/mono/runtime.h | 1 - modules/ui_mono/src/runtime.cpp | 40 +- .../ui/screens/chat/chat_ui_controller.h | 1 + modules/ui_shared/src/ui/app_runtime.cpp | 12 +- .../chat_presentation_source.cpp | 10 + .../ui/screens/chat/chat_ui_controller.cpp | 40 +- .../contacts/contacts_page_components.cpp | 54 +- .../contacts/contacts_page_runtime.cpp | 149 +- .../src/ui/screens/gps/gps_page_runtime.cpp | 4 + .../settings/settings_page_components.cpp | 1282 ++++--- .../tests/test_chat_presentation_source.cpp | 15 + .../arduino_common/include/app/app_context.h | 23 +- .../chat/infra/store/fixed_slot_journal.h | 45 + .../infra/store/sd_protocol_peer_repository.h | 133 +- .../chat/infra/store/sd_store.h | 177 +- .../storage/scoped_state_lock.h | 34 +- .../arduino_common/storage/storage_runtime.h | 14 +- .../esp/arduino_common/src/app_context.cpp | 245 +- .../src/app_context_platform_bindings.cpp | 2 +- .../src/chat/infra/lxmf/lxmf_adapter.cpp | 8 + .../src/chat/infra/lxmf/lxmf_rx_telemetry.cpp | 36 +- .../chat/infra/store/fixed_slot_journal.cpp | 90 + .../store/sd_protocol_peer_repository.cpp | 1721 +++++++--- .../src/chat/infra/store/sd_store.cpp | 2991 +++++++++++------ ...latform_ui_reticulum_directory_runtime.cpp | 6 + ...form_ui_reticulum_group_config_runtime.cpp | 62 + .../src/rnode_kiss/rnode_kiss_service.cpp | 45 +- .../src/storage/storage_runtime.cpp | 533 +-- .../src/ui/widgets/map/map_tiles.cpp | 6 + .../platform/esp/boards/board_runtime.h | 20 + platform/esp/boards/src/board_runtime.cpp | 35 + .../esp/common/storage/storage_contracts.h | 166 + .../storage/storage_maintenance_owner.h | 606 ++++ .../storage_maintenance_state_machine.h | 312 ++ ...test_storage_maintenance_state_machine.cpp | 172 + .../platform/esp/idf_common/storage_runtime.h | 8 +- .../src/platform_ui_device_runtime.cpp | 15 +- ...latform_ui_reticulum_directory_runtime.cpp | 10 + ...form_ui_reticulum_group_config_runtime.cpp | 64 + platform/esp/idf_common/src/screen_sleep.cpp | 4 +- .../esp/idf_common/src/storage_runtime.cpp | 730 +++- .../common/include/app/linux_app_services.h | 10 + .../common/src/app/linux_app_services.cpp | 75 +- .../ui/reticulum_directory_runtime.cpp | 10 + .../ui/reticulum_group_config_runtime.cpp | 54 + .../src/uconsole_map_workspace_model.cpp | 18 +- ...form_ui_reticulum_group_config_runtime.cpp | 16 + 78 files changed, 9577 insertions(+), 3737 deletions(-) create mode 100644 apps/esp32_lvgl/tests/esp32_lvgl_ui_lifecycle_contract_smoke.cpp delete mode 100644 modules/core_sys/include/app/app_config_save_plan.h create mode 100644 modules/core_sys/include/app/config_persistence_runtime.h create mode 100644 modules/core_sys/include/sys/persistence_contracts.h delete mode 100644 modules/core_sys/include/sys/persistence_runtime.h delete mode 100644 modules/core_sys/include/sys/storage_event_runtime.h create mode 100644 modules/core_sys/tests/test_config_persistence_runtime.cpp delete mode 100644 modules/core_sys/tests/test_storage_event_runtime.cpp create mode 100644 platform/esp/common/include/platform/esp/common/storage/storage_contracts.h create mode 100644 platform/esp/common/include/platform/esp/common/storage/storage_maintenance_owner.h create mode 100644 platform/esp/common/include/platform/esp/common/storage/storage_maintenance_state_machine.h create mode 100644 platform/esp/common/tests/test_storage_maintenance_state_machine.cpp diff --git a/AGENTS.md b/AGENTS.md index 1d44fc4f..e565ae14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,24 +1,24 @@ # GitNexus — Code Intelligence -This project is indexed by GitNexus as **trail-mate** (52705 symbols, 98350 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **trail-mate** (69850 symbols, 150359 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. -> Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). +> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "main"})`. +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. +- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. ## Never Do -- NEVER edit a function, class, or method without first running `impact` on it. +- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. -- NEVER commit changes without running `detect_changes()` to check affected scope. +- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. +- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. ## Resources diff --git a/apps/esp32_lvgl/CMakeLists.txt b/apps/esp32_lvgl/CMakeLists.txt index 1d654545..650a436e 100644 --- a/apps/esp32_lvgl/CMakeLists.txt +++ b/apps/esp32_lvgl/CMakeLists.txt @@ -68,6 +68,14 @@ if(BUILD_TESTING) COMMAND trailmate_esp32_lvgl_sd_coredump_contract_smoke "${TRAIL_MATE_REPO_ROOT}") + add_executable(trailmate_esp32_lvgl_ui_lifecycle_contract_smoke + tests/esp32_lvgl_ui_lifecycle_contract_smoke.cpp) + target_compile_features(trailmate_esp32_lvgl_ui_lifecycle_contract_smoke + PRIVATE cxx_std_17) + add_test(NAME trailmate_esp32_lvgl_ui_lifecycle_contract_smoke + COMMAND trailmate_esp32_lvgl_ui_lifecycle_contract_smoke + "${TRAIL_MATE_REPO_ROOT}") + add_executable(trailmate_esp32_lvgl_font_hot_path_contract_smoke tests/esp32_lvgl_font_hot_path_contract_smoke.cpp) target_compile_features(trailmate_esp32_lvgl_font_hot_path_contract_smoke diff --git a/apps/esp32_lvgl/src/esp32_lvgl_arduino_app_runtime_access.cpp b/apps/esp32_lvgl/src/esp32_lvgl_arduino_app_runtime_access.cpp index ed5d69a9..eba8a289 100644 --- a/apps/esp32_lvgl/src/esp32_lvgl_arduino_app_runtime_access.cpp +++ b/apps/esp32_lvgl/src/esp32_lvgl_arduino_app_runtime_access.cpp @@ -102,15 +102,6 @@ void tick() return; } - // SdStore and the peer repository hydrate under an exclusive state lock. - // Do not enter the foreground lifecycle while that phase is active: - // updateCoreServices() flushes those stores and would repeatedly wait on - // the hydration lock, starving the display loop. - if (platform::esp::arduino_common::storage::hydration_active()) - { - return; - } - platform::esp::arduino_common::tickBoundLifecycle(); } diff --git a/apps/esp32_lvgl/src/esp32_lvgl_idf_app_facade_runtime.cpp b/apps/esp32_lvgl/src/esp32_lvgl_idf_app_facade_runtime.cpp index 791a6978..a55a0113 100644 --- a/apps/esp32_lvgl/src/esp32_lvgl_idf_app_facade_runtime.cpp +++ b/apps/esp32_lvgl/src/esp32_lvgl_idf_app_facade_runtime.cpp @@ -4,6 +4,7 @@ #include "app/app_config.h" #include "app/app_facade_access.h" #include "app/app_facades.h" +#include "app/config_persistence_runtime.h" #include "board/BoardBase.h" #include "chat/delivery/chat_delivery_event_port.h" #include "chat/delivery/chat_delivery_event_projector.h" @@ -17,6 +18,7 @@ #include "chat/ports/i_mesh_peer_directory_blob_store.h" #include "chat/usecase/chat_service.h" #include "chat/usecase/contact_service.h" +#include "esp_heap_caps.h" #include "esp_log.h" #include "esp_mac.h" #include "esp_random.h" @@ -63,6 +65,7 @@ #include #include #include +#include #include #include #endif @@ -82,7 +85,6 @@ constexpr const char* kIdfMeshPeersFile = "/mesh/peers.bin"; constexpr size_t kIdfReadChunkBytes = 256; constexpr uint32_t kIdfAppConfigMagic = 0x50344346UL; // P4CF constexpr uint16_t kIdfAppConfigVersion = 1; -constexpr uint32_t kIdfPeerDirectoryFlushIntervalMs = 5000UL; constexpr size_t kIdfMaxMeshPeerBlobBytes = 768U * 1024U; constexpr const char* kIdfTeamTag = "idf-team"; constexpr size_t kTeamAeadTagBytes = 16; @@ -98,7 +100,29 @@ struct IdfPersistedAppConfig app::AppConfig config{}; }; -IdfPersistedAppConfig s_config_blob_scratch{}; +IdfPersistedAppConfig* s_config_blob_scratch = nullptr; + +IdfPersistedAppConfig* ensureConfigBlobScratch() +{ + if (s_config_blob_scratch) + { + return s_config_blob_scratch; + } + + void* psram_storage = + heap_caps_malloc(sizeof(IdfPersistedAppConfig), + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (psram_storage) + { + s_config_blob_scratch = new (psram_storage) IdfPersistedAppConfig(); + } + else + { + s_config_blob_scratch = + new (std::nothrow) IdfPersistedAppConfig(); + } + return s_config_blob_scratch; +} uint32_t fnv1a32(const void* data, size_t len) { @@ -195,45 +219,56 @@ void normalizeIdfAppConfig(app::AppConfig& config) bool loadIdfAppConfig(app::AppConfig& out) { - std::vector blob; - if (!platform::ui::settings_store::get_blob(kIdfSettingsNs, kIdfConfigKey, blob)) + IdfPersistedAppConfig* scratch = ensureConfigBlobScratch(); + if (!scratch) + { + ESP_LOGE(kIdfConfigTag, "config scratch allocation failed"); + return false; + } + + std::size_t blob_size = 0U; + if (!platform::ui::settings_store::get_blob_into( + kIdfSettingsNs, + kIdfConfigKey, + scratch, + sizeof(*scratch), + &blob_size)) { return false; } - if (blob.size() != sizeof(IdfPersistedAppConfig)) + if (blob_size != sizeof(IdfPersistedAppConfig)) { ESP_LOGW(kIdfConfigTag, "load rejected size=%u expected=%u", - static_cast(blob.size()), + static_cast(blob_size), static_cast(sizeof(IdfPersistedAppConfig))); return false; } - std::memcpy(&s_config_blob_scratch, blob.data(), sizeof(s_config_blob_scratch)); - if (s_config_blob_scratch.magic != kIdfAppConfigMagic || - s_config_blob_scratch.version != kIdfAppConfigVersion || - s_config_blob_scratch.payload_size != sizeof(app::AppConfig)) + if (scratch->magic != kIdfAppConfigMagic || + scratch->version != kIdfAppConfigVersion || + scratch->payload_size != sizeof(app::AppConfig)) { ESP_LOGW(kIdfConfigTag, "load rejected magic=%08lx version=%u payload=%u", - static_cast(s_config_blob_scratch.magic), - static_cast(s_config_blob_scratch.version), - static_cast(s_config_blob_scratch.payload_size)); + static_cast(scratch->magic), + static_cast(scratch->version), + static_cast(scratch->payload_size)); return false; } const uint32_t checksum = - fnv1a32(&s_config_blob_scratch.config, sizeof(s_config_blob_scratch.config)); - if (checksum != s_config_blob_scratch.checksum) + fnv1a32(&scratch->config, sizeof(scratch->config)); + if (checksum != scratch->checksum) { ESP_LOGW(kIdfConfigTag, "load rejected checksum stored=%08lx actual=%08lx", - static_cast(s_config_blob_scratch.checksum), + static_cast(scratch->checksum), static_cast(checksum)); return false; } - out = s_config_blob_scratch.config; + out = scratch->config; normalizeIdfAppConfig(out); ESP_LOGI(kIdfConfigTag, "loaded app config proto=%u region=%u tx=%d", @@ -245,19 +280,26 @@ bool loadIdfAppConfig(app::AppConfig& out) bool saveIdfAppConfig(const app::AppConfig& config) { - s_config_blob_scratch = IdfPersistedAppConfig{}; - s_config_blob_scratch.magic = kIdfAppConfigMagic; - s_config_blob_scratch.version = kIdfAppConfigVersion; - s_config_blob_scratch.payload_size = static_cast(sizeof(app::AppConfig)); - s_config_blob_scratch.config = config; - s_config_blob_scratch.checksum = - fnv1a32(&s_config_blob_scratch.config, sizeof(s_config_blob_scratch.config)); + IdfPersistedAppConfig* scratch = ensureConfigBlobScratch(); + if (!scratch) + { + ESP_LOGE(kIdfConfigTag, "config scratch allocation failed"); + return false; + } + + *scratch = IdfPersistedAppConfig{}; + scratch->magic = kIdfAppConfigMagic; + scratch->version = kIdfAppConfigVersion; + scratch->payload_size = static_cast(sizeof(app::AppConfig)); + scratch->config = config; + scratch->checksum = + fnv1a32(&scratch->config, sizeof(scratch->config)); const bool ok = platform::ui::settings_store::put_blob( kIdfSettingsNs, kIdfConfigKey, - &s_config_blob_scratch, - sizeof(s_config_blob_scratch)); + scratch, + sizeof(*scratch)); ESP_LOGI(kIdfConfigTag, "save app config proto=%u region=%u tx=%d ok=%u", static_cast(config.mesh_protocol), @@ -436,6 +478,60 @@ class IdfSdMeshPeerDirectoryBlobStore final return chat::MeshPeerDirectoryBlobLoadResult::Loaded; } + chat::MeshPeerDirectoryBlobLoadResult loadBlobTo( + chat::IMeshPeerDirectoryBlobSink& sink) override + { + if (!platform::esp::idf_common::bsp_runtime::ensure_sdcard_ready()) + { + return chat::MeshPeerDirectoryBlobLoadResult::Unavailable; + } + + const std::string path = makeSdPath(kIdfMeshPeersFile); + if (!platform::esp::arduino_common::storage::sd_exists(path.c_str())) + { + return chat::MeshPeerDirectoryBlobLoadResult::Missing; + } + + platform::esp::arduino_common::storage::SdRuntimeFile file; + if (!file.open(path.c_str(), "rb")) + { + return chat::MeshPeerDirectoryBlobLoadResult::IoError; + } + const uint64_t file_size = file.size(); + if (file_size == 0U || file_size > kIdfMaxMeshPeerBlobBytes || + !file.seek(0)) + { + file.close(); + return chat::MeshPeerDirectoryBlobLoadResult::IoError; + } + + const std::size_t size = static_cast(file_size); + if (!sink.begin(size)) + { + file.close(); + return chat::MeshPeerDirectoryBlobLoadResult::IoError; + } + + uint8_t buffer[kIdfReadChunkBytes]; + std::size_t total_read = 0U; + while (total_read < size) + { + const std::size_t chunk = + std::min(kIdfReadChunkBytes, size - total_read); + const int read = file.read(buffer, chunk); + if (read < 0 || static_cast(read) != chunk || + !sink.write(buffer, chunk)) + { + file.close(); + return chat::MeshPeerDirectoryBlobLoadResult::IoError; + } + total_read += chunk; + } + file.close(); + return sink.finish() ? chat::MeshPeerDirectoryBlobLoadResult::Loaded + : chat::MeshPeerDirectoryBlobLoadResult::IoError; + } + bool saveBlob(const uint8_t* data, size_t len) override { if ((!data && len != 0) || len > kIdfMaxMeshPeerBlobBytes || @@ -931,12 +1027,12 @@ class IdfAppFacadeRuntime final : public app::IAppFacade mesh_peer_directory_.setAutoSaveEnabled(false); const chat::MeshPeerDirectoryStatus peer_directory_status = - mesh_peer_directory_.begin(); + mesh_peer_directory_.beginEmpty(); mesh_peer_directory_ready_ = peer_directory_status.succeeded(); platform::ui::reticulum_directory::bind_mesh_peer_directory( mesh_peer_directory_ready_ ? &mesh_peer_directory_ : nullptr); ESP_LOGI(kIdfStoreTag, - "mesh peer directory path=%s status=%u", + "mesh peer directory path=%s status=%u hydration=deferred", kIdfMeshPeersFile, static_cast(peer_directory_status.code)); @@ -945,6 +1041,13 @@ class IdfAppFacadeRuntime final : public app::IAppFacade return false; } + chat_store_ = createIdfChatStore(&deferred_chat_store_); + if (!chat_store_) + { + ESP_LOGE(kIdfStoreTag, "chat store allocation failed"); + return false; + } + const auto identity = platform::esp::boards::defaultIdentity(); if (config_.node_name[0] == '\0') { @@ -969,14 +1072,9 @@ class IdfAppFacadeRuntime final : public app::IAppFacade applyUserInfo(); applyNetworkLimits(); applyPrivacyConfig(); + config_persistence_runtime_.initialize(config_); contact_service_.begin(); - chat_store_ = createIdfChatStore(&deferred_chat_store_); - if (!chat_store_) - { - ESP_LOGE(kIdfStoreTag, "chat store allocation failed"); - return false; - } chat_service_.reset(new chat::ChatService(chat_model_, meshAdapter(), *chat_store_)); chat_service_->setDeliveryEventPort(&delivery_event_port_); chat_service_->setActiveProtocol(config_.mesh_protocol); @@ -1000,7 +1098,8 @@ class IdfAppFacadeRuntime final : public app::IAppFacade } deferred_storage_started_ = true; platform::esp::idf_common::storage::start_deferred_storage( - deferred_chat_store_); + deferred_chat_store_, + mesh_peer_directory_ready_ ? &mesh_peer_directory_ : nullptr); } bool startBackgroundTasks() @@ -1018,6 +1117,32 @@ class IdfAppFacadeRuntime final : public app::IAppFacade return background_tasks_started_; } + bool waitForInitialStorageHydration(uint32_t timeout_ms = 15000U) + { + const uint32_t started_ms = persistenceNowMs(); + bool ready = platform::esp::idf_common::storage:: + consume_hydration_ready(); + while (!ready && + platform::esp::idf_common::storage::hydration_active()) + { + vTaskDelay(pdMS_TO_TICKS(10U)); + ready = platform::esp::idf_common::storage:: + consume_hydration_ready(); + if (static_cast(persistenceNowMs() - started_ms) >= + timeout_ms) + { + return false; + } + } + if (!ready) + { + ready = platform::esp::idf_common::storage:: + consume_hydration_ready(); + } + mesh_peer_directory_hydrated_ = ready; + return ready; + } + [[deprecated("Use beginConfigEdit() for configuration writes")]] app::AppConfig& getConfig() override { return config_; } const app::AppConfig& getConfig() const override { return config_; } @@ -1030,19 +1155,29 @@ class IdfAppFacadeRuntime final : public app::IAppFacade } void saveConfig() override + { + saveConfig(app::AppConfigChangeSet::allPersisted()); + } + + void saveConfig(app::AppConfigChangeSet changes) override { normalizeIdfAppConfig(config_); if (chat_service_) { chat_service_->setActiveProtocol(config_.mesh_protocol); } - (void)saveIdfAppConfig(config_); - } - void saveConfig(app::AppConfigChangeSet changes) override - { - (void)changes; - saveConfig(); + const uint32_t now_ms = persistenceNowMs(); + const auto submission = config_persistence_runtime_.submit( + config_, + changes, + now_ms, + app::ConfigPersistenceUrgency::Debounced); + ESP_LOGI(kIdfConfigTag, + "config intent queued=%u generation=%lu changes=0x%08lx", + submission.queued ? 1U : 0U, + static_cast(submission.generation), + static_cast(submission.changes.bits())); } void applyMeshConfig() override @@ -1244,10 +1379,20 @@ class IdfAppFacadeRuntime final : public app::IAppFacade void updateCoreServices() override { + flushConfigPersistence(persistenceNowMs()); + if (!mesh_peer_directory_hydrated_ && + platform::esp::idf_common::storage::consume_hydration_ready()) + { + mesh_peer_directory_hydrated_ = true; + ESP_LOGI(kIdfStoreTag, "mesh peer directory hydration ready"); + } + if (::platform::ui::reticulum_groups::hasPending()) + { + (void)::platform::ui::reticulum_groups::flushPending(); + } platform::ui::tracker::poll(); chat_service_->processIncoming(); chat_service_->flushStore(); - flushPeerDirectoryIfDue(); if (team_service_) { team_service_->processIncoming(); @@ -1315,6 +1460,28 @@ class IdfAppFacadeRuntime final : public app::IAppFacade bool initialized() const { return initialized_; } private: + static uint32_t persistenceNowMs() + { + return static_cast(esp_timer_get_time() / 1000ULL); + } + + void flushConfigPersistence(uint32_t now_ms) + { + app::ConfigPersistenceWork work{}; + if (!config_persistence_runtime_.takeDue(now_ms, work) || + work.snapshot == nullptr) + { + return; + } + + const bool ok = saveIdfAppConfig(*work.snapshot); + config_persistence_runtime_.complete( + work.generation, + ok ? app::ConfigPersistenceResultKind::Completed + : app::ConfigPersistenceResultKind::IoError, + persistenceNowMs()); + } + static void commitConfigEdit(void* context, app::AppConfigChangeSet changes) { auto* self = static_cast(context); @@ -1485,22 +1652,6 @@ class IdfAppFacadeRuntime final : public app::IAppFacade return null_mesh_adapter_; } - void flushPeerDirectoryIfDue() - { - const uint32_t now_ms = static_cast(esp_timer_get_time() / 1000ULL); - if ((now_ms - last_peer_directory_flush_ms_) < - kIdfPeerDirectoryFlushIntervalMs) - { - return; - } - last_peer_directory_flush_ms_ = now_ms; - if (mesh_peer_directory_ready_ && - !mesh_peer_directory_.flush().succeeded()) - { - ESP_LOGW(kIdfStoreTag, "mesh peer directory flush failed"); - } - } - static bool isTeamRuntimeEvent(sys::EventType type) { return type == sys::EventType::TeamKick || @@ -1645,6 +1796,7 @@ class IdfAppFacadeRuntime final : public app::IAppFacade BoardBase* board_ = nullptr; LoraBoard* lora_board_ = nullptr; app::AppConfig config_{}; + app::ConfigPersistenceRuntime config_persistence_runtime_{}; IdfSdMeshPeerDirectoryBlobStore mesh_peer_directory_blob_store_{}; chat::MeshPeerDirectoryCore mesh_peer_directory_{mesh_peer_directory_blob_store_}; chat::contacts::ContactService contact_service_{mesh_peer_directory_}; @@ -1652,6 +1804,7 @@ class IdfAppFacadeRuntime final : public app::IAppFacade std::unique_ptr chat_store_{}; chat::SdStore* deferred_chat_store_ = nullptr; bool deferred_storage_started_ = false; + bool mesh_peer_directory_hydrated_ = false; IdfNullMeshAdapter null_mesh_adapter_{}; chat::MeshAdapterRouter mesh_router_{}; chat::IMeshAdapter* mesh_adapter_ = &null_mesh_adapter_; @@ -1673,10 +1826,27 @@ class IdfAppFacadeRuntime final : public app::IAppFacade chat::ui::IChatUiRuntime* chat_ui_runtime_ = nullptr; bool mesh_peer_directory_ready_ = false; bool background_tasks_started_ = false; - uint32_t last_peer_directory_flush_ms_ = 0; }; -IdfAppFacadeRuntime s_runtime{}; +IdfAppFacadeRuntime* s_runtime = nullptr; + +IdfAppFacadeRuntime* createRuntime() +{ + void* psram_storage = + heap_caps_malloc(sizeof(IdfAppFacadeRuntime), + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (psram_storage) + { + ESP_LOGI(kIdfConfigTag, + "app facade runtime allocated in PSRAM bytes=%u", + static_cast(sizeof(IdfAppFacadeRuntime))); + return new (psram_storage) IdfAppFacadeRuntime(); + } + + ESP_LOGW(kIdfConfigTag, + "app facade runtime PSRAM allocation failed; using internal heap"); + return new (std::nothrow) IdfAppFacadeRuntime(); +} #endif } // namespace @@ -1696,14 +1866,27 @@ bool initialize(const platform::esp::boards::AppContextInitHandles& handles, return true; } - if (!s_runtime.begin(*handles.board, handles.lora_board)) + if (s_runtime == nullptr) + { + s_runtime = createRuntime(); + } + if (s_runtime == nullptr || + !s_runtime->begin(*handles.board, handles.lora_board)) { ESP_LOGE(config.log_tag, "IDF AppFacade runtime initialization failed for %s", config.target_name); return false; } - app::bindAppFacade(s_runtime); - if (!s_runtime.startBackgroundTasks()) + app::bindAppFacade(*s_runtime); + s_runtime->startDeferredStorage(); + if (!s_runtime->waitForInitialStorageHydration()) + { + ESP_LOGE(config.log_tag, + "initial storage hydration did not reach a ready state for %s", + config.target_name); + return false; + } + if (!s_runtime->startBackgroundTasks()) { ESP_LOGE(config.log_tag, "IDF shared ESP background tasks unavailable for %s; radio TX/RX remains disabled", @@ -1712,9 +1895,10 @@ bool initialize(const platform::esp::boards::AppContextInitHandles& handles, ESP_LOGI(config.log_tag, "IDF AppFacade runtime bound for %s self=%08lX mesh_backend=%s", config.target_name, - static_cast(s_runtime.getSelfNodeId()), - s_runtime.getMeshAdapter() != nullptr && s_runtime.getMeshAdapter()->isReady() - ? chat::infra::meshProtocolName(s_runtime.getMeshProtocol()) + static_cast(s_runtime->getSelfNodeId()), + s_runtime->getMeshAdapter() != nullptr && + s_runtime->getMeshAdapter()->isReady() + ? chat::infra::meshProtocolName(s_runtime->getMeshProtocol()) : "not_ready"); return true; #else @@ -1727,7 +1911,7 @@ bool initialize(const platform::esp::boards::AppContextInitHandles& handles, bool isInitialized() { #if defined(ESP_PLATFORM) - return s_runtime.initialized(); + return s_runtime != nullptr && s_runtime->initialized(); #else return false; #endif @@ -1736,7 +1920,10 @@ bool isInitialized() void startDeferredStorage() { #if defined(ESP_PLATFORM) - s_runtime.startDeferredStorage(); + if (s_runtime != nullptr) + { + s_runtime->startDeferredStorage(); + } #endif } diff --git a/apps/esp32_lvgl/tests/esp32_lvgl_ui_lifecycle_contract_smoke.cpp b/apps/esp32_lvgl/tests/esp32_lvgl_ui_lifecycle_contract_smoke.cpp new file mode 100644 index 00000000..94b79008 --- /dev/null +++ b/apps/esp32_lvgl/tests/esp32_lvgl_ui_lifecycle_contract_smoke.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include + +namespace +{ + +[[noreturn]] void fail(const char* requirement) +{ + std::fprintf(stderr, "UI lifecycle contract failed: %s\n", requirement); + std::exit(EXIT_FAILURE); +} + +std::string read_file(const std::filesystem::path& path) +{ + std::ifstream stream(path, std::ios::binary); + if (!stream.is_open()) + { + fail("required source file must be readable"); + } + + std::ostringstream out; + out << stream.rdbuf(); + return out.str(); +} + +std::size_t position_of(const std::string& source, const char* token) +{ + const std::size_t position = source.find(token); + if (position == std::string::npos) + { + fail(token); + } + return position; +} + +std::string function_body(const std::string& source, + const char* begin, + const char* end) +{ + const std::size_t start = position_of(source, begin); + const std::size_t finish = position_of(source.substr(start), end) + start; + if (finish <= start) + { + fail("function body must have a positive range"); + } + return source.substr(start, finish - start); +} + +void require_before(const std::string& source, + const char* first, + const char* second) +{ + if (position_of(source, first) >= position_of(source, second)) + { + fail("required lifecycle ordering"); + } +} + +} // namespace + +int main(int argc, char** argv) +{ + if (argc != 2) + { + fail("repository root argument"); + } + + const std::filesystem::path repo_root = argv[1]; + const std::string app_runtime = read_file( + repo_root / "modules/ui_shared/src/ui/app_runtime.cpp"); + + const std::string show_menu_internal = function_body( + app_runtime, + "void show_menu_internal()", + "uint32_t child_count"); + if (show_menu_internal.find("ui_clear_active_app()") != std::string::npos) + { + fail("menu presentation must not clear the active app"); + } + + const std::string menu_show = function_body( + app_runtime, + "void menu_show()", + "AppScreen* ui_get_active_app()"); + require_before(menu_show, + "if (s_active_app != nullptr)", + "ui_request_exit_to_menu();"); + require_before(menu_show, + "ui_request_exit_to_menu();", + "show_menu_internal();"); + + const std::string exit_to_menu = function_body( + app_runtime, + "void exit_to_menu_timer_cb", + "void rebuild_active_app_timer_cb"); + const std::string no_main_screen_exit = function_body( + exit_to_menu, + "if (main_screen == nullptr)", + "lv_obj_t* parent ="); + require_before(no_main_screen_exit, + "app->exit(nullptr);", + "s_active_app = nullptr;"); + const std::size_t parent_exit = position_of(exit_to_menu, "app->exit(parent);"); + const std::string normal_exit = exit_to_menu.substr(parent_exit); + require_before(normal_exit, "app->exit(parent);", "s_active_app = nullptr;"); + require_before(normal_exit, "s_active_app = nullptr;", "show_menu_internal();"); + + const std::string gps_runtime = read_file( + repo_root / "modules/ui_shared/src/ui/screens/gps/gps_page_runtime.cpp"); + const std::string gps_exit = function_body( + gps_runtime, + "void exit(lv_obj_t* parent)", + "} // namespace gps::ui::runtime"); + require_before(gps_exit, + "::ui::widgets::map::destroy(s_map_runtime);", + "lv_obj_del(s_root);"); + require_before(gps_exit, "lv_obj_del(s_root);", "s_top_bar = {};"); + + return EXIT_SUCCESS; +} diff --git a/apps/linux_sim_shell/CMakeLists.txt b/apps/linux_sim_shell/CMakeLists.txt index 2faefb06..fba7ee29 100644 --- a/apps/linux_sim_shell/CMakeLists.txt +++ b/apps/linux_sim_shell/CMakeLists.txt @@ -336,6 +336,17 @@ if(BUILD_TESTING) add_test(NAME trailmate_lxmf_deferred_discovery_queue_smoke COMMAND trailmate_lxmf_deferred_discovery_queue_smoke) + add_executable(trailmate_storage_maintenance_state_machine_smoke + "${TRAIL_MATE_REPO_ROOT}/platform/esp/common/tests/test_storage_maintenance_state_machine.cpp") + target_include_directories(trailmate_storage_maintenance_state_machine_smoke + PRIVATE + "${TRAIL_MATE_REPO_ROOT}/platform/esp/common/include" + "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/include") + target_compile_features(trailmate_storage_maintenance_state_machine_smoke + PRIVATE cxx_std_17) + add_test(NAME trailmate_storage_maintenance_state_machine_smoke + COMMAND trailmate_storage_maintenance_state_machine_smoke) + add_executable(trailmate_protocol_chat_storage_v2_codec_smoke "${TRAIL_MATE_REPO_ROOT}/platform/esp/arduino_common/tests/test_protocol_chat_storage_v2_codec.cpp" "${TRAIL_MATE_REPO_ROOT}/platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp") @@ -819,6 +830,22 @@ if(BUILD_TESTING) PRIVATE cxx_std_17) add_test(NAME trailmate_app_config_change_detection_smoke COMMAND trailmate_app_config_change_detection_smoke) + + add_executable(trailmate_config_persistence_runtime_smoke + "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/tests/test_config_persistence_runtime.cpp" + "${TRAIL_MATE_REPO_ROOT}/modules/core_chat/src/infra/mesh_protocol_utils.cpp") + target_include_directories(trailmate_config_persistence_runtime_smoke + PRIVATE + "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/include" + "${TRAIL_MATE_REPO_ROOT}/modules/core_chat/include" + "${TRAIL_MATE_REPO_ROOT}/modules/core_gps/include") + target_compile_definitions(trailmate_config_persistence_runtime_smoke + PRIVATE TRAIL_MATE_LORA_TX_POWER_MAX_DBM=22) + target_compile_features(trailmate_config_persistence_runtime_smoke + PRIVATE cxx_std_17) + add_test(NAME trailmate_config_persistence_runtime_smoke + COMMAND trailmate_config_persistence_runtime_smoke) + add_executable(trailmate_screen_brightness_steps_smoke "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/tests/test_screen_brightness_steps.cpp") target_include_directories(trailmate_screen_brightness_steps_smoke @@ -845,16 +872,6 @@ if(BUILD_TESTING) COMMAND trailmate_reticulum_call_runtime_smoke) endif() - add_executable(trailmate_storage_event_runtime_smoke - "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/tests/test_storage_event_runtime.cpp") - target_include_directories(trailmate_storage_event_runtime_smoke - PRIVATE - "${TRAIL_MATE_REPO_ROOT}/modules/core_sys/include") - target_compile_features(trailmate_storage_event_runtime_smoke - PRIVATE cxx_std_17) - add_test(NAME trailmate_storage_event_runtime_smoke - COMMAND trailmate_storage_event_runtime_smoke) - add_executable(trailmate_map_tile_async_runtime_smoke "${TRAIL_MATE_REPO_ROOT}/modules/ui_map_runtime/tests/test_map_tile_async_runtime.cpp" "${TRAIL_MATE_REPO_ROOT}/modules/ui_map_runtime/src/map_tiles/map_tile_async_runtime.cpp" diff --git a/builds/esp_idf/ESP_IDF_COMPONENT_SOURCES.cmake b/builds/esp_idf/ESP_IDF_COMPONENT_SOURCES.cmake index 02fd4dd9..e0c1724e 100644 --- a/builds/esp_idf/ESP_IDF_COMPONENT_SOURCES.cmake +++ b/builds/esp_idf/ESP_IDF_COMPONENT_SOURCES.cmake @@ -95,6 +95,7 @@ set(TRAILMATE_ESP_IDF_CORE_CHAT_SOURCES "${TRAILMATE_ROOT}/modules/core_chat/src/infra/reticulum/reticulum_wire.cpp" "${TRAILMATE_ROOT}/modules/core_chat/src/infra/rnode/rnode_packet_wire.cpp" "${TRAILMATE_ROOT}/modules/core_chat/src/infra/store/ram_store.cpp" + "${TRAILMATE_ROOT}/modules/core_chat/src/read/chat_read_state_ledger.cpp" "${TRAILMATE_ROOT}/modules/core_chat/src/runtime/meshtastic_position_core.cpp" "${TRAILMATE_ROOT}/modules/core_chat/src/runtime/meshtastic_self_announcement_core.cpp" "${TRAILMATE_ROOT}/modules/core_chat/src/runtime/meshtastic_waypoint_core.cpp" @@ -399,6 +400,9 @@ set(TRAILMATE_ESP_IDF_PLATFORM_COMMON_SOURCES "${TRAILMATE_ROOT}/platform/esp/idf_common/src/flash_storage_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/idf_common/src/sd_card_runtime_sdfat_adapter.cpp" "${TRAILMATE_ROOT}/platform/esp/idf_common/src/storage_runtime.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp" "${TRAILMATE_ROOT}/platform/esp/idf_common/src/chat_blob_store_io.cpp" "${TRAILMATE_ROOT}/platform/esp/idf_common/src/debug/sd_coredump_export.cpp" @@ -446,11 +450,13 @@ set(TRAILMATE_ESP_IDF_PLATFORM_COMMON_SOURCES "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter_call.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_announce_ingestor.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_announce_scheduler.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_delivery_attempt_ledger.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_delivery_notifier.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_delivery_planner.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_delivery_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_destination_registry.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_deferred_discovery_queue.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_identity.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_link_manager.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_link_runtime.cpp" @@ -465,6 +471,8 @@ set(TRAILMATE_ESP_IDF_PLATFORM_COMMON_SOURCES "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_propagation_stamp_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_propagation_service_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_resource_runtime.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_rx_telemetry.cpp" + "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_runtime_budget.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_transport_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/reticulum/reticulum_adapter.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/chat/infra/reticulum/reticulum_interfaces.cpp" @@ -480,6 +488,7 @@ set(TRAILMATE_ESP_IDF_PLATFORM_COMMON_SOURCES "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/ui/widgets/map/map_tiles.cpp" "${TRAILMATE_ROOT}/platform/esp/arduino_common/src/walkie/walkie_service.cpp" "${TRAILMATE_ROOT}/platform/esp/common/src/call_acoustic_echo_guard.cpp" + "${TRAILMATE_ROOT}/platform/esp/common/src/memory_budget.cpp" "${TRAILMATE_ROOT}/platform/esp/common/src/reticulum_call_audio_engine.cpp" "${TRAILMATE_ROOT}/platform/esp/idf_common/src/platform_ui_wireless_companion_runtime.cpp" "${TRAILMATE_ROOT}/platform/esp/radio/idf_lora_radio_pump.cpp" diff --git a/docs/config_persistence_architecture.md b/docs/config_persistence_architecture.md index 832a0b81..37b460a8 100644 --- a/docs/config_persistence_architecture.md +++ b/docs/config_persistence_architecture.md @@ -1,6 +1,6 @@ # Configuration Persistence Architecture -Status date: 2026-07-26 +Status date: 2026-07-27 This document describes only the technical mechanism for persisting `AppConfig`. Business specifications such as map behavior, protocol behavior, language packs, @@ -49,26 +49,41 @@ request a platform namespace such as `chat`, `settings`, or `gps`. ## ESP Persistence Flow 1. Startup loads `AppConfig` from the platform backend. -2. `AppContext` copies the loaded config into its PSRAM-backed save baseline. +2. `AppContext` initializes `ConfigPersistenceRuntime` with the loaded config as + its persisted baseline. The runtime's snapshots live with the PSRAM-backed + application context. 3. A writer opens `beginConfigEdit()`. The returned small token holds the configuration mutex while the caller makes one coherent update. 4. `commit(changes)` publishes the edit and replaces the pending snapshot with the current configuration. The token destructor cancels the edit without publishing it. -5. The async save worker debounces requests and persists the latest queued - snapshot. A later edit can replace an older pending snapshot, including by - reverting to the last persisted value. +5. `ConfigPersistenceRuntime` owns debounce, pending/in-flight snapshots, + generations, and retry state. The platform execution shell only takes an + immutable work view from the runtime and invokes the adapter. A later edit + can replace an older pending snapshot, including by reverting to the last + persisted value. 6. The ESP backend maps domains to Preferences/NVS sections and writes only the required sections. 7. On success, the in-flight snapshot becomes the new save baseline. Any pending snapshot is reconciled against that new baseline before another - write is allowed. -8. On failure, the baseline is invalidated and retry conservatively persists all - persisted domains until a save succeeds. + write is allowed. If the latest in-memory value reverted while the old + write was in flight, the runtime schedules that latest value rather than + replaying the old payload. +8. On failure, the baseline remains the last successfully persisted snapshot. + The failed payload's domains remain dirty and are retried; a newer pending + snapshot merges those domains with its own changes without escalating to a + full-domain rewrite. -The queue still uses PSRAM-backed `AppContext` member snapshots to keep the -worker's save input stable. It must not create large `AppConfig`, protocol -config, or byte-buffer automatic locals on ESP task stacks. +The long-lived snapshots are PSRAM-backed on ESP targets. The execution shell +must not create large `AppConfig`, protocol config, or byte-buffer automatic +locals on ESP task stacks. Arduino runs the shell from the application service +owner; IDF runs it from the IDF application owner; neither path creates a +dedicated configuration-save task. + +`ConfigPersistenceRuntime` is deterministic and platform-neutral. It shares +`sys::PersistenceGeneration` and `sys::PersistenceResultKind` with the +storage-maintenance foundation. FreeRTOS queues, task handles, Preferences, +NVS namespaces, and platform retry logging remain outside this module. ## ESP Domain-To-Store Mapping @@ -110,10 +125,11 @@ the remaining callers migrate. The scoped `saveConfig(AppConfigChangeSet)` contract is implemented explicitly by every facade. ESP Arduino uses the domains to select Preferences sections. -IDF, Linux, and nRF52 currently use full-blob or full-snapshot persistence, so -they intentionally expand the request to a complete save until their adapters -gain section-level writers. This fallback is visible in each implementation and -is not a silent default in the shared interface. +IDF and Linux submit an explicit full-snapshot adapter request to their +`ConfigPersistenceRuntime` and execute it from their application service tick. +nRF52 submits the same semantic request to its board-owned deferred settings +store, which currently persists a complete snapshot. These are platform +adapter choices, not alternate dirty-state machines. ## Non-Goals @@ -160,3 +176,73 @@ mechanism, not to the storage adapter. Configuration persistence logs should show the change-set and concrete store sections touched. A map source change should touch the `settings` section only. + +## Config Persistence Runtime + +The runtime is the sole owner of configuration persistence state. Its state is: + +```text +Idle + -> Debouncing + -> InFlight + -> Idle + +InFlight + -> Debouncing (a newer snapshot arrived) + -> Backoff (the adapter failed) + +Backoff + -> InFlight (retrying the latest immutable snapshot) +``` + +The runtime has three distinct snapshots: + +```text +baseline = last successfully persisted snapshot +pending = newest requested snapshot not yet started +active = immutable snapshot currently passed to the adapter +``` + +The adapter must never receive `AppConfig&` that can be changed by an edit +while the write is in progress. A completion is valid only when its generation +matches `active`; stale completions are ignored. A failed write keeps `baseline` +at the last successful snapshot and retries `active_changes`; any newer pending +changes are merged with those failed domains. A successful write reconciles +`pending` against the newly persisted baseline. + +The platform execution shells are intentionally thin: + +| Platform | Intent submission | Persistence owner execution | +| --- | --- | --- | +| ESP Arduino | `AppContext::beginConfigEdit()` / `requestSaveConfig()` | `AppContext::updateCoreServices()` calls `takeDue()` and the Preferences adapter | +| ESP IDF | `IdfAppFacadeRuntime::beginConfigEdit()` / `saveConfig()` | `IdfAppFacadeRuntime::updateCoreServices()` calls `takeDue()` and the full-blob adapter | +| Linux | `LinuxAppServices::beginConfigEdit()` / `saveConfig()` | `LinuxAppServices::tick()` calls `takeDue()` and the settings-store adapter | +| nRF52 | `AppConfigChangeSet` facade request | Board-owned deferred settings store; full snapshot is the explicit platform fallback | + +No caller performs routine configuration I/O in the callback that submits the +intent. Critical protocol-switch persistence may still use an immediate +platform path where the board contract requires it, but that path is separate +from routine debounce/retry persistence. + +Reticulum groups use the same ownership rule without pretending to be fields +owned by the AppConfig writer. Contacts edits a local candidate array, commits +the necessary runtime mirror through `AppConfigEdit`, and submits the candidate +to `platform::ui::reticulum_groups`. The Reticulum group owner performs the +physical SD write from the platform service tick. The legacy `save()` method is +retained only as the physical backend operation and is not a UI submission API. + +Mesh peer directory hydration follows the same split on IDF: the facade binds +an empty, valid directory during construction, and the storage maintenance +owner hydrates its immutable blob during `Hydrate`. The IDF startup sequence +waits for the owner readiness event before starting protocol background tasks +or exposing the operational facade, so no consumer can race the directory +commit. This keeps SD file I/O out of facade initialization while preserving +the old invariant that protocol tasks only see a hydrated directory. + +The SD maintenance adapter is a different contract from configuration +persistence. It owns hydration, bounded journal work, compaction, and the +repository persistence lease. Its `Persist` operation drains immutable +repository deltas in bounded batches; its `Compact` operation is demand-driven +by reset intents or journal growth and is admitted only after the idle gate is +stable. It does not receive configuration payloads and must not be used as a +generic configuration worker. diff --git a/docs/specification/RUNTIME_CONCURRENCY_SPEC.md b/docs/specification/RUNTIME_CONCURRENCY_SPEC.md index 422d6391..fd62708c 100644 --- a/docs/specification/RUNTIME_CONCURRENCY_SPEC.md +++ b/docs/specification/RUNTIME_CONCURRENCY_SPEC.md @@ -208,11 +208,12 @@ Valid slow-work owners include: ```text command worker -storage worker protocol runtime worker map tile worker track storage worker -persistence worker +StorageMaintenanceRuntime +ConfigPersistenceRuntime +domain store owner declared platform service task ``` @@ -239,3 +240,147 @@ UI event drain The simulator must assert that UI owner code does not execute blocking storage/device-I/O/filesystem calls and that background code does not execute concrete renderer calls. + +## Storage Maintenance Owner + +SD-backed maintenance is one active object, not one task per operation. The +owner task is created once, consumes a bounded command queue, and remains +blocked after maintenance reaches `Done`. A FreeRTOS task handle is an +implementation detail and must not be used as the business state. + +The maintenance owner publishes an immutable `StorageRuntimeSnapshot` with a +monotonic `StorageOperationGeneration`. Every operation completion carries the +operation and generation that produced it. A completion for an older +generation is ignored and cannot move the current state backward. + +The maintenance state machine is limited to: + +```text +Dormant + -> WaitingStartupGate + -> Hydrating(generation) + -> Ready + -> WaitingIdle + -> Persisting(generation) + -> Compacting(generation) + -> Done + +Hydrating / Persisting / Compacting -> Backoff -> the same operation + +Normal hydration, persistence, and compaction completion returns to `Ready`. +`Done` is reserved for an explicit stop or an exhausted retry policy. The +owner receives a latest-state maintenance demand rather than an unbounded +timer: pending immutable deltas request `Persist`, while reset intents or +compaction thresholds request `Compact`. When both are pending, active +foreground work is allowed to drain bounded persistence steps first; after +the idle gate is stable, compaction takes precedence because its snapshot +already includes the newest in-memory projections. +``` + +Foreground contexts may enqueue ticks and consume snapshots or one-shot +hydration-ready events. They may not mutate maintenance state or call the +maintenance backend directly. The owner sees only `ISemanticStorageAdapter`; +SPI tokens, chip-select pins, filesystem sessions, mutexes, and task handles +must remain behind that adapter. + +Maintenance readiness must not stop the foreground UI, input, or event loop. +Foreground presentation reads observe a semantic not-ready result while the +authoritative projection is being installed, and retain a retryable view +state instead of treating an empty projection as valid data. Device sessions, +bus ownership, and lock details remain inside the storage adapters. + +The adapter contract is intentionally incremental: + +```text +begin(operation, generation) +step(operation, generation, budget) +cancelAtStepBoundary(operation, generation) +``` + +`StorageOperationBudget` is expressed in logical work items. A backend must +perform filesystem/device work outside its logical state lock, then take a +short bounded lock only to apply the decoded result or commit a generation. +For SD-backed repositories, a separate persistence lease serializes the +multi-step physical writer; foreground mutations enqueue immutable projections +or reset intents while that lease is held. + +Retry semantics are part of the adapter contract, not an implementation +convention: + +- `begin(operation, generation)` with a new generation initializes a new + operation cursor. +- `begin(operation, generation)` with the same non-terminal operation and + generation resumes the existing cursor after `RetryLater`, `StateBusy`, + `DeviceUnavailable`, or another retryable result. +- The repository's logical maintenance-ownership lease remains held across + those retries. The physical filesystem/device transaction lease may be + released at the end of a bounded step, so the next `begin` resumes under + the existing ownership without competing with foreground persistence. +- Exhausting the owner's retry policy is a terminal cancellation boundary. + Before publishing `Done`, the owner must call + `cancelAtStepBoundary(operation, generation)` so the adapter discards the + cursor and releases every retained logical maintenance lease. +- A different generation, an explicit cancellation, or a terminal backend + phase is the only valid reason to discard that cursor. + +The three concurrency boundaries remain distinct: + +```text +logical repository state lock +physical filesystem/device session +shared SPI transaction +``` + +Hydration and compaction must acquire the logical state lock with a bounded +wait, release it between protocol/journal units where the backend permits, and +never use `portMAX_DELAY` across SD I/O. Shared-SPI boards must wait for a real +display transaction completion before hydration. SDMMC or independent-SPI +boards must use an already-satisfied gate and must not inherit a display delay. + +The board runtime exposes storage topology capabilities rather than making +the storage runtime maintain a board-name macro list. The capability +distinguishes shared-display SPI, dedicated SPI, and SDMMC. + +## Configuration Persistence Owner + +Configuration persistence is a separate owner from SD maintenance. It owns +configuration dirty state, debounce, immutable payloads, generation tracking, +critical flush requests, and retry decisions. It does not own SD hydration, +compaction, map tile reads, or repository maintenance. + +The configuration owner exposes only semantic work: + +```text +submit(snapshot, change_set, urgency) +takeDue(now_ms, work) +complete(generation, result) +``` + +The platform execution shell owns only task scheduling and adapter invocation. +It must not reconstruct the pending/in-flight state machine in a second set of +flags. `AppContext` facade methods are compatibility entry points; they submit +an edit or a persistence intent to the configuration owner. + +The owner keeps three snapshots: + +```text +baseline = last successful persisted snapshot +pending = newest requested snapshot not yet started +active = immutable snapshot currently being persisted +``` + +The following invariants are mandatory: + +- `active` is never mutated while an adapter call is in flight. +- A completion with a stale generation cannot change owner state. +- A successful completion advances `baseline` before reconciling `pending`. +- If the latest configuration reverted while an older write was in flight, the + latest value remains pending until it is persisted or explicitly cancelled. +- A failed completion invalidates `baseline` and retries conservatively. +- Configuration snapshots and protocol payloads must not be automatic locals on + ESP task stacks. + +Storage maintenance and configuration persistence may share result kinds, +generation semantics, retry policy conventions, and semantic adapter patterns, +but they must not share one giant state machine or business snapshot. Their +owners, payloads, and lifecycle remain independent. diff --git a/docs/specification/UI_STORAGE_EVENT_RUNTIME_DESIGN_SPEC.md b/docs/specification/UI_STORAGE_EVENT_RUNTIME_DESIGN_SPEC.md index 2ce9bb54..026a155d 100644 --- a/docs/specification/UI_STORAGE_EVENT_RUNTIME_DESIGN_SPEC.md +++ b/docs/specification/UI_STORAGE_EVENT_RUNTIME_DESIGN_SPEC.md @@ -14,6 +14,19 @@ The physical shared-device mechanism is a technical concern owned by `docs/spi_bus_architecture.md`. This document defines only UI/runtime ownership and semantic storage behavior. +There is no generic `sys::runtime::PersistenceRuntime` in production. Storage +responsibilities are deliberately split: + +- `StorageMaintenanceRuntime` owns SD-backed hydration, compaction, startup + gating, and maintenance retry state. +- `ConfigPersistenceRuntime` owns application-config dirty tracking, debounce, + immutable payloads, generations, critical flush, and config retry state. +- Domain stores own their domain state and domain-specific snapshot policy. + +These owners use semantic storage adapters. They do not expose SPI tokens, +chip-select lines, filesystem sessions, mutexes, or RTOS task handles to +callers, and neither runtime is a universal storage service. + ## Problem Statement Trail Mate currently has several paths where renderer-owned code, runtime @@ -113,7 +126,8 @@ Examples: - `MapTileWorker` - `TrackStorageWorker` -- `PersistenceWorker` +- `StorageMaintenanceRuntime` for SD-backed maintenance +- `ConfigPersistenceRuntime` for application-config persistence - `ProtocolRuntimeWorker` - `FeedbackDispatchWorker` when a target needs one @@ -332,7 +346,8 @@ flowchart TB subgraph Workers["Active Objects"] TileWorker["MapTileWorker"] TrackWorker["TrackStorageWorker"] - PersistWorker["PersistenceWorker"] + ConfigPersist["ConfigPersistenceRuntime"] + Maintenance["StorageMaintenanceRuntime"] end subgraph Platform["Platform Adapters"] @@ -347,15 +362,18 @@ flowchart TB Facade --> Policies Commands --> TileWorker Commands --> TrackWorker - Commands --> PersistWorker + Commands --> ConfigPersist + Commands --> Maintenance TileWorker --> DeviceIo TrackWorker --> DeviceIo - PersistWorker --> DeviceIo + ConfigPersist --> DeviceIo + Maintenance --> DeviceIo DeviceIo --> Storage TileWorker --> Decode TileWorker --> Events TrackWorker --> Events - PersistWorker --> Events + ConfigPersist --> Events + Maintenance --> Events Events --> State Events --> UiDrain UiDrain --> Renderer @@ -454,7 +472,8 @@ flowchart TB subgraph WorkerContext["Worker Context"] TileWorker["Tile worker"] TrackWorker["Track worker"] - PersistWorker["Persistence worker"] + ConfigPersist["Config persistence runtime"] + Maintenance["Storage maintenance runtime"] ProtocolWorker["Protocol worker"] end @@ -481,11 +500,13 @@ flowchart TB Facades --> Policies Commands --> TileWorker Commands --> TrackWorker - Commands --> PersistWorker + Commands --> ConfigPersist + Commands --> Maintenance Commands --> ProtocolWorker TileWorker --> StorageAdapter TrackWorker --> StorageAdapter - PersistWorker --> StorageAdapter + ConfigPersist --> StorageAdapter + Maintenance --> StorageAdapter TileWorker --> DecodeAdapter ProtocolWorker --> RadioAdapter WorkerContext --> Events @@ -528,7 +549,8 @@ flowchart LR CommandPump["Command pump"] TileWorker["Tile worker"] TrackWorker["Track worker"] - PersistWorker["Persistence worker"] + ConfigPersist["Config persistence runtime"] + Maintenance["Storage maintenance runtime"] end subgraph ResourceOwner["Resource owner"] @@ -547,10 +569,12 @@ flowchart LR ProtocolIntent --> CommandPump CommandPump --> TileWorker CommandPump --> TrackWorker - CommandPump --> PersistWorker + CommandPump --> ConfigPersist + CommandPump --> Maintenance TileWorker --> DeviceIo TrackWorker --> DeviceIo - PersistWorker --> DeviceIo + ConfigPersist --> DeviceIo + Maintenance --> DeviceIo DeviceIo --> Storage TileWorker --> Decode TileWorker --> UiDrain @@ -563,9 +587,9 @@ one physical thread, the same ownership model still applies cooperatively: slow work must be incremental, budgeted, and represented as commands/events rather than blocking UI execution. -The storage worker is not the device owner. It owns command state and calls a -device storage service. The service owns the physical transaction mechanism -described in `docs/spi_bus_architecture.md`. +The storage runtimes are not the device owner. They own operation state and +call semantic device services. Those services own the physical transaction +mechanism described in `docs/spi_bus_architecture.md`. ## Device I/O Boundary @@ -766,70 +790,71 @@ classDiagram TrackRuntime --> TrackEvent ``` -## UML Persistence Class Model +## UML Configuration Persistence Class Model ```mermaid classDiagram - class PersistenceRuntime { - +markDirty(store_key) - +requestSave(store_key, policy) - +handle(event) + class ConfigPersistenceRuntime { + +initialize(baseline) + +submit(desired, changes, now_ms, urgency) + +takeDue(now_ms, work) + +complete(generation, result, now_ms) } - class PersistenceCommand { - +command_id - +store_key - +policy - +deadline_ms + class AppConfigEdit { + +config() + +commit(change_set) + +cancel() } - class PersistencePolicy { - <> - DebouncedSave - BatchSave - ImmediateCriticalSave - DropDuplicateSave + class AppConfigChangeSet { + +domains + +generation } - class DirtyStoreRegistry { - +markDirty(store_key) - +takeDue(now_ms) - +hasPending(store_key) + class ConfigPersistenceWork { + +snapshot + +changes + +generation } - class PersistenceWorker { - +submit(command) - +tick(now_ms) + class PersistenceGeneration { + <> + +value } - class IStoreSnapshotProvider { + class PersistenceResultKind { + <> + Completed + InProgress + StateBusy + DeviceUnavailable + RetryLater + IoError + Cancelled + StaleGeneration + } + + class ISemanticStorageAdapter { <> - +snapshot(store_key) + +begin(operation, generation) + +step(operation, generation, budget) + +cancelAtStepBoundary(operation, generation) } - class IStoreStorageAdapter { - <> - +write(store_key, bytes) - +read(store_key) - } - - class PersistenceEvent { - +kind - +store_key - +command_id - +error - } - - PersistenceRuntime o-- DirtyStoreRegistry - PersistenceRuntime --> PersistenceCommand - PersistenceCommand --> PersistencePolicy - PersistenceWorker --> PersistenceCommand - PersistenceWorker o-- IStoreSnapshotProvider - PersistenceWorker o-- IStoreStorageAdapter - PersistenceWorker --> PersistenceEvent - PersistenceRuntime --> PersistenceEvent + ConfigPersistenceRuntime o-- ConfigPersistenceWork + ConfigPersistenceRuntime --> AppConfigChangeSet + ConfigPersistenceWork --> PersistenceGeneration + ConfigPersistenceRuntime --> PersistenceResultKind + AppConfigEdit --> AppConfigChangeSet ``` +`ConfigPersistenceRuntime` is the only owner of configuration persistence +state. `AppConfigEdit` creates an immutable intent from a caller-owned edit; +the runtime snapshots the authoritative configuration and owns the pending and +in-flight payloads. Domain stores such as contacts, peers, maps, and tracks do +not enter this model merely because they also use SD or flash. + ## UML Feedback Class Model ```mermaid @@ -1021,29 +1046,30 @@ sequenceDiagram Events->>UI: drain UI-safe events ``` -### NodeInfo Storm With Debounced Persistence +### NodeInfo Storm With Domain-Owned Maintenance ```mermaid sequenceDiagram participant Radio as Radio Task participant Runtime as Protocol Runtime participant Contacts as Contact/Node Runtime - participant Persist as PersistenceRuntime - participant Worker as PersistenceWorker - participant Store as Storage Adapter + participant StoreOwner as Contact/Node Store Owner + participant Maintenance as StorageMaintenanceRuntime + participant Adapter as Semantic Storage Adapter participant UI as UI Owner loop many packets Radio->>Runtime: incoming NodeInfo/Position Runtime->>Contacts: update in-memory projection - Contacts->>Persist: markDirty(nodes) + Contacts->>StoreOwner: apply domain update Contacts->>UI: publish projection update end - Persist->>Persist: coalesce dirty notifications - Persist->>Worker: enqueue SaveStoreCommand after debounce - Worker->>Store: write snapshot - Worker->>Persist: PersistenceSaved/PersistenceFailed - Persist->>UI: optional storage feedback event + StoreOwner->>StoreOwner: coalesce domain changes + StoreOwner->>Maintenance: request maintenance when policy is due + Maintenance->>Adapter: hydrate or compact store + Adapter-->>Maintenance: result(generation) + Maintenance-->>StoreOwner: completion or retry state + StoreOwner->>UI: optional storage feedback event ``` ### Chat Send Result While Page Changes @@ -1114,11 +1140,11 @@ classDiagram +drain() } - class FakeStorageBackend { + class FakeSemanticStorageAdapter { +scriptDelay(operation, ms) +scriptFailure(operation, error) - +read(request) - +write(request) + +execute(operation) + +result() } class FakeDeviceIo { @@ -1141,7 +1167,7 @@ classDiagram RuntimeHarness o-- FakeClock RuntimeHarness o-- FakeCommandQueue RuntimeHarness o-- FakeEventBus - RuntimeHarness o-- FakeStorageBackend + RuntimeHarness o-- FakeSemanticStorageAdapter RuntimeHarness o-- FakeDeviceIo RuntimeHarness o-- FakeUiOwner RuntimeHarness o-- FakeFeedbackPresenter @@ -1284,28 +1310,39 @@ Mandatory behavior: - The runtime state machine owns `Idle`, `Starting`, `Recording`, `Flushing`, `Stopping`, `Stopped`, `Error`, and `Recovering`. -## Persistence Runtime Design +## Configuration Persistence Design -Node/contact/config persistence must be decoupled from event dispatch. +Configuration persistence must be decoupled from event dispatch. Node/contact, +map, and track persistence remain domain-owned and do not share this runtime's +business state. ```text -Runtime event updates in-memory state. -Persistence intent is recorded. -PersistenceWorker batches/debounces writes. -Persistence result is published as an event. +Caller creates an AppConfigEdit. +The edit commits an AppConfigChangeSet. +ConfigPersistenceRuntime snapshots the authoritative configuration. +The runtime debounces, writes an immutable payload, and publishes the result. ``` -Required strategies: +Required configuration semantics: -- `DebouncedSave` for node/contact store updates. -- `ImmediateCriticalSave` only for explicit user settings or shutdown-critical - state. -- `BatchSave` for high-frequency updates. -- `DropDuplicateSave` for repeated dirty notifications while one save is - already pending. +- Debounce ordinary changes and coalesce them by configuration generation. +- Use an immediate critical flush only for explicit user settings or + shutdown-critical state. +- Keep pending and in-flight payloads immutable and independently owned. +- Retry the failed generation without overwriting a newer generation. +- Treat stale completions as observations, never as permission to mutate the + current runtime state. -Event dispatch may mark a store dirty. It must not synchronously write storage -from the UI owner context. +Event dispatch may submit a configuration intent. It must not synchronously +write storage from the UI owner context. + +## Domain Store Maintenance Design + +Domain stores own their state and decide when their snapshot is durable. The +maintenance runtime provides the shared lifecycle for SD-backed hydration, +compaction, startup gating, and retry, but it does not become a universal +`save(store_key, bytes)` service. A domain owner submits a semantic operation +and consumes a completion tagged with the corresponding generation. ## Feedback Runtime Design @@ -1382,7 +1419,7 @@ The simulator must provide: | `FakeUiThread` | records UI ticks and asserts no blocking operation runs on UI | | `FakeEventBus` | publishes and drains runtime events deterministically | | `FakeCommandQueue` | bounded queue, priorities, cancellation, dedupe | -| `FakeStorageBackend` | scripted read/write/list/flush delay and failure | +| `FakeSemanticStorageAdapter` | scripted semantic storage results, delay, and failure | | `FakeDeviceIo` | scripted device result, delay, and diagnostic outcome | | `FakeMapTileWorker` | completes tile commands in controlled order | | `FakeTrackStorageWorker` | batches points and emits track events | @@ -1545,7 +1582,8 @@ The burn-down should proceed in slices that each leave the system shippable. 3. Move map tile file access out of LVGL timer/input paths. 4. Move track start/stop/list/append/flush into an asynchronous track storage worker. -5. Move node/contact persistence to a debounced persistence worker. +5. Consolidate node/contact maintenance under domain store owners and + `StorageMaintenanceRuntime`; keep it separate from configuration persistence. 6. Replace direct hardware access in UI-facing code with device service calls. 7. Burn down adapter-owned business decisions and route them through shared runtimes/facades. diff --git a/docs/spi_bus_architecture.md b/docs/spi_bus_architecture.md index b0be6a8b..adcf5e8a 100644 --- a/docs/spi_bus_architecture.md +++ b/docs/spi_bus_architecture.md @@ -29,6 +29,12 @@ The current board profiles are intentionally different: may have no coordinator, one coordinator, or several independent coordinators; the business layer remains unaware of that topology. +The Arduino storage runtime consumes `BoardStorageCapabilities` from the board +runtime. Its `StorageBusTopology` values are `SharedDisplaySpi`, +`DedicatedSpi`, and `Sdmmc`; the storage worker does not maintain a second +board-name mapping. A shared-display board selects the display-transaction +startup gate, while `DedicatedSpi` and `Sdmmc` select the immediate gate. + ## Why the old model failed The old implementation had one physical mutex but several independent policy @@ -265,9 +271,11 @@ The display flush API returns a transaction result: - `Failed`: the transaction started but the driver reported failure; the caller completes the failure path and requests a full redraw. -`lv_display_flush_ready()` is called only after `Completed` or after the -explicit recovery path has recorded a dropped/invalidated frame. A lock timeout -must not be silently presented as success. +`lv_display_flush_ready()` is called only after `Completed`. When acquisition +returns `Busy` or `Unavailable`, the display adapter retains the exact area and +pixel-buffer ownership, and the LVGL flush-wait callback retries that transfer. +The buffer must not be reused and the flush must not be completed while the +pixels remain unsent. A lock timeout must not be silently presented as success. The first boot frame and the first wake redraw use the same path as normal frames. A completed display transaction means only that the coordinator granted @@ -306,9 +314,15 @@ The SD adapter has two distinct scopes: the SD device adapter and is invisible to business code. - The physical shared-SPI ownership is acquired by the SdFat driver at its `activate`/`deactivate` transaction boundary. On shared-SPI boards, payload - reads and writes are sliced to at most one 512-byte sector so the display or - radio can be granted between physical transactions. On SDMMC or independent - buses, this hook is not used. + reads and writes are sliced to at most one 512-byte sector where the SdFat + driver permits a transaction boundary. On SDMMC or independent buses, this + hook is not used. +- High-level SdFat metadata calls such as `open`, `exists`, directory + traversal, and `close` are not automatically preemptible. The coordinator + cannot interrupt an active SD command safely. Device adapters must therefore + measure these holds, keep them out of latency-critical interaction windows + where possible, and never claim a hard hold budget that the underlying + driver does not enforce. The logical session may remain open while an interactive read-only `FsFile` object is alive, but it must not hold the physical coordinator across sectors. @@ -352,15 +366,16 @@ flush requested -> lv_display_flush_ready() ``` -If the coordinator cannot grant the frame before the bounded retry budget: +If the coordinator cannot grant the frame on the first attempt: ```text flush requested -> coordinator reports Busy - -> frame is marked pending - -> lower-priority work is not granted while pending - -> retry is scheduled - -> LVGL is completed only by the defined recovery path + -> display adapter retains the area and LVGL buffer + -> LVGL flush remains incomplete + -> flush-wait callback retries the same transfer + -> transfer completes + -> lv_display_flush_ready() ``` The implementation must maintain counters for: @@ -369,7 +384,7 @@ The implementation must maintain counters for: - completed frames; - busy retries; - failed transfers; -- invalidated/dropped frames; +- deferred frames that are waiting for bus ownership; - maximum frame wait; - maximum frame hold; - current owner and waiter class. @@ -410,7 +425,9 @@ The design is internally consistent under the following assumptions: 2. No caller retains a direct physical mutex handle. 3. Every hardware transaction can be bounded and split from software work. 4. LVGL flush completion is coupled to a real transaction result. -5. Hydration and compaction re-check the foreground gate between operations. +5. Hydration, persistence, and compaction re-check the foreground gate between + operations. Persistence may drain one bounded immutable delta batch while + the foreground is active; compaction waits for a stable idle gate. 6. The coordinator is initialized before any display, SD, or radio request. Under those assumptions, the design addresses the observed failures: diff --git a/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h b/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h index 142c1df2..02e6baa4 100644 --- a/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h +++ b/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h @@ -30,6 +30,13 @@ class MeshPeerDirectoryCore final : public IMeshPeerDirectory void setAutoSaveEnabled(bool enabled); MeshPeerDirectoryStatus begin() override; + MeshPeerDirectoryStatus beginEmpty(); + MeshPeerDirectoryBlobLoadResult loadPersistenceBlob( + std::vector& out) const; + MeshPeerDirectoryBlobLoadResult streamPersistenceBlob( + IMeshPeerDirectoryBlobSink& sink) const; + MeshPeerDirectoryStatus hydratePersistenceBlob(const uint8_t* data, + std::size_t len); MeshPeerDirectoryStatus record(const MeshPeerRecord& record) override; MeshPeerDirectoryStatus find(const MeshPeerIdentity& identity, MeshPeerRecord& out_record) override; @@ -63,6 +70,16 @@ class MeshPeerDirectoryCore final : public IMeshPeerDirectory MeshPeerDirectoryCapacity capacityFor(MeshProtocol protocol) const override; MeshPeerDirectoryStatus flush() override; + bool persistencePending() const; + uint32_t persistenceRevision() const; + std::size_t persistenceSnapshotSize() const; + bool encodePersistenceSnapshot(uint8_t* out, + std::size_t out_len, + uint32_t* out_revision) const; + bool persistEncodedSnapshot(const uint8_t* data, + std::size_t len, + uint32_t revision); + std::size_t count(MeshProtocol protocol) const; void clear(); @@ -77,6 +94,7 @@ class MeshPeerDirectoryCore final : public IMeshPeerDirectory std::size_t findIndex(const MeshPeerIdentity& identity) const; std::size_t countForProtocol(MeshProtocol protocol) const; void evictOldest(MeshProtocol protocol); + void markDirty(); MeshPeerDirectoryStatus saveRecords(); void maybeSave(); @@ -85,6 +103,7 @@ class MeshPeerDirectoryCore final : public IMeshPeerDirectory std::vector records_; bool begun_ = false; bool dirty_ = false; + uint32_t persistence_revision_ = 0; }; } // namespace chat diff --git a/modules/core_chat/include/chat/ports/i_chat_store.h b/modules/core_chat/include/chat/ports/i_chat_store.h index b2c0e07e..ef834c5d 100644 --- a/modules/core_chat/include/chat/ports/i_chat_store.h +++ b/modules/core_chat/include/chat/ports/i_chat_store.h @@ -23,6 +23,17 @@ class IChatStore public: virtual ~IChatStore() = default; + /** + * Report whether the store's authoritative in-memory view is available. + * + * This is a logical data-availability contract. It does not expose a + * device session, bus ownership, or a lock state to callers. + * + * Volatile stores are ready by default. Persistent stores return false + * until their maintenance-owned hydration has completed. + */ + virtual bool isReady() const { return true; } + /** * @brief Append message to storage * @param msg Message to append diff --git a/modules/core_chat/include/chat/ports/i_mesh_peer_directory.h b/modules/core_chat/include/chat/ports/i_mesh_peer_directory.h index 7bcc2e82..e251c924 100644 --- a/modules/core_chat/include/chat/ports/i_mesh_peer_directory.h +++ b/modules/core_chat/include/chat/ports/i_mesh_peer_directory.h @@ -17,6 +17,8 @@ enum class MeshPeerDirectoryStatusCode : uint8_t IoError = 4, CapacityExceeded = 5, Unsupported = 6, + Busy = 7, + DeviceUnavailable = 8, }; struct MeshPeerDirectoryStatus diff --git a/modules/core_chat/include/chat/ports/i_mesh_peer_directory_blob_store.h b/modules/core_chat/include/chat/ports/i_mesh_peer_directory_blob_store.h index 6690e56f..fe463a17 100644 --- a/modules/core_chat/include/chat/ports/i_mesh_peer_directory_blob_store.h +++ b/modules/core_chat/include/chat/ports/i_mesh_peer_directory_blob_store.h @@ -15,12 +15,24 @@ enum class MeshPeerDirectoryBlobLoadResult : uint8_t IoError = 3, }; +class IMeshPeerDirectoryBlobSink +{ + public: + virtual ~IMeshPeerDirectoryBlobSink() = default; + + virtual bool begin(std::size_t expected_size) = 0; + virtual bool write(const uint8_t* data, std::size_t len) = 0; + virtual bool finish() = 0; +}; + class IMeshPeerDirectoryBlobStore { public: virtual ~IMeshPeerDirectoryBlobStore() = default; virtual MeshPeerDirectoryBlobLoadResult loadBlob(std::vector& out) = 0; + virtual MeshPeerDirectoryBlobLoadResult loadBlobTo( + IMeshPeerDirectoryBlobSink& sink); virtual bool saveBlob(const uint8_t* data, std::size_t len) = 0; virtual void clearBlob() = 0; }; diff --git a/modules/core_chat/include/chat/usecase/chat_service.h b/modules/core_chat/include/chat/usecase/chat_service.h index 73912176..7169ac0d 100644 --- a/modules/core_chat/include/chat/usecase/chat_service.h +++ b/modules/core_chat/include/chat/usecase/chat_service.h @@ -120,6 +120,15 @@ class ChatService size_t limit, size_t* total) const; std::vector getConversations(size_t offset, size_t limit, size_t* total) const; + + /** + * Report whether the store-backed conversation projection is available. + * + * This is intentionally a domain-level readiness signal. The service + * does not expose how the store obtains or protects its data. + */ + bool isDataReady() const { return store_.isReady(); } + int getTotalUnread() const; /** diff --git a/modules/core_chat/src/infra/mesh_peer_directory_core.cpp b/modules/core_chat/src/infra/mesh_peer_directory_core.cpp index 1e934888..3876bc59 100644 --- a/modules/core_chat/src/infra/mesh_peer_directory_core.cpp +++ b/modules/core_chat/src/infra/mesh_peer_directory_core.cpp @@ -2,6 +2,7 @@ #include #include +#include #if defined(_MSC_VER) #define TRAILMATE_PACK_PUSH __pragma(pack(push, 1)) @@ -590,6 +591,27 @@ NodeId reticulumNodeIdFromDestinationHash(const uint8_t* destination_hash) } // namespace +MeshPeerDirectoryBlobLoadResult IMeshPeerDirectoryBlobStore::loadBlobTo( + IMeshPeerDirectoryBlobSink& sink) +{ + std::vector buffer; + const MeshPeerDirectoryBlobLoadResult result = loadBlob(buffer); + if (result != MeshPeerDirectoryBlobLoadResult::Loaded) + { + return result; + } + if (!sink.begin(buffer.size())) + { + return MeshPeerDirectoryBlobLoadResult::IoError; + } + if (!buffer.empty() && !sink.write(buffer.data(), buffer.size())) + { + return MeshPeerDirectoryBlobLoadResult::IoError; + } + return sink.finish() ? MeshPeerDirectoryBlobLoadResult::Loaded + : MeshPeerDirectoryBlobLoadResult::IoError; +} + MeshPeerRecord mergeMeshPeerRecordFacts(const MeshPeerRecord& existing, const MeshPeerRecord& incoming) { @@ -617,7 +639,7 @@ void MeshPeerDirectoryCore::setAutoSaveEnabled(bool enabled) MeshPeerDirectoryStatus MeshPeerDirectoryCore::begin() { std::vector blob; - const auto loaded = blob_store_.loadBlob(blob); + const auto loaded = loadPersistenceBlob(blob); if (loaded == MeshPeerDirectoryBlobLoadResult::Unavailable) { begun_ = false; @@ -630,18 +652,65 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::begin() return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); } - records_.clear(); - dirty_ = false; - begun_ = true; + const MeshPeerDirectoryStatus empty_status = beginEmpty(); + if (!empty_status.succeeded()) + { + return empty_status; + } if (loaded == MeshPeerDirectoryBlobLoadResult::Missing || blob.empty()) { return MeshPeerDirectoryStatus::success(); } - if (!decodeBlob(records_, blob.data(), blob.size())) + + return hydratePersistenceBlob(blob.data(), blob.size()); +} + +MeshPeerDirectoryStatus MeshPeerDirectoryCore::beginEmpty() +{ + records_.clear(); + dirty_ = false; + begun_ = true; + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryBlobLoadResult MeshPeerDirectoryCore::loadPersistenceBlob( + std::vector& out) const +{ + return blob_store_.loadBlob(out); +} + +MeshPeerDirectoryBlobLoadResult MeshPeerDirectoryCore::streamPersistenceBlob( + IMeshPeerDirectoryBlobSink& sink) const +{ + return blob_store_.loadBlobTo(sink); +} + +MeshPeerDirectoryStatus MeshPeerDirectoryCore::hydratePersistenceBlob( + const uint8_t* data, + std::size_t len) +{ + if (!begun_ || (!data && len != 0U)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::InvalidArgument); + } + if (len == 0U) { records_.clear(); - return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); + dirty_ = false; + return MeshPeerDirectoryStatus::success(); } + + std::vector decoded; + if (!decodeBlob(decoded, data, len)) + { + records_.clear(); + dirty_ = false; + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::IoError); + } + records_ = std::move(decoded); + dirty_ = false; return MeshPeerDirectoryStatus::success(); } @@ -660,7 +729,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::record( const MeshPeerRecord& existing = records_[existing_index]; MeshPeerRecord next = mergeMeshPeerRecordFacts(existing, record); records_[existing_index] = next; - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -680,7 +749,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::record( next.last_seen_s = next.first_seen_s; } records_.push_back(next); - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -855,7 +924,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::setUserAlias( sizeof(records_[index].user_alias), alias); records_[index].flags.favorite = alias[0] != '\0'; - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -876,7 +945,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::setUserFlags( MeshPeerDirectoryStatusCode::NotFound); } records_[index].flags = flags; - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -908,7 +977,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::setKeyManuallyVerified( return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::Unsupported); } - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -923,7 +992,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::remove( MeshPeerDirectoryStatusCode::NotFound); } records_.erase(records_.begin() + static_cast(index)); - dirty_ = true; + markDirty(); maybeSave(); return MeshPeerDirectoryStatus::success(); } @@ -942,7 +1011,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::clearProtocol(MeshProtocol protoc records_.end()); if (records_.size() != old_size) { - dirty_ = true; + markDirty(); maybeSave(); } return MeshPeerDirectoryStatus::success(); @@ -975,6 +1044,72 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::flush() return saveRecords(); } +bool MeshPeerDirectoryCore::persistencePending() const +{ + return dirty_; +} + +uint32_t MeshPeerDirectoryCore::persistenceRevision() const +{ + return persistence_revision_; +} + +std::size_t MeshPeerDirectoryCore::persistenceSnapshotSize() const +{ + return sizeof(PersistedMeshPeerDirectoryHeaderV1) + + records_.size() * sizeof(PersistedMeshPeerEntryV2); +} + +bool MeshPeerDirectoryCore::encodePersistenceSnapshot( + uint8_t* out, + std::size_t out_len, + uint32_t* out_revision) const +{ + if (!out || !out_revision || out_len < persistenceSnapshotSize()) + { + return false; + } + + const std::size_t entries_len = + records_.size() * sizeof(PersistedMeshPeerEntryV2); + auto* entries_data = + out + sizeof(PersistedMeshPeerDirectoryHeaderV1); + for (std::size_t index = 0; index < records_.size(); ++index) + { + PersistedMeshPeerEntryV2 persisted{}; + copyIntoPersisted(persisted, records_[index]); + std::memcpy(entries_data + index * sizeof(PersistedMeshPeerEntryV2), + &persisted, + sizeof(persisted)); + } + + PersistedMeshPeerDirectoryHeaderV1 header{}; + header.count = static_cast(records_.size()); + header.crc = computeBlobCrc(entries_data, entries_len); + std::memcpy(out, &header, sizeof(header)); + *out_revision = persistence_revision_; + return true; +} + +bool MeshPeerDirectoryCore::persistEncodedSnapshot(const uint8_t* data, + std::size_t len, + uint32_t revision) +{ + if (!data || len == 0 || revision != persistence_revision_) + { + return false; + } + if (!blob_store_.saveBlob(data, len)) + { + return false; + } + if (revision == persistence_revision_) + { + dirty_ = false; + } + return true; +} + std::size_t MeshPeerDirectoryCore::count(MeshProtocol protocol) const { return countForProtocol(protocol); @@ -983,7 +1118,7 @@ std::size_t MeshPeerDirectoryCore::count(MeshProtocol protocol) const void MeshPeerDirectoryCore::clear() { records_.clear(); - dirty_ = true; + markDirty(); blob_store_.clearBlob(); dirty_ = false; } @@ -1135,15 +1270,29 @@ void MeshPeerDirectoryCore::evictOldest(MeshProtocol protocol) } } +void MeshPeerDirectoryCore::markDirty() +{ + dirty_ = true; + ++persistence_revision_; + if (persistence_revision_ == 0) + { + persistence_revision_ = 1; + } +} + MeshPeerDirectoryStatus MeshPeerDirectoryCore::saveRecords() { std::vector blob; encodeBlob(blob, records_); + const uint32_t revision = persistence_revision_; if (!blob_store_.saveBlob(blob.empty() ? nullptr : blob.data(), blob.size())) { return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); } - dirty_ = false; + if (revision == persistence_revision_) + { + dirty_ = false; + } return MeshPeerDirectoryStatus::success(); } diff --git a/modules/core_chat/src/usecase/contact_service.cpp b/modules/core_chat/src/usecase/contact_service.cpp index 03b7903d..eb7a646c 100644 --- a/modules/core_chat/src/usecase/contact_service.cpp +++ b/modules/core_chat/src/usecase/contact_service.cpp @@ -705,11 +705,21 @@ void ContactService::buildCache() const { return; } - cached_nodes_.clear(); - ProjectionVisitor visitor(cached_nodes_); - (void)directory_.visit(active_protocol_, - MeshPeerDirectoryView::All, - visitor); + std::vector next_nodes; + ProjectionVisitor visitor(next_nodes); + const MeshPeerDirectoryStatus status = + directory_.visit(active_protocol_, + MeshPeerDirectoryView::All, + visitor); + if (!status.succeeded()) + { + // A persistent directory may not have installed its authoritative + // projection yet. Keep the last visible cache instead of turning a + // transient retry into a blank contact list. + cache_timestamp_ = now_ms; + return; + } + cached_nodes_.swap(next_nodes); cache_timestamp_ = now_ms; } diff --git a/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp b/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp index bd1b3bfb..5b6d43c8 100644 --- a/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp +++ b/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp @@ -82,6 +82,14 @@ int main(int argc, char** argv) const std::string storage_runtime = readFile( repo_root / "platform/esp/arduino_common/src/storage/storage_runtime.cpp"); + const std::string board_runtime = readFile( + repo_root / "platform/esp/boards/src/board_runtime.cpp"); + const std::string storage_owner_header = readFile( + repo_root / + "platform/esp/common/include/platform/esp/common/storage/storage_maintenance_owner.h"); + const std::string state_lock_header = readFile( + repo_root / + "platform/esp/arduino_common/include/platform/esp/arduino_common/storage/scoped_state_lock.h"); const std::string idf_facade = readFile( repo_root / "apps/esp32_lvgl/src/esp32_lvgl_idf_app_facade_runtime.cpp"); @@ -91,6 +99,9 @@ int main(int argc, char** argv) const std::string idf_storage_runtime = readFile( repo_root / "platform/esp/idf_common/src/storage_runtime.cpp"); + const std::string linux_config_runtime = readFile( + repo_root / + "platform/linux/common/src/app/linux_app_services.cpp"); assert(contains(header, "kRoot = \"/data/v2\"")); assert(contains(header, "kMeshtasticRoot = \"/data/v2/mt/chat\"")); @@ -128,12 +139,75 @@ int main(int argc, char** argv) assert(contains(append_body, "projection_dirty_")); assert(contains(append_body, "authoritative=1")); assert(contains(append_body, "return true;")); + assert(contains(append_body, "ScopedPersistenceLease")); + const std::size_t append_projection_write = + positionOf(append_body, + "appendCatalogProjection(projection_snapshot)"); + assert(append_projection_write < + positionOfAfter(append_body, + "ScopedRecursiveStateLock state_lock", + append_projection_write)); const std::string unread_body = bodyBetween(source, "bool SdStore::setUnread", "int SdStore::getUnread"); assert(positionOf(unread_body, "appendReadProjection(projection)") < - positionOf(unread_body, "appendCatalogProjection(*catalog)")); + positionOf(unread_body, + "appendCatalogProjection(catalog_snapshot)")); assert(contains(unread_body, "last_read_sequence")); + const std::size_t unread_catalog_write = + positionOf(unread_body, + "appendCatalogProjection(catalog_snapshot)"); + assert(unread_catalog_write < + positionOfAfter(unread_body, + "ScopedRecursiveStateLock state_lock", + unread_catalog_write)); + + const std::string message_page_body = bodyBetween( + source, + "SdStore::loadPageFromLatest", + "SdStore::loadConversationPage"); + assert(contains(message_page_body, "ScopedPersistenceLease")); + assert(!contains(message_page_body, "ScopedRecursiveStateLock")); + + const std::string clear_conversation_body = bodyBetween( + source, + "void SdStore::clearConversation", + "void SdStore::clearAll"); + const std::size_t clear_read_tombstone = + positionOf(clear_conversation_body, + "appendReadProjection(read_tombstone)"); + assert(clear_read_tombstone < + positionOfAfter(clear_conversation_body, + "ScopedRecursiveStateLock state_lock", + clear_read_tombstone)); + + const std::string clear_all_body = bodyBetween( + source, + "void SdStore::clearAll", + "bool SdStore::updateMessageStatus"); + assert(positionOf(clear_all_body, "ensureLayout()") < + positionOf(clear_all_body, + "ScopedRecursiveStateLock state_lock")); + + const std::string status_update_body = bodyBetween( + source, + "bool SdStore::updateMessageStatusForProtocol", + "bool SdStore::getMessage"); + const std::size_t status_projection_write = + positionOf(status_update_body, + "appendStatusProjection(protocol, projection)"); + assert(status_projection_write < + positionOfAfter(status_update_body, + "ScopedRecursiveStateLock state_lock", + status_projection_write)); + + const std::string ordinal_read_body = bodyBetween( + source, + "bool SdStore::readMessageByOrdinal", + "bool SdStore::latestStoredMessage"); + assert(positionOf(ordinal_read_body, "journal_.read") < + positionOf(ordinal_read_body, + "ScopedRecursiveStateLock state_lock")); const std::string conversation_query = bodyBetween( source, @@ -162,6 +236,12 @@ int main(int argc, char** argv) assert(contains(peer_header, "pending_peer_observations_")); assert(contains(peer_source, "queueDeferredObservation")); assert(contains(peer_source, "StorageUnavailable")); + assert(contains(peer_source, "MeshPeerDirectoryStatusCode::Busy")); + assert(contains(peer_source, + "MeshPeerDirectoryStatusCode::DeviceUnavailable")); + assert(contains(peer_source, + "SdProtocolPeerRepository::flushPendingDeltas")); + assert(contains(peer_source, "operationFailureKind")); assert(contains(peer_codec, "validPeerIdentityForProtocol")); assert(contains(peer_codec, "validContactIdentityForProtocol")); const std::string peer_prefix = bodyBetween( @@ -182,18 +262,32 @@ int main(int argc, char** argv) repo_root / "platform/esp/arduino_common/src/chat/infra/contact_store.cpp")); - const std::string seen_load = bodyBetween( + assert(contains(source, "stepSeenRebuild")); + assert(contains(source, "stepProtocolCatalogReconcile")); + assert(contains(source, "stepConversationDirectoryReconcile")); + assert(!contains(source, "loadSeenJournal")); + assert(!contains(source, "rebuildSeenJournalFromMessages")); + assert(!contains(source, "reconcileProtocolCatalog")); + const std::string conversation_reconcile = bodyBetween( source, - "bool SdStore::loadSeenJournal", - "bool SdStore::reconcileProtocolCatalog"); - assert(contains(seen_load, "rebuildSeenJournalFromMessages")); - assert(contains(seen_load, "authoritative=messages")); + "SdStore::stepConversationDirectoryReconcile", + "std::size_t SdStore::slotsPerMessageSegment"); + assert(contains(conversation_reconcile, + "maintenance_reconcile_segment_")); + assert(contains(conversation_reconcile, + "ReconcileStepResult::InProgress")); + assert(contains(conversation_reconcile, + "ConversationReconcilePhase::ScanUnread")); + assert(!contains(conversation_reconcile, + "for (uint32_t segment")); + assert(!contains(conversation_reconcile, + "countUnreadAfter")); const std::string flush_body = bodyBetween( source, "void SdStore::flush", "bool SdStore::ensureLayout"); - assert(contains(flush_body, "projection_dirty_")); - assert(contains(flush_body, "kProjectionRetryIntervalMs")); + assert(!contains(flush_body, "journal_.")); + assert(!contains(flush_body, "compactProtocolProjections")); const std::string constructor_body = bodyBetween( source, @@ -207,7 +301,7 @@ int main(int argc, char** argv) const std::string begin_body = bodyBetween( peer_source, "MeshPeerDirectoryStatus SdProtocolPeerRepository::begin()", - "MeshPeerDirectoryStatus SdProtocolPeerRepository::hydrateFromStorage()"); + "bool SdProtocolPeerRepository::ensureLayout()"); assert(!contains(begin_body, "ensureLayout()")); assert(!contains(begin_body, "loadProtocol")); assert(!contains(begin_body, "compactProtocolAtBoot")); @@ -215,13 +309,51 @@ int main(int argc, char** argv) assert(positionOf(startup, "initializeShell()") < positionOf(startup, "startDeferredStorage()")); - assert(contains(storage_runtime, "xTaskCreatePinnedToCore")); - assert(contains(storage_runtime, "vTaskDelete(nullptr)")); + assert(contains(storage_runtime, "StorageMaintenanceOwner")); + assert(contains(storage_runtime, "stepMaintenance")); + assert(contains(storage_runtime, "StorageOperationBudget")); + assert(contains(storage_owner_header, "step_budget")); + assert(contains(journal, "FixedSlotJournalCursor")); + assert(contains(storage_runtime, "storageCapabilities")); + assert(!contains(storage_runtime, "ARDUINO_T_DECK")); + assert(contains(board_runtime, "StorageBusTopology::SharedDisplaySpi")); + assert(contains(board_runtime, "StorageBusTopology::Sdmmc")); + assert(contains(state_lock_header, "StateLockResult")); + assert(contains(state_lock_header, "Busy")); + assert(contains(state_lock_header, "Unavailable")); + assert(contains(storage_owner_header, "requestStop")); + assert(contains(storage_owner_header, "latest_tick_generation_")); + assert(contains(storage_runtime, "SdMaintenanceAdapter")); assert(contains(storage_runtime, "memory::admit")); - assert(contains(storage_runtime, "storage_worker")); - assert(contains(storage_runtime, "retry scheduled")); + assert(contains(storage_runtime, "generation")); assert(contains(storage_runtime, "tick_deferred_storage")); assert(contains(storage_runtime, "is_sleeping")); + const std::string maintenance_step_body = bodyBetween( + storage_runtime, + "Result step(Operation operation,", + "void cancelAtStepBoundary"); + const std::size_t peer_step = + positionOf(maintenance_step_body, + "context_.peer_directory->stepMaintenance"); + const std::size_t peer_result_check = + positionOfAfter(maintenance_step_body, + "if (!result.inProgress())", + peer_step); + assert(peer_step < peer_result_check); + assert(contains(storage_owner_header, "xTaskCreatePinnedToCore")); + assert(contains(storage_owner_header, "xQueueCreate")); + assert(contains(storage_owner_header, "xQueueSend")); + assert(contains(storage_owner_header, "StorageRuntimeSnapshot")); + assert(contains(storage_owner_header, + "std::atomic arm_event_pending_")); + assert(contains(storage_owner_header, "compare_exchange_strong")); + assert(contains(storage_owner_header, "terminal_without_completion")); + assert(contains(storage_owner_header, + "config_.adapter->cancelAtStepBoundary(command.operation")); + assert(!contains(storage_owner_header, "vTaskDelete(nullptr)")); + assert(!contains(storage_runtime, "vTaskDelete(nullptr)")); + assert(!contains(storage_runtime, "s_worker_task")); + assert(!contains(storage_runtime, "storage_worker")); assert(contains(idf_facade, "createIdfChatStore")); assert(!contains(bodyBetween(idf_facade, "std::unique_ptr createIdfChatStore", @@ -229,8 +361,185 @@ int main(int argc, char** argv) "isReady()")); assert(contains(idf_facade, "startDeferredStorage")); assert(contains(idf_startup, "idf_app_runtime_access::startDeferredStorage()")); - assert(contains(idf_storage_runtime, "retry scheduled")); - assert(contains(idf_storage_runtime, "xTaskCreatePinnedToCore")); + assert(contains(idf_storage_runtime, "StorageMaintenanceOwner")); + assert(contains(idf_storage_runtime, "storageCapabilities")); + assert(contains(idf_storage_runtime, "storageStartupGateSatisfied")); + assert(contains(idf_storage_runtime, "compactionPending")); + assert(contains(board_runtime, "storageStartupGateSatisfied")); + assert(contains(board_runtime, "displayFrameCompletions")); + const std::string idf_hydration_body = bodyBetween( + idf_storage_runtime, + "Result beginHydration", + "Result stepHydration"); + assert(contains(idf_hydration_body, "streamPersistenceBlob")); + assert(!contains(idf_hydration_body, "std::vector blob")); + const std::size_t idf_hydration_resume = + positionOf(idf_hydration_body, "const bool resume"); + const std::size_t idf_hydration_resume_store = + positionOfAfter(idf_hydration_body, + "store_->beginMaintenance", + idf_hydration_resume); + const std::size_t idf_hydration_fresh_reset = + positionOfAfter(idf_hydration_body, + "releasePeerHydrationPayload();", + idf_hydration_resume_store); + assert(contains(idf_hydration_body, + "peer_hydration_generation_ == generation")); + assert(contains(idf_hydration_body, + "peer_hydration_store_in_progress_ ||")); + assert(idf_hydration_resume < idf_hydration_resume_store); + assert(idf_hydration_resume_store < idf_hydration_fresh_reset); + const std::string idf_hydration_step_body = bodyBetween( + idf_storage_runtime, + "Result stepHydration", + "Result execute"); + const std::size_t idf_hydration_retry_guard = + positionOf(idf_hydration_step_body, + "if (!store_result.retryable())"); + const std::size_t idf_hydration_terminal_release = + positionOfAfter(idf_hydration_step_body, + "releasePeerHydrationPayload();", + idf_hydration_retry_guard); + assert(idf_hydration_retry_guard < idf_hydration_terminal_release); + const std::string idf_config_load_body = bodyBetween( + idf_facade, + "bool loadIdfAppConfig", + "bool saveIdfAppConfig"); + assert(contains(idf_config_load_body, "get_blob_into")); + assert(!contains(idf_config_load_body, "get_blob(")); + const std::string idf_config_submit_body = bodyBetween( + idf_facade, + "void saveConfig(app::AppConfigChangeSet changes) override", + "void applyMeshConfig()"); + assert(contains(idf_config_submit_body, + "ConfigPersistenceUrgency::Debounced")); + assert(!contains(idf_config_submit_body, + "ConfigPersistenceUrgency::Immediate")); + const std::string linux_config_submit_body = bodyBetween( + linux_config_runtime, + "void LinuxAppServices::saveConfig(::app::AppConfigChangeSet changes)", + "void LinuxAppServices::flushConfigPersistence"); + assert(contains(linux_config_submit_body, + "ConfigPersistenceUrgency::Debounced")); + assert(!contains(linux_config_submit_body, + "ConfigPersistenceUrgency::Immediate")); + assert(!contains(idf_storage_runtime, "PsramByteVector")); + assert(contains(idf_storage_runtime, "class PsramPayload")); + assert(contains(storage_owner_header, "StorageMaintenanceOwner")); + assert(!contains(idf_storage_runtime, "vTaskDelete(nullptr)")); + assert(!contains(idf_storage_runtime, "s_task")); + assert(contains(storage_owner_header, "compare_exchange_strong")); + assert(contains(storage_owner_header, "clearTickEventPending")); + + assert(!contains(source, "hydrateFromStorage")); + assert(!contains(source, "compactDeferred")); + assert(!contains(peer_source, "hydrateFromStorage")); + assert(!contains(peer_source, "compactDeferred")); + const std::string chat_begin_maintenance_body = bodyBetween( + source, + "SdStore::beginMaintenance", + "SdStore::stepMaintenance"); + assert(contains(chat_begin_maintenance_body, + "maintenance_.operation == operation")); + assert(contains(chat_begin_maintenance_body, + "maintenance_.generation == generation")); + assert(contains(chat_begin_maintenance_body, + "maintenance_persistence_locked_ = true")); + assert(contains(chat_begin_maintenance_body, + "A composite adapter may revisit this store")); + const std::size_t chat_resume = + positionOf(chat_begin_maintenance_body, + "if (maintenance_.operation == operation"); + const std::size_t chat_resume_ownership = + positionOfAfter(chat_begin_maintenance_body, + "if (!maintenance_persistence_locked_ &&", + chat_resume); + const std::size_t chat_resume_lease = + positionOfAfter(chat_begin_maintenance_body, + "!acquirePersistenceLease", + chat_resume_ownership); + const std::size_t chat_resume_lock = + positionOfAfter(chat_begin_maintenance_body, + "maintenance_persistence_locked_ = true", + chat_resume_lease); + const std::size_t chat_resume_return = + positionOfAfter(chat_begin_maintenance_body, + "inProgressResult(", + chat_resume_lock); + assert(chat_resume_ownership < chat_resume_lease); + assert(chat_resume_lease < chat_resume_lock); + assert(chat_resume_lock < chat_resume_return); + assert(chat_resume_return < + positionOf(chat_begin_maintenance_body, + "resetCatalogReconcileCursor()")); + assert(chat_resume_return < + positionOf(chat_begin_maintenance_body, "maintenance_ = {}")); + const std::string chat_step_maintenance_body = bodyBetween( + source, + "SdStore::stepMaintenance", + "void SdStore::cancelMaintenance"); + assert(contains(chat_step_maintenance_body, "result.completed()")); + assert(contains(chat_step_maintenance_body, + "maintenance_.phase == MaintenancePhase::Failed")); + const std::string chat_cancel_maintenance_body = bodyBetween( + source, + "void SdStore::cancelMaintenance", + "SdStore::maintenanceFailure"); + assert(contains(chat_cancel_maintenance_body, + "maintenance_.phase = MaintenancePhase::Failed")); + assert(contains(chat_cancel_maintenance_body, + "releaseMaintenanceLease()")); + const std::string peer_begin_maintenance_body = bodyBetween( + peer_source, + "SdProtocolPeerRepository::beginMaintenance", + "SdProtocolPeerRepository::stepMaintenance"); + assert(contains(peer_begin_maintenance_body, + "maintenance_.operation == operation")); + assert(contains(peer_begin_maintenance_body, + "maintenance_.generation == generation")); + assert(contains(peer_begin_maintenance_body, + "maintenance_persistence_locked_ = true")); + assert(contains(peer_begin_maintenance_body, + "A composite adapter may revisit this repository")); + const std::size_t peer_resume = + positionOf(peer_begin_maintenance_body, + "if (maintenance_.operation == operation"); + const std::size_t peer_resume_ownership = + positionOfAfter(peer_begin_maintenance_body, + "if (!maintenance_persistence_locked_ &&", + peer_resume); + const std::size_t peer_resume_lease = + positionOfAfter(peer_begin_maintenance_body, + "!acquirePersistenceLease", + peer_resume_ownership); + const std::size_t peer_resume_lock = + positionOfAfter(peer_begin_maintenance_body, + "maintenance_persistence_locked_ = true", + peer_resume_lease); + const std::size_t peer_resume_return = + positionOfAfter(peer_begin_maintenance_body, + "inProgressResult(", + peer_resume_lock); + assert(peer_resume_ownership < peer_resume_lease); + assert(peer_resume_lease < peer_resume_lock); + assert(peer_resume_lock < peer_resume_return); + assert(peer_resume_return < + positionOf(peer_begin_maintenance_body, "maintenance_ = {}")); + const std::string peer_step_maintenance_body = bodyBetween( + peer_source, + "SdProtocolPeerRepository::stepMaintenance", + "void SdProtocolPeerRepository::cancelMaintenance"); + assert(contains(peer_step_maintenance_body, "result.completed()")); + assert(contains(peer_step_maintenance_body, + "maintenance_.phase == MaintenancePhase::Failed")); + const std::string peer_persistence_body = bodyBetween( + peer_source, + "SdProtocolPeerRepository::stepPersistence", + "SdProtocolPeerRepository::stepCompaction"); + assert(!contains(peer_persistence_body, + "for (bool pending : protocol_reset_pending_)")); + assert(positionOf(peer_persistence_body, "flushPendingDeltas") < + positionOf(peer_persistence_body, "persistencePending()")); return 0; } diff --git a/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp b/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp index 6d6c6596..1d70e170 100644 --- a/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp +++ b/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp @@ -1,5 +1,6 @@ #include "chat/infra/mesh_peer_directory_core.h" #include "chat/usecase/contact_service.h" +#include "sys/clock.h" #include #include @@ -9,6 +10,13 @@ namespace { +uint32_t g_fake_millis = 0; + +uint32_t fake_millis() +{ + return g_fake_millis; +} + chat::NodeId reticulum_node_id_from_destination_hash(const uint8_t* destination_hash) { if (!destination_hash) @@ -221,6 +229,14 @@ class MemoryMeshPeerDirectory final : public chat::IMeshPeerDirectory chat::MeshPeerDirectoryView view, chat::IMeshPeerDirectoryVisitor& visitor) override { + ++visit_count; + if (!next_visit_status_.succeeded()) + { + const chat::MeshPeerDirectoryStatus status = next_visit_status_; + next_visit_status_ = chat::MeshPeerDirectoryStatus::success(); + return status; + } + for (const auto& record : records_) { const bool contact = chat::meshPeerIsContact(record); @@ -240,6 +256,11 @@ class MemoryMeshPeerDirectory final : public chat::IMeshPeerDirectory return chat::MeshPeerDirectoryStatus::success(); } + void failNextVisit(chat::MeshPeerDirectoryStatusCode code) + { + next_visit_status_ = chat::MeshPeerDirectoryStatus::fail(code); + } + chat::MeshPeerDirectoryStatus setUserAlias( const chat::MeshPeerIdentity& identity, const char* alias) override @@ -344,6 +365,7 @@ class MemoryMeshPeerDirectory final : public chat::IMeshPeerDirectory chat::MeshPeerDirectoryCapacity meshtastic_capacity{2, 1}; chat::MeshPeerDirectoryCapacity meshcore_capacity{1, 1}; chat::MeshPeerDirectoryCapacity reticulum_capacity{3, 1}; + std::size_t visit_count = 0; private: std::size_t countForProtocol(chat::MeshProtocol protocol) const @@ -381,6 +403,7 @@ class MemoryMeshPeerDirectory final : public chat::IMeshPeerDirectory } bool begun_ = false; + chat::MeshPeerDirectoryStatus next_visit_status_{}; std::vector records_; }; @@ -431,6 +454,41 @@ class CountingMeshPeerDirectoryBlobStore final int clear_count = 0; }; +class CollectingMeshPeerDirectoryBlobSink final + : public chat::IMeshPeerDirectoryBlobSink +{ + public: + bool begin(std::size_t expected_size) override + { + data.clear(); + data.reserve(expected_size); + return true; + } + + bool write(const uint8_t* bytes, std::size_t len) override + { + if (len == 0U) + { + return true; + } + if (!bytes) + { + return false; + } + data.insert(data.end(), bytes, bytes + len); + return true; + } + + bool finish() override + { + finished = true; + return true; + } + + std::vector data; + bool finished = false; +}; + chat::ReticulumPeerIdentity makeReticulumIdentity(uint8_t seed) { uint8_t destination[chat::kReticulumPeerHashSize] = {}; @@ -668,6 +726,53 @@ void core_persists_reticulum_ratchet() } } +void deferred_hydration_keeps_storage_out_of_directory_begin() +{ + CountingMeshPeerDirectoryBlobStore blob; + chat::MeshPeerDirectoryCore::Options options{}; + options.auto_save = false; + const auto original = makeMeshCorePeer(0x91, "deferred peer", 77); + + { + chat::MeshPeerDirectoryCore writer(blob, options); + assert(writer.beginEmpty().succeeded()); + assert(writer.record(original).succeeded()); + assert(writer.flush().succeeded()); + } + + chat::MeshPeerDirectoryCore reader(blob, options); + assert(reader.beginEmpty().succeeded()); + assert(reader.count(chat::MeshProtocol::MeshCore) == 0U); + + std::vector encoded; + assert(reader.loadPersistenceBlob(encoded) == + chat::MeshPeerDirectoryBlobLoadResult::Loaded); + CollectingMeshPeerDirectoryBlobSink streamed; + assert(reader.streamPersistenceBlob(streamed) == + chat::MeshPeerDirectoryBlobLoadResult::Loaded); + assert(streamed.finished); + assert(streamed.data == encoded); + assert(reader.hydratePersistenceBlob(encoded.data(), encoded.size()) + .succeeded()); + + chat::MeshPeerRecord loaded{}; + assert(reader.find(original.identity, loaded).succeeded()); + assert(std::strcmp(loaded.display_name, "deferred peer") == 0); + + assert(reader.record(makeMeshCorePeer(0x92, "new peer", 78)).succeeded()); + const std::size_t snapshot_size = reader.persistenceSnapshotSize(); + std::vector snapshot(snapshot_size); + uint32_t revision = 0U; + assert(reader.encodePersistenceSnapshot(snapshot.data(), + snapshot.size(), + &revision)); + assert(reader.record(makeMeshCorePeer(0x93, "newer peer", 79)).succeeded()); + assert(!reader.persistEncodedSnapshot(snapshot.data(), + snapshot.size(), + revision)); + assert(reader.persistencePending()); +} + void capacity_is_protocol_policy_not_interface_shape() { MemoryMeshPeerDirectory directory; @@ -973,6 +1078,37 @@ void contact_service_projects_one_directory_without_legacy_stores() assert(all.front().display_name == "Alias"); } +void contact_service_preserves_projection_when_visit_temporarily_fails() +{ + sys::set_millis_provider(fake_millis); + g_fake_millis = 1000; + + MemoryMeshPeerDirectory directory; + chat::contacts::ContactService contacts(directory); + contacts.begin(); + + constexpr chat::NodeId node_id = 0x10203040U; + assert(directory.record(makeMeshtasticPeer(node_id, "Cached", 1)) + .succeeded()); + assert(contacts.addContact(node_id, "Cached")); + + const auto initial = contacts.getContacts(); + assert(initial.size() == 1U); + assert(initial.front().node_id == node_id); + assert(initial.front().display_name == "Cached"); + const std::size_t visits_after_initial = directory.visit_count; + + g_fake_millis = 2501; + directory.failNextVisit(chat::MeshPeerDirectoryStatusCode::DeviceUnavailable); + const auto preserved = contacts.getContacts(); + assert(directory.visit_count == visits_after_initial + 1U); + assert(preserved.size() == 1U); + assert(preserved.front().node_id == node_id); + assert(preserved.front().display_name == "Cached"); + + sys::set_millis_provider(nullptr); +} + } // namespace int main() @@ -980,6 +1116,7 @@ int main() upsert_preserves_first_seen_and_updates_peer_facts(); protocol_identity_shapes_do_not_collapse(); core_persists_reticulum_ratchet(); + deferred_hydration_keeps_storage_out_of_directory_begin(); capacity_is_protocol_policy_not_interface_shape(); search_and_user_flags_are_directory_behaviors(); find_by_node_id_preserves_reticulum_destination_identity(); @@ -988,5 +1125,6 @@ int main() verified_keys_cannot_be_replaced_by_runtime_observations(); remove_and_clear_protocol_are_directory_behaviors(); contact_service_projects_one_directory_without_legacy_stores(); + contact_service_preserves_projection_when_visit_temporarily_fails(); return 0; } diff --git a/modules/core_sys/include/app/app_config_save_plan.h b/modules/core_sys/include/app/app_config_save_plan.h deleted file mode 100644 index 0132d7ce..00000000 --- a/modules/core_sys/include/app/app_config_save_plan.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "app/app_config_change_detection.h" - -namespace app -{ - -struct AppConfigSavePlan -{ - AppConfigChangeSet changes = AppConfigChangeSet::none(); - bool queue = false; -}; - -inline AppConfigSavePlan planAppConfigSave(const AppConfig& baseline, - bool baseline_valid, - const AppConfig& desired, - bool save_busy, - const AppConfig& active_save, - AppConfigChangeSet active_changes, - AppConfigChangeSet requested_changes) -{ - AppConfigSavePlan plan; - if (!save_busy) - { - plan.changes = baseline_valid - ? detectAppConfigChanges(baseline, desired) - : AppConfigChangeSet::allPersisted(); - plan.changes.mergeIn(requested_changes); - plan.queue = !plan.changes.empty(); - return plan; - } - - plan.changes = detectAppConfigChanges(active_save, desired); - if (!active_changes.containsAll(requested_changes)) - { - plan.changes.mergeIn(requested_changes); - } - plan.queue = !plan.changes.empty(); - return plan; -} - -} // namespace app diff --git a/modules/core_sys/include/app/config_persistence_runtime.h b/modules/core_sys/include/app/config_persistence_runtime.h new file mode 100644 index 00000000..5c43ea6d --- /dev/null +++ b/modules/core_sys/include/app/config_persistence_runtime.h @@ -0,0 +1,256 @@ +#pragma once + +#include "app/app_config.h" +#include "app/app_config_change_detection.h" +#include "sys/persistence_contracts.h" + +#include + +namespace app +{ + +enum class ConfigPersistenceState : uint8_t +{ + Idle, + Debouncing, + InFlight, + Backoff, +}; + +enum class ConfigPersistenceUrgency : uint8_t +{ + Debounced, + Immediate, +}; + +using ConfigPersistenceGeneration = sys::PersistenceGeneration; +using ConfigPersistenceResultKind = sys::PersistenceResultKind; + +struct ConfigPersistencePolicy +{ + uint32_t debounce_ms = 250U; + uint32_t retry_delay_ms = 1000U; +}; + +struct ConfigPersistenceSubmission +{ + bool queued = false; + ConfigPersistenceGeneration generation = 0U; + AppConfigChangeSet changes = AppConfigChangeSet::none(); +}; + +struct ConfigPersistenceWork +{ + const AppConfig* snapshot = nullptr; + AppConfigChangeSet changes = AppConfigChangeSet::none(); + ConfigPersistenceGeneration generation = 0U; +}; + +class ConfigPersistenceRuntime +{ + public: + explicit ConfigPersistenceRuntime( + ConfigPersistencePolicy policy = ConfigPersistencePolicy{}) + : policy_(policy) + { + } + + void initialize(const AppConfig& baseline) + { + baseline_ = baseline; + pending_ = baseline; + active_ = baseline; + baseline_valid_ = true; + initialized_ = true; + has_pending_ = false; + in_flight_ = false; + pending_changes_ = AppConfigChangeSet::none(); + pending_urgency_ = ConfigPersistenceUrgency::Debounced; + active_changes_ = AppConfigChangeSet::none(); + generation_ = 0U; + pending_generation_ = 0U; + active_generation_ = 0U; + last_completed_generation_ = 0U; + due_ms_ = 0U; + state_ = ConfigPersistenceState::Idle; + } + + ConfigPersistenceSubmission submit(const AppConfig& desired, + AppConfigChangeSet requested_changes, + uint32_t now_ms, + ConfigPersistenceUrgency urgency = + ConfigPersistenceUrgency::Debounced) + { + if (!initialized_) + { + return {}; + } + + AppConfigChangeSet changes = + in_flight_ + ? detectAppConfigChanges(active_, desired) + : (baseline_valid_ + ? detectAppConfigChanges(baseline_, desired) + : AppConfigChangeSet::allPersisted()); + changes.mergeIn(requested_changes); + + pending_ = desired; + if (changes.empty()) + { + has_pending_ = false; + pending_changes_ = AppConfigChangeSet::none(); + pending_urgency_ = ConfigPersistenceUrgency::Debounced; + if (!in_flight_) + { + state_ = ConfigPersistenceState::Idle; + } + return {}; + } + + ++generation_; + pending_generation_ = generation_; + pending_changes_ = changes; + pending_urgency_ = urgency; + has_pending_ = true; + due_ms_ = urgency == ConfigPersistenceUrgency::Immediate + ? now_ms + : now_ms + policy_.debounce_ms; + if (!in_flight_) + { + state_ = ConfigPersistenceState::Debouncing; + } + + return {true, pending_generation_, pending_changes_}; + } + + bool takeDue(uint32_t now_ms, ConfigPersistenceWork& out) + { + if (!initialized_ || in_flight_ || !has_pending_ || + !deadlineReached(now_ms, due_ms_)) + { + return false; + } + + active_ = pending_; + active_changes_ = pending_changes_; + active_generation_ = pending_generation_; + has_pending_ = false; + pending_changes_ = AppConfigChangeSet::none(); + pending_urgency_ = ConfigPersistenceUrgency::Debounced; + in_flight_ = true; + state_ = ConfigPersistenceState::InFlight; + + out.snapshot = &active_; + out.changes = active_changes_; + out.generation = active_generation_; + return true; + } + + ConfigPersistenceResultKind complete( + ConfigPersistenceGeneration generation, + ConfigPersistenceResultKind result, + uint32_t now_ms) + { + if (!in_flight_ || generation != active_generation_) + { + return ConfigPersistenceResultKind::StaleGeneration; + } + + in_flight_ = false; + if (result == ConfigPersistenceResultKind::Completed) + { + baseline_ = active_; + baseline_valid_ = true; + last_completed_generation_ = generation; + reconcilePendingAfterSuccess(now_ms); + return result; + } + + if (!has_pending_) + { + pending_ = active_; + pending_changes_ = active_changes_; + pending_generation_ = active_generation_; + pending_urgency_ = ConfigPersistenceUrgency::Debounced; + has_pending_ = true; + } + else + { + pending_changes_.mergeIn(active_changes_); + } + due_ms_ = now_ms + policy_.retry_delay_ms; + state_ = ConfigPersistenceState::Backoff; + return result; + } + + bool initialized() const { return initialized_; } + bool hasPending() const { return has_pending_; } + bool busy() const { return in_flight_; } + bool baselineValid() const { return baseline_valid_; } + ConfigPersistenceState state() const { return state_; } + ConfigPersistenceGeneration generation() const { return generation_; } + ConfigPersistenceGeneration pendingGeneration() const + { + return pending_generation_; + } + ConfigPersistenceGeneration activeGeneration() const + { + return active_generation_; + } + ConfigPersistenceGeneration lastCompletedGeneration() const + { + return last_completed_generation_; + } + uint32_t dueMs() const { return due_ms_; } + + private: + static bool deadlineReached(uint32_t now_ms, uint32_t due_ms) + { + return static_cast(now_ms - due_ms) >= 0; + } + + void reconcilePendingAfterSuccess(uint32_t now_ms) + { + if (!has_pending_) + { + state_ = ConfigPersistenceState::Idle; + return; + } + + pending_changes_ = detectAppConfigChanges(baseline_, pending_); + if (pending_changes_.empty()) + { + has_pending_ = false; + pending_changes_ = AppConfigChangeSet::none(); + pending_urgency_ = ConfigPersistenceUrgency::Debounced; + state_ = ConfigPersistenceState::Idle; + return; + } + + due_ms_ = pending_urgency_ == ConfigPersistenceUrgency::Immediate + ? now_ms + : now_ms + policy_.debounce_ms; + state_ = ConfigPersistenceState::Debouncing; + } + + ConfigPersistencePolicy policy_{}; + AppConfig baseline_{}; + AppConfig pending_{}; + AppConfig active_{}; + AppConfigChangeSet pending_changes_ = AppConfigChangeSet::none(); + AppConfigChangeSet active_changes_ = AppConfigChangeSet::none(); + ConfigPersistenceUrgency pending_urgency_ = + ConfigPersistenceUrgency::Debounced; + ConfigPersistenceGeneration generation_ = 0U; + ConfigPersistenceGeneration pending_generation_ = 0U; + ConfigPersistenceGeneration active_generation_ = 0U; + ConfigPersistenceGeneration last_completed_generation_ = 0U; + uint32_t due_ms_ = 0U; + bool initialized_ = false; + bool baseline_valid_ = false; + bool has_pending_ = false; + bool in_flight_ = false; + ConfigPersistenceState state_ = ConfigPersistenceState::Idle; +}; + +} // namespace app diff --git a/modules/core_sys/include/platform/ui/reticulum_group_config_runtime.h b/modules/core_sys/include/platform/ui/reticulum_group_config_runtime.h index bf07b659..6585e4dd 100644 --- a/modules/core_sys/include/platform/ui/reticulum_group_config_runtime.h +++ b/modules/core_sys/include/platform/ui/reticulum_group_config_runtime.h @@ -14,6 +14,7 @@ struct Status bool file_present = false; bool loaded = false; bool saved = false; + bool queued = false; char message[96] = {}; char detail[128] = {}; }; @@ -21,6 +22,10 @@ struct Status const char* config_path(); void clear(chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count); Status load(chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count); +Status submit(const chat::ReticulumGroupDestinationConfig* groups, + std::size_t group_count); +Status flushPending(); +bool hasPending(); Status save(const chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count); } // namespace platform::ui::reticulum_groups diff --git a/modules/core_sys/include/sys/persistence_contracts.h b/modules/core_sys/include/sys/persistence_contracts.h new file mode 100644 index 00000000..6a3392a5 --- /dev/null +++ b/modules/core_sys/include/sys/persistence_contracts.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace sys +{ + +using PersistenceGeneration = uint32_t; + +enum class PersistenceResultKind : uint8_t +{ + Completed, + InProgress, + StateBusy, + DeviceUnavailable, + RetryLater, + IoError, + Cancelled, + StaleGeneration, +}; + +} // namespace sys diff --git a/modules/core_sys/include/sys/persistence_runtime.h b/modules/core_sys/include/sys/persistence_runtime.h deleted file mode 100644 index 0b77eea9..00000000 --- a/modules/core_sys/include/sys/persistence_runtime.h +++ /dev/null @@ -1,381 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace sys -{ -namespace runtime -{ - -enum class PersistencePolicyMode : uint8_t -{ - DebouncedSave, - BatchSave, - ImmediateCriticalSave, - DropDuplicateSave, -}; - -enum class PersistenceEventKind : uint8_t -{ - SaveQueued, - SaveStarted, - SaveSucceeded, - SaveFailed, - SaveCoalesced, - SaveDropped, -}; - -struct StoreSnapshot -{ - const uint8_t* data = nullptr; - std::size_t len = 0; - bool valid = false; -}; - -struct StoreStorageResult -{ - bool ok = false; - std::size_t bytes = 0; - int32_t error = 0; -}; - -struct PersistenceCommand -{ - uint32_t command_id = 0; - const char* store_key = nullptr; - PersistencePolicyMode policy = PersistencePolicyMode::DebouncedSave; - uint32_t deadline_ms = 0; - uint32_t created_at_ms = 0; -}; - -struct PersistenceEvent -{ - PersistenceEventKind kind = PersistenceEventKind::SaveFailed; - const char* store_key = nullptr; - uint32_t command_id = 0; - uint32_t timestamp_ms = 0; - int32_t error = 0; -}; - -class PersistencePolicy -{ - public: - virtual ~PersistencePolicy() = default; - - virtual PersistencePolicyMode modeFor(const char* store_key) const = 0; - virtual uint32_t delayFor(PersistencePolicyMode policy) const = 0; -}; - -class DefaultPersistencePolicy : public PersistencePolicy -{ - public: - PersistencePolicyMode modeFor(const char* store_key) const override - { - (void)store_key; - return PersistencePolicyMode::DebouncedSave; - } - - uint32_t delayFor(PersistencePolicyMode policy) const override - { - switch (policy) - { - case PersistencePolicyMode::ImmediateCriticalSave: - return 0; - case PersistencePolicyMode::BatchSave: - return 500; - case PersistencePolicyMode::DropDuplicateSave: - case PersistencePolicyMode::DebouncedSave: - default: - return 150; - } - } - -}; - -class IStoreSnapshotProvider -{ - public: - virtual ~IStoreSnapshotProvider() = default; - - virtual StoreSnapshot snapshot(const char* store_key) = 0; -}; - -class IStoreStorageAdapter -{ - public: - virtual ~IStoreStorageAdapter() = default; - - virtual StoreStorageResult write(const char* store_key, - const uint8_t* bytes, - std::size_t len) = 0; - virtual StoreStorageResult read(const char* store_key, - uint8_t* bytes, - std::size_t capacity, - std::size_t& out_len) = 0; -}; - -class IPersistenceEventSink -{ - public: - virtual ~IPersistenceEventSink() = default; - - virtual bool publish(const PersistenceEvent& event) = 0; -}; - -template -class DirtyStoreRegistry -{ - public: - bool markDirty(const char* store_key, - PersistencePolicyMode policy, - uint32_t due_ms) - { - if (store_key == nullptr || store_key[0] == '\0') - { - return false; - } - - for (std::size_t i = 0; i < count_; ++i) - { - if (sameKey(records_[i].store_key, store_key)) - { - records_[i].policy = policy; - records_[i].due_ms = due_ms; - records_[i].dirty = true; - return true; - } - } - - if (count_ >= N) - { - return false; - } - - records_[count_++] = DirtyStoreRecord{store_key, due_ms, policy, true}; - return true; - } - - bool takeDue(uint32_t now_ms, PersistenceCommand& out) - { - for (std::size_t i = 0; i < count_; ++i) - { - DirtyStoreRecord& record = records_[i]; - if (!record.dirty || static_cast(record.due_ms - now_ms) > 0) - { - continue; - } - out.store_key = record.store_key; - out.policy = record.policy; - out.created_at_ms = now_ms; - record.dirty = false; - compact(); - return true; - } - return false; - } - - bool hasPending(const char* store_key) const - { - for (std::size_t i = 0; i < count_; ++i) - { - if (records_[i].dirty && sameKey(records_[i].store_key, store_key)) - { - return true; - } - } - return false; - } - - std::size_t size() const - { - return count_; - } - - private: - struct DirtyStoreRecord - { - const char* store_key = nullptr; - uint32_t due_ms = 0; - PersistencePolicyMode policy = PersistencePolicyMode::DebouncedSave; - bool dirty = false; - }; - - static bool sameKey(const char* lhs, const char* rhs) - { - return lhs != nullptr && rhs != nullptr && std::strcmp(lhs, rhs) == 0; - } - - void compact() - { - DirtyStoreRecord kept[N]{}; - std::size_t kept_count = 0; - for (std::size_t i = 0; i < count_; ++i) - { - if (records_[i].dirty) - { - kept[kept_count++] = records_[i]; - } - } - for (std::size_t i = 0; i < kept_count; ++i) - { - records_[i] = kept[i]; - } - count_ = kept_count; - } - - DirtyStoreRecord records_[N]{}; - std::size_t count_ = 0; -}; - -class PersistenceWorker -{ - public: - PersistenceWorker(IStoreSnapshotProvider& snapshots, - IStoreStorageAdapter& storage, - IPersistenceEventSink& events, - PersistencePolicy& policy) - : snapshots_(snapshots), storage_(storage), events_(events), policy_(policy) - { - } - - bool submit(const PersistenceCommand& command) - { - if (pending_) - { - return false; - } - pending_command_ = command; - pending_ = true; - return true; - } - - void tick(uint32_t now_ms) - { - if (!pending_) - { - return; - } - - PersistenceEvent started{}; - started.kind = PersistenceEventKind::SaveStarted; - started.store_key = pending_command_.store_key; - started.command_id = pending_command_.command_id; - started.timestamp_ms = now_ms; - (void)events_.publish(started); - - StoreSnapshot snapshot = snapshots_.snapshot(pending_command_.store_key); - StoreStorageResult result{}; - if (snapshot.valid) - { - result = storage_.write(pending_command_.store_key, snapshot.data, snapshot.len); - } - else - { - result.error = -12; - } - publishResult(result.ok ? PersistenceEventKind::SaveSucceeded - : PersistenceEventKind::SaveFailed, - now_ms, - result.ok ? 0 : result.error); - pending_ = false; - } - - bool busy() const - { - return pending_; - } - - private: - void publishResult(PersistenceEventKind kind, uint32_t now_ms, int32_t error) - { - PersistenceEvent event{}; - event.kind = kind; - event.store_key = pending_command_.store_key; - event.command_id = pending_command_.command_id; - event.timestamp_ms = now_ms; - event.error = error; - (void)events_.publish(event); - } - - IStoreSnapshotProvider& snapshots_; - IStoreStorageAdapter& storage_; - IPersistenceEventSink& events_; - PersistencePolicy& policy_; - PersistenceCommand pending_command_{}; - bool pending_ = false; -}; - -template -class PersistenceRuntime -{ - public: - PersistenceRuntime(DirtyStoreRegistry& registry, - PersistenceWorker& worker, - IPersistenceEventSink& events, - PersistencePolicy& policy) - : registry_(registry), worker_(worker), events_(events), policy_(policy) - { - } - - bool markDirty(const char* store_key, uint32_t now_ms) - { - const PersistencePolicyMode mode = policy_.modeFor(store_key); - const bool marked = registry_.markDirty(store_key, mode, now_ms + policy_.delayFor(mode)); - PersistenceEvent event{}; - event.kind = marked ? PersistenceEventKind::SaveQueued - : PersistenceEventKind::SaveDropped; - event.store_key = store_key; - event.timestamp_ms = now_ms; - (void)events_.publish(event); - return marked; - } - - bool requestSave(const char* store_key, - PersistencePolicyMode policy, - uint32_t now_ms) - { - return registry_.markDirty(store_key, policy, now_ms + policy_.delayFor(policy)); - } - - void tick(uint32_t now_ms) - { - if (!worker_.busy()) - { - PersistenceCommand command{}; - if (registry_.takeDue(now_ms, command)) - { - command.command_id = next_command_id_++; - if (!worker_.submit(command)) - { - (void)registry_.markDirty(command.store_key, - command.policy, - now_ms + policy_.delayFor(command.policy)); - } - } - } - worker_.tick(now_ms); - } - - void handle(const PersistenceEvent& event) - { - last_event_ = event; - } - - PersistenceEvent lastEvent() const - { - return last_event_; - } - - private: - DirtyStoreRegistry& registry_; - PersistenceWorker& worker_; - IPersistenceEventSink& events_; - PersistencePolicy& policy_; - PersistenceEvent last_event_{}; - uint32_t next_command_id_ = 1; -}; - -} // namespace runtime -} // namespace sys diff --git a/modules/core_sys/include/sys/runtime_harness.h b/modules/core_sys/include/sys/runtime_harness.h index 2f640b77..58c80264 100644 --- a/modules/core_sys/include/sys/runtime_harness.h +++ b/modules/core_sys/include/sys/runtime_harness.h @@ -1,7 +1,6 @@ #pragma once #include "sys/feedback_runtime.h" -#include "sys/persistence_runtime.h" #include "sys/runtime_async.h" #include "sys/shared_spi_access.h" @@ -59,7 +58,6 @@ class FakeCommandQueue : public ICommandQueue }; class FakeEventBus : public IEventSink, - public IPersistenceEventSink, public IFeedbackEventSink { public: @@ -68,16 +66,6 @@ class FakeEventBus : public IEventSink, return runtime_events_.publish(event); } - bool publish(const PersistenceEvent& event) override - { - if (persistence_count_ >= kMaxEvents) - { - return false; - } - persistence_events_[persistence_count_++] = event; - return true; - } - bool publish(const FeedbackEvent& event) override { if (feedback_count_ >= kMaxEvents) @@ -96,28 +84,16 @@ class FakeEventBus : public IEventSink, { ++count; } - count += persistence_count_; count += feedback_count_; - persistence_count_ = 0; feedback_count_ = 0; return count; } - std::size_t persistenceCount() const - { - return persistence_count_; - } - std::size_t feedbackCount() const { return feedback_count_; } - const PersistenceEvent& persistenceEvent(std::size_t index) const - { - return persistence_events_[index]; - } - const FeedbackEvent& feedbackEvent(std::size_t index) const { return feedback_events_[index]; @@ -127,127 +103,10 @@ class FakeEventBus : public IEventSink, static constexpr std::size_t kMaxEvents = 32; FixedEventSink<32> runtime_events_{}; - PersistenceEvent persistence_events_[kMaxEvents]{}; FeedbackEvent feedback_events_[kMaxEvents]{}; - std::size_t persistence_count_ = 0; std::size_t feedback_count_ = 0; }; -class FakeStorageBackend : public IPlatformStorageAdapter, - public IStoreSnapshotProvider, - public IStoreStorageAdapter -{ - public: - void scriptDelay(const char* operation, uint32_t ms) - { - (void)operation; - delay_ms_ = ms; - } - - void scriptFailure(const char* operation, int32_t error) - { - (void)operation; - fail_ = true; - error_ = error; - } - - PlatformStorageResult read(const PlatformStorageReadRequest& request) override - { - last_command_id_ = request.command_id; - return platformResult(request.capacity); - } - - PlatformStorageResult write(const PlatformStorageWriteRequest& request) override - { - last_command_id_ = request.command_id; - return platformResult(request.len); - } - - PlatformStorageResult list(const PlatformStorageListRequest& request) override - { - last_command_id_ = request.command_id; - return platformResult(0); - } - - PlatformStorageResult flush(const PlatformStorageFlushRequest& request) override - { - last_command_id_ = request.command_id; - return platformResult(0); - } - - StoreSnapshot snapshot(const char* store_key) override - { - (void)store_key; - StoreSnapshot snapshot{}; - snapshot.data = snapshot_; - snapshot.len = snapshot_len_; - snapshot.valid = !fail_; - return snapshot; - } - - StoreStorageResult write(const char* store_key, - const uint8_t* bytes, - std::size_t len) override - { - (void)store_key; - (void)bytes; - StoreStorageResult result{}; - result.ok = !fail_; - result.bytes = result.ok ? len : 0; - result.error = result.ok ? 0 : error_; - ++write_count_; - return result; - } - - StoreStorageResult read(const char* store_key, - uint8_t* bytes, - std::size_t capacity, - std::size_t& out_len) override - { - (void)store_key; - (void)bytes; - out_len = fail_ ? 0 : capacity; - StoreStorageResult result{}; - result.ok = !fail_; - result.bytes = out_len; - result.error = result.ok ? 0 : error_; - return result; - } - - uint32_t delayMs() const - { - return delay_ms_; - } - - uint32_t lastCommandId() const - { - return last_command_id_; - } - - std::size_t writeCount() const - { - return write_count_; - } - - private: - PlatformStorageResult platformResult(std::size_t bytes) - { - PlatformStorageResult result{}; - result.ok = !fail_; - result.bytes = result.ok ? bytes : 0; - result.error = result.ok ? 0 : error_; - return result; - } - - uint8_t snapshot_[4] = {1, 2, 3, 4}; - std::size_t snapshot_len_ = sizeof(snapshot_); - uint32_t delay_ms_ = 0; - uint32_t last_command_id_ = 0; - std::size_t write_count_ = 0; - bool fail_ = false; - int32_t error_ = -1; -}; - class FakeBusArbiter : public IBusArbiter { public: @@ -404,11 +263,6 @@ class RuntimeHarness return events_; } - FakeStorageBackend& storage() - { - return storage_; - } - FakeBusArbiter& bus() { return bus_; @@ -452,7 +306,6 @@ class RuntimeHarness FakeClock clock_{}; FakeCommandQueue commands_{}; FakeEventBus events_{}; - FakeStorageBackend storage_{}; FakeBusArbiter bus_{}; FakeUiOwner ui_{}; FakeFeedbackPresenter feedback_{}; diff --git a/modules/core_sys/include/sys/storage_event_runtime.h b/modules/core_sys/include/sys/storage_event_runtime.h deleted file mode 100644 index 2e84affc..00000000 --- a/modules/core_sys/include/sys/storage_event_runtime.h +++ /dev/null @@ -1,145 +0,0 @@ -#pragma once - -#include "sys/runtime_async.h" - -#include -#include -#include -#include - -namespace sys::runtime -{ - -enum class StorageWorkKind : uint8_t -{ - SnapshotSave, - LogAppend, - SnapshotRead, - DeleteBlob, -}; - -enum class StorageWorkState : uint8_t -{ - Idle, - Pending, - InFlight, - FailedPendingRetry, -}; - -struct StorageWorkItem -{ - StorageWorkKind kind = StorageWorkKind::SnapshotSave; - uint32_t key = 0; - uint32_t generation = 0; - const uint8_t* data = nullptr; - size_t len = 0; -}; - -template -class LatestSnapshotStorageRuntime -{ - public: - bool requestSave(uint32_t key, const uint8_t* data, size_t len) - { - if (len > MaxBytes || (len > 0 && data == nullptr)) - { - return false; - } - key_ = key; - len_ = len; - if (len > 0) - { - std::memcpy(buffer_.data(), data, len); - } - ++generation_; - pending_ = true; - state_ = (state_ == StorageWorkState::InFlight) - ? StorageWorkState::InFlight - : StorageWorkState::Pending; - return true; - } - - bool takeNext(StorageWorkItem& out) - { - if (!pending_) - { - return false; - } - pending_ = false; - in_flight_generation_ = generation_; - state_ = StorageWorkState::InFlight; - out.kind = StorageWorkKind::SnapshotSave; - out.key = key_; - out.generation = in_flight_generation_; - out.data = buffer_.data(); - out.len = len_; - return true; - } - - void complete(uint32_t generation, bool ok) - { - if (generation != in_flight_generation_) - { - return; - } - if (ok) - { - state_ = pending_ ? StorageWorkState::Pending : StorageWorkState::Idle; - last_completed_generation_ = generation; - return; - } - pending_ = true; - state_ = StorageWorkState::FailedPendingRetry; - } - - bool flushPending(IPlatformStorageAdapter& storage, - IEventSink& events, - const char* path, - uint32_t now_ms) - { - StorageWorkItem work{}; - if (!takeNext(work)) - { - return true; - } - - PlatformStorageWriteRequest request{}; - request.command_id = work.generation; - request.path = path; - request.data = work.data; - request.len = work.len; - request.durable = true; - const PlatformStorageResult result = storage.write(request); - complete(work.generation, result.ok); - - RuntimeEvent event{}; - event.event_id = work.generation; - event.kind = result.ok ? RuntimeEventKind::PersistenceSaved - : RuntimeEventKind::PersistenceFailed; - event.command_id = work.generation; - event.timestamp_ms = now_ms; - event.generation = work.generation; - event.error = result.error; - (void)events.publish(event); - return result.ok; - } - - bool pending() const { return pending_; } - bool busy() const { return state_ == StorageWorkState::InFlight; } - StorageWorkState state() const { return state_; } - uint32_t generation() const { return generation_; } - uint32_t lastCompletedGeneration() const { return last_completed_generation_; } - size_t len() const { return len_; } - - private: - std::array buffer_{}; - size_t len_ = 0; - uint32_t key_ = 0; - uint32_t generation_ = 0; - uint32_t in_flight_generation_ = 0; - uint32_t last_completed_generation_ = 0; - bool pending_ = false; - StorageWorkState state_ = StorageWorkState::Idle; -}; - -} // namespace sys::runtime diff --git a/modules/core_sys/tests/test_app_config_change_detection.cpp b/modules/core_sys/tests/test_app_config_change_detection.cpp index 17ae8f92..ff5695f9 100644 --- a/modules/core_sys/tests/test_app_config_change_detection.cpp +++ b/modules/core_sys/tests/test_app_config_change_detection.cpp @@ -1,5 +1,4 @@ #include "app/app_config_change_detection.h" -#include "app/app_config_save_plan.h" #include #include @@ -63,50 +62,5 @@ int main() assert(merged.containsAll(app::AppConfigChangeSet::gps())); assert(!merged.containsAll(app::AppConfigChangeSet::mesh())); - app::AppConfigSavePlan unchanged_plan = - app::planAppConfigSave(baseline, - true, - baseline, - false, - baseline, - app::AppConfigChangeSet::none(), - app::AppConfigChangeSet::none()); - assert(!unchanged_plan.queue); - assert(unchanged_plan.changes.empty()); - - app::AppConfigSavePlan busy_follow_up_plan = - app::planAppConfigSave(protocol_changed, - true, - baseline, - true, - protocol_changed, - app::AppConfigChangeSet::mesh(), - app::AppConfigChangeSet::none()); - assert(busy_follow_up_plan.queue); - assert(busy_follow_up_plan.changes.contains(app::AppConfigChangeDomain::Mesh)); - - app::AppConfigSavePlan changed_while_busy_plan = - app::planAppConfigSave(protocol_changed, - true, - map_changed, - true, - protocol_changed, - app::AppConfigChangeSet::mesh(), - app::AppConfigChangeSet::none()); - assert(changed_while_busy_plan.queue); - assert(changed_while_busy_plan.changes.contains(app::AppConfigChangeDomain::Map)); - - app::AppConfigSavePlan invalid_baseline_plan = - app::planAppConfigSave(baseline, - false, - baseline, - false, - baseline, - app::AppConfigChangeSet::none(), - app::AppConfigChangeSet::none()); - assert(invalid_baseline_plan.queue); - assert(invalid_baseline_plan.changes.containsAll( - app::AppConfigChangeSet::allPersisted())); - return 0; } diff --git a/modules/core_sys/tests/test_config_persistence_runtime.cpp b/modules/core_sys/tests/test_config_persistence_runtime.cpp new file mode 100644 index 00000000..7c00a5fd --- /dev/null +++ b/modules/core_sys/tests/test_config_persistence_runtime.cpp @@ -0,0 +1,193 @@ +#include "app/config_persistence_runtime.h" + +#include + +namespace +{ + +app::AppConfig changedMap(app::AppConfig value, uint8_t source) +{ + value.map_source = source; + return value; +} + +void testDebounceAndCompletion() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig desired = changedMap(baseline, 1U); + const app::ConfigPersistenceSubmission submission = + runtime.submit(desired, app::AppConfigChangeSet::none(), 100U); + assert(submission.queued); + assert(submission.generation == 1U); + assert(submission.changes.containsAll(app::AppConfigChangeSet::map())); + assert(!runtime.hasPending() || runtime.state() == + app::ConfigPersistenceState::Debouncing); + + app::ConfigPersistenceWork work; + assert(!runtime.takeDue(349U, work)); + assert(runtime.takeDue(350U, work)); + assert(work.snapshot != nullptr); + assert(work.snapshot->map_source == 1U); + assert(work.generation == 1U); + assert(runtime.busy()); + + assert(runtime.complete(work.generation, + app::ConfigPersistenceResultKind::Completed, + 351U) == + app::ConfigPersistenceResultKind::Completed); + assert(!runtime.busy()); + assert(!runtime.hasPending()); + assert(runtime.baselineValid()); + assert(runtime.lastCompletedGeneration() == 1U); + assert(runtime.state() == app::ConfigPersistenceState::Idle); +} + +void testPendingRollbackIsReconciledAgainstNewBaseline() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig first = changedMap(baseline, 1U); + assert(runtime.submit(first, app::AppConfigChangeSet::none(), 0U).queued); + app::ConfigPersistenceWork first_work; + assert(runtime.takeDue(250U, first_work)); + + const app::ConfigPersistenceSubmission rollback = + runtime.submit(baseline, app::AppConfigChangeSet::none(), 300U); + assert(rollback.queued); + assert(rollback.generation == 2U); + assert(runtime.hasPending()); + + assert(runtime.complete(first_work.generation, + app::ConfigPersistenceResultKind::Completed, + 400U) == + app::ConfigPersistenceResultKind::Completed); + assert(runtime.hasPending()); + assert(runtime.state() == app::ConfigPersistenceState::Debouncing); + assert(runtime.lastCompletedGeneration() == 1U); + + app::ConfigPersistenceWork rollback_work; + assert(!runtime.takeDue(649U, rollback_work)); + assert(runtime.takeDue(650U, rollback_work)); + assert(rollback_work.generation == 2U); + assert(rollback_work.snapshot->map_source == 0U); +} + +void testFailureRetriesTheFailedPayload() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig desired = changedMap(baseline, 2U); + assert(runtime.submit(desired, app::AppConfigChangeSet::none(), 10U).queued); + app::ConfigPersistenceWork work; + assert(runtime.takeDue(260U, work)); + + assert(runtime.complete(work.generation, + app::ConfigPersistenceResultKind::IoError, + 300U) == + app::ConfigPersistenceResultKind::IoError); + assert(runtime.baselineValid()); + assert(runtime.hasPending()); + assert(runtime.pendingGeneration() == work.generation); + assert(runtime.state() == app::ConfigPersistenceState::Backoff); + assert(!runtime.takeDue(1299U, work)); + assert(runtime.takeDue(1300U, work)); + assert(work.generation == 1U); + assert(work.snapshot->map_source == 2U); + assert(work.changes.contains(app::AppConfigChangeDomain::Map)); + assert(!work.changes.contains(app::AppConfigChangeDomain::Gps)); +} + +void testStaleCompletionCannotChangeState() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig first = changedMap(baseline, 1U); + assert(runtime.submit(first, app::AppConfigChangeSet::none(), 0U).queued); + app::ConfigPersistenceWork first_work; + assert(runtime.takeDue(250U, first_work)); + + const app::AppConfig second = changedMap(baseline, 2U); + assert(runtime.submit(second, app::AppConfigChangeSet::none(), 300U).queued); + assert(runtime.complete(2U, + app::ConfigPersistenceResultKind::Completed, + 301U) == + app::ConfigPersistenceResultKind::StaleGeneration); + assert(runtime.busy()); + assert(runtime.activeGeneration() == first_work.generation); + assert(runtime.complete(first_work.generation, + app::ConfigPersistenceResultKind::Completed, + 400U) == + app::ConfigPersistenceResultKind::Completed); + assert(runtime.hasPending()); + assert(runtime.takeDue(649U, first_work) == false); + assert(runtime.takeDue(650U, first_work)); + assert(first_work.generation == 2U); + assert(first_work.snapshot->map_source == 2U); +} + +void testCriticalSaveSkipsDebounce() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig desired = changedMap(baseline, 3U); + assert(runtime.submit(desired, + app::AppConfigChangeSet::none(), + 1000U, + app::ConfigPersistenceUrgency::Immediate) + .queued); + app::ConfigPersistenceWork work; + assert(runtime.takeDue(1000U, work)); + assert(work.generation == 1U); +} + +void testCriticalUrgencySurvivesGenerationHandoff() +{ + app::AppConfig baseline; + app::ConfigPersistenceRuntime runtime; + runtime.initialize(baseline); + + const app::AppConfig first = changedMap(baseline, 1U); + assert(runtime.submit(first, app::AppConfigChangeSet::none(), 0U).queued); + app::ConfigPersistenceWork first_work; + assert(runtime.takeDue(250U, first_work)); + + const app::AppConfig critical = changedMap(baseline, 2U); + assert(runtime.submit(critical, + app::AppConfigChangeSet::none(), + 300U, + app::ConfigPersistenceUrgency::Immediate) + .queued); + assert(runtime.complete(first_work.generation, + app::ConfigPersistenceResultKind::Completed, + 400U) == + app::ConfigPersistenceResultKind::Completed); + + app::ConfigPersistenceWork critical_work; + assert(runtime.takeDue(400U, critical_work)); + assert(critical_work.generation == 2U); + assert(critical_work.snapshot->map_source == 2U); +} + +} // namespace + +int main() +{ + testDebounceAndCompletion(); + testPendingRollbackIsReconciledAgainstNewBaseline(); + testFailureRetriesTheFailedPayload(); + testStaleCompletionCannotChangeState(); + testCriticalSaveSkipsDebounce(); + testCriticalUrgencySurvivesGenerationHandoff(); + return 0; +} diff --git a/modules/core_sys/tests/test_storage_event_runtime.cpp b/modules/core_sys/tests/test_storage_event_runtime.cpp deleted file mode 100644 index 46749182..00000000 --- a/modules/core_sys/tests/test_storage_event_runtime.cpp +++ /dev/null @@ -1,164 +0,0 @@ -#include "sys/storage_event_runtime.h" - -#include -#include - -using sys::runtime::LatestSnapshotStorageRuntime; -using sys::runtime::StorageWorkItem; -using sys::runtime::StorageWorkState; - -namespace -{ - -class FakeStorageAdapter final : public sys::runtime::IPlatformStorageAdapter -{ - public: - sys::runtime::PlatformStorageResult read( - const sys::runtime::PlatformStorageReadRequest& request) override - { - (void)request; - return {}; - } - - sys::runtime::PlatformStorageResult write( - const sys::runtime::PlatformStorageWriteRequest& request) override - { - writes += 1; - last_path = request.path; - last_len = request.len; - sys::runtime::PlatformStorageResult result{}; - result.ok = write_ok; - result.bytes = write_ok ? request.len : 0; - result.error = write_ok ? 0 : -7; - return result; - } - - sys::runtime::PlatformStorageResult list( - const sys::runtime::PlatformStorageListRequest& request) override - { - (void)request; - return {}; - } - - sys::runtime::PlatformStorageResult flush( - const sys::runtime::PlatformStorageFlushRequest& request) override - { - (void)request; - sys::runtime::PlatformStorageResult result{}; - result.ok = true; - return result; - } - - bool write_ok = true; - int writes = 0; - const char* last_path = nullptr; - std::size_t last_len = 0; -}; - -void burst_updates_coalesce_to_latest_snapshot() -{ - LatestSnapshotStorageRuntime<16> runtime; - const uint8_t first[] = {1, 2, 3}; - const uint8_t second[] = {9, 8}; - - assert(runtime.requestSave(42, first, sizeof(first))); - assert(runtime.requestSave(42, second, sizeof(second))); - - StorageWorkItem work; - assert(runtime.takeNext(work)); - assert(work.key == 42); - assert(work.generation == 2); - assert(work.len == sizeof(second)); - assert(std::memcmp(work.data, second, sizeof(second)) == 0); -} - -void completion_ignores_stale_generation() -{ - LatestSnapshotStorageRuntime<16> runtime; - const uint8_t first[] = {1}; - const uint8_t second[] = {2}; - - assert(runtime.requestSave(1, first, sizeof(first))); - StorageWorkItem work; - assert(runtime.takeNext(work)); - assert(runtime.requestSave(1, second, sizeof(second))); - - runtime.complete(work.generation + 100, true); - assert(runtime.busy()); - - runtime.complete(work.generation, true); - assert(runtime.pending()); - assert(runtime.state() == StorageWorkState::Pending); -} - -void failed_save_is_retried_with_same_generation() -{ - LatestSnapshotStorageRuntime<16> runtime; - const uint8_t blob[] = {4, 5, 6}; - - assert(runtime.requestSave(7, blob, sizeof(blob))); - StorageWorkItem first; - assert(runtime.takeNext(first)); - runtime.complete(first.generation, false); - - assert(runtime.pending()); - assert(runtime.state() == StorageWorkState::FailedPendingRetry); - - StorageWorkItem retry; - assert(runtime.takeNext(retry)); - assert(retry.generation == first.generation); - assert(retry.len == sizeof(blob)); - assert(std::memcmp(retry.data, blob, sizeof(blob)) == 0); -} - -void flush_pending_writes_through_storage_port_and_publishes_event() -{ - LatestSnapshotStorageRuntime<16> runtime; - FakeStorageAdapter storage; - sys::runtime::FixedEventSink<4> events; - const uint8_t blob[] = {1, 3, 5, 7}; - - assert(runtime.requestSave(12, blob, sizeof(blob))); - assert(runtime.flushPending(storage, events, "/nodes.bin", 50)); - assert(storage.writes == 1); - assert(storage.last_path != nullptr); - assert(std::strcmp(storage.last_path, "/nodes.bin") == 0); - assert(storage.last_len == sizeof(blob)); - assert(!runtime.pending()); - - sys::runtime::RuntimeEvent event{}; - assert(events.pop(event)); - assert(event.kind == sys::runtime::RuntimeEventKind::PersistenceSaved); - assert(event.timestamp_ms == 50); -} - -void failed_flush_keeps_latest_snapshot_pending_for_retry() -{ - LatestSnapshotStorageRuntime<16> runtime; - FakeStorageAdapter storage; - storage.write_ok = false; - sys::runtime::FixedEventSink<4> events; - const uint8_t blob[] = {2, 4, 6}; - - assert(runtime.requestSave(9, blob, sizeof(blob))); - assert(!runtime.flushPending(storage, events, "/nodes.bin", 60)); - assert(runtime.pending()); - assert(runtime.state() == StorageWorkState::FailedPendingRetry); - - sys::runtime::RuntimeEvent event{}; - assert(events.pop(event)); - assert(event.kind == sys::runtime::RuntimeEventKind::PersistenceFailed); - assert(event.error == -7); -} - -} // namespace - -int main() -{ - burst_updates_coalesce_to_latest_snapshot(); - completion_ignores_stale_generation(); - failed_save_is_retried_with_same_generation(); - flush_pending_writes_through_storage_port_and_publishes_event(); - failed_flush_keeps_latest_snapshot_pending_for_retry(); - return 0; -} diff --git a/modules/core_sys/tests/test_ui_storage_event_runtime_contract.cpp b/modules/core_sys/tests/test_ui_storage_event_runtime_contract.cpp index 9c935878..5dc0742d 100644 --- a/modules/core_sys/tests/test_ui_storage_event_runtime_contract.cpp +++ b/modules/core_sys/tests/test_ui_storage_event_runtime_contract.cpp @@ -1,6 +1,5 @@ #include "gps/track_runtime.h" #include "sys/feedback_runtime.h" -#include "sys/persistence_runtime.h" #include "sys/runtime_harness.h" #include "ui_map_runtime/map_tiles/map_tile_async_runtime.h" @@ -190,28 +189,6 @@ void test_map_tile_runtime_contract() assert(runtime.snapshot().ready_count == 1); } -void test_persistence_runtime_contract() -{ - sys::runtime::RuntimeHarness harness; - sys::runtime::DirtyStoreRegistry<4> registry; - sys::runtime::DefaultPersistencePolicy policy; - sys::runtime::PersistenceWorker worker(harness.storage(), - harness.storage(), - harness.events(), - policy); - sys::runtime::PersistenceRuntime<4> runtime(registry, - worker, - harness.events(), - policy); - - assert(runtime.markDirty("nodes", 0)); - runtime.tick(100); - assert(harness.storage().writeCount() == 0); - runtime.tick(150); - assert(harness.storage().writeCount() == 1); - assert(harness.events().persistenceCount() >= 3); -} - void test_feedback_runtime_contract() { sys::runtime::RuntimeHarness harness; @@ -379,7 +356,6 @@ void test_runtime_harness_keeps_ui_drain_separate() int main() { test_map_tile_runtime_contract(); - test_persistence_runtime_contract(); test_feedback_runtime_contract(); test_track_runtime_contract(); test_track_worker_failure_completes_semantically(); diff --git a/modules/ui_mono/include/ui/mono/runtime.h b/modules/ui_mono/include/ui/mono/runtime.h index e0666690..81128faf 100644 --- a/modules/ui_mono/include/ui/mono/runtime.h +++ b/modules/ui_mono/include/ui/mono/runtime.h @@ -223,7 +223,6 @@ class Runtime : public chat::ChatService::IncomingTextObserver, void retrySelectedMessage(); void executeNewChatPageItem(size_t index); void executeDiscoverPageItem(size_t index); - void commitConfig(); void ensureBootExit(); void ensureSleepTimeout(InputAction action); void beginSettingPopup(Page owner, size_t index); diff --git a/modules/ui_mono/src/runtime.cpp b/modules/ui_mono/src/runtime.cpp index 5f00fbe3..b6442312 100644 --- a/modules/ui_mono/src/runtime.cpp +++ b/modules/ui_mono/src/runtime.cpp @@ -6102,15 +6102,6 @@ void Runtime::executeDiscoverPageItem(size_t index) showTransientPopup("DISCOVER", meshOperationFailureLabel(result.failure)); } -void Runtime::commitConfig() -{ - if (!app()) - { - return; - } - app()->saveConfig(); -} - void Runtime::ensureBootExit() { if (page_ == Page::BootLog && (nowMs() - boot_started_ms_) >= kBootMinMs) @@ -6215,12 +6206,20 @@ void Runtime::confirmSettingPopup() } } - auto& cfg = app()->getConfig(); sanitizeMeshtasticChannelNum(setting_popup_config_); - cfg = setting_popup_config_; #if !TRAILMATE_NRF52_BLE_DISABLED - cfg.ble_enabled = setting_popup_ble_enabled_; + const bool ble_changed = app()->isBleEnabled() != setting_popup_ble_enabled_; #endif + auto edit = app()->beginConfigEdit(); + if (!edit) + { + return; + } + edit.config() = setting_popup_config_; +#if !TRAILMATE_NRF52_BLE_DISABLED + edit.config().ble_enabled = setting_popup_ble_enabled_; +#endif + edit.commit(app::AppConfigChangeSet::allPersisted()); if (host_.set_timezone_offset_min_fn) { host_.set_timezone_offset_min_fn(setting_popup_timezone_min_); @@ -6252,9 +6251,11 @@ void Runtime::confirmSettingPopup() } platform::ui::screen::set_timeout_ms(setting_popup_screen_timeout_ms_); #if !TRAILMATE_NRF52_BLE_DISABLED - app()->setBleEnabled(setting_popup_ble_enabled_); + if (ble_changed) + { + app()->setBleEnabled(setting_popup_ble_enabled_); + } #endif - app()->saveConfig(); if (setting_popup_owner_ == Page::RadioSettings) { app()->applyMeshConfig(); @@ -7325,7 +7326,13 @@ bool Runtime::saveEditedTextToConfig() return false; } - auto& cfg = app()->getConfig(); + auto edit = app()->beginConfigEdit(); + if (!edit) + { + return false; + } + + auto& cfg = edit.config(); bool mesh_config_changed = false; switch (edit_target_) { @@ -7360,7 +7367,8 @@ bool Runtime::saveEditedTextToConfig() default: break; } - commitConfig(); + edit.commit(mesh_config_changed ? app::AppConfigChangeSet::mesh() + : app::AppConfigChangeSet::none()); if (mesh_config_changed) { app()->applyMeshConfig(); diff --git a/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h b/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h index 3cf9085d..c936c5a5 100644 --- a/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h +++ b/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h @@ -182,6 +182,7 @@ class UiController : public IChatUiRefreshSink ::ui::chat::ChatWorkspaceSnapshot team_chat_snapshot_buffer_{}; bool conversation_list_dirty_ = true; bool conversation_list_loaded_ = false; + bool conversation_view_loaded_ = false; bool receive_status_visible_ = false; static void key_verify_submit_event_cb(lv_event_t* e); diff --git a/modules/ui_shared/src/ui/app_runtime.cpp b/modules/ui_shared/src/ui/app_runtime.cpp index b649f791..a8171150 100644 --- a/modules/ui_shared/src/ui/app_runtime.cpp +++ b/modules/ui_shared/src/ui/app_runtime.cpp @@ -219,7 +219,6 @@ void show_menu_internal() #endif std::printf("[UI][Lifecycle] menu visible=1 active=%s scene=menu\n", s_active_app ? s_active_app->name() : ""); - ui_clear_active_app(); set_default_group(menu_g); ui::menu_layout::setMenuVisible(true); ui::menu_runtime::setScene(ui::menu_runtime::Scene::Menu); @@ -377,6 +376,17 @@ void menu_show() { return; } + + // Showing the menu while an app is active is a lifecycle transition, not + // a visual shortcut. The app owns timers, LVGL children, and device + // leases that must be released by its exit callback before the menu can + // become the active scene. + if (s_active_app != nullptr) + { + ui_request_exit_to_menu(); + return; + } + show_menu_internal(); } diff --git a/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp b/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp index f831ecab..c74abfb1 100644 --- a/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp +++ b/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp @@ -401,6 +401,16 @@ bool ChatPresentationSource::buildChatWorkspaceSnapshot( static_cast(request.conversation_offset), static_cast(request.message_offset)); ui::chat::resetChatWorkspaceSnapshot(out); + if (!chat_service_.isDataReady()) + { + CHAT_SNAPSHOT_TRACE( + "[ChatUiTrace] stage=snapshot_source reject reason=data_not_ready elapsed_ms=%lld\n", + static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started) + .count())); + return false; + } out.header.valid = true; out.header.version = 1; ui::copyText(out.workspace_title, "Chat"); diff --git a/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp b/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp index fb40a328..f836c655 100644 --- a/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp +++ b/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp @@ -785,6 +785,14 @@ void UiController::update() { // Refresh UI only when an event marks the conversation list dirty. refreshUnreadCounts(false); + if (state_ == State::Conversation && conversation_ && + !team_conv_active_ && !conversation_view_loaded_) + { + // A visible thread has its own projection state. Do not let a + // temporary not-ready result during page entry become a permanent + // empty message list just because the conversation list has loaded. + reloadConversationView(); + } const auto receive = ::platform::ui::reticulum_receive::snapshot(); if (state_ == State::Conversation && conversation_ && (receive.active || receive_status_visible_)) @@ -1013,6 +1021,7 @@ void UiController::switchToConversation(chat::ConversationId conv) current_channel_ = conv.channel; current_conv_ = conv; team_conv_active_ = isTeamConversation(conv); + conversation_view_loaded_ = false; stopTeamConversationTimer(); CHAT_UI_LOG("[UiController] switchToConversation: parent=%p active=%p sleeping=%d conv_peer=%08lX\n", parent_, lv_screen_active(), platform::ui::screen::is_sleeping() ? 1 : 0, @@ -1092,6 +1101,7 @@ void UiController::switchToConversation(chat::ConversationId conv) { applySnapshotMessagesToConversation(team_chat_snapshot_buffer_, *conversation_); } + conversation_view_loaded_ = loaded; startTeamConversationTimer(); if (loaded && unread != 0) { @@ -1159,6 +1169,7 @@ void UiController::switchToConversation(chat::ConversationId conv) { applySnapshotMessagesToConversation(chat_snapshot_buffer_, *conversation_); } + conversation_view_loaded_ = snapshot_loaded; CHAT_UI_LOG("[ChatUiTrace] stage=switch_conversation mark_read begin elapsed_ms=%lu\n", static_cast(lv_tick_elaps(started_ms))); const ::ui::UiActionResult mark_read_result = chat_model_.markRead(ui_conv); @@ -1626,23 +1637,32 @@ void UiController::refreshUnreadCounts(const bool force_reload) void UiController::syncConversationListFromStore() { - cached_conversations_.clear(); - if (loadChatSnapshot()) + const bool chat_loaded = loadChatSnapshot(); + std::vector next_conversations; + if (chat_loaded) { appendSnapshotConversationsToControllerList(chat_snapshot_buffer_, - cached_conversations_); + next_conversations); } - normalizeConversationNames(cached_conversations_); + normalizeConversationNames(next_conversations); chat::ConversationMeta team_conv; - if (loadTeamChatSnapshot() && + if (chat_loaded && loadTeamChatSnapshot() && teamConversationMetaFromSnapshot(team_chat_snapshot_buffer_, team_conv)) { - cached_conversations_.insert(cached_conversations_.begin(), team_conv); + next_conversations.insert(next_conversations.begin(), team_conv); } - conversation_list_dirty_ = false; - conversation_list_loaded_ = true; + if (chat_loaded) + { + cached_conversations_.swap(next_conversations); + } + + // A not-ready store is retryable. Do not turn its temporary empty + // projection into a permanently loaded conversation list or erase the + // last visible projection when a retry fails. + conversation_list_dirty_ = !chat_loaded; + conversation_list_loaded_ = chat_loaded; } void UiController::normalizeConversationNames(std::vector& convs) const @@ -1777,7 +1797,9 @@ void UiController::reloadConversationView() return; } - if (!loadChatSnapshot()) + const bool snapshot_loaded = loadChatSnapshot(); + conversation_view_loaded_ = snapshot_loaded; + if (!snapshot_loaded) { return; } diff --git a/modules/ui_shared/src/ui/screens/contacts/contacts_page_components.cpp b/modules/ui_shared/src/ui/screens/contacts/contacts_page_components.cpp index 6b7b108c..98b84b43 100644 --- a/modules/ui_shared/src/ui/screens/contacts/contacts_page_components.cpp +++ b/modules/ui_shared/src/ui/screens/contacts/contacts_page_components.cpp @@ -2124,9 +2124,16 @@ static void show_reticulum_group_error(const char* message) static bool reticulum_group_storage_ready_for_edit() { - app::AppConfig& config = app::configFacade().getConfig(); + auto groups = std::unique_ptr( + new (std::nothrow) + chat::ReticulumGroupDestinationConfig[chat::kReticulumGroupDestinationMaxCount]); + if (!groups) + { + return false; + } + const auto status = ::platform::ui::reticulum_groups::load( - config.reticulumConfig().reticulum_groups, + groups.get(), chat::kReticulumGroupDestinationMaxCount); g_contacts_state.reticulum_group_storage_supported = status.supported; g_contacts_state.reticulum_group_storage_ready = status.sd_present; @@ -3530,12 +3537,29 @@ static void on_reticulum_group_save_clicked(lv_event_t* /*e*/) return; } - app::AppConfig& config = app::configFacade().getConfig(); - chat::MeshConfig& reticulum_config = config.reticulumConfig(); + auto groups = std::unique_ptr( + new (std::nothrow) + chat::ReticulumGroupDestinationConfig[chat::kReticulumGroupDestinationMaxCount]); + if (!groups) + { + show_reticulum_group_error("Group storage unavailable"); + return; + } + const auto load_status = ::platform::ui::reticulum_groups::load( + groups.get(), + chat::kReticulumGroupDestinationMaxCount); + if (!load_status.loaded) + { + show_reticulum_group_error( + load_status.message[0] != '\0' ? load_status.message + : "Cannot load groups"); + return; + } + int free_slot = -1; for (std::size_t index = 0; index < chat::kReticulumGroupDestinationMaxCount; ++index) { - auto& group = reticulum_config.reticulum_groups[index]; + auto& group = groups[index]; if (group.enabled && chat::hasReticulumDestinationIdentity(group.identity) && chat::sameReticulumDestinationHash(group.identity, identity)) { @@ -3555,16 +3579,23 @@ static void on_reticulum_group_save_clicked(lv_event_t* /*e*/) return; } - auto& group = reticulum_config.reticulum_groups[free_slot]; + auto& group = groups[free_slot]; group = chat::ReticulumGroupDestinationConfig{}; group.enabled = true; std::snprintf(group.name, sizeof(group.name), "%s", name); group.identity = identity; - const auto save_status = ::platform::ui::reticulum_groups::save( - reticulum_config.reticulum_groups, + app::AppConfigEdit edit = app::configFacade().beginConfigEdit(); + if (!edit) + { + show_reticulum_group_error("Configuration busy"); + return; + } + + const auto save_status = ::platform::ui::reticulum_groups::submit( + groups.get(), chat::kReticulumGroupDestinationMaxCount); - if (!save_status.saved) + if (!save_status.queued) { show_reticulum_group_error(save_status.message[0] != '\0' ? save_status.message @@ -3572,6 +3603,11 @@ static void on_reticulum_group_save_clicked(lv_event_t* /*e*/) return; } + std::memcpy(edit.config().reticulumConfig().reticulum_groups, + groups.get(), + chat::kReticulumGroupDestinationMaxCount * + sizeof(chat::ReticulumGroupDestinationConfig)); + edit.commit(app::AppConfigChangeSet::mesh()); app::configFacade().applyMeshConfig(); g_contacts_state.reticulum_group_name_textarea = nullptr; diff --git a/modules/ui_shared/src/ui/screens/contacts/contacts_page_runtime.cpp b/modules/ui_shared/src/ui/screens/contacts/contacts_page_runtime.cpp index 2fb560e4..f3855ed6 100644 --- a/modules/ui_shared/src/ui/screens/contacts/contacts_page_runtime.cpp +++ b/modules/ui_shared/src/ui/screens/contacts/contacts_page_runtime.cpp @@ -50,6 +50,54 @@ constexpr std::size_t kReticulumDirectoryProjectionLimit = 100; const Host* s_host = nullptr; +void destroy_contacts_page_runtime() +{ + if (g_contacts_state.compose_screen) + { + if (g_contacts_state.compose_ime) + { + g_contacts_state.compose_ime->detach(); + delete g_contacts_state.compose_ime; + g_contacts_state.compose_ime = nullptr; + } + delete g_contacts_state.compose_screen; + g_contacts_state.compose_screen = nullptr; + } + if (g_contacts_state.conversation_screen) + { + delete g_contacts_state.conversation_screen; + g_contacts_state.conversation_screen = nullptr; + } + if (g_contacts_state.conversation_timer != nullptr) + { + lv_timer_del(g_contacts_state.conversation_timer); + g_contacts_state.conversation_timer = nullptr; + } + if (g_contacts_state.discover_scan_timer != nullptr) + { + lv_timer_del(g_contacts_state.discover_scan_timer); + g_contacts_state.discover_scan_timer = nullptr; + } + if (g_contacts_state.refresh_timer != nullptr) + { + lv_timer_del(g_contacts_state.refresh_timer); + g_contacts_state.refresh_timer = nullptr; + } + + cleanup_modals(); + cleanup_contacts_input(); + + if (g_contacts_state.root != nullptr) + { + lv_obj_del(g_contacts_state.root); + g_contacts_state.root = nullptr; + } + + // Clear every handle after the object tree is gone. Re-entering the page + // must never let refresh_ui() observe children from the previous tree. + g_contacts_state = ContactsPageState{}; +} + static uint32_t hash_step(uint32_t hash, uint8_t byte) { return (hash ^ byte) * 16777619U; @@ -265,28 +313,6 @@ void copy_text(char* out, size_t out_len, const char* text) std::snprintf(out, out_len, "%s", text ? text : ""); } -bool same_reticulum_groups(const chat::ReticulumGroupDestinationConfig* lhs, - const chat::ReticulumGroupDestinationConfig* rhs, - std::size_t count) -{ - if (!lhs || !rhs) - { - return lhs == rhs; - } - for (std::size_t index = 0; index < count; ++index) - { - if (lhs[index].enabled != rhs[index].enabled || - std::strncmp(lhs[index].name, - rhs[index].name, - sizeof(lhs[index].name)) != 0 || - !chat::sameReticulumPeerIdentity(lhs[index].identity, rhs[index].identity)) - { - return false; - } - } - return true; -} - void refresh_reticulum_group_storage_state(const platform::ui::reticulum_groups::Status& status) { g_contacts_state.reticulum_group_storage_supported = status.supported; @@ -520,28 +546,27 @@ void refresh_reticulum_groups_data() return; } - app::AppConfig& config = app::configFacade().getConfig(); - chat::MeshConfig& reticulum_config = config.reticulumConfig(); - chat::ReticulumGroupDestinationConfig previous[chat::kReticulumGroupDestinationMaxCount] = {}; - std::memcpy(previous, - reticulum_config.reticulum_groups, - sizeof(previous)); - const auto status = platform::ui::reticulum_groups::load( - reticulum_config.reticulum_groups, - chat::kReticulumGroupDestinationMaxCount); - refresh_reticulum_group_storage_state(status); - if (!same_reticulum_groups(previous, - reticulum_config.reticulum_groups, - chat::kReticulumGroupDestinationMaxCount)) + auto groups = std::unique_ptr( + new (std::nothrow) + chat::ReticulumGroupDestinationConfig[chat::kReticulumGroupDestinationMaxCount]); + if (!groups) { - app::configFacade().applyMeshConfig(); + g_contacts_state.reticulum_group_storage_loaded = false; + std::snprintf(g_contacts_state.reticulum_group_storage_message, + sizeof(g_contacts_state.reticulum_group_storage_message), + "%s", + "Reticulum groups unavailable"); + return; } - const chat::MeshConfig& mesh_config = config.activeMeshConfig(); + const auto status = platform::ui::reticulum_groups::load( + groups.get(), + chat::kReticulumGroupDestinationMaxCount); + refresh_reticulum_group_storage_state(status); + for (std::size_t index = 0; index < chat::kReticulumGroupDestinationMaxCount; ++index) { - const chat::ReticulumGroupDestinationConfig& group = - mesh_config.reticulum_groups[index]; + const chat::ReticulumGroupDestinationConfig& group = groups[index]; if (!group.enabled || !chat::hasReticulumDestinationIdentity(group.identity)) { @@ -644,8 +669,7 @@ void enter(const shell::Host* host, lv_obj_t* parent) if (g_contacts_state.root != nullptr) { - lv_obj_del(g_contacts_state.root); - g_contacts_state.root = nullptr; + destroy_contacts_page_runtime(); } g_contacts_state.exiting = false; @@ -716,48 +740,7 @@ void exit(lv_obj_t* parent) CONTACTS_LOG("[Contacts] Exiting Contacts page\n"); - if (g_contacts_state.compose_screen) - { - if (g_contacts_state.compose_ime) - { - g_contacts_state.compose_ime->detach(); - delete g_contacts_state.compose_ime; - g_contacts_state.compose_ime = nullptr; - } - delete g_contacts_state.compose_screen; - g_contacts_state.compose_screen = nullptr; - } - if (g_contacts_state.conversation_screen) - { - delete g_contacts_state.conversation_screen; - g_contacts_state.conversation_screen = nullptr; - } - if (g_contacts_state.conversation_timer != nullptr) - { - lv_timer_del(g_contacts_state.conversation_timer); - g_contacts_state.conversation_timer = nullptr; - } - if (g_contacts_state.discover_scan_timer != nullptr) - { - lv_timer_del(g_contacts_state.discover_scan_timer); - g_contacts_state.discover_scan_timer = nullptr; - } - if (g_contacts_state.refresh_timer != nullptr) - { - lv_timer_del(g_contacts_state.refresh_timer); - g_contacts_state.refresh_timer = nullptr; - } - - cleanup_modals(); - cleanup_contacts_input(); - - if (g_contacts_state.root != nullptr) - { - lv_obj_del(g_contacts_state.root); - g_contacts_state.root = nullptr; - } - - g_contacts_state = ContactsPageState{}; + destroy_contacts_page_runtime(); CONTACTS_LOG("[Contacts] Contacts page cleaned up\n"); s_host = nullptr; } diff --git a/modules/ui_shared/src/ui/screens/gps/gps_page_runtime.cpp b/modules/ui_shared/src/ui/screens/gps/gps_page_runtime.cpp index 3ba2c8bf..f60dc0c9 100644 --- a/modules/ui_shared/src/ui/screens/gps/gps_page_runtime.cpp +++ b/modules/ui_shared/src/ui/screens/gps/gps_page_runtime.cpp @@ -4458,6 +4458,10 @@ void exit(lv_obj_t* parent) lv_obj_del(s_root); s_root = nullptr; } + // The LVGL object tree owns every label in the top bar. Clear this + // non-owning view immediately after deleting the tree so a stale callback + // cannot bind or render through a freed label. + s_top_bar = {}; const bool was_gps_status = s_projection == Projection::GpsStatus; s_host = nullptr; s_projection = Projection::Map; diff --git a/modules/ui_shared/src/ui/screens/settings/settings_page_components.cpp b/modules/ui_shared/src/ui/screens/settings/settings_page_components.cpp index cddd1c5a..be5e8e1e 100644 --- a/modules/ui_shared/src/ui/screens/settings/settings_page_components.cpp +++ b/modules/ui_shared/src/ui/screens/settings/settings_page_components.cpp @@ -1271,6 +1271,22 @@ static void apply_reticulum_bearer_policy(chat::MeshConfig& config, } } +template +static bool commit_app_config(app::IAppFacade& app_ctx, + app::AppConfigChangeSet changes, + Mutator mutator) +{ + auto edit = app_ctx.beginConfigEdit(); + if (!edit) + { + return false; + } + + mutator(edit.config()); + edit.commit(changes); + return true; +} + static bool reticulum_wifi_settings_visible() { return reticulum_bearer_policy_from_value(g_settings.rt_bearer_policy) != @@ -1286,15 +1302,24 @@ static bool reticulum_lora_settings_visible() static void reset_mesh_settings() { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config = chat::MeshConfig(); - app_ctx.getConfig().meshtastic_config.region = app::AppConfig::kDefaultRegionCode; - app_ctx.getConfig().applyMeshtasticMqttFactoryDefaults(); - app_ctx.getConfig().meshcore_config = chat::MeshConfig(); - app_ctx.getConfig().applyMeshCoreFactoryDefaults(); - app_ctx.getConfig().reticulumConfig() = chat::MeshConfig(); - app_ctx.getConfig().applyReticulumFactoryDefaults(); - app_ctx.getConfig().meshcore_config.resetMeshCoreChannels(); - app_ctx.saveConfig(); + if (!commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [](app::AppConfig& config) + { + config.meshtastic_config = chat::MeshConfig(); + config.meshtastic_config.region = + app::AppConfig::kDefaultRegionCode; + config.applyMeshtasticMqttFactoryDefaults(); + config.meshcore_config = chat::MeshConfig(); + config.applyMeshCoreFactoryDefaults(); + config.reticulumConfig() = chat::MeshConfig(); + config.applyReticulumFactoryDefaults(); + config.meshcore_config.resetMeshCoreChannels(); + })) + { + return; + } app_ctx.applyMeshConfig(); g_settings.chat_protocol = static_cast(app_ctx.getConfig().mesh_protocol); @@ -2201,20 +2226,32 @@ static void on_text_save_clicked(lv_event_t* e) if (is_user_name) { app::IAppFacade& app_ctx = app::appFacade(); - strncpy(app_ctx.getConfig().node_name, g_state.editing_item->text_value, - sizeof(app_ctx.getConfig().node_name) - 1); - app_ctx.getConfig().node_name[sizeof(app_ctx.getConfig().node_name) - 1] = '\0'; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::identity(), + [&](app::AppConfig& config) + { + strncpy(config.node_name, + g_state.editing_item->text_value, + sizeof(config.node_name) - 1); + config.node_name[sizeof(config.node_name) - 1] = '\0'; + }); app_ctx.applyUserInfo(); broadcast_nodeinfo = true; } if (is_short_name) { app::IAppFacade& app_ctx = app::appFacade(); - strncpy(app_ctx.getConfig().short_name, g_state.editing_item->text_value, - sizeof(app_ctx.getConfig().short_name) - 1); - app_ctx.getConfig().short_name[sizeof(app_ctx.getConfig().short_name) - 1] = '\0'; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::identity(), + [&](app::AppConfig& config) + { + strncpy(config.short_name, + g_state.editing_item->text_value, + sizeof(config.short_name) - 1); + config.short_name[sizeof(config.short_name) - 1] = '\0'; + }); app_ctx.applyUserInfo(); broadcast_nodeinfo = true; } @@ -2225,53 +2262,65 @@ static void on_text_save_clicked(lv_event_t* e) if (id == settings::ui::SettingId::MtPrimaryName) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshtastic_config.primary_channel_name, - sizeof(app_ctx.getConfig().meshtastic_config.primary_channel_name), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshtastic_config.primary_channel_name, + sizeof(config.meshtastic_config.primary_channel_name), + g_state.editing_item->text_value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::MtSecondaryName) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshtastic_config.secondary_channel_name, - sizeof(app_ctx.getConfig().meshtastic_config.secondary_channel_name), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshtastic_config.secondary_channel_name, + sizeof(config.meshtastic_config.secondary_channel_name), + g_state.editing_item->text_value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::MtPrimaryKey) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshtastic_config; - if (!::settings::ui::channel::parse_meshtastic_key_text( + auto edit = app_ctx.beginConfigEdit(); + if (!edit || + !::settings::ui::channel::parse_meshtastic_key_text( g_state.editing_item->text_value, - mesh.primary_key, - sizeof(mesh.primary_key), - &mesh.primary_key_len)) + edit.config().meshtastic_config.primary_key, + sizeof(edit.config().meshtastic_config.primary_key), + &edit.config().meshtastic_config.primary_key_len)) { ::ui::feedback::show_notice(::ui::i18n::tr("PSK must be 32/64 hex or 16/32 chars"), 4000); modal_close(); return; } - app_ctx.saveConfig(); + edit.commit(app::AppConfigChangeSet::channels()); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::MtSecondaryKey) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshtastic_config; - if (!::settings::ui::channel::parse_meshtastic_key_text( + auto edit = app_ctx.beginConfigEdit(); + if (!edit || + !::settings::ui::channel::parse_meshtastic_key_text( g_state.editing_item->text_value, - mesh.secondary_key, - sizeof(mesh.secondary_key), - &mesh.secondary_key_len)) + edit.config().meshtastic_config.secondary_key, + sizeof(edit.config().meshtastic_config.secondary_key), + &edit.config().meshtastic_config.secondary_key_len)) { ::ui::feedback::show_notice(::ui::i18n::tr("PSK must be 32/64 hex or 16/32 chars"), 4000); modal_close(); return; } - app_ctx.saveConfig(); + edit.commit(app::AppConfigChangeSet::channels()); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::ChatPsk) @@ -2292,9 +2341,15 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - if (app_ctx.getConfig().mesh_protocol == chat::MeshProtocol::MeshCore) + auto edit = app_ctx.beginConfigEdit(); + if (!edit) { - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; + modal_close(); + return; + } + if (edit.config().mesh_protocol == chat::MeshProtocol::MeshCore) + { + chat::MeshConfig& mesh = edit.config().meshcore_config; const uint8_t slot = chat::normalizeMeshCoreChannelSlot(mesh.meshcore_channel_slot); chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); @@ -2308,7 +2363,7 @@ static void on_text_save_clicked(lv_event_t* e) } else { - auto& mesh = app_ctx.getConfig().meshtastic_config; + auto& mesh = edit.config().meshtastic_config; memset(mesh.secondary_key, 0, sizeof(mesh.secondary_key)); memcpy(mesh.secondary_key, key, parsed_key_len); mesh.secondary_key_len = @@ -2316,7 +2371,7 @@ static void on_text_save_clicked(lv_event_t* e) sizeof(mesh.secondary_key), static_cast(parsed_key_len)); } - app_ctx.saveConfig(); + edit.commit(app::AppConfigChangeSet::channels()); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::NetFreqOffset) @@ -2329,8 +2384,13 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - app_ctx.getConfig().meshtastic_config.frequency_offset_mhz = value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshtastic_config.frequency_offset_mhz = value; + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::NetOverrideFreq) @@ -2343,15 +2403,20 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().override_frequency_mhz = value; - } - else - { - app_ctx.getConfig().meshtastic_config.override_frequency_mhz = value; - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + if (chat::infra::isReticulumMeshProtocol(config.mesh_protocol)) + { + config.reticulumConfig().override_frequency_mhz = value; + } + else + { + config.meshtastic_config.override_frequency_mhz = value; + } + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McFreq) @@ -2364,10 +2429,15 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - app_ctx.getConfig().meshcore_config.meshcore_freq_mhz = value; - app_ctx.getConfig().meshcore_config.meshcore_region_preset = 0; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshcore_config.meshcore_freq_mhz = value; + config.meshcore_config.meshcore_region_preset = 0; + }); g_settings.mc_region_preset = 0; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McBw) @@ -2380,10 +2450,15 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - app_ctx.getConfig().meshcore_config.meshcore_bw_khz = value; - app_ctx.getConfig().meshcore_config.meshcore_region_preset = 0; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshcore_config.meshcore_bw_khz = value; + config.meshcore_config.meshcore_region_preset = 0; + }); g_settings.mc_region_preset = 0; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McRxDelay) @@ -2396,8 +2471,13 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - app_ctx.getConfig().meshcore_config.meshcore_rx_delay_base = value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshcore_config.meshcore_rx_delay_base = value; + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McAirtime) @@ -2410,26 +2490,39 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - app_ctx.getConfig().meshcore_config.meshcore_airtime_factor = value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshcore_config.meshcore_airtime_factor = value; + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McChannelName) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; const uint8_t slot = chat::normalizeMeshCoreChannelSlot(static_cast(g_settings.mc_channel_slot)); - chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); - copy_bounded(channel.name, sizeof(channel.name), g_state.editing_item->text_value); - if (slot != 0 && channel.name[0] != '\0') - { - channel.enabled = true; - } - mesh.meshcore_channel_slot = slot; - mesh.syncMeshCoreLegacyChannelMirror(); - ::settings::ui::channel::sync_meshcore_channel_fields(app_ctx.getConfig(), g_settings); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [&](app::AppConfig& config) + { + chat::MeshConfig& mesh = config.meshcore_config; + chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); + copy_bounded(channel.name, + sizeof(channel.name), + g_state.editing_item->text_value); + if (slot != 0 && channel.name[0] != '\0') + { + channel.enabled = true; + } + mesh.meshcore_channel_slot = slot; + mesh.syncMeshCoreLegacyChannelMirror(); + ::settings::ui::channel::sync_meshcore_channel_fields(config, + g_settings); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McChannelKey) @@ -2444,19 +2537,25 @@ static void on_text_save_clicked(lv_event_t* e) modal_close(); return; } - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; const uint8_t slot = chat::normalizeMeshCoreChannelSlot(static_cast(g_settings.mc_channel_slot)); - chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); - memcpy(channel.key, key, sizeof(key)); - if (slot != 0) - { - channel.enabled = true; - } - mesh.meshcore_channel_slot = slot; - mesh.syncMeshCoreLegacyChannelMirror(); - ::settings::ui::channel::sync_meshcore_channel_fields(app_ctx.getConfig(), g_settings); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [&](app::AppConfig& config) + { + chat::MeshConfig& mesh = config.meshcore_config; + chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); + memcpy(channel.key, key, sizeof(key)); + if (slot != 0) + { + channel.enabled = true; + } + mesh.meshcore_channel_slot = slot; + mesh.syncMeshCoreLegacyChannelMirror(); + ::settings::ui::channel::sync_meshcore_channel_fields(config, + g_settings); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::GaugeDesignMah) @@ -2490,10 +2589,16 @@ static void on_text_save_clicked(lv_event_t* e) if (id == settings::ui::SettingId::RtWifiHost) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().reticulumConfig().reticulum_wifi_gateway_host, - sizeof(app_ctx.getConfig().reticulumConfig().reticulum_wifi_gateway_host), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded( + config.reticulumConfig().reticulum_wifi_gateway_host, + sizeof(config.reticulumConfig().reticulum_wifi_gateway_host), + g_state.editing_item->text_value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::RtWifiPort) @@ -2508,9 +2613,14 @@ static void on_text_save_clicked(lv_event_t* e) return; } app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().reticulumConfig().reticulum_wifi_gateway_port = - static_cast(value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.reticulumConfig().reticulum_wifi_gateway_port = + static_cast(value); + }); app_ctx.applyMeshConfig(); std::snprintf(g_settings.rt_wifi_gateway_port, sizeof(g_settings.rt_wifi_gateway_port), @@ -2520,10 +2630,15 @@ static void on_text_save_clicked(lv_event_t* e) if (id == settings::ui::SettingId::MtMqttHost) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshtastic_mqtt_host, - sizeof(app_ctx.getConfig().meshtastic_mqtt_host), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshtastic_mqtt_host, + sizeof(config.meshtastic_mqtt_host), + g_state.editing_item->text_value); + }); } if (id == settings::ui::SettingId::MtMqttPort) { @@ -2537,9 +2652,13 @@ static void on_text_save_clicked(lv_event_t* e) return; } app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_mqtt_port = - static_cast(value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshtastic_mqtt_port = static_cast(value); + }); std::snprintf(g_settings.mt_mqtt_port, sizeof(g_settings.mt_mqtt_port), "%u", @@ -2552,36 +2671,61 @@ static void on_text_save_clicked(lv_event_t* e) g_state.editing_item->text_value[0] != '\0' ? g_state.editing_item->text_value : app::AppConfig::kDefaultMeshtasticMqttRoot; - copy_bounded(app_ctx.getConfig().meshtastic_mqtt_root, - sizeof(app_ctx.getConfig().meshtastic_mqtt_root), - root); + const bool committed = commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [root](app::AppConfig& config) + { + copy_bounded(config.meshtastic_mqtt_root, + sizeof(config.meshtastic_mqtt_root), + root); + }); + if (!committed) + { + modal_close(); + return; + } copy_bounded(g_settings.mt_mqtt_root, sizeof(g_settings.mt_mqtt_root), root); - app_ctx.saveConfig(); std::printf("[Settings][MQTT] mt root saved root=%s\n", root); } if (id == settings::ui::SettingId::MtMqttUser) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshtastic_mqtt_username, - sizeof(app_ctx.getConfig().meshtastic_mqtt_username), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshtastic_mqtt_username, + sizeof(config.meshtastic_mqtt_username), + g_state.editing_item->text_value); + }); } if (id == settings::ui::SettingId::MtMqttPass) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshtastic_mqtt_password, - sizeof(app_ctx.getConfig().meshtastic_mqtt_password), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshtastic_mqtt_password, + sizeof(config.meshtastic_mqtt_password), + g_state.editing_item->text_value); + }); } if (id == settings::ui::SettingId::McMqttHost) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshcore_config.meshcore_mqtt_host, - sizeof(app_ctx.getConfig().meshcore_config.meshcore_mqtt_host), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshcore_config.meshcore_mqtt_host, + sizeof(config.meshcore_config.meshcore_mqtt_host), + g_state.editing_item->text_value); + }); } if (id == settings::ui::SettingId::McMqttPort) { @@ -2595,9 +2739,14 @@ static void on_text_save_clicked(lv_event_t* e) return; } app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_mqtt_port = - static_cast(value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value](app::AppConfig& config) + { + config.meshcore_config.meshcore_mqtt_port = + static_cast(value); + }); std::snprintf(g_settings.mc_mqtt_port, sizeof(g_settings.mc_mqtt_port), "%u", @@ -2610,27 +2759,47 @@ static void on_text_save_clicked(lv_event_t* e) g_state.editing_item->text_value[0] != '\0' ? g_state.editing_item->text_value : app::AppConfig::kDefaultMeshCoreMqttRoot; - copy_bounded(app_ctx.getConfig().meshcore_config.meshcore_mqtt_root, - sizeof(app_ctx.getConfig().meshcore_config.meshcore_mqtt_root), - root); + const bool committed = commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [root](app::AppConfig& config) + { + copy_bounded(config.meshcore_config.meshcore_mqtt_root, + sizeof(config.meshcore_config.meshcore_mqtt_root), + root); + }); + if (!committed) + { + modal_close(); + return; + } copy_bounded(g_settings.mc_mqtt_root, sizeof(g_settings.mc_mqtt_root), root); - app_ctx.saveConfig(); } if (id == settings::ui::SettingId::McMqttUser) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshcore_config.meshcore_mqtt_username, - sizeof(app_ctx.getConfig().meshcore_config.meshcore_mqtt_username), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshcore_config.meshcore_mqtt_username, + sizeof(config.meshcore_config.meshcore_mqtt_username), + g_state.editing_item->text_value); + }); } if (id == settings::ui::SettingId::McMqttPass) { app::IAppFacade& app_ctx = app::appFacade(); - copy_bounded(app_ctx.getConfig().meshcore_config.meshcore_mqtt_password, - sizeof(app_ctx.getConfig().meshcore_config.meshcore_mqtt_password), - g_state.editing_item->text_value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + copy_bounded(config.meshcore_config.meshcore_mqtt_password, + sizeof(config.meshcore_config.meshcore_mqtt_password), + g_state.editing_item->text_value); + }); } if (g_state.editing_item->pref_key && (id == settings::ui::SettingId::WifiSsid || @@ -3244,14 +3413,22 @@ static void on_option_clicked(lv_event_t* e) if (id == settings::ui::SettingId::RtBearer) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& reticulum = app_ctx.getConfig().reticulumConfig(); - apply_reticulum_bearer_policy(reticulum, - reticulum_bearer_policy_from_value(payload->value)); - g_settings.rt_bearer_policy = - reticulum_bearer_policy_to_value(reticulum.reticulum_interface_policy); - g_settings.rt_lora_enabled = reticulum.reticulum_lora_enabled; - g_settings.rt_wifi_gateway_enabled = reticulum.reticulum_wifi_gateway_enabled; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) + { + chat::MeshConfig& reticulum = config.reticulumConfig(); + apply_reticulum_bearer_policy( + reticulum, + reticulum_bearer_policy_from_value(payload->value)); + g_settings.rt_bearer_policy = + reticulum_bearer_policy_to_value( + reticulum.reticulum_interface_policy); + g_settings.rt_lora_enabled = reticulum.reticulum_lora_enabled; + g_settings.rt_wifi_gateway_enabled = + reticulum.reticulum_wifi_gateway_enabled; + }); app_ctx.applyMeshConfig(); rebuild_list = true; } @@ -3275,111 +3452,161 @@ static void on_option_clicked(lv_event_t* e) if (id == settings::ui::SettingId::ChatRegion) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mt_cfg = app_ctx.getConfig().meshtastic_config; - mt_cfg.region = static_cast(payload->value); - const auto* region = chat::meshtastic::findRegion( - static_cast(mt_cfg.region)); - if (region && region->power_limit_dbm > 0) - { - int8_t limit = static_cast(region->power_limit_dbm); - if (limit > kNetTxPowerMax) + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [&](app::AppConfig& config) { - limit = static_cast(kNetTxPowerMax); - } - if (mt_cfg.tx_power == 0 || mt_cfg.tx_power > limit) - { - mt_cfg.tx_power = limit; - } - if (g_settings.net_tx_power > limit) - { - g_settings.net_tx_power = limit; - } - } - app_ctx.saveConfig(); + chat::MeshConfig& mt_cfg = config.meshtastic_config; + mt_cfg.region = static_cast(payload->value); + const auto* region = chat::meshtastic::findRegion( + static_cast( + mt_cfg.region)); + if (region && region->power_limit_dbm > 0) + { + int8_t limit = static_cast(region->power_limit_dbm); + if (limit > kNetTxPowerMax) + { + limit = static_cast(kNetTxPowerMax); + } + if (mt_cfg.tx_power == 0 || mt_cfg.tx_power > limit) + { + mt_cfg.tx_power = limit; + } + if (g_settings.net_tx_power > limit) + { + g_settings.net_tx_power = limit; + } + } + }); app_ctx.applyMeshConfig(); app_ctx.applyNetworkLimits(); } if (id == settings::ui::SettingId::NetUsePreset) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.use_preset = (payload->value != 0); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshtastic_config.use_preset = (payload->value != 0); + }); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::NetPreset) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.modem_preset = static_cast(payload->value); - app_ctx.getConfig().meshtastic_config.use_preset = true; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshtastic_config.modem_preset = + static_cast(payload->value); + config.meshtastic_config.use_preset = true; + }); g_settings.net_use_preset = true; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::NetBw) { app::IAppFacade& app_ctx = app::appFacade(); - if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().bandwidth_khz = static_cast(payload->value); - } - else - { - app_ctx.getConfig().meshtastic_config.bandwidth_khz = static_cast(payload->value); - app_ctx.getConfig().meshtastic_config.use_preset = false; - g_settings.net_use_preset = false; - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + if (chat::infra::isReticulumMeshProtocol(config.mesh_protocol)) + { + config.reticulumConfig().bandwidth_khz = + static_cast(payload->value); + } + else + { + config.meshtastic_config.bandwidth_khz = + static_cast(payload->value); + config.meshtastic_config.use_preset = false; + g_settings.net_use_preset = false; + } + }); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::NetSf) { app::IAppFacade& app_ctx = app::appFacade(); - if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().spread_factor = static_cast(payload->value); - } - else - { - app_ctx.getConfig().meshtastic_config.spread_factor = static_cast(payload->value); - app_ctx.getConfig().meshtastic_config.use_preset = false; - g_settings.net_use_preset = false; - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + if (chat::infra::isReticulumMeshProtocol(config.mesh_protocol)) + { + config.reticulumConfig().spread_factor = + static_cast(payload->value); + } + else + { + config.meshtastic_config.spread_factor = + static_cast(payload->value); + config.meshtastic_config.use_preset = false; + g_settings.net_use_preset = false; + } + }); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::NetCr) { app::IAppFacade& app_ctx = app::appFacade(); - if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().coding_rate = static_cast(payload->value); - } - else - { - app_ctx.getConfig().meshtastic_config.coding_rate = static_cast(payload->value); - app_ctx.getConfig().meshtastic_config.use_preset = false; - g_settings.net_use_preset = false; - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + if (chat::infra::isReticulumMeshProtocol(config.mesh_protocol)) + { + config.reticulumConfig().coding_rate = + static_cast(payload->value); + } + else + { + config.meshtastic_config.coding_rate = + static_cast(payload->value); + config.meshtastic_config.use_preset = false; + g_settings.net_use_preset = false; + } + }); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::NetHopLimit) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.hop_limit = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshtastic_config.hop_limit = + static_cast(payload->value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::NetChannelNum) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.channel_num = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshtastic_config.channel_num = + static_cast(payload->value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::ScreenTimeout) @@ -3405,16 +3632,27 @@ static void on_option_clicked(lv_event_t* e) if (id == settings::ui::SettingId::GpsInterval) { app::IAppFacade& app_ctx = app::appFacade(); - uint32_t interval_ms = static_cast(payload->value) * 1000u; - app_ctx.getConfig().gps_interval_ms = interval_ms; - app_ctx.saveConfig(); + const uint32_t interval_ms = + static_cast(payload->value) * 1000u; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [interval_ms](app::AppConfig& config) + { + config.gps_interval_ms = interval_ms; + }); gps_runtime::set_collection_interval(interval_ms); } if (id == settings::ui::SettingId::GpsInitBaud) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_init_baud = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_init_baud = static_cast(payload->value); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsInitProbeMs) @@ -3429,76 +3667,131 @@ static void on_option_clicked(lv_event_t* e) { probe_ms = kGpsInitProbeMaxMs; } - app_ctx.getConfig().gps_init_probe_ms = static_cast(probe_ms); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [probe_ms](app::AppConfig& config) + { + config.gps_init_probe_ms = static_cast(probe_ms); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsInitProfile) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_init_profile = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_init_profile = static_cast(payload->value); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsInitRxm) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_init_rxm_policy = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_init_rxm_policy = static_cast(payload->value); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsInitGnss) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_init_gnss_policy = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_init_gnss_policy = static_cast(payload->value); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsInitNmea) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_init_nmea_policy = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_init_nmea_policy = static_cast(payload->value); + }); gps_runtime::set_receiver_init_config(make_gps_receiver_init_config(app_ctx.getConfig())); } if (id == settings::ui::SettingId::GpsMode) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_mode = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_mode = static_cast(payload->value); + }); gps_runtime::set_gnss_config(app_ctx.getConfig().gps_mode, app_ctx.getConfig().gps_sat_mask); } if (id == settings::ui::SettingId::GpsSatMask) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_sat_mask = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_sat_mask = static_cast(payload->value); + }); gps_runtime::set_gnss_config(app_ctx.getConfig().gps_mode, app_ctx.getConfig().gps_sat_mask); } if (id == settings::ui::SettingId::GpsStrategy) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_strategy = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_strategy = static_cast(payload->value); + }); gps_runtime::set_power_strategy(static_cast(payload->value)); } if (id == settings::ui::SettingId::GpsAltRef) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_alt_ref = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_alt_ref = static_cast(payload->value); + }); } if (id == settings::ui::SettingId::GpsCoordFmt) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().gps_coord_format = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.gps_coord_format = static_cast(payload->value); + }); } if (id == settings::ui::SettingId::MapCoord) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().map_coord_system = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [payload](app::AppConfig& config) + { + config.map_coord_system = static_cast(payload->value); + }); } if (id == settings::ui::SettingId::MapSource) { @@ -3508,165 +3801,266 @@ static void on_option_clicked(lv_event_t* e) { source = 0; } - app_ctx.getConfig().map_source = static_cast(source); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [source](app::AppConfig& config) + { + config.map_source = static_cast(source); + }); } if (id == settings::ui::SettingId::MapTrackInterval) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().map_track_interval = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [payload](app::AppConfig& config) + { + config.map_track_interval = + static_cast(payload->value); + }); apply_track_interval_runtime(static_cast(payload->value)); } if (id == settings::ui::SettingId::MapTrackFormat) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().map_track_format = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [payload](app::AppConfig& config) + { + config.map_track_format = static_cast(payload->value); + }); apply_track_format_runtime(static_cast(payload->value)); } if (id == settings::ui::SettingId::ChatChannel) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().chat_channel = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::chatUi(), + [payload](app::AppConfig& config) + { + config.chat_channel = static_cast(payload->value); + }); app_ctx.applyChatDefaults(); } if (id == settings::ui::SettingId::NetUtil) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().net_channel_util = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::network(), + [payload](app::AppConfig& config) + { + config.net_channel_util = static_cast(payload->value); + }); app_ctx.applyNetworkLimits(); } if (id == settings::ui::SettingId::NetTxPower) { app::IAppFacade& app_ctx = app::appFacade(); - if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().tx_power = static_cast(payload->value); - } - else - { - app_ctx.getConfig().meshtastic_config.tx_power = static_cast(payload->value); - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + if (chat::infra::isReticulumMeshProtocol(config.mesh_protocol)) + { + config.reticulumConfig().tx_power = + static_cast(payload->value); + } + else + { + config.meshtastic_config.tx_power = + static_cast(payload->value); + } + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McRegionPreset) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mc_cfg = app_ctx.getConfig().meshcore_config; uint8_t preset_id = static_cast(payload->value); if (!chat::meshcore::isValidRegionPresetId(preset_id)) { preset_id = 0; } - mc_cfg.meshcore_region_preset = preset_id; - g_settings.mc_region_preset = preset_id; - if (preset_id > 0) - { - const chat::meshcore::RegionPreset* preset = chat::meshcore::findRegionPresetById(preset_id); - if (preset) + const auto* preset = preset_id > 0 + ? chat::meshcore::findRegionPresetById(preset_id) + : nullptr; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [preset_id, preset](app::AppConfig& config) { - mc_cfg.meshcore_freq_mhz = preset->freq_mhz; - mc_cfg.meshcore_bw_khz = preset->bw_khz; - mc_cfg.meshcore_sf = preset->sf; - mc_cfg.meshcore_cr = preset->cr; - mc_cfg.tx_power = preset->tx_power_dbm; - float_to_text(mc_cfg.meshcore_freq_mhz, g_settings.mc_freq, sizeof(g_settings.mc_freq), 3); - float_to_text(mc_cfg.meshcore_bw_khz, g_settings.mc_bw, sizeof(g_settings.mc_bw), 3); - g_settings.mc_sf = mc_cfg.meshcore_sf; - g_settings.mc_cr = mc_cfg.meshcore_cr; - g_settings.mc_tx_power = mc_cfg.tx_power; - } + chat::MeshConfig& mc_cfg = config.meshcore_config; + mc_cfg.meshcore_region_preset = preset_id; + if (preset) + { + mc_cfg.meshcore_freq_mhz = preset->freq_mhz; + mc_cfg.meshcore_bw_khz = preset->bw_khz; + mc_cfg.meshcore_sf = preset->sf; + mc_cfg.meshcore_cr = preset->cr; + mc_cfg.tx_power = preset->tx_power_dbm; + } + }); + g_settings.mc_region_preset = preset_id; + if (preset) + { + float_to_text(preset->freq_mhz, g_settings.mc_freq, sizeof(g_settings.mc_freq), 3); + float_to_text(preset->bw_khz, g_settings.mc_bw, sizeof(g_settings.mc_bw), 3); + g_settings.mc_sf = preset->sf; + g_settings.mc_cr = preset->cr; + g_settings.mc_tx_power = preset->tx_power_dbm; } - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); rebuild_list = true; } if (id == settings::ui::SettingId::McSf) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_sf = static_cast(payload->value); - app_ctx.getConfig().meshcore_config.meshcore_region_preset = 0; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.meshcore_sf = + static_cast(payload->value); + config.meshcore_config.meshcore_region_preset = 0; + }); g_settings.mc_region_preset = 0; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McCr) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_cr = static_cast(payload->value); - app_ctx.getConfig().meshcore_config.meshcore_region_preset = 0; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.meshcore_cr = + static_cast(payload->value); + config.meshcore_config.meshcore_region_preset = 0; + }); g_settings.mc_region_preset = 0; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McTxPower) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.tx_power = static_cast(payload->value); - app_ctx.getConfig().meshcore_config.meshcore_region_preset = 0; + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.tx_power = + static_cast(payload->value); + config.meshcore_config.meshcore_region_preset = 0; + }); g_settings.mc_region_preset = 0; - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McFloodMax) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_flood_max = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.meshcore_flood_max = + static_cast(payload->value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McSendProfile) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_send_profile = - static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.meshcore_send_profile = + static_cast( + payload->value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McForwardProfile) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_forward_profile = - static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [payload](app::AppConfig& config) + { + config.meshcore_config.meshcore_forward_profile = + static_cast( + payload->value); + }); app_ctx.applyMeshConfig(); } if (id == settings::ui::SettingId::McChannelSlot) { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; - mesh.meshcore_channel_slot = - chat::normalizeMeshCoreChannelSlot(static_cast(payload->value)); - mesh.syncMeshCoreLegacyChannelMirror(); - ::settings::ui::channel::sync_meshcore_channel_fields(app_ctx.getConfig(), g_settings); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [payload](app::AppConfig& config) + { + chat::MeshConfig& mesh = config.meshcore_config; + mesh.meshcore_channel_slot = + chat::normalizeMeshCoreChannelSlot( + static_cast(payload->value)); + mesh.syncMeshCoreLegacyChannelMirror(); + ::settings::ui::channel::sync_meshcore_channel_fields(config, + g_settings); + }); app_ctx.applyMeshConfig(); refresh_visible_item_values(); } if (id == settings::ui::SettingId::PrivacyEncrypt) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().privacy_encrypt_mode = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::privacy(), + [payload](app::AppConfig& config) + { + config.privacy_encrypt_mode = + static_cast(payload->value); + }); app_ctx.applyPrivacyConfig(); } if (id == settings::ui::SettingId::ExternalNmea) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().external_nmea_output_hz = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.external_nmea_output_hz = + static_cast(payload->value); + }); gps_runtime::set_external_nmea_config(app_ctx.getConfig().external_nmea_output_hz, app_ctx.getConfig().external_nmea_sentence_mask); } if (id == settings::ui::SettingId::ExternalNmeaSent) { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().external_nmea_sentence_mask = static_cast(payload->value); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::gps(), + [payload](app::AppConfig& config) + { + config.external_nmea_sentence_mask = + static_cast(payload->value); + }); gps_runtime::set_external_nmea_config(app_ctx.getConfig().external_nmea_output_hz, app_ctx.getConfig().external_nmea_sentence_mask); } @@ -4988,15 +5382,15 @@ static void build_item_list() static void generate_meshtastic_channel_key(bool primary) { app::IAppFacade& app_ctx = app::appFacade(); - if (!::settings::ui::channel::generate_meshtastic_channel_key( - app_ctx.getConfig(), - g_settings, - primary)) + auto edit = app_ctx.beginConfigEdit(); + if (!edit || + !::settings::ui::channel::generate_meshtastic_channel_key( + edit.config(), g_settings, primary)) { ::ui::feedback::show_notice(::ui::i18n::tr("PSK generation failed"), 3000); return; } - app_ctx.saveConfig(); + edit.commit(app::AppConfigChangeSet::channels()); app_ctx.applyMeshConfig(); refresh_visible_item_values(); ::ui::feedback::show_notice(::ui::i18n::tr("Channel PSK generated"), 2200); @@ -5005,13 +5399,15 @@ static void generate_meshtastic_channel_key(bool primary) static void generate_meshcore_channel_key() { app::IAppFacade& app_ctx = app::appFacade(); - if (!::settings::ui::channel::generate_meshcore_channel_key(app_ctx.getConfig(), + auto edit = app_ctx.beginConfigEdit(); + if (!edit || + !::settings::ui::channel::generate_meshcore_channel_key(edit.config(), g_settings)) { ::ui::feedback::show_notice(::ui::i18n::tr("Key generation failed"), 3000); return; } - app_ctx.saveConfig(); + edit.commit(app::AppConfigChangeSet::channels()); app_ctx.applyMeshConfig(); refresh_visible_item_values(); ::ui::feedback::show_notice(::ui::i18n::tr("Channel key generated"), 2200); @@ -5020,20 +5416,29 @@ static void generate_meshcore_channel_key() static void clear_meshcore_channel() { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; const uint8_t slot = chat::normalizeMeshCoreChannelSlot(static_cast(g_settings.mc_channel_slot)); - chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); - channel = chat::MeshCoreChannelConfig(); - if (slot == 0) + if (!commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [slot](app::AppConfig& config) + { + chat::MeshConfig& mesh = config.meshcore_config; + chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); + channel = chat::MeshCoreChannelConfig(); + if (slot == 0) + { + channel.enabled = true; + copy_bounded(channel.name, sizeof(channel.name), "Public"); + } + mesh.meshcore_channel_slot = slot; + mesh.syncMeshCoreLegacyChannelMirror(); + ::settings::ui::channel::sync_meshcore_channel_fields(config, + g_settings); + })) { - channel.enabled = true; - copy_bounded(channel.name, sizeof(channel.name), "Public"); + return; } - mesh.meshcore_channel_slot = slot; - mesh.syncMeshCoreLegacyChannelMirror(); - ::settings::ui::channel::sync_meshcore_channel_fields(app_ctx.getConfig(), g_settings); - app_ctx.saveConfig(); app_ctx.applyMeshConfig(); refresh_visible_item_values(); ::ui::feedback::show_notice(::ui::i18n::tr("Channel cleared"), 1800); @@ -5067,24 +5472,39 @@ static bool activate_item_widget(settings::ui::ItemWidget& widget) case settings::ui::SettingId::NetRelay: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.enable_relay = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshtastic_config.enable_relay = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MapTrack: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().map_track_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [value = *item.bool_value](app::AppConfig& config) + { + config.map_track_enabled = value; + }); apply_track_recording_runtime(*item.bool_value); break; } case settings::ui::SettingId::MapContour: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().map_contour_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::map(), + [value = *item.bool_value](app::AppConfig& config) + { + config.map_contour_enabled = value; + }); break; } case settings::ui::SettingId::GpsEnabled: @@ -5098,60 +5518,93 @@ static bool activate_item_widget(settings::ui::ItemWidget& widget) case settings::ui::SettingId::NetDutyCycle: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().net_duty_cycle = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::network(), + [value = *item.bool_value](app::AppConfig& config) + { + config.net_duty_cycle = value; + }); app_ctx.applyNetworkLimits(); break; } case settings::ui::SettingId::NetTxEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - if (app_ctx.getConfig().mesh_protocol == chat::MeshProtocol::MeshCore) - { - app_ctx.getConfig().meshcore_config.tx_enabled = *item.bool_value; - } - else if (chat::infra::isReticulumMeshProtocol(app_ctx.getConfig().mesh_protocol)) - { - app_ctx.getConfig().reticulumConfig().tx_enabled = *item.bool_value; - } - else - { - app_ctx.getConfig().meshtastic_config.tx_enabled = *item.bool_value; - } - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + if (config.mesh_protocol == chat::MeshProtocol::MeshCore) + { + config.meshcore_config.tx_enabled = value; + } + else if (chat::infra::isReticulumMeshProtocol( + config.mesh_protocol)) + { + config.reticulumConfig().tx_enabled = value; + } + else + { + config.meshtastic_config.tx_enabled = value; + } + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::RtWifiAuto: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().reticulumConfig().reticulum_wifi_auto_connect = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.reticulumConfig().reticulum_wifi_auto_connect = + value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::RtAnonymousPeer: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().reticulumConfig().reticulum_anonymous_peer = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.reticulumConfig().reticulum_anonymous_peer = + value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::RtLocationRequests: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().reticulumConfig().reticulum_allow_location_requests = - *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.reticulumConfig() + .reticulum_allow_location_requests = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::NetOverrideDuty: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_config.override_duty_cycle = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshtastic_config.override_duty_cycle = value; + }); app_ctx.applyMeshConfig(); app_ctx.applyNetworkLimits(); break; @@ -5159,31 +5612,48 @@ static bool activate_item_widget(settings::ui::ItemWidget& widget) case settings::ui::SettingId::McRepeat: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_client_repeat = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshcore_config.meshcore_client_repeat = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::McMultiAcks: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_multi_acks = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshcore_config.meshcore_multi_acks = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::McChannelEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& mesh = app_ctx.getConfig().meshcore_config; const uint8_t slot = chat::normalizeMeshCoreChannelSlot(static_cast(g_settings.mc_channel_slot)); - chat::MeshCoreChannelConfig& channel = mesh.meshCoreChannel(slot); - channel.enabled = (slot == 0) ? true : *item.bool_value; - mesh.meshcore_channel_slot = slot; - mesh.syncMeshCoreLegacyChannelMirror(); - ::settings::ui::channel::sync_meshcore_channel_fields(app_ctx.getConfig(), g_settings); - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [slot, value = *item.bool_value](app::AppConfig& config) + { + chat::MeshConfig& mesh = config.meshcore_config; + chat::MeshCoreChannelConfig& channel = + mesh.meshCoreChannel(slot); + channel.enabled = (slot == 0) ? true : value; + mesh.meshcore_channel_slot = slot; + mesh.syncMeshCoreLegacyChannelMirror(); + ::settings::ui::channel::sync_meshcore_channel_fields( + config, g_settings); + }); app_ctx.applyMeshConfig(); refresh_visible_item_values(); break; @@ -5191,32 +5661,52 @@ static bool activate_item_widget(settings::ui::ItemWidget& widget) case settings::ui::SettingId::MtPrimaryEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().primary_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.primary_enabled = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MtPrimaryUplink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().primary_uplink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.primary_uplink_enabled = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MtPrimaryDownlink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().primary_downlink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.primary_downlink_enabled = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MtSecondaryEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().secondary_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.secondary_enabled = value; + }); app_ctx.applyMeshConfig(); build_item_list(); break; @@ -5224,61 +5714,103 @@ static bool activate_item_widget(settings::ui::ItemWidget& widget) case settings::ui::SettingId::MtSecondaryUplink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().secondary_uplink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.secondary_uplink_enabled = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MtSecondaryDownlink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().secondary_downlink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::channels(), + [value = *item.bool_value](app::AppConfig& config) + { + config.secondary_downlink_enabled = value; + }); app_ctx.applyMeshConfig(); break; } case settings::ui::SettingId::MtMqttEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_mqtt_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshtastic_mqtt_enabled = value; + }); build_item_list(); break; } case settings::ui::SettingId::MtMqttUplink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_mqtt_uplink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshtastic_mqtt_uplink_enabled = value; + }); break; } case settings::ui::SettingId::MtMqttDownlink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshtastic_mqtt_downlink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshtastic_mqtt_downlink_enabled = value; + }); break; } case settings::ui::SettingId::McMqttEnabled: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_mqtt_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshcore_config.meshcore_mqtt_enabled = value; + }); build_item_list(); break; } case settings::ui::SettingId::McMqttUplink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_mqtt_uplink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshcore_config.meshcore_mqtt_uplink_enabled = + value; + }); break; } case settings::ui::SettingId::McMqttDownlink: { app::IAppFacade& app_ctx = app::appFacade(); - app_ctx.getConfig().meshcore_config.meshcore_mqtt_downlink_enabled = *item.bool_value; - app_ctx.saveConfig(); + commit_app_config( + app_ctx, + app::AppConfigChangeSet::mesh(), + [value = *item.bool_value](app::AppConfig& config) + { + config.meshcore_config.meshcore_mqtt_downlink_enabled = + value; + }); break; } case settings::ui::SettingId::WifiEnabled: diff --git a/modules/ui_shared/tests/test_chat_presentation_source.cpp b/modules/ui_shared/tests/test_chat_presentation_source.cpp index 5f3bf856..cb646c89 100644 --- a/modules/ui_shared/tests/test_chat_presentation_source.cpp +++ b/modules/ui_shared/tests/test_chat_presentation_source.cpp @@ -121,6 +121,13 @@ class MemoryPeerDirectoryBlobStore final class PagingStore final : public ::chat::IChatStore { public: + bool isReady() const override { return ready_; } + + void setReady(const bool ready) + { + ready_ = ready; + } + void append(const ::chat::ChatMessage& msg) override { messages_.push_back(msg); @@ -298,6 +305,7 @@ class PagingStore final : public ::chat::IChatStore private: std::vector<::chat::ChatMessage> messages_; int unread_ = 0; + bool ready_ = true; }; ui::chat::ConversationId directPeer(uint32_t peer) @@ -616,6 +624,13 @@ int main() ui::chat::ChatWorkspaceRequest paging_request; paging_request.selected = paging; paging_request.message_offset = 0; + paging_store.setReady(false); + ui::chat::ChatWorkspaceSnapshot not_ready_snapshot; + assert(!paging_source.buildChatWorkspaceSnapshot(paging_request, + not_ready_snapshot)); + assert(!not_ready_snapshot.header.valid); + paging_store.setReady(true); + assert(paging_source.buildChatWorkspaceSnapshot(paging_request, snapshot)); assert(snapshot.message_count == ui::chat::ChatWorkspaceSnapshot::kMaxMessages); assert(snapshot.message_total_count == 25); diff --git a/platform/esp/arduino_common/include/app/app_context.h b/platform/esp/arduino_common/include/app/app_context.h index 11a6d12b..107e47d2 100644 --- a/platform/esp/arduino_common/include/app/app_context.h +++ b/platform/esp/arduino_common/include/app/app_context.h @@ -9,11 +9,10 @@ #include "app/app_context_platform_bindings.h" #include "app/app_event_runtime.h" #include "app/app_facades.h" +#include "app/config_persistence_runtime.h" #include "freertos/FreeRTOS.h" -#include "freertos/queue.h" #include "freertos/semphr.h" -#include "freertos/task.h" #include #include @@ -246,17 +245,16 @@ class AppContext final : public IAppBleFacade void initChatRuntime(bool use_mock_adapter); void initTeamServices(); void initContactServices(); - void ensureConfigSaveWorker(); + void ensureConfigPersistenceLock(); void enqueueConfigSave(AppConfigChangeSet requested_changes); bool enqueueConfigSaveLocked(const AppConfig& desired_config, AppConfigChangeSet requested_changes, uint32_t* out_generation, AppConfigChangeSet* out_changes); void finishConfigEdit(AppConfigChangeSet changes); + void flushConfigPersistence(uint32_t now_ms); static void commitConfigEdit(void* context, AppConfigChangeSet changes); static void cancelConfigEdit(void* context); - void configSaveLoop(); - static void configSaveTaskEntry(void* context); std::unique_ptr chat_model_; @@ -285,19 +283,8 @@ class AppContext final : public IAppBleFacade AppConfig config_; AppContextPlatformBindings platform_bindings_{}; - SemaphoreHandle_t config_save_mutex_ = nullptr; - QueueHandle_t config_save_queue_ = nullptr; - TaskHandle_t config_save_task_ = nullptr; - AppConfig pending_config_save_{}; - AppConfig active_config_save_{}; - AppConfigChangeSet pending_config_changes_{}; - AppConfigChangeSet active_config_changes_{}; - uint32_t pending_config_save_generation_ = 0; - uint32_t completed_config_save_generation_ = 0; - bool config_save_pending_ = false; - bool config_save_busy_ = false; - bool config_save_failed_ = false; - bool config_save_baseline_valid_ = false; + SemaphoreHandle_t config_state_mutex_ = nullptr; + ConfigPersistenceRuntime config_persistence_runtime_{}; bool deferred_storage_started_ = false; app::ChatServicesBundle::DeferredStorageStarter deferred_storage_starter_ = nullptr; diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h index cefc8660..933f2750 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h @@ -93,6 +93,51 @@ class FixedSlotJournalEngine final std::size_t slot_size); }; +// A cursor performs at most one bounded slot read per next() call. It keeps +// only metadata between calls, so no filesystem handle or logical store lock +// is held while the owner is waiting for the next maintenance tick. +class FixedSlotJournalCursor final +{ + public: + enum class StepStatus : uint8_t + { + Item = 0, + Complete, + Missing, + Invalid, + Unavailable, + }; + + bool begin(const FixedSlotJournalEngine& engine, + const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size); + + StepStatus next(const FixedSlotJournalEngine& engine, + void* out_slot, + std::size_t out_len); + + bool seek(uint32_t slot_index); + void reset(); + const FixedSlotJournalEngine::Inspection& inspection() const + { + return inspection_; + } + uint32_t nextIndex() const { return next_index_; } + std::size_t slotSize() const { return slot_size_; } + bool active() const { return active_; } + + private: + char path_[160] = {}; + MeshProtocol protocol_ = MeshProtocol::Meshtastic; + JournalKind kind_ = JournalKind::MessageSegment; + std::size_t slot_size_ = 0U; + FixedSlotJournalEngine::Inspection inspection_{}; + uint32_t next_index_ = 0U; + bool active_ = false; +}; + bool replaceFileAtomically(const char* temp_path, const char* final_path, const char* backup_path); diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h index 29fe09ad..f078e332 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h @@ -5,10 +5,12 @@ #include "platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h" #include "platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h" #include "platform/esp/arduino_common/memory/psram_allocator.h" +#include "platform/esp/common/storage/storage_contracts.h" #include #include +#include #include #include #include @@ -26,8 +28,33 @@ class SdProtocolPeerRepository final : public IProtocolPeerRepository SdProtocolPeerRepository& operator=(const SdProtocolPeerRepository&) = delete; MeshPeerDirectoryStatus begin() override; - MeshPeerDirectoryStatus hydrateFromStorage(); - MeshPeerDirectoryStatus compactDeferred(); + bool isHydrating() const + { + return hydrating_.load(std::memory_order_acquire); + } + bool persistencePending() const + { + return persistence_pending_.load(std::memory_order_acquire); + } + bool compactionPending() const + { + return compaction_pending_.load(std::memory_order_acquire); + } + + // Same operation/generation resumes the current maintenance cursor after + // a retryable physical SD/device transaction miss; logical maintenance + // ownership remains with this operation until completion or cancellation. + // A new generation starts fresh. + platform::esp::common::storage::StorageOperationResult beginMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation); + platform::esp::common::storage::StorageOperationResult stepMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& budget); + void cancelMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation); MeshPeerDirectoryStatus record(const MeshPeerRecord& record) override; MeshPeerDirectoryStatus find(const MeshPeerIdentity& identity, MeshPeerRecord& out_record) override; @@ -73,6 +100,10 @@ class SdProtocolPeerRepository final : public IProtocolPeerRepository storage::v2::PeerProjection, ::platform::esp::arduino_common::memory::PsramAllocator< storage::v2::PeerProjection>>; + using PendingContactVector = std::vector< + storage::v2::ContactProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::ContactProjection>>; using PsramByteVector = std::vector< uint8_t, ::platform::esp::arduino_common::memory::PsramAllocator>; @@ -85,19 +116,23 @@ class SdProtocolPeerRepository final : public IProtocolPeerRepository bool ensureLayout(); bool ensureProtocolLayout(MeshProtocol protocol); - bool loadProtocol(MeshProtocol protocol); - bool loadPeerJournal(MeshProtocol protocol, const char* name); - bool loadContactJournal(MeshProtocol protocol, const char* name); - bool compactProtocolAtBoot(MeshProtocol protocol); - bool rewritePeerSnapshot(MeshProtocol protocol); - bool rewriteContactSnapshot(MeshProtocol protocol); bool appendPeerDelta(const storage::v2::PeerProjection& projection); bool appendContactDelta( const storage::v2::ContactProjection& projection); - bool queueOrAppendPeerDelta( + bool queuePeerDelta( const storage::v2::PeerProjection& projection); - bool drainPendingPeerDeltas(std::size_t budget); + bool queueContactDelta( + const storage::v2::ContactProjection& projection); + MeshPeerDirectoryStatus flushProtocolReset(); + MeshPeerDirectoryStatus flushPendingDeltas(std::size_t budget); + void refreshPersistenceDemandLocked(); + bool rewritePeerSnapshotFrom(MeshProtocol protocol, + const PeerVector& snapshot); + bool acquirePersistenceLease(TickType_t wait_ticks); + void releasePersistenceLease(); + void releaseMaintenanceLease(); + void prunePendingDeltasLocked(); bool queueDeferredObservation(const MeshPeerRecord& record); void drainDeferredObservationsLocked(); MeshPeerDirectoryStatus recordLocked(const MeshPeerRecord& record); @@ -138,12 +173,64 @@ class SdProtocolPeerRepository final : public IProtocolPeerRepository char* out, std::size_t out_len); + enum class MaintenancePhase : uint8_t + { + Idle, + HydrationPrepare, + HydrationJournal, + HydrationFinalize, + PersistenceFlush, + CompactionPrepare, + CompactionInspect, + CompactionCreate, + CompactionWrite, + CompactionReplace, + CompactionAdvance, + Complete, + Failed, + }; + + struct MaintenanceState + { + MaintenancePhase phase = MaintenancePhase::Idle; + platform::esp::common::storage::StorageOperation operation = + platform::esp::common::storage::StorageOperation::None; + platform::esp::common::storage::StorageOperationGeneration generation = + 0U; + uint8_t protocol_index = 0U; + uint8_t journal_index = 0U; + bool journal_started = false; + uint8_t compaction_projection_index = 0U; + uint8_t compaction_inspection_index = 0U; + uint32_t compaction_record_index = 0U; + bool compact_peers = false; + bool compact_contacts = false; + }; + + bool prepareMaintenanceJournal(); + bool applyHydrationJournalSlot(MeshProtocol protocol, + storage::v2::JournalKind kind); + platform::esp::common::storage::StorageOperationResult stepHydration( + const platform::esp::common::storage::StorageOperationBudget& budget); + platform::esp::common::storage::StorageOperationResult stepPersistence( + const platform::esp::common::storage::StorageOperationBudget& budget); + platform::esp::common::storage::StorageOperationResult stepCompaction( + const platform::esp::common::storage::StorageOperationBudget& budget); + platform::esp::common::storage::StorageOperationResult maintenanceFailure( + platform::esp::common::storage::StorageOperationResultKind kind) const; + IChatStore& chat_store_; storage::v2::FixedSlotJournalEngine journal_{}; PeerVector peers_{}; ContactVector contacts_{}; PendingPeerVector pending_peer_deltas_{}; std::size_t pending_peer_head_ = 0U; + PendingContactVector pending_contact_deltas_{}; + std::size_t pending_contact_head_ = 0U; + uint32_t pending_peer_revision_ = 0U; + uint32_t pending_contact_revision_ = 0U; + bool protocol_reset_pending_[3]{}; + uint32_t protocol_reset_revision_[3]{}; PeerVector pending_peer_observations_{}; SemaphoreHandle_t pending_observation_mutex_ = nullptr; uint32_t dropped_peer_observations_ = 0U; @@ -151,8 +238,34 @@ class SdProtocolPeerRepository final : public IProtocolPeerRepository PartitionState partitions_[3]{}; MeshProtocol active_protocol_ = MeshProtocol::Meshtastic; bool begun_ = false; + std::atomic hydrating_{false}; bool hydrated_ = false; mutable SemaphoreHandle_t mutex_ = nullptr; + SemaphoreHandle_t persistence_mutex_ = nullptr; + bool maintenance_persistence_locked_ = false; + std::atomic persistence_pending_{false}; + std::atomic compaction_pending_{false}; + storage::v2::FixedSlotJournalCursor maintenance_journal_{}; + PsramByteVector maintenance_scratch_{}; + PeerVector compaction_peers_{}; + ContactVector compaction_contacts_{}; + PendingPeerVector flush_peer_batch_{}; + PendingContactVector flush_contact_batch_{}; + PeerVector flush_peer_snapshot_{}; + uint32_t compaction_peer_revision_ = 0U; + uint32_t compaction_contact_revision_ = 0U; + uint32_t compaction_reset_revision_[3]{}; + bool compaction_force_peers_[3]{}; + bool compaction_force_contacts_[3]{}; + MaintenanceState maintenance_{}; + char maintenance_path_[96] = {}; + char maintenance_final_path_[96] = {}; + char maintenance_backup_path_[96] = {}; + char maintenance_delta_path_[96] = {}; + MeshProtocol maintenance_protocol_ = MeshProtocol::Meshtastic; + storage::v2::JournalKind maintenance_kind_ = + storage::v2::JournalKind::PeerSnapshot; + std::size_t maintenance_slot_size_ = 0U; }; } // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h index c33e8b6e..3b236f67 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h @@ -9,6 +9,8 @@ #include "platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h" #include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" #include "platform/esp/arduino_common/memory/psram_allocator.h" +#include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/common/storage/storage_contracts.h" #include "freertos/FreeRTOS.h" #include "freertos/semphr.h" @@ -36,16 +38,36 @@ class SdStore final : public IChatStore SdStore(); ~SdStore() override; - bool isReady() const { return ready_.load(std::memory_order_acquire); } + bool isReady() const override + { + return ready_.load(std::memory_order_acquire); + } bool isHydrating() const { return hydrating_.load(std::memory_order_acquire); } + bool compactionPending() const + { + return maintenance_compaction_requested_.load( + std::memory_order_acquire); + } // Construction is intentionally empty. Disk recovery is an explicit // background lifecycle step so AppContext can become interactive first. - bool hydrateFromStorage(); - bool compactDeferred(); + // Same operation/generation resumes the current maintenance cursor after + // a retryable physical SD/device transaction miss; logical maintenance + // ownership remains with this operation until completion or cancellation. + // A new generation starts fresh. + platform::esp::common::storage::StorageOperationResult beginMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation); + platform::esp::common::storage::StorageOperationResult stepMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& budget); + void cancelMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation); void append(const ChatMessage& msg) override; bool appendDurably(const ChatMessage& msg) override; @@ -114,23 +136,32 @@ class SdStore final : public IChatStore static constexpr std::size_t kCatalogCompactThreshold = 512; static constexpr std::size_t kReadCompactThreshold = 256; static constexpr std::size_t kStatusCompactThreshold = 2048; - static constexpr uint32_t kProjectionRetryIntervalMs = 5000U; bool appendInternal(const ChatMessage& msg, bool incoming_commit); bool ensureLayout() const; bool ensureProtocolLayout(MeshProtocol protocol) const; - bool loadRuntimeState(); - bool loadProtocolState(MeshProtocol protocol); - bool loadCatalogJournal(MeshProtocol protocol, const char* name); - bool loadReadJournal(MeshProtocol protocol, const char* name); - bool loadStatusJournal(MeshProtocol protocol, const char* name); - bool loadSeenJournal(); - bool rebuildSeenJournalFromMessages(); bool recoverProjectionSnapshot(MeshProtocol protocol, const char* base_name); - bool reconcileProtocolCatalog(MeshProtocol protocol); - bool reconcileConversationDirectory(MeshProtocol protocol, - const char* directory_name); + enum class ReconcileStepResult : uint8_t + { + InProgress, + Complete, + Failed, + }; + enum class ConversationReconcilePhase : uint8_t + { + ScanSegments, + RepairSegment, + ReadLatest, + ScanUnread, + Commit, + }; + ReconcileStepResult stepConversationDirectoryReconcile( + MeshProtocol protocol, + const char* directory_name); + ReconcileStepResult stepProtocolCatalogReconcile(MeshProtocol protocol); + ReconcileStepResult stepSeenRebuild(); + bool beginSeenRebuild(); std::size_t slotsPerMessageSegment(MeshProtocol protocol) const; uint32_t messageCountOnDisk(const ConversationId& conv) const; @@ -160,6 +191,10 @@ class SdStore final : public IChatStore bool appendSeenProjection( const storage::v2::ReticulumSeenProjection& projection) const; bool rememberReticulumHash(const uint8_t* hash); + bool acquirePersistenceLease(TickType_t wait_ticks); + void releasePersistenceLease(); + void releaseMaintenanceLease(); + void resetCatalogReconcileCursor(); storage::v2::ChatCatalogProjection* findCatalog( const ConversationId& conversation); @@ -180,20 +215,6 @@ class SdStore final : public IChatStore uint32_t sequenceForUnread(const ConversationId& conversation, uint32_t unread) const; - bool rewriteCatalogSnapshot(MeshProtocol protocol); - bool rewriteReadSnapshot(MeshProtocol protocol); - bool rewriteStatusSnapshot(MeshProtocol protocol); - bool compactProtocolProjections(MeshProtocol protocol); - bool rewriteJournalFromCatalog(MeshProtocol protocol, - const char* final_path, - const char* temp_path); - bool rewriteJournalFromReadState(MeshProtocol protocol, - const char* final_path, - const char* temp_path); - bool rewriteJournalFromStatus(MeshProtocol protocol, - const char* final_path, - const char* temp_path); - static MeshProtocol normalizeProtocol(MeshProtocol protocol); static const char* protocolRoot(MeshProtocol protocol); static const char* protocolSlug(MeshProtocol protocol); @@ -216,18 +237,114 @@ class SdStore final : public IChatStore static bool ensureDirectory(const char* path); static bool removeTree(const char* path); + enum class MaintenancePhase : uint8_t + { + Idle, + HydrationPrepare, + HydrationRecover, + HydrationJournal, + HydrationReconcile, + HydrationSeen, + HydrationRebuildSeen, + CompactionPrepare, + CompactionInspect, + CompactionCreate, + CompactionWrite, + CompactionReplace, + CompactionRemove, + CompactionAdvance, + Complete, + Failed, + }; + + struct MaintenanceState + { + MaintenancePhase phase = MaintenancePhase::Idle; + platform::esp::common::storage::StorageOperation operation = + platform::esp::common::storage::StorageOperation::None; + platform::esp::common::storage::StorageOperationGeneration generation = + 0U; + uint8_t protocol_index = 0U; + uint8_t journal_index = 0U; + uint8_t recovery_index = 0U; + bool journal_started = false; + bool seen_journal_found = false; + bool seen_rebuild_required = false; + uint8_t compaction_projection_index = 0U; + uint8_t compaction_inspection_index = 0U; + uint32_t compaction_record_index = 0U; + bool compact_catalog = false; + bool compact_read = false; + bool compact_status = false; + }; + + bool prepareMaintenanceJournal(); + bool applyHydrationJournalSlot(MeshProtocol protocol, + storage::v2::JournalKind kind); + bool advanceHydrationJournal(); + bool recoverHydrationSnapshot(); + bool resetHydrationState(); + platform::esp::common::storage::StorageOperationResult stepHydration( + const platform::esp::common::storage::StorageOperationBudget& budget); + platform::esp::common::storage::StorageOperationResult stepCompaction( + const platform::esp::common::storage::StorageOperationBudget& budget); + platform::esp::common::storage::StorageOperationResult maintenanceFailure( + platform::esp::common::storage::StorageOperationResultKind kind) const; + static const char* hydrationRecoveryName(uint8_t index); + storage::v2::FixedSlotJournalEngine journal_{}; + storage::v2::FixedSlotJournalCursor maintenance_journal_{}; mutable SemaphoreHandle_t mutex_ = nullptr; + mutable SemaphoreHandle_t persistence_mutex_ = nullptr; + bool maintenance_persistence_locked_ = false; + ::platform::esp::arduino_common::storage::SdRuntimeDir + maintenance_directory_{}; + bool maintenance_directory_open_ = false; CatalogList catalog_{}; ReadStateList read_state_{}; StatusList statuses_{}; mutable SeenList seen_hot_{}; mutable ScratchBuffer scratch_{}; + mutable ScratchBuffer maintenance_scratch_{}; + ChatMessage maintenance_seen_message_{}; + storage::v2::ChatCatalogProjection maintenance_seen_catalog_{}; + ConversationId maintenance_seen_conversation_{}; + uint32_t maintenance_seen_catalog_index_ = 0U; + uint32_t maintenance_seen_message_count_ = 0U; + uint32_t maintenance_seen_message_ordinal_ = 0U; + bool maintenance_seen_rebuild_started_ = false; + char maintenance_reconcile_name_[80] = {}; + char maintenance_reconcile_directory_path_[128] = {}; + ChatMessage maintenance_reconcile_latest_message_{}; + storage::v2::ChatCatalogProjection + maintenance_reconcile_projection_{}; + bool maintenance_reconcile_conversation_active_ = false; + ConversationReconcilePhase maintenance_reconcile_phase_ = + ConversationReconcilePhase::ScanSegments; + uint32_t maintenance_reconcile_segment_ = 0U; + uint32_t maintenance_reconcile_total_count_ = 0U; + uint32_t maintenance_reconcile_last_segment_ = 0U; + uint32_t maintenance_reconcile_last_segment_count_ = 0U; + uint32_t maintenance_reconcile_unread_ordinal_ = 0U; + uint32_t maintenance_reconcile_unread_count_ = 0U; + bool maintenance_reconcile_found_segment_ = false; + bool maintenance_reconcile_catalog_current_ = false; + CatalogList compaction_catalog_{}; + ReadStateList compaction_read_state_{}; + StatusList compaction_statuses_{}; + MaintenanceState maintenance_{}; + char maintenance_path_[128] = {}; + char maintenance_final_path_[128] = {}; + char maintenance_backup_path_[128] = {}; + char maintenance_delta_path_[128] = {}; + MeshProtocol maintenance_protocol_ = MeshProtocol::Meshtastic; + storage::v2::JournalKind maintenance_kind_ = + storage::v2::JournalKind::MessageSegment; + std::size_t maintenance_slot_size_ = 0U; bool projection_dirty_[3] = {}; - uint8_t flush_protocol_cursor_ = 0; - uint32_t last_projection_retry_ms_ = 0; std::atomic ready_{false}; std::atomic hydrating_{false}; + mutable std::atomic maintenance_compaction_requested_{false}; }; } // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/scoped_state_lock.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/scoped_state_lock.h index 5de72d9b..7e89f8ed 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/scoped_state_lock.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/scoped_state_lock.h @@ -6,25 +6,42 @@ namespace platform::esp::arduino_common::storage { -// Storage recovery owns this lock for bounded SD transactions. Every caller -// must fail fast instead of turning SD latency into a UI/radio stall. +// This lock protects logical in-memory state only. Filesystem and device +// transactions must happen before or after its lifetime. constexpr TickType_t kStateLockWaitTicks = pdMS_TO_TICKS(50); +enum class StateLockResult : uint8_t +{ + Acquired = 0, + Busy, + Unavailable, +}; + class ScopedRecursiveStateLock final { public: explicit ScopedRecursiveStateLock( SemaphoreHandle_t mutex, TickType_t wait_ticks = kStateLockWaitTicks) - : mutex_(mutex), - locked_(mutex_ && - xSemaphoreTakeRecursive(mutex_, wait_ticks) == pdTRUE) + : mutex_(mutex) { + if (!mutex_) + { + result_ = StateLockResult::Unavailable; + } + else if (xSemaphoreTakeRecursive(mutex_, wait_ticks) == pdTRUE) + { + result_ = StateLockResult::Acquired; + } + else + { + result_ = StateLockResult::Busy; + } } ~ScopedRecursiveStateLock() { - if (locked_) + if (result_ == StateLockResult::Acquired) { xSemaphoreGiveRecursive(mutex_); } @@ -35,12 +52,13 @@ class ScopedRecursiveStateLock final bool locked() const { - return locked_; + return result_ == StateLockResult::Acquired; } + StateLockResult result() const { return result_; } private: SemaphoreHandle_t mutex_ = nullptr; - bool locked_ = false; + StateLockResult result_ = StateLockResult::Unavailable; }; } // namespace platform::esp::arduino_common::storage diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/storage_runtime.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/storage_runtime.h index 42993b33..aeb89bff 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/storage_runtime.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/storage/storage_runtime.h @@ -11,9 +11,8 @@ class SdProtocolPeerRepository; namespace platform::esp::arduino_common::storage { -// Starts the one-shot background recovery worker. The worker exits after -// hydration/maintenance, so storage recovery does not become another -// permanent always-on task. +// Arms the stable storage maintenance owner. The task remains blocked on its +// event queue after maintenance completes. void start_deferred_storage(chat::SdStore* store, chat::SdProtocolPeerRepository* peer_directory, chat::MeshProtocol active_protocol); @@ -21,11 +20,20 @@ void start_deferred_storage(chat::SdStore* store, // Advances the retry/maintenance state machine from the foreground loop. void tick_deferred_storage(); +// Requests cancellation at the next operation boundary and stops accepting +// foreground maintenance ticks. +void stop_deferred_storage(); + // Returns true while the initial SD-backed state is being hydrated. The // foreground lifecycle must not enter store/repository code during this // exclusive hydration phase. bool hydration_active(); +// Optional interactive SD reads, such as map tiles, must defer while the +// shared-display-SPI startup hydration barrier is active. The query is false +// for independent storage topologies. +bool interactive_storage_reads_deferred(); + // Returns true once after the initial hydration has completed successfully. // The foreground loop uses this edge to apply non-critical SD-backed state. bool consume_hydration_ready(); diff --git a/platform/esp/arduino_common/src/app_context.cpp b/platform/esp/arduino_common/src/app_context.cpp index 4e03f2ca..b89e1e0c 100644 --- a/platform/esp/arduino_common/src/app_context.cpp +++ b/platform/esp/arduino_common/src/app_context.cpp @@ -7,7 +7,6 @@ #include -#include "app/app_config_save_plan.h" #include "ble/ble_manager.h" #include "board/BoardBase.h" #include "board/GpsBoard.h" @@ -34,11 +33,7 @@ namespace app { namespace { -constexpr uint32_t kConfigSaveTaskStackBytes = 4 * 1024; -constexpr UBaseType_t kConfigSaveTaskPriority = 1; constexpr TickType_t kConfigSaveMutexWait = pdMS_TO_TICKS(20); -constexpr TickType_t kConfigSaveDebounceTicks = pdMS_TO_TICKS(250); -constexpr TickType_t kConfigSaveRetryDelayTicks = pdMS_TO_TICKS(1000); void normalize_reticulum_interface_strategy(AppConfig& config) { @@ -296,9 +291,9 @@ void AppContext::requestSaveConfig(AppConfigChangeSet changes) AppConfigEdit AppContext::beginConfigEdit() { - ensureConfigSaveWorker(); - if (config_save_mutex_ == nullptr || - xSemaphoreTake(config_save_mutex_, kConfigSaveMutexWait) != pdTRUE) + ensureConfigPersistenceLock(); + if (config_state_mutex_ == nullptr || + xSemaphoreTake(config_state_mutex_, kConfigSaveMutexWait) != pdTRUE) { Serial.println("[AppCfg][EDIT] unavailable"); return AppConfigEdit(); @@ -310,32 +305,11 @@ AppConfigEdit AppContext::beginConfigEdit() &AppContext::cancelConfigEdit); } -void AppContext::ensureConfigSaveWorker() +void AppContext::ensureConfigPersistenceLock() { - if (config_save_mutex_ == nullptr) + if (config_state_mutex_ == nullptr) { - config_save_mutex_ = xSemaphoreCreateMutex(); - } - if (config_save_queue_ == nullptr) - { - config_save_queue_ = xQueueCreate(1, sizeof(uint8_t)); - } - if (config_save_task_ == nullptr && - config_save_mutex_ != nullptr && - config_save_queue_ != nullptr) - { - BaseType_t ok = xTaskCreate(configSaveTaskEntry, - "app_cfg_io", - kConfigSaveTaskStackBytes, - this, - kConfigSaveTaskPriority, - &config_save_task_); - if (ok != pdPASS) - { - Serial.printf("[AppCfg][SAVE_ASYNC] task_create_failed rc=%ld\n", - static_cast(ok)); - config_save_task_ = nullptr; - } + config_state_mutex_ = xSemaphoreCreateMutex(); } } @@ -343,18 +317,16 @@ void AppContext::enqueueConfigSave(AppConfigChangeSet requested_changes) { if (platform_bindings_.save_app_config) { - ensureConfigSaveWorker(); - if (config_save_mutex_ == nullptr || - config_save_queue_ == nullptr || - config_save_task_ == nullptr) + ensureConfigPersistenceLock(); + if (config_state_mutex_ == nullptr) { - Serial.println("[AppCfg][SAVE_ASYNC] unavailable"); + Serial.println("[AppCfg][SAVE_OWNER] unavailable"); return; } - if (xSemaphoreTake(config_save_mutex_, kConfigSaveMutexWait) != pdTRUE) + if (xSemaphoreTake(config_state_mutex_, kConfigSaveMutexWait) != pdTRUE) { - Serial.println("[AppCfg][SAVE_ASYNC] enqueue_busy"); + Serial.println("[AppCfg][SAVE_OWNER] submit_busy"); return; } @@ -364,23 +336,15 @@ void AppContext::enqueueConfigSave(AppConfigChangeSet requested_changes) requested_changes, &generation, &queued_changes); - xSemaphoreGive(config_save_mutex_); + xSemaphoreGive(config_state_mutex_); if (!should_signal) { - Serial.println("[AppCfg][SAVE_ASYNC] noop"); + Serial.println("[AppCfg][SAVE_OWNER] noop"); return; } - const uint8_t signal = 1; - if (xQueueOverwrite(config_save_queue_, &signal) != pdTRUE) - { - Serial.printf("[AppCfg][SAVE_ASYNC] signal_failed gen=%lu\n", - static_cast(generation)); - return; - } - - Serial.printf("[AppCfg][SAVE_ASYNC] queued gen=%lu changes=0x%08lx\n", + Serial.printf("[AppCfg][SAVE_OWNER] submitted gen=%lu changes=0x%08lx\n", static_cast(generation), static_cast(queued_changes.bits())); } @@ -391,33 +355,22 @@ bool AppContext::enqueueConfigSaveLocked(const AppConfig& desired_config, uint32_t* out_generation, AppConfigChangeSet* out_changes) { - const AppConfigSavePlan plan = - planAppConfigSave(active_config_save_, - config_save_baseline_valid_, - desired_config, - config_save_busy_, - active_config_save_, - active_config_changes_, - requested_changes); - - pending_config_save_ = desired_config; - pending_config_changes_ = plan.changes; - config_save_pending_ = plan.queue; - config_save_failed_ = false; - if (!plan.queue) + const ConfigPersistenceSubmission submission = + config_persistence_runtime_.submit(desired_config, + requested_changes, + millis()); + if (!submission.queued) { - pending_config_changes_ = AppConfigChangeSet::none(); return false; } - ++pending_config_save_generation_; if (out_generation) { - *out_generation = pending_config_save_generation_; + *out_generation = submission.generation; } if (out_changes) { - *out_changes = pending_config_changes_; + *out_changes = submission.changes; } return true; } @@ -430,7 +383,7 @@ void AppContext::finishConfigEdit(AppConfigChangeSet changes) changes, &generation, &queued_changes); - xSemaphoreGive(config_save_mutex_); + xSemaphoreGive(config_state_mutex_); if (!should_signal) { @@ -438,16 +391,7 @@ void AppContext::finishConfigEdit(AppConfigChangeSet changes) return; } - const uint8_t signal = 1; - if (config_save_queue_ == nullptr || - xQueueOverwrite(config_save_queue_, &signal) != pdTRUE) - { - Serial.printf("[AppCfg][EDIT] signal_failed gen=%lu\n", - static_cast(generation)); - return; - } - - Serial.printf("[AppCfg][EDIT] queued gen=%lu changes=0x%08lx\n", + Serial.printf("[AppCfg][EDIT] submitted gen=%lu changes=0x%08lx\n", static_cast(generation), static_cast(queued_changes.bits())); } @@ -464,116 +408,49 @@ void AppContext::commitConfigEdit(void* context, AppConfigChangeSet changes) void AppContext::cancelConfigEdit(void* context) { auto* self = static_cast(context); - if (self && self->config_save_mutex_) + if (self && self->config_state_mutex_) { - xSemaphoreGive(self->config_save_mutex_); + xSemaphoreGive(self->config_state_mutex_); } } -void AppContext::configSaveLoop() +void AppContext::flushConfigPersistence(uint32_t now_ms) { - uint8_t signal = 0; - for (;;) + if (!platform_bindings_.save_app_config || config_state_mutex_ == nullptr) { - if (xQueueReceive(config_save_queue_, &signal, portMAX_DELAY) != pdTRUE) - { - continue; - } - - vTaskDelay(kConfigSaveDebounceTicks); - for (;;) - { - uint32_t generation = 0; - - if (xSemaphoreTake(config_save_mutex_, portMAX_DELAY) != pdTRUE) - { - break; - } - if (!config_save_pending_) - { - config_save_busy_ = false; - xSemaphoreGive(config_save_mutex_); - break; - } - active_config_save_ = pending_config_save_; - active_config_changes_ = pending_config_changes_; - pending_config_changes_ = AppConfigChangeSet::none(); - generation = pending_config_save_generation_; - config_save_pending_ = false; - config_save_busy_ = true; - xSemaphoreGive(config_save_mutex_); - - Serial.printf("[AppCfg][SAVE_ASYNC] flush begin gen=%lu changes=0x%08lx\n", - static_cast(generation), - static_cast(active_config_changes_.bits())); - const bool ok = platform_bindings_.save_app_config - ? platform_bindings_.save_app_config(active_config_save_, - active_config_changes_) - : false; - - bool has_more = false; - if (xSemaphoreTake(config_save_mutex_, portMAX_DELAY) == pdTRUE) - { - config_save_busy_ = false; - config_save_failed_ = !ok; - if (ok) - { - completed_config_save_generation_ = generation; - config_save_baseline_valid_ = true; - if (config_save_pending_) - { - pending_config_changes_ = - detectAppConfigChanges(active_config_save_, pending_config_save_); - if (pending_config_changes_.empty()) - { - config_save_pending_ = false; - } - } - } - else if (!config_save_pending_) - { - pending_config_save_ = active_config_save_; - pending_config_changes_ = AppConfigChangeSet::allPersisted(); - config_save_pending_ = true; - } - else - { - pending_config_changes_ = - pending_config_changes_.merged(AppConfigChangeSet::allPersisted()); - } - if (!ok) - { - config_save_baseline_valid_ = false; - } - has_more = config_save_pending_; - xSemaphoreGive(config_save_mutex_); - } - - Serial.printf("[AppCfg][SAVE_ASYNC] flush done gen=%lu ok=%u more=%u\n", - static_cast(generation), - ok ? 1U : 0U, - has_more ? 1U : 0U); - if (!ok) - { - vTaskDelay(kConfigSaveRetryDelayTicks); - } - if (!has_more) - { - break; - } - vTaskDelay(kConfigSaveDebounceTicks); - } + return; } -} -void AppContext::configSaveTaskEntry(void* context) -{ - auto* self = static_cast(context); - if (self) + ConfigPersistenceWork work{}; + if (xSemaphoreTake(config_state_mutex_, 0) != pdTRUE) { - self->configSaveLoop(); + return; } - vTaskDelete(nullptr); + const bool has_work = config_persistence_runtime_.takeDue(now_ms, work); + xSemaphoreGive(config_state_mutex_); + if (!has_work || !work.snapshot) + { + return; + } + + Serial.printf("[AppCfg][SAVE_OWNER] flush begin gen=%lu changes=0x%08lx\n", + static_cast(work.generation), + static_cast(work.changes.bits())); + const bool ok = platform_bindings_.save_app_config(*work.snapshot, + work.changes); + + if (xSemaphoreTake(config_state_mutex_, kConfigSaveMutexWait) == pdTRUE) + { + config_persistence_runtime_.complete( + work.generation, + ok ? ConfigPersistenceResultKind::Completed + : ConfigPersistenceResultKind::IoError, + millis()); + xSemaphoreGive(config_state_mutex_); + } + Serial.printf("[AppCfg][SAVE_OWNER] flush done gen=%lu ok=%u\n", + static_cast(work.generation), + ok ? 1U : 0U); } void AppContext::applyMeshConfig() @@ -669,8 +546,7 @@ bool AppContext::init(BoardBase& board, LoraBoard* lora_board, GpsBoard* gps_boa ::ui::boot::set_log_line("Loading app config..."); platform_bindings_.load_app_config(config_); } - active_config_save_ = config_; - config_save_baseline_valid_ = true; + config_persistence_runtime_.initialize(config_); const uint32_t after_config_ms = millis(); Serial.printf("[AppContext] phase=load_config elapsed_ms=%lu total_ms=%lu\n", static_cast(after_config_ms - init_started_ms), @@ -839,11 +715,16 @@ void AppContext::getEffectiveUserInfo(char* out_long, size_t long_len, void AppContext::updateCoreServices() { + flushConfigPersistence(millis()); + if (::platform::ui::reticulum_groups::hasPending()) + { + (void)::platform::ui::reticulum_groups::flushPending(); + } if (event_runtime_hooks_.update_core_services) { event_runtime_hooks_.update_core_services(*this); } - if (mesh_peer_directory_ && + if (mesh_peer_directory_ && !deferred_storage_started_ && !::platform::ui::reticulum_call::resource_preempt_active()) { (void)mesh_peer_directory_->flush(); diff --git a/platform/esp/arduino_common/src/app_context_platform_bindings.cpp b/platform/esp/arduino_common/src/app_context_platform_bindings.cpp index 2be7384d..ba51ccb8 100644 --- a/platform/esp/arduino_common/src/app_context_platform_bindings.cpp +++ b/platform/esp/arduino_common/src/app_context_platform_bindings.cpp @@ -370,7 +370,7 @@ void finalize_startup(app::IAppFacade& app_facade) (void)ui_get_timezone_offset_min(); // Team snapshot restore is performed by deferred_storage_ready() after - // the shell is interactive and the storage worker has hydrated. + // the shell is interactive and maintenance hydration has completed. } chat::NodeId get_self_node_id() diff --git a/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter.cpp b/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter.cpp index 772b6161..19eee192 100644 --- a/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter.cpp +++ b/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_adapter.cpp @@ -9718,6 +9718,8 @@ LxmfAdapter::PeerInfo* LxmfAdapter::findOrLoadPeerByNodeId(NodeId node_id) if (!result.status.succeeded()) { if (result.status.code != MeshPeerDirectoryStatusCode::StorageUnavailable && + result.status.code != MeshPeerDirectoryStatusCode::Busy && + result.status.code != MeshPeerDirectoryStatusCode::DeviceUnavailable && result.status.code != MeshPeerDirectoryStatusCode::InvalidArgument) { Serial.printf("[LXMF][Directory] peer_lookup miss node=%08lX status=%u\n", @@ -9749,6 +9751,8 @@ LxmfAdapter::PeerInfo* LxmfAdapter::findOrLoadPeerByDestinationHash( if (!result.status.succeeded()) { if (result.status.code != MeshPeerDirectoryStatusCode::StorageUnavailable && + result.status.code != MeshPeerDirectoryStatusCode::Busy && + result.status.code != MeshPeerDirectoryStatusCode::DeviceUnavailable && result.status.code != MeshPeerDirectoryStatusCode::InvalidArgument) { char dest[12] = {}; @@ -9796,6 +9800,8 @@ bool LxmfAdapter::recordPeerInDirectory(const PeerInfo& peer, if (!result.record_status.succeeded()) { if (result.record_status.code != MeshPeerDirectoryStatusCode::StorageUnavailable && + result.record_status.code != MeshPeerDirectoryStatusCode::Busy && + result.record_status.code != MeshPeerDirectoryStatusCode::DeviceUnavailable && result.record_status.code != MeshPeerDirectoryStatusCode::InvalidArgument) { Serial.printf("[LXMF][Directory] address_save failed status=%u\n", @@ -9807,6 +9813,8 @@ bool LxmfAdapter::recordPeerInDirectory(const PeerInfo& peer, if (!result.flags_status.succeeded()) { if (result.flags_status.code != MeshPeerDirectoryStatusCode::StorageUnavailable && + result.flags_status.code != MeshPeerDirectoryStatusCode::Busy && + result.flags_status.code != MeshPeerDirectoryStatusCode::DeviceUnavailable && result.flags_status.code != MeshPeerDirectoryStatusCode::InvalidArgument) { Serial.printf("[LXMF][Directory] flag_save failed status=%u\n", diff --git a/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_rx_telemetry.cpp b/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_rx_telemetry.cpp index a8c25c64..996209b6 100644 --- a/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_rx_telemetry.cpp +++ b/platform/esp/arduino_common/src/chat/infra/lxmf/lxmf_rx_telemetry.cpp @@ -5,7 +5,17 @@ #include "platform/esp/arduino_common/chat/infra/lxmf/lxmf_rx_telemetry.h" +#if defined(ARDUINO) #include +#else +#include +#endif + +#if defined(ARDUINO) +#define LXMF_RAW_RX_LOG(...) Serial.printf(__VA_ARGS__) +#else +#define LXMF_RAW_RX_LOG(...) std::printf(__VA_ARGS__) +#endif namespace chat::lxmf::runtime { @@ -40,9 +50,9 @@ bool RawRxTelemetry::shouldLogLoraDiscoveryDetail(uint32_t now_ms, } if (suppressed_lora_discovery_detail_logs_ != 0) { - Serial.printf("[LXMF][RawRX] detail_suppressed iface=lora public_discovery=1 phase=%s suppressed=%u\n", - phase ? phase : "-", - static_cast(suppressed_lora_discovery_detail_logs_)); + LXMF_RAW_RX_LOG("[LXMF][RawRX] detail_suppressed iface=lora public_discovery=1 phase=%s suppressed=%u\n", + phase ? phase : "-", + static_cast(suppressed_lora_discovery_detail_logs_)); suppressed_lora_discovery_detail_logs_ = 0; } last_lora_discovery_detail_log_ms_ = now_ms; @@ -60,8 +70,8 @@ bool RawRxTelemetry::shouldLogLoraAnnounceIgnore(uint32_t now_ms, } if (suppressed_lora_announce_ignore_logs_ != 0) { - Serial.printf("[LXMF][AnnounceRX] ignored_suppressed iface=lora suppressed=%u\n", - static_cast(suppressed_lora_announce_ignore_logs_)); + LXMF_RAW_RX_LOG("[LXMF][AnnounceRX] ignored_suppressed iface=lora suppressed=%u\n", + static_cast(suppressed_lora_announce_ignore_logs_)); suppressed_lora_announce_ignore_logs_ = 0; } last_lora_announce_ignore_log_ms_ = now_ms; @@ -125,14 +135,14 @@ void RawRxTelemetry::noteSummary(bool wifi_skipped, return; } - Serial.printf("[LXMF][RawRX] stats packets=%u wifi_skipped=%u duplicate=%u parse_failed=%u deferred=%u deferred_drop=%u throttled_discovery=%u\n", - static_cast(packets_), - static_cast(wifi_skipped_), - static_cast(duplicates_), - static_cast(parse_failed_), - static_cast(deferred_), - static_cast(deferred_dropped_), - static_cast(throttled_discovery_)); + LXMF_RAW_RX_LOG("[LXMF][RawRX] stats packets=%u wifi_skipped=%u duplicate=%u parse_failed=%u deferred=%u deferred_drop=%u throttled_discovery=%u\n", + static_cast(packets_), + static_cast(wifi_skipped_), + static_cast(duplicates_), + static_cast(parse_failed_), + static_cast(deferred_), + static_cast(deferred_dropped_), + static_cast(throttled_discovery_)); packets_ = 0; wifi_skipped_ = 0; duplicates_ = 0; diff --git a/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp b/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp index 7b605a2d..f3379447 100644 --- a/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp +++ b/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp @@ -184,6 +184,96 @@ FixedSlotJournalEngine::ReadStatus FixedSlotJournalEngine::readStatus( return ReadStatus::Ok; } +bool FixedSlotJournalCursor::begin(const FixedSlotJournalEngine& engine, + const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) +{ + reset(); + if (!path || path[0] == '\0' || slot_size == 0U || + std::strlen(path) >= sizeof(path_)) + { + return false; + } + + std::strncpy(path_, path, sizeof(path_) - 1U); + path_[sizeof(path_) - 1U] = '\0'; + protocol_ = protocol; + kind_ = kind; + slot_size_ = slot_size; + inspection_ = engine.inspect(path_, protocol_, kind_, slot_size_); + active_ = true; + return true; +} + +FixedSlotJournalCursor::StepStatus FixedSlotJournalCursor::next( + const FixedSlotJournalEngine& engine, + void* out_slot, + std::size_t out_len) +{ + if (!active_ || !out_slot || out_len < slot_size_) + { + return StepStatus::Invalid; + } + if (inspection_.state == + FixedSlotJournalEngine::State::Missing) + { + active_ = false; + return StepStatus::Missing; + } + if (inspection_.state != FixedSlotJournalEngine::State::Ready && + inspection_.state != FixedSlotJournalEngine::State::PartialTail) + { + active_ = false; + return StepStatus::Invalid; + } + if (next_index_ >= inspection_.slot_count) + { + active_ = false; + return StepStatus::Complete; + } + + const FixedSlotJournalEngine::ReadStatus status = + engine.readStatus(path_, + protocol_, + kind_, + slot_size_, + inspection_, + next_index_, + out_slot); + if (status != FixedSlotJournalEngine::ReadStatus::Ok) + { + return status == FixedSlotJournalEngine::ReadStatus::Unavailable + ? StepStatus::Unavailable + : StepStatus::Invalid; + } + + ++next_index_; + return StepStatus::Item; +} + +bool FixedSlotJournalCursor::seek(uint32_t slot_index) +{ + if (!active_ || slot_index > inspection_.slot_count) + { + return false; + } + next_index_ = slot_index; + return true; +} + +void FixedSlotJournalCursor::reset() +{ + path_[0] = '\0'; + protocol_ = MeshProtocol::Meshtastic; + kind_ = JournalKind::MessageSegment; + slot_size_ = 0U; + inspection_ = {}; + next_index_ = 0U; + active_ = false; +} + bool FixedSlotJournalEngine::validDescriptor(MeshProtocol protocol, JournalKind kind, std::size_t slot_size) diff --git a/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp b/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp index 4f2aadc9..cfd69b77 100644 --- a/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp +++ b/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp @@ -16,6 +16,81 @@ namespace { namespace storage_runtime = ::platform::esp::arduino_common::storage; namespace storage_v2 = ::chat::storage::v2; +namespace storage_contracts = ::platform::esp::common::storage; + +storage_contracts::StorageOperationResultKind stateLockFailure( + storage_runtime::StateLockResult result) +{ + return result == storage_runtime::StateLockResult::Unavailable + ? storage_contracts::StorageOperationResultKind:: + DeviceUnavailable + : storage_contracts::StorageOperationResultKind::StateBusy; +} + +MeshPeerDirectoryStatus stateLockStatus( + storage_runtime::StateLockResult result) +{ + return MeshPeerDirectoryStatus::fail( + result == storage_runtime::StateLockResult::Unavailable + ? MeshPeerDirectoryStatusCode::DeviceUnavailable + : MeshPeerDirectoryStatusCode::Busy); +} + +MeshPeerDirectoryStatus ioFailureStatus() +{ + return MeshPeerDirectoryStatus::fail( + storage_runtime::sd_card_ready() + ? MeshPeerDirectoryStatusCode::IoError + : MeshPeerDirectoryStatusCode::DeviceUnavailable); +} + +storage_contracts::StorageOperationResultKind operationFailureKind( + MeshPeerDirectoryStatusCode code) +{ + switch (code) + { + case MeshPeerDirectoryStatusCode::Busy: + return storage_contracts::StorageOperationResultKind::StateBusy; + case MeshPeerDirectoryStatusCode::DeviceUnavailable: + case MeshPeerDirectoryStatusCode::StorageUnavailable: + return storage_contracts::StorageOperationResultKind:: + DeviceUnavailable; + case MeshPeerDirectoryStatusCode::IoError: + default: + return storage_contracts::StorageOperationResultKind::IoError; + } +} + +MeshPeerDirectoryStatus maintenanceStatus( + const storage_contracts::StorageOperationResult& result) +{ + switch (result.kind) + { + case storage_contracts::StorageOperationResultKind::StateBusy: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Busy); + case storage_contracts::StorageOperationResultKind::DeviceUnavailable: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::DeviceUnavailable); + case storage_contracts::StorageOperationResultKind::IoError: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::IoError); + case storage_contracts::StorageOperationResultKind::RetryLater: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Busy); + case storage_contracts::StorageOperationResultKind::StaleGeneration: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Busy); + case storage_contracts::StorageOperationResultKind::Cancelled: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + case storage_contracts::StorageOperationResultKind::Completed: + case storage_contracts::StorageOperationResultKind::InProgress: + default: + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::IoError); + } +} constexpr const char* kRoot = "/data/v2"; constexpr MeshProtocol kProtocols[] = { @@ -29,6 +104,8 @@ constexpr std::size_t kPeerHotCacheCapacity[] = {16U, 128U, 64U}; constexpr uint32_t kBootCompactionDeltaThreshold = 1024U; constexpr std::size_t kPendingFlushBudget = 4U; constexpr std::size_t kPendingObservationCapacity = 64U; +constexpr uint8_t kMaintenanceJournalCount = 4U; +constexpr TickType_t kPersistenceLeaseWaitTicks = pdMS_TO_TICKS(50U); using ScopedRepositoryLock = storage_runtime::ScopedRecursiveStateLock; bool hasText(const char* text) @@ -63,9 +140,15 @@ SdProtocolPeerRepository::SdProtocolPeerRepository(IChatStore& chat_store) peers_.reserve(256U); contacts_.reserve(64U); pending_peer_deltas_.reserve(16U); + pending_contact_deltas_.reserve(16U); pending_peer_observations_.reserve(kPendingObservationCapacity); + flush_peer_batch_.reserve(kPendingFlushBudget); + flush_contact_batch_.reserve(kPendingFlushBudget); + flush_peer_snapshot_.reserve(256U); pending_observation_mutex_ = xSemaphoreCreateMutex(); + persistence_mutex_ = xSemaphoreCreateMutex(); slot_scratch_.resize(512U, 0U); + maintenance_scratch_.resize(512U, 0U); } SdProtocolPeerRepository::~SdProtocolPeerRepository() @@ -80,6 +163,885 @@ SdProtocolPeerRepository::~SdProtocolPeerRepository() vSemaphoreDelete(pending_observation_mutex_); pending_observation_mutex_ = nullptr; } + if (persistence_mutex_) + { + vSemaphoreDelete(persistence_mutex_); + persistence_mutex_ = nullptr; + } +} + +bool SdProtocolPeerRepository::acquirePersistenceLease(TickType_t wait_ticks) +{ + return persistence_mutex_ && + xSemaphoreTake(persistence_mutex_, wait_ticks) == pdTRUE; +} + +void SdProtocolPeerRepository::releasePersistenceLease() +{ + if (persistence_mutex_) + { + xSemaphoreGive(persistence_mutex_); + } +} + +void SdProtocolPeerRepository::releaseMaintenanceLease() +{ + if (maintenance_persistence_locked_) + { + maintenance_persistence_locked_ = false; + releasePersistenceLease(); + } +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::beginMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation) +{ + if (operation != storage_contracts::StorageOperation::Hydrate && + operation != storage_contracts::StorageOperation::Persist && + operation != storage_contracts::StorageOperation::Compact) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::Cancelled, + operation, + generation); + } + if (operation == storage_contracts::StorageOperation::Hydrate && hydrated_) + { + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + if (operation == storage_contracts::StorageOperation::Persist && + !persistencePending()) + { + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + // A composite adapter may revisit this repository after a later stage in + // the same generation asked the owner to retry. Preserve the completed + // stage instead of resetting its cursor. + if (maintenance_.operation == operation && + maintenance_.generation == generation && + maintenance_.phase == MaintenancePhase::Complete) + { + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + + // The persistence lease is the repository's logical maintenance + // ownership, not the physical SD/SPI transaction lease. Keep that + // ownership across a retry so foreground persistence cannot interleave + // between generations. + if (maintenance_.operation == operation && + maintenance_.generation == generation && + maintenance_.phase != MaintenancePhase::Idle && + maintenance_.phase != MaintenancePhase::Complete && + maintenance_.phase != MaintenancePhase::Failed) + { + if (!maintenance_persistence_locked_ && + !acquirePersistenceLease(kPersistenceLeaseWaitTicks)) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + maintenance_persistence_locked_ = true; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(true, std::memory_order_release); + } + return storage_contracts::StorageOperationResult::inProgressResult( + operation, + generation); + } + + if (!acquirePersistenceLease(kPersistenceLeaseWaitTicks)) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + if (maintenance_.phase != MaintenancePhase::Idle && + maintenance_.phase != MaintenancePhase::Complete && + maintenance_.phase != MaintenancePhase::Failed) + { + releasePersistenceLease(); + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + if (operation == storage_contracts::StorageOperation::Hydrate && + !begun_) + { + releasePersistenceLease(); + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + + maintenance_ = {}; + maintenance_.operation = operation; + maintenance_.generation = generation; + maintenance_.phase = + operation == storage_contracts::StorageOperation::Hydrate + ? MaintenancePhase::HydrationPrepare + : operation == storage_contracts::StorageOperation::Persist + ? MaintenancePhase::PersistenceFlush + : MaintenancePhase::CompactionPrepare; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(true, std::memory_order_release); + } + maintenance_persistence_locked_ = true; + return storage_contracts::StorageOperationResult::inProgressResult( + operation, + generation); +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::stepMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + if (maintenance_.operation != operation || + maintenance_.generation != generation) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StaleGeneration, + operation, + generation); + } + if (maintenance_.phase == MaintenancePhase::Complete) + { + releaseMaintenanceLease(); + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + if (maintenance_.phase == MaintenancePhase::Failed) + { + releaseMaintenanceLease(); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + const auto result = + operation == storage_contracts::StorageOperation::Hydrate + ? stepHydration(budget) + : operation == storage_contracts::StorageOperation::Persist + ? stepPersistence(budget) + : stepCompaction(budget); + if (result.completed() || + result.kind == storage_contracts::StorageOperationResultKind::Cancelled || + maintenance_.phase == MaintenancePhase::Failed) + { + releaseMaintenanceLease(); + } + return result; +} + +void SdProtocolPeerRepository::cancelMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation) +{ + if (maintenance_.operation != operation || + maintenance_.generation != generation) + { + return; + } + maintenance_journal_.reset(); + maintenance_.phase = MaintenancePhase::Failed; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(false, std::memory_order_release); + } + releaseMaintenanceLease(); +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::maintenanceFailure( + platform::esp::common::storage::StorageOperationResultKind kind) const +{ + return storage_contracts::StorageOperationResult::failure( + kind, + maintenance_.operation, + maintenance_.generation); +} + +bool SdProtocolPeerRepository::prepareMaintenanceJournal() +{ + const MeshProtocol protocol = kProtocols[maintenance_.protocol_index]; + const uint8_t index = maintenance_.journal_index; + const char* name = nullptr; + storage_v2::JournalKind kind = storage_v2::JournalKind::PeerSnapshot; + std::size_t slot_size = 0U; + switch (index) + { + case 0U: + name = "peers.snapshot"; + kind = storage_v2::JournalKind::PeerSnapshot; + slot_size = storage_v2::peerSlotSize(protocol); + break; + case 1U: + name = "peers.delta"; + kind = storage_v2::JournalKind::PeerDelta; + slot_size = storage_v2::peerSlotSize(protocol); + break; + case 2U: + name = "contacts.snapshot"; + kind = storage_v2::JournalKind::ContactSnapshot; + slot_size = storage_v2::contactSlotSize(protocol); + break; + case 3U: + name = "contacts.delta"; + kind = storage_v2::JournalKind::ContactDelta; + slot_size = storage_v2::contactSlotSize(protocol); + break; + default: + return false; + } + + buildProtocolPath(protocol, + name, + maintenance_path_, + sizeof(maintenance_path_)); + maintenance_protocol_ = protocol; + maintenance_kind_ = kind; + maintenance_slot_size_ = slot_size; + maintenance_.journal_started = maintenance_journal_.begin( + journal_, + maintenance_path_, + protocol, + kind, + slot_size); + return maintenance_.journal_started; +} + +bool SdProtocolPeerRepository::applyHydrationJournalSlot( + MeshProtocol protocol, + storage_v2::JournalKind kind) +{ + if (maintenance_slot_size_ > maintenance_scratch_.size()) + { + return false; + } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return false; + } + if (kind == storage_v2::JournalKind::PeerSnapshot || + kind == storage_v2::JournalKind::PeerDelta) + { + storage_v2::PeerProjection projection{}; + if (!storage_v2::decodePeerSlot(protocol, + maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + return true; + } + (void)applyPeerProjection(projection); + return true; + } + storage_v2::ContactProjection projection{}; + if (!storage_v2::decodeContactSlot(protocol, + maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + return true; + } + (void)applyContactProjection(projection); + return true; +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::stepHydration( + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + const uint8_t work_items = std::max(1U, budget.max_work_items); + for (uint8_t work = 0U; work < work_items; ++work) + { + switch (maintenance_.phase) + { + case MaintenancePhase::HydrationPrepare: + { + if (!storage_runtime::sd_card_ready() || !ensureLayout()) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + DeviceUnavailable); + } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + peers_.clear(); + contacts_.clear(); + pending_peer_deltas_.clear(); + pending_peer_head_ = 0U; + pending_contact_deltas_.clear(); + pending_contact_head_ = 0U; + pending_peer_revision_ = 0U; + pending_contact_revision_ = 0U; + std::memset(protocol_reset_pending_, 0, sizeof(protocol_reset_pending_)); + std::memset(protocol_reset_revision_, + 0, + sizeof(protocol_reset_revision_)); + std::memset(partitions_, 0, sizeof(partitions_)); + persistence_pending_.store(false, + std::memory_order_release); + compaction_pending_.store(false, + std::memory_order_release); + maintenance_.phase = MaintenancePhase::HydrationJournal; + break; + } + + case MaintenancePhase::HydrationJournal: + if (!maintenance_.journal_started && + !prepareMaintenanceJournal()) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + { + const auto status = maintenance_journal_.next( + journal_, + maintenance_scratch_.data(), + maintenance_scratch_.size()); + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Item) + { + if (!applyHydrationJournalSlot(maintenance_protocol_, + maintenance_kind_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + StateBusy); + } + break; + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Unavailable) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Invalid) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Complete) + { + const auto& inspection = maintenance_journal_.inspection(); + if (maintenance_kind_ == + storage_v2::JournalKind::PeerDelta) + { + partitions_[protocolIndex(maintenance_protocol_)] + .peer_delta_count = inspection.slot_count; + } + else if (maintenance_kind_ == + storage_v2::JournalKind::ContactDelta) + { + partitions_[protocolIndex(maintenance_protocol_)] + .contact_delta_count = inspection.slot_count; + } + } + maintenance_journal_.reset(); + maintenance_.journal_started = false; + ++maintenance_.journal_index; + if (maintenance_.journal_index >= kMaintenanceJournalCount) + { + maintenance_.journal_index = 0U; + ++maintenance_.protocol_index; + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + maintenance_.phase = MaintenancePhase::HydrationFinalize; + } + } + } + break; + + case MaintenancePhase::HydrationFinalize: + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + for (MeshProtocol protocol : kProtocols) + { + reconcileStableIdentities(protocol); + } + overlayContactFacts(); + drainDeferredObservationsLocked(); + hydrated_ = true; + hydrating_.store(false, std::memory_order_release); + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + } + + case MaintenancePhase::Complete: + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + + default: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + } + return storage_contracts::StorageOperationResult::inProgressResult( + maintenance_.operation, + maintenance_.generation); +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::stepPersistence( + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + if (!begun_ || !hydrated_ || + hydrating_.load(std::memory_order_acquire)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + + const MeshPeerDirectoryStatus flush_status = flushPendingDeltas( + std::max(1U, budget.max_work_items)); + if (!flush_status.succeeded()) + { + return maintenanceFailure(operationFailureKind(flush_status.code)); + } + if (!persistencePending()) + { + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + } + return storage_contracts::StorageOperationResult::inProgressResult( + maintenance_.operation, + maintenance_.generation); +} + +platform::esp::common::storage::StorageOperationResult +SdProtocolPeerRepository::stepCompaction( + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + if (!begun_ || !hydrated_ || + hydrating_.load(std::memory_order_acquire)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + + const uint8_t work_items = std::max(1U, budget.max_work_items); + for (uint8_t work = 0U; work < work_items; ++work) + { + switch (maintenance_.phase) + { + case MaintenancePhase::CompactionPrepare: + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + compaction_peers_ = peers_; + compaction_contacts_ = contacts_; + maintenance_.protocol_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_record_index = 0U; + maintenance_.compact_peers = false; + maintenance_.compact_contacts = false; + std::memset(compaction_force_peers_, + 0, + sizeof(compaction_force_peers_)); + std::memset(compaction_force_contacts_, + 0, + sizeof(compaction_force_contacts_)); + std::memcpy(compaction_reset_revision_, + protocol_reset_revision_, + sizeof(compaction_reset_revision_)); + compaction_peer_revision_ = pending_peer_revision_; + compaction_contact_revision_ = pending_contact_revision_; + for (std::size_t index = pending_peer_head_; + index < pending_peer_deltas_.size(); + ++index) + { + compaction_force_peers_[protocolIndex( + pending_peer_deltas_[index].record.identity.protocol)] = + true; + } + for (std::size_t index = pending_contact_head_; + index < pending_contact_deltas_.size(); + ++index) + { + compaction_force_contacts_[protocolIndex( + pending_contact_deltas_[index].identity.protocol)] = true; + } + for (std::size_t index = 0U; index < 3U; ++index) + { + compaction_force_peers_[index] = + compaction_force_peers_[index] || + protocol_reset_pending_[index]; + } + maintenance_.phase = MaintenancePhase::CompactionInspect; + break; + } + + case MaintenancePhase::CompactionInspect: + { + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult:: + completedResult(maintenance_.operation, + maintenance_.generation); + } + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const uint8_t index = maintenance_.compaction_inspection_index; + const char* name = index == 0U ? "peers.delta" + : "contacts.delta"; + const storage_v2::JournalKind kind = + index == 0U ? storage_v2::JournalKind::PeerDelta + : storage_v2::JournalKind::ContactDelta; + const std::size_t slot_size = + index == 0U ? storage_v2::peerSlotSize(protocol) + : storage_v2::contactSlotSize(protocol); + char path[96] = {}; + buildProtocolPath(protocol, name, path, sizeof(path)); + const auto inspection = + journal_.inspect(path, protocol, kind, slot_size); + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::IoError) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::Incompatible) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + const bool should_compact = + inspection.slot_count >= kBootCompactionDeltaThreshold || + (index == 0U ? compaction_force_peers_ + [protocolIndex(protocol)] + : compaction_force_contacts_ + [protocolIndex(protocol)]); + if (index == 0U) + { + maintenance_.compact_peers = should_compact; + } + else + { + maintenance_.compact_contacts = should_compact; + } + ++maintenance_.compaction_inspection_index; + if (maintenance_.compaction_inspection_index >= 2U) + { + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionCreate; + } + break; + } + + case MaintenancePhase::CompactionCreate: + { + while (maintenance_.compaction_projection_index < 2U) + { + const bool enabled = + maintenance_.compaction_projection_index == 0U + ? maintenance_.compact_peers + : maintenance_.compact_contacts; + if (enabled) + { + break; + } + ++maintenance_.compaction_projection_index; + } + if (maintenance_.compaction_projection_index >= 2U) + { + maintenance_.phase = MaintenancePhase::CompactionAdvance; + break; + } + + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const bool peers = + maintenance_.compaction_projection_index == 0U; + const char* base = peers ? "peers" : "contacts"; + maintenance_kind_ = + peers ? storage_v2::JournalKind::PeerSnapshot + : storage_v2::JournalKind::ContactSnapshot; + maintenance_slot_size_ = + peers ? storage_v2::peerSlotSize(protocol) + : storage_v2::contactSlotSize(protocol); + maintenance_protocol_ = protocol; + char final_name[40] = {}; + char temp_name[40] = {}; + char backup_name[40] = {}; + char delta_name[40] = {}; + std::snprintf(final_name, + sizeof(final_name), + "%s.snapshot", + base); + std::snprintf(temp_name, + sizeof(temp_name), + "%s.snapshot.tmp", + base); + std::snprintf(backup_name, + sizeof(backup_name), + "%s.snapshot.bak", + base); + std::snprintf(delta_name, + sizeof(delta_name), + "%s.delta", + base); + buildProtocolPath(protocol, + final_name, + maintenance_final_path_, + sizeof(maintenance_final_path_)); + buildProtocolPath(protocol, + temp_name, + maintenance_path_, + sizeof(maintenance_path_)); + buildProtocolPath(protocol, + backup_name, + maintenance_backup_path_, + sizeof(maintenance_backup_path_)); + buildProtocolPath(protocol, + delta_name, + maintenance_delta_path_, + sizeof(maintenance_delta_path_)); + (void)storage_runtime::sd_remove(maintenance_path_); + if (!journal_.create(maintenance_path_, + protocol, + maintenance_kind_, + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + maintenance_.compaction_record_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionWrite; + break; + } + + case MaintenancePhase::CompactionWrite: + { + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const bool peers = + maintenance_.compaction_projection_index == 0U; + while (true) + { + if (peers) + { + if (maintenance_.compaction_record_index >= + compaction_peers_.size()) + { + break; + } + const MeshPeerRecord& peer = + compaction_peers_[maintenance_.compaction_record_index++]; + if (!meshPeerSameProtocol(peer.identity.protocol, + protocol)) + { + continue; + } + const storage_v2::PeerProjection projection{peer, false}; + if (!storage_v2::encodePeerSlot( + protocol, + projection, + maintenance_scratch_.data(), + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + IoError); + } + } + else + { + if (maintenance_.compaction_record_index >= + compaction_contacts_.size()) + { + break; + } + const auto& contact = + compaction_contacts_[maintenance_.compaction_record_index++]; + if (!meshPeerSameProtocol(contact.identity.protocol, + protocol)) + { + continue; + } + if (!storage_v2::encodeContactSlot( + protocol, + contact, + maintenance_scratch_.data(), + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + IoError); + } + } + if (!journal_.append(maintenance_path_, + protocol, + maintenance_kind_, + maintenance_slot_size_, + maintenance_scratch_.data())) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + return storage_contracts::StorageOperationResult:: + inProgressResult(maintenance_.operation, + maintenance_.generation); + } + maintenance_.phase = MaintenancePhase::CompactionReplace; + break; + } + + case MaintenancePhase::CompactionReplace: + { + if (!storage_v2::replaceFileAtomically( + maintenance_path_, + maintenance_final_path_, + maintenance_backup_path_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const bool peers = + maintenance_.compaction_projection_index == 0U; + const storage_v2::JournalKind delta_kind = + peers ? storage_v2::JournalKind::PeerDelta + : storage_v2::JournalKind::ContactDelta; + if (!journal_.create(maintenance_delta_path_, + protocol, + delta_kind, + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + if (peers) + { + partitions_[protocolIndex(protocol)].peer_delta_count = + 0U; + } + else + { + partitions_[protocolIndex(protocol)].contact_delta_count = + 0U; + } + } + ++maintenance_.compaction_projection_index; + maintenance_.phase = MaintenancePhase::CompactionCreate; + break; + } + + case MaintenancePhase::CompactionAdvance: + ++maintenance_.protocol_index; + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + if (pending_peer_revision_ == compaction_peer_revision_) + { + pending_peer_deltas_.clear(); + pending_peer_head_ = 0U; + } + if (pending_contact_revision_ == compaction_contact_revision_) + { + pending_contact_deltas_.clear(); + pending_contact_head_ = 0U; + } + for (std::size_t index = 0U; index < 3U; ++index) + { + if (protocol_reset_pending_[index] && + protocol_reset_revision_[index] == + compaction_reset_revision_[index]) + { + protocol_reset_pending_[index] = false; + } + } + refreshPersistenceDemandLocked(); + compaction_peers_.clear(); + compaction_contacts_.clear(); + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult:: + completedResult(maintenance_.operation, + maintenance_.generation); + } + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionInspect; + break; + + case MaintenancePhase::Complete: + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + + default: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + } + return storage_contracts::StorageOperationResult::inProgressResult( + maintenance_.operation, + maintenance_.generation); } MeshPeerDirectoryStatus SdProtocolPeerRepository::begin() @@ -87,8 +1049,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::begin() ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } if (begun_) { @@ -100,98 +1061,6 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::begin() return MeshPeerDirectoryStatus::success(); } -MeshPeerDirectoryStatus SdProtocolPeerRepository::hydrateFromStorage() -{ - ScopedRepositoryLock lock(mutex_, portMAX_DELAY); - if (!lock.locked() || !begun_) - { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); - } - if (hydrated_) - { - return MeshPeerDirectoryStatus::success(); - } - const uint32_t started_ms = millis(); - if (!storage_runtime::sd_card_ready() || !ensureLayout()) - { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); - } - - PeerVector live_peers = std::move(peers_); - ContactVector live_contacts = std::move(contacts_); - PendingPeerVector live_pending = std::move(pending_peer_deltas_); - peers_.clear(); - contacts_.clear(); - pending_peer_deltas_.clear(); - pending_peer_head_ = 0U; - std::memset(partitions_, 0, sizeof(partitions_)); - - bool ok = true; - for (MeshProtocol protocol : kProtocols) - { - ok = loadProtocol(protocol) && ok; - } - if (!ok) - { - peers_ = std::move(live_peers); - contacts_ = std::move(live_contacts); - pending_peer_deltas_ = std::move(live_pending); - pending_peer_head_ = 0U; - return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); - } - - for (const MeshPeerRecord& peer : live_peers) - { - (void)applyPeerProjection({peer, false}); - } - for (const storage_v2::ContactProjection& contact : live_contacts) - { - (void)applyContactProjection(contact); - } - pending_peer_deltas_ = std::move(live_pending); - pending_peer_head_ = 0U; - for (MeshProtocol protocol : kProtocols) - { - reconcileStableIdentities(protocol); - } - overlayContactFacts(); - drainDeferredObservationsLocked(); - hydrated_ = true; - Serial.printf("[PeerStoreV2] hydration ready=1 peers=%u contacts=%u elapsed_ms=%lu\n", - static_cast(peers_.size()), - static_cast(contacts_.size()), - static_cast(millis() - started_ms)); - return MeshPeerDirectoryStatus::success(); -} - -MeshPeerDirectoryStatus SdProtocolPeerRepository::compactDeferred() -{ - ScopedRepositoryLock lock(mutex_, portMAX_DELAY); - if (!lock.locked() || !begun_ || !hydrated_) - { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); - } - const uint32_t started_ms = millis(); - bool ok = true; - for (MeshProtocol protocol : kProtocols) - { - if (!compactProtocolAtBoot(protocol)) - { - Serial.printf("[PeerStoreV2] deferred compaction failed protocol=%s\n", - protocolSlug(protocol)); - ok = false; - } - } - Serial.printf("[PeerStoreV2] deferred_compaction ok=%u elapsed_ms=%lu\n", - ok ? 1U : 0U, - static_cast(millis() - started_ms)); - return ok ? MeshPeerDirectoryStatus::success() - : MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); -} - bool SdProtocolPeerRepository::ensureLayout() { if (!storage_runtime::sd_card_ready() || !ensureDirectory("/data") || @@ -214,231 +1083,11 @@ bool SdProtocolPeerRepository::ensureProtocolLayout(MeshProtocol protocol) return ensureDirectory(path); } -bool SdProtocolPeerRepository::loadProtocol(MeshProtocol protocol) -{ - for (const char* base : {"peers", "contacts"}) - { - char final_path[96] = {}; - char temp_path[96] = {}; - char backup_path[96] = {}; - char name[32] = {}; - std::snprintf(name, sizeof(name), "%s.snapshot", base); - buildProtocolPath(protocol, - name, - final_path, - sizeof(final_path)); - std::snprintf(name, sizeof(name), "%s.snapshot.tmp", base); - buildProtocolPath(protocol, - name, - temp_path, - sizeof(temp_path)); - std::snprintf(name, sizeof(name), "%s.snapshot.bak", base); - buildProtocolPath(protocol, - name, - backup_path, - sizeof(backup_path)); - if (!storage_v2::recoverAtomicFile(final_path, - temp_path, - backup_path)) - { - return false; - } - } - return loadPeerJournal(protocol, "peers.snapshot") && - loadPeerJournal(protocol, "peers.delta") && - loadContactJournal(protocol, "contacts.snapshot") && - loadContactJournal(protocol, "contacts.delta"); -} - -bool SdProtocolPeerRepository::loadPeerJournal(MeshProtocol protocol, - const char* name) -{ - char path[96] = {}; - buildProtocolPath(protocol, name, path, sizeof(path)); - const std::size_t slot_size = storage_v2::peerSlotSize(protocol); - const storage_v2::JournalKind kind = - std::strstr(name, ".snapshot") - ? storage_v2::JournalKind::PeerSnapshot - : storage_v2::JournalKind::PeerDelta; - const auto inspection = journal_.inspect(path, protocol, kind, slot_size); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - return true; - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - Serial.printf("[PeerStoreV2] incompatible path=%s state=%u\n", - path, - static_cast(inspection.state)); - return false; - } - if (slot_size > slot_scratch_.size()) - { - return false; - } - uint32_t decode_failures = 0U; - uint32_t first_decode_failure = inspection.slot_count; - uint32_t last_decode_failure = 0U; - for (uint32_t index = 0; index < inspection.slot_count; ++index) - { - storage_v2::PeerProjection projection{}; - const auto read_status = journal_.readStatus(path, - protocol, - kind, - slot_size, - inspection, - index, - slot_scratch_.data()); - if (read_status != - storage_v2::FixedSlotJournalEngine::ReadStatus::Ok) - { - Serial.printf("[PeerStoreV2] hydration deferred path=%s index=%lu read_status=%u\n", - path, - static_cast(index), - static_cast(read_status)); - return false; - } - if (!storage_v2::decodePeerSlot(protocol, - slot_scratch_.data(), - slot_size, - projection)) - { - ++decode_failures; - first_decode_failure = - std::min(first_decode_failure, index); - last_decode_failure = index; - continue; - } - (void)applyPeerProjection(projection); - } - if (decode_failures > 0U) - { - Serial.printf("[PeerStoreV2] invalid peer slots path=%s count=%lu first=%lu last=%lu\n", - path, - static_cast(decode_failures), - static_cast(first_decode_failure), - static_cast(last_decode_failure)); - } - if (kind == storage_v2::JournalKind::PeerDelta) - { - partitions_[protocolIndex(protocol)].peer_delta_count = - inspection.slot_count; - } - if (inspection.state == - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - Serial.printf("[PeerStoreV2] partial peer tail path=%s valid=%lu\n", - path, - static_cast(inspection.slot_count)); - } - return true; -} - -bool SdProtocolPeerRepository::loadContactJournal(MeshProtocol protocol, - const char* name) -{ - char path[96] = {}; - buildProtocolPath(protocol, name, path, sizeof(path)); - const std::size_t slot_size = storage_v2::contactSlotSize(protocol); - const storage_v2::JournalKind kind = - std::strstr(name, ".snapshot") - ? storage_v2::JournalKind::ContactSnapshot - : storage_v2::JournalKind::ContactDelta; - const auto inspection = journal_.inspect(path, protocol, kind, slot_size); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - return true; - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - Serial.printf("[PeerStoreV2] incompatible path=%s state=%u\n", - path, - static_cast(inspection.state)); - return false; - } - if (slot_size > slot_scratch_.size()) - { - return false; - } - uint32_t decode_failures = 0U; - uint32_t first_decode_failure = inspection.slot_count; - uint32_t last_decode_failure = 0U; - for (uint32_t index = 0; index < inspection.slot_count; ++index) - { - storage_v2::ContactProjection projection{}; - const auto read_status = journal_.readStatus(path, - protocol, - kind, - slot_size, - inspection, - index, - slot_scratch_.data()); - if (read_status != - storage_v2::FixedSlotJournalEngine::ReadStatus::Ok) - { - Serial.printf("[PeerStoreV2] hydration deferred path=%s index=%lu read_status=%u\n", - path, - static_cast(index), - static_cast(read_status)); - return false; - } - if (!storage_v2::decodeContactSlot(protocol, - slot_scratch_.data(), - slot_size, - projection)) - { - ++decode_failures; - first_decode_failure = - std::min(first_decode_failure, index); - last_decode_failure = index; - continue; - } - (void)applyContactProjection(projection); - } - if (decode_failures > 0U) - { - Serial.printf("[PeerStoreV2] invalid contact slots path=%s count=%lu first=%lu last=%lu\n", - path, - static_cast(decode_failures), - static_cast(first_decode_failure), - static_cast(last_decode_failure)); - } - if (kind == storage_v2::JournalKind::ContactDelta) - { - partitions_[protocolIndex(protocol)].contact_delta_count = - inspection.slot_count; - } - if (inspection.state == - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - Serial.printf("[PeerStoreV2] partial contact tail path=%s valid=%lu\n", - path, - static_cast(inspection.slot_count)); - } - return true; -} - -bool SdProtocolPeerRepository::compactProtocolAtBoot(MeshProtocol protocol) -{ - PartitionState& state = partitions_[protocolIndex(protocol)]; - bool ok = true; - if (state.peer_delta_count >= kBootCompactionDeltaThreshold) - { - ok = rewritePeerSnapshot(protocol) && ok; - } - if (state.contact_delta_count >= kBootCompactionDeltaThreshold) - { - ok = rewriteContactSnapshot(protocol) && ok; - } - return ok; -} - -bool SdProtocolPeerRepository::rewritePeerSnapshot(MeshProtocol protocol) +bool SdProtocolPeerRepository::rewritePeerSnapshotFrom( + MeshProtocol protocol, + const PeerVector& snapshot) { + protocol = normalizeProtocol(protocol); char target[96] = {}; char temp[96] = {}; char backup[96] = {}; @@ -450,19 +1099,19 @@ bool SdProtocolPeerRepository::rewritePeerSnapshot(MeshProtocol protocol) backup, sizeof(backup)); buildProtocolPath(protocol, "peers.delta", delta, sizeof(delta)); - if (storage_runtime::sd_exists(temp)) - { - storage_runtime::sd_remove(temp); - } + + (void)storage_runtime::sd_remove(temp); const std::size_t slot_size = storage_v2::peerSlotSize(protocol); - if (!journal_.create(temp, + if (slot_size == 0U || slot_size > slot_scratch_.size() || + !journal_.create(temp, protocol, storage_v2::JournalKind::PeerSnapshot, slot_size)) { return false; } - for (const MeshPeerRecord& peer : peers_) + + for (const MeshPeerRecord& peer : snapshot) { if (!meshPeerSameProtocol(peer.identity.protocol, protocol)) { @@ -479,10 +1128,11 @@ bool SdProtocolPeerRepository::rewritePeerSnapshot(MeshProtocol protocol) slot_size, slot_scratch_.data())) { - storage_runtime::sd_remove(temp); + (void)storage_runtime::sd_remove(temp); return false; } } + if (!storage_v2::replaceFileAtomically(temp, target, backup) || !journal_.create(delta, protocol, @@ -491,64 +1141,6 @@ bool SdProtocolPeerRepository::rewritePeerSnapshot(MeshProtocol protocol) { return false; } - partitions_[protocolIndex(protocol)].peer_delta_count = 0U; - return true; -} - -bool SdProtocolPeerRepository::rewriteContactSnapshot(MeshProtocol protocol) -{ - char target[96] = {}; - char temp[96] = {}; - char backup[96] = {}; - char delta[96] = {}; - buildProtocolPath(protocol, "contacts.snapshot", target, sizeof(target)); - buildProtocolPath(protocol, "contacts.snapshot.tmp", temp, sizeof(temp)); - buildProtocolPath(protocol, - "contacts.snapshot.bak", - backup, - sizeof(backup)); - buildProtocolPath(protocol, "contacts.delta", delta, sizeof(delta)); - if (storage_runtime::sd_exists(temp)) - { - storage_runtime::sd_remove(temp); - } - const std::size_t slot_size = storage_v2::contactSlotSize(protocol); - if (!journal_.create(temp, - protocol, - storage_v2::JournalKind::ContactSnapshot, - slot_size)) - { - return false; - } - for (const storage_v2::ContactProjection& contact : contacts_) - { - if (!meshPeerSameProtocol(contact.identity.protocol, protocol)) - { - continue; - } - if (!storage_v2::encodeContactSlot(protocol, - contact, - slot_scratch_.data(), - slot_size) || - !journal_.append(temp, - protocol, - storage_v2::JournalKind::ContactSnapshot, - slot_size, - slot_scratch_.data())) - { - storage_runtime::sd_remove(temp); - return false; - } - } - if (!storage_v2::replaceFileAtomically(temp, target, backup) || - !journal_.create(delta, - protocol, - storage_v2::JournalKind::ContactDelta, - slot_size)) - { - return false; - } - partitions_[protocolIndex(protocol)].contact_delta_count = 0U; return true; } @@ -576,7 +1168,6 @@ bool SdProtocolPeerRepository::appendPeerDelta( { return false; } - ++partitions_[protocolIndex(protocol)].peer_delta_count; return true; } @@ -603,54 +1194,86 @@ bool SdProtocolPeerRepository::appendContactDelta( { return false; } - ++partitions_[protocolIndex(protocol)].contact_delta_count; return true; } -bool SdProtocolPeerRepository::queueOrAppendPeerDelta( +bool SdProtocolPeerRepository::queuePeerDelta( const storage_v2::PeerProjection& projection) { - if (pending_peer_head_ < pending_peer_deltas_.size()) + prunePendingDeltasLocked(); + if (!pending_peer_deltas_.empty()) { storage_v2::PeerProjection& newest = pending_peer_deltas_.back(); - if (newest.deleted == projection.deleted && - sameMeshPeerIdentity(newest.record.identity, + if (sameMeshPeerIdentity(newest.record.identity, projection.record.identity)) { newest = projection; + ++pending_peer_revision_; + refreshPersistenceDemandLocked(); return false; } - pending_peer_deltas_.push_back(projection); - return false; - } - if (appendPeerDelta(projection)) - { - return true; } pending_peer_deltas_.push_back(projection); + ++pending_peer_revision_; + refreshPersistenceDemandLocked(); return false; } -bool SdProtocolPeerRepository::drainPendingPeerDeltas(std::size_t budget) +bool SdProtocolPeerRepository::queueContactDelta( + const storage_v2::ContactProjection& projection) { - std::size_t drained = 0U; - while (pending_peer_head_ < pending_peer_deltas_.size() && - drained < budget) + prunePendingDeltasLocked(); + if (!pending_contact_deltas_.empty()) { - if (!appendPeerDelta(pending_peer_deltas_[pending_peer_head_])) + storage_v2::ContactProjection& newest = pending_contact_deltas_.back(); + if (sameMeshPeerIdentity(newest.identity, projection.identity)) { - return false; + newest = projection; + ++pending_contact_revision_; + refreshPersistenceDemandLocked(); + return true; } - ++pending_peer_head_; - ++drained; } + pending_contact_deltas_.push_back(projection); + ++pending_contact_revision_; + refreshPersistenceDemandLocked(); + return true; +} + +void SdProtocolPeerRepository::prunePendingDeltasLocked() +{ if (pending_peer_head_ == pending_peer_deltas_.size()) { pending_peer_deltas_.clear(); pending_peer_head_ = 0U; - return true; } - return false; + if (pending_contact_head_ == pending_contact_deltas_.size()) + { + pending_contact_deltas_.clear(); + pending_contact_head_ = 0U; + } +} + +void SdProtocolPeerRepository::refreshPersistenceDemandLocked() +{ + const bool has_pending_deltas = + pending_peer_head_ < pending_peer_deltas_.size() || + pending_contact_head_ < pending_contact_deltas_.size(); + persistence_pending_.store(has_pending_deltas, + std::memory_order_release); + + bool needs_compaction = false; + for (std::size_t index = 0U; index < 3U; ++index) + { + needs_compaction = + needs_compaction || protocol_reset_pending_[index] || + partitions_[index].peer_delta_count >= + kBootCompactionDeltaThreshold || + partitions_[index].contact_delta_count >= + kBootCompactionDeltaThreshold; + } + compaction_pending_.store(needs_compaction, + std::memory_order_release); } bool SdProtocolPeerRepository::applyPeerProjection( @@ -874,7 +1497,13 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::record( MeshPeerDirectoryStatusCode::InvalidArgument); } ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || !begun_) + if (!lock.locked()) + { + (void)queueDeferredObservation(input); + return stateLockStatus(lock.result()); + } + if (!begun_ || !hydrated_ || + hydrating_.load(std::memory_order_acquire)) { (void)queueDeferredObservation(input); return MeshPeerDirectoryStatus::fail( @@ -940,13 +1569,13 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::recordLocked( } const storage_v2::PeerProjection next_projection{next, false}; - const bool immediately_durable = queueOrAppendPeerDelta(next_projection); + (void)queuePeerDelta(next_projection); if (identity_upgrade) { storage_v2::PeerProjection tombstone{}; tombstone.record = peers_[merge_index]; tombstone.deleted = true; - (void)queueOrAppendPeerDelta(tombstone); + (void)queuePeerDelta(tombstone); } if (merge_index < peers_.size()) @@ -961,13 +1590,6 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::recordLocked( { overlayContactFactsForPeer(peers_[merge_index]); } - if (!immediately_durable) - { - Serial.printf("[PeerStoreV2] peer queued protocol=%s pending=%u\n", - protocolSlug(next.identity.protocol), - static_cast(pending_peer_deltas_.size() - - pending_peer_head_)); - } return MeshPeerDirectoryStatus::success(); } @@ -978,8 +1600,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::find( ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } const std::size_t index = findPeerIndex(identity); if (index >= peers_.size()) @@ -996,12 +1617,16 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::findByNodeId( NodeId node_id, MeshPeerRecord& out_record) { - ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || node_id == 0U) + if (node_id == 0U) { return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::InvalidArgument); } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } const std::size_t index = findPeerIndexByNodeId(protocol, node_id); if (index >= peers_.size()) { @@ -1018,12 +1643,16 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::loadRecent( std::size_t max_records, std::size_t* out_count) { - ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || !out_count || (!out_records && max_records > 0U)) + if (!out_count || (!out_records && max_records > 0U)) { return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::InvalidArgument); } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } protocol = normalizeProtocol(protocol); using PeerPtrVector = std::vector< const MeshPeerRecord*, @@ -1059,13 +1688,16 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::search( std::size_t max_records, std::size_t* out_count) { - ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || !query || !out_count || - (!out_records && max_records > 0U)) + if (!query || !out_count || (!out_records && max_records > 0U)) { return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::InvalidArgument); } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } protocol = normalizeProtocol(protocol); using PeerPtrVector = std::vector< const MeshPeerRecord*, @@ -1108,8 +1740,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::setUserFlags( ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } const std::size_t peer_index = findPeerIndex(identity); if (peer_index >= peers_.size()) @@ -1143,6 +1774,11 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::visit( { ScopedRepositoryLock lock(mutex_); if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + if (!begun_ || !hydrated_ || + hydrating_.load(std::memory_order_acquire)) { return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::StorageUnavailable); @@ -1182,8 +1818,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::setUserAlias( ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } const std::size_t peer_index = findPeerIndex(identity); if (peer_index >= peers_.size()) @@ -1214,8 +1849,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::setKeyManuallyVerified( ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } const std::size_t index = findPeerIndex(identity); if (index >= peers_.size()) @@ -1240,11 +1874,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::setKeyManuallyVerified( return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::Unsupported); } - if (!appendPeerDelta(storage_v2::PeerProjection{next, false})) - { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); - } + (void)queuePeerDelta(storage_v2::PeerProjection{next, false}); peers_[index] = next; return MeshPeerDirectoryStatus::success(); } @@ -1255,8 +1885,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::remove( ScopedRepositoryLock lock(mutex_); if (!lock.locked()) { - return MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + return stateLockStatus(lock.result()); } const std::size_t index = findPeerIndex(identity); if (index >= peers_.size()) @@ -1272,7 +1901,7 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::remove( storage_v2::PeerProjection tombstone{}; tombstone.record = peers_[index]; tombstone.deleted = true; - (void)queueOrAppendPeerDelta(tombstone); + (void)queuePeerDelta(tombstone); peers_.erase(peers_.begin() + static_cast(index)); return MeshPeerDirectoryStatus::success(); } @@ -1281,7 +1910,11 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::clearProtocol( MeshProtocol protocol) { ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || !begun_) + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + if (!begun_) { return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::StorageUnavailable); @@ -1314,12 +1947,19 @@ MeshPeerDirectoryStatus SdProtocolPeerRepository::clearProtocol( protocol); }), pending_peer_deltas_.end()); + prunePendingDeltasLocked(); + ++pending_peer_revision_; overlayContactFacts(); - - if (!rewritePeerSnapshot(protocol)) + const std::size_t index = protocolIndex(protocol); + protocol_reset_pending_[index] = true; + ++protocol_reset_revision_[index]; + if (protocol_reset_revision_[index] == 0U) { - return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); + protocol_reset_revision_[index] = 1U; } + refreshPersistenceDemandLocked(); + Serial.printf("[PeerStoreV2] protocol_reset queued protocol=%s\n", + protocolSlug(protocol)); return MeshPeerDirectoryStatus::success(); } @@ -1331,18 +1971,196 @@ MeshPeerDirectoryCapacity SdProtocolPeerRepository::capacityFor( kPeerHotCacheCapacity[index]}; } +MeshPeerDirectoryStatus SdProtocolPeerRepository::flushProtocolReset() +{ + MeshProtocol protocol = MeshProtocol::Meshtastic; + uint32_t reset_revision = 0U; + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + std::size_t index = 0U; + while (index < 3U && !protocol_reset_pending_[index]) + { + ++index; + } + if (index >= 3U) + { + return MeshPeerDirectoryStatus::success(); + } + protocol = kProtocols[index]; + reset_revision = protocol_reset_revision_[index]; + flush_peer_snapshot_ = peers_; + } + + if (!rewritePeerSnapshotFrom(protocol, flush_peer_snapshot_)) + { + return ioFailureStatus(); + } + + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + const std::size_t index = protocolIndex(protocol); + if (protocol_reset_revision_[index] == reset_revision) + { + protocol_reset_pending_[index] = false; + partitions_[index].peer_delta_count = 0U; + } + refreshPersistenceDemandLocked(); + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::flushPendingDeltas( + std::size_t budget) +{ + flush_peer_batch_.clear(); + flush_contact_batch_.clear(); + std::size_t peer_start = 0U; + std::size_t contact_start = 0U; + uint32_t peer_revision = 0U; + uint32_t contact_revision = 0U; + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + prunePendingDeltasLocked(); + peer_start = pending_peer_head_; + contact_start = pending_contact_head_; + peer_revision = pending_peer_revision_; + contact_revision = pending_contact_revision_; + const std::size_t peer_end = + std::min(pending_peer_deltas_.size(), peer_start + budget); + const std::size_t contact_end = + std::min(pending_contact_deltas_.size(), contact_start + budget); + flush_peer_batch_.assign(pending_peer_deltas_.begin() + + static_cast(peer_start), + pending_peer_deltas_.begin() + + static_cast(peer_end)); + flush_contact_batch_.assign(pending_contact_deltas_.begin() + + static_cast( + contact_start), + pending_contact_deltas_.begin() + + static_cast(contact_end)); + } + + std::size_t peer_written = 0U; + std::size_t contact_written = 0U; + uint32_t peer_counts[3] = {}; + uint32_t contact_counts[3] = {}; + for (const auto& projection : flush_peer_batch_) + { + if (!appendPeerDelta(projection)) + { + break; + } + ++peer_written; + ++peer_counts[protocolIndex(projection.record.identity.protocol)]; + } + if (peer_written != flush_peer_batch_.size()) + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + for (std::size_t index = 0U; index < 3U; ++index) + { + partitions_[index].peer_delta_count += peer_counts[index]; + } + if (pending_peer_revision_ == peer_revision) + { + pending_peer_head_ = peer_start + peer_written; + prunePendingDeltasLocked(); + } + refreshPersistenceDemandLocked(); + return ioFailureStatus(); + } + for (const auto& projection : flush_contact_batch_) + { + if (!appendContactDelta(projection)) + { + break; + } + ++contact_written; + ++contact_counts[protocolIndex(projection.identity.protocol)]; + } + + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return stateLockStatus(lock.result()); + } + for (std::size_t index = 0U; index < 3U; ++index) + { + partitions_[index].peer_delta_count += peer_counts[index]; + partitions_[index].contact_delta_count += contact_counts[index]; + } + if (pending_peer_revision_ == peer_revision) + { + pending_peer_head_ = peer_start + peer_written; + prunePendingDeltasLocked(); + } + if (pending_contact_revision_ == contact_revision) + { + pending_contact_head_ = contact_start + contact_written; + prunePendingDeltasLocked(); + } + refreshPersistenceDemandLocked(); + return contact_written == flush_contact_batch_.size() + ? MeshPeerDirectoryStatus::success() + : ioFailureStatus(); +} + MeshPeerDirectoryStatus SdProtocolPeerRepository::flush() { - ScopedRepositoryLock lock(mutex_); - if (!lock.locked() || !begun_) + if (!acquirePersistenceLease(kPersistenceLeaseWaitTicks)) { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Busy); + } + bool begun = false; + MeshPeerDirectoryStatus lock_status = + MeshPeerDirectoryStatus::success(); + { + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + lock_status = stateLockStatus(lock.result()); + } + else + { + begun = begun_; + } + } + if (!lock_status.succeeded()) + { + releasePersistenceLease(); + return lock_status; + } + if (!begun) + { + releasePersistenceLease(); return MeshPeerDirectoryStatus::fail( MeshPeerDirectoryStatusCode::StorageUnavailable); } - return drainPendingPeerDeltas(kPendingFlushBudget) - ? MeshPeerDirectoryStatus::success() - : MeshPeerDirectoryStatus::fail( - MeshPeerDirectoryStatusCode::StorageUnavailable); + + const MeshPeerDirectoryStatus reset_status = flushProtocolReset(); + if (!reset_status.succeeded()) + { + releasePersistenceLease(); + return reset_status; + } + const MeshPeerDirectoryStatus deltas_status = + flushPendingDeltas(kPendingFlushBudget); + releasePersistenceLease(); + return deltas_status; } std::size_t SdProtocolPeerRepository::findPeerIndex( @@ -1535,7 +2353,7 @@ bool SdProtocolPeerRepository::evictOldestEphemeral(MeshProtocol protocol) storage_v2::PeerProjection tombstone{}; tombstone.record = peers_[candidate]; tombstone.deleted = true; - (void)queueOrAppendPeerDelta(tombstone); + (void)queuePeerDelta(tombstone); peers_.erase(peers_.begin() + static_cast(candidate)); return true; } @@ -1566,10 +2384,7 @@ bool SdProtocolPeerRepository::persistContactFacts( { projection.node_id_hint = projectedNodeId(peers_[peer_index]); } - if (!appendContactDelta(projection)) - { - return false; - } + (void)queueContactDelta(projection); (void)applyContactProjection(projection); if (peer_index < peers_.size()) { diff --git a/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp b/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp index 1a060c74..7e18eb44 100644 --- a/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp +++ b/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp @@ -24,6 +24,44 @@ namespace { namespace storage_runtime = ::platform::esp::arduino_common::storage; namespace storage_v2 = ::chat::storage::v2; +namespace storage_contracts = ::platform::esp::common::storage; + +storage_contracts::StorageOperationResultKind stateLockFailure( + storage_runtime::StateLockResult result) +{ + return result == storage_runtime::StateLockResult::Unavailable + ? storage_contracts::StorageOperationResultKind:: + DeviceUnavailable + : storage_contracts::StorageOperationResultKind::StateBusy; +} + +class ScopedPersistenceLease final +{ + public: + ScopedPersistenceLease(SemaphoreHandle_t mutex, TickType_t wait_ticks) + : mutex_(mutex) + { + locked_ = mutex_ && + xSemaphoreTakeRecursive(mutex_, wait_ticks) == pdTRUE; + } + + ~ScopedPersistenceLease() + { + if (locked_) + { + xSemaphoreGiveRecursive(mutex_); + } + } + + ScopedPersistenceLease(const ScopedPersistenceLease&) = delete; + ScopedPersistenceLease& operator=(const ScopedPersistenceLease&) = delete; + + bool locked() const { return locked_; } + + private: + SemaphoreHandle_t mutex_ = nullptr; + bool locked_ = false; +}; uint32_t monotonic_millis() { @@ -46,6 +84,10 @@ constexpr MeshProtocol kProtocols[] = { MeshProtocol::Reticulum, }; +constexpr uint8_t kHydrationJournalCount = 6U; +constexpr uint8_t kHydrationRecoveryCount = 3U; +constexpr TickType_t kPersistenceLeaseWaitTicks = pdMS_TO_TICKS(50U); + std::size_t protocolIndex(MeshProtocol protocol) { if (protocol == MeshProtocol::MeshCore) @@ -120,22 +162,6 @@ bool hasSuffix(const char* value, const char* suffix) suffix_len) == 0; } -bool replaceSnapshot(const char* temp_path, const char* final_path) -{ - if (!temp_path || !final_path) - { - return false; - } - char backup_path[160] = {}; - std::snprintf(backup_path, - sizeof(backup_path), - "%s.bak", - final_path); - return storage_v2::replaceFileAtomically(temp_path, - final_path, - backup_path); -} - void hashToHex(const uint8_t* hash, char* out, std::size_t out_len) { static constexpr char kHex[] = "0123456789abcdef"; @@ -157,76 +183,1440 @@ SdStore::SdStore() : mutex_(xSemaphoreCreateRecursiveMutex()) { scratch_.resize(kScratchCapacity); + maintenance_scratch_.resize(kScratchCapacity); catalog_.reserve(64); read_state_.reserve(64); statuses_.reserve(256); seen_hot_.reserve(256); + persistence_mutex_ = xSemaphoreCreateRecursiveMutex(); CHAT_STORE_LOG("[ChatStoreV2] constructed ready=0 hydration=pending root=%s\n", kRoot); } SdStore::~SdStore() { + resetCatalogReconcileCursor(); if (mutex_) { vSemaphoreDelete(mutex_); mutex_ = nullptr; } + if (persistence_mutex_) + { + vSemaphoreDelete(persistence_mutex_); + persistence_mutex_ = nullptr; + } } -bool SdStore::hydrateFromStorage() +bool SdStore::acquirePersistenceLease(TickType_t wait_ticks) { - if (ready_.load(std::memory_order_acquire)) - { - return true; - } - if (hydrating_.exchange(true, std::memory_order_acq_rel)) - { - return false; - } - - const uint32_t started_ms = monotonic_millis(); - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_, portMAX_DELAY); - const bool ok = state_lock.locked() && storage_runtime::sd_card_ready() && - ensureLayout() && loadRuntimeState(); - if (ok) - { - ready_.store(true, std::memory_order_release); - } - hydrating_.store(false, std::memory_order_release); - CHAT_STORE_LOG("[ChatStoreV2] hydration ready=%u elapsed_ms=%lu conversations=%u statuses=%u seen_hot=%u\n", - ok ? 1U : 0U, - static_cast(monotonic_millis() - started_ms), - static_cast(catalog_.size()), - static_cast(statuses_.size()), - static_cast(seen_hot_.size())); - return ok; + return persistence_mutex_ && + xSemaphoreTakeRecursive(persistence_mutex_, wait_ticks) == pdTRUE; } -bool SdStore::compactDeferred() +void SdStore::releasePersistenceLease() { - if (!ready_.load(std::memory_order_acquire)) + if (persistence_mutex_) { - return false; + xSemaphoreGiveRecursive(persistence_mutex_); } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_, portMAX_DELAY); +} + +void SdStore::releaseMaintenanceLease() +{ + if (maintenance_persistence_locked_) + { + maintenance_persistence_locked_ = false; + releasePersistenceLease(); + } +} + +void SdStore::resetCatalogReconcileCursor() +{ + maintenance_directory_.close(); + maintenance_directory_open_ = false; + maintenance_reconcile_name_[0] = '\0'; + maintenance_reconcile_directory_path_[0] = '\0'; + maintenance_reconcile_projection_ = {}; + maintenance_reconcile_conversation_active_ = false; + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ScanSegments; + maintenance_reconcile_segment_ = 0U; + maintenance_reconcile_total_count_ = 0U; + maintenance_reconcile_last_segment_ = 0U; + maintenance_reconcile_last_segment_count_ = 0U; + maintenance_reconcile_unread_ordinal_ = 0U; + maintenance_reconcile_unread_count_ = 0U; + maintenance_reconcile_found_segment_ = false; + maintenance_reconcile_catalog_current_ = false; +} + +const char* SdStore::hydrationRecoveryName(uint8_t index) +{ + switch (index) + { + case 0U: + return "catalog"; + case 1U: + return "read"; + case 2U: + return "status"; + default: + return nullptr; + } +} + +bool SdStore::resetHydrationState() +{ + resetCatalogReconcileCursor(); + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); if (!state_lock.locked()) { return false; } - const uint32_t started_ms = monotonic_millis(); - bool ok = true; - for (MeshProtocol protocol : kProtocols) + catalog_.clear(); + read_state_.clear(); + statuses_.clear(); + seen_hot_.clear(); + projection_dirty_[0] = false; + projection_dirty_[1] = false; + projection_dirty_[2] = false; + maintenance_.protocol_index = 0U; + maintenance_.journal_index = 0U; + maintenance_.recovery_index = 0U; + maintenance_.journal_started = false; + maintenance_.seen_journal_found = false; + maintenance_.seen_rebuild_required = false; + maintenance_journal_.reset(); + maintenance_path_[0] = '\0'; + maintenance_seen_catalog_index_ = 0U; + maintenance_seen_message_count_ = 0U; + maintenance_seen_message_ordinal_ = 0U; + maintenance_seen_rebuild_started_ = false; + maintenance_reconcile_name_[0] = '\0'; + return true; +} + +bool SdStore::beginSeenRebuild() +{ + if (maintenance_seen_rebuild_started_) { - if (!compactProtocolProjections(protocol)) + return true; + } + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot", + maintenance_final_path_, + sizeof(maintenance_final_path_)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot.tmp", + maintenance_path_, + sizeof(maintenance_path_)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot.bak", + maintenance_backup_path_, + sizeof(maintenance_backup_path_)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.delta", + maintenance_delta_path_, + sizeof(maintenance_delta_path_)); + (void)storage_runtime::sd_remove(maintenance_path_); + const std::size_t slot_size = storage_v2::reticulumSeenSlotSize(); + if (!journal_.create(maintenance_path_, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size)) + { + return false; + } + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return false; + } + seen_hot_.clear(); + maintenance_seen_catalog_index_ = 0U; + maintenance_seen_message_count_ = 0U; + maintenance_seen_message_ordinal_ = 0U; + maintenance_seen_rebuild_started_ = true; + return true; +} + +SdStore::ReconcileStepResult SdStore::stepSeenRebuild() +{ + if (!beginSeenRebuild()) + { + return ReconcileStepResult::Failed; + } + + bool rebuild_complete = false; + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) { - projection_dirty_[protocolIndex(protocol)] = true; - ok = false; + return ReconcileStepResult::InProgress; + } + if (maintenance_seen_catalog_index_ >= catalog_.size()) + { + rebuild_complete = true; + } + else + { + maintenance_seen_catalog_ = + catalog_[maintenance_seen_catalog_index_]; } } - CHAT_STORE_LOG("[ChatStoreV2] deferred_compaction ok=%u elapsed_ms=%lu\n", - ok ? 1U : 0U, - static_cast(monotonic_millis() - started_ms)); - return ok; + if (rebuild_complete) + { + if (!storage_v2::replaceFileAtomically(maintenance_path_, + maintenance_final_path_, + maintenance_backup_path_)) + { + return ReconcileStepResult::Failed; + } + (void)storage_runtime::sd_remove(maintenance_delta_path_); + maintenance_seen_rebuild_started_ = false; + return ReconcileStepResult::Complete; + } + + if (maintenance_seen_catalog_.deleted || + !sameProtocol(maintenance_seen_catalog_.conversation.protocol, + MeshProtocol::Reticulum)) + { + ++maintenance_seen_catalog_index_; + maintenance_seen_message_count_ = 0U; + maintenance_seen_message_ordinal_ = 0U; + return ReconcileStepResult::InProgress; + } + + if (maintenance_seen_message_count_ == 0U) + { + maintenance_seen_conversation_ = + maintenance_seen_catalog_.conversation; + maintenance_seen_message_count_ = + messageCountOnDisk(maintenance_seen_conversation_); + } + if (maintenance_seen_message_ordinal_ >= + maintenance_seen_message_count_) + { + ++maintenance_seen_catalog_index_; + maintenance_seen_message_count_ = 0U; + maintenance_seen_message_ordinal_ = 0U; + return ReconcileStepResult::InProgress; + } + + if (!readMessageByOrdinal(maintenance_seen_conversation_, + maintenance_seen_message_ordinal_, + maintenance_seen_message_)) + { + return ReconcileStepResult::Failed; + } + if (chat::hasReticulumLxmfMessageHash(maintenance_seen_message_)) + { + storage_v2::ReticulumSeenProjection projection{}; + std::memcpy(projection.hash, + maintenance_seen_message_.reticulum_lxmf_hash, + sizeof(projection.hash)); + const std::size_t slot_size = storage_v2::reticulumSeenSlotSize(); + if (!storage_v2::encodeReticulumSeenSlot(projection, + maintenance_scratch_.data(), + slot_size) || + !journal_.append(maintenance_path_, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size, + maintenance_scratch_.data())) + { + return ReconcileStepResult::Failed; + } + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return ReconcileStepResult::InProgress; + } + if (seen_hot_.size() == kSeenHotCapacity) + { + seen_hot_.erase(seen_hot_.begin()); + } + seen_hot_.push_back(projection); + } + ++maintenance_seen_message_ordinal_; + return ReconcileStepResult::InProgress; +} + +SdStore::ReconcileStepResult SdStore::stepProtocolCatalogReconcile( + MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + if (!maintenance_directory_open_) + { + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return ReconcileStepResult::InProgress; + } + for (storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (sameProtocol(projection.conversation.protocol, protocol)) + { + projection.deleted = true; + } + } + std::snprintf(maintenance_path_, + sizeof(maintenance_path_), + "%s/conversations", + protocolRoot(protocol)); + } + if (!maintenance_directory_.open(maintenance_path_)) + { + return ReconcileStepResult::Failed; + } + maintenance_directory_open_ = true; + } + + if (maintenance_reconcile_conversation_active_) + { + const ReconcileStepResult result = + stepConversationDirectoryReconcile( + protocol, + maintenance_reconcile_name_); + if (result == ReconcileStepResult::Complete) + { + maintenance_reconcile_conversation_active_ = false; + return ReconcileStepResult::InProgress; + } + if (result == ReconcileStepResult::Failed) + { + maintenance_reconcile_conversation_active_ = false; + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (state_lock.locked()) + { + projection_dirty_[protocolIndex(protocol)] = true; + } + return ReconcileStepResult::InProgress; + } + return result; + } + + bool is_directory = false; + if (!maintenance_directory_.read_next(maintenance_reconcile_name_, + sizeof(maintenance_reconcile_name_), + &is_directory)) + { + maintenance_directory_.close(); + maintenance_directory_open_ = false; + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return ReconcileStepResult::InProgress; + } + catalog_.erase(std::remove_if(catalog_.begin(), + catalog_.end(), + [&](const auto& value) + { + return sameProtocol( + value.conversation.protocol, + protocol) && + value.deleted; + }), + catalog_.end()); + return ReconcileStepResult::Complete; + } + if (is_directory && maintenance_reconcile_name_[0] != '\0') + { + std::snprintf(maintenance_reconcile_directory_path_, + sizeof(maintenance_reconcile_directory_path_), + "%s/conversations/%s", + protocolRoot(protocol), + maintenance_reconcile_name_); + maintenance_reconcile_conversation_active_ = true; + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ScanSegments; + maintenance_reconcile_projection_ = {}; + maintenance_reconcile_segment_ = 0U; + maintenance_reconcile_total_count_ = 0U; + maintenance_reconcile_last_segment_ = 0U; + maintenance_reconcile_last_segment_count_ = 0U; + maintenance_reconcile_unread_ordinal_ = 0U; + maintenance_reconcile_unread_count_ = 0U; + maintenance_reconcile_found_segment_ = false; + maintenance_reconcile_catalog_current_ = false; + } + return ReconcileStepResult::InProgress; +} + +platform::esp::common::storage::StorageOperationResult +SdStore::beginMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation) +{ + if (operation != storage_contracts::StorageOperation::Hydrate && + operation != storage_contracts::StorageOperation::Compact) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::Cancelled, + operation, + generation); + } + if (operation == storage_contracts::StorageOperation::Hydrate && + ready_.load(std::memory_order_acquire)) + { + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + // A composite adapter may revisit this store after a later store in the + // same generation asked the owner to retry. Do not reset a completed + // first stage, especially when the operation is Compaction. + if (maintenance_.operation == operation && + maintenance_.generation == generation && + maintenance_.phase == MaintenancePhase::Complete) + { + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + + // The persistence lease is the store's logical maintenance ownership, not + // the physical SD/SPI transaction lease. Keep that ownership across a + // retry so foreground persistence cannot interleave between generations. + if (maintenance_.operation == operation && + maintenance_.generation == generation && + maintenance_.phase != MaintenancePhase::Idle && + maintenance_.phase != MaintenancePhase::Complete && + maintenance_.phase != MaintenancePhase::Failed) + { + if (!maintenance_persistence_locked_ && + !acquirePersistenceLease(kPersistenceLeaseWaitTicks)) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + maintenance_persistence_locked_ = true; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(true, std::memory_order_release); + } + return storage_contracts::StorageOperationResult::inProgressResult( + operation, + generation); + } + + if (!acquirePersistenceLease(kPersistenceLeaseWaitTicks)) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + if (maintenance_.phase != MaintenancePhase::Idle && + maintenance_.phase != MaintenancePhase::Complete && + maintenance_.phase != MaintenancePhase::Failed) + { + releasePersistenceLease(); + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StateBusy, + operation, + generation); + } + + resetCatalogReconcileCursor(); + maintenance_seen_rebuild_started_ = false; + maintenance_ = {}; + maintenance_.operation = operation; + maintenance_.generation = generation; + maintenance_.phase = operation == storage_contracts::StorageOperation::Hydrate + ? MaintenancePhase::HydrationPrepare + : MaintenancePhase::CompactionPrepare; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(true, std::memory_order_release); + } + maintenance_persistence_locked_ = true; + return storage_contracts::StorageOperationResult::inProgressResult( + operation, + generation); +} + +platform::esp::common::storage::StorageOperationResult +SdStore::stepMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + if (maintenance_.operation != operation || + maintenance_.generation != generation) + { + return storage_contracts::StorageOperationResult::failure( + storage_contracts::StorageOperationResultKind::StaleGeneration, + operation, + generation); + } + if (maintenance_.phase == MaintenancePhase::Failed) + { + releaseMaintenanceLease(); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + if (maintenance_.phase == MaintenancePhase::Complete) + { + releaseMaintenanceLease(); + return storage_contracts::StorageOperationResult::completedResult( + operation, + generation); + } + + const auto result = operation == storage_contracts::StorageOperation::Hydrate + ? stepHydration(budget) + : operation == storage_contracts::StorageOperation::Compact + ? stepCompaction(budget) + : maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + Cancelled); + if (result.completed() || + result.kind == storage_contracts::StorageOperationResultKind::Cancelled || + maintenance_.phase == MaintenancePhase::Failed) + { + releaseMaintenanceLease(); + } + return result; +} + +void SdStore::cancelMaintenance( + platform::esp::common::storage::StorageOperation operation, + platform::esp::common::storage::StorageOperationGeneration generation) +{ + if (maintenance_.operation != operation || + maintenance_.generation != generation) + { + return; + } + maintenance_journal_.reset(); + resetCatalogReconcileCursor(); + maintenance_seen_rebuild_started_ = false; + maintenance_.phase = MaintenancePhase::Failed; + if (operation == storage_contracts::StorageOperation::Hydrate) + { + hydrating_.store(false, std::memory_order_release); + } + releaseMaintenanceLease(); +} + +platform::esp::common::storage::StorageOperationResult +SdStore::maintenanceFailure( + platform::esp::common::storage::StorageOperationResultKind kind) const +{ + return storage_contracts::StorageOperationResult::failure( + kind, + maintenance_.operation, + maintenance_.generation); +} + +bool SdStore::prepareMaintenanceJournal() +{ + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const uint8_t index = maintenance_.journal_index; + const char* name = nullptr; + storage_v2::JournalKind kind = storage_v2::JournalKind::MessageSegment; + std::size_t slot_size = 0U; + + switch (index) + { + case 0U: + name = "catalog.snapshot"; + kind = storage_v2::JournalKind::CatalogSnapshot; + slot_size = storage_v2::catalogSlotSize(protocol); + break; + case 1U: + name = "catalog.delta"; + kind = storage_v2::JournalKind::CatalogDelta; + slot_size = storage_v2::catalogSlotSize(protocol); + break; + case 2U: + name = "read.snapshot"; + kind = storage_v2::JournalKind::ReadStateSnapshot; + slot_size = storage_v2::readStateSlotSize(protocol); + break; + case 3U: + name = "read.delta"; + kind = storage_v2::JournalKind::ReadStateDelta; + slot_size = storage_v2::readStateSlotSize(protocol); + break; + case 4U: + name = "status.snapshot"; + kind = storage_v2::JournalKind::StatusSnapshot; + slot_size = storage_v2::statusSlotSize(); + break; + case 5U: + name = "status.delta"; + kind = storage_v2::JournalKind::StatusDelta; + slot_size = storage_v2::statusSlotSize(); + break; + default: + return false; + } + + buildProjectionPath(protocol, + name, + maintenance_path_, + sizeof(maintenance_path_)); + maintenance_protocol_ = protocol; + maintenance_kind_ = kind; + maintenance_slot_size_ = slot_size; + maintenance_.journal_started = maintenance_journal_.begin( + journal_, + maintenance_path_, + protocol, + kind, + slot_size); + return maintenance_.journal_started; +} + +bool SdStore::applyHydrationJournalSlot(MeshProtocol protocol, + storage_v2::JournalKind kind) +{ + if (maintenance_slot_size_ > maintenance_scratch_.size()) + { + return false; + } + + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return false; + } + + if (kind == storage_v2::JournalKind::CatalogSnapshot || + kind == storage_v2::JournalKind::CatalogDelta) + { + storage_v2::ChatCatalogProjection projection{}; + if (!storage_v2::decodeCatalogSlot(protocol, + maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + projection_dirty_[protocolIndex(protocol)] = true; + return true; + } + storage_v2::ChatCatalogProjection* existing = + findCatalog(projection.conversation); + if (projection.deleted) + { + if (existing) + { + catalog_.erase(catalog_.begin() + + static_cast(existing - + catalog_.data())); + } + } + else if (existing) + { + *existing = projection; + } + else + { + catalog_.push_back(projection); + } + return true; + } + + if (kind == storage_v2::JournalKind::ReadStateSnapshot || + kind == storage_v2::JournalKind::ReadStateDelta) + { + storage_v2::ChatReadProjection projection{}; + if (!storage_v2::decodeReadStateSlot(protocol, + maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + projection_dirty_[protocolIndex(protocol)] = true; + return true; + } + storage_v2::ChatReadProjection* existing = + findReadState(projection.conversation); + if (projection.deleted) + { + if (existing) + { + read_state_.erase( + read_state_.begin() + + static_cast(existing - read_state_.data())); + } + } + else if (existing) + { + *existing = projection; + } + else + { + read_state_.push_back(projection); + } + return true; + } + + storage_v2::ChatStatusProjection projection{}; + if (!storage_v2::decodeStatusSlot(maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + projection_dirty_[protocolIndex(protocol)] = true; + return true; + } + storage_v2::ChatStatusProjection* existing = + findStatus(projection.message_id, protocol); + if (existing) + { + *existing = projection; + } + else + { + ProtocolStatusProjection state{}; + state.protocol = normalizeProtocol(protocol); + state.value = projection; + statuses_.push_back(state); + } + return true; +} + +bool SdStore::advanceHydrationJournal() +{ + maintenance_journal_.reset(); + maintenance_.journal_started = false; + ++maintenance_.journal_index; + if (maintenance_.journal_index < kHydrationJournalCount) + { + return true; + } + maintenance_.journal_index = 0U; + ++maintenance_.protocol_index; + if (maintenance_.protocol_index < + static_cast(sizeof(kProtocols) / sizeof(kProtocols[0]))) + { + maintenance_.phase = MaintenancePhase::HydrationRecover; + maintenance_.recovery_index = 0U; + } + else + { + maintenance_.phase = MaintenancePhase::HydrationSeen; + maintenance_.protocol_index = 0U; + maintenance_.recovery_index = 0U; + } + return true; +} + +bool SdStore::recoverHydrationSnapshot() +{ + const char* base_name = + hydrationRecoveryName(maintenance_.recovery_index); + if (!base_name) + { + maintenance_.phase = MaintenancePhase::HydrationJournal; + return true; + } + const bool ok = recoverProjectionSnapshot( + kProtocols[maintenance_.protocol_index], + base_name); + ++maintenance_.recovery_index; + if (!ok) + { + return false; + } + if (maintenance_.recovery_index >= kHydrationRecoveryCount) + { + maintenance_.phase = MaintenancePhase::HydrationJournal; + } + return true; +} + +platform::esp::common::storage::StorageOperationResult +SdStore::stepHydration( + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + const uint8_t work_items = std::max(1U, budget.max_work_items); + for (uint8_t work = 0U; work < work_items; ++work) + { + switch (maintenance_.phase) + { + case MaintenancePhase::HydrationPrepare: + if (!storage_runtime::sd_card_ready() || !ensureLayout() || + !resetHydrationState()) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + DeviceUnavailable); + } + maintenance_.phase = MaintenancePhase::HydrationRecover; + break; + + case MaintenancePhase::HydrationRecover: + if (!recoverHydrationSnapshot()) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + break; + + case MaintenancePhase::HydrationJournal: + if (!maintenance_.journal_started && + !prepareMaintenanceJournal()) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + { + const auto status = maintenance_journal_.next( + journal_, + maintenance_scratch_.data(), + maintenance_scratch_.size()); + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Item) + { + if (!applyHydrationJournalSlot(maintenance_protocol_, + maintenance_kind_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + StateBusy); + } + break; + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Unavailable) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Invalid) + { + { + storage_runtime::ScopedRecursiveStateLock state_lock( + mutex_); + if (!state_lock.locked()) + { + return maintenanceFailure( + stateLockFailure(state_lock.result())); + } + projection_dirty_[protocolIndex(maintenance_protocol_)] = + true; + } + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Missing || + status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Complete || + status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Invalid) + { + if (!advanceHydrationJournal()) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + IoError); + } + } + } + break; + + case MaintenancePhase::HydrationSeen: + if (maintenance_.recovery_index == 0U) + { + if (!recoverProjectionSnapshot(MeshProtocol::Reticulum, + "seen")) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + IoError); + } + maintenance_.recovery_index = 1U; + break; + } + if (!maintenance_.journal_started) + { + if (maintenance_.journal_index >= 2U) + { + if (!maintenance_.seen_journal_found) + { + storage_runtime::ScopedRecursiveStateLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + for (const auto& projection : catalog_) + { + if (!projection.deleted && + sameProtocol( + projection.conversation.protocol, + MeshProtocol::Reticulum) && + projection.message_count > 0U) + { + maintenance_.seen_rebuild_required = true; + break; + } + } + } + maintenance_.protocol_index = 0U; + maintenance_.phase = + maintenance_.seen_rebuild_required + ? MaintenancePhase::HydrationRebuildSeen + : MaintenancePhase::HydrationReconcile; + break; + } + const char* name = maintenance_.journal_index == 0U + ? "seen.snapshot" + : "seen.delta"; + buildProjectionPath(MeshProtocol::Reticulum, + name, + maintenance_path_, + sizeof(maintenance_path_)); + maintenance_protocol_ = MeshProtocol::Reticulum; + maintenance_kind_ = storage_v2::JournalKind::ReticulumSeen; + maintenance_slot_size_ = + storage_v2::reticulumSeenSlotSize(); + if (!maintenance_journal_.begin( + journal_, + maintenance_path_, + maintenance_protocol_, + maintenance_kind_, + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + IoError); + } + const auto& inspection = maintenance_journal_.inspection(); + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::IoError) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + maintenance_.seen_journal_found |= + inspection.state != + storage_v2::FixedSlotJournalEngine::State::Missing; + maintenance_.seen_rebuild_required |= + inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail; + const uint32_t start = + inspection.slot_count > kSeenHotCapacity + ? inspection.slot_count - + static_cast(kSeenHotCapacity) + : 0U; + (void)maintenance_journal_.seek(start); + maintenance_.journal_started = true; + } + { + const auto status = maintenance_journal_.next( + journal_, + maintenance_scratch_.data(), + maintenance_scratch_.size()); + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Item) + { + storage_v2::ReticulumSeenProjection projection{}; + if (!storage_v2::decodeReticulumSeenSlot( + maintenance_scratch_.data(), + maintenance_slot_size_, + projection)) + { + maintenance_.seen_rebuild_required = true; + break; + } + storage_runtime::ScopedRecursiveStateLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + if (seen_hot_.size() == kSeenHotCapacity) + { + seen_hot_.erase(seen_hot_.begin()); + } + seen_hot_.push_back(projection); + break; + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Unavailable) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + if (status == storage_v2::FixedSlotJournalCursor::StepStatus:: + Invalid) + { + maintenance_.seen_rebuild_required = true; + } + maintenance_journal_.reset(); + maintenance_.journal_started = false; + ++maintenance_.journal_index; + } + break; + + case MaintenancePhase::HydrationRebuildSeen: + { + const ReconcileStepResult result = stepSeenRebuild(); + if (result == ReconcileStepResult::Failed) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + if (result == ReconcileStepResult::Complete) + { + maintenance_.protocol_index = 0U; + maintenance_.phase = MaintenancePhase::HydrationReconcile; + } + break; + } + + case MaintenancePhase::HydrationReconcile: + { + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + ready_.store(true, std::memory_order_release); + maintenance_compaction_requested_.store( + projection_dirty_[0] || projection_dirty_[1] || + projection_dirty_[2], + std::memory_order_release); + hydrating_.store(false, std::memory_order_release); + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult:: + completedResult(maintenance_.operation, + maintenance_.generation); + } + const ReconcileStepResult result = + stepProtocolCatalogReconcile( + kProtocols[maintenance_.protocol_index]); + if (result == ReconcileStepResult::Failed) + { + maintenance_.phase = MaintenancePhase::Failed; + hydrating_.store(false, std::memory_order_release); + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + if (result == ReconcileStepResult::Complete) + { + ++maintenance_.protocol_index; + } + break; + } + + case MaintenancePhase::Complete: + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + + case MaintenancePhase::Failed: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + + default: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + } + return storage_contracts::StorageOperationResult::inProgressResult( + maintenance_.operation, + maintenance_.generation); +} + +platform::esp::common::storage::StorageOperationResult +SdStore::stepCompaction( + const platform::esp::common::storage::StorageOperationBudget& budget) +{ + if (!ready_.load(std::memory_order_acquire)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + + const uint8_t work_items = std::max(1U, budget.max_work_items); + for (uint8_t work = 0U; work < work_items; ++work) + { + switch (maintenance_.phase) + { + case MaintenancePhase::CompactionPrepare: + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return maintenanceFailure( + stateLockFailure(state_lock.result())); + } + compaction_catalog_ = catalog_; + compaction_read_state_ = read_state_; + compaction_statuses_ = statuses_; + maintenance_.protocol_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_record_index = 0U; + maintenance_.compact_catalog = false; + maintenance_.compact_read = false; + maintenance_.compact_status = false; + maintenance_.phase = MaintenancePhase::CompactionInspect; + break; + } + + case MaintenancePhase::CompactionInspect: + { + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + maintenance_compaction_requested_.store( + false, + std::memory_order_release); + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult:: + completedResult(maintenance_.operation, + maintenance_.generation); + } + + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const uint8_t index = maintenance_.compaction_inspection_index; + const char* name = nullptr; + storage_v2::JournalKind kind = + storage_v2::JournalKind::CatalogDelta; + std::size_t slot_size = 0U; + switch (index) + { + case 0U: + name = "catalog.delta"; + kind = storage_v2::JournalKind::CatalogDelta; + slot_size = storage_v2::catalogSlotSize(protocol); + break; + case 1U: + name = "read.delta"; + kind = storage_v2::JournalKind::ReadStateDelta; + slot_size = storage_v2::readStateSlotSize(protocol); + break; + case 2U: + name = "status.delta"; + kind = storage_v2::JournalKind::StatusDelta; + slot_size = storage_v2::statusSlotSize(); + break; + default: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + char path[128] = {}; + buildProjectionPath(protocol, name, path, sizeof(path)); + const auto inspection = + journal_.inspect(path, protocol, kind, slot_size); + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::IoError) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind:: + RetryLater); + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::Incompatible) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + bool dirty_catalog = false; + if (index == 0U) + { + storage_runtime::ScopedRecursiveStateLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + dirty_catalog = projection_dirty_[protocolIndex(protocol)]; + } + const bool should_compact = dirty_catalog || + inspection.slot_count >= + (index == 0U + ? kCatalogCompactThreshold + : index == 1U + ? kReadCompactThreshold + : kStatusCompactThreshold); + if (index == 0U) + { + maintenance_.compact_catalog = should_compact; + } + else if (index == 1U) + { + maintenance_.compact_read = should_compact; + } + else + { + maintenance_.compact_status = should_compact; + } + ++maintenance_.compaction_inspection_index; + if (maintenance_.compaction_inspection_index >= 3U) + { + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionCreate; + } + break; + } + + case MaintenancePhase::CompactionCreate: + { + while (maintenance_.compaction_projection_index < 3U) + { + const bool enabled = + maintenance_.compaction_projection_index == 0U + ? maintenance_.compact_catalog + : maintenance_.compaction_projection_index == 1U + ? maintenance_.compact_read + : maintenance_.compact_status; + if (enabled) + { + break; + } + ++maintenance_.compaction_projection_index; + } + if (maintenance_.compaction_projection_index >= 3U) + { + maintenance_.phase = MaintenancePhase::CompactionAdvance; + break; + } + + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + const uint8_t index = maintenance_.compaction_projection_index; + const char* base = index == 0U + ? "catalog" + : index == 1U ? "read" + : "status"; + const storage_v2::JournalKind snapshot_kind = + index == 0U + ? storage_v2::JournalKind::CatalogSnapshot + : index == 1U + ? storage_v2::JournalKind::ReadStateSnapshot + : storage_v2::JournalKind::StatusSnapshot; + maintenance_slot_size_ = + index == 0U + ? storage_v2::catalogSlotSize(protocol) + : index == 1U ? storage_v2::readStateSlotSize(protocol) + : storage_v2::statusSlotSize(); + maintenance_protocol_ = protocol; + maintenance_kind_ = snapshot_kind; + char final_name[40] = {}; + char temp_name[40] = {}; + char backup_name[40] = {}; + char delta_name[40] = {}; + std::snprintf(final_name, + sizeof(final_name), + "%s.snapshot", + base); + std::snprintf(temp_name, + sizeof(temp_name), + "%s.snapshot.tmp", + base); + std::snprintf(backup_name, + sizeof(backup_name), + "%s.snapshot.bak", + base); + std::snprintf(delta_name, + sizeof(delta_name), + "%s.delta", + base); + buildProjectionPath(protocol, + final_name, + maintenance_final_path_, + sizeof(maintenance_final_path_)); + buildProjectionPath(protocol, + temp_name, + maintenance_path_, + sizeof(maintenance_path_)); + buildProjectionPath(protocol, + backup_name, + maintenance_backup_path_, + sizeof(maintenance_backup_path_)); + buildProjectionPath(protocol, + delta_name, + maintenance_delta_path_, + sizeof(maintenance_delta_path_)); + (void)storage_runtime::sd_remove(maintenance_path_); + if (!journal_.create(maintenance_path_, + protocol, + snapshot_kind, + maintenance_slot_size_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + maintenance_.compaction_record_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionWrite; + break; + } + + case MaintenancePhase::CompactionWrite: + { + const uint8_t index = maintenance_.compaction_projection_index; + const MeshProtocol protocol = + kProtocols[maintenance_.protocol_index]; + bool has_record = false; + while (true) + { + if (index == 0U) + { + if (maintenance_.compaction_record_index >= + compaction_catalog_.size()) + { + break; + } + const auto& projection = + compaction_catalog_[maintenance_.compaction_record_index++]; + if (!sameProtocol(projection.conversation.protocol, + protocol) || + projection.deleted) + { + continue; + } + has_record = + storage_v2::encodeCatalogSlot(protocol, + projection, + maintenance_scratch_.data(), + maintenance_slot_size_); + } + else if (index == 1U) + { + if (maintenance_.compaction_record_index >= + compaction_read_state_.size()) + { + break; + } + const auto& projection = + compaction_read_state_[maintenance_.compaction_record_index++]; + if (!sameProtocol(projection.conversation.protocol, + protocol) || + projection.deleted) + { + continue; + } + has_record = + storage_v2::encodeReadStateSlot(protocol, + projection, + maintenance_scratch_.data(), + maintenance_slot_size_); + } + else + { + if (maintenance_.compaction_record_index >= + compaction_statuses_.size()) + { + break; + } + const auto& state = + compaction_statuses_[maintenance_.compaction_record_index++]; + if (!sameProtocol(state.protocol, protocol)) + { + continue; + } + has_record = + storage_v2::encodeStatusSlot( + state.value, + maintenance_scratch_.data(), + maintenance_slot_size_); + } + if (!has_record || + !journal_.append(maintenance_path_, + protocol, + maintenance_kind_, + maintenance_slot_size_, + maintenance_scratch_.data())) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + return storage_contracts::StorageOperationResult:: + inProgressResult(maintenance_.operation, + maintenance_.generation); + } + maintenance_.phase = MaintenancePhase::CompactionReplace; + break; + } + + case MaintenancePhase::CompactionReplace: + if (!storage_v2::replaceFileAtomically( + maintenance_path_, + maintenance_final_path_, + maintenance_backup_path_)) + { + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::IoError); + } + maintenance_.phase = MaintenancePhase::CompactionRemove; + break; + + case MaintenancePhase::CompactionRemove: + (void)storage_runtime::sd_remove(maintenance_delta_path_); + if (maintenance_.compaction_projection_index == 0U) + { + storage_runtime::ScopedRecursiveStateLock lock(mutex_); + if (!lock.locked()) + { + return maintenanceFailure( + stateLockFailure(lock.result())); + } + projection_dirty_[protocolIndex(maintenance_protocol_)] = + false; + } + ++maintenance_.compaction_projection_index; + maintenance_.phase = MaintenancePhase::CompactionCreate; + break; + + case MaintenancePhase::CompactionAdvance: + ++maintenance_.protocol_index; + if (maintenance_.protocol_index >= + static_cast(sizeof(kProtocols) / + sizeof(kProtocols[0]))) + { + compaction_catalog_.clear(); + compaction_read_state_.clear(); + compaction_statuses_.clear(); + maintenance_compaction_requested_.store( + false, + std::memory_order_release); + maintenance_.phase = MaintenancePhase::Complete; + return storage_contracts::StorageOperationResult:: + completedResult(maintenance_.operation, + maintenance_.generation); + } + maintenance_.compaction_inspection_index = 0U; + maintenance_.compaction_projection_index = 0U; + maintenance_.phase = MaintenancePhase::CompactionInspect; + break; + + case MaintenancePhase::Complete: + return storage_contracts::StorageOperationResult::completedResult( + maintenance_.operation, + maintenance_.generation); + + default: + return maintenanceFailure( + storage_contracts::StorageOperationResultKind::StateBusy); + } + } + return storage_contracts::StorageOperationResult::inProgressResult( + maintenance_.operation, + maintenance_.generation); } void SdStore::append(const ChatMessage& msg) @@ -263,8 +1653,13 @@ bool SdStore::appendInternal(const ChatMessage& input, bool incoming_commit) { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) + { + return false; + } + if (!ready_.load(std::memory_order_acquire)) { return false; } @@ -310,44 +1705,78 @@ bool SdStore::appendInternal(const ChatMessage& input, bool incoming_commit) return false; } - storage_v2::ChatCatalogProjection* projection = findCatalog(conversation); - const bool new_projection = projection == nullptr; - if (!projection) + storage_v2::ChatCatalogProjection projection_snapshot{}; + uint32_t last_read_sequence = 0U; + bool projection_was_current = false; { - storage_v2::ChatCatalogProjection created{}; - created.conversation = conversation; - catalog_.push_back(created); - projection = &catalog_.back(); - } - const bool projection_was_current = - projection->message_count == stored_count && - projection->last_message_id == message.msg_id; - if (!projection_was_current || new_projection) - { - projection->conversation = conversation; - projection->message_count = stored_count; - projection->last_sequence = stored_count; - projection->last_message_id = message.msg_id; - projection->last_timestamp = message.timestamp; - projection->last_status = message.status; - projection->deleted = false; - const storage_v2::ChatReadProjection* read_state = - findReadState(conversation); - projection->unread = countUnreadAfter( - conversation, - read_state ? read_state->last_read_sequence : 0U); - copyTextPreview(projection->preview, - sizeof(projection->preview), - message.text); - if (!appendCatalogProjection(*projection)) + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) { - projection_dirty_[protocolIndex(message.protocol)] = true; - CHAT_STORE_LOG("[ChatStoreV2] projection deferred protocol=%s msg=%08lX authoritative=1\n", - protocolSlug(message.protocol), - static_cast(message.msg_id)); + return false; + } + if (const storage_v2::ChatCatalogProjection* projection = + findCatalog(conversation)) + { + projection_snapshot = *projection; + projection_was_current = + projection->message_count == stored_count && + projection->last_message_id == message.msg_id; + } + else + { + projection_snapshot.conversation = conversation; + } + if (const storage_v2::ChatReadProjection* read_state = + findReadState(conversation)) + { + last_read_sequence = read_state->last_read_sequence; } } + bool projection_persisted = true; + if (!projection_was_current) + { + projection_snapshot.conversation = conversation; + projection_snapshot.message_count = stored_count; + projection_snapshot.last_sequence = stored_count; + projection_snapshot.last_message_id = message.msg_id; + projection_snapshot.last_timestamp = message.timestamp; + projection_snapshot.last_status = message.status; + projection_snapshot.deleted = false; + projection_snapshot.unread = + countUnreadAfter(conversation, last_read_sequence); + copyTextPreview(projection_snapshot.preview, + sizeof(projection_snapshot.preview), + message.text); + projection_persisted = + appendCatalogProjection(projection_snapshot); + + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return false; + } + if (storage_v2::ChatCatalogProjection* projection = + findCatalog(conversation)) + { + *projection = projection_snapshot; + } + else + { + catalog_.push_back(projection_snapshot); + } + if (!projection_persisted) + { + projection_dirty_[protocolIndex(message.protocol)] = true; + } + } + + if (!projection_persisted) + { + CHAT_STORE_LOG("[ChatStoreV2] projection deferred protocol=%s msg=%08lX authoritative=1\n", + protocolSlug(message.protocol), + static_cast(message.msg_id)); + } CHAT_STORE_LOG("[ChatStoreV2] commit protocol=%s msg=%08lX seq=%lu duplicate=%u publish=1\n", protocolSlug(message.protocol), static_cast(message.msg_id), @@ -380,8 +1809,10 @@ std::vector SdStore::loadPageFromLatest( } return {}; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked() || + !ready_.load(std::memory_order_acquire)) { if (total) { @@ -533,18 +1964,34 @@ bool SdStore::setUnread(const ConversationId& input, int unread) { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) + { + return false; + } + if (!ready_.load(std::memory_order_acquire)) { return false; } ConversationId conversation = input; conversation.protocol = normalizeProtocol(conversation.protocol); - storage_v2::ChatCatalogProjection* catalog = findCatalog(conversation); - if (!catalog) + storage_v2::ChatCatalogProjection catalog_snapshot{}; { - return unread == 0; + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return false; + } + const storage_v2::ChatCatalogProjection* catalog = + findCatalog(conversation); + if (!catalog) + { + return unread == 0; + } + catalog_snapshot = *catalog; } + const uint32_t bounded_unread = unread <= 0 ? 0U : static_cast(unread); storage_v2::ChatReadProjection projection{}; @@ -555,8 +2002,19 @@ bool SdStore::setUnread(const ConversationId& input, int unread) { return false; } - storage_v2::ChatReadProjection* current = findReadState(conversation); - if (current) + catalog_snapshot.unread = + countUnreadAfter(conversation, + projection.last_read_sequence); + const bool catalog_persisted = + appendCatalogProjection(catalog_snapshot); + + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return false; + } + if (storage_v2::ChatReadProjection* current = + findReadState(conversation)) { *current = projection; } @@ -564,9 +2022,16 @@ bool SdStore::setUnread(const ConversationId& input, int unread) { read_state_.push_back(projection); } - catalog->unread = countUnreadAfter(conversation, - projection.last_read_sequence); - if (!appendCatalogProjection(*catalog)) + if (storage_v2::ChatCatalogProjection* catalog = + findCatalog(conversation)) + { + *catalog = catalog_snapshot; + } + else + { + catalog_.push_back(catalog_snapshot); + } + if (!catalog_persisted) { projection_dirty_[protocolIndex(conversation.protocol)] = true; } @@ -597,13 +2062,32 @@ void SdStore::clearConversation(const ConversationId& input) { return; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) + { + return; + } + if (!ready_.load(std::memory_order_acquire)) { return; } ConversationId conversation = input; conversation.protocol = normalizeProtocol(conversation.protocol); + storage_v2::ChatCatalogProjection tombstone{}; + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return; + } + if (const storage_v2::ChatCatalogProjection* existing = + findCatalog(conversation)) + { + tombstone = *existing; + } + } + char path[128]{}; buildConversationDirectory(conversation, path, sizeof(path)); if (!removeTree(path)) @@ -611,21 +2095,22 @@ void SdStore::clearConversation(const ConversationId& input) return; } - storage_v2::ChatCatalogProjection tombstone{}; - if (const storage_v2::ChatCatalogProjection* existing = - findCatalog(conversation)) - { - tombstone = *existing; - } tombstone.conversation = conversation; tombstone.deleted = true; - (void)appendCatalogProjection(tombstone); + const bool catalog_persisted = + appendCatalogProjection(tombstone); storage_v2::ChatReadProjection read_tombstone{}; read_tombstone.conversation = conversation; read_tombstone.deleted = true; - (void)appendReadProjection(read_tombstone); + const bool read_persisted = + appendReadProjection(read_tombstone); + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return; + } catalog_.erase(std::remove_if(catalog_.begin(), catalog_.end(), [&](const auto& value) @@ -644,6 +2129,10 @@ void SdStore::clearConversation(const ConversationId& input) conversation); }), read_state_.end()); + if (!catalog_persisted || !read_persisted) + { + projection_dirty_[protocolIndex(conversation.protocol)] = true; + } } void SdStore::clearAll() @@ -652,8 +2141,9 @@ void SdStore::clearAll() { return; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked()) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) { return; } @@ -661,12 +2151,22 @@ void SdStore::clearAll() { (void)removeTree(protocolRoot(protocol)); } + const bool layout_ready = ensureLayout(); + + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return; + } catalog_.clear(); read_state_.clear(); statuses_.clear(); seen_hot_.clear(); std::memset(projection_dirty_, 0, sizeof(projection_dirty_)); - ready_ = ensureLayout(); + ready_.store(layout_ready, std::memory_order_release); + maintenance_compaction_requested_.store( + false, + std::memory_order_release); } bool SdStore::updateMessageStatus(MessageId msg_id, MessageStatus status) @@ -675,6 +2175,12 @@ bool SdStore::updateMessageStatus(MessageId msg_id, MessageStatus status) { return false; } + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) + { + return false; + } ChatMessage message{}; if (!getMessage(msg_id, &message)) { @@ -691,8 +2197,13 @@ bool SdStore::updateMessageStatusForProtocol(MessageId msg_id, { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked()) + { + return false; + } + if (!ready_.load(std::memory_order_acquire)) { return false; } @@ -711,21 +2222,49 @@ bool SdStore::updateMessageStatusForProtocol(MessageId msg_id, storage_v2::ChatStatusProjection projection{}; projection.message_id = msg_id; projection.status = status; - if (const storage_v2::ChatStatusProjection* current = - findStatus(msg_id, protocol)) + storage_v2::ChatCatalogProjection catalog_snapshot{}; + bool update_catalog = false; { - projection.sequence = current->sequence + 1U; - } - else - { - projection.sequence = 1U; + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return false; + } + if (const storage_v2::ChatStatusProjection* current = + findStatus(msg_id, protocol)) + { + projection.sequence = current->sequence + 1U; + } + else + { + projection.sequence = 1U; + } + const ConversationId conversation = + conversationIdForMessage(message); + if (const storage_v2::ChatCatalogProjection* catalog = + findCatalog(conversation); + catalog && catalog->last_message_id == msg_id) + { + catalog_snapshot = *catalog; + catalog_snapshot.last_status = status; + update_catalog = true; + } } + if (!appendStatusProjection(protocol, projection)) { return false; } - storage_v2::ChatStatusProjection* current = findStatus(msg_id, protocol); - if (current) + const bool catalog_persisted = + !update_catalog || appendCatalogProjection(catalog_snapshot); + + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) + { + return false; + } + if (storage_v2::ChatStatusProjection* current = + findStatus(msg_id, protocol)) { *current = projection; } @@ -736,13 +2275,14 @@ bool SdStore::updateMessageStatusForProtocol(MessageId msg_id, state.value = projection; statuses_.push_back(state); } - - const ConversationId conversation = conversationIdForMessage(message); - storage_v2::ChatCatalogProjection* catalog = findCatalog(conversation); - if (catalog && catalog->last_message_id == msg_id) + if (update_catalog) { - catalog->last_status = status; - if (!appendCatalogProjection(*catalog)) + if (storage_v2::ChatCatalogProjection* catalog = + findCatalog(catalog_snapshot.conversation)) + { + *catalog = catalog_snapshot; + } + if (!catalog_persisted) { projection_dirty_[protocolIndex(protocol)] = true; } @@ -756,8 +2296,10 @@ bool SdStore::getMessage(MessageId msg_id, ChatMessage* out) const { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked() || + !ready_.load(std::memory_order_acquire)) { return false; } @@ -779,8 +2321,10 @@ bool SdStore::getMessageForProtocol(MessageId msg_id, { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked() || + !ready_.load(std::memory_order_acquire)) { return false; } @@ -789,13 +2333,28 @@ bool SdStore::getMessageForProtocol(MessageId msg_id, { return false; } - for (const storage_v2::ChatCatalogProjection& projection : catalog_) + + CatalogList catalog_snapshot{}; { - if (projection.deleted || - !sameProtocol(projection.conversation.protocol, protocol)) + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) { - continue; + return false; } + catalog_snapshot.reserve(catalog_.size()); + for (const storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (!projection.deleted && + sameProtocol(projection.conversation.protocol, protocol)) + { + catalog_snapshot.push_back(projection); + } + } + } + + for (const storage_v2::ChatCatalogProjection& projection : + catalog_snapshot) + { for (uint32_t ordinal = projection.message_count; ordinal > 0U; --ordinal) { @@ -823,20 +2382,29 @@ bool SdStore::hasReticulumLxmfMessageHash(const uint8_t* hash) const { return false; } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked() || !ready_) - { - return false; - } if (!hash || isAllZeroKeyBytes(hash, kReticulumLxmfHashSize)) { return false; } - for (const storage_v2::ReticulumSeenProjection& seen : seen_hot_) + ScopedPersistenceLease persistence_lease(persistence_mutex_, + kPersistenceLeaseWaitTicks); + if (!persistence_lease.locked() || + !ready_.load(std::memory_order_acquire)) { - if (std::memcmp(seen.hash, hash, sizeof(seen.hash)) == 0) + return false; + } + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked() || !ready_) { - return true; + return false; + } + for (const storage_v2::ReticulumSeenProjection& seen : seen_hot_) + { + if (std::memcmp(seen.hash, hash, sizeof(seen.hash)) == 0) + { + return true; + } } } @@ -879,41 +2447,9 @@ bool SdStore::hasReticulumLxmfMessageHash(const uint8_t* hash) const void SdStore::flush() { - if (!ready_.load(std::memory_order_acquire)) - { - return; - } - storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); - if (!state_lock.locked()) - { - return; - } - const uint32_t now_ms = monotonic_millis(); - if (last_projection_retry_ms_ != 0U && - now_ms - last_projection_retry_ms_ < kProjectionRetryIntervalMs) - { - return; - } - MeshProtocol protocol = MeshProtocol::Meshtastic; - bool found_dirty = false; - for (std::size_t offset = 0U; offset < 3U; ++offset) - { - const std::size_t index = - (static_cast(flush_protocol_cursor_) + offset) % 3U; - if (projection_dirty_[index]) - { - protocol = kProtocols[index]; - flush_protocol_cursor_ = static_cast((index + 1U) % 3U); - found_dirty = true; - break; - } - } - if (!found_dirty) - { - return; - } - last_projection_retry_ms_ = now_ms; - (void)compactProtocolProjections(protocol); + // Record appends are already durable. Projection compaction belongs to + // StorageMaintenanceOwner, so a foreground flush never starts filesystem + // work or competes with the logical state lock. } bool SdStore::ensureLayout() const @@ -953,400 +2489,6 @@ bool SdStore::ensureProtocolLayout(MeshProtocol protocol) const ensureDirectory(conversations); } -bool SdStore::loadRuntimeState() -{ - catalog_.clear(); - read_state_.clear(); - statuses_.clear(); - seen_hot_.clear(); - bool ok = true; - for (MeshProtocol protocol : kProtocols) - { - ok = loadProtocolState(protocol) && ok; - } - ok = loadSeenJournal() && ok; - return ok; -} - -bool SdStore::loadProtocolState(MeshProtocol protocol) -{ - if (!recoverProjectionSnapshot(protocol, "catalog") || - !recoverProjectionSnapshot(protocol, "read") || - !recoverProjectionSnapshot(protocol, "status")) - { - return false; - } - bool ok = true; - ok = loadCatalogJournal(protocol, "catalog.snapshot") && ok; - ok = loadCatalogJournal(protocol, "catalog.delta") && ok; - ok = loadReadJournal(protocol, "read.snapshot") && ok; - ok = loadReadJournal(protocol, "read.delta") && ok; - ok = loadStatusJournal(protocol, "status.snapshot") && ok; - ok = loadStatusJournal(protocol, "status.delta") && ok; - return reconcileProtocolCatalog(protocol) && ok; -} - -bool SdStore::loadCatalogJournal(MeshProtocol protocol, const char* name) -{ - char path[128]{}; - buildProjectionPath(protocol, name, path, sizeof(path)); - const auto inspection = journal_.inspect( - path, - protocol, - hasSuffix(name, ".snapshot") - ? storage_v2::JournalKind::CatalogSnapshot - : storage_v2::JournalKind::CatalogDelta, - storage_v2::catalogSlotSize(protocol)); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - return true; - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - projection_dirty_[protocolIndex(protocol)] = true; - return true; - } - const storage_v2::JournalKind kind = - hasSuffix(name, ".snapshot") - ? storage_v2::JournalKind::CatalogSnapshot - : storage_v2::JournalKind::CatalogDelta; - for (uint32_t index = 0; index < inspection.slot_count; ++index) - { - storage_v2::ChatCatalogProjection projection{}; - if (!journal_.read(path, - protocol, - kind, - storage_v2::catalogSlotSize(protocol), - index, - scratch_.data()) || - !storage_v2::decodeCatalogSlot(protocol, - scratch_.data(), - storage_v2::catalogSlotSize(protocol), - projection)) - { - projection_dirty_[protocolIndex(protocol)] = true; - continue; - } - storage_v2::ChatCatalogProjection* existing = - findCatalog(projection.conversation); - if (projection.deleted) - { - if (existing) - { - catalog_.erase(catalog_.begin() + - static_cast(existing - - catalog_.data())); - } - } - else if (existing) - { - *existing = projection; - } - else - { - catalog_.push_back(projection); - } - } - return true; -} - -bool SdStore::loadReadJournal(MeshProtocol protocol, const char* name) -{ - char path[128]{}; - buildProjectionPath(protocol, name, path, sizeof(path)); - const storage_v2::JournalKind kind = - hasSuffix(name, ".snapshot") - ? storage_v2::JournalKind::ReadStateSnapshot - : storage_v2::JournalKind::ReadStateDelta; - const auto inspection = journal_.inspect( - path, - protocol, - kind, - storage_v2::readStateSlotSize(protocol)); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - return true; - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - projection_dirty_[protocolIndex(protocol)] = true; - return true; - } - for (uint32_t index = 0; index < inspection.slot_count; ++index) - { - storage_v2::ChatReadProjection projection{}; - if (!journal_.read(path, - protocol, - kind, - storage_v2::readStateSlotSize(protocol), - index, - scratch_.data()) || - !storage_v2::decodeReadStateSlot( - protocol, - scratch_.data(), - storage_v2::readStateSlotSize(protocol), - projection)) - { - projection_dirty_[protocolIndex(protocol)] = true; - continue; - } - storage_v2::ChatReadProjection* existing = - findReadState(projection.conversation); - if (projection.deleted) - { - if (existing) - { - read_state_.erase( - read_state_.begin() + - static_cast(existing - read_state_.data())); - } - } - else if (existing) - { - *existing = projection; - } - else - { - read_state_.push_back(projection); - } - } - return true; -} - -bool SdStore::loadStatusJournal(MeshProtocol protocol, const char* name) -{ - char path[128]{}; - buildProjectionPath(protocol, name, path, sizeof(path)); - const storage_v2::JournalKind kind = - hasSuffix(name, ".snapshot") - ? storage_v2::JournalKind::StatusSnapshot - : storage_v2::JournalKind::StatusDelta; - const auto inspection = journal_.inspect(path, - protocol, - kind, - storage_v2::statusSlotSize()); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - return true; - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - projection_dirty_[protocolIndex(protocol)] = true; - return true; - } - for (uint32_t index = 0; index < inspection.slot_count; ++index) - { - storage_v2::ChatStatusProjection projection{}; - if (!journal_.read(path, - protocol, - kind, - storage_v2::statusSlotSize(), - index, - scratch_.data()) || - !storage_v2::decodeStatusSlot(scratch_.data(), - storage_v2::statusSlotSize(), - projection)) - { - projection_dirty_[protocolIndex(protocol)] = true; - continue; - } - storage_v2::ChatStatusProjection* existing = - findStatus(projection.message_id, protocol); - if (existing) - { - *existing = projection; - } - else - { - ProtocolStatusProjection state{}; - state.protocol = normalizeProtocol(protocol); - state.value = projection; - statuses_.push_back(state); - } - } - return true; -} - -bool SdStore::loadSeenJournal() -{ - if (!recoverProjectionSnapshot(MeshProtocol::Reticulum, "seen")) - { - return false; - } - bool journal_found = false; - bool rebuild_required = false; - for (const char* name : {"seen.snapshot", "seen.delta"}) - { - char path[128]{}; - buildProjectionPath(MeshProtocol::Reticulum, - name, - path, - sizeof(path)); - const auto inspection = journal_.inspect( - path, - MeshProtocol::Reticulum, - storage_v2::JournalKind::ReticulumSeen, - storage_v2::reticulumSeenSlotSize()); - if (inspection.state == - storage_v2::FixedSlotJournalEngine::State::Missing) - { - continue; - } - journal_found = true; - if (inspection.state != - storage_v2::FixedSlotJournalEngine::State::Ready && - inspection.state != - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - rebuild_required = true; - continue; - } - if (inspection.state == - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - rebuild_required = true; - } - const uint32_t start = - inspection.slot_count > kSeenHotCapacity - ? inspection.slot_count - static_cast(kSeenHotCapacity) - : 0U; - for (uint32_t index = start; index < inspection.slot_count; ++index) - { - storage_v2::ReticulumSeenProjection projection{}; - if (!journal_.read(path, - MeshProtocol::Reticulum, - storage_v2::JournalKind::ReticulumSeen, - storage_v2::reticulumSeenSlotSize(), - index, - scratch_.data()) || - !storage_v2::decodeReticulumSeenSlot( - scratch_.data(), - storage_v2::reticulumSeenSlotSize(), - projection)) - { - rebuild_required = true; - continue; - } - if (seen_hot_.size() == kSeenHotCapacity) - { - seen_hot_.erase(seen_hot_.begin()); - } - seen_hot_.push_back(projection); - } - } - if (!journal_found) - { - for (const storage_v2::ChatCatalogProjection& projection : catalog_) - { - if (!projection.deleted && - sameProtocol(projection.conversation.protocol, - MeshProtocol::Reticulum) && - messageCountOnDisk(projection.conversation) > 0U) - { - rebuild_required = true; - break; - } - } - } - return !rebuild_required || rebuildSeenJournalFromMessages(); -} - -bool SdStore::rebuildSeenJournalFromMessages() -{ - char final_path[128] = {}; - char temp_path[128] = {}; - char backup_path[128] = {}; - char delta_path[128] = {}; - buildProjectionPath(MeshProtocol::Reticulum, - "seen.snapshot", - final_path, - sizeof(final_path)); - buildProjectionPath(MeshProtocol::Reticulum, - "seen.snapshot.tmp", - temp_path, - sizeof(temp_path)); - buildProjectionPath(MeshProtocol::Reticulum, - "seen.snapshot.bak", - backup_path, - sizeof(backup_path)); - buildProjectionPath(MeshProtocol::Reticulum, - "seen.delta", - delta_path, - sizeof(delta_path)); - (void)storage_runtime::sd_remove(temp_path); - const std::size_t slot_size = storage_v2::reticulumSeenSlotSize(); - if (!journal_.create(temp_path, - MeshProtocol::Reticulum, - storage_v2::JournalKind::ReticulumSeen, - slot_size)) - { - return false; - } - - seen_hot_.clear(); - uint32_t rebuilt = 0U; - for (const storage_v2::ChatCatalogProjection& catalog : catalog_) - { - if (catalog.deleted || - !sameProtocol(catalog.conversation.protocol, - MeshProtocol::Reticulum)) - { - continue; - } - const uint32_t message_count = - messageCountOnDisk(catalog.conversation); - for (uint32_t ordinal = 0U; ordinal < message_count; ++ordinal) - { - ChatMessage message{}; - if (!readMessageByOrdinal(catalog.conversation, - ordinal, - message) || - !chat::hasReticulumLxmfMessageHash(message)) - { - continue; - } - storage_v2::ReticulumSeenProjection projection{}; - std::memcpy(projection.hash, - message.reticulum_lxmf_hash, - sizeof(projection.hash)); - if (!storage_v2::encodeReticulumSeenSlot(projection, - scratch_.data(), - slot_size) || - !journal_.append(temp_path, - MeshProtocol::Reticulum, - storage_v2::JournalKind::ReticulumSeen, - slot_size, - scratch_.data())) - { - (void)storage_runtime::sd_remove(temp_path); - return false; - } - if (seen_hot_.size() == kSeenHotCapacity) - { - seen_hot_.erase(seen_hot_.begin()); - } - seen_hot_.push_back(projection); - ++rebuilt; - } - } - if (!storage_v2::replaceFileAtomically(temp_path, - final_path, - backup_path)) - { - return false; - } - (void)storage_runtime::sd_remove(delta_path); - CHAT_STORE_LOG("[ChatStoreV2] seen rebuilt hashes=%lu authoritative=messages\n", - static_cast(rebuilt)); - return true; -} - bool SdStore::recoverProjectionSnapshot(MeshProtocol protocol, const char* base_name) { @@ -1369,174 +2511,276 @@ bool SdStore::recoverProjectionSnapshot(MeshProtocol protocol, backup_path); } -bool SdStore::reconcileProtocolCatalog(MeshProtocol protocol) +SdStore::ReconcileStepResult +SdStore::stepConversationDirectoryReconcile( + MeshProtocol protocol, + const char* directory_name) { protocol = normalizeProtocol(protocol); - for (storage_v2::ChatCatalogProjection& projection : catalog_) + if (!directory_name || directory_name[0] == '\0') { - if (sameProtocol(projection.conversation.protocol, protocol)) - { - projection.deleted = true; - } + return ReconcileStepResult::Failed; } - char conversations_path[96]{}; - std::snprintf(conversations_path, - sizeof(conversations_path), - "%s/conversations", - protocolRoot(protocol)); - storage_runtime::SdRuntimeDir directory; - if (!directory.open(conversations_path)) - { - return false; - } - char name[80]{}; - bool is_directory = false; - while (directory.read_next(name, sizeof(name), &is_directory)) - { - if (is_directory && name[0] != '\0' && - !reconcileConversationDirectory(protocol, name)) - { - projection_dirty_[protocolIndex(protocol)] = true; - } - } - catalog_.erase(std::remove_if(catalog_.begin(), - catalog_.end(), - [&](const auto& value) - { - return sameProtocol( - value.conversation.protocol, - protocol) && - value.deleted; - }), - catalog_.end()); - return true; -} - -bool SdStore::reconcileConversationDirectory(MeshProtocol protocol, - const char* directory_name) -{ - char directory_path[128]{}; - std::snprintf(directory_path, - sizeof(directory_path), - "%s/conversations/%s", - protocolRoot(protocol), - directory_name); const std::size_t slot_size = storage_v2::messageSlotSize(protocol); - uint32_t total_count = 0; - uint32_t last_segment = 0; - uint32_t last_segment_count = 0; - bool found_segment = false; - for (uint32_t segment = 0; segment < 10000U; ++segment) + if (slot_size == 0U || slot_size > scratch_.size()) + { + return ReconcileStepResult::Failed; + } + + switch (maintenance_reconcile_phase_) + { + case ConversationReconcilePhase::ScanSegments: + { + if (maintenance_reconcile_segment_ >= 10000U) + { + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ReadLatest; + return ReconcileStepResult::InProgress; + } + + char path[160]{}; + std::snprintf(path, + sizeof(path), + "%s/%04lu.msg", + maintenance_reconcile_directory_path_, + static_cast( + maintenance_reconcile_segment_)); + const auto inspection = journal_.inspect( + path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::Missing) + { + if (!maintenance_reconcile_found_segment_) + { + return ReconcileStepResult::Complete; + } + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ReadLatest; + return ReconcileStepResult::InProgress; + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + maintenance_reconcile_phase_ = + ConversationReconcilePhase::RepairSegment; + return ReconcileStepResult::InProgress; + } + if (inspection.state != + storage_v2::FixedSlotJournalEngine::State::Ready) + { + return ReconcileStepResult::Failed; + } + + maintenance_reconcile_found_segment_ = true; + maintenance_reconcile_total_count_ += inspection.slot_count; + maintenance_reconcile_last_segment_ = + maintenance_reconcile_segment_; + maintenance_reconcile_last_segment_count_ = + inspection.slot_count; + ++maintenance_reconcile_segment_; + if (inspection.slot_count < slotsPerMessageSegment(protocol)) + { + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ReadLatest; + } + return ReconcileStepResult::InProgress; + } + + case ConversationReconcilePhase::RepairSegment: { char path[160]{}; std::snprintf(path, sizeof(path), "%s/%04lu.msg", - directory_path, - static_cast(segment)); - auto inspection = journal_.inspect( - path, - protocol, - storage_v2::JournalKind::MessageSegment, - slot_size); - if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) - { - break; - } - if (inspection.state == - storage_v2::FixedSlotJournalEngine::State::PartialTail) - { - if (!repairPartialJournal(path, - protocol, - storage_v2::JournalKind::MessageSegment, - slot_size)) - { - return false; - } - inspection = journal_.inspect( + maintenance_reconcile_directory_path_, + static_cast( + maintenance_reconcile_segment_)); + if (!repairPartialJournal( path, protocol, storage_v2::JournalKind::MessageSegment, - slot_size); - } - if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready) + slot_size)) { - return false; + return ReconcileStepResult::Failed; } - found_segment = true; - total_count += inspection.slot_count; - last_segment = segment; - last_segment_count = inspection.slot_count; - if (inspection.slot_count < slotsPerMessageSegment(protocol)) - { - break; - } - } - if (!found_segment || total_count == 0U || last_segment_count == 0U) - { - return true; + maintenance_reconcile_phase_ = + ConversationReconcilePhase::ScanSegments; + return ReconcileStepResult::InProgress; } - char last_path[160]{}; - std::snprintf(last_path, - sizeof(last_path), - "%s/%04lu.msg", - directory_path, - static_cast(last_segment)); - if (!journal_.read(last_path, - protocol, - storage_v2::JournalKind::MessageSegment, - slot_size, - last_segment_count - 1U, - scratch_.data())) + case ConversationReconcilePhase::ReadLatest: { - return false; + if (!maintenance_reconcile_found_segment_ || + maintenance_reconcile_total_count_ == 0U || + maintenance_reconcile_last_segment_count_ == 0U) + { + return ReconcileStepResult::Complete; + } + + char last_path[160]{}; + std::snprintf( + last_path, + sizeof(last_path), + "%s/%04lu.msg", + maintenance_reconcile_directory_path_, + static_cast( + maintenance_reconcile_last_segment_)); + if (!journal_.read( + last_path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size, + maintenance_reconcile_last_segment_count_ - 1U, + scratch_.data())) + { + return ReconcileStepResult::Failed; + } + uint32_t sequence = 0U; + if (!storage_v2::decodeMessageSlot( + protocol, + scratch_.data(), + slot_size, + maintenance_reconcile_latest_message_, + &sequence)) + { + return ReconcileStepResult::Failed; + } + + ConversationId conversation{}; + uint32_t last_read = 0U; + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return ReconcileStepResult::InProgress; + } + ChatMessage& latest = + maintenance_reconcile_latest_message_; + applyStoredStatus(latest); + conversation = conversationIdForMessage(latest); + if (const storage_v2::ChatCatalogProjection* projection = + findCatalog(conversation)) + { + maintenance_reconcile_projection_ = *projection; + maintenance_reconcile_catalog_current_ = + projection->message_count == + maintenance_reconcile_total_count_ && + projection->last_message_id == latest.msg_id && + projection->last_sequence == sequence; + } + else + { + maintenance_reconcile_projection_ = {}; + maintenance_reconcile_catalog_current_ = false; + } + if (const storage_v2::ChatReadProjection* read_state = + findReadState(conversation)) + { + last_read = read_state->last_read_sequence; + } + } + + ChatMessage& latest = maintenance_reconcile_latest_message_; + storage_v2::ChatCatalogProjection& projection = + maintenance_reconcile_projection_; + projection.conversation = conversation; + projection.message_count = + maintenance_reconcile_total_count_; + projection.last_sequence = sequence; + projection.last_message_id = latest.msg_id; + projection.last_timestamp = latest.timestamp; + projection.last_status = latest.status; + projection.deleted = false; + copyTextPreview(projection.preview, + sizeof(projection.preview), + latest.text); + + maintenance_reconcile_unread_ordinal_ = last_read; + maintenance_reconcile_unread_count_ = 0U; + if (maintenance_reconcile_catalog_current_) + { + maintenance_reconcile_phase_ = + ConversationReconcilePhase::Commit; + } + else + { + projection.unread = 0U; + maintenance_reconcile_phase_ = + last_read < maintenance_reconcile_total_count_ + ? ConversationReconcilePhase::ScanUnread + : ConversationReconcilePhase::Commit; + } + return ReconcileStepResult::InProgress; } - ChatMessage latest{}; - uint32_t sequence = 0; - if (!storage_v2::decodeMessageSlot(protocol, - scratch_.data(), - slot_size, - latest, - &sequence)) + + case ConversationReconcilePhase::ScanUnread: { - return false; + if (maintenance_reconcile_unread_ordinal_ >= + maintenance_reconcile_total_count_) + { + maintenance_reconcile_projection_.unread = + maintenance_reconcile_unread_count_; + maintenance_reconcile_phase_ = + ConversationReconcilePhase::Commit; + return ReconcileStepResult::InProgress; + } + + uint32_t sequence = 0U; + if (!readMessageByOrdinal( + maintenance_reconcile_projection_.conversation, + maintenance_reconcile_unread_ordinal_, + maintenance_reconcile_latest_message_, + &sequence)) + { + return ReconcileStepResult::Failed; + } + if (sequence > maintenance_reconcile_unread_ordinal_ && + maintenance_reconcile_latest_message_.status == + MessageStatus::Incoming) + { + ++maintenance_reconcile_unread_count_; + } + ++maintenance_reconcile_unread_ordinal_; + if (maintenance_reconcile_unread_ordinal_ >= + maintenance_reconcile_total_count_) + { + maintenance_reconcile_projection_.unread = + maintenance_reconcile_unread_count_; + maintenance_reconcile_phase_ = + ConversationReconcilePhase::Commit; + } + return ReconcileStepResult::InProgress; } - applyStoredStatus(latest); - const ConversationId conversation = conversationIdForMessage(latest); - storage_v2::ChatCatalogProjection* projection = findCatalog(conversation); - const bool catalog_current = - projection && projection->message_count == total_count && - projection->last_message_id == latest.msg_id && - projection->last_sequence == sequence; - if (!projection) + + case ConversationReconcilePhase::Commit: { - storage_v2::ChatCatalogProjection created{}; - created.conversation = conversation; - catalog_.push_back(created); - projection = &catalog_.back(); + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return ReconcileStepResult::InProgress; + } + if (storage_v2::ChatCatalogProjection* projection = + findCatalog( + maintenance_reconcile_projection_.conversation)) + { + *projection = maintenance_reconcile_projection_; + } + else + { + catalog_.push_back(maintenance_reconcile_projection_); + } + if (!maintenance_reconcile_catalog_current_) + { + projection_dirty_[protocolIndex(protocol)] = true; + } + return ReconcileStepResult::Complete; } - const uint32_t last_read = - findReadState(conversation) - ? findReadState(conversation)->last_read_sequence - : 0U; - projection->conversation = conversation; - projection->message_count = total_count; - projection->last_sequence = sequence; - projection->last_message_id = latest.msg_id; - projection->last_timestamp = latest.timestamp; - projection->last_status = latest.status; - projection->deleted = false; - copyTextPreview(projection->preview, - sizeof(projection->preview), - latest.text); - if (!catalog_current) - { - projection->unread = countUnreadAfter(conversation, last_read); - projection_dirty_[protocolIndex(protocol)] = true; } - return true; + return ReconcileStepResult::Failed; } std::size_t SdStore::slotsPerMessageSegment(MeshProtocol protocol) const @@ -1617,7 +2861,14 @@ bool SdStore::readMessageByOrdinal(const ConversationId& input, { return false; } - applyStoredStatus(out_message); + { + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return false; + } + applyStoredStatus(out_message); + } return true; } @@ -1798,11 +3049,18 @@ bool SdStore::appendCatalogProjection( } char path[128]{}; buildProjectionPath(protocol, "catalog.delta", path, sizeof(path)); - return journal_.append(path, - protocol, - storage_v2::JournalKind::CatalogDelta, - slot_size, - scratch_.data()); + const bool ok = journal_.append(path, + protocol, + storage_v2::JournalKind::CatalogDelta, + slot_size, + scratch_.data()); + if (ok) + { + maintenance_compaction_requested_.store( + true, + std::memory_order_release); + } + return ok; } bool SdStore::appendReadProjection( @@ -1840,11 +3098,18 @@ bool SdStore::appendReadProjection( { return false; } - return journal_.append(path, - protocol, - storage_v2::JournalKind::ReadStateDelta, - slot_size, - scratch_.data()); + const bool ok = journal_.append(path, + protocol, + storage_v2::JournalKind::ReadStateDelta, + slot_size, + scratch_.data()); + if (ok) + { + maintenance_compaction_requested_.store( + true, + std::memory_order_release); + } + return ok; } bool SdStore::appendStatusProjection( @@ -1880,11 +3145,18 @@ bool SdStore::appendStatusProjection( { return false; } - return journal_.append(path, - protocol, - storage_v2::JournalKind::StatusDelta, - slot_size, - scratch_.data()); + const bool ok = journal_.append(path, + protocol, + storage_v2::JournalKind::StatusDelta, + slot_size, + scratch_.data()); + if (ok) + { + maintenance_compaction_requested_.store( + true, + std::memory_order_release); + } + return ok; } bool SdStore::appendSeenProjection( @@ -1922,11 +3194,19 @@ bool SdStore::appendSeenProjection( { return false; } - return journal_.append(path, - MeshProtocol::Reticulum, - storage_v2::JournalKind::ReticulumSeen, - slot_size, - scratch_.data()); + const bool ok = journal_.append( + path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size, + scratch_.data()); + if (ok) + { + maintenance_compaction_requested_.store( + true, + std::memory_order_release); + } + return ok; } bool SdStore::rememberReticulumHash(const uint8_t* hash) @@ -1941,6 +3221,11 @@ bool SdStore::rememberReticulumHash(const uint8_t* hash) { return false; } + storage_runtime::ScopedRecursiveStateLock state_lock(mutex_); + if (!state_lock.locked()) + { + return false; + } if (seen_hot_.size() == kSeenHotCapacity) { seen_hot_.erase(seen_hot_.begin()); @@ -2095,228 +3380,6 @@ uint32_t SdStore::sequenceForUnread(const ConversationId& conversation, return 0U; } -bool SdStore::rewriteCatalogSnapshot(MeshProtocol protocol) -{ - char final_path[128]{}; - char temp_path[128]{}; - buildProjectionPath(protocol, - "catalog.snapshot", - final_path, - sizeof(final_path)); - buildProjectionPath(protocol, - "catalog.snapshot.tmp", - temp_path, - sizeof(temp_path)); - return rewriteJournalFromCatalog(protocol, final_path, temp_path); -} - -bool SdStore::rewriteReadSnapshot(MeshProtocol protocol) -{ - char final_path[128]{}; - char temp_path[128]{}; - buildProjectionPath(protocol, - "read.snapshot", - final_path, - sizeof(final_path)); - buildProjectionPath(protocol, - "read.snapshot.tmp", - temp_path, - sizeof(temp_path)); - return rewriteJournalFromReadState(protocol, final_path, temp_path); -} - -bool SdStore::rewriteStatusSnapshot(MeshProtocol protocol) -{ - char final_path[128]{}; - char temp_path[128]{}; - buildProjectionPath(protocol, - "status.snapshot", - final_path, - sizeof(final_path)); - buildProjectionPath(protocol, - "status.snapshot.tmp", - temp_path, - sizeof(temp_path)); - return rewriteJournalFromStatus(protocol, final_path, temp_path); -} - -bool SdStore::compactProtocolProjections(MeshProtocol protocol) -{ - protocol = normalizeProtocol(protocol); - char catalog_delta[128]{}; - char read_delta[128]{}; - char status_delta[128]{}; - buildProjectionPath(protocol, - "catalog.delta", - catalog_delta, - sizeof(catalog_delta)); - buildProjectionPath(protocol, - "read.delta", - read_delta, - sizeof(read_delta)); - buildProjectionPath(protocol, - "status.delta", - status_delta, - sizeof(status_delta)); - const auto catalog_inspection = journal_.inspect( - catalog_delta, - protocol, - storage_v2::JournalKind::CatalogDelta, - storage_v2::catalogSlotSize(protocol)); - const auto read_inspection = journal_.inspect( - read_delta, - protocol, - storage_v2::JournalKind::ReadStateDelta, - storage_v2::readStateSlotSize(protocol)); - const auto status_inspection = journal_.inspect( - status_delta, - protocol, - storage_v2::JournalKind::StatusDelta, - storage_v2::statusSlotSize()); - const bool compact_catalog = - projection_dirty_[protocolIndex(protocol)] || - catalog_inspection.slot_count >= kCatalogCompactThreshold; - const bool compact_read = - read_inspection.slot_count >= kReadCompactThreshold; - const bool compact_status = - status_inspection.slot_count >= kStatusCompactThreshold; - if (compact_catalog && !rewriteCatalogSnapshot(protocol)) - { - return false; - } - if (compact_read && !rewriteReadSnapshot(protocol)) - { - return false; - } - if (compact_status && !rewriteStatusSnapshot(protocol)) - { - return false; - } - if (compact_catalog) - { - (void)storage_runtime::sd_remove(catalog_delta); - projection_dirty_[protocolIndex(protocol)] = false; - } - if (compact_read) - { - (void)storage_runtime::sd_remove(read_delta); - } - if (compact_status) - { - (void)storage_runtime::sd_remove(status_delta); - } - return true; -} - -bool SdStore::rewriteJournalFromCatalog(MeshProtocol protocol, - const char* final_path, - const char* temp_path) -{ - const std::size_t slot_size = storage_v2::catalogSlotSize(protocol); - (void)storage_runtime::sd_remove(temp_path); - if (!journal_.create(temp_path, - protocol, - storage_v2::JournalKind::CatalogSnapshot, - slot_size)) - { - return false; - } - for (const storage_v2::ChatCatalogProjection& projection : catalog_) - { - if (!sameProtocol(projection.conversation.protocol, protocol) || - projection.deleted) - { - continue; - } - if (!storage_v2::encodeCatalogSlot(protocol, - projection, - scratch_.data(), - slot_size) || - !journal_.append(temp_path, - protocol, - storage_v2::JournalKind::CatalogSnapshot, - slot_size, - scratch_.data())) - { - (void)storage_runtime::sd_remove(temp_path); - return false; - } - } - return replaceSnapshot(temp_path, final_path); -} - -bool SdStore::rewriteJournalFromReadState(MeshProtocol protocol, - const char* final_path, - const char* temp_path) -{ - const std::size_t slot_size = storage_v2::readStateSlotSize(protocol); - (void)storage_runtime::sd_remove(temp_path); - if (!journal_.create(temp_path, - protocol, - storage_v2::JournalKind::ReadStateSnapshot, - slot_size)) - { - return false; - } - for (const storage_v2::ChatReadProjection& projection : read_state_) - { - if (!sameProtocol(projection.conversation.protocol, protocol) || - projection.deleted) - { - continue; - } - if (!storage_v2::encodeReadStateSlot(protocol, - projection, - scratch_.data(), - slot_size) || - !journal_.append(temp_path, - protocol, - storage_v2::JournalKind::ReadStateSnapshot, - slot_size, - scratch_.data())) - { - (void)storage_runtime::sd_remove(temp_path); - return false; - } - } - return replaceSnapshot(temp_path, final_path); -} - -bool SdStore::rewriteJournalFromStatus(MeshProtocol protocol, - const char* final_path, - const char* temp_path) -{ - const std::size_t slot_size = storage_v2::statusSlotSize(); - (void)storage_runtime::sd_remove(temp_path); - if (!journal_.create(temp_path, - protocol, - storage_v2::JournalKind::StatusSnapshot, - slot_size)) - { - return false; - } - for (const ProtocolStatusProjection& state : statuses_) - { - if (!sameProtocol(state.protocol, protocol)) - { - continue; - } - if (!storage_v2::encodeStatusSlot(state.value, - scratch_.data(), - slot_size) || - !journal_.append(temp_path, - protocol, - storage_v2::JournalKind::StatusSnapshot, - slot_size, - scratch_.data())) - { - (void)storage_runtime::sd_remove(temp_path); - return false; - } - } - return replaceSnapshot(temp_path, final_path); -} - MeshProtocol SdStore::normalizeProtocol(MeshProtocol protocol) { return protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum : protocol; diff --git a/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp b/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp index 9d37ec76..730135a2 100644 --- a/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp +++ b/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp @@ -231,6 +231,12 @@ void set_mesh_peer_directory_failure(Status& out, case chat::MeshPeerDirectoryStatusCode::StorageUnavailable: set_status(out, "Mesh peer directory storage unavailable", kLxmfAddressesPath); break; + case chat::MeshPeerDirectoryStatusCode::Busy: + set_status(out, "Mesh peer directory busy", kLxmfAddressesPath); + break; + case chat::MeshPeerDirectoryStatusCode::DeviceUnavailable: + set_status(out, "Mesh peer directory device unavailable", kLxmfAddressesPath); + break; case chat::MeshPeerDirectoryStatusCode::IoError: set_status(out, "Cannot access mesh peer directory", kLxmfAddressesPath); break; diff --git a/platform/esp/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp b/platform/esp/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp index e41d0251..edb6caad 100644 --- a/platform/esp/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp +++ b/platform/esp/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp @@ -18,6 +18,9 @@ constexpr const char* kConfigDir = "/trailmate/reticulum"; constexpr const char* kConfigPath = "/trailmate/reticulum/groups.tsv"; constexpr const char* kConfigTempPath = "/trailmate/reticulum/groups.tmp"; constexpr std::size_t kMaxConfigBytes = 2048; +chat::ReticulumGroupDestinationConfig + s_pending_groups[chat::kReticulumGroupDestinationMaxCount] = {}; +bool s_pending = false; void copy_text(char* out, std::size_t out_len, const char* text) { @@ -247,6 +250,65 @@ Status load(chat::ReticulumGroupDestinationConfig* groups, std::size_t group_cou return out; } +Status submit(const chat::ReticulumGroupDestinationConfig* groups, + std::size_t group_count) +{ + Status out{}; + out.supported = true; + out.sd_present = sd_available(); + if (!groups || group_count == 0 || + group_count > chat::kReticulumGroupDestinationMaxCount) + { + set_status(out, "Group storage unavailable", kConfigPath); + return out; + } + if (!out.sd_present) + { + set_status(out, "SD card required", kConfigPath); + return out; + } + + std::memcpy(s_pending_groups, + groups, + group_count * sizeof(chat::ReticulumGroupDestinationConfig)); + for (std::size_t index = group_count; + index < chat::kReticulumGroupDestinationMaxCount; + ++index) + { + s_pending_groups[index] = chat::ReticulumGroupDestinationConfig{}; + } + s_pending = true; + out.queued = true; + set_status(out, "Reticulum groups save queued", kConfigPath); + return out; +} + +Status flushPending() +{ + if (!s_pending) + { + Status out{}; + out.supported = true; + out.sd_present = sd_available(); + out.saved = true; + set_status(out, "Reticulum groups idle", kConfigPath); + return out; + } + + const Status out = save(s_pending_groups, + chat::kReticulumGroupDestinationMaxCount); + if (out.saved) + { + s_pending = false; + } + return out; +} + +bool hasPending() +{ + return s_pending; +} + Status save(const chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count) { Status out{}; diff --git a/platform/esp/arduino_common/src/rnode_kiss/rnode_kiss_service.cpp b/platform/esp/arduino_common/src/rnode_kiss/rnode_kiss_service.cpp index 9f35bc38..6519a692 100644 --- a/platform/esp/arduino_common/src/rnode_kiss/rnode_kiss_service.cpp +++ b/platform/esp/arduino_common/src/rnode_kiss/rnode_kiss_service.cpp @@ -121,6 +121,20 @@ bool apply_live_config() return true; } +template +bool edit_reticulum_config(app::IAppFacade& app_ctx, Mutator mutator) +{ + auto edit = app_ctx.beginConfigEdit(); + if (!edit) + { + return false; + } + + mutator(edit.config().reticulumConfig()); + edit.commit(app::AppConfigChangeSet::mesh()); + return true; +} + void note_rx() { hostlink::note_rx(s_session); @@ -329,7 +343,6 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) note_rx(); app::IAppFacade& app_ctx = app::appFacade(); - chat::MeshConfig& cfg = app_ctx.getConfig().reticulumConfig(); chat::rnode::RNodeAdapter* backend = get_backend(); switch (command) @@ -352,8 +365,10 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) case kCmdFrequency: if (len >= 4) { - cfg.override_frequency_mhz = static_cast(decode_u32(payload, len)) / 1000000.0f; - if (s_radio_state == kRadioStateOn) + const float frequency_mhz = static_cast(decode_u32(payload, len)) / 1000000.0f; + if (edit_reticulum_config(app_ctx, [frequency_mhz](chat::MeshConfig& config) + { config.override_frequency_mhz = frequency_mhz; }) && + s_radio_state == kRadioStateOn) { (void)apply_live_config(); } @@ -363,8 +378,10 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) case kCmdBandwidth: if (len >= 4) { - cfg.bandwidth_khz = static_cast(decode_u32(payload, len)) / 1000.0f; - if (s_radio_state == kRadioStateOn) + const float bandwidth_khz = static_cast(decode_u32(payload, len)) / 1000.0f; + if (edit_reticulum_config(app_ctx, [bandwidth_khz](chat::MeshConfig& config) + { config.bandwidth_khz = bandwidth_khz; }) && + s_radio_state == kRadioStateOn) { (void)apply_live_config(); } @@ -374,8 +391,10 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) case kCmdTxPower: if (len >= 1) { - cfg.tx_power = static_cast(payload[0]); - if (s_radio_state == kRadioStateOn) + const int8_t tx_power = static_cast(payload[0]); + if (edit_reticulum_config(app_ctx, [tx_power](chat::MeshConfig& config) + { config.tx_power = tx_power; }) && + s_radio_state == kRadioStateOn) { (void)apply_live_config(); } @@ -385,8 +404,10 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) case kCmdSf: if (len >= 1) { - cfg.spread_factor = payload[0]; - if (s_radio_state == kRadioStateOn) + const uint8_t spread_factor = payload[0]; + if (edit_reticulum_config(app_ctx, [spread_factor](chat::MeshConfig& config) + { config.spread_factor = spread_factor; }) && + s_radio_state == kRadioStateOn) { (void)apply_live_config(); } @@ -396,8 +417,10 @@ void process_command(uint8_t command, const uint8_t* payload, size_t len) case kCmdCr: if (len >= 1) { - cfg.coding_rate = payload[0]; - if (s_radio_state == kRadioStateOn) + const uint8_t coding_rate = payload[0]; + if (edit_reticulum_config(app_ctx, [coding_rate](chat::MeshConfig& config) + { config.coding_rate = coding_rate; }) && + s_radio_state == kRadioStateOn) { (void)apply_live_config(); } diff --git a/platform/esp/arduino_common/src/storage/storage_runtime.cpp b/platform/esp/arduino_common/src/storage/storage_runtime.cpp index df2e2125..208601f6 100644 --- a/platform/esp/arduino_common/src/storage/storage_runtime.cpp +++ b/platform/esp/arduino_common/src/storage/storage_runtime.cpp @@ -2,7 +2,10 @@ #include "platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h" #include "platform/esp/arduino_common/chat/infra/store/sd_store.h" +#include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/boards/board_runtime.h" #include "platform/esp/common/memory_budget.h" +#include "platform/esp/common/storage/storage_maintenance_owner.h" #include "platform/ui/screen_runtime.h" #include @@ -10,26 +13,17 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" -#include +#include namespace platform::esp::arduino_common::storage { namespace { -// ESP-IDF changes the FreeRTOS task API contract: the stack-depth argument -// and uxTaskGetStackHighWaterMark() are expressed in bytes, not words. -// Hydration crosses SdFat, the store/repository loaders, and Arduino logging; -// 2 KiB is not a viable budget for that call chain. constexpr uint32_t kStorageTaskStackBytes = 8U * 1024U; -// Keep hydration below the Arduino loop task. Storage is deliberately -// throughput-oriented; the UI/input loop owns responsiveness on this core. constexpr UBaseType_t kStorageTaskPriority = 0; constexpr size_t kStorageInternalReservation = kStorageTaskStackBytes; constexpr size_t kStorageInternalFloor = 40U * 1024U; -constexpr uint32_t kRetryBaseMs = 2000U; -constexpr uint32_t kRetryMaxMs = 60000U; -constexpr uint32_t kIdleStableMs = 1500U; struct WorkerContext { @@ -38,150 +32,311 @@ struct WorkerContext chat::MeshProtocol active_protocol = chat::MeshProtocol::Meshtastic; }; -enum class WorkerMode : uint8_t -{ - Hydrate, - Compact, -}; +using Owner = platform::esp::common::storage::StorageMaintenanceOwner; +using OwnerConfig = + platform::esp::common::storage::StorageMaintenanceOwnerConfig; +using Adapter = platform::esp::common::storage::ISemanticStorageAdapter; +using Operation = platform::esp::common::storage::StorageOperation; +using OperationGeneration = + platform::esp::common::storage::StorageOperationGeneration; +using Demand = + platform::esp::common::storage::StorageMaintenanceDemand; +using Result = platform::esp::common::storage::StorageOperationResult; +using ResultKind = + platform::esp::common::storage::StorageOperationResultKind; WorkerContext s_context{}; -TaskHandle_t s_worker_task = nullptr; -WorkerMode s_worker_mode = WorkerMode::Hydrate; -uint32_t s_retry_due_ms = 0U; -uint8_t s_retry_attempt = 0U; -uint32_t s_idle_since_ms = 0U; -bool s_armed = false; -bool s_first_frame_pending = false; -volatile bool s_hydrating = false; -bool s_hydration_ready_event = false; -bool s_maintenance_pending = false; +Owner s_owner{}; +std::atomic s_defer_interactive_storage_reads{false}; -void storage_worker(void*); - -uint32_t retry_delay_ms() +Result makeResult(Operation operation, + OperationGeneration generation, + bool ok) { - const uint8_t shift = std::min(s_retry_attempt, 5U); - return std::min(kRetryBaseMs << shift, kRetryMaxMs); -} - -bool deadline_reached(uint32_t now_ms, uint32_t deadline_ms) -{ - return deadline_ms == 0U || - static_cast(now_ms - deadline_ms) >= 0; -} - -void schedule_retry(const char* reason) -{ - ++s_retry_attempt; - const uint32_t delay_ms = retry_delay_ms(); - s_retry_due_ms = millis() + delay_ms; - Serial.printf("[Storage] retry scheduled reason=%s attempt=%u retry_in_ms=%lu\n", - reason, - static_cast(s_retry_attempt), - static_cast(delay_ms)); -} - -bool start_worker(WorkerMode mode) -{ - if (s_worker_task || !s_armed) + if (ok) { - return false; + return Result::completedResult(operation, generation); } - if (!::platform::esp::common::memory::admit("storage_worker", - kStorageInternalReservation, - 0, - 0, - kStorageInternalFloor, - 0)) + if (!sd_card_ready()) { - schedule_retry("low_internal"); - return false; + return Result::failure(ResultKind::DeviceUnavailable, + operation, + generation); + } + return Result::failure(ResultKind::IoError, operation, generation); +} + +class SdMaintenanceAdapter final : public Adapter +{ + public: + explicit SdMaintenanceAdapter(WorkerContext& context) : context_(context) {} + + Result begin(Operation operation, OperationGeneration generation) override + { + next_step_ = Step::None; + if (operation == Operation::Hydrate) + { + const Result chat_result = hydrateChat(generation); + if (!chat_result.completed()) + { + return chat_result; + } + if (!context_.peer_directory) + { + return chat_result; + } + const auto peer_result = + context_.peer_directory->beginMaintenance(operation, generation); + if (!peer_result.inProgress()) + { + return peer_result; + } + next_step_ = Step::HydratePeer; + return Result::inProgressResult(operation, generation); + } + + if (operation == Operation::Persist) + { + if (!context_.peer_directory) + { + return Result::completedResult(operation, generation); + } + const auto peer_result = + context_.peer_directory->beginMaintenance(operation, + generation); + if (!peer_result.inProgress()) + { + return peer_result; + } + next_step_ = Step::PersistPeer; + return Result::inProgressResult(operation, generation); + } + + if (operation == Operation::Compact) + { + const Result chat_result = compactChat(generation); + if (!chat_result.completed()) + { + return chat_result; + } + if (!context_.peer_directory) + { + return chat_result; + } + const auto peer_result = + context_.peer_directory->beginMaintenance(operation, generation); + if (!peer_result.inProgress()) + { + return peer_result; + } + next_step_ = Step::CompactPeer; + return Result::inProgressResult(operation, generation); + } + + return Result::failure(ResultKind::Cancelled, operation, generation); } - s_worker_mode = mode; - s_hydrating = mode == WorkerMode::Hydrate; - const BaseType_t result = - xTaskCreatePinnedToCore(&storage_worker, - mode == WorkerMode::Hydrate ? "storage_hydrate" - : "storage_compact", - kStorageTaskStackBytes, - nullptr, - kStorageTaskPriority, - &s_worker_task, - 1); - if (result != pdPASS) + Result step(Operation operation, + OperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& + budget) override { - s_worker_task = nullptr; - s_hydrating = false; - schedule_retry("task_create_failed"); - return false; + const Step step = next_step_; + if (step == Step::Chat) + { + if (!context_.chat_store) + { + if (!context_.peer_directory) + { + next_step_ = Step::None; + return makeResult(operation, generation, true); + } + const auto peer_result = + context_.peer_directory->beginMaintenance(operation, + generation); + if (!peer_result.inProgress()) + { + next_step_ = Step::None; + return peer_result; + } + next_step_ = + operation == Operation::Hydrate + ? Step::HydratePeer + : operation == Operation::Persist + ? Step::PersistPeer + : Step::CompactPeer; + return peer_result; + } + const Result result = context_.chat_store->stepMaintenance( + operation, + generation, + budget); + if (!result.completed()) + { + return result; + } + if (context_.peer_directory) + { + const auto peer_result = + context_.peer_directory->beginMaintenance(operation, + generation); + if (!peer_result.inProgress()) + { + return peer_result; + } + next_step_ = operation == Operation::Hydrate + ? Step::HydratePeer + : Step::CompactPeer; + } + else + { + next_step_ = Step::None; + } + return next_step_ == Step::None + ? result + : Result::inProgressResult(operation, generation); + } + + if ((operation == Operation::Hydrate && step == Step::HydratePeer) || + (operation == Operation::Persist && step == Step::PersistPeer) || + (operation == Operation::Compact && step == Step::CompactPeer)) + { + if (!context_.peer_directory) + { + next_step_ = Step::None; + return Result::failure(ResultKind::DeviceUnavailable, + operation, + generation); + } + const Result result = context_.peer_directory->stepMaintenance( + operation, + generation, + budget); + if (!result.inProgress()) + { + next_step_ = Step::None; + } + return result; + } + next_step_ = Step::None; + return Result::failure(ResultKind::Cancelled, operation, generation); } - s_retry_due_ms = 0U; - Serial.printf("[Storage] worker started mode=%s stack_bytes=%u\n", - mode == WorkerMode::Hydrate ? "hydrate" : "compact", - static_cast(kStorageTaskStackBytes)); - return true; + + void cancelAtStepBoundary(Operation operation, + OperationGeneration generation) override + { + if (context_.chat_store) + { + context_.chat_store->cancelMaintenance(operation, generation); + } + if (context_.peer_directory) + { + context_.peer_directory->cancelMaintenance(operation, generation); + } + next_step_ = Step::None; + } + + private: + enum class Step : uint8_t + { + None, + Chat, + HydratePeer, + PersistPeer, + CompactPeer, + }; + + Result hydrateChat(OperationGeneration generation) + { + if (context_.chat_store == nullptr) + { + return Result::completedResult(Operation::Hydrate, generation); + } + next_step_ = Step::Chat; + return context_.chat_store->beginMaintenance( + Operation::Hydrate, + generation); + } + + Result compactChat(OperationGeneration generation) + { + if (context_.chat_store == nullptr) + { + return Result::completedResult(Operation::Compact, generation); + } + next_step_ = Step::Chat; + return context_.chat_store->beginMaintenance( + Operation::Compact, + generation); + } + + WorkerContext& context_; + Step next_step_ = Step::None; +}; + +SdMaintenanceAdapter s_adapter(s_context); + +bool admitOwner(void*) +{ + return ::platform::esp::common::memory::admit("storage_owner", + kStorageInternalReservation, + 0, + 0, + kStorageInternalFloor, + 0); } -void storage_worker(void*) +uint32_t ownerNow(void*) { - const uint32_t started_ms = millis(); - const WorkerMode mode = s_worker_mode; - Serial.printf("[Storage] worker begin mode=%s active_protocol=%u\n", - mode == WorkerMode::Hydrate ? "hydrate" : "compact", + return millis(); +} + +const char* operationName(Operation operation) +{ + if (operation == Operation::Hydrate) + { + return "hydrate"; + } + if (operation == Operation::Persist) + { + return "persist"; + } + return "compact"; +} + +void ownerStarted(void*, + Operation operation, + OperationGeneration generation) +{ + Serial.printf("[Storage] owner begin mode=%s generation=%lu active_protocol=%u\n", + operationName(operation), + static_cast(generation), static_cast(s_context.active_protocol)); +} - bool ok = true; - if (mode == WorkerMode::Hydrate) - { - const bool chat_ready = - s_context.chat_store == nullptr || - s_context.chat_store->hydrateFromStorage(); - const bool peer_ready = - s_context.peer_directory == nullptr || - s_context.peer_directory->hydrateFromStorage().succeeded(); - ok = chat_ready && peer_ready; - if (ok) - { - s_retry_attempt = 0U; - s_hydration_ready_event = true; - s_maintenance_pending = true; - s_idle_since_ms = 0U; - } - } - else - { - if (s_context.chat_store) - { - ok = s_context.chat_store->compactDeferred() && ok; - } - if (s_context.peer_directory) - { - ok = s_context.peer_directory->compactDeferred().succeeded() && ok; - } - if (ok) - { - s_maintenance_pending = false; - s_retry_attempt = 0U; - } - } +void ownerFinished(void*, + Operation operation, + OperationGeneration generation, + ResultKind result, + uint32_t elapsed_ms, + uint32_t stack_free_bytes) +{ + Serial.printf("[Storage] owner end mode=%s generation=%lu ok=%u result=%u " + "elapsed_ms=%lu stack_free_bytes=%lu\n", + operationName(operation), + static_cast(generation), + (result == ResultKind::Completed || + result == ResultKind::InProgress) + ? 1U + : 0U, + static_cast(result), + static_cast(elapsed_ms), + static_cast(stack_free_bytes)); +} - // ESP-IDF returns the high-water mark in bytes (unlike vanilla FreeRTOS). - const unsigned long stack_free_bytes = - static_cast(uxTaskGetStackHighWaterMark(nullptr)); - Serial.printf("[Storage] worker end mode=%s ok=%u elapsed_ms=%lu stack_free_bytes=%lu\n", - mode == WorkerMode::Hydrate ? "hydrate" : "compact", - ok ? 1U : 0U, - static_cast(millis() - started_ms), - stack_free_bytes); - s_worker_task = nullptr; - s_hydrating = false; - if (!ok) - { - schedule_retry(mode == WorkerMode::Hydrate ? "hydrate_failed" - : "compact_failed"); - } - vTaskDelete(nullptr); +bool startupGateSatisfied() +{ + return ::platform::esp::boards::storageStartupGateSatisfied(); } } // namespace @@ -190,85 +345,89 @@ void start_deferred_storage(chat::SdStore* chat_store, chat::SdProtocolPeerRepository* peer_store, chat::MeshProtocol active_protocol) { - if (s_armed) + if (s_owner.isArmed()) { return; } if (!chat_store && !peer_store) { - Serial.printf("[Storage] deferred recovery skipped backend=ram\n"); + Serial.printf("[Storage] maintenance skipped backend=ram\n"); return; } s_context.chat_store = chat_store; s_context.peer_directory = peer_store; s_context.active_protocol = active_protocol; - s_armed = true; - s_retry_attempt = 0U; - s_retry_due_ms = 0U; - // Arm the state machine now, but let the first foreground loop present a - // complete LVGL frame before any SD worker can contend for the shared SPI - // bus. - s_first_frame_pending = true; + const bool shared_storage_topology = + ::platform::esp::boards::storageCapabilities() + .requiresDisplayTransactionGate(); + + OwnerConfig config{}; + config.task_name = "storage_owner"; + config.stack_bytes = kStorageTaskStackBytes; + config.priority = kStorageTaskPriority; + config.core = 1; + config.startup_gate = shared_storage_topology + ? platform::esp::common::storage::StorageStartupGate:: + DisplayTransaction + : platform::esp::common::storage::StorageStartupGate::Immediate; + config.context = &s_context; + config.adapter = &s_adapter; + config.admit = &admitOwner; + config.now = &ownerNow; + config.on_started = &ownerStarted; + config.on_finished = &ownerFinished; + s_owner.configure(config); + const bool armed = s_owner.arm(millis(), startupGateSatisfied()); + s_defer_interactive_storage_reads.store(armed && shared_storage_topology, + std::memory_order_release); } void tick_deferred_storage() { - if (!s_armed || s_worker_task) + if (!s_owner.isArmed()) { + s_defer_interactive_storage_reads.store(false, std::memory_order_release); return; } - if (s_first_frame_pending) - { - s_first_frame_pending = false; - return; - } + Demand demand{}; + demand.persistence_pending = + s_context.peer_directory && + s_context.peer_directory->persistencePending(); + demand.compaction_pending = + (s_context.chat_store && + s_context.chat_store->compactionPending()) || + (s_context.peer_directory && + s_context.peer_directory->compactionPending()); + (void)s_owner.submitTick(millis(), + ::platform::ui::screen::is_sleeping(), + ::platform::ui::screen::is_saver_active(), + startupGateSatisfied(), + demand); +} - const uint32_t now_ms = millis(); - if (s_retry_due_ms != 0U && !deadline_reached(now_ms, s_retry_due_ms)) - { - return; - } - - if (s_maintenance_pending) - { - if (!::platform::ui::screen::is_sleeping() || - ::platform::ui::screen::is_saver_active()) - { - s_idle_since_ms = 0U; - return; - } - if (s_idle_since_ms == 0U) - { - s_idle_since_ms = now_ms; - return; - } - if (now_ms - s_idle_since_ms < kIdleStableMs) - { - return; - } - (void)start_worker(WorkerMode::Compact); - return; - } - - (void)start_worker(WorkerMode::Hydrate); +void stop_deferred_storage() +{ + (void)s_owner.requestStop(); + s_defer_interactive_storage_reads.store(false, std::memory_order_release); } bool hydration_active() { - return s_hydrating; + return s_owner.hydrationActive(); +} + +bool interactive_storage_reads_deferred() +{ + return s_defer_interactive_storage_reads.load(std::memory_order_acquire) && + s_owner.initialHydrationPending(); } bool consume_hydration_ready() { - if (!s_hydration_ready_event) - { - return false; - } - s_hydration_ready_event = false; - return true; + return s_owner.consumeHydrationReady(); } } // namespace platform::esp::arduino_common::storage diff --git a/platform/esp/arduino_common/src/ui/widgets/map/map_tiles.cpp b/platform/esp/arduino_common/src/ui/widgets/map/map_tiles.cpp index a5fd785d..fdc89ab8 100644 --- a/platform/esp/arduino_common/src/ui/widgets/map/map_tiles.cpp +++ b/platform/esp/arduino_common/src/ui/widgets/map/map_tiles.cpp @@ -9,6 +9,7 @@ #include "freertos/task.h" #include "lvgl.h" #include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/arduino_common/storage/storage_runtime.h" #include "src/draw/lv_image_decoder_private.h" #include "src/misc/cache/instance/lv_image_cache.h" #include "sys/clock.h" @@ -173,6 +174,11 @@ class SdMapTileFileSystem final : public ui::map_tiles::IMapTileFileSystem uint8_t* buffer, std::size_t capacity) const override { + if (::platform::esp::arduino_common::storage:: + interactive_storage_reads_deferred()) + { + return {ui::map_tiles::MapTileReadStatus::RetryLater, 0U, 0}; + } const auto result = ::platform::esp::arduino_common::storage::sd_read_file( path, diff --git a/platform/esp/boards/include/platform/esp/boards/board_runtime.h b/platform/esp/boards/include/platform/esp/boards/board_runtime.h index 38c6bdd0..64cd10e7 100644 --- a/platform/esp/boards/include/platform/esp/boards/board_runtime.h +++ b/platform/esp/boards/include/platform/esp/boards/board_runtime.h @@ -37,6 +37,24 @@ struct BoardIdentity const char* ble_name = "trail-mate"; }; +enum class StorageBusTopology : uint8_t +{ + None = 0, + DedicatedSpi, + SharedDisplaySpi, + Sdmmc, +}; + +struct BoardStorageCapabilities +{ + StorageBusTopology topology = StorageBusTopology::None; + + constexpr bool requiresDisplayTransactionGate() const + { + return topology == StorageBusTopology::SharedDisplaySpi; + } +}; + void initializeBoard(bool waking_from_sleep); void initializeBoardDisplayHardware(bool waking_from_sleep); void initializeBoardServices(bool waking_from_sleep); @@ -49,5 +67,7 @@ void unlockDisplay(); bool syncSystemTimeFromBoardRtc(); bool applySystemTimeAndSyncBoardRtc(std::time_t epoch_seconds, const char* source); BoardIdentity defaultIdentity(); +BoardStorageCapabilities storageCapabilities(); +bool storageStartupGateSatisfied(); } // namespace platform::esp::boards diff --git a/platform/esp/boards/src/board_runtime.cpp b/platform/esp/boards/src/board_runtime.cpp index 8f0494b9..5a303787 100644 --- a/platform/esp/boards/src/board_runtime.cpp +++ b/platform/esp/boards/src/board_runtime.cpp @@ -16,6 +16,11 @@ #include "boards/tlora_pager/platform_esp_board_runtime.h" #endif +#if defined(ARDUINO_T_DECK_PRO) || defined(ARDUINO_T_DECK) || \ + defined(ARDUINO_T_LORA_PAGER) +#include "platform/esp/common/shared_spi_coordinator.h" +#endif + namespace platform::esp::boards { @@ -102,4 +107,34 @@ BoardIdentity defaultIdentity() #endif } +BoardStorageCapabilities storageCapabilities() +{ +#if defined(TRAIL_MATE_ESP_BOARD_TAB5) || \ + defined(TRAIL_MATE_ESP_BOARD_T_DISPLAY_P4) + return {StorageBusTopology::Sdmmc}; +#elif defined(ARDUINO_T_DECK_PRO) || defined(ARDUINO_T_DECK) || \ + defined(ARDUINO_T_LORA_PAGER) + return {StorageBusTopology::SharedDisplaySpi}; +#else + return {StorageBusTopology::DedicatedSpi}; +#endif +} + +bool storageStartupGateSatisfied() +{ + if (!storageCapabilities().requiresDisplayTransactionGate()) + { + return true; + } +#if defined(ARDUINO_T_DECK_PRO) || defined(ARDUINO_T_DECK) || \ + defined(ARDUINO_T_LORA_PAGER) + return platform::esp::common::shared_spi_coordinator() + .displayFrameCompletions() > 0U; +#else + // A shared-display-SPI IDF board must provide a board-local completion + // signal before hydration can be enabled. + return false; +#endif +} + } // namespace platform::esp::boards diff --git a/platform/esp/common/include/platform/esp/common/storage/storage_contracts.h b/platform/esp/common/include/platform/esp/common/storage/storage_contracts.h new file mode 100644 index 00000000..d46eee37 --- /dev/null +++ b/platform/esp/common/include/platform/esp/common/storage/storage_contracts.h @@ -0,0 +1,166 @@ +#pragma once + +#include "sys/persistence_contracts.h" + +#include + +namespace platform::esp::common::storage +{ + +enum class StorageOperation : uint8_t +{ + None, + Hydrate, + Persist, + Compact, +}; + +using StorageOperationGeneration = sys::PersistenceGeneration; +using StorageOperationResultKind = sys::PersistenceResultKind; + +struct StorageOperationResult +{ + StorageOperationResultKind kind = StorageOperationResultKind::IoError; + StorageOperation operation = StorageOperation::None; + StorageOperationGeneration generation = 0U; + + bool completed() const + { + return kind == StorageOperationResultKind::Completed; + } + + bool inProgress() const + { + return kind == StorageOperationResultKind::InProgress; + } + + bool retryable() const + { + return kind == StorageOperationResultKind::StateBusy || + kind == StorageOperationResultKind::DeviceUnavailable || + kind == StorageOperationResultKind::RetryLater || + kind == StorageOperationResultKind::IoError; + } + + static StorageOperationResult completedResult( + StorageOperation operation, + StorageOperationGeneration generation) + { + return {StorageOperationResultKind::Completed, operation, generation}; + } + + static StorageOperationResult inProgressResult( + StorageOperation operation, + StorageOperationGeneration generation) + { + return {StorageOperationResultKind::InProgress, operation, generation}; + } + + static StorageOperationResult failure( + StorageOperationResultKind kind, + StorageOperation operation, + StorageOperationGeneration generation) + { + return {kind, operation, generation}; + } +}; + +struct StorageOperationBudget +{ + // A step is deliberately expressed in logical work items rather than + // milliseconds. Physical adapters can then keep each transaction bounded + // without making the owner depend on a particular clock or filesystem. + uint8_t max_work_items = 1U; +}; + +struct StorageMaintenanceDemand +{ + bool persistence_pending = false; + bool compaction_pending = false; +}; + +struct StorageRetryPolicy +{ + uint32_t base_delay_ms = 2000U; + uint32_t maximum_delay_ms = 60000U; + uint8_t maximum_attempts = 0U; + + uint32_t delayForAttempt(uint8_t attempt) const + { + if (attempt == 0U) + { + return 0U; + } + + uint32_t delay = base_delay_ms; + for (uint8_t shift = 1U; shift < attempt && delay < maximum_delay_ms; + ++shift) + { + if (delay > maximum_delay_ms / 2U) + { + delay = maximum_delay_ms; + break; + } + delay *= 2U; + } + return delay > maximum_delay_ms ? maximum_delay_ms : delay; + } +}; + +enum class StorageRuntimeState : uint8_t +{ + Dormant, + WaitingStartupGate, + Hydrating, + Ready, + WaitingIdle, + Persisting, + Compacting, + Backoff, + Done, +}; + +struct StorageRuntimeSnapshot +{ + StorageRuntimeState state = StorageRuntimeState::Dormant; + StorageOperation active_operation = StorageOperation::None; + StorageOperation pending_operation = StorageOperation::None; + StorageOperationGeneration generation = 0U; + uint8_t retry_attempt = 0U; + uint32_t retry_due_ms = 0U; + bool startup_gate_satisfied = false; +}; + +// This is the only contract the maintenance owner sees for a storage backend. +// Physical sessions, SPI arbitration, filesystem handles, and task handles +// remain implementation details of the adapter. +class ISemanticStorageAdapter +{ + public: + virtual ~ISemanticStorageAdapter() = default; + + // Begin starts a new operation for a new generation, or resumes the + // backend cursor for the same operation and generation after a retryable + // result. + // A retry must not discard partially completed work merely because the + // previous step released a physical/device lease. + virtual StorageOperationResult begin( + StorageOperation operation, + StorageOperationGeneration generation) = 0; + + // A retryable result may release a transient device/transaction lease, + // but the backend's logical maintenance ownership remains held until the + // operation completes, is cancelled, or reaches an explicit failed + // boundary. The adapter retains enough operation state for begin() to + // resume the same generation without competing with foreground writes. + virtual StorageOperationResult step( + StorageOperation operation, + StorageOperationGeneration generation, + const StorageOperationBudget& budget) = 0; + + virtual void cancelAtStepBoundary( + StorageOperation operation, + StorageOperationGeneration generation) = 0; +}; + +} // namespace platform::esp::common::storage diff --git a/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_owner.h b/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_owner.h new file mode 100644 index 00000000..c228edab --- /dev/null +++ b/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_owner.h @@ -0,0 +1,606 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "platform/esp/common/storage/storage_maintenance_state_machine.h" + +#include +#include +#include + +namespace platform::esp::common::storage +{ + +using StorageMaintenanceAdmitFn = bool (*)(void* context); +using StorageMaintenanceNowFn = uint32_t (*)(void* context); +using StorageMaintenanceStartedFn = void (*)( + void* context, + StorageOperation operation, + StorageOperationGeneration generation); +using StorageMaintenanceFinishedFn = void (*)( + void* context, + StorageOperation operation, + StorageOperationGeneration generation, + StorageOperationResultKind result, + uint32_t elapsed_ms, + uint32_t stack_free_bytes); + +struct StorageMaintenanceOwnerConfig +{ + const char* task_name = "storage_owner"; + uint32_t stack_bytes = 8U * 1024U; + UBaseType_t priority = 1U; + BaseType_t core = 1; + StorageStartupGate startup_gate = StorageStartupGate::Immediate; + StorageOperationBudget step_budget{}; + void* context = nullptr; + ISemanticStorageAdapter* adapter = nullptr; + StorageMaintenanceAdmitFn admit = nullptr; + StorageMaintenanceNowFn now = nullptr; + StorageMaintenanceStartedFn on_started = nullptr; + StorageMaintenanceFinishedFn on_finished = nullptr; +}; + +// A stable active object for maintenance. Its task is created once and stays +// blocked on the command queue after maintenance reaches Done. Business state +// is owned by the task-local state machine; callers only enqueue events and +// read the published snapshot. +class StorageMaintenanceOwner final +{ + public: + static constexpr std::size_t kQueueLength = 8U; + + StorageMaintenanceOwner() = default; + + StorageMaintenanceOwner(const StorageMaintenanceOwner&) = delete; + StorageMaintenanceOwner& operator=(const StorageMaintenanceOwner&) = delete; + + bool arm(uint32_t now_ms, bool startup_gate_satisfied) + { + if (armed_.exchange(true, std::memory_order_acq_rel)) + { + return false; + } + + initial_now_ms_ = now_ms; + initial_gate_satisfied_ = startup_gate_satisfied; + foreground_storage_barrier_.store( + startup_gate_satisfied || + config_.startup_gate == StorageStartupGate::Immediate, + std::memory_order_release); + arm_event_pending_.store(true, std::memory_order_release); + ensureTask(); + enqueuePendingArm(); + + if (startup_gate_satisfied && + config_.startup_gate == StorageStartupGate::DisplayTransaction) + { + const StorageRuntimeSnapshot current = snapshot(); + if (arm_event_pending_.load(std::memory_order_acquire) || + current.state == StorageRuntimeState::Dormant || + current.state == StorageRuntimeState::WaitingStartupGate || + current.pending_operation == StorageOperation::Hydrate) + { + foreground_storage_barrier_.store(true, + std::memory_order_release); + } + } + return true; + } + + bool submitTick(uint32_t now_ms, + bool is_sleeping, + bool saver_active, + bool startup_gate_satisfied, + StorageMaintenanceDemand demand = {}) + { + if (!armed_.load(std::memory_order_acquire)) + { + return false; + } + + latest_tick_now_ms_.store(now_ms, std::memory_order_release); + latest_is_sleeping_.store(is_sleeping, std::memory_order_release); + latest_saver_active_.store(saver_active, std::memory_order_release); + latest_gate_satisfied_.store(startup_gate_satisfied, + std::memory_order_release); + latest_persistence_pending_.store( + demand.persistence_pending, + std::memory_order_release); + latest_compaction_pending_.store( + demand.compaction_pending, + std::memory_order_release); + latest_tick_generation_.fetch_add(1U, std::memory_order_acq_rel); + tick_event_pending_.store(true, std::memory_order_release); + + ensureTask(); + enqueuePendingArm(); + enqueuePendingStop(); + return enqueueLatestTick(); + } + + bool requestStop() + { + if (!armed_.load(std::memory_order_acquire)) + { + return false; + } + stop_event_pending_.store(true, std::memory_order_release); + ensureTask(); + enqueuePendingArm(); + return enqueuePendingStop(); + } + + void configure(const StorageMaintenanceOwnerConfig& config) + { + config_ = config; + } + + // The owner is the sole authority for whether its command stream may + // accept ticks or a new configuration. A queued Stop remains armed until + // the owner task has cancelled the active operation at its boundary. + bool isArmed() const + { + return armed_.load(std::memory_order_acquire); + } + + StorageRuntimeSnapshot snapshot() const + { + StorageRuntimeSnapshot snapshot{}; + snapshot.state = static_cast( + published_state_.load(std::memory_order_acquire)); + snapshot.active_operation = static_cast( + published_active_operation_.load(std::memory_order_acquire)); + snapshot.pending_operation = static_cast( + published_pending_operation_.load(std::memory_order_acquire)); + snapshot.generation = + published_generation_.load(std::memory_order_acquire); + snapshot.retry_attempt = + published_retry_attempt_.load(std::memory_order_acquire); + snapshot.retry_due_ms = + published_retry_due_ms_.load(std::memory_order_acquire); + snapshot.startup_gate_satisfied = + published_startup_gate_satisfied_.load(std::memory_order_acquire); + return snapshot; + } + + bool consumeHydrationReady(StorageOperationGeneration* generation = nullptr) + { + const StorageOperationGeneration ready_generation = + hydration_ready_generation_.exchange(0U, std::memory_order_acq_rel); + if (generation) + { + *generation = ready_generation; + } + return ready_generation != 0U; + } + + bool hydrationActive() const + { + return foreground_storage_barrier_.load(std::memory_order_acquire); + } + + // Covers the arm-to-owner handoff as well as the display gate, active + // hydration, and hydration retry backoff. Optional interactive reads use + // this semantic state so they cannot preempt the first hydration attempt. + bool initialHydrationPending() const + { + if (!armed_.load(std::memory_order_acquire)) + { + return false; + } + + const StorageRuntimeSnapshot current = snapshot(); + return current.state == StorageRuntimeState::Dormant || + isInitialHydrationPending(current); + } + + private: + static constexpr uint32_t kTaskRetryDelayMs = 2000U; + + enum class EventKind : uint8_t + { + Arm, + Tick, + Stop, + }; + + struct Event + { + EventKind kind = EventKind::Tick; + uint32_t now_ms = 0U; + bool is_sleeping = false; + bool saver_active = false; + bool startup_gate_satisfied = false; + uint32_t tick_generation = 0U; + }; + + static void taskEntry(void* arg) + { + static_cast(arg)->taskLoop(); + } + + bool ensureTask() + { + const uint32_t now_ms = + config_.now ? config_.now(config_.context) : 0U; + if (task_retry_due_ms_ != 0U && + static_cast(now_ms - task_retry_due_ms_) < 0) + { + return false; + } + + if (queue_ == nullptr) + { + queue_ = xQueueCreate(kQueueLength, sizeof(Event)); + if (queue_ == nullptr) + { + task_retry_due_ms_ = now_ms + kTaskRetryDelayMs; + return false; + } + } + if (task_ != nullptr) + { + return true; + } + if (config_.admit && !config_.admit(config_.context)) + { + task_retry_due_ms_ = now_ms + kTaskRetryDelayMs; + return false; + } + + if (xTaskCreatePinnedToCore(&taskEntry, + config_.task_name, + config_.stack_bytes, + this, + config_.priority, + &task_, + config_.core) != pdPASS) + { + task_ = nullptr; + task_retry_due_ms_ = now_ms + kTaskRetryDelayMs; + return false; + } + task_retry_due_ms_ = 0U; + return true; + } + + void enqueuePendingArm() + { + if (!queue_) + { + return; + } + + bool expected = true; + if (!arm_event_pending_.compare_exchange_strong( + expected, + false, + std::memory_order_acq_rel)) + { + return; + } + + Event event{}; + event.kind = EventKind::Arm; + event.now_ms = initial_now_ms_; + event.startup_gate_satisfied = initial_gate_satisfied_; + if (xQueueSend(queue_, &event, 0U) != pdPASS) + { + arm_event_pending_.store(true, std::memory_order_release); + } + } + + bool enqueueLatestTick() + { + if (!tick_event_pending_.load(std::memory_order_acquire) || !queue_) + { + return false; + } + bool expected = false; + if (!tick_event_queued_.compare_exchange_strong( + expected, + true, + std::memory_order_acq_rel)) + { + return true; + } + + Event event{}; + event.kind = EventKind::Tick; + event.tick_generation = + latest_tick_generation_.load(std::memory_order_acquire); + if (xQueueSend(queue_, &event, 0U) == pdPASS) + { + return true; + } + tick_event_queued_.store(false, std::memory_order_release); + return false; + } + + bool enqueuePendingStop() + { + if (!stop_event_pending_.load(std::memory_order_acquire) || !queue_) + { + return false; + } + bool expected = true; + if (!stop_event_pending_.compare_exchange_strong( + expected, + false, + std::memory_order_acq_rel)) + { + return false; + } + Event event{}; + event.kind = EventKind::Stop; + if (xQueueSend(queue_, &event, 0U) != pdPASS) + { + stop_event_pending_.store(true, std::memory_order_release); + return false; + } + return true; + } + + void clearTickEventPending(uint32_t processed_generation) + { + if (latest_tick_generation_.load(std::memory_order_acquire) != + processed_generation) + { + return; + } + + tick_event_pending_.store(false, std::memory_order_release); + if (latest_tick_generation_.load(std::memory_order_acquire) != + processed_generation) + { + tick_event_pending_.store(true, std::memory_order_release); + } + } + + void taskLoop() + { + for (;;) + { + Event event{}; + if (xQueueReceive(queue_, &event, portMAX_DELAY) != pdPASS) + { + continue; + } + + if (event.kind == EventKind::Stop) + { + const StorageRuntimeSnapshot current = + state_machine_.snapshot(); + const StorageOperation operation = + current.active_operation != StorageOperation::None + ? current.active_operation + : current.pending_operation; + if (config_.adapter && operation != StorageOperation::None) + { + config_.adapter->cancelAtStepBoundary(operation, + current.generation); + } + state_machine_.stop(); + publish(); + armed_.store(false, std::memory_order_release); + tick_event_pending_.store(false, std::memory_order_release); + tick_event_queued_.store(false, std::memory_order_release); + continue; + } + + StorageMaintenanceCommand command{}; + if (event.kind == EventKind::Arm) + { + command = state_machine_.arm(event.now_ms, + config_.startup_gate, + event.startup_gate_satisfied); + } + else + { + tick_event_queued_.store(false, std::memory_order_release); + event.tick_generation = + latest_tick_generation_.load(std::memory_order_acquire); + event.now_ms = + latest_tick_now_ms_.load(std::memory_order_acquire); + event.is_sleeping = + latest_is_sleeping_.load(std::memory_order_acquire); + event.saver_active = + latest_saver_active_.load(std::memory_order_acquire); + event.startup_gate_satisfied = + latest_gate_satisfied_.load(std::memory_order_acquire); + clearTickEventPending(event.tick_generation); + command = state_machine_.tick(event.now_ms, + event.is_sleeping, + event.saver_active, + event.startup_gate_satisfied, + StorageMaintenanceDemand{ + latest_persistence_pending_ + .load( + std::memory_order_acquire), + latest_compaction_pending_ + .load( + std::memory_order_acquire)}); + } + publish(); + execute(command, event.now_ms); + enqueuePendingStop(); + enqueueLatestTick(); + } + } + + void execute(const StorageMaintenanceCommand& command, uint32_t event_now_ms) + { + if (command.kind == StorageMaintenanceCommandKind::None || + config_.adapter == nullptr) + { + return; + } + + const uint32_t started_ms = currentTime(event_now_ms); + if (command.kind == StorageMaintenanceCommandKind::Begin) + { + active_operation_started_ms_ = started_ms; + active_operation_generation_ = command.generation; + if (config_.on_started) + { + config_.on_started(config_.context, + command.operation, + command.generation); + } + } + + StorageOperationResult result = + command.kind == StorageMaintenanceCommandKind::Begin + ? config_.adapter->begin(command.operation, command.generation) + : config_.adapter->step(command.operation, + command.generation, + config_.step_budget); + if (result.operation == StorageOperation::None) + { + result.operation = command.operation; + } + if (result.generation == 0U) + { + result.generation = command.generation; + } + + const uint32_t finished_ms = currentTime(event_now_ms); + state_machine_.complete(result, finished_ms); + const StorageRuntimeSnapshot completed_snapshot = + state_machine_.snapshot(); + const bool terminal_without_completion = + completed_snapshot.state == StorageRuntimeState::Done && + result.kind != StorageOperationResultKind::Completed && + result.kind != StorageOperationResultKind::InProgress; + if (terminal_without_completion) + { + // A bounded retry budget is a terminal cancellation too. The + // adapter must release any logical maintenance lease retained + // across retryable physical I/O steps. + config_.adapter->cancelAtStepBoundary(command.operation, + command.generation); + } + publish(); + + if (result.kind != StorageOperationResultKind::InProgress && + config_.on_finished) + { + const uint32_t operation_started_ms = + active_operation_generation_ == command.generation + ? active_operation_started_ms_ + : started_ms; + config_.on_finished(config_.context, + command.operation, + command.generation, + result.kind, + static_cast(finished_ms - + operation_started_ms), + static_cast( + uxTaskGetStackHighWaterMark(nullptr) * + sizeof(StackType_t))); + } + if (result.kind != StorageOperationResultKind::InProgress && + active_operation_generation_ == command.generation) + { + active_operation_started_ms_ = 0U; + active_operation_generation_ = 0U; + } + if (completed_snapshot.state == StorageRuntimeState::Done) + { + // Done is terminal for this command stream, whether it was + // reached through an explicit cancellation or retry exhaustion. + // The stable task remains alive, but the next lifecycle may arm + // it with a fresh configuration and generation. + armed_.store(false, std::memory_order_release); + } + } + + uint32_t currentTime(uint32_t fallback_ms) const + { + return config_.now ? config_.now(config_.context) : fallback_ms; + } + + void publish() + { + const StorageRuntimeSnapshot current = state_machine_.snapshot(); + published_state_.store(static_cast(current.state), + std::memory_order_release); + published_active_operation_.store( + static_cast(current.active_operation), + std::memory_order_release); + published_pending_operation_.store( + static_cast(current.pending_operation), + std::memory_order_release); + published_generation_.store(current.generation, + std::memory_order_release); + published_retry_attempt_.store(current.retry_attempt, + std::memory_order_release); + published_retry_due_ms_.store(current.retry_due_ms, + std::memory_order_release); + published_startup_gate_satisfied_.store( + current.startup_gate_satisfied, + std::memory_order_release); + + const StorageOperationGeneration ready_generation = + state_machine_.takeHydrationReadyGeneration(); + if (ready_generation != 0U) + { + hydration_ready_generation_.store(ready_generation, + std::memory_order_release); + } + + if (current.state == StorageRuntimeState::Ready || + current.state == StorageRuntimeState::WaitingIdle || + current.state == StorageRuntimeState::Done) + { + foreground_storage_barrier_.store(false, + std::memory_order_release); + } + else if (current.pending_operation == StorageOperation::Hydrate && + (current.state == StorageRuntimeState::Hydrating || + current.state == StorageRuntimeState::Backoff)) + { + foreground_storage_barrier_.store(true, + std::memory_order_release); + } + } + + StorageMaintenanceOwnerConfig config_{}; + StorageMaintenanceStateMachine state_machine_{}; + QueueHandle_t queue_ = nullptr; + TaskHandle_t task_ = nullptr; + std::atomic armed_{false}; + std::atomic arm_event_pending_{false}; + uint32_t initial_now_ms_ = 0U; + bool initial_gate_satisfied_ = false; + uint32_t task_retry_due_ms_ = 0U; + std::atomic latest_tick_now_ms_{0U}; + std::atomic latest_tick_generation_{0U}; + std::atomic latest_is_sleeping_{false}; + std::atomic latest_saver_active_{false}; + std::atomic latest_gate_satisfied_{false}; + std::atomic latest_persistence_pending_{false}; + std::atomic latest_compaction_pending_{false}; + std::atomic tick_event_pending_{false}; + std::atomic tick_event_queued_{false}; + std::atomic stop_event_pending_{false}; + uint32_t active_operation_started_ms_ = 0U; + StorageOperationGeneration active_operation_generation_ = 0U; + + std::atomic published_state_{ + static_cast(StorageRuntimeState::Dormant)}; + std::atomic published_active_operation_{ + static_cast(StorageOperation::None)}; + std::atomic published_pending_operation_{ + static_cast(StorageOperation::None)}; + std::atomic published_generation_{0U}; + std::atomic published_retry_attempt_{0U}; + std::atomic published_retry_due_ms_{0U}; + std::atomic published_startup_gate_satisfied_{false}; + std::atomic foreground_storage_barrier_{false}; + std::atomic hydration_ready_generation_{0U}; +}; + +} // namespace platform::esp::common::storage diff --git a/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_state_machine.h b/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_state_machine.h new file mode 100644 index 00000000..0a2c12cd --- /dev/null +++ b/platform/esp/common/include/platform/esp/common/storage/storage_maintenance_state_machine.h @@ -0,0 +1,312 @@ +#pragma once + +#include "platform/esp/common/storage/storage_contracts.h" + +#include +#include + +namespace platform::esp::common::storage +{ + +enum class StorageStartupGate : uint8_t +{ + Immediate, + DisplayTransaction, +}; + +enum class StorageMaintenanceCommandKind : uint8_t +{ + None, + Begin, + Step, +}; + +struct StorageMaintenanceCommand +{ + StorageMaintenanceCommandKind kind = StorageMaintenanceCommandKind::None; + StorageOperation operation = StorageOperation::None; + StorageOperationGeneration generation = 0U; +}; + +// Optional interactive reads must yield until the initial authoritative +// projection has either been installed or reached a terminal result. This is +// intentionally narrower than the runtime's general maintenance activity: +// persistence and compaction do not block interactive reads. +constexpr bool isInitialHydrationPending(const StorageRuntimeSnapshot& snapshot) +{ + return snapshot.pending_operation == StorageOperation::Hydrate && + (snapshot.state == StorageRuntimeState::WaitingStartupGate || + snapshot.state == StorageRuntimeState::Hydrating || + snapshot.state == StorageRuntimeState::Backoff); +} + +class StorageMaintenanceStateMachine final +{ + public: + explicit StorageMaintenanceStateMachine( + StorageRetryPolicy retry_policy = {}) + : retry_policy_(retry_policy) + { + } + + StorageMaintenanceCommand arm(uint32_t now_ms, + StorageStartupGate startup_gate, + bool startup_gate_satisfied) + { + (void)now_ms; + if (snapshot_.state == StorageRuntimeState::Done) + { + snapshot_ = {}; + retry_operation_ = StorageOperation::Hydrate; + idle_since_ms_ = 0U; + hydration_ready_generation_ = 0U; + } + if (snapshot_.state != StorageRuntimeState::Dormant) + { + return {}; + } + + snapshot_.generation = 0U; + snapshot_.pending_operation = StorageOperation::Hydrate; + snapshot_.startup_gate_satisfied = startup_gate_satisfied; + if (startup_gate == StorageStartupGate::DisplayTransaction && + !startup_gate_satisfied) + { + snapshot_.state = StorageRuntimeState::WaitingStartupGate; + return {}; + } + + return beginOperation(StorageOperation::Hydrate, true); + } + + StorageMaintenanceCommand tick( + uint32_t now_ms, + bool is_sleeping, + bool saver_active, + bool startup_gate_satisfied, + StorageMaintenanceDemand demand = {}) + { + snapshot_.startup_gate_satisfied |= startup_gate_satisfied; + + switch (snapshot_.state) + { + case StorageRuntimeState::WaitingStartupGate: + if (snapshot_.startup_gate_satisfied) + { + return beginOperation(StorageOperation::Hydrate, true); + } + return {}; + + case StorageRuntimeState::Ready: + snapshot_.state = StorageRuntimeState::WaitingIdle; + [[fallthrough]]; + + case StorageRuntimeState::WaitingIdle: + if (demand.compaction_pending && is_sleeping && !saver_active) + { + if (idle_since_ms_ == 0U) + { + idle_since_ms_ = now_ms; + return {}; + } + if (static_cast(now_ms - idle_since_ms_) < + kIdleStableMs) + { + return {}; + } + return beginOperation(StorageOperation::Compact, true); + } + if (demand.persistence_pending) + { + idle_since_ms_ = 0U; + return beginOperation(StorageOperation::Persist, true); + } + if (demand.compaction_pending) + { + idle_since_ms_ = 0U; + return {}; + } + if (!is_sleeping || saver_active) + { + idle_since_ms_ = 0U; + return {}; + } + if (idle_since_ms_ == 0U) + { + idle_since_ms_ = now_ms; + return {}; + } + if (static_cast(now_ms - idle_since_ms_) < + kIdleStableMs) + { + return {}; + } + return {}; + + case StorageRuntimeState::Backoff: + if (!deadlineReached(now_ms, snapshot_.retry_due_ms)) + { + return {}; + } + return beginOperation(retry_operation_, false); + + case StorageRuntimeState::Hydrating: + return stepOperation(StorageOperation::Hydrate); + + case StorageRuntimeState::Persisting: + return stepOperation(StorageOperation::Persist); + + case StorageRuntimeState::Compacting: + return stepOperation(StorageOperation::Compact); + + case StorageRuntimeState::Dormant: + case StorageRuntimeState::Done: + default: + return {}; + } + } + + void complete(const StorageOperationResult& result, uint32_t now_ms) + { + if (result.generation != snapshot_.generation || + result.operation != snapshot_.active_operation) + { + return; + } + + if (result.kind == StorageOperationResultKind::InProgress) + { + return; + } + + if (result.completed()) + { + snapshot_.retry_attempt = 0U; + snapshot_.retry_due_ms = 0U; + idle_since_ms_ = 0U; + if (result.operation == StorageOperation::Hydrate) + { + snapshot_.state = StorageRuntimeState::Ready; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + hydration_ready_generation_ = snapshot_.generation; + } + else if (result.operation == StorageOperation::Persist) + { + snapshot_.state = StorageRuntimeState::Ready; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + } + else if (result.operation == StorageOperation::Compact) + { + snapshot_.state = StorageRuntimeState::Ready; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + } + return; + } + + if (result.kind == StorageOperationResultKind::StaleGeneration) + { + return; + } + + if (result.kind == StorageOperationResultKind::Cancelled) + { + snapshot_.state = StorageRuntimeState::Done; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + snapshot_.retry_due_ms = 0U; + return; + } + + retry_operation_ = result.operation; + snapshot_.state = StorageRuntimeState::Backoff; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = result.operation; + snapshot_.retry_attempt = static_cast( + std::min(snapshot_.retry_attempt + 1U, 255U)); + if (retry_policy_.maximum_attempts != 0U && + snapshot_.retry_attempt >= retry_policy_.maximum_attempts) + { + snapshot_.state = StorageRuntimeState::Done; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + snapshot_.retry_due_ms = 0U; + return; + } + snapshot_.retry_due_ms = + now_ms + retry_policy_.delayForAttempt(snapshot_.retry_attempt); + } + + StorageRuntimeSnapshot snapshot() const + { + return snapshot_; + } + + void stop() + { + snapshot_.state = StorageRuntimeState::Done; + snapshot_.active_operation = StorageOperation::None; + snapshot_.pending_operation = StorageOperation::None; + snapshot_.retry_due_ms = 0U; + retry_operation_ = StorageOperation::None; + idle_since_ms_ = 0U; + hydration_ready_generation_ = 0U; + } + + StorageOperationGeneration takeHydrationReadyGeneration() + { + const StorageOperationGeneration generation = + hydration_ready_generation_; + hydration_ready_generation_ = 0U; + return generation; + } + + private: + static constexpr uint32_t kIdleStableMs = 1500U; + + static bool deadlineReached(uint32_t now_ms, uint32_t deadline_ms) + { + return static_cast(now_ms - deadline_ms) >= 0; + } + + StorageMaintenanceCommand beginOperation(StorageOperation operation, + bool new_generation) + { + if (new_generation) + { + ++snapshot_.generation; + if (snapshot_.generation == 0U) + { + snapshot_.generation = 1U; + } + } + snapshot_.state = operation == StorageOperation::Hydrate + ? StorageRuntimeState::Hydrating + : operation == StorageOperation::Persist + ? StorageRuntimeState::Persisting + : StorageRuntimeState::Compacting; + snapshot_.active_operation = operation; + snapshot_.pending_operation = operation; + snapshot_.retry_due_ms = 0U; + return {StorageMaintenanceCommandKind::Begin, + operation, + snapshot_.generation}; + } + + StorageMaintenanceCommand stepOperation(StorageOperation operation) const + { + return {StorageMaintenanceCommandKind::Step, + operation, + snapshot_.generation}; + } + + StorageRetryPolicy retry_policy_{}; + StorageRuntimeSnapshot snapshot_{}; + StorageOperation retry_operation_ = StorageOperation::Hydrate; + StorageOperationGeneration hydration_ready_generation_ = 0U; + uint32_t idle_since_ms_ = 0U; +}; + +} // namespace platform::esp::common::storage diff --git a/platform/esp/common/tests/test_storage_maintenance_state_machine.cpp b/platform/esp/common/tests/test_storage_maintenance_state_machine.cpp new file mode 100644 index 00000000..a02d6287 --- /dev/null +++ b/platform/esp/common/tests/test_storage_maintenance_state_machine.cpp @@ -0,0 +1,172 @@ +#include "platform/esp/common/storage/storage_maintenance_state_machine.h" + +#include + +using namespace platform::esp::common::storage; + +int main() +{ + StorageMaintenanceStateMachine machine; + + StorageMaintenanceCommand command = + machine.arm(100U, StorageStartupGate::DisplayTransaction, false); + assert(command.kind == StorageMaintenanceCommandKind::None); + assert(machine.snapshot().state == + StorageRuntimeState::WaitingStartupGate); + assert(machine.snapshot().pending_operation == StorageOperation::Hydrate); + assert(isInitialHydrationPending(machine.snapshot())); + + command = machine.tick(101U, false, false, false); + assert(command.kind == StorageMaintenanceCommandKind::None); + + command = machine.tick(102U, false, false, true); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Hydrate); + assert(command.generation == 1U); + assert(machine.snapshot().active_operation == StorageOperation::Hydrate); + assert(isInitialHydrationPending(machine.snapshot())); + + machine.complete(StorageOperationResult::failure( + StorageOperationResultKind::StaleGeneration, + StorageOperation::Hydrate, + command.generation + 1U), + 103U); + assert(machine.snapshot().state == StorageRuntimeState::Hydrating); + + machine.complete(StorageOperationResult::inProgressResult( + StorageOperation::Hydrate, + command.generation), + 104U); + command = machine.tick(105U, false, false, true); + assert(command.kind == StorageMaintenanceCommandKind::Step); + assert(command.generation == 1U); + + machine.complete(StorageOperationResult::failure( + StorageOperationResultKind::IoError, + StorageOperation::Hydrate, + command.generation), + 200U); + assert(machine.snapshot().state == StorageRuntimeState::Backoff); + assert(machine.snapshot().pending_operation == StorageOperation::Hydrate); + assert(machine.snapshot().retry_attempt == 1U); + assert(machine.snapshot().retry_due_ms == 2200U); + assert(isInitialHydrationPending(machine.snapshot())); + + command = machine.tick(2199U, false, false, true); + assert(command.kind == StorageMaintenanceCommandKind::None); + command = machine.tick(2200U, false, false, true); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.generation == 1U); + + machine.complete(StorageOperationResult::completedResult( + StorageOperation::Hydrate, + command.generation), + 2300U); + assert(machine.snapshot().state == StorageRuntimeState::Ready); + assert(machine.snapshot().active_operation == StorageOperation::None); + assert(machine.snapshot().pending_operation == StorageOperation::None); + assert(!isInitialHydrationPending(machine.snapshot())); + assert(machine.takeHydrationReadyGeneration() == 1U); + assert(machine.takeHydrationReadyGeneration() == 0U); + + StorageMaintenanceDemand demand{}; + demand.compaction_pending = true; + command = machine.tick(2301U, true, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::None); + command = machine.tick(3801U, true, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Compact); + assert(command.generation == 2U); + + machine.complete(StorageOperationResult::completedResult( + StorageOperation::Compact, + command.generation), + 3900U); + assert(machine.snapshot().state == StorageRuntimeState::Ready); + assert(machine.tick(10000U, true, false, true).kind == + StorageMaintenanceCommandKind::None); + + demand = {}; + demand.persistence_pending = true; + command = machine.tick(10001U, false, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Persist); + machine.complete(StorageOperationResult::completedResult( + StorageOperation::Persist, + command.generation), + 10002U); + assert(machine.snapshot().state == StorageRuntimeState::Ready); + + demand.compaction_pending = true; + command = machine.tick(10003U, false, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Persist); + machine.complete(StorageOperationResult::completedResult( + StorageOperation::Persist, + command.generation), + 10004U); + command = machine.tick(10005U, true, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::None); + command = machine.tick(11505U, true, false, true, demand); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Compact); + + const StorageRetryPolicy retry_policy{}; + assert(retry_policy.delayForAttempt(1U) == 2000U); + assert(retry_policy.delayForAttempt(2U) == 4000U); + assert(retry_policy.delayForAttempt(6U) == 60000U); + + StorageMaintenanceStateMachine immediate_machine; + command = immediate_machine.arm( + 0U, + StorageStartupGate::Immediate, + true); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.generation == 1U); + assert(isInitialHydrationPending(immediate_machine.snapshot())); + + StorageRetryPolicy bounded_retry{}; + bounded_retry.maximum_attempts = 2U; + StorageMaintenanceStateMachine bounded_machine(bounded_retry); + command = bounded_machine.arm( + 0U, + StorageStartupGate::Immediate, + true); + bounded_machine.complete( + StorageOperationResult::failure(StorageOperationResultKind::IoError, + StorageOperation::Hydrate, + command.generation), + 10U); + command = bounded_machine.tick(2010U, false, false, true); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + bounded_machine.complete( + StorageOperationResult::failure(StorageOperationResultKind::IoError, + StorageOperation::Hydrate, + command.generation), + 2020U); + assert(bounded_machine.snapshot().state == StorageRuntimeState::Done); + assert(!isInitialHydrationPending(bounded_machine.snapshot())); + + StorageMaintenanceStateMachine cancelled_machine; + command = cancelled_machine.arm( + 0U, + StorageStartupGate::Immediate, + true); + cancelled_machine.complete( + StorageOperationResult::failure(StorageOperationResultKind::Cancelled, + StorageOperation::Hydrate, + command.generation), + 1U); + assert(cancelled_machine.snapshot().state == StorageRuntimeState::Done); + assert(cancelled_machine.snapshot().pending_operation == + StorageOperation::None); + assert(!isInitialHydrationPending(cancelled_machine.snapshot())); + command = cancelled_machine.arm( + 2U, + StorageStartupGate::Immediate, + true); + assert(command.kind == StorageMaintenanceCommandKind::Begin); + assert(command.operation == StorageOperation::Hydrate); + assert(command.generation == 1U); + return 0; +} diff --git a/platform/esp/idf_common/include/platform/esp/idf_common/storage_runtime.h b/platform/esp/idf_common/include/platform/esp/idf_common/storage_runtime.h index 620231e3..b630338c 100644 --- a/platform/esp/idf_common/include/platform/esp/idf_common/storage_runtime.h +++ b/platform/esp/idf_common/include/platform/esp/idf_common/storage_runtime.h @@ -3,13 +3,17 @@ namespace chat { class SdStore; -} +class MeshPeerDirectoryCore; +} // namespace chat namespace platform::esp::idf_common::storage { -void start_deferred_storage(chat::SdStore* store); +void start_deferred_storage(chat::SdStore* store, + chat::MeshPeerDirectoryCore* peer_directory); void tick_deferred_storage(); +void stop_deferred_storage(); +bool hydration_active(); bool consume_hydration_ready(); } // namespace platform::esp::idf_common::storage diff --git a/platform/esp/idf_common/src/platform_ui_device_runtime.cpp b/platform/esp/idf_common/src/platform_ui_device_runtime.cpp index 15f5aba4..92b5322a 100644 --- a/platform/esp/idf_common/src/platform_ui_device_runtime.cpp +++ b/platform/esp/idf_common/src/platform_ui_device_runtime.cpp @@ -26,7 +26,8 @@ namespace platform::ui::device namespace { -uint8_t s_brightness_level = DEVICE_MAX_BRIGHTNESS_LEVEL; +constexpr uint8_t kIdfMaxBrightnessLevel = 16U; +uint8_t s_brightness_level = kIdfMaxBrightnessLevel; constexpr ::time_t kMinValidEpochSeconds = 1577836800; // 2020-01-01 UTC bool is_valid_epoch(::time_t value) @@ -142,7 +143,7 @@ uint8_t screen_brightness() uint8_t screen_brightness_max() { - return DEVICE_MAX_BRIGHTNESS_LEVEL; + return kIdfMaxBrightnessLevel; } void set_screen_brightness(uint8_t level) @@ -150,10 +151,11 @@ void set_screen_brightness(uint8_t level) const uint8_t max_level = screen_brightness_max(); const uint8_t clamped = level > max_level ? max_level : level; s_brightness_level = clamped; - const int percent = (DEVICE_MAX_BRIGHTNESS_LEVEL <= 0) + const int percent = (kIdfMaxBrightnessLevel == 0U) ? 100 : static_cast((static_cast(clamped) * 100U) / - static_cast(DEVICE_MAX_BRIGHTNESS_LEVEL)); + static_cast( + kIdfMaxBrightnessLevel)); (void)platform::esp::idf_common::bsp_runtime::set_display_brightness(percent); } @@ -169,7 +171,7 @@ uint8_t keyboard_backlight() uint8_t keyboard_backlight_max() { - return DEVICE_MAX_BRIGHTNESS_LEVEL; + return kIdfMaxBrightnessLevel; } void set_keyboard_backlight(uint8_t level) @@ -180,7 +182,8 @@ void set_keyboard_backlight(uint8_t level) { return; } - const uint8_t clamped = level > DEVICE_MAX_BRIGHTNESS_LEVEL ? DEVICE_MAX_BRIGHTNESS_LEVEL : level; + const uint8_t clamped = + level > kIdfMaxBrightnessLevel ? kIdfMaxBrightnessLevel : level; board.keyboardSetBrightness(clamped); #else (void)level; diff --git a/platform/esp/idf_common/src/platform_ui_reticulum_directory_runtime.cpp b/platform/esp/idf_common/src/platform_ui_reticulum_directory_runtime.cpp index 2f4df816..a190d01d 100644 --- a/platform/esp/idf_common/src/platform_ui_reticulum_directory_runtime.cpp +++ b/platform/esp/idf_common/src/platform_ui_reticulum_directory_runtime.cpp @@ -473,6 +473,16 @@ void set_mesh_peer_failure(Status& out, "Mesh peer directory storage unavailable", kLxmfAddressesPath); break; + case chat::MeshPeerDirectoryStatusCode::Busy: + set_status(out, + "Mesh peer directory busy", + kLxmfAddressesPath); + break; + case chat::MeshPeerDirectoryStatusCode::DeviceUnavailable: + set_status(out, + "Mesh peer directory device unavailable", + kLxmfAddressesPath); + break; case chat::MeshPeerDirectoryStatusCode::IoError: set_status(out, "Cannot access mesh peer directory", kLxmfAddressesPath); break; diff --git a/platform/esp/idf_common/src/platform_ui_reticulum_group_config_runtime.cpp b/platform/esp/idf_common/src/platform_ui_reticulum_group_config_runtime.cpp index 73c011ef..a23decb7 100644 --- a/platform/esp/idf_common/src/platform_ui_reticulum_group_config_runtime.cpp +++ b/platform/esp/idf_common/src/platform_ui_reticulum_group_config_runtime.cpp @@ -17,6 +17,9 @@ constexpr const char* kConfigDir = "/trailmate/reticulum"; constexpr const char* kConfigPath = "/trailmate/reticulum/groups.tsv"; constexpr const char* kConfigTempPath = "/trailmate/reticulum/groups.tmp"; constexpr std::size_t kMaxConfigBytes = 2048; +chat::ReticulumGroupDestinationConfig + s_pending_groups[chat::kReticulumGroupDestinationMaxCount] = {}; +bool s_pending = false; void copy_text(char* out, std::size_t out_len, const char* text) { @@ -268,6 +271,67 @@ Status load(chat::ReticulumGroupDestinationConfig* groups, std::size_t group_cou return out; } +Status submit(const chat::ReticulumGroupDestinationConfig* groups, + std::size_t group_count) +{ + Status out{}; + out.supported = true; + out.sd_present = + ::platform::esp::idf_common::bsp_runtime::ensure_sdcard_ready(); + if (!groups || group_count == 0 || + group_count > chat::kReticulumGroupDestinationMaxCount) + { + set_status(out, "Group storage unavailable", kConfigPath); + return out; + } + if (!out.sd_present) + { + set_status(out, "SD card required", kConfigPath); + return out; + } + + std::memcpy(s_pending_groups, + groups, + group_count * sizeof(chat::ReticulumGroupDestinationConfig)); + for (std::size_t index = group_count; + index < chat::kReticulumGroupDestinationMaxCount; + ++index) + { + s_pending_groups[index] = chat::ReticulumGroupDestinationConfig{}; + } + s_pending = true; + out.queued = true; + set_status(out, "Reticulum groups save queued", kConfigPath); + return out; +} + +Status flushPending() +{ + if (!s_pending) + { + Status out{}; + out.supported = true; + out.sd_present = + ::platform::esp::idf_common::bsp_runtime::ensure_sdcard_ready(); + out.saved = true; + set_status(out, "Reticulum groups idle", kConfigPath); + return out; + } + + const Status out = save(s_pending_groups, + chat::kReticulumGroupDestinationMaxCount); + if (out.saved) + { + s_pending = false; + } + return out; +} + +bool hasPending() +{ + return s_pending; +} + Status save(const chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count) { Status out{}; diff --git a/platform/esp/idf_common/src/screen_sleep.cpp b/platform/esp/idf_common/src/screen_sleep.cpp index 0ba20041..0599935d 100644 --- a/platform/esp/idf_common/src/screen_sleep.cpp +++ b/platform/esp/idf_common/src/screen_sleep.cpp @@ -7,6 +7,7 @@ #include +#include "board/BoardBase.h" #include "esp_log.h" #include "esp_timer.h" #include "freertos/FreeRTOS.h" @@ -43,13 +44,14 @@ constexpr const char* kSettingsNs = "settings"; constexpr const char* kScreenTimeoutKey = "screen_timeout"; constexpr std::uint32_t kQueueDepth = 32; constexpr std::uint32_t kTaskPeriodMs = 100; +constexpr std::uint8_t kIdfMaxBrightnessLevel = 16U; ScreenSleepHooks s_hooks{}; StateMachine s_machine{}; SemaphoreHandle_t s_state_mutex = nullptr; QueueHandle_t s_event_queue = nullptr; TaskHandle_t s_task = nullptr; -std::uint8_t s_saved_screen_brightness = DEVICE_MAX_BRIGHTNESS_LEVEL; +std::uint8_t s_saved_screen_brightness = kIdfMaxBrightnessLevel; std::uint32_t now_ms() { diff --git a/platform/esp/idf_common/src/storage_runtime.cpp b/platform/esp/idf_common/src/storage_runtime.cpp index 98721611..e21f91be 100644 --- a/platform/esp/idf_common/src/storage_runtime.cpp +++ b/platform/esp/idf_common/src/storage_runtime.cpp @@ -1,14 +1,18 @@ #include "platform/esp/idf_common/storage_runtime.h" +#include "chat/infra/mesh_peer_directory_core.h" #include "esp_log.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "platform/esp/arduino_common/chat/infra/store/sd_store.h" +#include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/boards/board_runtime.h" #include "platform/esp/common/memory_budget.h" +#include "platform/esp/common/storage/storage_maintenance_owner.h" #include "platform/ui/screen_runtime.h" -#include #include +#include namespace platform::esp::idf_common::storage { @@ -16,183 +20,631 @@ namespace { constexpr const char* kTag = "idf-storage"; -// ESP-IDF's task API uses bytes for both the requested stack depth and the -// high-water mark. Keep this explicit so the Arduino and IDF storage workers -// cannot regress to the vanilla-FreeRTOS words convention. constexpr uint32_t kStackBytes = 8U * 1024U; constexpr std::size_t kInternalReservation = kStackBytes; constexpr std::size_t kInternalFloor = 40U * 1024U; -constexpr uint32_t kRetryBaseMs = 2000U; -constexpr uint32_t kRetryMaxMs = 60000U; -constexpr uint32_t kIdleStableMs = 1500U; +constexpr std::size_t kMaxPeerPayloadBytes = 768U * 1024U; -enum class Mode : uint8_t +using Owner = platform::esp::common::storage::StorageMaintenanceOwner; +using OwnerConfig = + platform::esp::common::storage::StorageMaintenanceOwnerConfig; +using Adapter = platform::esp::common::storage::ISemanticStorageAdapter; +using Operation = platform::esp::common::storage::StorageOperation; +using OperationGeneration = + platform::esp::common::storage::StorageOperationGeneration; +using Result = platform::esp::common::storage::StorageOperationResult; +using ResultKind = + platform::esp::common::storage::StorageOperationResultKind; + +class PsramBlobSink final : public chat::IMeshPeerDirectoryBlobSink { - Hydrate, - Compact, + public: + PsramBlobSink(uint8_t*& out, std::size_t& out_size) + : out_(out), out_size_(out_size) + { + } + + bool begin(std::size_t expected_size) override + { + release(); + if (expected_size > kMaxPeerPayloadBytes) + { + return false; + } + if (expected_size == 0U) + { + return true; + } + out_ = static_cast( + heap_caps_malloc(expected_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (!out_) + { + return false; + } + capacity_ = expected_size; + return true; + } + + bool write(const uint8_t* data, std::size_t len) override + { + if (len == 0U) + { + return true; + } + if (!data) + { + return false; + } + if (out_size_ > capacity_ || len > capacity_ - out_size_) + { + return false; + } + std::memcpy(out_ + out_size_, data, len); + out_size_ += len; + return true; + } + + bool finish() override { return out_size_ == capacity_; } + + private: + void release() + { + if (out_) + { + heap_caps_free(out_); + out_ = nullptr; + } + out_size_ = 0U; + capacity_ = 0U; + } + + uint8_t*& out_; + std::size_t& out_size_; + std::size_t capacity_ = 0U; +}; + +class PsramPayload final +{ + public: + ~PsramPayload() { release(); } + + bool allocate(std::size_t size) + { + release(); + if (size == 0U || size > kMaxPeerPayloadBytes) + { + return false; + } + data_ = static_cast( + heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (!data_) + { + return false; + } + size_ = size; + return true; + } + + void release() + { + if (data_) + { + heap_caps_free(data_); + data_ = nullptr; + } + size_ = 0U; + } + + uint8_t* data() { return data_; } + const uint8_t* data() const { return data_; } + std::size_t size() const { return size_; } + bool empty() const { return data_ == nullptr || size_ == 0U; } + + private: + uint8_t* data_ = nullptr; + std::size_t size_ = 0U; }; chat::SdStore* s_store = nullptr; -TaskHandle_t s_task = nullptr; -Mode s_mode = Mode::Hydrate; -uint32_t s_retry_due_ms = 0U; -uint8_t s_retry_attempt = 0U; -uint32_t s_idle_since_ms = 0U; -bool s_armed = false; -bool s_ready_event = false; -bool s_maintenance_pending = false; +chat::MeshPeerDirectoryCore* s_peer_directory = nullptr; +Owner s_owner{}; -void worker(void*); +Result makeResult(Operation operation, + OperationGeneration generation, + bool ok) +{ + if (ok) + { + return Result::completedResult(operation, generation); + } + if (!platform::esp::arduino_common::storage::sd_card_ready()) + { + return Result::failure(ResultKind::DeviceUnavailable, + operation, + generation); + } + return Result::failure(ResultKind::IoError, operation, generation); +} -uint32_t now_ms() +class SdMaintenanceAdapter final : public Adapter +{ + public: + SdMaintenanceAdapter(chat::SdStore*& store, + chat::MeshPeerDirectoryCore*& peer_directory) + : store_(store), peer_directory_(peer_directory) + { + } + + Result begin(Operation operation, OperationGeneration generation) override + { + if (operation == Operation::Hydrate) + { + return beginHydration(generation); + } + if (operation == Operation::Persist) + { + return beginPeerPersistence(generation); + } + return execute(operation, generation); + } + + Result step(Operation operation, + OperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& + budget) override + { + if (operation == Operation::Hydrate) + { + return stepHydration(generation, budget); + } + if (operation == Operation::Persist) + { + return stepPeerPersistence(generation); + } + if (!store_) + { + return Result::failure(ResultKind::DeviceUnavailable, + operation, + generation); + } + return store_->stepMaintenance(operation, generation, budget); + } + + void cancelAtStepBoundary(Operation operation, + OperationGeneration generation) override + { + if (store_) + { + store_->cancelMaintenance(operation, generation); + } + if (operation == Operation::Persist) + { + peer_payload_.release(); + peer_payload_generation_ = 0U; + peer_payload_revision_ = 0U; + } + if (operation == Operation::Hydrate) + { + releasePeerHydrationPayload(); + peer_hydration_generation_ = 0U; + peer_hydration_store_in_progress_ = false; + peer_hydration_pending_ = false; + } + } + + private: + Result beginHydration(OperationGeneration generation) + { + const bool resume = + peer_hydration_generation_ == generation && + (peer_hydration_store_in_progress_ || peer_hydration_pending_); + if (resume) + { + if (!peer_hydration_store_in_progress_) + { + return Result::inProgressResult(Operation::Hydrate, + generation); + } + if (!store_) + { + return Result::failure(ResultKind::DeviceUnavailable, + Operation::Hydrate, + generation); + } + const Result store_result = + store_->beginMaintenance(Operation::Hydrate, generation); + if (!store_result.inProgress() && !store_result.completed()) + { + if (!store_result.retryable()) + { + releasePeerHydrationPayload(); + peer_hydration_pending_ = false; + peer_hydration_store_in_progress_ = false; + } + return store_result; + } + peer_hydration_store_in_progress_ = + store_result.inProgress(); + if (!peer_hydration_store_in_progress_ && + !peer_hydration_pending_) + { + return Result::completedResult(Operation::Hydrate, + generation); + } + return Result::inProgressResult(Operation::Hydrate, generation); + } + + releasePeerHydrationPayload(); + peer_hydration_generation_ = generation; + peer_hydration_store_in_progress_ = false; + peer_hydration_pending_ = false; + + if (peer_directory_) + { + PsramBlobSink sink(peer_hydration_payload_, + peer_hydration_payload_size_); + const chat::MeshPeerDirectoryBlobLoadResult loaded = + peer_directory_->streamPersistenceBlob(sink); + if (loaded == chat::MeshPeerDirectoryBlobLoadResult::Unavailable) + { + releasePeerHydrationPayload(); + return Result::failure(ResultKind::DeviceUnavailable, + Operation::Hydrate, + generation); + } + if (loaded == chat::MeshPeerDirectoryBlobLoadResult::IoError) + { + releasePeerHydrationPayload(); + return Result::failure(ResultKind::IoError, + Operation::Hydrate, + generation); + } + if (loaded == chat::MeshPeerDirectoryBlobLoadResult::Loaded && + peer_hydration_payload_size_ != 0U) + { + peer_hydration_pending_ = true; + } + } + + if (!store_) + { + if (peer_hydration_pending_) + { + return Result::inProgressResult(Operation::Hydrate, + generation); + } + return Result::completedResult(Operation::Hydrate, generation); + } + + const Result store_result = + store_->beginMaintenance(Operation::Hydrate, generation); + if (!store_result.inProgress() && !store_result.completed()) + { + releasePeerHydrationPayload(); + peer_hydration_pending_ = false; + return store_result; + } + peer_hydration_store_in_progress_ = store_result.inProgress(); + if (!peer_hydration_store_in_progress_ && !peer_hydration_pending_) + { + return Result::completedResult(Operation::Hydrate, generation); + } + return Result::inProgressResult(Operation::Hydrate, generation); + } + + Result stepHydration( + OperationGeneration generation, + const platform::esp::common::storage::StorageOperationBudget& budget) + { + if (peer_hydration_generation_ != generation) + { + return Result::failure(ResultKind::StaleGeneration, + Operation::Hydrate, + generation); + } + + if (peer_hydration_store_in_progress_) + { + if (!store_) + { + return Result::failure(ResultKind::DeviceUnavailable, + Operation::Hydrate, + generation); + } + const Result store_result = + store_->stepMaintenance(Operation::Hydrate, generation, budget); + if (!store_result.inProgress()) + { + if (!store_result.completed()) + { + if (!store_result.retryable()) + { + releasePeerHydrationPayload(); + peer_hydration_pending_ = false; + peer_hydration_store_in_progress_ = false; + } + return store_result; + } + peer_hydration_store_in_progress_ = false; + } + else + { + return store_result; + } + } + + if (peer_hydration_pending_) + { + if (!peer_directory_) + { + return Result::failure(ResultKind::DeviceUnavailable, + Operation::Hydrate, + generation); + } + const chat::MeshPeerDirectoryStatus status = + peer_directory_->hydratePersistenceBlob( + peer_hydration_payload_, + peer_hydration_payload_size_); + releasePeerHydrationPayload(); + peer_hydration_pending_ = false; + if (!status.succeeded()) + { + return Result::failure(ResultKind::IoError, + Operation::Hydrate, + generation); + } + } + + return Result::completedResult(Operation::Hydrate, generation); + } + + Result execute(Operation operation, OperationGeneration generation) + { + if (!store_) + { + return Result::failure(ResultKind::DeviceUnavailable, + operation, + generation); + } + + if (operation != Operation::Hydrate && + operation != Operation::Compact) + { + return Result::failure(ResultKind::Cancelled, + operation, + generation); + } + return store_->beginMaintenance(operation, generation); + } + + Result beginPeerPersistence(OperationGeneration generation) + { + if (!peer_directory_) + { + return Result::failure(ResultKind::DeviceUnavailable, + Operation::Persist, + generation); + } + if (!peer_directory_->persistencePending()) + { + return Result::completedResult(Operation::Persist, generation); + } + + const std::size_t snapshot_size = + peer_directory_->persistenceSnapshotSize(); + if (snapshot_size == 0U || snapshot_size > kMaxPeerPayloadBytes) + { + return Result::failure(ResultKind::IoError, + Operation::Persist, + generation); + } + + if (!peer_payload_.allocate(snapshot_size)) + { + return Result::failure(ResultKind::RetryLater, + Operation::Persist, + generation); + } + uint32_t revision = 0U; + if (!peer_directory_->encodePersistenceSnapshot(peer_payload_.data(), + peer_payload_.size(), + &revision)) + { + peer_payload_.release(); + return Result::failure(ResultKind::RetryLater, + Operation::Persist, + generation); + } + peer_payload_generation_ = generation; + peer_payload_revision_ = revision; + return Result::inProgressResult(Operation::Persist, generation); + } + + Result stepPeerPersistence(OperationGeneration generation) + { + if (!peer_directory_ || peer_payload_generation_ != generation || + peer_payload_.empty()) + { + return Result::failure(ResultKind::StaleGeneration, + Operation::Persist, + generation); + } + + if (peer_directory_->persistenceRevision() != + peer_payload_revision_) + { + peer_payload_.release(); + return Result::failure(ResultKind::RetryLater, + Operation::Persist, + generation); + } + + const bool saved = peer_directory_->persistEncodedSnapshot( + peer_payload_.data(), peer_payload_.size(), peer_payload_revision_); + peer_payload_.release(); + peer_payload_generation_ = 0U; + peer_payload_revision_ = 0U; + if (saved) + { + return Result::completedResult(Operation::Persist, generation); + } + return makeResult(Operation::Persist, generation, false); + } + + chat::SdStore*& store_; + chat::MeshPeerDirectoryCore*& peer_directory_; + PsramPayload peer_payload_{}; + OperationGeneration peer_payload_generation_ = 0U; + uint32_t peer_payload_revision_ = 0U; + uint8_t* peer_hydration_payload_ = nullptr; + std::size_t peer_hydration_payload_size_ = 0U; + OperationGeneration peer_hydration_generation_ = 0U; + bool peer_hydration_store_in_progress_ = false; + bool peer_hydration_pending_ = false; + + void releasePeerHydrationPayload() + { + if (peer_hydration_payload_) + { + heap_caps_free(peer_hydration_payload_); + peer_hydration_payload_ = nullptr; + } + peer_hydration_payload_size_ = 0U; + } +}; + +SdMaintenanceAdapter s_adapter(s_store, s_peer_directory); + +bool admitOwner(void*) +{ + return platform::esp::common::memory::admit("idf_storage_owner", + kInternalReservation, + 0, + 0, + kInternalFloor, + 0); +} + +uint32_t ownerNow(void*) { return static_cast(xTaskGetTickCount() * portTICK_PERIOD_MS); } -uint32_t retry_delay_ms() +const char* operationName(Operation operation) { - const uint8_t shift = std::min(s_retry_attempt, 5U); - return std::min(kRetryBaseMs << shift, kRetryMaxMs); + if (operation == Operation::Hydrate) + { + return "hydrate"; + } + if (operation == Operation::Persist) + { + return "persist"; + } + return "compact"; } -bool deadline_reached(uint32_t now, uint32_t deadline) +void ownerStarted(void*, + Operation operation, + OperationGeneration generation) { - return deadline == 0U || static_cast(now - deadline) >= 0; -} - -void schedule_retry(const char* reason) -{ - ++s_retry_attempt; - const uint32_t delay = retry_delay_ms(); - s_retry_due_ms = now_ms() + delay; - ESP_LOGW(kTag, - "retry scheduled reason=%s attempt=%u retry_in_ms=%lu", - reason, - static_cast(s_retry_attempt), - static_cast(delay)); -} - -bool start_worker(Mode mode) -{ - if (!s_armed || s_task) - { - return false; - } - if (!platform::esp::common::memory::admit("idf_storage_worker", - kInternalReservation, - 0, - 0, - kInternalFloor, - 0)) - { - schedule_retry("low_internal"); - return false; - } - s_mode = mode; - if (xTaskCreatePinnedToCore(&worker, - mode == Mode::Hydrate ? "idf_store_hydrate" - : "idf_store_compact", - kStackBytes, - nullptr, - 1, - &s_task, - 1) != pdPASS) - { - s_task = nullptr; - schedule_retry("task_create_failed"); - return false; - } - s_retry_due_ms = 0U; - ESP_LOGI(kTag, "worker started mode=%s", mode == Mode::Hydrate ? "hydrate" : "compact"); - return true; -} - -void worker(void*) -{ - const Mode mode = s_mode; - bool ok = mode == Mode::Hydrate ? s_store->hydrateFromStorage() - : s_store->compactDeferred(); - if (ok && mode == Mode::Hydrate) - { - s_ready_event = true; - s_maintenance_pending = true; - s_retry_attempt = 0U; - s_idle_since_ms = 0U; - } - else if (ok) - { - s_maintenance_pending = false; - s_retry_attempt = 0U; - } - if (!ok) - { - schedule_retry(mode == Mode::Hydrate ? "hydrate_failed" - : "compact_failed"); - } ESP_LOGI(kTag, - "worker finished mode=%s ok=%d stack_free=%u", - mode == Mode::Hydrate ? "hydrate" : "compact", - ok ? 1 : 0, - static_cast(uxTaskGetStackHighWaterMark(nullptr))); - s_task = nullptr; - vTaskDelete(nullptr); + "owner begin mode=%s generation=%lu", + operationName(operation), + static_cast(generation)); +} + +void ownerFinished(void*, + Operation operation, + OperationGeneration generation, + ResultKind result, + uint32_t elapsed_ms, + uint32_t stack_free_bytes) +{ + ESP_LOGI(kTag, + "owner end mode=%s generation=%lu ok=%u result=%u " + "elapsed_ms=%lu stack_free_bytes=%lu", + operationName(operation), + static_cast(generation), + (result == ResultKind::Completed || + result == ResultKind::InProgress) + ? 1U + : 0U, + static_cast(result), + static_cast(elapsed_ms), + static_cast(stack_free_bytes)); +} + +bool startupGateSatisfied() +{ + return platform::esp::boards::storageStartupGateSatisfied(); } } // namespace -void start_deferred_storage(chat::SdStore* store) +void start_deferred_storage(chat::SdStore* store, + chat::MeshPeerDirectoryCore* peer_directory) { - if (!store || s_armed) + if ((!store && !peer_directory) || s_owner.isArmed()) { return; } + s_store = store; - s_armed = true; - (void)start_worker(Mode::Hydrate); + s_peer_directory = peer_directory; + + OwnerConfig config{}; + config.task_name = "idf_storage_owner"; + config.stack_bytes = kStackBytes; + config.priority = 1; + config.core = 1; + config.startup_gate = + platform::esp::boards::storageCapabilities() + .requiresDisplayTransactionGate() + ? platform::esp::common::storage::StorageStartupGate:: + DisplayTransaction + : platform::esp::common::storage::StorageStartupGate::Immediate; + config.context = nullptr; + config.adapter = &s_adapter; + config.admit = &admitOwner; + config.now = &ownerNow; + config.on_started = &ownerStarted; + config.on_finished = &ownerFinished; + s_owner.configure(config); + (void)s_owner.arm(ownerNow(nullptr), startupGateSatisfied()); } void tick_deferred_storage() { - if (!s_armed || s_task) + if (!s_owner.isArmed()) { return; } - const uint32_t now = now_ms(); - if (!deadline_reached(now, s_retry_due_ms)) - { - return; - } - if (s_maintenance_pending) - { - if (!platform::ui::screen::is_sleeping() || - platform::ui::screen::is_saver_active()) - { - s_idle_since_ms = 0U; - return; - } - if (s_idle_since_ms == 0U) - { - s_idle_since_ms = now; - return; - } - if (now - s_idle_since_ms < kIdleStableMs) - { - return; - } - (void)start_worker(Mode::Compact); - return; - } - (void)start_worker(Mode::Hydrate); + + platform::esp::common::storage::StorageMaintenanceDemand demand{}; + demand.persistence_pending = + s_peer_directory && s_peer_directory->persistencePending(); + demand.compaction_pending = + s_store && s_store->compactionPending(); + (void)s_owner.submitTick(ownerNow(nullptr), + platform::ui::screen::is_sleeping(), + platform::ui::screen::is_saver_active(), + startupGateSatisfied(), + demand); +} + +void stop_deferred_storage() +{ + (void)s_owner.requestStop(); +} + +bool hydration_active() +{ + return s_owner.hydrationActive(); } bool consume_hydration_ready() { - if (!s_ready_event) - { - return false; - } - s_ready_event = false; - return true; + return s_owner.consumeHydrationReady(); } } // namespace platform::esp::idf_common::storage diff --git a/platform/linux/common/include/app/linux_app_services.h b/platform/linux/common/include/app/linux_app_services.h index c784504b..63ed32af 100644 --- a/platform/linux/common/include/app/linux_app_services.h +++ b/platform/linux/common/include/app/linux_app_services.h @@ -1,9 +1,12 @@ #pragma once #include +#include #include #include "app/app_config.h" +#include "app/app_config_edit.h" +#include "app/config_persistence_runtime.h" #include "chat/domain/chat_types.h" #include "platform/linux/runtime_mode.h" @@ -76,6 +79,7 @@ class LinuxAppServices final [[nodiscard]] const ::app::AppConfig& config() const; [[nodiscard]] ::app::AppConfig& getConfig(); [[nodiscard]] const ::app::AppConfig& getConfig() const; + [[nodiscard]] ::app::AppConfigEdit beginConfigEdit(); void saveConfig(); void saveConfig(::app::AppConfigChangeSet changes); void applyMeshConfig(); @@ -129,9 +133,15 @@ class LinuxAppServices final void seedDefaultIdentity(); void syncLocalIdentity(); void ensureServicesReady(); + void flushConfigPersistence(uint32_t now_ms); + bool writePersistedConfig(const ::app::AppConfig& config); + static void commitConfigEdit(void* context, + ::app::AppConfigChangeSet changes); + static void cancelConfigEdit(void* context); LinuxAppServicesOptions options_{}; ::app::AppConfig config_{}; + ::app::ConfigPersistenceRuntime config_persistence_runtime_{}; std::unique_ptr impl_; UiEventDispatcher ui_event_dispatcher_ = nullptr; void* ui_event_context_ = nullptr; diff --git a/platform/linux/common/src/app/linux_app_services.cpp b/platform/linux/common/src/app/linux_app_services.cpp index 7457f016..b3085502 100644 --- a/platform/linux/common/src/app/linux_app_services.cpp +++ b/platform/linux/common/src/app/linux_app_services.cpp @@ -1755,6 +1755,7 @@ bool LinuxAppServices::initialize() loadPersistedConfig(); seedDefaultIdentity(); + config_persistence_runtime_.initialize(config_); (void)::sys::EventBus::init(); ensureServicesReady(); syncLocalIdentity(); @@ -1795,7 +1796,15 @@ bool LinuxAppServices::dispatchUiEvent(::sys::Event* event) void LinuxAppServices::tick(std::size_t max_events) { - if (!initialized_) return; + if (!initialized_) + { + return; + } + flushConfigPersistence(::sys::millis_now()); + if (::platform::ui::reticulum_groups::hasPending()) + { + (void)::platform::ui::reticulum_groups::flushPending(); + } updateCoreServices(); tickEventRuntime(); dispatchPendingEvents(max_events); @@ -1821,23 +1830,69 @@ const ::app::AppConfig& LinuxAppServices::getConfig() const return config_; } +::app::AppConfigEdit LinuxAppServices::beginConfigEdit() +{ + return ::app::AppConfigEdit(&config_, + this, + &LinuxAppServices::commitConfigEdit, + &LinuxAppServices::cancelConfigEdit); +} + void LinuxAppServices::saveConfig() { - ::app::AppConfig persisted = config_; - persisted.mesh_protocol = ::chat::infra::normalizeMeshProtocol(persisted.mesh_protocol); - const PersistedConfigBlob blob{.magic = kConfigBlobMagic, - .version = kConfigBlobVersion, - .config = persisted}; - (void)::platform::ui::settings_store::put_blob( - kConfigNamespace, kConfigBlobKey, &blob, sizeof(blob)); + saveConfig(::app::AppConfigChangeSet::allPersisted()); } void LinuxAppServices::saveConfig(::app::AppConfigChangeSet changes) { // The Linux blob backend has no section-level writer yet. Keep the // scoped request explicit while preserving the complete blob contract. - (void)changes; - saveConfig(); + config_.mesh_protocol = + ::chat::infra::normalizeMeshProtocol(config_.mesh_protocol); + const uint32_t now_ms = ::sys::millis_now(); + const auto submission = config_persistence_runtime_.submit( + config_, changes, now_ms, ::app::ConfigPersistenceUrgency::Debounced); + (void)submission; +} + +void LinuxAppServices::flushConfigPersistence(uint32_t now_ms) +{ + ::app::ConfigPersistenceWork work{}; + if (!config_persistence_runtime_.takeDue(now_ms, work) || + work.snapshot == nullptr) + { + return; + } + + const bool ok = writePersistedConfig(*work.snapshot); + config_persistence_runtime_.complete( + work.generation, + ok ? ::app::ConfigPersistenceResultKind::Completed + : ::app::ConfigPersistenceResultKind::IoError, + ::sys::millis_now()); +} + +bool LinuxAppServices::writePersistedConfig(const ::app::AppConfig& config) +{ + const PersistedConfigBlob blob{.magic = kConfigBlobMagic, + .version = kConfigBlobVersion, + .config = config}; + return ::platform::ui::settings_store::put_blob( + kConfigNamespace, kConfigBlobKey, &blob, sizeof(blob)); +} + +void LinuxAppServices::commitConfigEdit(void* context, + ::app::AppConfigChangeSet changes) +{ + auto* self = static_cast(context); + if (self != nullptr) + { + self->saveConfig(changes); + } +} + +void LinuxAppServices::cancelConfigEdit(void*) +{ } void LinuxAppServices::applyMeshConfig() diff --git a/platform/linux/common/src/platform/ui/reticulum_directory_runtime.cpp b/platform/linux/common/src/platform/ui/reticulum_directory_runtime.cpp index 918965fd..004273bf 100644 --- a/platform/linux/common/src/platform/ui/reticulum_directory_runtime.cpp +++ b/platform/linux/common/src/platform/ui/reticulum_directory_runtime.cpp @@ -272,6 +272,16 @@ void set_mesh_peer_directory_failure(Status& out, "Mesh peer directory storage unavailable", kMeshPeerDirectoryLogicalPath); break; + case chat::MeshPeerDirectoryStatusCode::Busy: + set_status(out, + "Mesh peer directory busy", + kMeshPeerDirectoryLogicalPath); + break; + case chat::MeshPeerDirectoryStatusCode::DeviceUnavailable: + set_status(out, + "Mesh peer directory device unavailable", + kMeshPeerDirectoryLogicalPath); + break; case chat::MeshPeerDirectoryStatusCode::IoError: set_status(out, "Cannot access mesh peer directory", diff --git a/platform/linux/common/src/platform/ui/reticulum_group_config_runtime.cpp b/platform/linux/common/src/platform/ui/reticulum_group_config_runtime.cpp index 34ab3de7..eaa38f82 100644 --- a/platform/linux/common/src/platform/ui/reticulum_group_config_runtime.cpp +++ b/platform/linux/common/src/platform/ui/reticulum_group_config_runtime.cpp @@ -17,6 +17,9 @@ namespace constexpr const char* kRelativePath = "trailmate/reticulum/groups.tsv"; constexpr const char* kLogicalPath = "/trailmate/reticulum/groups.tsv"; constexpr std::size_t kMaxConfigBytes = 2048; +chat::ReticulumGroupDestinationConfig + s_pending_groups[chat::kReticulumGroupDestinationMaxCount] = {}; +bool s_pending = false; void copy_text(char* out, std::size_t out_len, const char* text) { @@ -197,6 +200,57 @@ Status load(chat::ReticulumGroupDestinationConfig* groups, std::size_t group_cou return out; } +Status submit(const chat::ReticulumGroupDestinationConfig* groups, + std::size_t group_count) +{ + Status out{}; + out.supported = true; + if (!groups || group_count == 0 || + group_count > chat::kReticulumGroupDestinationMaxCount) + { + set_status(out, "Group storage unavailable", kLogicalPath); + return out; + } + std::memcpy(s_pending_groups, + groups, + group_count * sizeof(chat::ReticulumGroupDestinationConfig)); + for (std::size_t index = group_count; + index < chat::kReticulumGroupDestinationMaxCount; + ++index) + { + s_pending_groups[index] = chat::ReticulumGroupDestinationConfig{}; + } + s_pending = true; + out.queued = true; + set_status(out, "Reticulum groups save queued", kLogicalPath); + return out; +} + +Status flushPending() +{ + if (!s_pending) + { + Status out{}; + out.supported = true; + out.saved = true; + set_status(out, "Reticulum groups idle", kLogicalPath); + return out; + } + + const Status out = save(s_pending_groups, + chat::kReticulumGroupDestinationMaxCount); + if (out.saved) + { + s_pending = false; + } + return out; +} + +bool hasPending() +{ + return s_pending; +} + Status save(const chat::ReticulumGroupDestinationConfig* groups, std::size_t group_count) { Status out{}; diff --git a/platform/linux/uconsole/src/uconsole_map_workspace_model.cpp b/platform/linux/uconsole/src/uconsole_map_workspace_model.cpp index 57170c8f..b6e5009a 100644 --- a/platform/linux/uconsole/src/uconsole_map_workspace_model.cpp +++ b/platform/linux/uconsole/src/uconsole_map_workspace_model.cpp @@ -498,8 +498,13 @@ UConsoleMapWorkspaceModel::ensureContourTiles( void UConsoleMapWorkspaceModel::setSource( ::platform::linux_runtime::MapBaseSource source_value) { - services_.config().map_source = static_cast(source_value); - services_.saveConfig(); + auto edit = services_.beginConfigEdit(); + if (!edit) + { + return; + } + edit.config().map_source = static_cast(source_value); + edit.commit(::app::AppConfigChangeSet::map()); } void UConsoleMapWorkspaceModel::setZoom(int zoom) @@ -516,8 +521,13 @@ void UConsoleMapWorkspaceModel::setShowMqttNodes(bool enabled) void UConsoleMapWorkspaceModel::setContourEnabled(bool enabled) { - services_.config().map_contour_enabled = enabled; - services_.saveConfig(); + auto edit = services_.beginConfigEdit(); + if (!edit) + { + return; + } + edit.config().map_contour_enabled = enabled; + edit.commit(::app::AppConfigChangeSet::map()); } void UConsoleMapWorkspaceModel::setContourUltraFineEnabled(bool enabled) diff --git a/platform/nrf52/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp b/platform/nrf52/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp index a712ae7d..15cab7fd 100644 --- a/platform/nrf52/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp +++ b/platform/nrf52/arduino_common/src/platform_ui_reticulum_group_config_runtime.cpp @@ -61,4 +61,20 @@ Status save(const chat::ReticulumGroupDestinationConfig*, std::size_t) return out; } +Status submit(const chat::ReticulumGroupDestinationConfig* groups, + std::size_t group_count) +{ + return save(groups, group_count); +} + +Status flushPending() +{ + return save(nullptr, 0); +} + +bool hasPending() +{ + return false; +} + } // namespace platform::ui::reticulum_groups