From a74a09da56a2dfd1f412f45f2ce69b63d3384220 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Thu, 24 Sep 2026 17:13:22 -0700 Subject: [PATCH] Complete ESP32 partition expander and adaptive LoRa OTA pacing --- docs/_javascript/firmware_picker.js | 20 +- docs/cli_commands.md | 6 +- docs/lora_ota_automation.md | 4 +- docs/ota_user_guide.md | 9 +- examples/companion_radio/MyMesh.cpp | 5 +- examples/esp32_partition_migrator/main.cpp | 323 +++++++++++++++++- examples/partition_expander/main.cpp | 211 ++++++++++++ scripts/build_esp32_partition_migration.py | 4 +- scripts/package_esp32_partition_migration.py | 66 +++- src/Dispatcher.cpp | 4 +- src/Dispatcher.h | 1 + src/Mesh.cpp | 10 + src/Mesh.h | 1 + src/helpers/CommonCLI.cpp | 27 +- src/helpers/ota/OtaCli.cpp | 14 +- src/helpers/ota/OtaManager.cpp | 67 +++- src/helpers/ota/OtaManager.h | 18 +- src/helpers/ota/OtaSpeedConfig.cpp | 23 +- src/helpers/ota/OtaSpeedConfig.h | 7 +- test/test_companion_ota_config.py | 4 + test/test_esp32_partition_migration_recipe.py | 38 ++- test/test_firmware_picker.js | 15 + test/test_heltec_v4_partition_migrator.py | 37 +- test/test_ota/test_ota_core.cpp | 48 +++ test/test_ota_speed.py | 20 +- tools/lora_ota/lora_ota.py | 6 +- tools/lora_ota/rak3401_mota_chain.py | 2 +- tools/lora_ota/rebuild_rak3401_bundle.py | 2 +- tools/lora_ota/test_lora_ota.py | 9 +- tools/mota/motalib.py | 2 +- tools/mota/test_mota.py | 2 + variants/heltec_v4/platformio.ini | 27 +- variants/xiao_s3_wio/platformio.ini | 20 ++ 33 files changed, 986 insertions(+), 66 deletions(-) create mode 100644 examples/partition_expander/main.cpp diff --git a/docs/_javascript/firmware_picker.js b/docs/_javascript/firmware_picker.js index 70725dc3..12947512 100644 --- a/docs/_javascript/firmware_picker.js +++ b/docs/_javascript/firmware_picker.js @@ -37,6 +37,7 @@ repeater: "Repeater", room: "Room Server", sensor: "Sensor / telemetry", + "partition-expander": "Partition Expander", terminal: "Terminal Chat", kiss: "KISS modem", other: "Other", @@ -264,6 +265,10 @@ role: "sensor", pattern: /_sensor(?=$|[_-])/i, }, + { + role: "partition-expander", + pattern: /_partition_expander(?=$|[_-])/i, + }, { role: "terminal", pattern: /_terminal_chat(?=$|[_-])/i, @@ -365,7 +370,8 @@ (parts.role === "companion" && mode === "full") ? "full" : "standard"; - const explicitOta = lowerTarget.includes("lora_ota") + const explicitOta = parts.role === "partition-expander" || + lowerTarget.includes("lora_ota") ? "lora-receiver" : parts.role === "companion" && mode === "full" ? "lora-source" @@ -979,7 +985,8 @@ } function optionSort(field, a, b) { - const roleOrder = ["companion", "repeater", "room", "sensor", "terminal", "kiss", "other"]; + const roleOrder = ["companion", "repeater", "room", "sensor", + "partition-expander", "terminal", "kiss", "other"]; const loggingOrder = ["none", "usb", "wifi", "both"]; const otaOrder = ["none", "lora-receiver", "lora-source"]; const featureOrder = ["standard", "full"]; @@ -1025,6 +1032,15 @@ "Verify that the hardware name and every displayed variant match the physical board.", "Back up configuration, keys, and radio settings before changing roles or profiles.", ]; + if (profile.role === "partition-expander") { + return common.concat( + kind === "zip" + ? ["This ZIP is a staged ESP32 partition-migration package, not an nRF52 Serial DFU package. Follow its included README for the existing Wi-Fi or LoRa update route."] + : ["Install this temporary bridge only through the existing OTA route documented for this exact board and source layout; do not flash it as permanent node firmware."], + ["Keep power stable while the partition table changes. Confirm that private identity restoration succeeded before transferring the exact Full application over LoRa or Wi-Fi.", + "The Partition Expander is a temporary update receiver, not a repeater, client, or sensor firmware."] + ); + } const memory = profile.controls; const selectedAsset = canonicalAsset(profile.files, kind); if (memory && memory.memoryNote && memory.memorySource && diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 1826cdf4..b5f49e64 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -196,8 +196,10 @@ set ota.speed <0.05..3> `1` preserves the current OTA timing. Values below `1` slow OTA traffic; values above `1` shorten adjustable delays. For example, `0.5` doubles those delays and `3` divides them by three. Decimal values such as `.05` work. -`ota config speed ` is an equivalent setter, and `ota config` shows -the value. The setting is saved and can be changed during a transfer. +`ota config speed ` is an equivalent setter. `get ota.speed` and +`ota config` show both the saved value and the effective `packet` pacing +factor. Automatic loss response may make packet spacing slower without +changing the saved setting. The setting can be changed during a transfer. Apply it on the source, destination, and relays. It controls all LoRa OTA packet types on both profiles, relay timing, discovery, and OTA adverts. diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index 591f3868..a6a03dbd 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -472,10 +472,10 @@ required `ver` command instead of entering an operator continuation loop. The default TempRadio tuple is: ```text -909.950,250,5,5,120 +909.5,500,5,5,120 ``` -The test default is 250 kHz bandwidth, SF5, and CR5. The frequency is only a +The lab test default is 909.5 MHz, 500 kHz bandwidth, SF5, and CR5. The frequency is only a North American example: choose a legal frequency supported by every participating radio and appropriate to your location. Older radios that do not support SF5 require a complete replacement tuple passed with `--temp-radio`. diff --git a/docs/ota_user_guide.md b/docs/ota_user_guide.md index d061aeff..5f982ddc 100644 --- a/docs/ota_user_guide.md +++ b/docs/ota_user_guide.md @@ -430,8 +430,13 @@ already received; queue congestion alone does not trigger legacy fallback. Retry timing includes the primary channel when an RX-only temporary radio2 uses crossover to send requests there. -`ota config speed 0.5` is an equivalent setter. `ota config speed` and -`ota speed` read the current factor; the `ota config` summary also includes it. +`ota config speed 0.5` is an equivalent setter. `get ota.speed`, +`ota config speed`, and `ota speed` show both the saved `ota.speed` setting and +the effective `packet` pace. The latter is the slower of the saved speed and +the sender's automatic loss-responsive packet-spacing factor; it does not +replace the saved setting or change non-packet OTA timers. The `ota config` +summary also shows both values. If no OTA workspace is active, the automatic +factor starts at 1, so `packet` reflects the saved setting up to 1x. The setting is available on OTA-enabled repeaters, rooms, sensors, Companions, and seeder-only builds. Its separate `/ota_speed` settings file preserves the existing preference layouts and does not need an active OTA workspace. diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index d2db88b8..59b97daa 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -2544,7 +2544,10 @@ bool MyMesh::handleLocalControlCommand(const char* command, char* reply, if (handleTxRoutingCommand(command, reply, reply_size)) return true; #if COMPANION_FEATURE_OTA_CLI - if (mesh::ota::handleSpeedCommand(command, reply, reply_size)) return true; + const auto* ota_context = mesh::ota::ota_context_if_active(); + const float adaptive_ota_pace = ota_context + ? ota_context->manager.adaptivePacketSpeed() : mesh::ota::OTA_SPEED_DEFAULT; + if (mesh::ota::handleSpeedCommand(command, reply, reply_size, adaptive_ota_pace)) return true; #endif if (handleCompanionBluetoothCommand(command, reply, reply_size)) return true; if (handleCompanionWirelessCommand(command, reply, reply_size)) return true; diff --git a/examples/esp32_partition_migrator/main.cpp b/examples/esp32_partition_migrator/main.cpp index f886bb96..43934e0e 100644 --- a/examples/esp32_partition_migrator/main.cpp +++ b/examples/esp32_partition_migrator/main.cpp @@ -9,7 +9,8 @@ #include #include #include -#if defined(MESHCORE_MIGRATION_RESUME_OTA) +#include +#if defined(MESHCORE_MIGRATION_RESUME_OTA) || defined(MOTA_MIGRATION_TARGET_ID) #include #include #endif @@ -17,6 +18,10 @@ #include #include +#if defined(MOTA_MIGRATION_TARGET_ID) +String partitionExpanderStatus(); +#endif + namespace migration = mesh::esp32_partition_migration; namespace { @@ -37,8 +42,48 @@ constexpr char kMigrationNvsNamespace[] = "mesh-pt-migrate"; constexpr char kMigrationIdentityKey[] = "identity"; // NVS key names are limited to 15 characters. constexpr char kMigrationIdentityPendingKey[] = "id-pending"; +constexpr char kMigrationConfigKey[] = "cfg-meta"; +constexpr char kMigrationConfigPendingKey[] = "cfg-pending"; +constexpr char kMigrationConfigRestoredKey[] = "cfg-restored"; +#if defined(MOTA_MIGRATION_TARGET_ID) +constexpr char kExpanderHandoffKey[] = "full-handoff"; +#endif +constexpr uint32_t kConfigStageMagic = 0x43464731U; +constexpr size_t kMaxConfigFileBytes = 16384; +// The common preferences hold node name, password and primary radio settings. +// Other entries preserve access policy, extra radio profiles and the credentials +// needed to remain manageable after the SPIFFS partition moves. +struct ConfigFile { const char* path; const char* nvs_key; }; +constexpr ConfigFile kConfigFiles[] = { + {"/com_prefs", "cfg00"}, {"/node_prefs", "cfg01"}, + {"/s_contacts", "cfg02"}, {"/s_login_replay", "cfg03"}, + {"/regions2", "cfg04"}, {"/radio_profiles", "cfg05"}, + {"/mqtt_prefs", "cfg06"}, {"/mqtt.json", "cfg07"}, + {"/ota_config", "cfg08"}, {"/flood_filter", "cfg09"}, + {"/flood_filter_bl", "cfg10"}, {"/flood_ch_scope", "cfg11"}, + {"/flood_ch_req", "cfg12"}, {"/flood_grp_mod", "cfg13"}, + {"/clock_sync", "cfg14"}, {"/display_prefs", "cfg15"}, + {"/telemetry_tx", "cfg16"}, {"/data_tx", "cfg17"}, + {"/com_prefs.bak", "cfg18"}, {"/radio_profiles.bak", "cfg19"}, + {"/s_contacts.bak", "cfg20"}, {"/mqtt_prefs.bak", "cfg21"}, + {"/regions2.bak", "cfg22"}, {"/prefs.json", "cfg23"}, + {"/management", "cfg24"}, {"/management.bak", "cfg25"}, + {"/s_login_replay.bak", "cfg26"}, {"/ota_config.bak", "cfg27"}, + {"/ota_speed", "cfg28"}, {"/flood_ch_block", "cfg29"}, + {"/bsec_state.bin", "cfg30"}, {"/display_prefs.bak", "cfg31"}, +}; +constexpr size_t kConfigFileCount = sizeof(kConfigFiles) / sizeof(kConfigFiles[0]); +static_assert(kConfigFileCount <= 32, "config presence mask is 32 bits"); +struct ConfigStage { + uint32_t magic; + uint32_t present; + uint32_t sizes[kConfigFileCount]; + uint32_t crcs[kConfigFileCount]; +}; #if defined(MESHCORE_MIGRATION_RESUME_OTA) constexpr char kMigrationResumeSlotKey[] = "resume-slot"; +#endif +#if defined(MESHCORE_MIGRATION_RESUME_OTA) || defined(MOTA_MIGRATION_TARGET_ID) constexpr size_t kEndfBytes = 56; constexpr char kBridgeImageMarker[] = "MeshCore ESP32 partition migration bridge image"; // Earlier bridge packages predate the dedicated marker but contain this page title. @@ -344,7 +389,197 @@ bool restoreStagedIdentity() { return true; } -#if defined(MESHCORE_MIGRATION_RESUME_OTA) +#if !defined(MESHCORE_MIGRATION_RESUME_OTA) +bool verifyExpandedIdentityFile() { + if (!SPIFFS.begin(false)) { + strcpy(status_text, "Refused: expanded identity filesystem is unavailable"); + return false; + } + File identity = SPIFFS.open("/identity/_main.id", "r"); + const bool available = identity && identity.size() >= kIdentityFileBytes + && identity.read(copy_buffer, kIdentityFileBytes) == kIdentityFileBytes; + if (identity) identity.close(); + SPIFFS.end(); + if (!available) strcpy(status_text, "Refused: restored private key is unavailable"); + return available; +} +#endif + +bool readConfigStage(Preferences& nvs, ConfigStage& stage) { + if (nvs.getBytes(kMigrationConfigKey, &stage, sizeof(stage)) != sizeof(stage) + || stage.magic != kConfigStageMagic) { + strcpy(status_text, "Refused: staged configuration manifest is incomplete"); + return false; + } + for (size_t i = 0; i < kConfigFileCount; ++i) { + const bool present = (stage.present & (1UL << i)) != 0; + if ((!present && stage.sizes[i] != 0) + || (present && (stage.sizes[i] == 0 + || stage.sizes[i] > kMaxConfigFileBytes + || nvs.getBytesLength(kConfigFiles[i].nvs_key) != stage.sizes[i]))) { + snprintf(status_text, sizeof(status_text), + "Refused: staged %s is incomplete", kConfigFiles[i].path); + return false; + } + } + return true; +} + +bool verifyStagedConfigBlob(Preferences& nvs, const ConfigStage& stage, + size_t i, uint8_t* buffer) { + const size_t size = stage.sizes[i]; + return nvs.getBytes(kConfigFiles[i].nvs_key, buffer, size) == size + && (~crc32(buffer, size)) == stage.crcs[i]; +} + +void clearConfigStageBlobs(Preferences& nvs) { + nvs.remove(kMigrationConfigKey); + for (const ConfigFile& file : kConfigFiles) nvs.remove(file.nvs_key); +} + +bool validateStagedConfig() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, true)) return true; + const bool pending = nvs.getBool(kMigrationConfigPendingKey, false); + if (!pending) { nvs.end(); return true; } + ConfigStage stage = {}; + bool valid = readConfigStage(nvs, stage); + for (size_t i = 0; valid && i < kConfigFileCount; ++i) { + if (!(stage.present & (1UL << i))) continue; + uint8_t* buffer = static_cast(heap_caps_malloc( + stage.sizes[i], MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); + valid = buffer && verifyStagedConfigBlob(nvs, stage, i, buffer); + heap_caps_free(buffer); + if (!valid) snprintf(status_text, sizeof(status_text), + "Refused: staged %s failed verification", kConfigFiles[i].path); + } + nvs.end(); + return valid; +} + +bool stageLegacyConfig() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, false)) { + strcpy(status_text, "Could not open NVS configuration staging"); + return false; + } + if (nvs.getBool(kMigrationConfigPendingKey, false)) { + nvs.end(); + return validateStagedConfig(); + } + // A power failure before the commit marker can leave partial blobs. They + // are never authoritative and would otherwise consume the small NVS area. + if (!nvs.putBool(kMigrationConfigRestoredKey, false)) { + nvs.end(); + strcpy(status_text, "Could not reset configuration handoff record"); + return false; + } + clearConfigStageBlobs(nvs); + if (!SPIFFS.begin(false)) { + nvs.end(); + strcpy(status_text, "Could not mount legacy configuration filesystem"); + return false; + } + ConfigStage stage = {}; + stage.magic = kConfigStageMagic; + bool saved = true; + for (size_t i = 0; saved && i < kConfigFileCount; ++i) { + if (!SPIFFS.exists(kConfigFiles[i].path)) continue; + File source = SPIFFS.open(kConfigFiles[i].path, "r"); + const size_t size = source ? source.size() : 0; + if (size == 0 || size > kMaxConfigFileBytes) { + snprintf(status_text, sizeof(status_text), + "Refused: %s cannot fit NVS staging", kConfigFiles[i].path); + saved = false; + } else { + uint8_t* buffer = static_cast(heap_caps_malloc( + size, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); + saved = buffer && source.read(buffer, size) == size; + if (saved) { + stage.sizes[i] = size; + stage.crcs[i] = ~crc32(buffer, size); + saved = nvs.putBytes(kConfigFiles[i].nvs_key, buffer, size) == size + && verifyStagedConfigBlob(nvs, stage, i, buffer); + } + heap_caps_free(buffer); + if (!saved) snprintf(status_text, sizeof(status_text), + "Could not stage and verify %s", kConfigFiles[i].path); + } + if (source) source.close(); + if (saved) stage.present |= (1UL << i); + delay(1); + } + SPIFFS.end(); + if (saved) { + saved = nvs.putBytes(kMigrationConfigKey, &stage, sizeof(stage)) + == sizeof(stage) + && nvs.putBool(kMigrationConfigPendingKey, true); + if (!saved) strcpy(status_text, "Could not finish NVS configuration staging"); + } + if (!saved) clearConfigStageBlobs(nvs); + nvs.end(); + return saved && validateStagedConfig(); +} + +bool restoreStagedConfig() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, false)) return true; + if (!nvs.getBool(kMigrationConfigPendingKey, false)) { + // Cleanup may have been interrupted after the restored files were + // committed. This does not require another SPIFFS write. + clearConfigStageBlobs(nvs); + nvs.end(); + return true; + } + ConfigStage stage = {}; + if (!readConfigStage(nvs, stage) || !validateStagedConfig()) { + nvs.end(); + return false; + } + if (!SPIFFS.begin(false)) { + nvs.end(); + strcpy(status_text, "Could not mount expanded configuration filesystem"); + return false; + } + bool restored = true; + for (size_t i = 0; restored && i < kConfigFileCount; ++i) { + if (!(stage.present & (1UL << i))) continue; + const size_t size = stage.sizes[i]; + uint8_t* buffer = static_cast(heap_caps_malloc( + size, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); + restored = buffer && verifyStagedConfigBlob(nvs, stage, i, buffer); + if (restored) { + File destination = SPIFFS.open(kConfigFiles[i].path, "w"); + restored = destination && destination.write(buffer, size) == size; + if (destination) { destination.flush(); destination.close(); } + File verify = SPIFFS.open(kConfigFiles[i].path, "r"); + restored = restored && verify && verify.size() == size + && verify.read(buffer, size) == size + && (~crc32(buffer, size)) == stage.crcs[i]; + if (verify) verify.close(); + } + heap_caps_free(buffer); + if (!restored) snprintf(status_text, sizeof(status_text), + "Could not restore and verify %s", kConfigFiles[i].path); + delay(1); + } + SPIFFS.end(); + if (restored) { + // Record the verified restore before clearing the pending marker. A lost + // pending flag cannot otherwise prove that saved settings survived. + restored = nvs.putBool(kMigrationConfigRestoredKey, true); + if (restored) restored = nvs.remove(kMigrationConfigPendingKey); + if (restored) { + clearConfigStageBlobs(nvs); + } else { + strcpy(status_text, "Configuration restored; NVS commit needs retry"); + } + } + nvs.end(); + return restored; +} + +#if defined(MESHCORE_MIGRATION_RESUME_OTA) || defined(MOTA_MIGRATION_TARGET_ID) uint32_t readLe32(const uint8_t* bytes) { return static_cast(bytes[0]) | (static_cast(bytes[1]) << 8) @@ -467,6 +702,62 @@ bool validOtherLoRaFirmware(const esp_partition_t& bridge, return true; } +#if defined(MOTA_MIGRATION_TARGET_ID) +bool stageExpanderHandoff() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, false)) { + strcpy(status_text, "Could not open Partition Expander handoff record"); + return false; + } + const bool saved = nvs.putUChar(kExpanderHandoffKey, 0xA5) == 1; + nvs.end(); + if (!saved) strcpy(status_text, "Could not save Partition Expander handoff record"); + return saved; +} + +bool hasExpanderHandoff() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, true)) return false; + const bool pending = nvs.getUChar(kExpanderHandoffKey, 0) == 0xA5; + nvs.end(); + return pending; +} + +bool hasVerifiedConfigHandoff() { + Preferences nvs; + if (!nvs.begin(kMigrationNvsNamespace, true)) return false; + const bool restored = nvs.getBool(kMigrationConfigRestoredKey, false); + nvs.end(); + return restored; +} + +bool returnToVerifiedFull() { + const esp_partition_t* running = esp_ota_get_running_partition(); + const esp_partition_t* other = esp_ota_get_next_update_partition(nullptr); + if (!running || !other || running->address == other->address) return false; + OtaImageIdentity bridge; + OtaImageIdentity full; + if (!readVerifiedOtaIdentity(*running, bridge) + || !readVerifiedOtaIdentity(*other, full) + || full.target_id != (uint32_t)MOTA_MIGRATION_TARGET_ID + || memcmp(full.hardware_id, bridge.hardware_id, + sizeof(full.hardware_id)) != 0 + || containsBridgeMarker(*other, full.body_bytes) + != BridgeMarkerResult::Missing) return false; + const esp_err_t selected = esp_ota_set_boot_partition(other); + if (selected != ESP_OK) { + snprintf(status_text, sizeof(status_text), + "Could not return to verified Full image: %s", errName(selected)); + return false; + } + strcpy(status_text, "Already expanded; returning to verified Full image"); + reboot_at = millis() + 1000; + return true; +} +#endif + +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + bool stageResumeSlot(uint8_t slot) { Preferences migration_nvs; const bool opened = migration_nvs.begin(kMigrationNvsNamespace, false); @@ -529,6 +820,7 @@ bool resumeLegacyOtaReceiver(const migration::PartitionGeometry& geometry) { return true; } #endif +#endif // MESHCORE_MIGRATION_RESUME_OTA || MOTA_MIGRATION_TARGET_ID bool publishExpandedPartitionTable(const migration::TargetPlan& plan) { // Preserve ESP-IDF's normal OS flash hooks. In particular, their start/end @@ -632,6 +924,19 @@ void runMigration() { return; } Serial.println("Migration: private key safely staged in NVS"); +#if !defined(MESHCORE_MIGRATION_RESUME_OTA) + if (!stageLegacyConfig()) { + Serial.println(status_text); + return; + } + Serial.println("Migration: ACL, radio profiles and node configuration safely staged in NVS"); +#if defined(MOTA_MIGRATION_TARGET_ID) + if (!stageExpanderHandoff()) { + Serial.println(status_text); + return; + } +#endif +#endif // The Wi-Fi bridge always runs from target app0. The LoRa bridge instead // keeps the old application in the other expanded slot, restores identity, @@ -736,6 +1041,13 @@ void sendHome(AsyncWebServerRequest* request) { page += "

MeshCore Wi-Fi partition migration

"; page += mode; page += "

"; +#if defined(MOTA_MIGRATION_TARGET_ID) + if (expanded_layout_ready && identity_ready) { + page += "

"; + page += partitionExpanderStatus(); + page += "

"; + } +#endif if (expanded_layout_ready && identity_ready) { page += "

The expanded partition layout is active. If the bridge could " "not return to a verified LoRa image, Wi-Fi recovery is available " @@ -809,7 +1121,12 @@ void setup() { if (findPartitions(refs, geometry) && migration::isTargetLayout(flash_bytes, geometry)) { expanded_layout_ready = true; - if (restoreStagedIdentity()) { + if (validateStagedConfig() && restoreStagedIdentity() +#if !defined(MESHCORE_MIGRATION_RESUME_OTA) + && restoreStagedConfig() + && verifyExpandedIdentityFile() +#endif + ) { identity_ready = true; strcpy(status_text, "Expanded layout ready"); #if defined(MESHCORE_MIGRATION_RESUME_OTA) diff --git a/examples/partition_expander/main.cpp b/examples/partition_expander/main.cpp new file mode 100644 index 00000000..86371ef6 --- /dev/null +++ b/examples/partition_expander/main.cpp @@ -0,0 +1,211 @@ +// A self-hosted partition bridge. Reuse the tested Wi-Fi migration engine so +// both legacy OTA placements preserve the running image and node configuration. +// After its restart into expanded app0, this role also receives the final +// Full application over LoRa; the old application need not remain bootable. +#define setup partitionMigrationSetup +#define loop partitionMigrationLoop +#include "../esp32_partition_migrator/main.cpp" +#undef setup +#undef loop + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef MOTA_MIGRATION_TARGET_ID +#error "Partition Expander must pin the exact successor Full target ID" +#endif + +#ifndef LORA_FREQ +#define LORA_FREQ 915.0 +#endif +#ifndef LORA_BW +#define LORA_BW 250.0 +#endif +#ifndef LORA_SF +#define LORA_SF 10 +#endif +#ifndef LORA_CR +#define LORA_CR 5 +#endif + +namespace { + +class ExpanderMesh final : public mesh::Mesh { + public: + ExpanderMesh(mesh::Radio& radio, mesh::MillisecondClock& millis_clock, + mesh::RNG& rng, mesh::RTCClock& rtc, + mesh::PacketManager& packets, mesh::MeshTables& tables) + : mesh::Mesh(radio, millis_clock, rng, rtc, packets, tables) {} + + bool isTempRadioActive() const override { return true; } +}; + +ArduinoMillis uptime_clock; +StdRNG fast_rng; +StaticPoolPacketManager packets(32); +SimpleMeshTables tables; +ExpanderMesh the_mesh(radio_driver, uptime_clock, fast_rng, rtc_clock, packets, tables); +bool lora_ready = false; +bool install_attempted = false; +mesh::RadioProfileParams active_profile; + +// /com_prefs stores these fields at fixed offsets in both stock 1.17.1 and +// keymindCascade. Keep the migration receiver on the user's saved radio1 +// channel so an existing LoRa seeder can reach it without a USB-only retune. +bool loadPrimaryRadio(mesh::RadioProfileParams& profile) { + bool ok = false; + if (SPIFFS.exists("/com_prefs") || SPIFFS.exists("/node_prefs")) { + File prefs = SPIFFS.open(SPIFFS.exists("/com_prefs") + ? "/com_prefs" : "/node_prefs", "r"); + if (prefs && prefs.size() >= 290) { + ok = prefs.seek(72) && prefs.readBytes(reinterpret_cast(&profile.freq), 4) == 4; + ok = ok && prefs.seek(112) && prefs.readBytes(reinterpret_cast(&profile.sf), 1) == 1; + ok = ok && prefs.readBytes(reinterpret_cast(&profile.cr), 1) == 1; + ok = ok && prefs.seek(116) && prefs.readBytes(reinterpret_cast(&profile.bw), 4) == 4; + } + if (prefs) prefs.close(); + } else if (SPIFFS.exists("/prefs.json")) { + File prefs = SPIFFS.open("/prefs.json", "r"); + NodePrefs legacy; + ok = prefs && legacy.loadSerial(prefs); + if (prefs) prefs.close(); + if (ok) { + profile.freq = legacy.freq; + profile.bw = legacy.bw; + profile.sf = legacy.sf; + profile.cr = legacy.cr; + } + } + return ok && profile.freq >= 100.0f && profile.freq <= 2500.0f + && profile.bw > 0.0f && profile.bw <= 500.0f + && profile.sf >= 5 && profile.sf <= 12 + && profile.cr >= 5 && profile.cr <= 8; +} + +void startLoRaReceiver() { + board.begin(); + if (!radio_init()) { + Serial.println("Partition Expander: radio init failed; Wi-Fi recovery remains available"); + return; + } + fast_rng.begin(radio_driver.getRngSeed()); + + if (!SPIFFS.begin(false)) { + Serial.println("Partition Expander: identity filesystem unavailable"); + return; + } + IdentityStore identity(SPIFFS, "/identity"); + if (identity.loadResult("_main", the_mesh.self_id) != IdentityLoadResult::Loaded) { + Serial.println("Partition Expander: private identity unavailable; LoRa disabled"); + return; + } + + mesh::RadioProfileParams profile; + profile.freq = LORA_FREQ; + profile.bw = LORA_BW; + profile.sf = LORA_SF; + profile.cr = LORA_CR; + // HIL recipe: save 909.5 MHz / 500 kHz / SF5 / CR5 as radio1 on + // both bench nodes before migration. Production recovery must still use + // the node's own saved radio1, never a hard-coded lab frequency. + if (!loadPrimaryRadio(profile)) { + Serial.println("Partition Expander: saved radio1 unavailable; using build preset"); + } + if (radio_driver.trySetPrimaryParams(profile, true) + != mesh::RadioParamApplyResult::APPLIED) { + Serial.println("Partition Expander: OTA radio profile refused"); + return; + } + active_profile = profile; + + the_mesh.begin(); + auto& ota = mesh::ota::ota_ctx(); + ota.manager.set_auto_migration_target((uint32_t)MOTA_MIGRATION_TARGET_ID); + ota.manager.set_auto_version_floor(0, false); + ota.manager.set_autofetch(mesh::ota::OtaManager::AUTOFETCH_ANY); + lora_ready = true; + Serial.println("Partition Expander: waiting for exact Full target over LoRa"); +} + +} // namespace + +String partitionExpanderStatus() { + if (reboot_at) return "Returning to the verified Full image"; + if (!lora_ready) return "LoRa receiver unavailable; use Wi-Fi recovery"; + const auto& manager = mesh::ota::ota_ctx().manager; + char detail[160]; + snprintf(detail, sizeof(detail), + "LoRa receiver %.3f MHz / %.1f kHz / SF%u / CR%u; " + "OTA state %u; blocks %lu/%lu; request window %u/%u", + active_profile.freq, active_profile.bw, + (unsigned)active_profile.sf, (unsigned)active_profile.cr, + (unsigned)manager.fetchState(), + (unsigned long)manager.blocksHave(), + (unsigned long)manager.blocksTotal(), + (unsigned)manager.fetchPipelineWidth(), + (unsigned)manager.fetchPipelineCapacity()); + return detail; +} + +void setup() { + partitionMigrationSetup(); + // Never start a LoRa fetch against the 1.25 MiB layout or before the key is + // restored. The migration engine restarts after verifying the new table. + if (!expanded_layout_ready || !identity_ready) return; + // On an already-expanded board the temporary role may have landed next to + // a valid Full image. Return to it instead of needlessly replacing it. + if (returnToVerifiedFull()) return; + if (!hasExpanderHandoff()) { + strcpy(status_text, + "Already expanded; no migration handoff. Wi-Fi recovery available"); + return; + } + if (!hasVerifiedConfigHandoff()) { + strcpy(status_text, + "Refused: saved configuration was not verified; Wi-Fi recovery available"); + return; + } + startLoRaReceiver(); + if (lora_ready) strcpy(status_text, "Expanded layout ready; LoRa receiver active"); +} + +void loop() { + partitionMigrationLoop(); + if (!lora_ready) return; + the_mesh.loop(); + auto& ota = mesh::ota::ota_ctx(); + if (!install_attempted + && ota.manager.fetchState() == mesh::ota::OtaManager::COMPLETE) { + install_attempted = true; + uint8_t manifest_bytes[mesh::ota::MOTA_MFL]; + mesh::ota::MotaManifest manifest; + if (!ota.fetch_store.read(8, manifest_bytes, sizeof(manifest_bytes)) + || !mesh::ota::mota_parse_manifest(manifest_bytes, + sizeof(manifest_bytes), manifest) + || !manifest.is_full() + || manifest.target_id != (uint32_t)MOTA_MIGRATION_TARGET_ID) { + Serial.println("Partition Expander: staged image is not the pinned Full target"); + return; + } + char message[100] = {}; + const bool installed = ota.apply_fetched(message); + Serial.println(message); + if (installed) { + // This role has no CLI reply queue or role-specific reboot scheduler. + // apply_fetched() only arms the ESP32 OTA slot; it does not restart. + Serial.flush(); + delay(200); + mesh::ota::ota_reboot_to_apply(); + } else { + Serial.println("Partition Expander: Full install refused; restart to retry"); + } + } +} diff --git a/scripts/build_esp32_partition_migration.py b/scripts/build_esp32_partition_migration.py index de99b836..ce4f8571 100644 --- a/scripts/build_esp32_partition_migration.py +++ b/scripts/build_esp32_partition_migration.py @@ -47,8 +47,8 @@ def build_steps(boards: list[str], version: str, radio_preset: str, # Full builds can clear .pio/build; every bridge must be built afterwards. built_bridges: set[str] = set() for name in boards: - for key in ("wifi_bridge", "lora_bridge"): - bridge = BOARDS[name][key] + for key in ("wifi_bridge", "lora_bridge", "expander_bridge"): + bridge = BOARDS[name].get(key) if bridge and bridge not in built_bridges: steps.append((["pio", "run", "-e", bridge, "-j", str(jobs)], False)) diff --git a/scripts/package_esp32_partition_migration.py b/scripts/package_esp32_partition_migration.py index 84de6fcd..269d1b8f 100644 --- a/scripts/package_esp32_partition_migration.py +++ b/scripts/package_esp32_partition_migration.py @@ -14,8 +14,9 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tools" / "mota")) from motalib import ( # noqa: E402 - CODEC_FULL, FwIdent, build_container, build_manifest, ensure_endf, - hardware_id_for_env, pack_version, parse_container, parse_endf_ident, + CODEC_FULL, FwIdent, build_container, build_endf, build_manifest, has_endf, + hardware_id_for_env, pack_version, parse_container, parse_endf, + parse_endf_ident, target_id_for_env, verify, ) from check_esp32_app_size import app_partition_size # noqa: E402 @@ -25,6 +26,7 @@ from firmware_memory_manifest import validate_package # noqa: E402 BOARDS = { "heltec-v4": { "target": "heltec_v4_repeater", + "expander_bridge": "heltec_v4_partition_expander", "wifi_bridge": "heltec_v4_partition_migrator", "lora_bridge": "heltec_v4_partition_migrator_lora_repeater", "flash_bytes": 16 * 1024 * 1024, @@ -33,6 +35,7 @@ BOARDS = { }, "xiao-s3-wio": { "target": "Xiao_S3_WIO_repeater", + "expander_bridge": "Xiao_S3_WIO_partition_expander", "wifi_bridge": "xiao_s3_partition_migrator", "lora_bridge": "xiao_s3_partition_migrator_lora_repeater", "flash_bytes": 8 * 1024 * 1024, @@ -312,6 +315,16 @@ def mota_full(image: bytes, identity: FwIdent, block_size: int) -> bytes: return package +def bridge_with_successor_endf(image: bytes, successor: FwIdent) -> bytes: + """Bind a temporary role image to the old node's exact LoRa target. + + PlatformIO can already append EndF for the temporary role. Keeping that + trailer would make the old repeater reject stage one on target mismatch. + """ + body = parse_endf(image)[0] if has_endf(image) else image + return body + build_endf(body, successor) + + def check_esp32_stage(package: bytes, image: bytes, slot_bytes: int) -> None: # Mirror the ESP32 FULL staging geometry: the payload is written at the # start of the inactive app, with metadata in aligned sectors at its end. @@ -324,6 +337,32 @@ def check_esp32_stage(package: bytes, image: bytes, slot_bytes: int) -> None: def readme(board: str, spec: dict, version: str, source: str, has_full_mota: bool = True, full_mota_blocks: int = 0) -> str: + expander_instructions = "" + if spec.get("expander_bridge"): + expander_instructions = f"""## Partition Expander route (automatic LoRa finish) + +For this exact **{spec['target']}** hardware/role, `partition-expander.bin` +is an alternative to `wifi-bridge.bin` in step 1 above. It works from either +legacy OTA slot. It stages and verifies the private identity, saved node name, +radio profiles and ACL before changing the table. After **Expanded layout +ready**, it listens on the saved primary radio profile and automatically +fetches only this package's exact-target Full application. Serve +`full-application.mota` on that profile; do not upload it to the browser. +The bridge's status page reports the receiver profile and block progress. +Keep only the intended Full build for this target on the seeder during the +migration; the bridge pins the hardware/role target, not an individual MID. +If installed on an already-expanded board, it returns to a verified Full image +in the other slot. Without a recorded migration handoff or safe Full image, it +does not fetch or replace an image; the status page explains whether Wi-Fi +recovery is available. Firmware upload is disabled if identity verification +failed. + +`partition-expander.mota` offers the same bridge as a first-stage LoRa update +only if the installed old firmware already supports compatible mOTA and its +target ID matches this repeater. A stock image without mOTA must use its +existing Wi-Fi updater for stage one. Keep a compatible seeder available before +installing the bridge; the bridge does not itself repeat normal traffic. +""" wifi_instructions = f"""## Wi-Fi route 1. On the old node, run `start ota ap` through its authenticated terminal. @@ -414,7 +453,9 @@ Source: `{source}`; firmware version: `{version}`. This package is only for the physical board/role represented by `{spec['target']}` with {spec['flash_bytes'] // 1048576} MiB flash. A legacy Wi-Fi-updatable observer or bridge variant on the same hardware may migrate to this canonical Full target; -its previous feature settings are not preserved. The LoRa route, where present, +the Wi-Fi bridge stages its saved node/radio configuration and ACL when NVS has +enough room, and refuses migration otherwise. The older LoRa handoff bridge +preserves only the private identity. The LoRa route, where present, requires the old image's exact target ID. The current partition table must have stable NVS at 0x9000, OTA metadata at 0xE000, two OTA apps, and SPIFFS with `/identity/_main.id`. @@ -426,13 +467,17 @@ the bridge restores and verifies the identity in expanded SPIFFS; on the 4 MiB LoRa route the new Full firmware does so at first boot. No full-chip erase or USB connection is used. The table sector and inactive app sectors are erased as part of migration. -SPIFFS may be reformatted; other SPIFFS settings can be recreated. +SPIFFS may be reformatted. The Wi-Fi bridge restores verified radio 1 and +radio 2 settings, node name, ACL and login replay state, region keys, and +selected network/OTA settings. Temporary radio sessions and logs are not +preserved. The older LoRa handoff route may reset saved settings. The stock bootloader has no atomic backup partition table: power loss during the table-sector erase/write can still require cable recovery. Do not use this on an inaccessible node without accepting that risk and testing a sacrificial example of the same board first. {wifi_instructions} +{expander_instructions} {lora_instructions} """ @@ -488,6 +533,16 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, "target-partitions.bin": table, "capabilities.json": capability_path.read_bytes(), } + if spec.get("expander_bridge"): + expander_body = (ROOT / ".pio" / "build" / spec["expander_bridge"] / + "firmware.bin").read_bytes() + expander_image = bridge_with_successor_endf(expander_body, bridge_ident) + if len(expander_image) > LEGACY_SLOT_BYTES: + raise ValueError(f"{name}: Partition Expander does not fit the legacy slot") + expander_mota = mota_full(expander_image, bridge_ident, 1024) + check_esp32_stage(expander_mota, expander_image, LEGACY_SLOT_BYTES) + files["partition-expander.bin"] = expander_body + files["partition-expander.mota"] = expander_mota full_mota_blocks = (len(full_image) + 2047) // 2048 if full_mota_blocks <= 4096: candidate_mota = mota_full(full_image, full_ident, 2048) @@ -502,7 +557,7 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, raise ValueError(f"{name}: LoRa Full image exceeds the mOTA block limit") if spec["lora_bridge"]: lora_bridge_body = (ROOT / ".pio" / "build" / spec["lora_bridge"] / "firmware.bin").read_bytes() - lora_bridge, _ = ensure_endf(lora_bridge_body, bridge_ident) + lora_bridge = bridge_with_successor_endf(lora_bridge_body, bridge_ident) if len(lora_bridge) > LEGACY_SLOT_BYTES: raise ValueError(f"{name}: LoRa bridge does not fit the legacy slot") files["lora-bridge.mota"] = mota_full(lora_bridge, bridge_ident, 1024) @@ -525,6 +580,7 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, "lora_bridge_mode": ("slot-b-only-full-identity-recovery" if spec["flash_bytes"] == 4 * 1024 * 1024 else "either-slot-preserve-receiver"), + "partition_expander": spec.get("expander_bridge"), "full_mota_seeder_scratch_bytes": full_mota_blocks * 4, "files": {filename: {"bytes": len(data), "sha256": sha256(data)} for filename, data in files.items()}, diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index 9ea1c7b2..4da03b11 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -849,7 +849,7 @@ void Dispatcher::checkSend() { // override. RX processing and ordinary watchdogs continue in loop(). if (!restoreOutboundTxOverrides()) return; const uint32_t now = _ms->getMillis(); - if (ota_tx_airtime && now - ota_tx_finished_at >= ota::packetQuietTime(ota_tx_airtime, getOtaSpeedFactor())) { + if (ota_tx_airtime && now - ota_tx_finished_at >= ota::packetQuietTime(ota_tx_airtime, getOtaPacketSpeedFactor())) { ota_tx_airtime = 0; } uint32_t next_outbound; @@ -870,7 +870,7 @@ void Dispatcher::checkSend() { return; } if (ota_tx_airtime) { - const uint32_t gap = ota::packetQuietTime(ota_tx_airtime, getOtaSpeedFactor()); + const uint32_t gap = ota::packetQuietTime(ota_tx_airtime, getOtaPacketSpeedFactor()); const uint32_t elapsed = now - ota_tx_finished_at; if (elapsed >= gap) ota_tx_airtime = 0; else if (pending && pending->getPayloadType() == PAYLOAD_TYPE_OTA) { diff --git a/src/Dispatcher.h b/src/Dispatcher.h index cc114d81..2787980f 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -423,6 +423,7 @@ protected: virtual float getAirtimeBudgetFactor() const; virtual float getOtaSpeedFactor() const { return 1.0f; } + virtual float getOtaPacketSpeedFactor() const { return getOtaSpeedFactor(); } virtual int calcRxDelay(float score, uint32_t air_time) const; virtual bool shouldBypassRxDelay(const Packet* packet) { (void)packet; diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 754513c8..6dc9d8a6 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -663,6 +663,16 @@ float Mesh::getOtaSpeedFactor() const { #endif } +float Mesh::getOtaPacketSpeedFactor() const { + const float configured = getOtaSpeedFactor(); +#if defined(ENABLE_OTA) + const float adaptive = ota::ota_ctx().manager.adaptivePacketSpeed(); + return configured < adaptive ? configured : adaptive; +#else + return configured; +#endif +} + uint32_t Mesh::getOtaPacketAirtime() const { const auto* profiles = _radio->profiles(); if (!profiles || !profiles->enabled()) return _radio->getProfileAirtime(0, MAX_TRANS_UNIT); diff --git a/src/Mesh.h b/src/Mesh.h index c302d131..8354a259 100644 --- a/src/Mesh.h +++ b/src/Mesh.h @@ -542,6 +542,7 @@ protected: bool _ota_temp_was_active = false; // detects entry into a temporary-radio window #endif float getOtaSpeedFactor() const override; + float getOtaPacketSpeedFactor() const override; uint32_t getOtaPacketAirtime() const; /** diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index fd262e46..d7671ed1 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -792,6 +792,28 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { } _com_prefs_needs_upgrade = false; #endif + } else if (fs->exists("/prefs.json")) { + // Stock 1.17.1 stored NodePrefs as JSON. A partition migration preserves + // that file byte-for-byte; import it before writing the current binary + // image so node name, passwords and primary radio survive the Full handoff. +#if defined(NRF52_PLATFORM) + File legacy(*fs); + legacy.open("/prefs.json", FILE_O_READ); +#elif defined(STM32_PLATFORM) + File legacy = fs->open("/prefs.json", FILE_O_READ); +#else + File legacy = fs->open("/prefs.json", "r"); +#endif + if (legacy && _prefs->loadSerial(legacy)) { + loaded = true; + is_upgrade = true; +#ifdef WITH_MQTT_BRIDGE + node_prefs_needs_migration = true; +#else + savePrefs(fs); +#endif + } + if (legacy) legacy.close(); } else { // File doesn't exist - set defaults for a fresh install. Dual R1/R2 // scanning keeps the node awake, so only that configuration starts with @@ -2579,7 +2601,10 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re _callbacks->wirelessCommandSource(sender_timestamp))) return; if (_radio_profiles.handle(command, reply, 160, sender_timestamp != 0)) return; #if defined(ENABLE_OTA) - if (mesh::ota::handleSpeedCommand(command, reply, 160)) return; + const auto* ota_context = mesh::ota::ota_context_if_active(); + const float adaptive_ota_pace = ota_context + ? ota_context->manager.adaptivePacketSpeed() : mesh::ota::OTA_SPEED_DEFAULT; + if (mesh::ota::handleSpeedCommand(command, reply, 160, adaptive_ota_pace)) return; #endif if (strncmp(command, "set tempradio ", 14) == 0) { handleCommand(sender_timestamp, command + 4, reply); diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index bcce3995..6134f80d 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -147,7 +147,9 @@ static bool is_cmd(const char* a, const char* names, const char** rest) { static bool handle_dev(const char* d, char* reply, OtaContext& c); bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board) { - if (handleSpeedCommand(command, reply, 160)) return true; + const auto* active = ota_context_if_active(); + const float adaptive_pace = active ? active->manager.adaptivePacketSpeed() : OTA_SPEED_DEFAULT; + if (handleSpeedCommand(command, reply, 160, adaptive_pace)) return true; const char* a = command + 3; if (*a != 0 && *a != ' ') return false; while (*a == ' ') a++; @@ -1002,10 +1004,12 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board strcpy(reply, "ERR unknown OTA config setting"); } else { // show current policy uint8_t af = c.manager.autofetch(); - char speed[16]; formatSpeed(speed, sizeof(speed)); + char speed[16], packet[16]; + formatSpeed(speed, sizeof(speed)); + formatSpeedFactor(effectivePacketPace(c.manager.adaptivePacketSpeed()), packet, sizeof(packet)); #if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) bool cache_ready = c.ensureSdCache(); - snprintf(reply, 160, "ota config: speed=%sx cache=%s/%u autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u", speed, + snprintf(reply, 160, "ota config: speed=%sx packet=%sx cache=%s/%u autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u", speed, packet, cache_ready ? (c.sd_cache.autoCaptureEnabled() ? "on" : "off") : "unavailable", cache_ready ? (unsigned)c.sd_cache.capturedCount() : 0, af == OtaManager::AUTOFETCH_ANY ? "any" : af == OtaManager::AUTOFETCH_SIGNED ? "signed" : "off", @@ -1014,11 +1018,11 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board (unsigned)c.manager.max_hops(), (unsigned)c.allow.count()); #else #if defined(OTA_SEEDER_ONLY) - snprintf(reply, 160, "ota config: speed=%sx mode=seeder-only autofetch=off autoinstall=off checkpoint=%u advert=%umin hops=%u", speed, + snprintf(reply, 160, "ota config: speed=%sx packet=%sx mode=seeder-only autofetch=off autoinstall=off checkpoint=%u advert=%umin hops=%u", speed, packet, (unsigned)c.manager.checkpoint_blocks(), (unsigned)c.manager.advert_mins(), (unsigned)c.manager.max_hops()); #else - snprintf(reply, 160, "ota config: speed=%sx autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u (persisted)", speed, + snprintf(reply, 160, "ota config: speed=%sx packet=%sx autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u (persisted)", speed, packet, af == OtaManager::AUTOFETCH_ANY ? "any" : af == OtaManager::AUTOFETCH_SIGNED ? "signed" : "off", c.autoinstall == OtaContext::AUTOINSTALL_TRUSTED ? "trusted" : "off", (unsigned)c.manager.checkpoint_blocks(), (unsigned)c.manager.advert_mins(), diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index 0bafeb3d..80f05e64 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -94,6 +94,10 @@ void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) { memset(_src_advertised, 0, sizeof(_src_advertised)); _n_src = 0; _n_cat = 0; clearPendingEgress(); + _adaptive_packet_speed = 1.0f; + _pace_clean_blocks = 0; + _pace_has_mid = false; + _pace_has_loss = false; } // ---------------- serve (multi-mota registry) ---------------- @@ -682,6 +686,45 @@ bool OtaManager::loadActiveServeBlock() { return true; } +void OtaManager::noteServedRequestPacing(const uint8_t* mid, uint16_t block, + uint16_t want, uint16_t full_mask) { + if (want != full_mask) { + // A sparse mask is the receiver's explicit evidence that our previous + // DATA burst left holes. One extra packet-airtime gap is much cheaper than + // repeated multi-second request turns on a fast link. + if (!_pace_has_loss || memcmp(_pace_loss_mid, mid, sizeof(_pace_loss_mid)) != 0 + || _pace_last_loss_block != block) { + memcpy(_pace_loss_mid, mid, sizeof(_pace_loss_mid)); + _pace_last_loss_block = block; + _pace_has_loss = true; + if (_adaptive_packet_speed > 0.5f) _adaptive_packet_speed = 0.5f; + else if (_adaptive_packet_speed > 0.25f) { + _adaptive_packet_speed -= 0.1f; + if (_adaptive_packet_speed < 0.25f) _adaptive_packet_speed = 0.25f; + } + } + _pace_clean_blocks = 0; + return; + } + if (!_pace_has_mid || memcmp(_pace_mid, mid, sizeof(_pace_mid)) != 0) { + memcpy(_pace_mid, mid, sizeof(_pace_mid)); + _pace_has_mid = true; + _pace_last_full_block = block; + _pace_clean_blocks = 0; + return; + } + // Count only forward progress; duplicates of a full request are not proof + // that the receiver tolerated the current packet rate. + if (block <= _pace_last_full_block) return; + _pace_last_full_block = block; + if (_adaptive_packet_speed >= 1.0f) return; + if (++_pace_clean_blocks >= 16) { + _adaptive_packet_speed += 0.1f; + if (_adaptive_packet_speed > 1.0f) _adaptive_packet_speed = 1.0f; + _pace_clean_blocks = 0; + } +} + bool OtaManager::handleReq(const uint8_t* m, uint16_t n) { ReqWindowMsg rq; if (!decode_req_window(m, n, rq)) return false; @@ -708,8 +751,11 @@ bool OtaManager::handleReq(const uint8_t* m, uint16_t n) { const uint16_t requested = wire_v2 ? ota_req_v2_fragments(rq.items[i].want_mask) : rq.items[i].want_mask; const uint16_t want = (uint16_t)(requested & valid_mask); - if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want, - wire_v2, allow_deflate, extended_length); + if (want != 0 && queueServeJob(v->m.merkle_root, (uint16_t)idx, want, + wire_v2, allow_deflate, extended_length)) { + accepted = true; + noteServedRequestPacing(v->m.merkle_root, (uint16_t)idx, want, valid_mask); + } } return accepted; } @@ -1223,7 +1269,22 @@ uint32_t OtaManager::fetchRetryTimeoutMs() const { service = (service * _tx_spacing_permille + 999u) / 1000u; service = (service * 5u + 3u) / 4u; // 25% CAD/relay contention allowance service += OTA_FETCH_RETRY_GUARD_MS; - if (service < OTA_FETCH_RETRY_MIN_MS) service = OTA_FETCH_RETRY_MIN_MS; + uint64_t floor = OTA_FETCH_RETRY_MIN_MS; + if (_observed_path_transmissions != 0 && anyWireDataReceived()) { + // A valid fragment proves this source has already answered the current flight. + // Missing-fragment requests need not inherit the five-second allowance for + // an unanswered host/relay request. Four full-packet service intervals cover + // queued DATA/proof/turnaround; measured airtime and the observed path raise + // the floor automatically on slower or relayed links. + uint64_t partial_floor = (uint64_t)_radio_packet_airtime_ms + * _observed_path_transmissions * _tx_spacing_permille * 4u; + partial_floor = (partial_floor + 999u) / 1000u; + partial_floor += OTA_FETCH_RETRY_GUARD_MS; + if (partial_floor < OTA_FETCH_RETRY_PARTIAL_MIN_MS) + partial_floor = OTA_FETCH_RETRY_PARTIAL_MIN_MS; + if (partial_floor < floor) floor = partial_floor; + } + if (service < floor) service = floor; if (service > OTA_FETCH_RETRY_MAX_MS) service = OTA_FETCH_RETRY_MAX_MS; return retryDelay((uint32_t)service); } diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index 310eb630..b5b83b3a 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -215,7 +215,10 @@ static constexpr uint16_t ota_max_block_capability() { return (uint16_t)OTA_MAX_ #error "OTA_FETCH_PIPELINE_INITIAL must be between 1 and OTA_FETCH_PIPELINE" #endif #ifndef OTA_FETCH_RETRY_MIN_MS -#define OTA_FETCH_RETRY_MIN_MS 5000 // quiet-time floor: covers host/queue/turnaround latency on fast links +#define OTA_FETCH_RETRY_MIN_MS 5000 // unanswered/empty flight: allow host and relay turnaround +#endif +#ifndef OTA_FETCH_RETRY_PARTIAL_MIN_MS +#define OTA_FETCH_RETRY_PARTIAL_MIN_MS 1500 // after valid DATA, recover missing holes promptly on fast links #endif #ifndef OTA_FETCH_RETRY_MAX_MS #define OTA_FETCH_RETRY_MAX_MS 60000 // bound loss recovery when configured radio settings are impractical @@ -409,6 +412,9 @@ public: void set_clock(uint32_t ms) { _now_ms = ms; } bool set_speed(float speed); float speed() const { return _speed; } + // Sender-only, loss-responsive packet pacing. This does not alter discovery, + // proof, or retry timers; the user's persisted speed remains an upper bound. + float adaptivePacketSpeed() const { return _adaptive_packet_speed; } uint32_t pacedDelay(uint32_t ms) const { return scaleDelay(ms, _speed); } // Faster pacing cannot shorten a loss-recovery deadline below the existing // physical packet-flight allowance. Slower pacing expands that allowance. @@ -620,6 +626,8 @@ private: bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask, bool wire_v2 = false, bool allow_deflate = false, bool extended_length = false); + void noteServedRequestPacing(const uint8_t* mid, uint16_t block, + uint16_t want, uint16_t full_mask); bool queueManifestJob(const uint8_t* mid, uint16_t want_mask); uint32_t manifestEgressGapMs() const; uint32_t proofEgressGapMs() const; @@ -699,6 +707,14 @@ private: }; ServeJob _serve_jobs[OTA_SERVE_QUEUE]; uint8_t _n_serve_jobs = 0; + float _adaptive_packet_speed = 1.0f; + uint8_t _pace_clean_blocks = 0; + uint8_t _pace_mid[4] = {0}; + uint16_t _pace_last_full_block = 0; + bool _pace_has_mid = false; + uint8_t _pace_loss_mid[4] = {0}; + uint16_t _pace_last_loss_block = 0; + bool _pace_has_loss = false; struct ManifestServeJob { OtaReplyRoute route; uint8_t mid[4]; diff --git a/src/helpers/ota/OtaSpeedConfig.cpp b/src/helpers/ota/OtaSpeedConfig.cpp index 159729d6..d59f9ca4 100644 --- a/src/helpers/ota/OtaSpeedConfig.cpp +++ b/src/helpers/ota/OtaSpeedConfig.cpp @@ -156,8 +156,8 @@ void beginSpeedConfig(FILESYSTEM* fs) { float speedFactor() { return factor; } -void formatSpeed(char* text, size_t capacity) { - const uint32_t scaled = (uint32_t)(factor * 1000000.0f + 0.5f); +void formatSpeedFactor(float speed, char* text, size_t capacity) { + const uint32_t scaled = (uint32_t)(speed * 1000000.0f + 0.5f); snprintf(text, capacity, "%u.%06u", (unsigned)(scaled / 1000000), (unsigned)(scaled % 1000000)); if (!capacity) return; size_t n = strlen(text); @@ -165,7 +165,17 @@ void formatSpeed(char* text, size_t capacity) { if (n && text[n - 1] == '.') text[n - 1] = 0; } -bool handleSpeedCommand(const char* command, char* reply, size_t capacity) { +void formatSpeed(char* text, size_t capacity) { + formatSpeedFactor(factor, text, capacity); +} + +float effectivePacketPace(float adaptive_speed) { + if (!validSpeed(adaptive_speed)) adaptive_speed = OTA_SPEED_DEFAULT; + return factor < adaptive_speed ? factor : adaptive_speed; +} + +bool handleSpeedCommand(const char* command, char* reply, size_t capacity, + float adaptive_speed) { const char* value = nullptr; bool require_value = false, read_only = false; const char* const forms[] = {"set ota.speed", "get ota.speed", "ota config speed", "ota cfg speed", "ota set speed", "ota speed"}; @@ -194,8 +204,11 @@ bool handleSpeedCommand(const char* command, char* reply, size_t capacity) { return true; } } - char number[16]; formatSpeed(number, sizeof(number)); - snprintf(reply, capacity, *value ? "OK ota.speed=%sx (saved)" : "> ota.speed=%sx", number); + char number[16], packet[16]; + formatSpeed(number, sizeof(number)); + formatSpeedFactor(effectivePacketPace(adaptive_speed), packet, sizeof(packet)); + if (*value) snprintf(reply, capacity, "OK ota.speed=%sx (saved) packet=%sx", number, packet); + else snprintf(reply, capacity, "> ota.speed=%sx packet=%sx", number, packet); return true; } diff --git a/src/helpers/ota/OtaSpeedConfig.h b/src/helpers/ota/OtaSpeedConfig.h index 36e8e349..b4d31c58 100644 --- a/src/helpers/ota/OtaSpeedConfig.h +++ b/src/helpers/ota/OtaSpeedConfig.h @@ -10,6 +10,11 @@ namespace mesh { namespace ota { void beginSpeedConfig(FILESYSTEM* fs); float speedFactor(); void formatSpeed(char* text, size_t capacity); -bool handleSpeedCommand(const char* command, char* reply, size_t capacity); +void formatSpeedFactor(float speed, char* text, size_t capacity); +// Automatic sender pacing only adds packet quiet time. The saved setting is +// still the upper bound and continues to control the other OTA timers. +float effectivePacketPace(float adaptive_speed); +bool handleSpeedCommand(const char* command, char* reply, size_t capacity, + float adaptive_speed = OTA_SPEED_DEFAULT); } } diff --git a/test/test_companion_ota_config.py b/test/test_companion_ota_config.py index 1b55a5b5..c5a6fba4 100644 --- a/test/test_companion_ota_config.py +++ b/test/test_companion_ota_config.py @@ -34,6 +34,7 @@ struct Manager { uint8_t max_hops() const { return hops; } uint16_t checkpoint_blocks() const { return checkpoint; } uint16_t advert_mins() const { return advert; } + float adaptivePacketSpeed() const { return 0.5f; } void set_autofetch(uint8_t v) { af=v; } void set_max_hops(uint8_t v) { hops=v; } void set_checkpoint_blocks(uint16_t v) { checkpoint=v; } @@ -53,6 +54,8 @@ static bool ota_acquire_context(char* reply, size_t cap) { static OtaContext& ota_ctx() { return context; } static OtaContext* ota_context_if_active() { return &context; } static void formatSpeed(char* text, size_t) { strcpy(text, "1"); } +static float effectivePacketPace(float adaptive) { return adaptive; } +static void formatSpeedFactor(float factor, char* text, size_t cap) { snprintf(text, cap, "%g", factor); } @IS_CMD@ static bool config(const char* rest, char* reply, OtaContext& c) { @CONFIG@ @@ -111,6 +114,7 @@ int main() { assert(!context.config_dirty); }; reboot(fs); + command("ota config", "ota config: speed=1x packet=0.5x"); for (const auto* setting : {"hops", "checkpoint", "advert"}) { const auto before=OtaConfigState::capture(context); for (const auto* bad : {"", " ", "x", "-1", "1x", "1 2", "1.0", "4294967296", "9999999999999999999999"}) { diff --git a/test/test_esp32_partition_migration_recipe.py b/test/test_esp32_partition_migration_recipe.py index ccb1f761..b09e855a 100644 --- a/test/test_esp32_partition_migration_recipe.py +++ b/test/test_esp32_partition_migration_recipe.py @@ -17,8 +17,10 @@ sys.path.insert(0, str(ROOT / "scripts")) from build_esp32_partition_migration import ( # noqa: E402 build_steps, bundle_release, verify_archive, ) -from package_esp32_partition_migration import BOARDS, mota_full, readme # noqa: E402 -from motalib import FwIdent # noqa: E402 +from package_esp32_partition_migration import ( # noqa: E402 + BOARDS, bridge_with_successor_endf, mota_full, readme, +) +from motalib import FwIdent, build_endf, parse_endf_ident # noqa: E402 class Esp32MigrationRecipeTest(unittest.TestCase): @@ -35,6 +37,10 @@ class Esp32MigrationRecipeTest(unittest.TestCase): BOARDS["generic-e22-sx1262-repeater"]["wifi_bridge"]) self.assertEqual("esp32_s3_8mb_partition_migrator", BOARDS["heltec-v3-sensor"]["wifi_bridge"]) + self.assertEqual("heltec_v4_partition_expander", + BOARDS["heltec-v4"]["expander_bridge"]) + self.assertEqual("Xiao_S3_WIO_partition_expander", + BOARDS["xiao-s3-wio"]["expander_bridge"]) self.assertEqual("esp32_8mb_partition_migrator", BOARDS["heltec-v2-room-server"]["wifi_bridge"]) expected_slots = {4 * 1024 * 1024: 0x1F0000, @@ -49,6 +55,8 @@ class Esp32MigrationRecipeTest(unittest.TestCase): self.assertIn(spec["target"], env_names) self.assertIn(spec["wifi_bridge"], env_names) self.assertIn(spec["lora_bridge"], env_names) + if spec.get("expander_bridge"): + self.assertIn(spec["expander_bridge"], env_names) self.assertEqual(expected_slots[spec["flash_bytes"]], spec["slot_bytes"]) @@ -85,8 +93,8 @@ class Esp32MigrationRecipeTest(unittest.TestCase): def test_full_images_precede_every_bridge_and_builds_are_serial(self): steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test", "usa-cascadia", "cascade", 4) - self.assertEqual(6, len(steps)) - self.assertEqual([True, True, False, False, False, False], + self.assertEqual(8, len(steps)) + self.assertEqual([True, True] + [False] * 6, [full for _, full in steps]) self.assertEqual("heltec_v4_repeater", steps[0][0][3]) self.assertEqual("Xiao_S3_WIO_repeater", steps[1][0][3]) @@ -94,9 +102,16 @@ class Esp32MigrationRecipeTest(unittest.TestCase): self.assertEqual([ "heltec_v4_partition_migrator", "heltec_v4_partition_migrator_lora_repeater", + "heltec_v4_partition_expander", "xiao_s3_partition_migrator", "xiao_s3_partition_migrator_lora_repeater", + "Xiao_S3_WIO_partition_expander", ], [command[3] for command, _ in steps[2:]]) + guide = readme("heltec-v4", BOARDS["heltec-v4"], + "v1.17.1.7-test", "01234567", full_mota_blocks=960) + self.assertIn("partition-expander.mota", guide) + self.assertIn("automatically", guide) + self.assertIn("already-expanded board", guide) def test_4mb_roles_share_two_chip_family_bridges(self): steps = build_steps(["thinknode-m2-repeater", "thinknode-m2-room-server"], @@ -132,6 +147,15 @@ class Esp32MigrationRecipeTest(unittest.TestCase): package = mota_full(image, FwIdent(0x01020304, 0xAABBCCDD, "TEST"), 2048) self.assertGreater(len(package), len(image)) + def test_bridge_mota_rebinds_preexisting_temporary_role_endf(self): + body = b"\xe9" + b"bridge image" + temporary = body + build_endf(body, FwIdent(1, 0x12345678, "BOARD")) + successor = FwIdent(2, 0x87654321, "BOARD") + rebound = bridge_with_successor_endf(temporary, successor) + self.assertEqual(len(temporary), len(rebound)) + self.assertEqual(successor, parse_endf_ident(rebound)) + self.assertEqual(body, rebound[:-56]) + def test_dry_run_has_no_build_side_effects(self): with tempfile.TemporaryDirectory(prefix="esp32-migration-plan-") as temporary: output_root = Path(temporary) / "release" @@ -146,6 +170,12 @@ class Esp32MigrationRecipeTest(unittest.TestCase): self.assertIn("pio run -e heltec_v4_partition_migrator", result.stdout) self.assertFalse(output_root.exists()) + def test_partition_expanders_use_adaptive_four_block_transfer_window(self): + for variant in ("heltec_v4", "xiao_s3_wio"): + config = (ROOT / "variants" / variant / "platformio.ini").read_text() + expander = config.split("partition_expander]", 1)[1].split("\n[env:", 1)[0] + self.assertIn("-D OTA_FETCH_PIPELINE=4", expander) + def test_archive_verification_rejects_changed_payload(self): with tempfile.TemporaryDirectory(prefix="esp32-migration-zip-") as temporary: archive_path = Path(temporary) / "test.zip" diff --git a/test/test_firmware_picker.js b/test/test_firmware_picker.js index ae0e8ee2..070bbf06 100644 --- a/test/test_firmware_picker.js +++ b/test/test_firmware_picker.js @@ -296,6 +296,21 @@ const fullUsbLogging = picker.parseTargetProfile( assert.strictEqual(fullUsbLogging.mode, "usb"); assert.strictEqual(fullUsbLogging.variant, "default"); +const partitionExpander = picker.parseTargetProfile( + "Xiao_S3_WIO_partition_expander" +); +assert.strictEqual(partitionExpander.role, "partition-expander"); +assert.strictEqual(partitionExpander.hardware, "Xiao_S3_WIO"); +assert.strictEqual(partitionExpander.explicitOta, "lora-receiver"); +assert.strictEqual(picker.ROLE_LABELS[partitionExpander.role], + "Partition Expander"); +assert(picker.installSteps(partitionExpander, "zip").some(function (step) { + return step.includes("not an nRF52 Serial DFU package"); +})); +assert(picker.installSteps(partitionExpander, "bin").some(function (step) { + return step.includes("temporary bridge"); +})); + const heltecV4Full = picker.parseTargetProfile( "heltec_v4_2_v4_3_companion_radio_full_femon" ); diff --git a/test/test_heltec_v4_partition_migrator.py b/test/test_heltec_v4_partition_migrator.py index 475a84eb..3f6f22a8 100644 --- a/test/test_heltec_v4_partition_migrator.py +++ b/test/test_heltec_v4_partition_migrator.py @@ -15,6 +15,7 @@ PROFILE = ROOT / "variants/heltec_v4/platformio.ini" MIGRATOR_BOARD = ROOT / "boards/heltec_v4_migrator.json" XIAO_PROFILE = ROOT / "variants/xiao_s3_wio/platformio.ini" XIAO_MIGRATOR_BOARD = ROOT / "boards/seeed_xiao_esp32s3_migrator.json" +EXPANDER = ROOT / "examples/partition_expander/main.cpp" PARTITIONS = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_16MB.csv" PARTITIONS_8MB = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_8MB.csv" PARTITIONS_4MB = ROOT / "variants/dual_ota_full_4MB.csv" @@ -55,14 +56,24 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase): migration_body = source[source.index("void runMigration()") :] self.assertLess(migration_body.index("stageLegacyIdentity()"), migration_body.index("copyAndVerify(*running")) - self.assertLess(migration_body.index("esp_ota_set_boot_partition(\n bridge_in_app0 ? refs.app0 : refs.app1)"), + self.assertLess(migration_body.index("esp_ota_set_boot_partition(\n boot_in_app0 ? refs.app0 : refs.app1)"), + migration_body.index("publishExpandedPartitionTable(*plan)")) + self.assertLess(migration_body.index("stageLegacyConfig()"), + migration_body.index("copyAndVerify(*running")) + self.assertLess(migration_body.index("stageExpanderHandoff()"), migration_body.index("publishExpandedPartitionTable(*plan)")) setup_body = source[source.index("void setup()") :] self.assertIn("restoreStagedIdentity()", setup_body) - self.assertIn("resumeLegacyOtaReceiver()", setup_body) + self.assertIn("restoreStagedConfig()", setup_body) + self.assertIn("verifyExpandedIdentityFile()", setup_body) + self.assertIn('kMigrationConfigRestoredKey[] = "cfg-restored"', source) + self.assertIn("nvs.putBool(kMigrationConfigRestoredKey, true)", source) + self.assertIn("resumeLegacyOtaReceiver(geometry)", setup_body) self.assertLess(setup_body.index("restoreStagedIdentity()"), - setup_body.index("resumeLegacyOtaReceiver()")) - self.assertIn("validLegacyOtaReceiver(*old_receiver)", migration_body) + setup_body.index("restoreStagedConfig()")) + self.assertLess(setup_body.index("restoreStagedConfig()"), + setup_body.index("resumeLegacyOtaReceiver(geometry)")) + self.assertIn("validOtherLoRaFirmware(*running, *old_receiver)", migration_body) self.assertIn("copyAndVerify(*refs.app1", migration_body) self.assertIn("stageResumeSlot(resume.resume_slot)", migration_body) self.assertIn("AsyncElegantOTA.begin(&server)", source) @@ -73,6 +84,11 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase): self.assertIn("esp_flash_erase_region(chip", source) self.assertIn("esp_flash_write(chip", source) self.assertNotIn("#include ", source) + for path in ("/com_prefs", "/node_prefs", "/s_contacts", + "/s_login_replay", "/radio_profiles", "/regions2", + "/com_prefs.bak", "/radio_profiles.bak", "/prefs.json", + "/management", "/ota_speed", "/bsec_state.bin"): + self.assertIn(path, source) board = MIGRATOR_BOARD.read_text(encoding="utf-8") self.assertIn('"-DARDUINO_USB_CDC_ON_BOOT=0"', board) @@ -109,6 +125,19 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase): self.assertEqual(0xC0, len(embedded_4mb)) self.assertEqual(embedded_4mb, generated_4mb.read_bytes()[:len(embedded_4mb)]) + def test_expander_pins_full_target_and_reboots_after_apply(self): + expander = EXPANDER.read_text(encoding="utf-8") + self.assertIn("MOTA_MIGRATION_TARGET_ID", expander) + self.assertIn("manifest.is_full()", expander) + self.assertIn("loadPrimaryRadio(profile)", expander) + self.assertIn("returnToVerifiedFull()", expander) + self.assertIn("hasExpanderHandoff()", expander) + self.assertIn("hasVerifiedConfigHandoff()", expander) + self.assertIn("ota.apply_fetched(message)", expander) + self.assertIn("ota_reboot_to_apply()", expander) + self.assertLess(expander.index("ota.apply_fetched(message)"), + expander.index("ota_reboot_to_apply()")) + if __name__ == "__main__": unittest.main() diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 0b89461e..b9acac5b 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -2781,6 +2781,47 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(server.pendingServeJobs(), 0u); } +TEST(OtaTransfer, SenderPacketPaceBacksOffOnDistinctLostBlocks) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + ASSERT_EQ(manifest.block_count, 3u); + + OtaManager server; + CapturedMessages sent; + server.begin(0, capture_send, &sent); + ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f); + + auto request = [&](uint16_t block, uint16_t mask) { + ReqMsg req{}; + memcpy(req.manifest_id, manifest.merkle_root, 4); + req.block_idx = block; + req.want_mask = mask; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t len = encode_req(wire, sizeof(wire), req); + return len != 0 && server.on_message(wire, len); + }; + + ASSERT_TRUE(request(0, 0xFFFF)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f); + ASSERT_TRUE(request(0, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.5f); + ASSERT_TRUE(request(0, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.5f); // same hole cannot ratchet down + ASSERT_TRUE(request(1, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.4f); + ASSERT_TRUE(request(0, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.3f); + ASSERT_TRUE(request(1, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.25f); // automatic floor + ASSERT_TRUE(request(1, 0x0001)); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.25f); + + // Reinitializing a source starts with the configured maximum again. + server.begin(0, capture_send, &sent); + EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f); +} + TEST(OtaTransfer, LiteralLegacyFullMaskServesTwoKilobyteBlockInThirteenFragments) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); @@ -3744,6 +3785,7 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) { client.begin(SIM_TARGET_ID, capture_send, &sent); client.set_fetch_store(&store); client.set_clock(100); + client.set_link_timing(20, 2000); // direct SF5-class link client.pull(manifest.merkle_root, manifest.target_id); sent.items.clear(); deliver_manifest_fragment(client, manifest.merkle_root, 0, @@ -3752,6 +3794,10 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) { manifest.manifest_start + OTA_MF_FRAG, (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); ASSERT_EQ(sent.items.size(), 1u); // conservative one-block probe flight + const uint32_t unanswered_timeout = client.fetchRetryTimeoutMs(); + EXPECT_GE(unanswered_timeout, (uint32_t)OTA_FETCH_RETRY_MIN_MS); + client.note_rx_path_hops(0); + EXPECT_EQ(client.fetchRetryTimeoutMs(), unanswered_timeout); // path alone is not proof of a live response DataMsg first_fragment; memcpy(first_fragment.manifest_id, manifest.merkle_root, 4); @@ -3765,6 +3811,8 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) { client.on_message(wire, wire_len); const uint32_t timeout = client.fetchRetryTimeoutMs(); + EXPECT_EQ(timeout, (uint32_t)OTA_FETCH_RETRY_PARTIAL_MIN_MS); + EXPECT_LT(timeout, unanswered_timeout); client.set_clock(100 + timeout - 1); client.loop(); // no premature fixed-tick retry EXPECT_EQ(sent.items.size(), 1u); diff --git a/test/test_ota_speed.py b/test/test_ota_speed.py index 4c620285..d4e5bb68 100644 --- a/test/test_ota_speed.py +++ b/test/test_ota_speed.py @@ -20,20 +20,26 @@ int main() { assert(!strncmp(reply, prefix, strlen(prefix))); }; beginSpeedConfig(&fs); - command("get ota.speed", "> ota.speed=1x"); + command("get ota.speed", "> ota.speed=1x packet=1x"); + assert(handleSpeedCommand("get ota.speed", reply, sizeof reply, 0.25f)); + assert(!strcmp(reply, "> ota.speed=1x packet=0.25x")); + assert(effectivePacketPace(0.25f) == 0.25f); + assert(effectivePacketPace(1.0f) == 1.0f); for (const auto* bad : {"", "0", "-1", "0.049", "3.01", "nan", "inf", "1e0", "1abc", "1 2"}) { const std::string text = std::string("set ota.speed ") + bad; command(text.c_str(), "ERR"); assert(speedFactor() == 1.0f && fs.files.empty()); } - command("set ota.speed .05", "OK ota.speed=0.05x (saved)"); + command("set ota.speed .05", "OK ota.speed=0.05x (saved) packet=0.05x"); + assert(handleSpeedCommand("get ota.speed", reply, sizeof reply, 0.25f)); + assert(!strcmp(reply, "> ota.speed=0.05x packet=0.05x")); assert(validSpeed(speedFactor())); beginSpeedConfig(&fs); - command("get ota.speed", "> ota.speed=0.05x"); - command("ota config speed 3", "OK ota.speed=3x (saved)"); - command("ota config speed", "> ota.speed=3x"); - command("ota cfg speed 0.5", "OK ota.speed=0.5x"); - command("ota speed", "> ota.speed=0.5x"); + command("get ota.speed", "> ota.speed=0.05x packet=0.05x"); + command("ota config speed 3", "OK ota.speed=3x (saved) packet=1x"); + command("ota config speed", "> ota.speed=3x packet=1x"); + command("ota cfg speed 0.5", "OK ota.speed=0.5x (saved) packet=0.5x"); + command("ota speed", "> ota.speed=0.5x packet=0.5x"); command("get ota.speed 1", "ERR"); assert(!handleSpeedCommand("get ota.speed.extra", reply, sizeof reply)); assert(!handleSpeedCommand("ota config advert 10", reply, sizeof reply)); diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index 61aa8335..eef34b74 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -63,6 +63,10 @@ MAX_ARCHIVE_MEMBER_SIZE = 64 * 1024 * 1024 MAX_FIRMWARE_IMAGE_SIZE = 64 * 1024 * 1024 LEGACY_TARGET_MAX_BLOCK_SIZE = 1024 MOTA_MAX_BLOCK_SIZE = 2048 +# Lab/HIL OTA tests use one explicit tuple on every participating radio. +# This is a bench default, not a regional or production firmware preset. +# Override --temp-radio only when the hardware or local rules require it. +DEFAULT_LAB_TEMP_RADIO = "909.5,500,5,5,120" TRANSMISSION_RETRY_LIMIT = 3 TRANSMISSION_RETRY_WINDOW_SECONDS = 90 TRANSMISSION_RETRY_DELAY_SECONDS = 2 @@ -7150,7 +7154,7 @@ def build_parser() -> argparse.ArgumentParser: ), ) parser.add_argument( - "--temp-radio", default="909.950,250,5,5,120", + "--temp-radio", default=DEFAULT_LAB_TEMP_RADIO, help="frequency,bw,sf,cr,minutes", ) parser.add_argument( diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index 6e1bae91..ed6911b6 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -2452,7 +2452,7 @@ def build_parser() -> argparse.ArgumentParser: default=ota.DEFAULT_RELAY_TX_DELAY, help="temporary flood txdelay for managed intermediate relays", ) - parser.add_argument("--temp-radio", default="909.950,250,5,5,120") + parser.add_argument("--temp-radio", default=ota.DEFAULT_LAB_TEMP_RADIO) parser.add_argument( "--ota-hops", type=int, diff --git a/tools/lora_ota/rebuild_rak3401_bundle.py b/tools/lora_ota/rebuild_rak3401_bundle.py index 3c45dceb..a8508aff 100755 --- a/tools/lora_ota/rebuild_rak3401_bundle.py +++ b/tools/lora_ota/rebuild_rak3401_bundle.py @@ -378,7 +378,7 @@ ZIP and a persistent work directory. The runner enforces exact target, start body hash, package order, watchdog state, post-boot identity, and normal-radio restoration. -Direct bench: use `--temp-radio 909.950,500,5,5,120 --ota-hops 0`. Add +Direct bench: use `--temp-radio 909.5,500,5,5,120 --ota-hops 0`. Add `--legacy-full-airtime` only where the frequency and local duty-cycle rules permit it; omit it otherwise. The runner saves and restores the original airtime factor and OTA hop reach. diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index 70fb0cac..1edeb159 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -360,11 +360,12 @@ class FormatTests(unittest.TestCase): with self.assertRaisesRegex(argparse.ArgumentTypeError, "bandwidth must be"): ota.parse_temp_radio("909.950,200,5,5,120") - def test_ota_runners_default_to_sf5_and_250_khz(self) -> None: + def test_ota_runners_use_the_same_lab_tuple(self) -> None: generic = ota.build_parser().parse_args(["release.mota", "remote"]) chain = rak_chain.build_parser().parse_args([]) - self.assertEqual(generic.temp_radio, "909.950,250,5,5,120") - self.assertEqual(chain.temp_radio, "909.950,250,5,5,120") + self.assertEqual(generic.temp_radio, ota.DEFAULT_LAB_TEMP_RADIO) + self.assertEqual(chain.temp_radio, ota.DEFAULT_LAB_TEMP_RADIO) + self.assertEqual(generic.temp_radio, "909.5,500,5,5,120") self.assertFalse(chain.legacy_full_airtime) def test_package_build_timeout_is_configurable_and_positive(self) -> None: @@ -1736,7 +1737,7 @@ class SourceCliTests(unittest.TestCase): self.assertIn( mock.call( mock.ANY, - "tempradio 909.950,250,5,5,120", + f"tempradio {ota.DEFAULT_LAB_TEMP_RADIO}", retry=False, ), source_cli.call_args_list, diff --git a/tools/mota/motalib.py b/tools/mota/motalib.py index 49307501..cb64c63d 100644 --- a/tools/mota/motalib.py +++ b/tools/mota/motalib.py @@ -252,7 +252,7 @@ def hardware_id_for_env(env_name: str) -> str: return hardware_id role = re.search( - r"[_-](?:repeater|repeatr|room_server|room_svr|sensor|terminal_chat|kiss_modem|" + r"[_-](?:partition_expander|repeater|repeatr|room_server|room_svr|sensor|terminal_chat|kiss_modem|" r"companion_radio|companion|comp_radio)(?=[_-]|$)", env_name, re.IGNORECASE) family = (env_name[:role.start()] if role else env_name).strip("_-") or env_name.strip("_-") # Match the explicit MOTA_HW_ID shared by every role in these variant diff --git a/tools/mota/test_mota.py b/tools/mota/test_mota.py index 7abd41d4..926f246c 100644 --- a/tools/mota/test_mota.py +++ b/tools/mota/test_mota.py @@ -1033,6 +1033,8 @@ def test_rak_nrf52_ota_profiles_keep_ina_and_gps_where_uart_is_available(): def test_hardware_id_for_env(): + assert ml.hardware_id_for_env("Xiao_S3_WIO_partition_expander") == "Xiao_S3_WIO" + assert ml.hardware_id_for_env("Xiao_S3_WIO_partition_expander") == ml.hardware_id_for_env("Xiao_S3_WIO_repeater") assert ml.hardware_id_for_env("RAK_4631_repeater") == "RAK4631" assert ml.hardware_id_for_env("RAK_4631_companion_radio_usb") == "RAK4631" assert ( diff --git a/variants/heltec_v4/platformio.ini b/variants/heltec_v4/platformio.ini index 7027caf0..6543597c 100644 --- a/variants/heltec_v4/platformio.ini +++ b/variants/heltec_v4/platformio.ini @@ -860,13 +860,28 @@ build_flags = build_src_filter = ${Heltec_lora32_v4.build_src_filter} +<../examples/kiss_modem/> +; Self-hosted partition migration receiver for a legacy V4 1.25 MiB slot. +; This role keeps Wi-Fi recovery and receives the exact Full repeater over LoRa. +[env:heltec_v4_partition_expander] +extends = Heltec_lora32_v4 +board = heltec_v4_migrator +board_build.partitions = default.csv +board_build.flash_mode = dio +board_upload.maximum_size = 1310720 +build_src_filter = ${Heltec_lora32_v4.build_src_filter} + +<../examples/partition_expander/*.cpp> +build_flags = ${Heltec_lora32_v4.build_flags} + -D MESH_MIN_RUNTIME_HEAP=100352 + -D MOTA_MIGRATION_TARGET_ID=0xe792a051 + ; Pipeline verified OTA blocks across one request turn. The adaptive window + ; starts at one and grows only after clean flights, then shrinks on loss. + -D OTA_FETCH_PIPELINE=4 +lib_deps = ${Heltec_lora32_v4.lib_deps} + ${esp32_ota.lib_deps} + ; One-time Wi-Fi-only migration bridge for legacy Heltec V4/V4.3 OLED -; repeaters that still use Arduino's 1.25 MiB OTA slots. It is deliberately -; linked against default.csv so the existing browser updater can install it. -; On first boot it stages /identity/_main.id in unchanged NVS, makes a -; CRC-verified copy of legacy SPIFFS, copies itself to app0 when necessary, -; then publishes default_16MB. It restores the identity after the larger -; SPIFFS region initializes and exposes a browser updater for the full image. +; repeaters that still use Arduino's 1.25 MiB OTA slots. It is linked against +; default.csv so the existing browser updater can install it. [env:heltec_v4_partition_migrator] platform = platformio/espressif32@6.11.0 board = heltec_v4_migrator diff --git a/variants/xiao_s3_wio/platformio.ini b/variants/xiao_s3_wio/platformio.ini index 2faf86f7..13227062 100644 --- a/variants/xiao_s3_wio/platformio.ini +++ b/variants/xiao_s3_wio/platformio.ini @@ -58,6 +58,26 @@ lib_deps = ${Xiao_S3_WIO.lib_deps} ${esp32_ota.lib_deps} +; Self-hosted Wi-Fi/LoRa receiver for the 1.25 MiB -> full-layout migration. +; The target ID is the exact successor Xiao_S3_WIO_repeater; this build must +; not automatically select an unrelated role. Hardware deployment still needs +; an A/B and power-interruption test before it is included in a release. +[env:Xiao_S3_WIO_partition_expander] +extends = Xiao_S3_WIO +board_build.partitions = default.csv +board_build.flash_mode = dio +board_upload.maximum_size = 1310720 +build_src_filter = ${Xiao_S3_WIO.build_src_filter} + +<../examples/partition_expander/*.cpp> +build_flags = ${Xiao_S3_WIO.build_flags} + -D MESH_MIN_RUNTIME_HEAP=100352 + -D MOTA_MIGRATION_TARGET_ID=0x9f19bcd1 + ; Same adaptive four-block window as the accelerated LoRa bridges: fewer + ; request/proof turnarounds without weakening per-block verification. + -D OTA_FETCH_PIPELINE=4 +lib_deps = ${Xiao_S3_WIO.lib_deps} + ${esp32_ota.lib_deps} + ; Generic source, XIAO-specific board definition. It is intentionally linked ; to default.csv so the stock legacy browser updater can install it. [env:xiao_s3_partition_migrator]