Complete ESP32 partition expander and adaptive LoRa OTA pacing

test/test_companion_ota_config.py

@@ -34,6 +34,7 @@ struct Manager {
   uint8_t max_hops() const { return hops; }
   uint16_t checkpoint_blocks() const { return checkpoint; }
   uint16_t advert_mins() const { return advert; }
+  float adaptivePacketSpeed() const { return 0.5f; }
   void set_autofetch(uint8_t v) { af=v; }
   void set_max_hops(uint8_t v) { hops=v; }
   void set_checkpoint_blocks(uint16_t v) { checkpoint=v; }
@@ -53,6 +54,8 @@ static bool ota_acquire_context(char* reply, size_t cap) {
 static OtaContext& ota_ctx() { return context; }
 static OtaContext* ota_context_if_active() { return &context; }
 static void formatSpeed(char* text, size_t) { strcpy(text, "1"); }
+static float effectivePacketPace(float adaptive) { return adaptive; }
+static void formatSpeedFactor(float factor, char* text, size_t cap) { snprintf(text, cap, "%g", factor); }
 @IS_CMD@
 static bool config(const char* rest, char* reply, OtaContext& c) {
   @CONFIG@
@@ -111,6 +114,7 @@ int main() {
     assert(!context.config_dirty);
   };
   reboot(fs);
+  command("ota config", "ota config: speed=1x packet=0.5x");
   for (const auto* setting : {"hops", "checkpoint", "advert"}) {
     const auto before=OtaConfigState::capture(context);
     for (const auto* bad : {"", " ", "x", "-1", "1x", "1 2", "1.0", "4294967296", "9999999999999999999999"}) {

test/test_esp32_partition_migration_recipe.py

@@ -17,8 +17,10 @@ sys.path.insert(0, str(ROOT / "scripts"))
 from build_esp32_partition_migration import (  # noqa: E402
     build_steps, bundle_release, verify_archive,
 )
-from package_esp32_partition_migration import BOARDS, mota_full, readme  # noqa: E402
-from motalib import FwIdent  # noqa: E402
+from package_esp32_partition_migration import (  # noqa: E402
+    BOARDS, bridge_with_successor_endf, mota_full, readme,
+)
+from motalib import FwIdent, build_endf, parse_endf_ident  # noqa: E402
 
 
 class Esp32MigrationRecipeTest(unittest.TestCase):
@@ -35,6 +37,10 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
                          BOARDS["generic-e22-sx1262-repeater"]["wifi_bridge"])
         self.assertEqual("esp32_s3_8mb_partition_migrator",
                          BOARDS["heltec-v3-sensor"]["wifi_bridge"])
+        self.assertEqual("heltec_v4_partition_expander",
+                         BOARDS["heltec-v4"]["expander_bridge"])
+        self.assertEqual("Xiao_S3_WIO_partition_expander",
+                         BOARDS["xiao-s3-wio"]["expander_bridge"])
         self.assertEqual("esp32_8mb_partition_migrator",
                          BOARDS["heltec-v2-room-server"]["wifi_bridge"])
         expected_slots = {4 * 1024 * 1024: 0x1F0000,
@@ -49,6 +55,8 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
                 self.assertIn(spec["target"], env_names)
                 self.assertIn(spec["wifi_bridge"], env_names)
                 self.assertIn(spec["lora_bridge"], env_names)
+                if spec.get("expander_bridge"):
+                    self.assertIn(spec["expander_bridge"], env_names)
                 self.assertEqual(expected_slots[spec["flash_bytes"]],
                                  spec["slot_bytes"])
 
@@ -85,8 +93,8 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
     def test_full_images_precede_every_bridge_and_builds_are_serial(self):
         steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test",
                             "usa-cascadia", "cascade", 4)
-        self.assertEqual(6, len(steps))
-        self.assertEqual([True, True, False, False, False, False],
+        self.assertEqual(8, len(steps))
+        self.assertEqual([True, True] + [False] * 6,
                          [full for _, full in steps])
         self.assertEqual("heltec_v4_repeater", steps[0][0][3])
         self.assertEqual("Xiao_S3_WIO_repeater", steps[1][0][3])
@@ -94,9 +102,16 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
         self.assertEqual([
             "heltec_v4_partition_migrator",
             "heltec_v4_partition_migrator_lora_repeater",
+            "heltec_v4_partition_expander",
             "xiao_s3_partition_migrator",
             "xiao_s3_partition_migrator_lora_repeater",
+            "Xiao_S3_WIO_partition_expander",
         ], [command[3] for command, _ in steps[2:]])
+        guide = readme("heltec-v4", BOARDS["heltec-v4"],
+                       "v1.17.1.7-test", "01234567", full_mota_blocks=960)
+        self.assertIn("partition-expander.mota", guide)
+        self.assertIn("automatically", guide)
+        self.assertIn("already-expanded board", guide)
 
     def test_4mb_roles_share_two_chip_family_bridges(self):
         steps = build_steps(["thinknode-m2-repeater", "thinknode-m2-room-server"],
@@ -132,6 +147,15 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
         package = mota_full(image, FwIdent(0x01020304, 0xAABBCCDD, "TEST"), 2048)
         self.assertGreater(len(package), len(image))
 
+    def test_bridge_mota_rebinds_preexisting_temporary_role_endf(self):
+        body = b"\xe9" + b"bridge image"
+        temporary = body + build_endf(body, FwIdent(1, 0x12345678, "BOARD"))
+        successor = FwIdent(2, 0x87654321, "BOARD")
+        rebound = bridge_with_successor_endf(temporary, successor)
+        self.assertEqual(len(temporary), len(rebound))
+        self.assertEqual(successor, parse_endf_ident(rebound))
+        self.assertEqual(body, rebound[:-56])
+
     def test_dry_run_has_no_build_side_effects(self):
         with tempfile.TemporaryDirectory(prefix="esp32-migration-plan-") as temporary:
             output_root = Path(temporary) / "release"
@@ -146,6 +170,12 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
             self.assertIn("pio run -e heltec_v4_partition_migrator", result.stdout)
             self.assertFalse(output_root.exists())
 
+    def test_partition_expanders_use_adaptive_four_block_transfer_window(self):
+        for variant in ("heltec_v4", "xiao_s3_wio"):
+            config = (ROOT / "variants" / variant / "platformio.ini").read_text()
+            expander = config.split("partition_expander]", 1)[1].split("\n[env:", 1)[0]
+            self.assertIn("-D OTA_FETCH_PIPELINE=4", expander)
+
     def test_archive_verification_rejects_changed_payload(self):
         with tempfile.TemporaryDirectory(prefix="esp32-migration-zip-") as temporary:
             archive_path = Path(temporary) / "test.zip"

test/test_firmware_picker.js

@@ -296,6 +296,21 @@ const fullUsbLogging = picker.parseTargetProfile(
 assert.strictEqual(fullUsbLogging.mode, "usb");
 assert.strictEqual(fullUsbLogging.variant, "default");
 
+const partitionExpander = picker.parseTargetProfile(
+  "Xiao_S3_WIO_partition_expander"
+);
+assert.strictEqual(partitionExpander.role, "partition-expander");
+assert.strictEqual(partitionExpander.hardware, "Xiao_S3_WIO");
+assert.strictEqual(partitionExpander.explicitOta, "lora-receiver");
+assert.strictEqual(picker.ROLE_LABELS[partitionExpander.role],
+  "Partition Expander");
+assert(picker.installSteps(partitionExpander, "zip").some(function (step) {
+  return step.includes("not an nRF52 Serial DFU package");
+}));
+assert(picker.installSteps(partitionExpander, "bin").some(function (step) {
+  return step.includes("temporary bridge");
+}));
+
 const heltecV4Full = picker.parseTargetProfile(
   "heltec_v4_2_v4_3_companion_radio_full_femon"
 );

test/test_heltec_v4_partition_migrator.py

@@ -15,6 +15,7 @@ PROFILE = ROOT / "variants/heltec_v4/platformio.ini"
 MIGRATOR_BOARD = ROOT / "boards/heltec_v4_migrator.json"
 XIAO_PROFILE = ROOT / "variants/xiao_s3_wio/platformio.ini"
 XIAO_MIGRATOR_BOARD = ROOT / "boards/seeed_xiao_esp32s3_migrator.json"
+EXPANDER = ROOT / "examples/partition_expander/main.cpp"
 PARTITIONS = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_16MB.csv"
 PARTITIONS_8MB = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_8MB.csv"
 PARTITIONS_4MB = ROOT / "variants/dual_ota_full_4MB.csv"
@@ -55,14 +56,24 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase):
         migration_body = source[source.index("void runMigration()") :]
         self.assertLess(migration_body.index("stageLegacyIdentity()"),
                         migration_body.index("copyAndVerify(*running"))
-        self.assertLess(migration_body.index("esp_ota_set_boot_partition(\n      bridge_in_app0 ? refs.app0 : refs.app1)"),
+        self.assertLess(migration_body.index("esp_ota_set_boot_partition(\n      boot_in_app0 ? refs.app0 : refs.app1)"),
+                        migration_body.index("publishExpandedPartitionTable(*plan)"))
+        self.assertLess(migration_body.index("stageLegacyConfig()"),
+                        migration_body.index("copyAndVerify(*running"))
+        self.assertLess(migration_body.index("stageExpanderHandoff()"),
                         migration_body.index("publishExpandedPartitionTable(*plan)"))
         setup_body = source[source.index("void setup()") :]
         self.assertIn("restoreStagedIdentity()", setup_body)
-        self.assertIn("resumeLegacyOtaReceiver()", setup_body)
+        self.assertIn("restoreStagedConfig()", setup_body)
+        self.assertIn("verifyExpandedIdentityFile()", setup_body)
+        self.assertIn('kMigrationConfigRestoredKey[] = "cfg-restored"', source)
+        self.assertIn("nvs.putBool(kMigrationConfigRestoredKey, true)", source)
+        self.assertIn("resumeLegacyOtaReceiver(geometry)", setup_body)
         self.assertLess(setup_body.index("restoreStagedIdentity()"),
-                        setup_body.index("resumeLegacyOtaReceiver()"))
-        self.assertIn("validLegacyOtaReceiver(*old_receiver)", migration_body)
+                        setup_body.index("restoreStagedConfig()"))
+        self.assertLess(setup_body.index("restoreStagedConfig()"),
+                        setup_body.index("resumeLegacyOtaReceiver(geometry)"))
+        self.assertIn("validOtherLoRaFirmware(*running, *old_receiver)", migration_body)
         self.assertIn("copyAndVerify(*refs.app1", migration_body)
         self.assertIn("stageResumeSlot(resume.resume_slot)", migration_body)
         self.assertIn("AsyncElegantOTA.begin(&server)", source)
@@ -73,6 +84,11 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase):
         self.assertIn("esp_flash_erase_region(chip", source)
         self.assertIn("esp_flash_write(chip", source)
         self.assertNotIn("#include <esp_flash_internal.h>", source)
+        for path in ("/com_prefs", "/node_prefs", "/s_contacts",
+                     "/s_login_replay", "/radio_profiles", "/regions2",
+                     "/com_prefs.bak", "/radio_profiles.bak", "/prefs.json",
+                     "/management", "/ota_speed", "/bsec_state.bin"):
+            self.assertIn(path, source)
 
         board = MIGRATOR_BOARD.read_text(encoding="utf-8")
         self.assertIn('"-DARDUINO_USB_CDC_ON_BOOT=0"', board)
@@ -109,6 +125,19 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase):
             self.assertEqual(0xC0, len(embedded_4mb))
             self.assertEqual(embedded_4mb, generated_4mb.read_bytes()[:len(embedded_4mb)])
 
+    def test_expander_pins_full_target_and_reboots_after_apply(self):
+        expander = EXPANDER.read_text(encoding="utf-8")
+        self.assertIn("MOTA_MIGRATION_TARGET_ID", expander)
+        self.assertIn("manifest.is_full()", expander)
+        self.assertIn("loadPrimaryRadio(profile)", expander)
+        self.assertIn("returnToVerifiedFull()", expander)
+        self.assertIn("hasExpanderHandoff()", expander)
+        self.assertIn("hasVerifiedConfigHandoff()", expander)
+        self.assertIn("ota.apply_fetched(message)", expander)
+        self.assertIn("ota_reboot_to_apply()", expander)
+        self.assertLess(expander.index("ota.apply_fetched(message)"),
+                        expander.index("ota_reboot_to_apply()"))
+
 
 if __name__ == "__main__":
     unittest.main()

test/test_ota/test_ota_core.cpp

@@ -2781,6 +2781,47 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure)
   EXPECT_EQ(server.pendingServeJobs(), 0u);
 }
 
+TEST(OtaTransfer, SenderPacketPaceBacksOffOnDistinctLostBlocks) {
+  MotaManifest manifest;
+  ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
+  ASSERT_EQ(manifest.block_count, 3u);
+
+  OtaManager server;
+  CapturedMessages sent;
+  server.begin(0, capture_send, &sent);
+  ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f);
+
+  auto request = [&](uint16_t block, uint16_t mask) {
+    ReqMsg req{};
+    memcpy(req.manifest_id, manifest.merkle_root, 4);
+    req.block_idx = block;
+    req.want_mask = mask;
+    uint8_t wire[MAX_PACKET_PAYLOAD];
+    const uint16_t len = encode_req(wire, sizeof(wire), req);
+    return len != 0 && server.on_message(wire, len);
+  };
+
+  ASSERT_TRUE(request(0, 0xFFFF));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f);
+  ASSERT_TRUE(request(0, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.5f);
+  ASSERT_TRUE(request(0, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.5f); // same hole cannot ratchet down
+  ASSERT_TRUE(request(1, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.4f);
+  ASSERT_TRUE(request(0, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.3f);
+  ASSERT_TRUE(request(1, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.25f); // automatic floor
+  ASSERT_TRUE(request(1, 0x0001));
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 0.25f);
+
+  // Reinitializing a source starts with the configured maximum again.
+  server.begin(0, capture_send, &sent);
+  EXPECT_FLOAT_EQ(server.adaptivePacketSpeed(), 1.0f);
+}
+
 TEST(OtaTransfer, LiteralLegacyFullMaskServesTwoKilobyteBlockInThirteenFragments) {
   MotaManifest manifest;
   ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest));
@@ -3744,6 +3785,7 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) {
   client.begin(SIM_TARGET_ID, capture_send, &sent);
   client.set_fetch_store(&store);
   client.set_clock(100);
+  client.set_link_timing(20, 2000);                      // direct SF5-class link
   client.pull(manifest.merkle_root, manifest.target_id);
   sent.items.clear();
   deliver_manifest_fragment(client, manifest.merkle_root, 0,
@@ -3752,6 +3794,10 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) {
                             manifest.manifest_start + OTA_MF_FRAG,
                             (uint16_t)(MOTA_MFL - OTA_MF_FRAG));
   ASSERT_EQ(sent.items.size(), 1u);                       // conservative one-block probe flight
+  const uint32_t unanswered_timeout = client.fetchRetryTimeoutMs();
+  EXPECT_GE(unanswered_timeout, (uint32_t)OTA_FETCH_RETRY_MIN_MS);
+  client.note_rx_path_hops(0);
+  EXPECT_EQ(client.fetchRetryTimeoutMs(), unanswered_timeout); // path alone is not proof of a live response
 
   DataMsg first_fragment;
   memcpy(first_fragment.manifest_id, manifest.merkle_root, 4);
@@ -3765,6 +3811,8 @@ TEST(OtaTransfer, FlightRetriesOnlyMissingFragmentsAfterItsDeadline) {
   client.on_message(wire, wire_len);
 
   const uint32_t timeout = client.fetchRetryTimeoutMs();
+  EXPECT_EQ(timeout, (uint32_t)OTA_FETCH_RETRY_PARTIAL_MIN_MS);
+  EXPECT_LT(timeout, unanswered_timeout);
   client.set_clock(100 + timeout - 1);
   client.loop();                                         // no premature fixed-tick retry
   EXPECT_EQ(sent.items.size(), 1u);

test/test_ota_speed.py

@@ -20,20 +20,26 @@ int main() {
     assert(!strncmp(reply, prefix, strlen(prefix)));
   };
   beginSpeedConfig(&fs);
-  command("get ota.speed", "> ota.speed=1x");
+  command("get ota.speed", "> ota.speed=1x packet=1x");
+  assert(handleSpeedCommand("get ota.speed", reply, sizeof reply, 0.25f));
+  assert(!strcmp(reply, "> ota.speed=1x packet=0.25x"));
+  assert(effectivePacketPace(0.25f) == 0.25f);
+  assert(effectivePacketPace(1.0f) == 1.0f);
   for (const auto* bad : {"", "0", "-1", "0.049", "3.01", "nan", "inf", "1e0", "1abc", "1 2"}) {
     const std::string text = std::string("set ota.speed ") + bad;
     command(text.c_str(), "ERR");
     assert(speedFactor() == 1.0f && fs.files.empty());
   }
-  command("set ota.speed .05", "OK ota.speed=0.05x (saved)");
+  command("set ota.speed .05", "OK ota.speed=0.05x (saved) packet=0.05x");
+  assert(handleSpeedCommand("get ota.speed", reply, sizeof reply, 0.25f));
+  assert(!strcmp(reply, "> ota.speed=0.05x packet=0.05x"));
   assert(validSpeed(speedFactor()));
   beginSpeedConfig(&fs);
-  command("get ota.speed", "> ota.speed=0.05x");
-  command("ota config speed 3", "OK ota.speed=3x (saved)");
-  command("ota config speed", "> ota.speed=3x");
-  command("ota cfg speed 0.5", "OK ota.speed=0.5x");
-  command("ota speed", "> ota.speed=0.5x");
+  command("get ota.speed", "> ota.speed=0.05x packet=0.05x");
+  command("ota config speed 3", "OK ota.speed=3x (saved) packet=1x");
+  command("ota config speed", "> ota.speed=3x packet=1x");
+  command("ota cfg speed 0.5", "OK ota.speed=0.5x (saved) packet=0.5x");
+  command("ota speed", "> ota.speed=0.5x packet=0.5x");
   command("get ota.speed 1", "ERR");
   assert(!handleSpeedCommand("get ota.speed.extra", reply, sizeof reply));
   assert(!handleSpeedCommand("ota config advert 10", reply, sizeof reply));