From 7372ad6c26567f4cd6bab232acdd7fc818d2a27c Mon Sep 17 00:00:00 2001 From: mikecarper Date: Thu, 13 Aug 2026 23:58:32 -0700 Subject: [PATCH] Probe OTA readiness after reboot --- docs/lora_ota_automation.md | 8 +- docs/rak3401_mota_chain.md | 7 +- tools/lora_ota/lora_ota.py | 136 +++++++++++++++++++++++++-- tools/lora_ota/rak3401_mota_chain.py | 10 +- tools/lora_ota/test_lora_ota.py | 99 +++++++++++++++++++ 5 files changed, 248 insertions(+), 12 deletions(-) diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index a86b586b9..1c9ae04cd 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -418,9 +418,11 @@ the destination. 7. Recheck that exact ID, give the target a short final TempRadio safety window, and request `ota install`. Then shorten each relay's TempRadio window so the normal multi-hop route returns, stop the seeder, shorten the source window, - restore the controller, wait for reboot, and require the new running identity - and exact package version. A source supplied with `--source-already-temp` is - never modified. + restore the controller, and probe `ota self` at 10 and 20 seconds instead of + sleeping for 90 seconds. The exact new body hash is the readiness signal; + only then does the runner require the exact package version. A relayed run + continues the 10-second probes through the mandatory relay-return window. A + source supplied with `--source-already-temp` is never modified. `--leave-controller-radio` moves the controller back to TempRadio only after this normal-channel verification. diff --git a/docs/rak3401_mota_chain.md b/docs/rak3401_mota_chain.md index a77bab6e4..48c2a4800 100644 --- a/docs/rak3401_mota_chain.md +++ b/docs/rak3401_mota_chain.md @@ -199,8 +199,11 @@ The runner performs the following guarded sequence: 2. Recheck destination identity, OTA reach, source identity, and exact package base before every step. 3. Require `system.watchdog` to report `off` immediately before every install. -4. After every reboot, require the exact version and EndF body hash, then prove - the rescue command is present before exposing that bridge to another step. +4. After every install, probe `ota self` at 10 and 20 seconds rather than + sleeping for 90 seconds. Require the exact version and EndF body hash, then + prove the rescue command is present before exposing that bridge to another + step. Relayed runs keep the same 10-second probe cadence until the relays' + mandatory return-to-normal window has elapsed. 5. Re-enable the watchdog only after the exact step-29 endpoint boots. Success requires all of the following: diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index cf2d55c53..c899585cf 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -63,6 +63,8 @@ TEMP_RADIO_SWITCH_DELAY_SECONDS = 3 TEMP_RADIO_RETURN_MINUTES = 1 TEMP_RADIO_RETURN_MARGIN_SECONDS = 15 INSTALL_TARGET_WINDOW_MINUTES = 3 +DEFAULT_POST_INSTALL_READY_WAIT_SECONDS = 20 +POST_INSTALL_READY_PROBE_INTERVAL_SECONDS = 10 COMPANION_TERMINAL_START = "+++MESHCORE-TERM-START" COMPANION_TERMINAL_STOP = "+++MESHCORE-TERM-STOP" # Firmware may hold the apply reboot for up to 15 seconds while its reply @@ -1289,7 +1291,13 @@ class Controller: target: str, command_text: str, password: str, + reply_timeout: int | None = None, ) -> str: + command_reply_timeout = ( + self.reply_timeout if reply_timeout is None else reply_timeout + ) + if command_reply_timeout <= 0: + raise OtaError("remote-command reply timeout must be positive") marker = f"MESHCORE_OTA_{secrets.token_hex(16)}" login_required = not self._remote_auth_is_cached(target) commands = ["contact_info", target] @@ -1299,7 +1307,7 @@ class Controller: "sync_msgs", "echo", marker, "cmd", target, command_text, - "trywait_msg", str(self.reply_timeout), + "trywait_msg", str(command_reply_timeout), "sync_msgs", ]) try: @@ -1364,6 +1372,7 @@ class Controller: *, password: str | None = None, retry: bool = True, + reply_timeout: int | None = None, ) -> str: login_password = self.password if password is None else password normalized = command_text.strip().lower() @@ -1374,7 +1383,7 @@ class Controller: f"{command_text!r} requires state-aware retry handling" ) action = lambda: self._remote_command_once( - target, command_text, login_password + target, command_text, login_password, reply_timeout ) if retry: return retry_transmission( @@ -2259,8 +2268,10 @@ def verify_installed( package: MotaInfo, expected_body_hash: bytes | None, previous_body_hash: bytes | None = None, + identity_reply: str | None = None, ) -> None: - identity_reply = controller.remote_command(args.target, "ota self") + if identity_reply is None: + identity_reply = controller.remote_command(args.target, "ota self") hash_match = re.search(r"\bbase_hash=([0-9A-Fa-f]{16})\b", identity_reply) if not hash_match: raise OtaError("post-reboot `ota self` did not report a running body hash") @@ -2321,6 +2332,99 @@ def verify_installed( ) +def wait_for_post_install_identity( + controller: Controller, + args: argparse.Namespace, + expected_body_hash: bytes | None, + previous_body_hash: bytes | None, + wait_seconds: int, +) -> str: + """Probe the exact running identity instead of sleeping through reboot.""" + if wait_seconds <= 0: + raise OtaError("post-install ready-probe window must be positive") + if expected_body_hash is None and previous_body_hash is None: + raise OtaError( + "post-install readiness needs an expected or previous body hash" + ) + + started = time.monotonic() + probe_at = min(POST_INSTALL_READY_PROBE_INTERVAL_SECONDS, wait_seconds) + last_failure = "no identity reply" + while True: + delay = started + probe_at - time.monotonic() + if delay > 0: + time.sleep(delay) + print( + f"[reboot] probing `ota self` at {probe_at}s " + f"(window {wait_seconds}s)" + ) + try: + reply = controller.remote_command( + args.target, + "ota self", + retry=False, + reply_timeout=min( + POST_INSTALL_READY_PROBE_INTERVAL_SECONDS, + getattr( + controller, + "reply_timeout", + POST_INSTALL_READY_PROBE_INTERVAL_SECONDS, + ), + ), + ) + except TransmissionError as exc: + last_failure = str(exc) + else: + match = re.search(r"\bbase_hash=([0-9A-Fa-f]{16})\b", reply) + if match is None: + raise OtaError( + "post-reboot `ota self` replied without a running body " + f"hash: {reply}" + ) + running_hash = bytes.fromhex(match.group(1)) + if expected_body_hash is not None: + if running_hash == expected_body_hash: + print( + f"[reboot] expected body " + f"{running_hash.hex().upper()} is ready" + ) + return reply + if ( + previous_body_hash is not None + and running_hash == previous_body_hash + ): + last_failure = ( + "destination still reports its pre-install body " + f"{running_hash.hex().upper()}" + ) + else: + raise OtaError( + "post-reboot `ota self` reports an unexpected running " + f"body {running_hash.hex().upper()}" + ) + elif running_hash != previous_body_hash: + print( + f"[reboot] new body {running_hash.hex().upper()} is ready" + ) + return reply + else: + last_failure = ( + "destination still reports its pre-install body " + f"{running_hash.hex().upper()}" + ) + + if probe_at >= wait_seconds: + raise OtaError( + f"destination did not report its installed identity during " + f"the {wait_seconds}s ready-probe window; last result: " + f"{last_failure}" + ) + probe_at = min( + probe_at + POST_INSTALL_READY_PROBE_INTERVAL_SECONDS, + wait_seconds, + ) + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Verify, transfer, and install a MeshCore LoRa OTA package.", @@ -2385,7 +2489,15 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--poll-seconds", type=int, default=30) parser.add_argument("--transfer-timeout-minutes", type=int, default=110) parser.add_argument("--seeder-start-wait", type=int, default=5) - parser.add_argument("--reboot-wait", type=int, default=90) + parser.add_argument( + "--reboot-wait", + type=int, + default=DEFAULT_POST_INSTALL_READY_WAIT_SECONDS, + help=( + "post-install identity-probe window in seconds; `ota self` is " + "scheduled every 10 seconds" + ), + ) parser.add_argument( "--source-already-temp", action="store_true", help="do not configure a TCP source; assert it is already on --temp-radio", @@ -2772,10 +2884,22 @@ def main( TEMP_RADIO_RETURN_MINUTES * 60 + TEMP_RADIO_RETURN_MARGIN_SECONDS if args.relay_values else 0 ) - time.sleep(max(args.reboot_wait, relay_wait)) + ready_probe_window = max(args.reboot_wait, relay_wait) + identity_reply = wait_for_post_install_identity( + controller, + args, + expected_body_hash, + target.base_hash, + ready_probe_window, + ) try: verify_installed( - controller, args, package, expected_body_hash, target.base_hash + controller, + args, + package, + expected_body_hash, + target.base_hash, + identity_reply, ) finally: if args.leave_controller_radio: diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index f454cea10..287cb6cc4 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -1035,7 +1035,15 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--transfer-timeout-minutes", type=int, default=90) parser.add_argument("--seeder-start-wait", type=int, default=5) - parser.add_argument("--reboot-wait", type=int, default=90) + parser.add_argument( + "--reboot-wait", + type=int, + default=ota.DEFAULT_POST_INSTALL_READY_WAIT_SECONDS, + help=( + "post-install identity-probe window in seconds; `ota self` is " + "scheduled every 10 seconds" + ), + ) parser.add_argument("--keep-watchdog-off", action="store_true") mode = parser.add_mutually_exclusive_group() mode.add_argument( diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index f17c208dc..2cba51505 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -708,6 +708,31 @@ class ReliabilityTests(unittest.TestCase): controller._remote_command_once("remote", "ota status", "secret") self.assertIn("login", commands_seen[2]) + def test_remote_command_can_bound_one_reply_wait(self) -> None: + controller = object.__new__(ota.Controller) + controller.reply_timeout = 45 + controller._authenticated_targets = {"remote"} + key = "A1" * 32 + commands_seen: list[str] = [] + + def run_marked(commands: list[str], _label: str, _marker: str): + commands_seen.extend(commands) + return ( + [{"adv_name": "remote", "public_key": key}], + [{ + "txt_type": 1, + "text": "OTA | no download | target:1234ABCD", + "pubkey_prefix": key[:12], + }], + ) + + controller._run_marked = run_marked + controller._remote_command_once( + "remote", "ota status", "secret", reply_timeout=10 + ) + wait_index = commands_seen.index("trywait_msg") + self.assertEqual(commands_seen[wait_index + 1], "10") + def test_silent_commands_retain_cached_admin_session(self) -> None: controller = object.__new__(ota.Controller) controller.reply_timeout = 20 @@ -1016,6 +1041,80 @@ class ReliabilityTests(unittest.TestCase): Controller("v1.16.9 (Build: old)"), args, package, expected_hash ) + def test_post_install_ready_probe_runs_at_ten_and_twenty_seconds(self) -> None: + expected_hash = bytes.fromhex("0011223344556677") + old_hash = bytes.fromhex("8899AABBCCDDEEFF") + + class Controller: + reply_timeout = 45 + + def __init__(self) -> None: + self.calls: list[tuple[str, dict[str, object]]] = [] + + def remote_command( + self, _target: str, command: str, **kwargs: object + ) -> str: + self.calls.append((command, kwargs)) + if len(self.calls) == 1: + raise ota.TransmissionError("still rebooting") + return ( + "self body=1 image=2 " + f"base_hash={expected_hash.hex().upper()}" + ) + + controller = Controller() + with ( + mock.patch.object( + ota.time, "monotonic", side_effect=[0.0, 0.0, 10.0] + ), + mock.patch.object(ota.time, "sleep") as sleep, + ): + reply = ota.wait_for_post_install_identity( + controller, + argparse.Namespace(target="remote"), + expected_hash, + old_hash, + 20, + ) + + self.assertIn(expected_hash.hex().upper(), reply) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [10, 10]) + self.assertEqual([item[0] for item in controller.calls], ["ota self"] * 2) + self.assertTrue(all( + item[1] == {"retry": False, "reply_timeout": 10} + for item in controller.calls + )) + + def test_post_install_ready_probe_rejects_no_endf_reply(self) -> None: + class Controller: + reply_timeout = 45 + + def remote_command(self, *_args: object, **_kwargs: object) -> str: + return "ERR no EndF (firmware lacks the trailer?)" + + with ( + mock.patch.object(ota.time, "monotonic", side_effect=[0.0, 0.0]), + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "without a running body hash"), + ): + ota.wait_for_post_install_identity( + Controller(), + argparse.Namespace(target="remote"), + bytes.fromhex("0011223344556677"), + bytes.fromhex("8899AABBCCDDEEFF"), + 20, + ) + + def test_reboot_ready_probe_defaults_to_twenty_seconds(self) -> None: + parser = ota.build_parser() + args = parser.parse_args([ + "release.mota", "remote", + "--controller-serial", "/dev/controller", + "--source-serial", "/dev/source", + ]) + self.assertEqual(args.reboot_wait, 20) + self.assertEqual(rak_chain.build_parser().parse_args([]).reboot_wait, 20) + def test_post_install_version_falls_back_to_runtime_ver(self) -> None: image = firmware(b"verify-fallback" * 700, VERSION_NEW) package = ota.parse_mota(mota_blob(image))