Probe OTA readiness after reboot

docs/lora_ota_automation.md

@@ -418,9 +418,11 @@ the destination.
 7. Recheck that exact ID, give the target a short final TempRadio safety window,
    and request `ota install`. Then shorten each relay's TempRadio window so the
    normal multi-hop route returns, stop the seeder, shorten the source window,
-   restore the controller, wait for reboot, and require the new running identity
-   and exact package version. A source supplied with `--source-already-temp` is
-   never modified.
+   restore the controller, and probe `ota self` at 10 and 20 seconds instead of
+   sleeping for 90 seconds. The exact new body hash is the readiness signal;
+   only then does the runner require the exact package version. A relayed run
+   continues the 10-second probes through the mandatory relay-return window. A
+   source supplied with `--source-already-temp` is never modified.
    `--leave-controller-radio` moves the controller back to TempRadio only after
    this normal-channel verification.
 

docs/rak3401_mota_chain.md

@@ -199,8 +199,11 @@ The runner performs the following guarded sequence:
 2. Recheck destination identity, OTA reach, source identity, and exact package
    base before every step.
 3. Require `system.watchdog` to report `off` immediately before every install.
-4. After every reboot, require the exact version and EndF body hash, then prove
-   the rescue command is present before exposing that bridge to another step.
+4. After every install, probe `ota self` at 10 and 20 seconds rather than
+   sleeping for 90 seconds. Require the exact version and EndF body hash, then
+   prove the rescue command is present before exposing that bridge to another
+   step. Relayed runs keep the same 10-second probe cadence until the relays'
+   mandatory return-to-normal window has elapsed.
 5. Re-enable the watchdog only after the exact step-29 endpoint boots.
 
 Success requires all of the following:

tools/lora_ota/lora_ota.py

@@ -63,6 +63,8 @@ TEMP_RADIO_SWITCH_DELAY_SECONDS = 3
 TEMP_RADIO_RETURN_MINUTES = 1
 TEMP_RADIO_RETURN_MARGIN_SECONDS = 15
 INSTALL_TARGET_WINDOW_MINUTES = 3
+DEFAULT_POST_INSTALL_READY_WAIT_SECONDS = 20
+POST_INSTALL_READY_PROBE_INTERVAL_SECONDS = 10
 COMPANION_TERMINAL_START = "+++MESHCORE-TERM-START"
 COMPANION_TERMINAL_STOP = "+++MESHCORE-TERM-STOP"
 # Firmware may hold the apply reboot for up to 15 seconds while its reply
@@ -1289,7 +1291,13 @@ class Controller:
         target: str,
         command_text: str,
         password: str,
+        reply_timeout: int | None = None,
     ) -> str:
+        command_reply_timeout = (
+            self.reply_timeout if reply_timeout is None else reply_timeout
+        )
+        if command_reply_timeout <= 0:
+            raise OtaError("remote-command reply timeout must be positive")
         marker = f"MESHCORE_OTA_{secrets.token_hex(16)}"
         login_required = not self._remote_auth_is_cached(target)
         commands = ["contact_info", target]
@@ -1299,7 +1307,7 @@ class Controller:
             "sync_msgs",
             "echo", marker,
             "cmd", target, command_text,
-            "trywait_msg", str(self.reply_timeout),
+            "trywait_msg", str(command_reply_timeout),
             "sync_msgs",
         ])
         try:
@@ -1364,6 +1372,7 @@ class Controller:
         *,
         password: str | None = None,
         retry: bool = True,
+        reply_timeout: int | None = None,
     ) -> str:
         login_password = self.password if password is None else password
         normalized = command_text.strip().lower()
@@ -1374,7 +1383,7 @@ class Controller:
                 f"{command_text!r} requires state-aware retry handling"
             )
         action = lambda: self._remote_command_once(
-            target, command_text, login_password
+            target, command_text, login_password, reply_timeout
         )
         if retry:
             return retry_transmission(
@@ -2259,8 +2268,10 @@ def verify_installed(
     package: MotaInfo,
     expected_body_hash: bytes | None,
     previous_body_hash: bytes | None = None,
+    identity_reply: str | None = None,
 ) -> None:
-    identity_reply = controller.remote_command(args.target, "ota self")
+    if identity_reply is None:
+        identity_reply = controller.remote_command(args.target, "ota self")
     hash_match = re.search(r"\bbase_hash=([0-9A-Fa-f]{16})\b", identity_reply)
     if not hash_match:
         raise OtaError("post-reboot `ota self` did not report a running body hash")
@@ -2321,6 +2332,99 @@ def verify_installed(
     )
 
 
+def wait_for_post_install_identity(
+    controller: Controller,
+    args: argparse.Namespace,
+    expected_body_hash: bytes | None,
+    previous_body_hash: bytes | None,
+    wait_seconds: int,
+) -> str:
+    """Probe the exact running identity instead of sleeping through reboot."""
+    if wait_seconds <= 0:
+        raise OtaError("post-install ready-probe window must be positive")
+    if expected_body_hash is None and previous_body_hash is None:
+        raise OtaError(
+            "post-install readiness needs an expected or previous body hash"
+        )
+
+    started = time.monotonic()
+    probe_at = min(POST_INSTALL_READY_PROBE_INTERVAL_SECONDS, wait_seconds)
+    last_failure = "no identity reply"
+    while True:
+        delay = started + probe_at - time.monotonic()
+        if delay > 0:
+            time.sleep(delay)
+        print(
+            f"[reboot] probing `ota self` at {probe_at}s "
+            f"(window {wait_seconds}s)"
+        )
+        try:
+            reply = controller.remote_command(
+                args.target,
+                "ota self",
+                retry=False,
+                reply_timeout=min(
+                    POST_INSTALL_READY_PROBE_INTERVAL_SECONDS,
+                    getattr(
+                        controller,
+                        "reply_timeout",
+                        POST_INSTALL_READY_PROBE_INTERVAL_SECONDS,
+                    ),
+                ),
+            )
+        except TransmissionError as exc:
+            last_failure = str(exc)
+        else:
+            match = re.search(r"\bbase_hash=([0-9A-Fa-f]{16})\b", reply)
+            if match is None:
+                raise OtaError(
+                    "post-reboot `ota self` replied without a running body "
+                    f"hash: {reply}"
+                )
+            running_hash = bytes.fromhex(match.group(1))
+            if expected_body_hash is not None:
+                if running_hash == expected_body_hash:
+                    print(
+                        f"[reboot] expected body "
+                        f"{running_hash.hex().upper()} is ready"
+                    )
+                    return reply
+                if (
+                    previous_body_hash is not None
+                    and running_hash == previous_body_hash
+                ):
+                    last_failure = (
+                        "destination still reports its pre-install body "
+                        f"{running_hash.hex().upper()}"
+                    )
+                else:
+                    raise OtaError(
+                        "post-reboot `ota self` reports an unexpected running "
+                        f"body {running_hash.hex().upper()}"
+                    )
+            elif running_hash != previous_body_hash:
+                print(
+                    f"[reboot] new body {running_hash.hex().upper()} is ready"
+                )
+                return reply
+            else:
+                last_failure = (
+                    "destination still reports its pre-install body "
+                    f"{running_hash.hex().upper()}"
+                )
+
+        if probe_at >= wait_seconds:
+            raise OtaError(
+                f"destination did not report its installed identity during "
+                f"the {wait_seconds}s ready-probe window; last result: "
+                f"{last_failure}"
+            )
+        probe_at = min(
+            probe_at + POST_INSTALL_READY_PROBE_INTERVAL_SECONDS,
+            wait_seconds,
+        )
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Verify, transfer, and install a MeshCore LoRa OTA package.",
@@ -2385,7 +2489,15 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--poll-seconds", type=int, default=30)
     parser.add_argument("--transfer-timeout-minutes", type=int, default=110)
     parser.add_argument("--seeder-start-wait", type=int, default=5)
-    parser.add_argument("--reboot-wait", type=int, default=90)
+    parser.add_argument(
+        "--reboot-wait",
+        type=int,
+        default=DEFAULT_POST_INSTALL_READY_WAIT_SECONDS,
+        help=(
+            "post-install identity-probe window in seconds; `ota self` is "
+            "scheduled every 10 seconds"
+        ),
+    )
     parser.add_argument(
         "--source-already-temp", action="store_true",
         help="do not configure a TCP source; assert it is already on --temp-radio",
@@ -2772,10 +2884,22 @@ def main(
             TEMP_RADIO_RETURN_MINUTES * 60 + TEMP_RADIO_RETURN_MARGIN_SECONDS
             if args.relay_values else 0
         )
-        time.sleep(max(args.reboot_wait, relay_wait))
+        ready_probe_window = max(args.reboot_wait, relay_wait)
+        identity_reply = wait_for_post_install_identity(
+            controller,
+            args,
+            expected_body_hash,
+            target.base_hash,
+            ready_probe_window,
+        )
         try:
             verify_installed(
-                controller, args, package, expected_body_hash, target.base_hash
+                controller,
+                args,
+                package,
+                expected_body_hash,
+                target.base_hash,
+                identity_reply,
             )
         finally:
             if args.leave_controller_radio:

tools/lora_ota/rak3401_mota_chain.py

@@ -1035,7 +1035,15 @@ def build_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument("--transfer-timeout-minutes", type=int, default=90)
     parser.add_argument("--seeder-start-wait", type=int, default=5)
-    parser.add_argument("--reboot-wait", type=int, default=90)
+    parser.add_argument(
+        "--reboot-wait",
+        type=int,
+        default=ota.DEFAULT_POST_INSTALL_READY_WAIT_SECONDS,
+        help=(
+            "post-install identity-probe window in seconds; `ota self` is "
+            "scheduled every 10 seconds"
+        ),
+    )
     parser.add_argument("--keep-watchdog-off", action="store_true")
     mode = parser.add_mutually_exclusive_group()
     mode.add_argument(

tools/lora_ota/test_lora_ota.py

@@ -708,6 +708,31 @@ class ReliabilityTests(unittest.TestCase):
         controller._remote_command_once("remote", "ota status", "secret")
         self.assertIn("login", commands_seen[2])
 
+    def test_remote_command_can_bound_one_reply_wait(self) -> None:
+        controller = object.__new__(ota.Controller)
+        controller.reply_timeout = 45
+        controller._authenticated_targets = {"remote"}
+        key = "A1" * 32
+        commands_seen: list[str] = []
+
+        def run_marked(commands: list[str], _label: str, _marker: str):
+            commands_seen.extend(commands)
+            return (
+                [{"adv_name": "remote", "public_key": key}],
+                [{
+                    "txt_type": 1,
+                    "text": "OTA | no download | target:1234ABCD",
+                    "pubkey_prefix": key[:12],
+                }],
+            )
+
+        controller._run_marked = run_marked
+        controller._remote_command_once(
+            "remote", "ota status", "secret", reply_timeout=10
+        )
+        wait_index = commands_seen.index("trywait_msg")
+        self.assertEqual(commands_seen[wait_index + 1], "10")
+
     def test_silent_commands_retain_cached_admin_session(self) -> None:
         controller = object.__new__(ota.Controller)
         controller.reply_timeout = 20
@@ -1016,6 +1041,80 @@ class ReliabilityTests(unittest.TestCase):
                 Controller("v1.16.9 (Build: old)"), args, package, expected_hash
             )
 
+    def test_post_install_ready_probe_runs_at_ten_and_twenty_seconds(self) -> None:
+        expected_hash = bytes.fromhex("0011223344556677")
+        old_hash = bytes.fromhex("8899AABBCCDDEEFF")
+
+        class Controller:
+            reply_timeout = 45
+
+            def __init__(self) -> None:
+                self.calls: list[tuple[str, dict[str, object]]] = []
+
+            def remote_command(
+                self, _target: str, command: str, **kwargs: object
+            ) -> str:
+                self.calls.append((command, kwargs))
+                if len(self.calls) == 1:
+                    raise ota.TransmissionError("still rebooting")
+                return (
+                    "self body=1 image=2 "
+                    f"base_hash={expected_hash.hex().upper()}"
+                )
+
+        controller = Controller()
+        with (
+            mock.patch.object(
+                ota.time, "monotonic", side_effect=[0.0, 0.0, 10.0]
+            ),
+            mock.patch.object(ota.time, "sleep") as sleep,
+        ):
+            reply = ota.wait_for_post_install_identity(
+                controller,
+                argparse.Namespace(target="remote"),
+                expected_hash,
+                old_hash,
+                20,
+            )
+
+        self.assertIn(expected_hash.hex().upper(), reply)
+        self.assertEqual([call.args[0] for call in sleep.call_args_list], [10, 10])
+        self.assertEqual([item[0] for item in controller.calls], ["ota self"] * 2)
+        self.assertTrue(all(
+            item[1] == {"retry": False, "reply_timeout": 10}
+            for item in controller.calls
+        ))
+
+    def test_post_install_ready_probe_rejects_no_endf_reply(self) -> None:
+        class Controller:
+            reply_timeout = 45
+
+            def remote_command(self, *_args: object, **_kwargs: object) -> str:
+                return "ERR no EndF (firmware lacks the trailer?)"
+
+        with (
+            mock.patch.object(ota.time, "monotonic", side_effect=[0.0, 0.0]),
+            mock.patch.object(ota.time, "sleep"),
+            self.assertRaisesRegex(ota.OtaError, "without a running body hash"),
+        ):
+            ota.wait_for_post_install_identity(
+                Controller(),
+                argparse.Namespace(target="remote"),
+                bytes.fromhex("0011223344556677"),
+                bytes.fromhex("8899AABBCCDDEEFF"),
+                20,
+            )
+
+    def test_reboot_ready_probe_defaults_to_twenty_seconds(self) -> None:
+        parser = ota.build_parser()
+        args = parser.parse_args([
+            "release.mota", "remote",
+            "--controller-serial", "/dev/controller",
+            "--source-serial", "/dev/source",
+        ])
+        self.assertEqual(args.reboot_wait, 20)
+        self.assertEqual(rak_chain.build_parser().parse_args([]).reboot_wait, 20)
+
     def test_post_install_version_falls_back_to_runtime_ver(self) -> None:
         image = firmware(b"verify-fallback" * 700, VERSION_NEW)
         package = ota.parse_mota(mota_blob(image))