From b40d2e6cb2a185db740175e8b161ea71736d19d9 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Sat, 29 Aug 2026 06:58:48 -0700 Subject: [PATCH] fix: harden LoRa OTA chain and storage --- .gitignore | 6 + build.sh | 7 + docs/lora_ota_automation.md | 115 +- docs/ota_protocol.md | 11 +- docs/ota_user_guide.md | 13 +- docs/rak3401_mota_chain.md | 332 +++- src/helpers/ota/OtaCli.cpp | 48 +- src/helpers/ota/OtaFlashLayout_esp32.h | 129 ++ src/helpers/ota/OtaFlashLayout_nrf52.h | 67 + src/helpers/ota/OtaStore.h | 19 + src/helpers/ota/OtaStoreFlashEsp32.cpp | 202 ++- src/helpers/ota/OtaStoreFlashEsp32.h | 19 +- src/helpers/ota/OtaStoreFlashNrf52.cpp | 67 +- src/helpers/ota/OtaStoreFlashNrf52.h | 11 +- src/helpers/ota/OtaStoreQspiNrf52.cpp | 11 +- src/helpers/ota/OtaStoreQspiNrf52.h | 1 + src/helpers/ota/OtaStoreSdNrf52.cpp | 21 +- src/helpers/ota/OtaStoreSdNrf52.h | 1 + .../sensors/EnvironmentSensorManager.cpp | 15 + test/test_build_profiles.sh | 19 + test/test_ota/test_ota_core.cpp | 69 + test/test_ota/test_ota_flashplan.cpp | 302 ++++ .../lora_ota/build_rak3401_compact_bundle.py | 1178 +++++++++++- tools/lora_ota/lora_ota.py | 893 +++++++-- tools/lora_ota/rak3401_mota_chain.py | 1165 ++++++++++-- tools/lora_ota/rak3401_route_search.py | 638 +++++++ .../test_build_rak3401_compact_bundle.py | 688 +++++++ tools/lora_ota/test_lora_ota.py | 1597 ++++++++++++++++- tools/lora_ota/test_rak3401_route_search.py | 381 ++++ 29 files changed, 7497 insertions(+), 528 deletions(-) create mode 100644 src/helpers/ota/OtaFlashLayout_esp32.h create mode 100755 tools/lora_ota/rak3401_route_search.py create mode 100644 tools/lora_ota/test_build_rak3401_compact_bundle.py create mode 100644 tools/lora_ota/test_rak3401_route_search.py diff --git a/.gitignore b/.gitignore index f17f4df4..e2568fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,12 @@ site/ out.*/ out-*/ build-logs/ +# Local resume state, extracted firmware, and physical-test logs from the +# pinned RAK3401 chain runner. These may contain device-specific settings. +rak3401-mota-chain-work/ +# Generic runner work directories can also contain endpoint-bound recovery +# settings and diagnostic logs. Never stage them with a broad `git add`. +meshcore-lora-ota-*/ .direnv/ .DS_Store .vscode/settings.json diff --git a/build.sh b/build.sh index 78a5d40a..8e5c30b4 100755 --- a/build.sh +++ b/build.sh @@ -2714,6 +2714,13 @@ declare_build_capability_contract() { fi if is_rak_i2c_voltage_monitor_ota_target "$env_name"; then + # These two reduced RAK OTA profiles deliberately retain their board GPS + # provider as well as the compact INA monitor set. Keep the release + # manifest honest so a space-oriented target name is not mistaken for a + # GPS-less image. This evidence is emitted only by the linked WisBlock GPS + # provider; generic CLI command text is not sufficient proof. + record_build_expectation \ + "sensor.gps" "meshcore.capability.rak_wisblock_gps.v1" record_build_expectation "sensor.ina219" "INA219" record_build_expectation "sensor.ina226" "INA226" record_build_expectation "sensor.ina260" "INA260" diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index 1e624af2..ebf3c313 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -262,6 +262,22 @@ runner can restore those periods but cannot reconstruct an unreported saved level. A radio change later recalculates from a preserved saved minimum, so moving back to a slower tuple returns to the operator's saved level. +The OTA source has a stricter policy than the destination. For every source +with a managed serial or TCP CLI, the runner reads and retains its exact RXPS +preference, using the legacy fixed-period query only when the detailed query is +unavailable. It then disables RXPS and verifies the readback before any target +discovery or TempRadio change. Source RXPS stays off through catalog serving, +download, installation, and post-install identity verification. Cleanup first +proves that the source has returned to its normal radio, then restores and +verifies the exact saved level/preamble or fixed-period state once. A source +whose RXPS state cannot be read, disabled, or restored safely fails closed. +If current firmware explicitly rejects an RXPS disable or restore with `radio +busy; retry`, the runner retries that idempotent mutation at staggered +210–378 ms intervals. All 32 delays are distinct and contribute about 9.4 +seconds of waits; source-CLI command round-trip time is additional. This avoids +repeatedly sampling one radio phase while retaining a strict attempt cap. Other +rejections are not replayed. + ## 4. Run an ESP32 update The ZIP can contain a compatible ready `.mota` or the exact board-and-role @@ -427,7 +443,9 @@ later as a catalog timeout. If the source is already on the exact TempRadio tuple through a scheduled or manual operation, `--source-already-temp` lets a TCP source run without a raw CLI link. The script cannot verify, extend, or shorten that source window, so -leave a comfortable time margin. +leave a comfortable time margin. It also cannot inspect or change that +unmanaged source's RXPS state; disable source RXPS yourself before starting +the run and restore it only after returning the source to its normal radio. Use `--controller-baud` or `--source-baud` only for a build whose corresponding interface is genuinely configured to another speed. @@ -458,7 +476,9 @@ Useful controls: - `--no-install` downloads and verifies the image but leaves it staged. By default the runner then schedules the target, relays, and a script-configured source back to their normal radios. Combining it with - `--leave-controller-radio` deliberately preserves the TempRadio topology. + `--leave-controller-radio` deliberately preserves the destination, relays, + and separate controller on the TempRadio topology; a managed source is still + returned to normal so its exact RXPS setting can be restored. If the version gate required RXPS off, it stays off while that topology is preserved; use `target-rxps-settings.json` to restore it only after sending `normalradio`. @@ -467,10 +487,14 @@ Useful controls: downloading or staged on the target. Without it, that update is preserved. - `--source-shares-controller` is for a Full Companion whose USB Binary API is the controller while its TCP port `5001` is the source. It verifies that the - source's port-`5000` public key equals the controller key. The Binary API - first moves the shared physical radio, port `5002` then enables the local OTA - egress gate, and cleanup sends `normalradio` before restoring the saved - Binary radio tuple. + source's port-`5000` public key equals the controller key. Port `5002` uses a + bounded local `tempradio` override to move the shared physical radio without + overwriting its saved normal tuple; the Binary API remains the authenticated + transport. Cleanup sends local `normalradio`, proves that override inactive, + and then reasserts the saved Binary tuple. It cannot be combined with + `--leave-controller-radio`, + because exact source RXPS restoration requires that shared physical radio to + be back on its verified normal tuple. - `--require-system-watchdog-off` checks `get system.watchdog` immediately before every `ota install` transmission and refuses installation unless the destination reports `> off`. Use it for nRF52 chains whose bootloader cannot @@ -511,12 +535,14 @@ the destination. hashes, Merkle root, full-image hash where applicable, identity fields, signature, codec, base, and the firmware's 1024-byte maximum block size. Version-3 bootloader packages are refused before any target state changes. -4. Save the controller's normal radio tuple, read every participant's version, - save the destination's RXPS state, select the qualified RXPS policy, and - show the confirmation prompt. -5. Apply and verify that RXPS policy, then start TempRadio on the target, - far-to-near relays, and the source; - finally switch the controller to the same tuple and read it back. The runner +4. Read and save a managed source's exact RXPS preference, disable and verify + source RXPS, save the controller's normal radio tuple, read every + participant's version, save the destination's RXPS state, select the + qualified destination policy, and show the confirmation prompt. +5. Apply and verify the destination RXPS policy, then start TempRadio on the + target, far-to-near relays, and source. A separate controller is moved and + read back through Binary; a shared Full Companion instead schedules its + bounded local override while Binary remains the transport. The runner rejects a TempRadio window that cannot cover setup, seeder startup, discovery, the transfer timeout, final polling, and install checks. 6. Start `motatool serve`, discover the exact eight-hex manifest ID, request @@ -527,14 +553,17 @@ the destination. 7. Recheck that exact ID, give the target a short final TempRadio safety window, and request `ota install`. Then shorten each relay's TempRadio window so the normal multi-hop route returns, stop the seeder, shorten the source window, - restore the controller, and probe `ota self` at 10 and 20 seconds instead of - sleeping for 90 seconds. The exact new body hash is the readiness signal; + restore the controller, and probe `ota self` every 10 seconds through the + configured readiness window (five minutes by default). The exact new body + hash is the readiness signal; only then does the runner require the exact package version. A relayed run continues the 10-second probes through the mandatory relay-return window. A source supplied with `--source-already-temp` is never modified. `--leave-controller-radio` moves the controller back to TempRadio only after this normal-channel verification. Restore the destination's exact original - RXPS setting after normal-channel identity is proven. + RXPS setting after normal-channel identity is proven. A managed source stays + RXPS-off through that verification and its exact setting is restored only + after its own TempRadio state is proven inactive. Remote replies are matched only after queued messages have been drained and only when they come from the intended contact and fit the command. A ready @@ -565,26 +594,54 @@ budget. If a bounded window expires, rerun the same package after the nodes return to their normal channel; the manifest-ID check resumes its partial download without replacing it. -The working directory is retained and printed at exit. It contains the exact +The working directory is created before a managed source can be changed, then +retained and printed at exit. It contains the exact served mOTA, `motatool-serve.log`, extracted build inputs when needed, and -`controller-radio.txt`. When the destination started with RXPS enabled it also -contains protected `target-rxps-settings.json` for manual recovery. It contains -no saved admin password. +`controller-radio.txt`. A managed source also gets a protected +`source-rxps-settings.json` containing its exact original preference and +idempotent restore command. Its contents and directory entry are flushed before +RXPS is disabled. When the destination +started with RXPS enabled, protected `target-rxps-settings.json` records its +manual recovery state. The RAK3401 chain points every nested step at one +chain-root source record, so a rerun after host power loss does not adopt the +temporary RXPS-off state as the original. A retained record is accepted only +for the same managed CLI endpoint. These files contain no saved admin password. +After exact source restoration, a standalone run atomically retires its record; +the chain retains its shared record between steps and retires it only after the +verified endpoint restoration completes. ## Interruption and recovery -Ctrl-C stops the seeder, detaches its serial folder, makes one best-effort -request to shorten a source TempRadio window started by the script, and attempts -to restore the controller. The target and relays remain on TempRadio only until -their bounded windows end; rebooting also restores their saved radio settings. -A normal cleanup restores the destination's exact RXPS periods. If that remote -restore cannot be confirmed, use `target-rxps-settings.json` after the target -returns to its normal channel. +Ctrl-C stops the seeder, detaches its serial folder, makes a bounded attempt to +shorten a source TempRadio window started by the script, and attempts to +restore the controller. For a managed source it leaves RXPS off until the +source is proven back on its normal radio, then restores and verifies the +saved source preference. A transient success-path restore failure remains +armed for one more idempotent cleanup attempt. The target and relays remain on +TempRadio only until their bounded windows end; rebooting also restores their +saved radio settings. A normal cleanup restores the destination's exact RXPS +periods. If that remote restore cannot be confirmed, use +`target-rxps-settings.json` after the target returns to its normal channel. A partial download remains safe. Once the target is reachable again (after its TempRadio window ends, or after putting the controller back on that tuple), rerunning the same package recognizes its manifest ID and resumes the existing session instead of clearing it. +When a chained run has already proved the newly running body hash, its retained +previous package can briefly report `verifying staged blocks` after TempRadio +reactivates the OTA manager. The runner waits only through the configured +discovery timeout, keeps checking source liveness, and accepts only the same +manifest becoming `ready to install` or the manager becoming idle. A changed +ID, failed or incomplete state, or timeout stops the chain. It then proves the +exact installed body again. If that same session is still attached and ready, +the runner detaches it with `ota cancel`; if the manager has become idle, it +sends no cancel. An ordinary-channel `no download` status proves only that the +manager is idle, not that persistent staging was erased. The runner therefore +does not issue or describe an IDLE cancel as durable cleanup. The next chain +transition explicitly re-adopts and proves the expected previous MID before detaching it; +after the final install OTAFIX has consumed the approval word, so any retained +container is inert and is replaced by the next valid pull. + A hard process kill or host power loss cannot run cleanup. Recover a serial controller using the tuple saved in the printed work directory: @@ -598,6 +655,12 @@ $radio = (Get-Content '.\meshcore-lora-ota-...\controller-radio.txt' -Raw).Trim( meshcli -s COM7 set radio $radio ``` +For a managed source, first return it to its ordinary radio, then inspect +`source-rxps-settings.json` and issue its exact `restore_command` through the +same serial or TCP-console endpoint recorded in that file. The command is +idempotent; confirm the full setting with `get radio.rxps.config` before +resuming an update. + If you stop during final confirmation, reconnect on the node's normal channel and run `ota self` and `ver`. A completed run returns success only when `ota self` reports a valid new body hash and `ver` exactly matches the package; diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index ea5f4632..603412b7 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -824,7 +824,7 @@ ota rescue install internal-flash nRF52 only: recover from failed ota bootloader [status] capable allowlisted nRF52 repeater: installed BL identity/caps + staged confirmation ota bootloader install explicitly verify/arm one complete trusted v3 package; never automatic -ota cancel | drop | stop drop the current fetch session (frees the slot) +ota cancel | drop | stop drop the fetch; durably invalidate device staging, or retain a folder partial for resume ota announce | adv serve self + send a beacon now ota self | id print this firmware's EndF (body/image size, base_hash) ota qspi | storage QSPI nRF52 only: JEDEC/SR1/stage/latched storage error (read-only) @@ -834,6 +834,15 @@ ota key | keys [add|rm ] trusted signer allowlist ; bare = list ota dev ... bring-up helpers (stage/recv/serve/resume /verify) ``` +For a device-backed pull, current firmware returns success only after the +persistent store can no longer be reopened; flash/SD/QSPI I/O or readback +failure is reported as an error even though the in-memory manager session was +dropped. For a `folder` pull, cancellation detaches the live transfer but +deliberately leaves the host `.part` file available for a later resume. +If the shared receive engine is currently performing the MeshTower SD +auto-archive capture, cancellation likewise detaches that archive transfer and +retains its `.part` file; it does not erase the unrelated manual-install store. + --- diff --git a/docs/ota_user_guide.md b/docs/ota_user_guide.md index 20fc044a..9d041285 100644 --- a/docs/ota_user_guide.md +++ b/docs/ota_user_guide.md @@ -212,6 +212,15 @@ falls back to flash. To **stop** a download you no longer want: ota cancel ``` +For a download staged on the node, success means current firmware also +invalidated the persistent flash/SD/QSPI copy; it reports an error if that +media operation cannot be verified. For a `folder` capture, cancel only +detaches the live session and retains the host partial so it can be resumed or +removed on the host deliberately. On a MeshTower SD auto-archive capture, +cancel also detaches the live archive session and retains its card partial for +the archive service to resume later; use `ota cache off` to stop new archive +captures. + ### 4. Install a downloaded update Once `ota status` shows the download is **ready to install**: @@ -292,7 +301,9 @@ complete target inventory, storage layouts, and safety contract. ### 5. If something goes wrong -- A download that stalls or gets interrupted just **resumes** later, or you can `ota cancel` and try again. +- A download that stalls or gets interrupted just **resumes** later. Use + `ota cancel` to durably discard device staging; a folder capture keeps its + host partial for a later resume. - A legacy app-only internal-flash **nRF52** that still runs but reports `no EndF` can use the pre-provisioned rescue path if its physical EndF is intact and only app-side validation is failing. Fetch the exact `[rescue]` in-place delta with an explicit acknowledgement, obtain its 16-hex-digit `base_hash` from the package diff --git a/docs/rak3401_mota_chain.md b/docs/rak3401_mota_chain.md index acb9b148..65642c87 100644 --- a/docs/rak3401_mota_chain.md +++ b/docs/rak3401_mota_chain.md @@ -1,24 +1,32 @@ # RAK3401 1W repeater compact LoRa update chain -> Status: physically qualified on 19-Aug-2026. All nine exact transitions -> completed on a deployed RAK3401 1W with its existing bootloader, using a -> Heltec V4 source at 909.950 MHz / 500 kHz / SF5 / CR5. Every post-boot EndF -> hash matched. The same files also pass independent reconstruction, container -> verification, and both deployed Preview 6 and current OTAFIX simulators. +> Status: **unreleased candidate with physically passed package transitions.** +> On 29-Aug-2026 all ten exact package transitions completed on the target +> RAK3401 with its deployed `0.9.2-OTAFIX2.4` bootloader and a Heltec V4 source at +> 909.950 MHz / 500 kHz / SF5 / CR5. Every intermediate EndF hash matched. +> Independent SWD readback then matched the endpoint application, original +> bootloader, SoftDevice/MBR, and UICR byte-for-byte. The host runner received +> cleanup/recovery fixes after that test and has not had a new clean end-to-end +> physical rerun. Multi-hop and alternate-bandwidth estimates remain planning +> data, not physical qualification. -Use this asset: +The candidate is a local qualification artifact, not a GitHub release: ```text -RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip +RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip ``` -- ZIP SHA-256: `9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a` -- inner `SHA256SUMS.txt` SHA-256: `73d96e23237896a3e342fe736be12d94087a813bf09ad609fb55330bbe586055` -- endpoint image SHA-256: `2784e4b645bc3dc198de0b8b18d3d7369cd02eca61cd71c46a51b61854da5345` -- endpoint EndF body hash: `4BB1526BF647547D` +- candidate ZIP SHA-256: `c0b33f4568985e8b2b8dc99411295907212cf2bad21764b6333d5e0ba298fd61` +- inner `SHA256SUMS.txt` SHA-256: `3f8c4af8096b96a4aa6506825c387cc8a06f74d5213a29c9387bd11689546881` +- endpoint image SHA-256: `31c182c888ceb1135e5afb2376610d93cee2e807b556c838e07fd4486c79d095` +- endpoint EndF body hash: `9179B98432895924` -Until that asset is uploaded to the release tag pinned by the runner, pass its -local path with `--bundle`. +Always pass the exact local path with `--bundle`. The runner has no download +URL for this candidate. `--verify-only` remains the safe default. The hidden +`--accept-test-candidate` override is intentionally absent from `--help` and +remains required until this exact artifact is published with its qualification +record; direct physical success does not silently turn a local file into a +released deployment asset. ## Exact destination @@ -29,29 +37,32 @@ target: - hardware: `RAK_3401` - role: `RAK_3401_repeater_lora_ota_no_external_sensors` - start: `v1.16.7.0-c1caa5ad`, EndF `71F4026CBE4B8B74` -- endpoint source: MeshCore `e742333a0ee393b0d55d2414f25b28f2f67e1ea1` -- endpoint label: `v1.17.1.02-halo-keymind-cascade-dev-e742333a` -- endpoint packed version: `0x01110102` (EndF tools render this as `1.17.1.2`) +- endpoint source: MeshCore `fd98bc90f7fb25d5956facd550e227b217abbe95` +- endpoint label: `v1.17.1.5-halo-keymind-cascade-dev-fd98bc90` +- endpoint packed version: `0x01110105` - deployment target key: `63d8df6387eaffd2e25db7d2a8ad967a65202182a48d681d7e7a9260f917280d` -### Historical endpoint GPS limitation +### Retained and omitted hardware support -This chain's exact `RAK_3401_repeater_lora_ota_no_external_sensors` endpoint -compiles with `ENV_INCLUDE_GPS` removed. It will not detect, configure, or read -a RAK12501 GPS module. GPS requires the ordinary full-sensor -`RAK_3401_repeater` firmware and the RAK12501 must be installed in sensor slot -A. Do not use slot D with the RAK13302 1 W radio: the GPS reset/PPS signals -would overlap the radio's BUSY/DIO1 signals. The full-sensor build is a -different target and is not an endpoint of this compact OTA chain. +Despite the legacy `no_external_sensors` target name, the endpoint retains the +RAK12501 GPS provider and the common INA3221, INA219, INA226, and INA260 I2C +voltage/current monitors. Install RAK12501 in sensor slot A. Do not use slot D +with the RAK13302 1 W radio because the GPS reset/PPS signals overlap the +radio's BUSY/DIO1 wiring. -This limitation describes the immutable released chain endpoint identified -above. Newly built RAK3401 OTA repeater firmware retains RAK12501 support in -slot A, but that newer image is not retroactively part of this exact chain. +The reduced profile omits optional external environmental and ranging devices: +AHTX0, BME280, BMP280, SHTC3, SHT4x, LPS22HB, MLX90614, VL53L0X, BME680, +BMP085, RAK12035, and BME680 BSEC. Board radio, display, buttons, battery +telemetry, GPS, and the INA monitor family remain. No additional “remove I2C +temperature” image is needed: the optional I2C temperature/environment drivers +are already in the omitted set, while removing the remaining I2C/INA support +would discard desired voltage/current monitoring and create another firmware +identity without helping this route. Do not use the chain on another target ID, hardware family, starting image, or firmware body hash. The runner checks all four. -## Why the old bootloader works +## Why Preview 5 and the old staging ceiling work The deployed bootloader is not changed and no package uses the newer expanded `0xED000` ceiling. Each package remains bottom-aligned below the old @@ -66,31 +77,43 @@ The old bootloader already reads and validates that encoded detools geometry before its first destructive write. The previous application receiver was the limiting part: it reserved a fixed `0x98000` workspace before accepting a download. Step 1 therefore remains byte-for-byte identical to the physically -passed 89,844-byte package. Step 2 is a 46,363-byte fixed-workspace package +passed 89,844-byte package. Step 2 is a 46,364-byte fixed-workspace package that installs the compatibility receiver. Later receivers stage above the real EndF image extent and let the bootloader enforce each package's selected workspace. -An exhaustive search tested every page-aligned workspace relevant to a route -shorter than nine packages: 9,430 candidate geometries across 272 possible -layer-skipping edges. None fit. The 61 edges on the shortest-path graph were -then swept across another 2,296 workspace choices; 840 nine-package routes -were compared by total transfer size. The selected route is the byte-minimum -shortest route: +Every package in this candidate is an ordinary application container using +format ABI 2 and in-place codec 2. The historical bridge applications scan the +bootloader capability marker byte-by-byte, so they recognize the released +Preview 5 RAK4631 marker at absolute address `0xFCDEA` even though it is two +bytes off a word boundary. The `fd98bc90` endpoint deliberately retains that +exact Preview 5 application-update fallback. Its separately compiled internal +bootloader-update feature remains unavailable on Preview 5 because privileged +self-update requires a current ABI 3 marker with the exact storage profile. +Installing this chain therefore does not replace the bootloader, and the final +application can still accept a future valid ABI-2/codec-2 application update. + +The schema-2 exhaustive search checked 17,402 candidate geometries and found +121 feasible forward edges in the declared 33-image inventory. Fourteen +ten-package routes tied for the minimum package count; the selected route has +the smallest total container size among them. That minimum applies to the +declared inventory, not every conceivable firmware image. The selected route +is: | Step | From | To | Workspace | Package | Margin | |---:|---|---|---:|---:|---:| | 1 | 1.16.7.0 | 1.16.7.9 | `0x98000` | 89,844 | 0 | -| 2 | 1.16.7.9 | 1.16.7.10 | `0x98000` | 46,363 | 40,960 | -| 3 | 1.16.7.10 | 1.16.8.7 | `0x91000` | 106,030 | 12,288 | +| 2 | 1.16.7.9 | 1.16.7.10 | `0x98000` | 46,364 | 40,960 | +| 3 | 1.16.7.10 | 1.16.8.7 | `0x91000` | 106,029 | 12,288 | | 4 | 1.16.8.7 | 1.16.9.105 | `0x89000` | 149,927 | 0 | -| 5 | 1.16.9.105 | 1.16.9.110 | `0x8E000` | 111,307 | 16,384 | -| 6 | 1.16.9.110 | 1.16.9.111 | `0x8D000` | 70,679 | 61,440 | -| 7 | 1.16.9.111 | 1.16.9.116 | `0x89000` | 143,441 | 4,096 | -| 8 | 1.16.9.116 | 1.16.9.118 | `0x81000` | 98,188 | 86,016 | -| 9 | 1.16.9.118 | 1.17.1.02 | `0x7D000` | 186,385 | 12,288 | +| 5 | 1.16.9.105 | 1.16.9.110 | `0x8E000` | 111,312 | 16,384 | +| 6 | 1.16.9.110 | 1.16.9.113 | `0x8F000` | 120,624 | 4,096 | +| 7 | 1.16.9.113 | 1.16.9.117 | `0x8B000` | 139,637 | 0 | +| 8 | 1.16.9.117 | 1.16.10.0 | `0x7E000` | 174,202 | 20,480 | +| 9 | 1.16.10.0 | 1.17.1.3 | `0x6B000` | 271,569 | 0 | +| 10 | 1.17.1.3 | 1.17.1.5 | `0x74000` | 173,334 | 61,440 | -Total mOTA transfer data is 1,002,164 bytes. `ROUTE.json`, `CHAIN.csv`, and +Total mOTA transfer data is 1,382,842 bytes. `ROUTE.json`, `CHAIN.csv`, and `validation-results.json` in the bundle pin the exact geometry and image hash for every transition. @@ -105,17 +128,27 @@ No password or device is needed: ```bash python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip \ --work-dir ./rak3401-mota-chain-work \ --motatool /path/to/motatool \ --verify-only ``` -The runner checks the outer ZIP, complete inner checksum coverage, all nine -manifests, continuity, image anchors, final recovery image, and every container -with `motatool`. +When given the ZIP, the runner checks its pinned outer hash and binds the +extraction cache to that exact archive. An explicitly supplied extracted root +starts at the pinned inner checksum list. In either case it checks complete +inner coverage, all ten manifests, continuity, image anchors, both recovery +images, and every container with `motatool`. Passing this command means offline +and simulator qualification only; it does not unlock live use. -## Direct recoverable bench run +## Current guarded reproduction command + +These are the same direct-link arguments used for the physical transition run, +but the current host runner also contains later cleanup and recovery fixes. +That revised runner has not been cleanly rerun through all ten transitions. +Keep local USB/SWD recovery available while the artifact remains unreleased. +The hidden override bypasses only the candidate-status block; all identity, +checksum, bootloader, route, watchdog, and post-boot gates still run. Restore the test start locally with `recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.uf2`, then run: @@ -124,7 +157,7 @@ Restore the test start locally with export MESHCORE_ADMIN_PASSWORD='password' python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip \ --work-dir ./rak3401-mota-chain-work \ --controller-serial /dev/ttyACM0 \ --source-tcp 192.168.1.51:5001 \ @@ -135,6 +168,7 @@ python3 tools/lora_ota/rak3401_mota_chain.py \ --ota-hops 0 \ --legacy-full-airtime \ --motatool /path/to/motatool \ + --accept-test-candidate \ --yes ``` @@ -145,23 +179,76 @@ zero and restores it at the endpoint. Use that option only where the selected frequency and local duty-cycle rules permit a full transmit budget; omit it otherwise. +### Measured direct-link timing + +The first `ready to install` transfer ages were: + +| Step | Blocks | Transfer | +|---:|---:|---:| +| 1 | 88 | 598 s | +| 2 | 45 | 42 s | +| 3 | 103 | 81 s | +| 4 | 146 | 105 s | +| 5 | 109 | 76 s | +| 6 | 118 | 94 s | +| 7 | 136 | 102 s | +| 8 | 170 | 127 s | +| 9 | 264 | 187 s | +| 10 | 169 | 193 s | + +The physically observed bulk-transfer total for those exact packages is 1,605 +seconds (26m45s). The log came from the pre-cleanup-fix runner revision. Step 1 +was measured before the host began forcing source RXPS off and is therefore a +conservative outlier. A new complete run with that fix is projected at roughly +18–19 minutes of bulk transfer, but that projection is not a substitute for +the measured total. Allow about 70–75 minutes direct at BW500 for validation, +ten transfers, installs/reboots, retained-store checks, and final restoration. + +Step 10's longer per-block time was not a V4 TempRadio delay: the source RXPS +readback was off. Its v1.17.1.3 requester uses non-overlapping adaptive flights +and a five-second recovery floor and logged more block reloads than step 8. +The V4 itself intentionally schedules its local radio handoff after 1.5 seconds +so the normal-channel command reply can drain; the host waits three seconds and +then checks the live TempRadio state. + +For planning only, scale the conservative measured 26m45s bulk baseline by the +qualified adaptive-preamble airtime and by each additional relay transmission: + +| Bandwidth, SF5/CR5 | Direct / 0 relays | 1 relay | 2 relays | +|---:|---:|---:|---:| +| 500 kHz | 26m45s | 53m30s | 1h20m15s | +| 250 kHz | 46m51s | 1h33m42s | 2h20m33s | +| 125 kHz | 1h27m03s | 2h54m07s | 4h21m11s | +| 62.5 kHz | 2h54m07s | 5h48m15s | 8h42m23s | + +These are transfer-only estimates, not measured alternate-bandwidth or relay +results. Packet loss, relay contention, discovery, administrative commands, +staged verification, and ten reboot cycles add time. The bandwidth factors are +not simple powers of two because the adaptive wire preamble is 128 symbols at +500 kHz, 64 at 250 kHz, and 32 at 125/62.5 kHz. + ## Manual operation -The pinned runner is the safer method. If it cannot be used, follow every -check below and keep a written copy of each original setting. Do not skip a -step even when a later package appears in `ota ls`. +The automated runner is preferred because it binds every transition to the +expected MID and body hash. The manual details below document the checks used +during qualification and recovery; they are not permission to skip the +runner's candidate gate. Keep a written copy of each original setting and +never skip a step even when a later package appears in `ota ls`. ### 1. Verify and extract the asset ```bash -sha256sum RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip -unzip RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip -cd RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a +sha256sum RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip +unzip RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip +cd RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90 sha256sum -c SHA256SUMS.txt for package in motas/*.mota; do motatool verify "$package" || exit 1; done ``` -The outer hash must be `9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a`. +The outer hash must be +`c0b33f4568985e8b2b8dc99411295907212cf2bad21764b6333d5e0ba298fd61`, +and the SHA-256 of the extracted `SHA256SUMS.txt` must be +`3f8c4af8096b96a4aa6506825c387cc8a06f74d5213a29c9387bd11689546881`. Do not continue after any checksum or verification error. ### 2. Record and prepare the destination @@ -218,13 +305,12 @@ tests use `hops 0`: `hops 1` makes the target echo each source response despite there being no intermediate relay, increasing half-duplex loss and legacy three-second retries. -The exact compact chain never qualifies for fast RXPS: its bridge and endpoint -versions predate the v1.17.1.5 adaptive-preamble contract. Keep RXPS off for -this historical chain whether using SF5/BW250 or the faster direct-bench -SF5/BW500 tuple. For a later update whose target, controller, source, and -relays are all v1.17.1.5 or newer, use the generic LoRa OTA runner; it can -retain RXPS at the qualified level-8/preamble-64 boundary for SF5/BW250 or the -level-8/preamble-128 boundary for SF5/BW500. +The exact candidate cannot use fast RXPS throughout because its historical +bridge applications predate the v1.17.1.5 adaptive-preamble contract. Keep +RXPS off for all ten steps whether using SF5/BW250 or the faster direct-bench +SF5/BW500 tuple. After the final endpoint is proven, a later update whose +target, controller, source, and relays are all v1.17.1.5 or newer can use the +generic LoRa OTA runner and its qualified adaptive-preamble boundaries. Put the source on the identical TempRadio tuple. A current ASCII-first Full Companion recognizes `motatool`'s initial `ota folder on` line directly: @@ -243,30 +329,48 @@ every step so the source emits a fresh catalog advert. Leave it running during the download and stop it with Ctrl-C only after the destination reports `ready to install`. -### 3. Install all nine packages in order +### 3. Install all ten packages in order | Step | Manifest ID | Before hash | Expected version | After hash | |---:|---|---|---|---| | 1 | `C147BCEF` | `71F4026CBE4B8B74` | `1.16.7.9` | `42BC53A64288E845` | -| 2 | `C510B628` | `42BC53A64288E845` | `1.16.7.10` | `6F6E51D63BF3E93F` | -| 3 | `8CAD7045` | `6F6E51D63BF3E93F` | `1.16.8.7` | `F04DA8DB515E5C6D` | -| 4 | `8B1EEFF6` | `F04DA8DB515E5C6D` | `1.16.9.105` | `3894A4D7B11ED282` | -| 5 | `687E3BEA` | `3894A4D7B11ED282` | `1.16.9.110` | `CDC5CA630881DA15` | -| 6 | `0D59A34B` | `CDC5CA630881DA15` | `1.16.9.111` | `D0CC4A9E135B9A4D` | -| 7 | `2C5C2082` | `D0CC4A9E135B9A4D` | `1.16.9.116` | `66272A0E7FEF2773` | -| 8 | `994AB743` | `66272A0E7FEF2773` | `1.16.9.118` | `8E00019AA08E00BF` | -| 9 | `FE986948` | `8E00019AA08E00BF` | `1.17.1.02` | `4BB1526BF647547D` | +| 2 | `BEB578FC` | `42BC53A64288E845` | `1.16.7.10` | `1AD2AA8B6C478DA6` | +| 3 | `6D8EF814` | `1AD2AA8B6C478DA6` | `1.16.8.7` | `D709A25308BC1831` | +| 4 | `40AC4CEB` | `D709A25308BC1831` | `1.16.9.105` | `457BEDA5E6406C81` | +| 5 | `F6C8890B` | `457BEDA5E6406C81` | `1.16.9.110` | `E76BFA205634CAB8` | +| 6 | `82405E89` | `E76BFA205634CAB8` | `1.16.9.113` | `65328FC8A1FBED2D` | +| 7 | `80F40DB6` | `65328FC8A1FBED2D` | `1.16.9.117` | `AF7532E13337FADD` | +| 8 | `DBF6310D` | `AF7532E13337FADD` | `1.16.10.0` | `22002359BBDFA76E` | +| 9 | `C2979E08` | `22002359BBDFA76E` | `1.17.1.3` | `FE65A6135A1E7B3F` | +| 10 | `319DEC92` | `FE65A6135A1E7B3F` | `1.17.1.5` | `9179B98432895924` | -For each row, first prove that `ota self` exactly matches its **Before hash**. -Then run: +For each row, first prove that `ota self` exactly matches its **Before hash**, +then inspect the manager before changing it: + +```text +ota status +``` + +Proceed to discovery only after that status is reconciled. When the manager is +idle, run: ```text -ota cancel ota ls ota pull MANIFEST_ID flash ota status ``` +If the first status reports this row's MID as partial, use `ota pull` with that +same MID to resume it. If this row's MID is already ready, do not pull or +cancel it; continue with the install checks below. If status reports the +immediately previous row's MID as `verifying staged blocks`, wait through the +bounded discovery window; require either that exact MID to become `ready to +install` or the manager to become `no download`, and re-prove the current +**Before hash**. Cancel only the same previous MID while it is visibly +attached and ready. If it has become idle, send no cancel. Stop on any other +MID, failed or incomplete state, ambiguous reply, or timeout. Never issue a +blind `ota cancel`: it can discard a resumable or unrelated session. + Repeat `ota status` at a restrained interval until it says `ready to install`. If the manifest is initially absent, wait for a fresh source advert, run `ota ls` again, or restart `motatool`; do not pull a different ID. Stop the @@ -277,19 +381,35 @@ get system.watchdog ota install ``` -The watchdog reply must still be `> off`. Allow up to 90 seconds for the USB -port to disappear and return. Then run `ver`, `ota self`, and `ota help`. +The watchdog reply must still be `> off`. Allow the current default five-minute +readiness window for the USB/LoRa identity to return; automation probes every +10 seconds and returns early as soon as the exact body is visible. Then run +`ver`, `ota self`, and `ota help`. Require the row's exact **After hash**, require `bootloader: apply OK`, and require `rescue install ` in the help before exposing that bridge to -the next package. Clear only the now-proven retained download with `ota cancel`, -reapply the transfer guardrails and TempRadio tuple, restart `motatool`, and -continue with the next row. +the next package. This rescue-help gate applies to steps 1-9; step 10 is the +endpoint and is not exposed to another package in this chain. A normal-radio +`ota status` response of `no download` proves only that the manager is idle; +legacy internal-flash firmware does not durably erase a retained container by +accepting an IDLE `ota cancel`. Do not claim otherwise. When the same expected +manifest reports `verifying staged blocks` after the next TempRadio starts, +wait only for the bounded discovery window and require that it become `ready +to install` or `no download`; stop on a changed ID, failed/incomplete state, or +timeout. Prove the exact running body before cancelling an attached completed +session. OTAFIX consumes the approval word during a successful install, so a +retained endpoint container is inert and a later valid pull replaces it. +Reapply the transfer +guardrails and TempRadio tuple, restart `motatool`, and continue with the next +row. If the board boots but `ota self` says the EndF is invalid, stop. Only when `ota status` still identifies that row's complete staged package may the guarded `ota rescue install BEFORE_HASH` command be used. It is not a force option and must use that row's exact Before hash. If the board does not boot, -recover the documented start/recovery UF2 locally over USB. +recover the documented start/recovery UF2 locally over USB. The fd98 endpoint +deliberately disables rescue on its shared internal bootloader-update build; +if its EndF is invalid, use local DFU/SWD rather than attempting another LoRa +package. ### 4. Intermediate relays and restoration @@ -307,7 +427,7 @@ window. The source's manifest-fragment gap independently follows active packet airtime and duty spacing, clamped to 100-1000 ms. These two delays solve different problems. -After step 9 is proven, restore each saved destination and relay value exactly, +After step 10 is proven, restore each saved destination and relay value exactly, including `rxdelay`, RXPS, CPU power saving, `af`, `ota config hops`, and relay timing. Current full-parser repeater firmware accepts `normalradio` and restores the saved tuple after replying on the temporary channel: @@ -316,7 +436,7 @@ the saved tuple after replying on the temporary channel: normalradio ``` -The historical endpoint in this pinned bundle predates that command. If it +The fd98 endpoint supports that command. If a recovery build unexpectedly replies `Unknown command`, shorten its lease to one minute with the same tuple: ```text @@ -335,20 +455,26 @@ A Full Companion source is the exception: its local TCP console supports `normalradio` and the automated runner uses it before restoring the shared Binary API radio tuple. -Final success requires version `v1.17.1.02-halo-keymind-cascade-dev-e742333a`, -body hash `4BB1526BF647547D`, target `2FA509C1`, hardware `RAK_3401`, and +Final lab success requires version +`v1.17.1.5-halo-keymind-cascade-dev-fd98bc90`, body hash +`9179B98432895924`, target `2FA509C1`, hardware `RAK_3401`, and `get system.watchdog` reporting `> on`. If a relay cannot be restored before its TempRadio lease ends, wait for it to return to the normal channel and restore its saved `rxdelay` and `txdelay` there. ## Two-relay deployment -For two intermediate relays, list them farthest-to-nearest and use three OTA -hops. Run a non-mutating preflight first: +The exact direct package transitions above were physically observed; this +post-run host-runner revision and multi-hop operation are not cleanly +end-to-end qualified. A controlled, recoverable multi-hop lab can exercise two +intermediate relays by +listing them farthest-to-nearest and using three OTA hops. Run a non-mutating +preflight first; even preflight requires the hidden candidate override because +the artifact remains unreleased and the command connects to live devices: ```bash python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip \ --work-dir ./rak3401-mota-chain-work \ --controller-serial /dev/ttyACM0 \ --source-tcp SOURCE_IP:5001 \ @@ -360,6 +486,7 @@ python3 tools/lora_ota/rak3401_mota_chain.py \ --temp-radio 909.950,250,5,5,120 \ --ota-hops 3 \ --motatool /path/to/motatool \ + --accept-test-candidate \ --preflight-only ``` @@ -370,16 +497,27 @@ legal at the deployment location. ## Rescue and completion The guarded `ota rescue install ` command is present after -step 1. It is not a force command: it refuses a valid normal EndF, a base -mismatch, wrong target/hardware, or invalid payload. A failure before step 1 -completes still requires local USB recovery because the deployed start image -predates the rescue command. +step 1 and remains available through the step-9 bridge. It is not a force +command: it refuses a valid normal EndF, a base mismatch, wrong +target/hardware, or invalid payload. A failure before step 1 completes still +requires local USB recovery because the deployed start image predates the +rescue command. The final fd98 endpoint uses the stricter shared-slot profile; +its invalid-EndF recovery is local DFU/SWD, not the rescue command. Before the first mutation, the runner saves the destination's RXPS periods, CPU power-saving state, RX flood delay, airtime factor, and OTA hop reach in -the persistent work directory. It reads the destination, controller, source, -and relay versions before applying the same RXPS policy as the generic runner. -The pinned historical chain therefore verifies `radio.rxps off` after every +the persistent work directory, flushing both the file and its directory entry +before changing a guardrail. It also saves the managed source's exact RXPS +preference in the chain-root `source-rxps-settings.json` before disabling it. +Every step reuses that protected record, so a rerun after a killed host process +restores the chain-start preference instead of treating the temporary off state +as original. The record is accepted only for the same CLI endpoint. The runner +retires that chain-root record only after it has proved the source is back on +its normal radio and restored the exact preference at the verified endpoint. +The runner +reads the destination, controller, source, and relay versions before applying +the same RXPS policy as the generic runner. +The mixed historical chain therefore verifies `radio.rxps off` after every bridge reboot; a future all-v1.17.1.5-or-newer SF5/BW250 chain would instead keep RXPS on under the qualified level-8/preamble-64 boundary. It also verifies `powersaving off` and `rxdelay 0`, plus `af 0` when @@ -398,8 +536,8 @@ the relay leaves TempRadio. The runner also keeps the watchdog off across the chain, rechecks identity and OTA reach before every package, requires the exact post-boot EndF hash, and -only re-enables the watchdog after step 9. Success requires endpoint body hash -`4BB1526BF647547D`, target `2FA509C1`, hardware `RAK_3401`, normal radio +only re-enables the watchdog after step 10. Lab success requires endpoint body +hash `9179B98432895924`, target `2FA509C1`, hardware `RAK_3401`, normal radio `910.525 MHz / 62.5 kHz / SF7 / CR5`, and the watchdog verified on. ## External-radio startup and manual USB recovery diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index 6e4544ff..f3f8a84d 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -647,19 +647,45 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board } else if (is_cmd(a, "drop|cancel|stop", &rest)) { if (c.apply_pending) { strcpy(reply, "ERR update is armed; reboot is pending"); return true; } OtaManager::FetchState fs = c.manager.fetchState(); + const bool was_folder = c.fetch_to_folder; + bool was_sd_archive = false; char midhx[9]; strcpy(midhx, "-"); if (fs != OtaManager::IDLE) mesh::Utils::toHex(midhx, c.manager.fetchManifestId(), 4); #if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + was_sd_archive = c.sdCacheFetching(); c.stopSdCacheFetch(); #endif c.manager.reset_session(); c.manager.want(0); c.manager.want_mid(nullptr); + if (was_folder && c.folder_dest) c.folder_dest->clear(); c.fetch_to_folder = false; c.manager.set_fetch_store(&c.fetch_store); // revert to the default flash store (a folder pull switched it) #if defined(NRF52_PLATFORM) && !defined(OTA_SD_STORE) && !defined(OTA_QSPI_STORE) c.manager.set_accept_full(false); #endif - c.fetch_store.clear(); c.serving = false; c.serve_expected = 0; c.session_started_ms = 0; - snprintf(reply, 160, "OK dropped session (was %c mid=%s); slot free for a new pull", fstate_char(fs), midhx); + const bool discarded = !was_folder && !was_sd_archive && + c.fetch_store.discard(); + // Fetch cancellation and serving are independent. In particular, a + // manual ESP32 serve view can point into serve_buf, so leave the manager + // view and its caller-owned buffer intact until `ota dev clear` (which + // detaches the view before releasing the buffer). + c.session_started_ms = 0; + if (was_folder) { + snprintf(reply, 160, + "OK dropped folder session (was %c mid=%s); host file left untouched", + fstate_char(fs), midhx); + } else if (was_sd_archive) { + snprintf(reply, 160, + "OK dropped SD archive session (was %c mid=%s); partial retained for resume", + fstate_char(fs), midhx); + } else if (!discarded) { + snprintf(reply, 160, + "ERR dropped live session (was %c mid=%s), but persistent OTA slot invalidation failed", + fstate_char(fs), midhx); + } else { + snprintf(reply, 160, + "OK dropped session (was %c mid=%s); OTA receive slot confirmed clear", + fstate_char(fs), midhx); + } // ---- broadcast our tiny beacon so peers discover us. If not already serving, set up flash-backed // self-serve first (so we're a real, fetchable source of our own running firmware). ---- @@ -1141,13 +1167,27 @@ static bool handle_dev(const char* d, char* reply, OtaContext& c) { } } else if (strncmp(d, "clear", 5) == 0) { + const bool was_folder = c.fetch_to_folder; + bool was_sd_archive = false; #if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + was_sd_archive = c.sdCacheFetching(); c.stopSdCacheFetch(); #endif c.manager.clear_primary(); c.serve_expected = 0; c.serving = false; c.releaseServeBuffer(); - c.fetch_store.clear(); c.manager.reset_session(); c.fetch_to_folder = false; - strcpy(reply, "OK cleared"); + c.manager.reset_session(); + if (was_folder && c.folder_dest) c.folder_dest->clear(); + c.fetch_to_folder = false; + c.manager.set_fetch_store(&c.fetch_store); + if (was_folder) { + strcpy(reply, "OK cleared folder session; host file left untouched"); + } else if (was_sd_archive) { + strcpy(reply, "OK cleared SD archive session; partial retained for resume"); + } else { + const bool discarded = c.fetch_store.discard(); + strcpy(reply, discarded ? "OK OTA receive slot confirmed clear" + : "ERR RAM state cleared; persistent OTA slot invalidation failed"); + } } else { strcpy(reply, "ota dev: stage|recv|serve|announce|resume [mid8]|verify|want|apply|clear"); diff --git a/src/helpers/ota/OtaFlashLayout_esp32.h b/src/helpers/ota/OtaFlashLayout_esp32.h new file mode 100644 index 00000000..a5c08303 --- /dev/null +++ b/src/helpers/ota/OtaFlashLayout_esp32.h @@ -0,0 +1,129 @@ +#pragma once + +#include + +namespace mesh { +namespace ota { + +// Physical placement of one ESP32 flash-store container. Keep this independent +// of ESP-IDF so reopen/discard geometry can be tested on the native host. +struct MotaEsp32StageLayout { + uint32_t total = 0; + uint32_t meta_span = 0; + uint32_t meta_flush = 0; + uint32_t write_start = 0; + uint32_t meta_part = 0; + uint32_t pay_log0 = 0; + uint32_t pay_part0 = 0; +}; + +inline bool mota_esp32_align_up(uint32_t value, uint32_t alignment, + uint32_t& result) { + if (alignment == 0 || value > UINT32_MAX - (alignment - 1u)) return false; + result = ((value + alignment - 1u) / alignment) * alignment; + return true; +} + +inline uint32_t mota_esp32_align_down(uint32_t value, + uint32_t alignment) { + return alignment ? (value / alignment) * alignment : 0u; +} + +inline bool mota_esp32_stage_layout( + uint32_t partition_size, uint32_t sector_size, uint32_t meta_capacity, + bool is_full, uint32_t image_size, uint32_t meta_bytes, + uint32_t payload_size, MotaEsp32StageLayout& out) { + if (sector_size == 0 || partition_size < sector_size || + partition_size % sector_size != 0) { + return false; + } + const uint64_t total64 = + (uint64_t)meta_bytes + (uint64_t)payload_size + 5u; + if (total64 > UINT32_MAX) return false; + + MotaEsp32StageLayout layout; + layout.total = (uint32_t)total64; + if (is_full) { + // A FULL payload is the final application image, byte for byte. Reject a + // malformed manifest before begin() can erase or stream anything, and + // prove that the complete logical container fits the inactive partition. + if (payload_size != image_size || layout.total > partition_size) { + return false; + } + layout.meta_span = meta_bytes; + if (meta_bytes > UINT32_MAX - 5u || + !mota_esp32_align_up(meta_bytes + 5u, sector_size, + layout.meta_flush) || + layout.meta_flush > meta_capacity || + layout.meta_flush > partition_size) { + return false; + } + layout.meta_part = mota_esp32_align_down( + partition_size - layout.meta_flush, sector_size); + layout.pay_log0 = meta_bytes; + layout.pay_part0 = 0; + layout.write_start = 0; + if (image_size > layout.meta_part) return false; + } else { + if (!mota_esp32_align_up(meta_bytes, sector_size, + layout.meta_span) || + layout.meta_span > meta_capacity || layout.total > partition_size) { + return false; + } + layout.meta_flush = layout.meta_span; + layout.write_start = mota_esp32_align_down( + partition_size - layout.total, sector_size); + layout.meta_part = layout.write_start; + layout.pay_log0 = layout.meta_span; + if (layout.write_start > UINT32_MAX - layout.meta_span) return false; + layout.pay_part0 = layout.write_start + layout.meta_span; + if (image_size > layout.write_start) return false; + } + out = layout; + return true; +} + +// The probe separates ordinary non-candidates from partition I/O failure: +// true/false is I/O success, while `reopenable` says whether this sector holds +// a header that the store's reopen path would adopt. +typedef bool (*MotaEsp32ProbeStagedHeader)(void* context, uint32_t offset, + bool& reopenable); +typedef bool (*MotaEsp32InvalidateStagedHeader)(void* context, + uint32_t offset); + +inline bool mota_esp32_discard_staged_headers( + uint32_t partition_size, uint32_t sector_size, void* context, + MotaEsp32ProbeStagedHeader probe_header, + MotaEsp32InvalidateStagedHeader invalidate_header, + uint32_t* invalidated_count = nullptr) { + if (invalidated_count) *invalidated_count = 0; + if (sector_size == 0 || partition_size < sector_size || + partition_size % sector_size != 0 || !probe_header || + !invalidate_header) { + return false; + } + + bool ok = true; + uint32_t count = 0; + uint32_t offset = mota_esp32_align_down( + partition_size - sector_size, sector_size); + for (;;) { + bool reopenable = false; + if (!probe_header(context, offset, reopenable)) { + ok = false; + } else if (reopenable) { + if (invalidate_header(context, offset)) { + ++count; + } else { + ok = false; + } + } + if (offset < sector_size) break; + offset -= sector_size; + } + if (invalidated_count) *invalidated_count = count; + return ok; +} + +} // namespace ota +} // namespace mesh diff --git a/src/helpers/ota/OtaFlashLayout_nrf52.h b/src/helpers/ota/OtaFlashLayout_nrf52.h index a87522f2..e8bf6432 100644 --- a/src/helpers/ota/OtaFlashLayout_nrf52.h +++ b/src/helpers/ota/OtaFlashLayout_nrf52.h @@ -47,6 +47,7 @@ static const uint32_t MOTA_NRF52_STAGE_CEILING_EXPANDED = MOTA_NRF52_APP_END; // explicit ceiling so it cannot accidentally cross an Internal ExtraFS. static const uint32_t MOTA_NRF52_FS_START = MOTA_NRF52_EXTRAFS_START; static const uint32_t MOTA_NRF52_FLASH_PAGE = 4096u; +static const uint32_t MOTA_NRF52_CONTAINER_MIN_SIZE = 8u + 197u + 5u; static const uint8_t GPREGRET_OTA_APPLY = 0x6Au; // distinct from DFU magics 0x57/0x4E/0xA8 static const uint8_t GPREGRET_OTA_BOOTLOADER_UPDATE = 0x6Bu; static const uint8_t GPREGRET2_OTA_STAGE_LEGACY = 0xD4u; @@ -272,5 +273,71 @@ inline bool mota_nrf52_stage_plan(uint32_t total_size, uint32_t app_base, uint32 mota_nrf52_layout_stage_ceiling(), out_start); } +// Durable internal-store cancellation has to work even when the live +// OtaStore object is empty (for example, autofetch is off after a reboot). It +// also has to invalidate every older bottom-aligned header: invalidating only +// the highest container could expose a larger stale container below it on the +// next reopen scan. Keep the page walk and layout recognition pure so native +// tests can exercise fresh-object and multiple-header cases without nRF flash. +typedef bool (*MotaNrf52ReadStagedHeader)(void* context, uint32_t address, + uint32_t& total_size); +typedef bool (*MotaNrf52InvalidateStagedHeader)(void* context, + uint32_t address); + +inline bool mota_nrf52_staged_header_matches_layout( + uint32_t address, uint32_t total_size, uint32_t app_base, + uint32_t app_end, uint32_t effective_stage_ceiling) { + uint32_t planned = 0; + if (mota_nrf52_stage_plan(total_size, app_base, app_end, + effective_stage_ceiling, planned) && + planned == address) { + return true; + } + + // A device that moved from the legacy D4000 ceiling to the expanded ED000 + // ceiling can still contain an older, otherwise reopenable legacy header. + // The expanded layout has no filesystem in that range, so it is safe to + // invalidate both geometries. A legacy-only build never scans above D4000. + return effective_stage_ceiling == MOTA_NRF52_STAGE_CEILING_EXPANDED && + mota_nrf52_stage_plan(total_size, app_base, app_end, + MOTA_NRF52_STAGE_CEILING_LEGACY, planned) && + planned == address; +} + +inline bool mota_nrf52_discard_staged_headers( + uint32_t app_base, uint32_t app_end, uint32_t effective_stage_ceiling, + void* context, MotaNrf52ReadStagedHeader read_header, + MotaNrf52InvalidateStagedHeader invalidate_header, + uint32_t* invalidated_count = nullptr) { + if (invalidated_count) *invalidated_count = 0; + if (!read_header || !invalidate_header || + !mota_nrf52_layout_valid(app_base, effective_stage_ceiling) || + app_end < app_base || app_end > effective_stage_ceiling) { + return false; + } + + bool ok = true; + uint32_t count = 0; + uint32_t address = effective_stage_ceiling - MOTA_NRF52_FLASH_PAGE; + while (address >= app_end) { + uint32_t total_size = 0; + if (read_header(context, address, total_size) && + total_size >= MOTA_NRF52_CONTAINER_MIN_SIZE && + mota_nrf52_staged_header_matches_layout( + address, total_size, app_base, app_end, + effective_stage_ceiling)) { + if (invalidate_header(context, address)) { + ++count; + } else { + ok = false; + } + } + if (address < MOTA_NRF52_FLASH_PAGE) break; + address -= MOTA_NRF52_FLASH_PAGE; + } + if (invalidated_count) *invalidated_count = count; + return ok; +} + } // namespace ota } // namespace mesh diff --git a/src/helpers/ota/OtaStore.h b/src/helpers/ota/OtaStore.h index 7b186196..37f0e756 100644 --- a/src/helpers/ota/OtaStore.h +++ b/src/helpers/ota/OtaStore.h @@ -24,6 +24,17 @@ public: virtual uint32_t staged_size() const = 0; // total_size from begin(), 0 if none virtual void clear() = 0; + // Durably discard a staged container. `clear()` resets the store object's + // live session state and is intentionally cheap enough for begin()/retry + // paths; a persistent backend may therefore leave its resumable bytes in + // place. User-facing cancellation needs the stronger operation: after a + // successful discard, a fresh store object must not be able to reopen the + // old container. The conservative default clears only the live object and + // returns false: a backend must opt in only when it can prove that its + // durable copy is gone. A false result means cancellation must not be + // reported as having cleared the persistent slot. + virtual bool discard() { clear(); return false; } + // Optional: declare the size of the leading metadata (header + manifest + merkle leaves, i.e. // everything before the payload). A flash-backed store keeps that region - which is updated // throughout the transfer (a leaf is committed per block) - pinned in one RAM page, so it can @@ -90,9 +101,17 @@ public: uint32_t capacity() const override { return CAP; } uint32_t staged_size() const override { return _total; } void clear() override { _total = 0; } + bool discard() override { + _total = 0; + const size_t marker_bytes = + CAP < sizeof(MOTA_MAGIC) ? CAP : sizeof(MOTA_MAGIC); + memset(_buf, 0, marker_bytes); + return true; + } // RAM doesn't survive a real reboot, but the buffer persists within a process - enough to exercise the // manager's resume path in native tests. Recover `total` from the stored header so read() bounds work. bool reopen() override { + if (CAP < 8u) return false; if (memcmp(_buf, MOTA_MAGIC, 4) != 0) return false; uint32_t t = (uint32_t)_buf[4] | ((uint32_t)_buf[5] << 8) | ((uint32_t)_buf[6] << 16) | ((uint32_t)_buf[7] << 24); if (t < 13 || t > CAP) return false; diff --git a/src/helpers/ota/OtaStoreFlashEsp32.cpp b/src/helpers/ota/OtaStoreFlashEsp32.cpp index d75f11e7..697b9bbe 100644 --- a/src/helpers/ota/OtaStoreFlashEsp32.cpp +++ b/src/helpers/ota/OtaStoreFlashEsp32.cpp @@ -3,7 +3,7 @@ #if defined(ESP32_PLATFORM) && defined(OTA_FLASH_STORE) #include "OtaDebug.h" -#include "OtaByteIO.h" // align_up / align_down (flash-sector geometry) +#include "OtaByteIO.h" // rd_u32le (persisted header total) #include "MotaContainer.h" // mota_parse_manifest (reopen: rebuild geometry from the staged manifest) #include #include // malloc/free (the meta buffer is sized per fetch) @@ -22,33 +22,29 @@ bool OtaStoreFlashEsp32::acquire() { return _part != nullptr; } +void OtaStoreFlashEsp32::set_layout(const MotaEsp32StageLayout& layout) { + _total = layout.total; + _meta_span = layout.meta_span; + _meta_flush = layout.meta_flush; + _write_start = layout.write_start; + _meta_part = layout.meta_part; + _pay_log0 = layout.pay_log0; + _pay_part0 = layout.pay_part0; + _trailer_part = _full ? _meta_part + _meta_bytes + : _write_start + _total - 5u; +} + // Compute the slot placement from _full/_image_size/_meta_bytes/_pay_size (already set). Returns false if -// it won't fit. Shared by plan_layout (fresh fetch) and reopen (resume) so both derive identical geometry. +// it won't fit. Shared by plan_layout (fresh fetch), reopen (resume), and discard validation so all three +// derive identical geometry. bool OtaStoreFlashEsp32::layout() { - uint32_t total = _meta_bytes + _pay_size + 5; - if (_full) { - // payload streams to slot offset 0 (it IS the image); header+manifest+leaves+trailer persist at the - // bottom so the container survives a reboot (resume / re-serve). - _meta_span = _meta_bytes; // routing boundary: [0,meta) -> RAM meta buffer - _meta_flush = align_up(_meta_bytes + 5, SEC); // meta + 5-byte trailer, whole sectors - if (_meta_flush > OTA_ESP32_META_CAP) return false; - uint32_t bottom = align_down(_psize - _meta_flush, SEC); - _meta_part = bottom; - _pay_log0 = _meta_bytes; _pay_part0 = 0; - _write_start = 0; - if (_image_size > bottom) return false; // image would overrun the bottom meta region - } else { - // whole container staged bottom-aligned; the decoded image fills the slot from offset 0. - _meta_span = align_up(_meta_bytes, SEC); // pin whole sectors covering meta (+ spillover payload) - _meta_flush = _meta_span; - if (_meta_flush > OTA_ESP32_META_CAP) return false; - if (total > _psize) return false; - _write_start = align_down(_psize - total, SEC); - _meta_part = _write_start; - _pay_log0 = _meta_span; _pay_part0 = _write_start + _meta_span; - if (_image_size > _write_start) return false; // decoded output would overlap the staged container + MotaEsp32StageLayout planned; + if (!mota_esp32_stage_layout( + _psize, SEC, OTA_ESP32_META_CAP, _full, _image_size, + _meta_bytes, _pay_size, planned)) { + return false; } - _total = total; + set_layout(planned); return true; } @@ -84,6 +80,120 @@ void OtaStoreFlashEsp32::clear() { _total = 0; _pay_open = false; _flushed = false; // _part kept (re-acquire is fine) } +OtaStoreFlashEsp32::CandidateProbe OtaStoreFlashEsp32::probe_candidate( + uint32_t offset, StagedCandidate& candidate) const { + if (!_part || _psize < SEC || offset > _psize - SEC) { + return CandidateProbe::INVALID; + } + + uint8_t header[8]; + if (esp_partition_read(_part, offset, header, sizeof(header)) != ESP_OK) { + return CandidateProbe::IO_ERROR; + } + if (memcmp(header, MOTA_MAGIC, sizeof(MOTA_MAGIC)) != 0) { + return CandidateProbe::INVALID; + } + + const uint32_t total = rd_u32le(header + sizeof(MOTA_MAGIC)); + if (total < 8u + MOTA_MFL + 5u || total > _psize) { + return CandidateProbe::INVALID; + } + + uint8_t manifest_bytes[256]; + uint32_t manifest_read = total - 8u; + if (manifest_read > sizeof(manifest_bytes)) { + manifest_read = sizeof(manifest_bytes); + } + if (esp_partition_read(_part, offset + 8u, manifest_bytes, + manifest_read) != ESP_OK) { + return CandidateProbe::IO_ERROR; + } + + MotaManifest manifest; + if (!mota_parse_manifest(manifest_bytes, manifest_read, manifest)) { + return CandidateProbe::INVALID; + } + const uint32_t manifest_len = + (uint32_t)(manifest.approval - manifest.manifest_start) + + sizeof(APPROVAL_YES); + const uint64_t payload_off64 = + 8u + (uint64_t)manifest_len + (uint64_t)manifest.block_count * 4u; + if (payload_off64 > UINT32_MAX || + payload_off64 + (uint64_t)manifest.payload_size + 5u != total) { + return CandidateProbe::INVALID; + } + + MotaEsp32StageLayout planned; + if (!mota_esp32_stage_layout( + _psize, SEC, OTA_ESP32_META_CAP, manifest.is_full(), + manifest.image_size, (uint32_t)payload_off64, + manifest.payload_size, planned) || + planned.meta_part != offset || planned.total != total) { + return CandidateProbe::INVALID; + } + + candidate.layout = planned; + candidate.full = manifest.is_full(); + candidate.image_size = manifest.image_size; + candidate.meta_bytes = (uint32_t)payload_off64; + candidate.pay_size = manifest.payload_size; + return CandidateProbe::VALID; +} + +bool OtaStoreFlashEsp32::probe_staged_header( + void* context, uint32_t offset, bool& reopenable) { + OtaStoreFlashEsp32* store = static_cast(context); + StagedCandidate candidate; + const CandidateProbe result = store->probe_candidate(offset, candidate); + reopenable = result == CandidateProbe::VALID; + return result != CandidateProbe::IO_ERROR; +} + +bool OtaStoreFlashEsp32::invalidate_staged_header( + void* context, uint32_t offset) { + OtaStoreFlashEsp32* store = static_cast(context); + if (!store->_part || store->_psize < SEC || offset > store->_psize - SEC || + esp_partition_read(store->_part, offset, store->_pay, SEC) != ESP_OK) { + return false; + } + + // Both markers are inside the already-validated first metadata sector. + // One sector RMW avoids erasing a delta's adjacent metadata/payload twice. + memset(store->_pay, 0, sizeof(MOTA_MAGIC)); + memset(store->_pay + 8u + MOTA_OFF_APPROVAL, 0, + sizeof(APPROVAL_YES)); + if (esp_partition_erase_range(store->_part, offset, SEC) != ESP_OK || + esp_partition_write(store->_part, offset, store->_pay, SEC) != ESP_OK) { + return false; + } + + uint8_t check[sizeof(MOTA_MAGIC)]; + if (esp_partition_read(store->_part, offset, check, sizeof(check)) != ESP_OK || + memcmp(check, store->_pay, sizeof(check)) != 0 || + esp_partition_read(store->_part, offset + 8u + MOTA_OFF_APPROVAL, + check, sizeof(check)) != ESP_OK || + memcmp(check, store->_pay + 8u + MOTA_OFF_APPROVAL, + sizeof(check)) != 0) { + return false; + } + return true; +} + +bool OtaStoreFlashEsp32::discard() { + bool ok = acquire(); + uint32_t invalidated = 0; + if (ok) { + ok = mota_esp32_discard_staged_headers( + _psize, SEC, this, probe_staged_header, + invalidate_staged_header, &invalidated); + } + OTA_DBG("OTA esp32: discard invalidated=%u ok=%d\n", + (unsigned)invalidated, (int)ok); + clear(); + _io_ok = ok; + return ok; +} + uint8_t* OtaStoreFlashEsp32::meta_slot(uint32_t L) { if (in_trailer(L)) return _full ? (_meta + _meta_bytes + (L - (_total - 5))) : (_trailer + (L - (_total - 5))); @@ -207,34 +317,26 @@ void OtaStoreFlashEsp32::checkpoint() { // so a stray/stale match can only cost a restart, never a corrupt adopt. bool OtaStoreFlashEsp32::reopen() { if (!acquire() || _psize < SEC) return false; - uint8_t hb[8]; // The meta/container is staged at the bottom of the slot. Large delta containers can begin well above // the final 512 KB, so scan the complete partition rather than silently losing resumability after reboot. - for (uint32_t o = align_down(_psize - SEC, SEC); ; o -= SEC) { - if (esp_partition_read(_part, o, hb, 8) == ESP_OK && memcmp(hb, MOTA_MAGIC, 4) == 0) { - uint32_t total = rd_u32le(hb + 4); - if (total >= 13 && total <= _psize) { - uint8_t mbuf[256]; uint32_t mread = total - 8; if (mread > sizeof(mbuf)) mread = sizeof(mbuf); - MotaManifest m; - if (esp_partition_read(_part, o + 8, mbuf, mread) == ESP_OK && mota_parse_manifest(mbuf, mread, m)) { - uint32_t mfl = (uint32_t)(m.approval - m.manifest_start) + 4; - uint32_t payload_off = 8 + mfl + m.block_count * 4; - if ((uint64_t)payload_off + m.payload_size + 5 == total) { - _full = m.is_full(); _image_size = m.image_size; _meta_bytes = payload_off; _pay_size = m.payload_size; - if (layout() && _meta_part == o) { // geometry agrees AND magic is where we'd place meta - free(_meta); _meta = (uint8_t*)malloc(_meta_flush); - if (!_meta) { _total = 0; return false; } - if (esp_partition_read(_part, _meta_part, _meta, _meta_flush) != ESP_OK) { - free(_meta); _meta = nullptr; _total = 0; return false; } - memset(_trailer, 0xFF, sizeof(_trailer)); // delta trailer (re-written at finalize); full reads it from _meta - _pay_open = false; _pay_sec = 0; _flushed = false; _io_ok = true; - _pay_max_sec = (_pay_part0 + _pay_size + SEC) / SEC; // treat all payload sectors as seen -> RMW preserves committed blocks - OTA_DBG("OTA esp32: reopen %s total=%u meta_part=%u\n", _full ? "FULL" : "DELTA", (unsigned)total, (unsigned)o); - return true; - } - } - } + for (uint32_t o = mota_esp32_align_down(_psize - SEC, SEC); ; o -= SEC) { + StagedCandidate candidate; + if (probe_candidate(o, candidate) == CandidateProbe::VALID) { + _full = candidate.full; + _image_size = candidate.image_size; + _meta_bytes = candidate.meta_bytes; + _pay_size = candidate.pay_size; + set_layout(candidate.layout); + free(_meta); _meta = (uint8_t*)malloc(_meta_flush); + if (!_meta) { _total = 0; return false; } + if (esp_partition_read(_part, _meta_part, _meta, _meta_flush) != ESP_OK) { + free(_meta); _meta = nullptr; _total = 0; return false; } + memset(_trailer, 0xFF, sizeof(_trailer)); // delta trailer (re-written at finalize); full reads it from _meta + _pay_open = false; _pay_sec = 0; _flushed = false; _io_ok = true; + _pay_max_sec = (_pay_part0 + _pay_size + SEC) / SEC; // treat all payload sectors as seen -> RMW preserves committed blocks + OTA_DBG("OTA esp32: reopen %s total=%u meta_part=%u\n", _full ? "FULL" : "DELTA", (unsigned)_total, (unsigned)o); + return true; } if (o == 0) break; } diff --git a/src/helpers/ota/OtaStoreFlashEsp32.h b/src/helpers/ota/OtaStoreFlashEsp32.h index 36bacce8..db8cdd19 100644 --- a/src/helpers/ota/OtaStoreFlashEsp32.h +++ b/src/helpers/ota/OtaStoreFlashEsp32.h @@ -3,6 +3,7 @@ #if defined(ESP32_PLATFORM) && defined(OTA_FLASH_STORE) #include "OtaStore.h" +#include "OtaFlashLayout_esp32.h" #include "esp_partition.h" // Persistent flash-backed OtaStore for ESP32 (A/B). Stages the received `.mota` in the INACTIVE OTA @@ -41,6 +42,15 @@ namespace ota { class OtaStoreFlashEsp32 : public OtaStore { static const uint32_t SEC = 4096; // ESP32 NOR flash erase unit + enum class CandidateProbe : uint8_t { INVALID, VALID, IO_ERROR }; + struct StagedCandidate { + MotaEsp32StageLayout layout; + bool full = false; + uint32_t image_size = 0; + uint32_t meta_bytes = 0; + uint32_t pay_size = 0; + }; + const esp_partition_t* _part = nullptr; // inactive OTA slot (acquired in plan_layout/begin) uint32_t _psize = 0; // slot size @@ -62,7 +72,7 @@ class OtaStoreFlashEsp32 : public OtaStore { // RX-safe staging buffers uint8_t* _meta = nullptr; // heap, sized per fetch: header+manifest+leaves(+full trailer) - uint8_t _pay[SEC]; // one sliding payload sector (slot-sector aligned) + alignas(4) uint8_t _pay[SEC]; // one sliding payload sector (slot-sector aligned) uint32_t _pay_sec = 0; // slot sector index currently in _pay (0 = none open) uint8_t _trailer[5]; uint32_t _meta_flush = 0; // whole-sector byte count to program for the meta buffer @@ -73,6 +83,12 @@ class OtaStoreFlashEsp32 : public OtaStore { bool acquire(); // resolve the inactive slot (idempotent) bool layout(); // compute placement from _full/_image_size/_meta_bytes/_pay_size + void set_layout(const MotaEsp32StageLayout& layout); + CandidateProbe probe_candidate(uint32_t offset, + StagedCandidate& candidate) const; + static bool probe_staged_header(void* context, uint32_t offset, + bool& reopenable); + static bool invalidate_staged_header(void* context, uint32_t offset); uint32_t pay_part(uint32_t L) const { return _pay_part0 + (L - _pay_log0); } // payload slot offset bool in_trailer(uint32_t L) const { return L >= _total - 5; } uint32_t run(uint32_t pos, uint32_t remain) const; // bytes from `pos` that stay in one region+sector @@ -92,6 +108,7 @@ public: uint32_t capacity() const override { return _psize; } // loose bound; plan_layout does the real check uint32_t staged_size() const override { return _total; } void clear() override; + bool discard() override; bool set_meta_size(uint32_t meta_bytes) override { return meta_bytes < OTA_ESP32_META_CAP; } bool finalize() override; void checkpoint() override; // persist meta(leaves) + open payload sector so a reboot can resume diff --git a/src/helpers/ota/OtaStoreFlashNrf52.cpp b/src/helpers/ota/OtaStoreFlashNrf52.cpp index 72818fe3..ee567b8b 100644 --- a/src/helpers/ota/OtaStoreFlashNrf52.cpp +++ b/src/helpers/ota/OtaStoreFlashNrf52.cpp @@ -13,6 +13,18 @@ namespace mesh { namespace ota { +void OtaStoreFlashNrf52::reset_session() { + _write_start = 0; + _stage_ceiling = MOTA_NRF52_STAGE_CEILING_LEGACY; + _total = 0; + _pay_idx = 0; + _flushed = false; + _io_ok = true; + _planned_bootloader = false; + _planned_total = 0; + _planned_start = 0; +} + // A valid EndF gives the exact live-image extent. Legacy app-only builds keep // the conservative rescue fallback. Shared internal bootloader-update builds // reject every package kind without EndF because their normal linker may place @@ -73,6 +85,59 @@ static bool flash_matches(uint32_t addr, const uint8_t* expected, uint32_t n) { return true; } +bool OtaStoreFlashNrf52::read_staged_header( + void*, uint32_t address, uint32_t& total_size) { + const uint8_t* header = (const uint8_t*)(uintptr_t)address; + if (memcmp(header, MOTA_MAGIC, sizeof(MOTA_MAGIC)) != 0) return false; + total_size = rd_u32le(header + sizeof(MOTA_MAGIC)); + return true; +} + +bool OtaStoreFlashNrf52::invalidate_staged_header( + void* context, uint32_t address) { + OtaStoreFlashNrf52* store = static_cast(context); + const uint8_t* page = (const uint8_t*)(uintptr_t)address; + memcpy(store->_meta_page, page, PG); + + // Magic prevents application reopen; approval prevents an explicitly + // triggered legacy bootloader from accepting a half-cleared container. + // Rewrite the page once so both durable markers are consumed together. + memset(store->_meta_page, 0, sizeof(MOTA_MAGIC)); + memset(store->_meta_page + 8u + MOTA_OFF_APPROVAL, 0, + sizeof(APPROVAL_YES)); + if (flash_nrf5x_write(address, store->_meta_page, PG) < 0) { + store->_io_ok = false; + return false; + } + flash_nrf5x_flush(); + if (!flash_matches(address, store->_meta_page, PG)) { + store->_io_ok = false; + return false; + } + return true; +} + +bool OtaStoreFlashNrf52::discard() { + const uint32_t app_base = mota_nrf52_app_base(); + const uint32_t stage_ceiling = ota_nrf52_effective_stage_ceiling(); + uint32_t app_end = 0; + if (!protected_app_end(app_base, stage_ceiling, app_end)) { + reset_session(); + _io_ok = false; + return false; + } + + uint32_t invalidated = 0; + const bool ok = mota_nrf52_discard_staged_headers( + app_base, app_end, stage_ceiling, this, read_staged_header, + invalidate_staged_header, &invalidated); + OTA_DBG("OTA flash: discard invalidated=%u ok=%d\n", + (unsigned)invalidated, (int)ok); + reset_session(); + _io_ok = ok; + return ok; +} + bool OtaStoreFlashNrf52::flush_page(uint32_t page_idx, const uint8_t* buf) { if (!_io_ok || _write_start > _stage_ceiling - PG || page_idx > (_stage_ceiling - PG - _write_start) / PG) { @@ -133,7 +198,7 @@ bool OtaStoreFlashNrf52::begin(uint32_t total_size) { const bool planned_bootloader = _planned_bootloader; const uint32_t planned_total = _planned_total; const uint32_t planned_start = _planned_start; - clear(); + reset_session(); // never collide with the running application image (its extent comes from its EndF trailer) const uint32_t app_base = mota_nrf52_app_base(); diff --git a/src/helpers/ota/OtaStoreFlashNrf52.h b/src/helpers/ota/OtaStoreFlashNrf52.h index 8911b365..14f759a3 100644 --- a/src/helpers/ota/OtaStoreFlashNrf52.h +++ b/src/helpers/ota/OtaStoreFlashNrf52.h @@ -57,6 +57,10 @@ class OtaStoreFlashNrf52 : public OtaStore { uint8_t* write_slot(uint32_t pos); bool flush_pay(); // commit _pay_page to flash (erase + program, one page) bool flush_page(uint32_t page_idx, const uint8_t* buf); // write + verify a full page + void reset_session(); // RAM bookkeeping only; never touches staged flash + static bool read_staged_header(void* context, uint32_t address, + uint32_t& total_size); + static bool invalidate_staged_header(void* context, uint32_t address); public: bool plan_layout(bool is_full, uint32_t image_size, uint32_t payload_off, @@ -66,11 +70,8 @@ public: bool read(uint32_t offset, uint8_t* buf, uint32_t len) const override; uint32_t capacity() const override; uint32_t staged_size() const override { return _total; } - void clear() override { - _stage_ceiling = MOTA_NRF52_STAGE_CEILING_LEGACY; - _total = 0; _pay_idx = 0; _flushed = false; _io_ok = true; - _planned_bootloader = false; _planned_total = 0; _planned_start = 0; - } + void clear() override { reset_session(); } + bool discard() override; bool set_meta_size(uint32_t meta_bytes) override { return meta_bytes <= PG; } // leaves must fit page 0 bool finalize() override; void checkpoint() override; // persist page 0 (leaves) + the open payload page so a reboot can resume diff --git a/src/helpers/ota/OtaStoreQspiNrf52.cpp b/src/helpers/ota/OtaStoreQspiNrf52.cpp index 042ad448..e89dd7a3 100644 --- a/src/helpers/ota/OtaStoreQspiNrf52.cpp +++ b/src/helpers/ota/OtaStoreQspiNrf52.cpp @@ -864,7 +864,7 @@ bool OtaStoreQspiNrf52::reopen() { return true; } -void OtaStoreQspiNrf52::clear() { +bool OtaStoreQspiNrf52::discard() { bool ok = ensureFlash(); uint8_t zero[4] = { 0, 0, 0, 0 }; uint8_t check[sizeof(zero)]; @@ -872,7 +872,11 @@ void OtaStoreQspiNrf52::clear() { ok = rawWrite(0, zero, sizeof(zero)) && rawRead(0, check, sizeof(check)) && memcmp(check, zero, sizeof(zero)) == 0; } - if (ok && _total >= 8 + MOTA_OFF_APPROVAL + sizeof(zero)) { + // The raw QSPI store is dedicated to OTA, and APRV has a fixed manifest + // offset. A fresh store object has _total == 0 even when an older approved + // container remains in flash, so invalidate both gates from flash capacity + // rather than trusting this session's RAM state. + if (ok && _flash_size >= 8 + MOTA_OFF_APPROVAL + sizeof(zero)) { const uint32_t approval = 8 + MOTA_OFF_APPROVAL; ok = rawWrite(approval, zero, sizeof(zero)) && rawRead(approval, check, sizeof(check)) && memcmp(check, zero, sizeof(zero)) == 0; @@ -883,8 +887,11 @@ void OtaStoreQspiNrf52::clear() { resetSession(); if (!ok) fail(saved_error[0] ? saved_error : "QSPI container invalidation failed"); releaseFlash(); + return ok; } +void OtaStoreQspiNrf52::clear() { (void)discard(); } + bool OtaStoreQspiNrf52::approve_for_bootloader() { if (!finalize()) return false; const uint32_t approval = 8 + MOTA_OFF_APPROVAL; diff --git a/src/helpers/ota/OtaStoreQspiNrf52.h b/src/helpers/ota/OtaStoreQspiNrf52.h index 13888da1..ded474f4 100644 --- a/src/helpers/ota/OtaStoreQspiNrf52.h +++ b/src/helpers/ota/OtaStoreQspiNrf52.h @@ -155,6 +155,7 @@ public: uint32_t capacity() const override; uint32_t staged_size() const override { return _total; } void clear() override; + bool discard() override; bool set_meta_size(uint32_t meta_bytes) override; bool finalize() override; void checkpoint() override; diff --git a/src/helpers/ota/OtaStoreSdNrf52.cpp b/src/helpers/ota/OtaStoreSdNrf52.cpp index 2da36c91..4497b074 100644 --- a/src/helpers/ota/OtaStoreSdNrf52.cpp +++ b/src/helpers/ota/OtaStoreSdNrf52.cpp @@ -509,17 +509,32 @@ bool OtaStoreSdNrf52::reopen() { return true; } -void OtaStoreSdNrf52::clear() { +bool OtaStoreSdNrf52::discard() { _total = 0; _planned_bootloader = false; _first_sector = 0; _allocated_sectors = 0; clear_sd_auth_record(); - if (!mount()) return; + if (!mount()) return false; if (_file && *_file) _file->close(); - _sd->remove(PATH); + if (_sd->exists(PATH) && !_sd->remove(PATH)) { + fail("SD OTA file removal failed"); + return false; + } + if (_sd->exists(PATH)) { + fail("SD OTA file still exists after removal"); + return false; + } + SdCard* const card = _sd->card(); + if (card && !card->syncDevice()) { + fail("SD OTA removal sync failed"); + return false; + } + return true; } +void OtaStoreSdNrf52::clear() { (void)discard(); } + bool OtaStoreSdNrf52::approve_for_bootloader( const uint8_t expected_boot_image_hash[32], const uint8_t authorized_container_hash[32]) { diff --git a/src/helpers/ota/OtaStoreSdNrf52.h b/src/helpers/ota/OtaStoreSdNrf52.h index d73239fe..1b8bc68d 100644 --- a/src/helpers/ota/OtaStoreSdNrf52.h +++ b/src/helpers/ota/OtaStoreSdNrf52.h @@ -30,6 +30,7 @@ public: uint32_t capacity() const override; uint32_t staged_size() const override { return _total; } void clear() override; + bool discard() override; bool set_meta_size(uint32_t meta_bytes) override; bool finalize() override; void checkpoint() override; diff --git a/src/helpers/sensors/EnvironmentSensorManager.cpp b/src/helpers/sensors/EnvironmentSensorManager.cpp index b0e61730..7d84f83e 100644 --- a/src/helpers/sensors/EnvironmentSensorManager.cpp +++ b/src/helpers/sensors/EnvironmentSensorManager.cpp @@ -172,6 +172,17 @@ static RAK12035_SoilMoisture RAK12035; #endif #ifdef RAK_WISBLOCK_GPS +// Release capability manifests scan the linked firmware image rather than +// trusting build flags. Give the actual WisBlock provider a unique marker and +// keep an observable reference from begin() so LTO/section GC cannot discard +// it. The marker lives inside this provider guard, so generic GPS CLI text +// cannot produce a false positive. +extern "C" { +extern const char meshcore_capability_rak_wisblock_gps[]; +const char meshcore_capability_rak_wisblock_gps[] __attribute__((used)) = + "meshcore.capability.rak_wisblock_gps.v1"; +} + // -1 = no enable pin; out-of-range values are no-ops in pinMode/digitalWrite, // while 0 would be a real GPIO (P0.00 = LFXO crystal on nRF52) static uint32_t gpsResetPin = -1; @@ -702,6 +713,10 @@ static const size_t SENSOR_TABLE_SIZE = (sizeof(SENSOR_TABLE) / sizeof(SENSOR_TA bool EnvironmentSensorManager::begin() { #if ENV_INCLUDE_GPS #ifdef RAK_WISBLOCK_GPS + // A volatile read anchors the externally linked marker even under LTO. + const volatile char* capability_marker = + meshcore_capability_rak_wisblock_gps; + (void)*capability_marker; rakGPSInit(); #else initBasicGPS(); diff --git a/test/test_build_profiles.sh b/test/test_build_profiles.sh index 7d4fab50..8ca824d5 100644 --- a/test/test_build_profiles.sh +++ b/test/test_build_profiles.sh @@ -783,4 +783,23 @@ expectations=" ${BUILD_EXPECTATIONS[*]} " [[ "$expectations" != *"companion.network_terminal="* ]] \ || fail "nRF52 Full contract promised an ESP32-only network terminal" +BUILD_PROFILE_FOR_TARGET=standard +for rak_target in \ + RAK_3401_repeater_lora_ota_no_external_sensors \ + RAK_4631_repeater_lora_ota_no_external_sensors; do + BUILD_CAPABILITIES=() + BUILD_REDUCTIONS=() + BUILD_EXPECTATIONS=() + declare_build_capability_contract "$rak_target" NRF52_PLATFORM + expectations=" ${BUILD_EXPECTATIONS[*]} " + [[ "$expectations" == *"sensor.gps=meshcore.capability.rak_wisblock_gps.v1"* ]] \ + || fail "$rak_target reduced OTA contract omitted retained GPS" + [[ "$expectations" != *"sensor.gps=gps setloc"* ]] \ + || fail "$rak_target reduced OTA contract still accepts generic GPS CLI text" + for ina in 219 226 260 3221; do + [[ "$expectations" == *"sensor.ina${ina}=INA${ina}"* ]] \ + || fail "$rak_target reduced OTA contract omitted retained INA${ina}" + done +done + echo "test_build_profiles: OK" diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 422e6e83..efe826c3 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -1274,6 +1274,60 @@ TEST(OtaStoreRamTest, RandomAccessAndErasedSentinel) { for (int i = 0; i < 8; i++) EXPECT_EQ(rd[i], 0xFF); } +TEST(OtaStoreRamTest, ClearKeepsResumeFixtureButDiscardConsumesHeader) { + OtaStoreRam<4096> s; + constexpr uint32_t total = 8u + MOTA_MFL + 5u; + uint8_t header[8] = {'m', 'O', 'T', 'A', + static_cast(total), + static_cast(total >> 8), + static_cast(total >> 16), + static_cast(total >> 24)}; + ASSERT_TRUE(s.begin(total)); + ASSERT_TRUE(s.write(0, header, sizeof(header))); + + // clear() remains the cheap RAM-session reset used by reboot/resume tests. + s.clear(); + EXPECT_TRUE(s.reopen()); + EXPECT_EQ(s.staged_size(), total); + + // User-facing cancellation uses discard(), which must prevent a fresh + // reopen even for the native in-process persistence model. + EXPECT_TRUE(s.discard()); + EXPECT_EQ(s.staged_size(), 0u); + EXPECT_FALSE(s.reopen()); +} + +TEST(OtaStoreRamTest, SeederSizedStoreDiscardsWithoutAssumingAHeaderFits) { + // OTA_SEEDER_ONLY intentionally uses OtaStoreRam<1>: it needs a valid + // manager destination object but never stages firmware locally. + OtaStoreRam<1> s; + EXPECT_TRUE(s.discard()); + EXPECT_EQ(s.staged_size(), 0u); + EXPECT_FALSE(s.reopen()); +} + +namespace { + +class SessionOnlyOtaStore : public OtaStore { +public: + bool begin(uint32_t) override { return false; } + bool write(uint32_t, const uint8_t*, uint32_t) override { return false; } + bool read(uint32_t, uint8_t*, uint32_t) const override { return false; } + uint32_t capacity() const override { return 0; } + uint32_t staged_size() const override { return 0; } + void clear() override { cleared = true; } + + bool cleared = false; +}; + +} // namespace + +TEST(OtaStoreDiscardTest, ConservativeDefaultCannotClaimDurableInvalidation) { + SessionOnlyOtaStore store; + EXPECT_FALSE(store.discard()); + EXPECT_TRUE(store.cleared); +} + // --- merkle proof GENERATION (server side) matches the Python oracle --------------------------- TEST(OtaMerkle, GenProofMatchesPythonAndVerifies) { @@ -1605,6 +1659,21 @@ TEST(OtaServe, ClearPrimaryInvalidatesCallerOwnedView) { EXPECT_FALSE(manager.remove_source(&folder)); } +TEST(OtaServe, FetchSessionResetPreservesIndependentPrimaryView) { + OtaManager manager; + manager.begin(0, nullptr, nullptr); + ASSERT_TRUE(manager.serve(SIM_MOTA, SIM_MOTA_LEN)); + ASSERT_EQ(manager.servedCount(), 1); + ASSERT_TRUE(manager.servedEntry(0)->is_self); + + // `ota cancel` resets only the receive side. A served image (and its + // caller-owned backing buffer) must remain registered until the serving + // side explicitly calls clear_primary(). + manager.reset_session(); + ASSERT_EQ(manager.servedCount(), 1); + EXPECT_TRUE(manager.servedEntry(0)->is_self); +} + TEST(OtaTransfer, TwoManagersFullTransfer) { g_q.clear(); OtaManager server, client; diff --git a/test/test_ota/test_ota_flashplan.cpp b/test/test_ota/test_ota_flashplan.cpp index 1b72871e..15449ba5 100644 --- a/test/test_ota/test_ota_flashplan.cpp +++ b/test/test_ota/test_ota_flashplan.cpp @@ -2,6 +2,7 @@ #include #include "helpers/ota/OtaFlashLayout_nrf52.h" +#include "helpers/ota/OtaFlashLayout_esp32.h" #include "helpers/ota/OtaStoreQspiNrf52.h" using namespace mesh::ota; @@ -318,6 +319,307 @@ TEST(OtaFlashPlan, ReopenBoundsUntrustedTotalBeforeManifestRead) { 8u + 197u + 5u)); } +namespace { + +struct FakeStagedHeader { + uint32_t address; + uint32_t total; + bool present; + bool invalidate_ok; + bool invalidated; +}; + +struct FakeStagedFlash { + FakeStagedHeader* headers; + size_t count; + uint32_t invalidate_calls; +}; + +bool fake_read_staged_header(void* context, uint32_t address, + uint32_t& total) { + FakeStagedFlash* flash = static_cast(context); + for (size_t i = 0; i < flash->count; ++i) { + FakeStagedHeader& header = flash->headers[i]; + if (header.address == address && header.present) { + total = header.total; + return true; + } + } + return false; +} + +bool fake_invalidate_staged_header(void* context, uint32_t address) { + FakeStagedFlash* flash = static_cast(context); + ++flash->invalidate_calls; + for (size_t i = 0; i < flash->count; ++i) { + FakeStagedHeader& header = flash->headers[i]; + if (header.address != address || !header.present) continue; + if (!header.invalidate_ok) return false; + header.present = false; + header.invalidated = true; + return true; + } + return false; +} + +} // namespace + +TEST(OtaFlashDiscard, FreshIdleObjectStillInvalidatesPersistedHeader) { + // The callback context models flash only: there is deliberately no live + // OtaStore session or staged_size. Cancellation after reboot must still + // find the bottom-aligned persistent header. + FakeStagedHeader headers[] = { + {0xE2000u, EXPANDED - 0xE2000u, true, true, false}, + }; + FakeStagedFlash flash{headers, 1u, 0u}; + uint32_t invalidated = 99u; + + EXPECT_TRUE(mota_nrf52_discard_staged_headers( + APP_V6, APP_END_V6, EXPANDED, &flash, fake_read_staged_header, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(invalidated, 1u); + EXPECT_EQ(flash.invalidate_calls, 1u); + EXPECT_TRUE(headers[0].invalidated); + + // Durable discard is idempotent: a new scan cannot reopen or re-invalidate + // the header that the first call consumed. + invalidated = 99u; + EXPECT_TRUE(mota_nrf52_discard_staged_headers( + APP_V6, APP_END_V6, EXPANDED, &flash, fake_read_staged_header, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(invalidated, 0u); + EXPECT_EQ(flash.invalidate_calls, 1u); +} + +TEST(OtaFlashDiscard, InvalidatesEveryCurrentAndLegacyLayoutHeader) { + FakeStagedHeader headers[] = { + // Two expanded-ceiling containers from transfers of different sizes. + {0xE2000u, EXPANDED - 0xE2000u, true, true, false}, + {0xD8000u, EXPANDED - 0xD8000u, true, true, false}, + // A retained container from the older D4000 ceiling geometry. + {0xC0000u, LEGACY - 0xC0000u, true, true, false}, + // Magic at a page whose total does not bottom-align there is not a + // reopenable store header and must remain untouched. + {0xD9000u, 0x8000u, true, true, false}, + }; + FakeStagedFlash flash{headers, 4u, 0u}; + uint32_t invalidated = 0; + + EXPECT_TRUE(mota_nrf52_discard_staged_headers( + APP_V6, APP_END_V6, EXPANDED, &flash, fake_read_staged_header, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(invalidated, 3u); + EXPECT_EQ(flash.invalidate_calls, 3u); + EXPECT_TRUE(headers[0].invalidated); + EXPECT_TRUE(headers[1].invalidated); + EXPECT_TRUE(headers[2].invalidated); + EXPECT_TRUE(headers[3].present); + EXPECT_FALSE(headers[3].invalidated); +} + +TEST(OtaFlashDiscard, ReportsFailureButContinuesInvalidatingOtherHeaders) { + FakeStagedHeader headers[] = { + {0xE2000u, EXPANDED - 0xE2000u, true, false, false}, + {0xD8000u, EXPANDED - 0xD8000u, true, true, false}, + }; + FakeStagedFlash flash{headers, 2u, 0u}; + uint32_t invalidated = 0; + + EXPECT_FALSE(mota_nrf52_discard_staged_headers( + APP_V6, APP_END_V6, EXPANDED, &flash, fake_read_staged_header, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(invalidated, 1u); + EXPECT_EQ(flash.invalidate_calls, 2u); + EXPECT_TRUE(headers[0].present); + EXPECT_TRUE(headers[1].invalidated); +} + +TEST(OtaFlashDiscard, UnsafeBoundsFailBeforeAnyFlashCallback) { + FakeStagedHeader headers[] = { + {0xE2000u, EXPANDED - 0xE2000u, true, true, false}, + }; + FakeStagedFlash flash{headers, 1u, 0u}; + uint32_t invalidated = 99u; + + EXPECT_FALSE(mota_nrf52_discard_staged_headers( + APP_V6, EXPANDED + 1u, EXPANDED, &flash, fake_read_staged_header, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(invalidated, 0u); + EXPECT_EQ(flash.invalidate_calls, 0u); + EXPECT_TRUE(headers[0].present); + + EXPECT_FALSE(mota_nrf52_discard_staged_headers( + APP_V6, APP_END_V6, EXPANDED, &flash, nullptr, + fake_invalidate_staged_header, &invalidated)); + EXPECT_EQ(flash.invalidate_calls, 0u); +} + +TEST(OtaEsp32FlashPlan, FullAndDeltaPlacementStaySectorAligned) { + constexpr uint32_t partition = 2u * 1024u * 1024u; + constexpr uint32_t sector = 4096u; + constexpr uint32_t meta_capacity = 65536u; + MotaEsp32StageLayout full; + ASSERT_TRUE(mota_esp32_stage_layout( + partition, sector, meta_capacity, true, 1024u * 1024u, 40000u, + 1024u * 1024u, full)); + EXPECT_EQ(full.total, 40000u + 1024u * 1024u + 5u); + EXPECT_EQ(full.meta_flush, 40960u); + EXPECT_EQ(full.meta_part, partition - full.meta_flush); + EXPECT_EQ(full.meta_part % sector, 0u); + EXPECT_EQ(full.pay_part0, 0u); + + MotaEsp32StageLayout delta; + ASSERT_TRUE(mota_esp32_stage_layout( + partition, sector, meta_capacity, false, 900000u, 1234u, + 100000u, delta)); + EXPECT_EQ(delta.total, 101239u); + EXPECT_EQ(delta.meta_span, sector); + EXPECT_EQ(delta.meta_part, delta.write_start); + EXPECT_EQ(delta.write_start % sector, 0u); + EXPECT_EQ(delta.pay_part0, delta.write_start + sector); +} + +TEST(OtaEsp32FlashPlan, RejectsOverflowAndUnfittableMetadata) { + MotaEsp32StageLayout layout; + EXPECT_FALSE(mota_esp32_stage_layout( + 4097u, 4096u, 65536u, true, 1u, 200u, 1u, layout)); + EXPECT_FALSE(mota_esp32_stage_layout( + 2u * 1024u * 1024u, 4096u, 65536u, true, 1u, UINT32_MAX, + 1u, layout)); + EXPECT_FALSE(mota_esp32_stage_layout( + 32768u, 4096u, 65536u, true, 1u, 40000u, 1u, layout)); + EXPECT_FALSE(mota_esp32_stage_layout( + 2u * 1024u * 1024u, 4096u, 65536u, false, + 2u * 1024u * 1024u, 1000u, 1000u, layout)); +} + +TEST(OtaEsp32FlashPlan, RejectsMalformedOrOversizedFullPayloadBeforeErase) { + constexpr uint32_t partition = 2u * 1024u * 1024u; + constexpr uint32_t sector = 4096u; + constexpr uint32_t meta_capacity = 65536u; + MotaEsp32StageLayout layout; + + // FULL means that the payload is exactly the final application image. + EXPECT_FALSE(mota_esp32_stage_layout( + partition, sector, meta_capacity, true, 100000u, 2000u, + 99999u, layout)); + EXPECT_FALSE(mota_esp32_stage_layout( + partition, sector, meta_capacity, true, 100000u, 2000u, + 100001u, layout)); + + // The logical container and rounded metadata/payload regions must both fit + // wholly inside the inactive slot before OtaStoreFlashEsp32::begin(). + EXPECT_FALSE(mota_esp32_stage_layout( + partition, sector, meta_capacity, true, partition, 2000u, + partition, layout)); + EXPECT_FALSE(mota_esp32_stage_layout( + 8192u, sector, meta_capacity, true, 4097u, 1u, 4097u, + layout)); +} + +namespace { + +struct FakeEspStagedHeader { + uint32_t offset; + bool reopenable; + bool probe_ok; + bool invalidate_ok; + bool invalidated; +}; + +struct FakeEspPartition { + FakeEspStagedHeader* headers; + size_t count; + uint32_t probe_calls; + uint32_t invalidate_calls; +}; + +bool fake_probe_esp_header(void* context, uint32_t offset, + bool& reopenable) { + FakeEspPartition* partition = static_cast(context); + ++partition->probe_calls; + reopenable = false; + for (size_t i = 0; i < partition->count; ++i) { + FakeEspStagedHeader& header = partition->headers[i]; + if (header.offset != offset || header.invalidated) continue; + reopenable = header.reopenable; + return header.probe_ok; + } + return true; +} + +bool fake_invalidate_esp_header(void* context, uint32_t offset) { + FakeEspPartition* partition = static_cast(context); + ++partition->invalidate_calls; + for (size_t i = 0; i < partition->count; ++i) { + FakeEspStagedHeader& header = partition->headers[i]; + if (header.offset != offset || header.invalidated) continue; + if (!header.invalidate_ok) return false; + header.invalidated = true; + return true; + } + return false; +} + +} // namespace + +TEST(OtaEsp32FlashDiscard, FreshIdleScanInvalidatesEveryReopenableHeader) { + constexpr uint32_t sector = 4096u; + constexpr uint32_t partition_size = 6u * sector; + FakeEspStagedHeader headers[] = { + {5u * sector, true, true, true, false}, + {3u * sector, false, true, true, false}, + {1u * sector, true, true, true, false}, + }; + FakeEspPartition partition{headers, 3u, 0u, 0u}; + uint32_t invalidated = 99u; + + EXPECT_TRUE(mota_esp32_discard_staged_headers( + partition_size, sector, &partition, fake_probe_esp_header, + fake_invalidate_esp_header, &invalidated)); + EXPECT_EQ(partition.probe_calls, 6u); + EXPECT_EQ(partition.invalidate_calls, 2u); + EXPECT_EQ(invalidated, 2u); + EXPECT_TRUE(headers[0].invalidated); + EXPECT_FALSE(headers[1].invalidated); + EXPECT_TRUE(headers[2].invalidated); +} + +TEST(OtaEsp32FlashDiscard, IoAndInvalidateFailuresAreReportedAfterFullScan) { + constexpr uint32_t sector = 4096u; + constexpr uint32_t partition_size = 6u * sector; + FakeEspStagedHeader headers[] = { + {4u * sector, false, false, true, false}, + {3u * sector, true, true, false, false}, + {1u * sector, true, true, true, false}, + }; + FakeEspPartition partition{headers, 3u, 0u, 0u}; + uint32_t invalidated = 0u; + + EXPECT_FALSE(mota_esp32_discard_staged_headers( + partition_size, sector, &partition, fake_probe_esp_header, + fake_invalidate_esp_header, &invalidated)); + EXPECT_EQ(partition.probe_calls, 6u); + EXPECT_EQ(partition.invalidate_calls, 2u); + EXPECT_EQ(invalidated, 1u); + EXPECT_FALSE(headers[1].invalidated); + EXPECT_TRUE(headers[2].invalidated); +} + +TEST(OtaEsp32FlashDiscard, InvalidScanArgumentsFailWithoutCallbacks) { + FakeEspPartition partition{nullptr, 0u, 0u, 0u}; + uint32_t invalidated = 99u; + EXPECT_FALSE(mota_esp32_discard_staged_headers( + 4095u, 4096u, &partition, fake_probe_esp_header, + fake_invalidate_esp_header, &invalidated)); + EXPECT_FALSE(mota_esp32_discard_staged_headers( + 4097u, 4096u, &partition, fake_probe_esp_header, + fake_invalidate_esp_header, &invalidated)); + EXPECT_EQ(invalidated, 0u); + EXPECT_EQ(partition.probe_calls, 0u); + EXPECT_EQ(partition.invalidate_calls, 0u); +} + // out_start is only written on success - a rejected plan must not clobber the caller's variable. TEST(OtaFlashPlan, LeavesOutputUntouchedOnReject) { uint32_t start = 0x1234ABCD; diff --git a/tools/lora_ota/build_rak3401_compact_bundle.py b/tools/lora_ota/build_rak3401_compact_bundle.py index ee74e74f..e49a8ff1 100755 --- a/tools/lora_ota/build_rak3401_compact_bundle.py +++ b/tools/lora_ota/build_rak3401_compact_bundle.py @@ -16,8 +16,10 @@ import json from pathlib import Path import re import shutil +import struct import subprocess import sys +import tempfile import zipfile @@ -27,6 +29,7 @@ sys.path.insert(0, str(REPO_ROOT / "tools" / "mota")) import motalib # noqa: E402 import rebuild_rak3401_bundle as common # noqa: E402 +import rak3401_route_search as route_search # noqa: E402 PREVIOUS_BUNDLE_SHA256 = ( @@ -34,16 +37,49 @@ PREVIOUS_BUNDLE_SHA256 = ( ) EXPECTED_TARGET_ID = 0x2FA509C1 EXPECTED_HARDWARE = "RAK_3401" -EXPECTED_ENDPOINT_VERSION = "1.17.1.02" -EXPECTED_ENDPOINT_VERSION_PACKED = motalib.pack_version(EXPECTED_ENDPOINT_VERSION) -EXPECTED_ROUTE_NODES = (0, 1, 2, 7, 13, 16, 17, 22, 24, 30) -EXPECTED_STEPS = len(EXPECTED_ROUTE_NODES) - 1 +ENDPOINT_NODE = 30 APP_BASE = 0x26000 STAGE_CEILING = 0xD4000 FIXED_WORKSPACE = 0x98000 FLASH_PAGE = 4096 SEGMENT_SIZE = 4096 BLOCK_SIZE = 1024 +UF2_BLOCK_SIZE = 512 +UF2_DATA_OFFSET = 32 +UF2_DATA_CAPACITY = 476 +UF2_MAGIC_START0 = 0x0A324655 +UF2_MAGIC_START1 = 0x9E5D5157 +UF2_MAGIC_END = 0x0AB16F30 +UF2_FLAG_NOT_MAIN_FLASH = 0x00000001 +UF2_FLAG_FAMILY_ID = 0x00002000 +UF2_NRF52840_FAMILY_ID = 0xADA52840 +REQUIRED_SIMULATORS = { + "preview5": { + "sha256": "c6b3c6bc0b284b0b2c7a9eef1b3ea7afeb05ce4241d81309d4ab6381a575b897", + "tag": "0.9.2-OTAFIX2.4.1-preview.5", + "commit": "f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48", + }, + "current": { + "sha256": "bf57fc4b62f11d6d9ee1d8b7a863d22d3e8048ad750f6bee5ceffd7b2d4d83a7", + "commit": "ffb1580d8fa59d333e1e944a198ec9b231be5a40", + }, +} +MOTATOOL_VERSION = "0.1.0" +MOTATOOL_COMMIT = "b0bfa1c16fa758f622cacaea6e427756921c4271" +DETOOLS_VERSION = "0.53.0" +PHYSICAL_VALIDATION_KIND = "rak3401-mota-exact-chain" +MAX_PHYSICAL_VALIDATION_BYTES = 256 * 1024 +PHYSICAL_VALIDATION_FIELDS = { + "schema", "kind", "status", "chain_sha256", "start_sha256", + "endpoint_sha256", "endpoint_body_hash", "endpoint_version", + "step_count", "steps", "final_swd", +} + +# Capture the source text at module load. The bundle must archive the code +# that this process loaded, even if another process edits the shared checkout +# while the relatively long patch/simulator pass is running. +EXECUTING_BUILDER_SOURCE = Path(__file__).resolve().read_bytes() +EXECUTING_ROUTE_SEARCH_SOURCE = Path(route_search.__file__).resolve().read_bytes() class CompactBuildError(RuntimeError): @@ -79,7 +115,7 @@ def align_down(value: int, unit: int) -> int: def parse_int(value: object, label: str) -> int: - if isinstance(value, int): + if isinstance(value, int) and not isinstance(value, bool): result = value elif isinstance(value, str) and re.fullmatch(r"(?:0[xX][0-9a-fA-F]+|[0-9]+)", value): result = int(value, 0) @@ -100,27 +136,170 @@ def parse_simulators(values: list[str]) -> list[tuple[str, Path]]: if not path.is_file(): raise CompactBuildError(f"apply simulator does not exist: {path}") simulators.append((label, path)) + expected = {label: details["sha256"] for label, details in REQUIRED_SIMULATORS.items()} + actual = {label: common.sha256_file(path) for label, path in simulators} + if len(actual) != len(simulators): + raise CompactBuildError("apply simulator labels must be unique") + if actual != expected: + raise CompactBuildError( + "legacy RAK3401 builds require exactly preview5=" + f"{expected['preview5']} and current={expected['current']} apply simulators" + ) return simulators +def snapshot_file(source: Path, destination: Path) -> Path: + """Freeze a regular build input before any package generation starts.""" + destination.parent.mkdir(parents=True, exist_ok=True) + try: + shutil.copy2(source, destination, follow_symlinks=True) + except OSError as exc: + raise CompactBuildError(f"cannot snapshot build input {source}: {exc}") from exc + if not destination.is_file(): + raise CompactBuildError(f"snapshotted build input is not a file: {destination}") + return destination + + +def make_reproducible_zip(root: Path, output: Path) -> None: + """Archive content with stable metadata so identical inputs hash identically.""" + resolved_root = root.resolve(strict=True) + resolved_output = output.resolve(strict=False) + if resolved_output.is_relative_to(resolved_root): + raise CompactBuildError( + f"archive output must be outside the archived tree: {output}" + ) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise CompactBuildError(f"output already exists: {output}") + archived_files = sorted(item for item in root.rglob("*") if item.is_file()) + with zipfile.ZipFile( + output, "x", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: + for path in archived_files: + archive_name = (Path(root.name) / path.relative_to(root)).as_posix() + info = zipfile.ZipInfo(archive_name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, path.read_bytes()) + + +def snapshot_schema2_inventory(source: Path, destination: Path) -> Path: + """Freeze a schema-2 manifest and every firmware image it references.""" + try: + document = json.loads(source.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError(f"cannot read image inventory {source}: {exc}") from exc + records = document.get("images") if isinstance(document, dict) else None + if not isinstance(records, list): + raise CompactBuildError("image inventory must contain an images list") + image_directory = destination.parent / "inventory-images" + for node, record in enumerate(records): + if not isinstance(record, dict): + raise CompactBuildError(f"image inventory node {node} is invalid") + raw_path = record.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise CompactBuildError(f"image inventory node {node} has no path") + image_source = Path(raw_path) + if not image_source.is_absolute(): + image_source = source.parent / image_source + image_name = f"image-{node:02d}.bin" + snapshot_file(image_source, image_directory / image_name) + record["path"] = str(Path("inventory-images") / image_name) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + return destination + + +def snapshot_schema1_manifest(source: Path, destination: Path) -> Path: + """Freeze a schema-1 manifest together with its sibling firmware ZIPs.""" + snapshot_file(source, destination) + try: + document = json.loads(destination.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError(f"cannot read accelerated image manifest: {exc}") from exc + records = document.get("targets") if isinstance(document, dict) else None + if not isinstance(records, list): + raise CompactBuildError("accelerated image manifest must contain a targets list") + for index, record in enumerate(records, 1): + filename = record.get("zip") if isinstance(record, dict) else None + if not isinstance(filename, str): + raise CompactBuildError(f"accelerated image {index} has no ZIP filename") + try: + relative = common.safe_relative(filename, f"accelerated image {index} ZIP") + except common.RebuildError as exc: + raise CompactBuildError(str(exc)) from exc + if len(relative.parts) != 1: + raise CompactBuildError( + f"accelerated image {index} ZIP must be a basename" + ) + snapshot_file(source.parent / relative.name, destination.parent / relative.name) + return destination + + +def tool_provenance(motatool: Path, detools: Path) -> dict[str, object]: + """Validate portable tool versions and record the exact executing launchers.""" + motatool_output = run([str(motatool), "-V"], "read motatool version").strip() + if motatool_output != f"motatool {MOTATOOL_VERSION}": + raise CompactBuildError( + f"motatool version is {motatool_output!r}, expected {MOTATOOL_VERSION}" + ) + detools_output = run([str(detools), "--version"], "read detools version").strip() + if detools_output != DETOOLS_VERSION: + raise CompactBuildError( + f"detools version is {detools_output!r}, expected {DETOOLS_VERSION}" + ) + return { + "motatool": { + "version": MOTATOOL_VERSION, + "asserted_source_commit": MOTATOOL_COMMIT, + "executable_sha256": common.sha256_file(motatool), + }, + "detools": { + "version": DETOOLS_VERSION, + "launcher_sha256": common.sha256_file(detools), + }, + } + + def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: try: document = json.loads(path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise CompactBuildError(f"cannot read route plan {path}: {exc}") from exc - if not isinstance(document, dict) or document.get("schema") != 1: - raise CompactBuildError("route plan must use schema 1") + if not isinstance(document, dict) or document.get("schema") not in (1, 2): + raise CompactBuildError("route plan must use schema 1 or 2") + schema = int(document["schema"]) + if schema == 2: + if document.get("status") != "reachable": + raise CompactBuildError("schema-2 route plan does not reach the endpoint") + if document.get("search_complete") is not True: + raise CompactBuildError("schema-2 route plan search is incomplete") + node_count = parse_int(document.get("node_count"), "route node_count") + if node_count < 3: + raise CompactBuildError("route node_count is invalid") + endpoint_node = node_count - 1 + else: + node_count = ENDPOINT_NODE + 1 + endpoint_node = ENDPOINT_NODE if parse_int(document.get("app_base"), "route app_base") != APP_BASE: raise CompactBuildError("route plan has the wrong application base") if parse_int(document.get("stage_ceiling"), "route stage_ceiling") != STAGE_CEILING: raise CompactBuildError("route plan does not target the deployed 0xD4000 ceiling") - if document.get("shortest_package_count") != EXPECTED_STEPS: - raise CompactBuildError( - f"route plan shortest_package_count must be {EXPECTED_STEPS}" - ) + expected_steps = document.get("shortest_package_count") + if ( + not isinstance(expected_steps, int) + or isinstance(expected_steps, bool) + or expected_steps < 2 + or expected_steps > endpoint_node + ): + raise CompactBuildError("route plan shortest_package_count is invalid") raw_steps = document.get("steps") - if not isinstance(raw_steps, list) or len(raw_steps) != EXPECTED_STEPS: - raise CompactBuildError(f"route plan must contain {EXPECTED_STEPS} steps") + if not isinstance(raw_steps, list) or len(raw_steps) != expected_steps: + raise CompactBuildError(f"route plan must contain {expected_steps} steps") steps: list[dict[str, object]] = [] nodes = [] for number, raw in enumerate(raw_steps, 1): @@ -150,7 +329,7 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: reuse = raw.get("reuse_baseline_package", False) if not isinstance(reuse, bool): raise CompactBuildError(f"route step {number} reuse flag must be boolean") - if source >= target or memory % FLASH_PAGE: + if source >= target or target >= node_count or memory % FLASH_PAGE: raise CompactBuildError(f"route step {number} has invalid nodes or workspace alignment") if memory < FLASH_PAGE or APP_BASE + memory >= STAGE_CEILING: raise CompactBuildError(f"route step {number} workspace is outside legacy flash") @@ -169,16 +348,64 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: "expected_target_sha256": expected_sha, "expected_target_version": expected_version, }) - if tuple(nodes) != EXPECTED_ROUTE_NODES: + if not nodes or nodes[0] != 0 or nodes[-1] != endpoint_node: raise CompactBuildError( - f"route nodes are {tuple(nodes)}, expected {EXPECTED_ROUTE_NODES}" + f"route nodes must run from 0 through {endpoint_node}: {tuple(nodes)}" ) + if len(nodes) < 2 or nodes[1] != 1: + raise CompactBuildError("route must retain the pinned 0->1 physical bridge") if not steps[0]["reuse_baseline_package"] or any( step["reuse_baseline_package"] for step in steps[1:] ): raise CompactBuildError("only the first route step may reuse the baseline package") - if steps[0]["inplace_memory"] != FIXED_WORKSPACE or steps[1]["inplace_memory"] != FIXED_WORKSPACE: - raise CompactBuildError("the two fixed-receiver steps require a 0x98000 workspace") + if steps[0]["inplace_memory"] != FIXED_WORKSPACE: + raise CompactBuildError("the pinned first bridge requires a 0x98000 workspace") + if steps[1]["inplace_memory"] != FIXED_WORKSPACE: + raise CompactBuildError("the fixed-receiver second step requires a 0x98000 workspace") + search = document.get("search") if schema == 1 else document + if not isinstance(search, dict): + raise CompactBuildError("route plan is missing its exhaustive-search summary") + if search.get("objective") != "minimum packages, then minimum total container bytes": + raise CompactBuildError("route plan has the wrong search objective") + if search.get("shortest_package_count") != expected_steps: + raise CompactBuildError("route plan search package count is inconsistent") + if schema == 1: + if parse_int(search.get("page_size"), "route search page_size") != FLASH_PAGE: + raise CompactBuildError("route plan search used the wrong page size") + selected_bytes = sum(int(step["expected_container_size"]) for step in steps) + if ( + parse_int( + search.get("selected_total_bytes"), + "route search selected_total_bytes", + ) + != selected_bytes + ): + raise CompactBuildError("route plan search byte total is inconsistent") + search_aliases = { + "candidate_geometries": ( + "candidate_geometries", "shorter_path_workspace_candidates" + ), + "feasible_edges": ("feasible_edges", "shortest_dag_edges"), + } + if schema == 1: + search_aliases["candidate_pairs"] = ("candidate_pairs", "shorter_path_pairs") + for field, aliases in search_aliases.items(): + value = next((search.get(alias) for alias in aliases if alias in search), None) + if parse_int(value, f"route search {field}") < 1: + raise CompactBuildError(f"route plan search {field} must be positive") + if parse_int(search.get("shortest_route_count"), "route search shortest_route_count") < 1: + raise CompactBuildError("route plan search shortest_route_count must be positive") + document["endpoint_node"] = endpoint_node + if schema == 2: + document["search"] = { + key: search[key] + for key in ( + "candidate_geometries", "feasible_edges", "objective", + "selected_total_bytes", "shortest_package_count", + "shortest_route_count", "search_complete", "node_count", + ) + if key in search + } return steps, document @@ -206,6 +433,405 @@ def firmware_identity(image: bytes) -> tuple[int, str, str, int]: return ident.fw_version, body_hash.hex(), motalib.unpack_version(ident.fw_version), len(body) +def read_image_inventory( + path: Path, + expected_node_count: int, +) -> tuple[dict[int, bytes], dict[int, str]]: + """Read a schema-2 optimizer inventory without trusting its paths or pins.""" + try: + document = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError(f"cannot read image inventory {path}: {exc}") from exc + raw_images = document.get("images") if isinstance(document, dict) else None + if not isinstance(raw_images, list) or len(raw_images) != expected_node_count: + raise CompactBuildError( + f"image inventory must contain {expected_node_count} ordered nodes" + ) + + images: dict[int, bytes] = {} + sources: dict[int, str] = {} + seen_hashes: set[str] = set() + for node, raw in enumerate(raw_images): + if not isinstance(raw, dict) or parse_int(raw.get("node", node), f"inventory node {node}") != node: + raise CompactBuildError("image inventory nodes must be contiguous and zero-based") + raw_path = raw.get("path") + if not isinstance(raw_path, str) or not raw_path: + raw_path = f"image-{node:02d}.bin" + image_path = Path(raw_path) + if not image_path.is_absolute(): + image_path = path.parent / image_path + try: + image = image_path.read_bytes() + except OSError as exc: + raise CompactBuildError( + f"cannot read inventory node {node} image {image_path}: {exc}" + ) from exc + actual_sha = common.sha256_bytes(image) + expected_sha = raw.get("sha256") + if not isinstance(expected_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_sha): + raise CompactBuildError(f"inventory node {node} SHA-256 is invalid") + if actual_sha != expected_sha: + raise CompactBuildError(f"inventory node {node} SHA-256 mismatch") + if actual_sha in seen_hashes: + raise CompactBuildError(f"inventory node {node} duplicates an earlier image") + seen_hashes.add(actual_sha) + if parse_int(raw.get("size"), f"inventory node {node} size") != len(image): + raise CompactBuildError(f"inventory node {node} size mismatch") + _packed, actual_body_hash, version, _body_size = firmware_identity(image) + declared_body_hash = raw.get("body_hash") + if ( + not isinstance(declared_body_hash, str) + or not re.fullmatch(r"[0-9a-f]{16}", declared_body_hash) + or declared_body_hash != actual_body_hash + ): + raise CompactBuildError(f"inventory node {node} body hash mismatch") + expected_version = raw.get("version") + if not isinstance(expected_version, str) or expected_version != version: + raise CompactBuildError(f"inventory node {node} version mismatch") + source_commit = raw.get("source_commit") + if source_commit is not None and ( + not isinstance(source_commit, str) + or not re.fullmatch(r"[0-9a-f]{40}", source_commit) + ): + raise CompactBuildError(f"inventory node {node} source commit is invalid") + images[node] = image + # A complete firmware hash is still stable provenance for an optimizer-created + # bridge whose source commit was not recorded by the temporary inventory tool. + sources[node] = source_commit or f"image-sha256:{actual_sha}" + return images, sources + + +def write_inventory_provenance(source: Path, destination: Path) -> None: + """Emit only the schema-2 content pins needed for public provenance.""" + document = json.loads(source.read_text(encoding="ascii")) + raw_images = document.get("images") if isinstance(document, dict) else None + if not isinstance(raw_images, list) or not all( + isinstance(record, dict) for record in raw_images + ): + raise CompactBuildError("cannot normalize invalid image inventory") + allowed = ( + "node", "size", "sha256", "body_hash", "version", + "source_commit", "baseline_container_size", + ) + document = { + "schema": 2, + "images": [ + {key: record[key] for key in allowed if key in record} + for record in raw_images + ], + } + destination.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + + +def write_accelerated_provenance(source: Path, destination: Path) -> None: + """Emit only validated schema-1 bridge-image content pins.""" + document = json.loads(source.read_text(encoding="ascii")) + raw_targets = document.get("targets") if isinstance(document, dict) else None + if not isinstance(raw_targets, list) or not all( + isinstance(record, dict) for record in raw_targets + ): + raise CompactBuildError("cannot normalize invalid accelerated image manifest") + allowed = ( + "version", "zip", "zip_sha256", "firmware_sha256", "firmware_size", + "body_hash", "body_size", "source_commit", + ) + normalized = { + "schema": 1, + "targets": [ + {key: record[key] for key in allowed if key in record} + for record in raw_targets + ], + } + destination.write_text( + json.dumps(normalized, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + + +def public_route_search( + schema: int, evidence: dict[str, object] +) -> dict[str, object]: + """Return only validated route-search metrics safe for public artifacts.""" + raw_search = evidence.get("search") + search = raw_search if isinstance(raw_search, dict) else evidence + if schema == 2: + allowed = ( + "candidate_geometries", "feasible_edges", "objective", + "selected_total_bytes", "shortest_package_count", + "shortest_route_count", "search_complete", "node_count", + ) + return {key: search[key] for key in allowed if key in search} + if schema != 1 or not isinstance(raw_search, dict): + raise CompactBuildError("schema-1 route has no validated search summary") + normalized = { + key: search[key] + for key in ( + "objective", "shortest_package_count", "selected_total_bytes", + "page_size", "candidate_pairs", "candidate_geometries", + "feasible_edges", "shortest_route_count", + ) + if key in search + } + # Older schema-1 plans used aliases. Publish canonical names only. + aliases = { + "candidate_pairs": "shorter_path_pairs", + "candidate_geometries": "shorter_path_workspace_candidates", + "feasible_edges": "shortest_dag_edges", + } + for canonical, legacy in aliases.items(): + if canonical not in normalized: + normalized[canonical] = search[legacy] + return normalized + + +def write_route_provenance( + destination: Path, + schema: int, + steps: list[dict[str, object]], + evidence: dict[str, object], + source_commit: str, + endpoint_version: str, +) -> None: + """Emit a reusable route plan containing only validated public fields.""" + normalized_steps = [ + { + "source_node": int(step["source_node"]), + "target_node": int(step["target_node"]), + "inplace_memory": f"0x{int(step['inplace_memory']):X}", + "reuse_baseline_package": bool(step["reuse_baseline_package"]), + "expected_container_size": int(step["expected_container_size"]), + "expected_staging_margin": int(step["expected_staging_margin"]), + "expected_target_sha256": str(step["expected_target_sha256"]), + "expected_target_version": str(step["expected_target_version"]), + } + for step in steps + ] + if schema == 2: + allowed = ( + "schema", "status", "search_complete", "app_base", + "stage_ceiling", "node_count", "candidate_geometries", + "feasible_edges", "objective", "nodes", + "shortest_package_count", "shortest_route_count", + "selected_total_bytes", + ) + normalized = {key: evidence[key] for key in allowed if key in evidence} + else: + normalized = { + "schema": 1, + "app_base": f"0x{APP_BASE:X}", + "stage_ceiling": f"0x{STAGE_CEILING:X}", + "shortest_package_count": len(normalized_steps), + "search": public_route_search(schema, evidence), + } + normalized.update({ + "endpoint_node": int(steps[-1]["target_node"]), + "endpoint_version": endpoint_version, + "endpoint_source_commit": source_commit, + "endpoint_name": "halo-keymind-cascade-dev", + "steps": normalized_steps, + }) + destination.write_text( + json.dumps(normalized, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + + +def validate_geometry_results( + geometry_path: Path, + inventory_path: Path, + route_path: Path, +) -> dict[str, object]: + """Prove a schema-2 route from a complete, internally consistent geometry set.""" + try: + inventory = route_search.load_inventory(inventory_path) + except route_search.RouteSearchError as exc: + raise CompactBuildError(f"invalid route-search inventory: {exc}") from exc + jobs = route_search.all_jobs(inventory) + required = { + route_search.cache_key(job[5], job[6], job[2]): (job[0], job[1]) + for job in jobs + } + rows: list[dict[str, object]] = [] + seen: set[tuple[str, str, int]] = set() + try: + source = geometry_path.open(newline="", encoding="ascii") + except OSError as exc: + raise CompactBuildError(f"cannot read geometry results {geometry_path}: {exc}") from exc + with source: + reader = csv.DictReader(source) + if reader.fieldnames != route_search.FIELDS: + raise CompactBuildError("geometry results have unexpected columns") + for line, raw in enumerate(reader, 2): + try: + source_node = parse_int(raw["source"], f"geometry line {line} source") + target_node = parse_int(raw["target"], f"geometry line {line} target") + memory = parse_int(raw["memory"], f"geometry line {line} memory") + except (KeyError, TypeError) as exc: + raise CompactBuildError(f"geometry line {line} is malformed") from exc + source_sha_raw = raw.get("source_sha256") + target_sha_raw = raw.get("target_sha256") + if ( + not isinstance(source_sha_raw, str) + or not re.fullmatch(r"[0-9a-fA-F]{64}", source_sha_raw) + or not isinstance(target_sha_raw, str) + or not re.fullmatch(r"[0-9a-fA-F]{64}", target_sha_raw) + ): + raise CompactBuildError(f"geometry line {line} has invalid image hashes") + source_sha = source_sha_raw.lower() + target_sha = target_sha_raw.lower() + key = route_search.cache_key(source_sha, target_sha, memory) + if key in seen: + raise CompactBuildError(f"duplicate geometry result at line {line}") + seen.add(key) + legacy_pinned_source_measurement = ( + source_node == 0 + and 2 <= target_node < len(inventory) + and memory == FIXED_WORKSPACE + and source_sha == str(inventory[0]["sha256"]) + and target_sha == str(inventory[target_node]["sha256"]) + ) + if ( + (key not in required or required[key] != (source_node, target_node)) + and not legacy_pinned_source_measurement + ): + raise CompactBuildError(f"extra or misindexed geometry result at line {line}") + try: + payload = int(raw["payload"]) + container = int(raw["container"]) + stage_start = int(raw["stage_start"]) + margin = int(raw["margin"]) + except (TypeError, ValueError) as exc: + raise CompactBuildError(f"geometry line {line} has invalid numeric results") from exc + feasible_raw = raw.get("feasible") + if not isinstance(feasible_raw, str): + raise CompactBuildError( + f"geometry line {line} has invalid feasible value" + ) + feasible_text = feasible_raw.lower() + if feasible_text not in ("true", "false"): + raise CompactBuildError(f"geometry line {line} has invalid feasible value") + feasible = feasible_text == "true" + error_raw = raw.get("error") + if not isinstance(error_raw, str): + raise CompactBuildError(f"geometry line {line} has invalid error value") + error = error_raw + if payload < 0: + raise CompactBuildError( + f"geometry line {line} is a failed patch job and cannot prove " + "an exhaustive route search" + ) + else: + expected_container = route_search.container_size(payload) + expected_stage = route_search.align_down(STAGE_CEILING - expected_container) + expected_margin = expected_stage - (APP_BASE + memory) + if ( + container != expected_container + or stage_start != expected_stage + or margin != expected_margin + or feasible != (expected_margin >= 0) + or error + ): + raise CompactBuildError(f"geometry line {line} has inconsistent geometry") + row = dict(raw) + row.update(source=source_node, target=target_node, memory=memory, + payload=payload, container=container, stage_start=stage_start, + margin=margin, feasible=feasible, error=error, + source_sha256=source_sha, target_sha256=target_sha) + rows.append(row) + missing = set(required) - seen + if missing: + raise CompactBuildError(f"geometry results are missing {len(missing)} required rows") + baseline_size = parse_int( + inventory[1].get("baseline_container_size"), + "inventory node 1 baseline_container_size", + ) + try: + with tempfile.TemporaryDirectory() as directory: + recomputed = route_search.select_route( + rows, inventory, baseline_size, Path(directory) / "route.json", + complete=True, + ) + except route_search.RouteSearchError as exc: + raise CompactBuildError(f"route recomputation failed: {exc}") from exc + try: + declared = json.loads(route_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError(f"cannot reread route plan: {exc}") from exc + compared = ( + "schema", "status", "search_complete", "app_base", "stage_ceiling", + "node_count", "candidate_geometries", "feasible_edges", "objective", + "nodes", "shortest_package_count", "shortest_route_count", + "selected_total_bytes", "steps", + ) + for field in compared: + if declared.get(field) != recomputed.get(field): + raise CompactBuildError(f"route plan does not match recomputed {field}") + return recomputed + + +def validate_uf2_firmware(path: Path, image: bytes, label: str) -> None: + """Require an app-only nRF52840 UF2 carrying this exact firmware image.""" + try: + raw = path.read_bytes() + except OSError as exc: + raise CompactBuildError(f"cannot read {label}: {exc}") from exc + if not raw or len(raw) % UF2_BLOCK_SIZE: + raise CompactBuildError(f"{label} is not a complete UF2 block stream") + + blocks = len(raw) // UF2_BLOCK_SIZE + block_numbers: set[int] = set() + flash: dict[int, int] = {} + payload_size: int | None = None + for index in range(blocks): + block = raw[index * UF2_BLOCK_SIZE:(index + 1) * UF2_BLOCK_SIZE] + ( + magic0, magic1, flags, address, size, block_number, + declared_blocks, family_id, + ) = struct.unpack_from(" UF2_DATA_CAPACITY: + raise CompactBuildError(f"{label} block {index} has invalid payload size") + if payload_size is None: + payload_size = size + elif size != payload_size: + raise CompactBuildError(f"{label} uses inconsistent UF2 payload sizes") + if declared_blocks != blocks or block_number >= blocks: + raise CompactBuildError(f"{label} block {index} has inconsistent numbering") + if block_number in block_numbers: + raise CompactBuildError(f"{label} repeats UF2 block {block_number}") + block_numbers.add(block_number) + for offset, value in enumerate(block[UF2_DATA_OFFSET:UF2_DATA_OFFSET + size]): + byte_address = address + offset + if byte_address in flash: + raise CompactBuildError(f"{label} contains overlapping flash data") + flash[byte_address] = value + + if block_numbers != set(range(blocks)): + raise CompactBuildError(f"{label} is missing a numbered UF2 block") + assert payload_size is not None + expected_end = align_up(APP_BASE + len(image), payload_size) + if set(flash) != set(range(APP_BASE, expected_end)): + raise CompactBuildError(f"{label} is not a contiguous app-only UF2 image") + encoded = bytes(flash[address] for address in range(APP_BASE, expected_end)) + if encoded[:len(image)] != image: + raise CompactBuildError(f"{label} firmware does not match the endpoint ZIP") + if any(value != 0xFF for value in encoded[len(image):]): + raise CompactBuildError(f"{label} has non-erased data after the endpoint image") + + def apply_in_place( detools: Path, base: bytes, @@ -266,33 +892,195 @@ def write_chain(path: Path, rows: list[dict[str, object]]) -> None: writer.writerows(rows) +def read_physical_validation_record(path: Path) -> dict[str, object]: + """Read a structured publisher assertion; arbitrary transcripts are unsafe.""" + try: + document = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError( + f"cannot read physical validation record {path}: {exc}" + ) from exc + if not isinstance(document, dict): + raise CompactBuildError("physical validation record must be a JSON object") + if set(document) != PHYSICAL_VALIDATION_FIELDS: + raise CompactBuildError( + "physical validation record has unexpected or missing fields; " + f"expected {sorted(PHYSICAL_VALIDATION_FIELDS)}" + ) + return document + + +def validate_physical_validation_record( + document: dict[str, object], + *, + chain_sha256: str, + start_sha256: str, + endpoint_sha256: str, + endpoint_body_hash: str, + endpoint_version: str, + output_rows: list[dict[str, object]], + validation_steps: list[dict[str, object]], +) -> dict[str, object]: + """Bind a publisher's hardware/SWD assertion to every exact package.""" + if set(document) != PHYSICAL_VALIDATION_FIELDS: + raise CompactBuildError( + "physical validation record has unexpected or missing fields; " + f"expected {sorted(PHYSICAL_VALIDATION_FIELDS)}" + ) + if ( + not isinstance(document.get("schema"), int) + or isinstance(document.get("schema"), bool) + or document["schema"] != 1 + ): + raise CompactBuildError("physical validation record must use schema 1") + if document.get("kind") != PHYSICAL_VALIDATION_KIND: + raise CompactBuildError("physical validation record has the wrong kind") + if document.get("status") != "passed": + raise CompactBuildError("physical validation record is not passed") + + def require_hash(value: object, expected: str, label: str, length: int = 64) -> str: + if not isinstance(value, str) or not re.fullmatch( + rf"[0-9a-fA-F]{{{length}}}", value + ): + raise CompactBuildError(f"physical validation {label} is invalid") + normalized = value.lower() + if normalized != expected.lower(): + raise CompactBuildError(f"physical validation {label} does not match this chain") + return normalized + + normalized_chain = require_hash( + document.get("chain_sha256"), chain_sha256, "CHAIN.csv SHA-256" + ) + normalized_start = require_hash( + document.get("start_sha256"), start_sha256, "start image SHA-256" + ) + normalized_endpoint = require_hash( + document.get("endpoint_sha256"), endpoint_sha256, + "endpoint image SHA-256", + ) + normalized_body = require_hash( + document.get("endpoint_body_hash"), endpoint_body_hash, + "endpoint body hash", 16, + ) + if document.get("endpoint_version") != endpoint_version: + raise CompactBuildError("physical validation endpoint version does not match") + if ( + not isinstance(document.get("step_count"), int) + or isinstance(document.get("step_count"), bool) + or document["step_count"] != len(validation_steps) + ): + raise CompactBuildError("physical validation step count does not match") + raw_steps = document.get("steps") + if not isinstance(raw_steps, list) or len(raw_steps) != len(validation_steps): + raise CompactBuildError("physical validation must contain every chain step") + + normalized_steps = [] + step_fields = { + "step", "status", "mota_sha256", "target_sha256", "target_body_hash" + } + for number, (raw, expected, output_row) in enumerate( + zip(raw_steps, validation_steps, output_rows), 1 + ): + if not isinstance(raw, dict) or set(raw) != step_fields: + raise CompactBuildError( + f"physical validation step {number} has unexpected or missing fields" + ) + if ( + not isinstance(raw.get("step"), int) + or isinstance(raw.get("step"), bool) + or raw["step"] != number + ): + raise CompactBuildError(f"physical validation step {number} is misnumbered") + if raw.get("status") != "passed": + raise CompactBuildError(f"physical validation step {number} is not passed") + normalized_steps.append({ + "step": number, + "status": "passed", + "mota_sha256": require_hash( + raw.get("mota_sha256"), str(expected["mota_sha256"]), + f"step {number} mOTA SHA-256", + ), + "target_sha256": require_hash( + raw.get("target_sha256"), str(expected["target_sha256"]), + f"step {number} target SHA-256", + ), + "target_body_hash": require_hash( + raw.get("target_body_hash"), str(output_row["target_body_hash"]), + f"step {number} target body hash", 16, + ), + }) + + final_swd = document.get("final_swd") + if not isinstance(final_swd, dict) or set(final_swd) != {"status", "app_sha256"}: + raise CompactBuildError("physical validation final_swd record is invalid") + if final_swd.get("status") != "passed": + raise CompactBuildError("physical validation final SWD verification is not passed") + normalized_swd = require_hash( + final_swd.get("app_sha256"), endpoint_sha256, + "final SWD application SHA-256", + ) + return { + "schema": 1, + "kind": PHYSICAL_VALIDATION_KIND, + "status": "passed", + "chain_sha256": normalized_chain, + "start_sha256": normalized_start, + "endpoint_sha256": normalized_endpoint, + "endpoint_body_hash": normalized_body, + "endpoint_version": endpoint_version, + "step_count": len(normalized_steps), + "steps": normalized_steps, + "final_swd": {"status": "passed", "app_sha256": normalized_swd}, + } + + def write_docs( root: Path, source_commit: str, + endpoint_version: str, endpoint: bytes, previous_sha: str, bridge_sha: str, route_sha: str, + geometry_sha: str | None, + route_search_sha: str | None, + builder_source_sha: str, + tools: dict[str, object], rows: list[dict[str, object]], + physically_qualified: bool, ) -> None: endpoint_sha = common.sha256_bytes(endpoint) _packed, endpoint_body, _version, _body_size = firmware_identity(endpoint) total_bytes = sum(int(row["mota_size"]) for row in rows) largest = max(int(row["mota_size"]) for row in rows) minimum_margin = min(int(row["staging_margin"]) for row in rows) + expected_steps = len(rows) + physical_text = ( + "The publisher supplied an unsigned operator assertion that the exact " + "package sequence passed physical/SWD validation; every hash in that " + "assertion is bound to this archive. It is provenance, not a " + "cryptographic attestation, so consumers must trust the publisher." + if physically_qualified else + "A publisher physical/SWD validation assertion has not yet been attached to this archive." + ) + live_text = ( + "Live use requires a runner revision that pins this exact archive SHA, checksum list, and step anchors." + if physically_qualified else + "Do not use this candidate live until its exact package sequence is physically qualified." + ) (root / "README.md").write_text( f"""# RAK3401 compact mOTA update chain -This archive contains {EXPECTED_STEPS} mandatory mOTA packages from the deployed +This archive contains {expected_steps} mandatory mOTA packages from the deployed `1.16.7.0-c1caa5ad` image to -`v{EXPECTED_ENDPOINT_VERSION}-halo-keymind-cascade-dev-{source_commit[:8]}`. +`v{endpoint_version}-halo-keymind-cascade-dev-{source_commit[:8]}`. The deployed bootloader is unchanged. Every flash package is bottom-aligned below its original `0x{STAGE_CEILING:X}` scan ceiling, and every encoded detools workspace ends at or below that package. No byte from `0x{STAGE_CEILING:X}` through `0xED000` is used by this chain. -- Packages: {EXPECTED_STEPS} (the replaced chain used 30) +- Packages: {expected_steps} (the replaced chain used 30) - Total transferred bytes: {total_bytes:,} - Largest package: {largest:,} bytes - Smallest workspace-to-stage margin: {minimum_margin:,} bytes @@ -300,14 +1088,17 @@ through `0xED000` is used by this chain. - Endpoint EndF body hash: `{endpoint_body.upper()}` Step 1 is byte-for-byte identical to the physically passed baseline package. -Step 2 installs the dynamic-workspace receiver while still obeying the old -fixed `0x{FIXED_WORKSPACE:X}` receiver limit. Steps 3-{EXPECTED_STEPS} use the -page-aligned workspace selected in `ROUTE.json`. +The next package leaves the fixed-workspace receiver while still obeying its +old `0x{FIXED_WORKSPACE:X}` limit. Every later package uses the page-aligned +workspace selected in `ROUTE.json`. -All {EXPECTED_STEPS} transitions passed container verification, zero-filled and -erased-workspace reconstruction, the supplied legacy bootloader simulator, and -the exact physical chain on a deployed RAK3401 1W on 19-Aug-2026. Use the pinned -chain runner and do not install packages out of order. +All {expected_steps} transitions passed container verification, zero-filled and +erased-workspace reconstruction, OTAFIX 2.4.1 Preview 5 simulator validation +(tag `0.9.2-OTAFIX2.4.1-preview.5`, commit +`f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48`) and current simulator validation +(commit `ffb1580d8fa59d333e1e944a198ec9b231be5a40`). +{physical_text} Use the pinned chain runner and do not install packages out of +order. """, encoding="ascii", ) @@ -315,8 +1106,8 @@ chain runner and do not install packages out of order. f"""# Build and validation provenance Endpoint source commit: `{source_commit}` -Endpoint requested version: `{EXPECTED_ENDPOINT_VERSION}` -Endpoint packed version: `0x{EXPECTED_ENDPOINT_VERSION_PACKED:08X}` +Endpoint requested version: `{endpoint_version}` +Endpoint packed version: `0x{motalib.pack_version(endpoint_version):08X}` Endpoint profile name: `halo-keymind-cascade-dev` Legacy stage ceiling: `0x{STAGE_CEILING:X}` Application base: `0x{APP_BASE:X}` @@ -326,22 +1117,44 @@ Logical mOTA block size: `{BLOCK_SIZE}` Pinned 30-step reconstruction input SHA-256: `{previous_sha}` Dynamic bridge-image manifest SHA-256: `{bridge_sha}` Compact route plan SHA-256: `{route_sha}` +Geometry-results SHA-256: `{geometry_sha or "not applicable (schema 1)"}` +Bundled route-selection/verifier source SHA-256: +`{route_search_sha or "not applicable (schema 1)"}` +Executing bundle-builder source SHA-256: `{builder_source_sha}` -The route plan records the shortest path found by sweeping every page-aligned -workspace relevant to a path shorter than {EXPECTED_STEPS} packages. The -bundle generator independently checks the selected geometry against the old -bootloader ceiling before invoking either simulator. +Tool provenance: + +- motatool `{tools['motatool']['version']}`, asserted source commit + `{tools['motatool']['asserted_source_commit']}`, executable SHA-256 + `{tools['motatool']['executable_sha256']}` +- detools `{tools['detools']['version']}`, launcher SHA-256 + `{tools['detools']['launcher_sha256']}` + +Required bootloader simulators: + +- OTAFIX 2.4.1 Preview 5 (`0.9.2-OTAFIX2.4.1-preview.5`, commit + `f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48`): + `c6b3c6bc0b284b0b2c7a9eef1b3ea7afeb05ce4241d81309d4ab6381a575b897` +- Current OTAFIX (commit `ffb1580d8fa59d333e1e944a198ec9b231be5a40`): + `bf57fc4b62f11d6d9ee1d8b7a863d22d3e8048ad750f6bee5ceffd7b2d4d83a7` + +For schema 2, the route plan is independently selected again from every +successful row in the complete, content-pinned `GEOMETRY.csv` table. Failed +tool jobs are not admissible evidence. The minimum-package, then minimum-byte +claim applies only to that declared firmware inventory and geometry table; it +does not claim that no other conceivable bridge firmware could improve the +route. The bundle generator rebuilds every selected package and checks it +against the old bootloader ceiling before invoking either simulator. """, encoding="ascii", ) (root / "RUNBOOK.md").write_text( - """# Runbook + f"""# Runbook Use `tools/lora_ota/rak3401_mota_chain.py` from the matching MeshCore commit. -Run `--verify-only`, then `--preflight-only`, before a live attempt. Live use -is enabled for this physically qualified exact chain. Keep the persistent work -directory so an interrupted transfer resumes from the target's exact EndF body -hash. +Run `--verify-only`, then `--preflight-only`, before a live attempt. {live_text} +Keep the persistent work directory so an interrupted transfer resumes from the +target's exact EndF body hash. Do not use `--inplace-memory 0x98000` as a manual override for these files. Each package already carries its audited workspace in the detools payload. @@ -355,8 +1168,22 @@ def main() -> int: parser.add_argument("--previous-bundle", type=Path, required=True) parser.add_argument("--endpoint-zip", type=Path, required=True) parser.add_argument("--endpoint-uf2", type=Path, required=True) - parser.add_argument("--accelerated-images", type=Path, required=True) + image_input = parser.add_mutually_exclusive_group(required=True) + image_input.add_argument( + "--accelerated-images", + type=Path, + help="legacy 28-image bridge manifest used by schema-1 routes", + ) + image_input.add_argument( + "--image-inventory", + type=Path, + help="ordered variable-node image inventory used by schema-2 routes", + ) parser.add_argument("--route-plan", type=Path, required=True) + parser.add_argument( + "--geometry-results", type=Path, + help="complete geometry CSV required by schema-2 routes", + ) parser.add_argument("--work-dir", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--motatool", type=Path, required=True) @@ -369,20 +1196,50 @@ def main() -> int: help="apply simulator and stable provenance label; repeat to validate multiple bootloaders", ) parser.add_argument("--source-commit", required=True) + parser.add_argument( + "--physical-validation-record", "--physical-validation-log", + dest="physical_validation_record", + type=Path, + help=( + "unsigned schema-1 publisher assertion binding every passed " + "package and the final SWD application hash to this exact chain" + ), + ) args = parser.parse_args() for path, label in ( (args.previous_bundle, "previous bundle"), (args.endpoint_zip, "endpoint ZIP"), (args.endpoint_uf2, "endpoint UF2"), - (args.accelerated_images, "accelerated image manifest"), (args.route_plan, "route plan"), (args.motatool, "motatool"), (args.detools, "detools"), ): if not path.is_file(): raise CompactBuildError(f"{label} does not exist: {path}") - simulators = parse_simulators(args.apply_sim) + original_image_manifest = args.image_inventory or args.accelerated_images + assert original_image_manifest is not None + if not original_image_manifest.is_file(): + raise CompactBuildError( + f"image manifest does not exist: {original_image_manifest}" + ) + if args.geometry_results is not None and not args.geometry_results.is_file(): + raise CompactBuildError( + f"geometry results do not exist: {args.geometry_results}" + ) + if args.physical_validation_record is not None and \ + not args.physical_validation_record.is_file(): + raise CompactBuildError( + "physical validation record does not exist: " + f"{args.physical_validation_record}" + ) + if ( + args.physical_validation_record is not None + and args.physical_validation_record.stat().st_size + > MAX_PHYSICAL_VALIDATION_BYTES + ): + raise CompactBuildError("physical validation record exceeds 256 KiB") + original_simulators = parse_simulators(args.apply_sim) if not re.fullmatch(r"[0-9a-f]{40}", args.source_commit): raise CompactBuildError("source commit must be a full lowercase 40-hex commit") resolved = run( @@ -393,17 +1250,103 @@ def main() -> int: raise CompactBuildError( f"endpoint source resolves to {resolved}, expected {args.source_commit}" ) + source_epoch_text = run( + [ + "git", "-C", str(REPO_ROOT), "show", "-s", "--format=%ct", + args.source_commit, + ], + "read endpoint source timestamp", + ).strip() + try: + source_epoch = int(source_epoch_text) + generated_at = datetime.fromtimestamp(source_epoch, timezone.utc).isoformat() + except (OverflowError, OSError, ValueError) as exc: + raise CompactBuildError( + f"endpoint source timestamp is invalid: {source_epoch_text!r}" + ) from exc if args.work_dir.exists(): raise CompactBuildError(f"work directory already exists: {args.work_dir}") args.work_dir.mkdir(parents=True) + # All subsequent file reads and launcher invocations use stable snapshots. + # A Python launcher can still reference its pinned pipx environment, whose + # version and launcher hash are recorded below rather than called hermetic. + # This checkout is shared by multiple agents, and a compact-chain build is + # long enough that copying provenance only at the end would be racy. + frozen = args.work_dir / "frozen-inputs" + args.previous_bundle = snapshot_file( + args.previous_bundle, frozen / "previous-bundle.zip" + ) + args.endpoint_zip = snapshot_file(args.endpoint_zip, frozen / "endpoint.zip") + args.endpoint_uf2 = snapshot_file(args.endpoint_uf2, frozen / "endpoint.uf2") + args.route_plan = snapshot_file(args.route_plan, frozen / "route.json") + args.motatool = snapshot_file(args.motatool, frozen / "tools" / "motatool") + args.detools = snapshot_file(args.detools, frozen / "tools" / "detools") + if args.geometry_results is not None: + args.geometry_results = snapshot_file( + args.geometry_results, frozen / "geometry.csv" + ) + physical_validation_document = None + if args.physical_validation_record is not None: + args.physical_validation_record = snapshot_file( + args.physical_validation_record, frozen / "physical-validation.json" + ) + physical_validation_document = read_physical_validation_record( + args.physical_validation_record + ) + frozen_simulators = [ + ( + label, + snapshot_file(path, frozen / "simulators" / label), + ) + for label, path in original_simulators + ] + simulators = parse_simulators( + [f"{label}={path}" for label, path in frozen_simulators] + ) + tools = tool_provenance(args.motatool, args.detools) + route, route_document = read_route(args.route_plan) - if ( - route_document.get("endpoint_source_commit") != args.source_commit - or route_document.get("endpoint_version") != EXPECTED_ENDPOINT_VERSION - or route_document.get("endpoint_name") != "halo-keymind-cascade-dev" - ): + route_evidence = route_document + route_schema = int(route_document["schema"]) + if route_schema == 1 and args.accelerated_images is None: + raise CompactBuildError("schema-1 routes require --accelerated-images") + if route_schema == 2 and args.image_inventory is None: + raise CompactBuildError("schema-2 routes require --image-inventory") + if route_schema == 2 and args.geometry_results is None: + raise CompactBuildError("schema-2 routes require --geometry-results") + if route_schema == 1 and args.geometry_results is not None: + raise CompactBuildError("--geometry-results is only valid for schema-2 routes") + if route_schema == 1: + assert args.accelerated_images is not None + args.accelerated_images = snapshot_schema1_manifest( + original_image_manifest, + frozen / "accelerated" / original_image_manifest.name, + ) + image_manifest = args.accelerated_images + else: + assert args.image_inventory is not None + args.image_inventory = snapshot_schema2_inventory( + original_image_manifest, + frozen / "inventory" / "images.json", + ) + image_manifest = args.image_inventory + endpoint_node = int(route_document["endpoint_node"]) + endpoint_version = route_document.get( + "endpoint_version", route[-1]["expected_target_version"] + ) + if not isinstance(endpoint_version, str): + raise CompactBuildError("route plan endpoint version is invalid") + try: + endpoint_version_packed = motalib.pack_version(endpoint_version) + except (TypeError, ValueError): + raise CompactBuildError("route plan endpoint version is invalid") from None + route_source_commit = route_document.get("endpoint_source_commit", args.source_commit) + route_endpoint_name = route_document.get("endpoint_name", "halo-keymind-cascade-dev") + if route_source_commit != args.source_commit or \ + route_endpoint_name != "halo-keymind-cascade-dev": raise CompactBuildError("route plan endpoint identity does not match this build") + expected_steps = len(route) previous_sha = common.sha256_file(args.previous_bundle) if previous_sha != PREVIOUS_BUNDLE_SHA256: raise CompactBuildError( @@ -432,34 +1375,57 @@ def main() -> int: if image1 != image1_zero or common.sha256_bytes(image1) != old_rows[0]["target_sha256"]: raise CompactBuildError("baseline step 1 does not reconstruct its pinned target") - accelerated, accelerated_sources = common.read_accelerated_images(args.accelerated_images) endpoint = common.read_firmware_zip(args.endpoint_zip) endpoint_packed, _body_hash, _version_text, _body_size = firmware_identity(endpoint) - if endpoint_packed != EXPECTED_ENDPOINT_VERSION_PACKED: + validate_uf2_firmware(args.endpoint_uf2, endpoint, "endpoint UF2") + if endpoint_packed != endpoint_version_packed: raise CompactBuildError( f"endpoint packed version is 0x{endpoint_packed:08X}, " - f"expected 0x{EXPECTED_ENDPOINT_VERSION_PACKED:08X}" + f"expected 0x{endpoint_version_packed:08X}" ) - expected_label = f"v{EXPECTED_ENDPOINT_VERSION}-halo-keymind-cascade-dev-{args.source_commit[:8]}" + expected_label = f"v{endpoint_version}-halo-keymind-cascade-dev-{args.source_commit[:8]}" if expected_label.encode("ascii") not in endpoint: raise CompactBuildError(f"endpoint firmware does not contain {expected_label}") - images = {0: image0, 1: image1} - sources = {1: old_rows[0]["source_commit"]} - for node, (image, source_commit) in enumerate(zip(accelerated, accelerated_sources), 2): - images[node] = image - sources[node] = source_commit - images[30] = endpoint - sources[30] = args.source_commit - expected_nodes = set(range(31)) - if set(images) != expected_nodes: - # Do not silently accept a shortened or reordered bridge manifest. + if route_schema == 1: + assert args.accelerated_images is not None + accelerated, accelerated_sources = common.read_accelerated_images( + args.accelerated_images + ) + images = {0: image0, 1: image1} + sources = {1: old_rows[0]["source_commit"]} + for node, (image, source_commit) in enumerate( + zip(accelerated, accelerated_sources), 2 + ): + images[node] = image + sources[node] = source_commit + images[endpoint_node] = endpoint + sources[endpoint_node] = args.source_commit + else: + assert args.image_inventory is not None + assert args.geometry_results is not None + images, sources = read_image_inventory( + args.image_inventory, endpoint_node + 1 + ) + route_evidence = validate_geometry_results( + args.geometry_results, args.image_inventory, args.route_plan + ) + if images[0] != image0: + raise CompactBuildError("inventory node 0 does not match the pinned start image") + if images[1] != image1: + raise CompactBuildError("inventory node 1 does not match the pinned first bridge") + if images[endpoint_node] != endpoint: + raise CompactBuildError("inventory endpoint does not match the endpoint ZIP") + sources[1] = old_rows[0]["source_commit"] + sources[endpoint_node] = args.source_commit + expected_nodes = set(range(endpoint_node + 1)) + if set(images) != expected_nodes or not (expected_nodes - {0}).issubset(sources): raise CompactBuildError(f"assembled image nodes are {sorted(images)}") short_commit = args.source_commit[:8] root_name = ( "RAK3401-update-chain-v1.16.7-c1caa5ad-to-" - f"v{EXPECTED_ENDPOINT_VERSION}-{short_commit}" + f"v{endpoint_version}-{short_commit}" ) root = args.work_dir / root_name motas = root / "motas" @@ -482,7 +1448,7 @@ def main() -> int: base_packed, base_body, base_version, _base_body_size = firmware_identity(base_image) target_packed, target_body, target_version, _target_body_size = firmware_identity(target_image) from_version = base_version - to_version = EXPECTED_ENDPOINT_VERSION if target_node == 30 else target_version + to_version = endpoint_version if target_node == endpoint_node else target_version filename = f"step-{number:02d}__v{from_version}-to-v{to_version}.mota" output_path = motas / filename base_path = validation_work / f"base-{number:02d}.bin" @@ -615,46 +1581,104 @@ def main() -> int: "apply_simulators": simulator_results, }) print( - f"[compact] {number:02d}/{EXPECTED_STEPS} nodes={source_node}->{target_node} " + f"[compact] {number:02d}/{expected_steps} nodes={source_node}->{target_node} " f"memory=0x{memory_size:X} size={package_size} margin={staging_margin}", flush=True, ) - write_chain(root / "CHAIN.csv", output_rows) + chain_path = root / "CHAIN.csv" + write_chain(chain_path, output_rows) + chain_sha = common.sha256_file(chain_path) + physical_validation = None + physical_validation_path = root / "PHYSICAL-VALIDATION.json" + if physical_validation_document is not None: + _endpoint_packed, endpoint_body_hash, _endpoint_text, _endpoint_body_size = ( + firmware_identity(endpoint) + ) + physical_validation = validate_physical_validation_record( + physical_validation_document, + chain_sha256=chain_sha, + start_sha256=common.sha256_bytes(image0), + endpoint_sha256=common.sha256_bytes(endpoint), + endpoint_body_hash=endpoint_body_hash, + endpoint_version=endpoint_version, + output_rows=output_rows, + validation_steps=validation_steps, + ) + physical_validation_path.write_text( + json.dumps(physical_validation, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) shutil.copy2(start_zip, recovery_start) start_uf2 = old_root / "recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.uf2" shutil.copy2(start_uf2, recovery_start) shutil.copy2(args.endpoint_zip, recovery_final) shutil.copy2(args.endpoint_uf2, recovery_final) - shutil.copy2(args.accelerated_images, root / "BRIDGE-IMAGES.json") - shutil.copy2(args.route_plan, root / "ROUTE.json") + manifest_name = ( + "BRIDGE-IMAGES.json" if route_schema == 1 else "IMAGE-INVENTORY.json" + ) + bundled_manifest = root / manifest_name + if route_schema == 1: + write_accelerated_provenance(image_manifest, bundled_manifest) + else: + write_inventory_provenance(image_manifest, bundled_manifest) + bundled_route = root / "ROUTE.json" + write_route_provenance( + bundled_route, route_schema, route, route_evidence, + args.source_commit, endpoint_version, + ) + builder_source_copy = root / "build_rak3401_compact_bundle.py" + builder_source_copy.write_bytes(EXECUTING_BUILDER_SOURCE) + builder_source_sha = common.sha256_file(builder_source_copy) + geometry_sha = None + route_search_sha = None + if route_schema == 2: + assert args.geometry_results is not None + geometry_copy = root / "GEOMETRY.csv" + route_search_copy = root / "rak3401_route_search.py" + shutil.copy2(args.geometry_results, geometry_copy) + route_search_copy.write_bytes(EXECUTING_ROUTE_SEARCH_SOURCE) + geometry_sha = common.sha256_file(geometry_copy) + route_search_sha = common.sha256_file(route_search_copy) baseline = old_root / "PHYSICAL-BASELINE-cd824765.jsonl" if baseline.is_file(): shutil.copy2(baseline, root) - - route_sha = common.sha256_file(args.route_plan) - bridge_sha = common.sha256_file(args.accelerated_images) + route_sha = common.sha256_file(bundled_route) + bridge_sha = common.sha256_file(bundled_manifest) write_docs( - root, args.source_commit, endpoint, previous_sha, bridge_sha, - route_sha, output_rows, + root, args.source_commit, endpoint_version, endpoint, previous_sha, + bridge_sha, route_sha, geometry_sha, route_search_sha, + builder_source_sha, tools, output_rows, + physical_validation is not None, ) validation = { - "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at": generated_at, "generator": "tools/lora_ota/build_rak3401_compact_bundle.py", "source_commit": args.source_commit, - "endpoint_requested_version": EXPECTED_ENDPOINT_VERSION, - "endpoint_packed_version": EXPECTED_ENDPOINT_VERSION_PACKED, + "endpoint_requested_version": endpoint_version, + "endpoint_packed_version": endpoint_version_packed, "endpoint_name": "halo-keymind-cascade-dev", "previous_bundle_sha256": previous_sha, "bridge_images_sha256": bridge_sha, "route_plan_sha256": route_sha, - "route_search": route_document.get("search"), + "route_search": public_route_search(route_schema, route_evidence), + "required_simulator_provenance": REQUIRED_SIMULATORS, + "bundle_builder_source_sha256": builder_source_sha, + "tool_provenance": tools, "app_base": APP_BASE, "stage_ceiling": STAGE_CEILING, "segment_size": SEGMENT_SIZE, "block_size": BLOCK_SIZE, "steps": validation_steps, } + if route_schema == 2: + validation["geometry_results_sha256"] = geometry_sha + validation["route_search_source_sha256"] = route_search_sha + validation["image_inventory_sha256"] = bridge_sha + if physical_validation is not None: + validation["physical_validation_record_sha256"] = common.sha256_file( + physical_validation_path + ) (root / "validation-results.json").write_text( json.dumps(validation, indent=2, sort_keys=True) + "\n", encoding="ascii", @@ -662,7 +1686,7 @@ def main() -> int: checksum_sha = common.write_checksums(root) args.output_dir.mkdir(parents=True, exist_ok=True) output_path = args.output_dir / f"{root_name}.zip" - common.make_zip(root, output_path) + make_reproducible_zip(root, output_path) print(f"[bundle] {output_path}") print(f"[bundle] sha256={common.sha256_file(output_path)}") print(f"[bundle] inner_sha256={checksum_sha}") diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index 79833b79..2cb74b93 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -28,6 +28,7 @@ import shlex import shutil import signal import socket +import stat import struct import subprocess import sys @@ -62,12 +63,21 @@ TRANSMISSION_RETRY_WINDOW_SECONDS = 90 TRANSMISSION_RETRY_DELAY_SECONDS = 2 TRANSMISSION_RETRY_MAX_DELAY_SECONDS = 8 TRANSMISSION_PROMPT_SECONDS = 10 +SOURCE_RXPS_BUSY_RETRY_LIMIT = 32 ADAPTIVE_POLL_MAX_FACTOR = 3 TEMP_RADIO_SWITCH_DELAY_SECONDS = 3 TEMP_RADIO_RETURN_MINUTES = 1 TEMP_RADIO_RETURN_MARGIN_SECONDS = 15 +SHARED_SOURCE_NORMAL_TIMEOUT_SECONDS = 30 +SHARED_SOURCE_NORMAL_POLL_SECONDS = 1 INSTALL_TARGET_WINDOW_MINUTES = 3 -DEFAULT_POST_INSTALL_READY_WAIT_SECONDS = 20 +# Internal-flash nRF52 installs can spend minutes applying a large in-place +# patch before USB and LoRa are available again. The RAK3401 hardware run +# measured a 106-KiB package taking 64 seconds from USB disconnect to +# re-enumeration, while later chain packages are substantially larger. Keep +# the readiness probe bounded at five minutes; it still returns immediately +# when the exact installed identity replies. +DEFAULT_POST_INSTALL_READY_WAIT_SECONDS = 300 POST_INSTALL_READY_PROBE_INTERVAL_SECONDS = 10 COMPANION_TERMINAL_START = "+++MESHCORE-TERM-START" COMPANION_TERMINAL_STOP = "+++MESHCORE-TERM-STOP" @@ -78,6 +88,7 @@ DEFAULT_RELAY_TX_DELAY = 0.3 RELAY_TIMING_COMMANDS_PER_RELAY = 12 RELAY_TIMING_RECOVERY_FILE = "relay-timing-settings.json" TARGET_RXPS_RECOVERY_FILE = "target-rxps-settings.json" +SOURCE_RXPS_RECOVERY_FILE = "source-rxps-settings.json" MIN_MESHCLI_VERSION = (1, 6, 0) # v1.17.1.5 is the first release-version contract in which every packet, # including retries, uses the same tuple-selected physical preamble: normally @@ -1021,19 +1032,19 @@ def redact_text(value: str, sensitive_values: tuple[str, ...] = ()) -> str: def _debug_text( - value: str | bytes | None, + value: str | bytes | bytearray | memoryview | None, sensitive_values: tuple[str, ...] = (), ) -> str: if value is None: return "" - if isinstance(value, bytes): - value = value.decode("utf-8", "replace") + if not isinstance(value, str): + value = bytes(value).decode("utf-8", "replace") return redact_text(value, sensitive_values).rstrip() or "" def debug_stream( label: str, - value: str | bytes | None, + value: str | bytes | bytearray | memoryview | None, sensitive_values: tuple[str, ...] = (), ) -> None: if not DEBUG: @@ -1299,7 +1310,16 @@ def reply_matches_command(command_text: str, reply: str) -> bool: ) ) if command == "ota cancel": - return lowered.startswith("ok dropped ") or is_unknown or needs_temp + # Current persistent stores can report that the manager/RAM session + # was dropped but durable media invalidation failed. Treat that as the + # command's reply so callers surface the real rejection instead of + # waiting through reply-timeout retries for an impossible OK. + return ( + lowered.startswith("ok dropped ") + or (is_error and "persistent ota slot" in lowered) + or is_unknown + or needs_temp + ) if command == "ota install": return ( lowered.startswith(("ok |", "err |")) @@ -2134,6 +2154,15 @@ def preflight_source_cli(args: argparse.Namespace) -> None: ) +def has_managed_source_cli(args: argparse.Namespace) -> bool: + """Return whether this run can safely inspect and restore its OTA source.""" + return bool( + getattr(args, "source_serial", None) + or getattr(args, "source_cli_serial", None) + or getattr(args, "source_cli_tcp", None) + ) + + def read_lora_ota_participant_versions( controller: Controller, args: argparse.Namespace, @@ -2463,6 +2492,300 @@ def read_remote_rxps( ) from detailed_error +def read_source_rxps(args: argparse.Namespace) -> RxpsSettings: + """Read the managed source's persisted RXPS preference exactly.""" + try: + reply = source_cli_command(args, "get radio.rxps.config") + return parse_rxps_settings(reply, "OTA source") + except TransmissionStopped: + raise + except OtaError as detailed_error: + # Older full-parser sources expose only the fixed receive/sleep + # periods. They are still sufficient for an exact legacy restore. + try: + reply = source_cli_command(args, "get radio.rxps") + return parse_rxps_settings(reply, "OTA source") + except TransmissionStopped: + raise + except OtaError as legacy_error: + raise OtaError( + "could not read OTA source RXPS state using current or " + f"legacy CLI: {legacy_error}" + ) from detailed_error + + +def source_rxps_busy_retry_delay(retry_number: int) -> float: + """Return a unique short cadence for each retry in the bounded window.""" + # 73 and 173 are coprime, so none of the 32 delays repeat. Their sum is + # 9.366 seconds, and the varying 210-378 ms spacing avoids phase-locking + # retries to a periodic RX/TX scheduler slot. + return (210 + (((retry_number - 1) * 73) % 173)) / 1000 + + +def mutate_source_rxps(args: argparse.Namespace, command: str) -> str: + """Run one idempotent source RXPS mutation across short busy phases.""" + busy_retries = 0 + while True: + try: + return source_cli_command(args, command) + except TransmissionStopped: + raise + except OtaError as exc: + # The Full Companion emits this exact response only when the + # driver rejected the change before updating or saving the RXPS + # preference. It is therefore safe to replay an idempotent RXPS + # mutation, unlike an arbitrary rejected or reply-lost command. + busy = re.search( + r"\bradio busy;\s*retry\b", str(exc), re.IGNORECASE + ) + if busy is None: + raise + if busy_retries >= SOURCE_RXPS_BUSY_RETRY_LIMIT: + raise OtaError( + "OTA source remained radio busy after " + f"{SOURCE_RXPS_BUSY_RETRY_LIMIT} bounded RXPS retries " + "and about 9.4 seconds of retry waits (command round-trip " + "time is additional)" + ) from exc + busy_retries += 1 + delay = source_rxps_busy_retry_delay(busy_retries) + print( + "[rxps] OTA source radio busy; retrying RXPS mutation in " + f"{delay:.2f}s ({busy_retries}/{SOURCE_RXPS_BUSY_RETRY_LIMIT})" + ) + time.sleep(delay) + + +def disable_source_rxps( + args: argparse.Namespace, + saved: RxpsSettings, +) -> bool: + """Disable RXPS for a managed source and prove continuous receive mode.""" + if not saved.enabled: + print("[rxps] OTA source was already off; leaving it unchanged") + return False + + reply = mutate_source_rxps(args, "set radio.rxps off") + if re.search(r"\boff\b", reply, re.IGNORECASE) is None: + raise OtaError(f"OTA source did not confirm RXPS off: {reply}") + verified = read_source_rxps(args) + if verified.enabled: + raise OtaError("OTA source RXPS did not read back as off") + print("[rxps] OTA source temporarily off for TempRadio transfer") + return True + + +def rxps_restore_command(saved: RxpsSettings) -> str: + if saved.level is not None and 1 <= saved.level <= 10: + if saved.preamble in (16, 32): + return ( + f"set radio.rxps level {saved.level} " + f"preamble {saved.preamble}" + ) + return f"set radio.rxps level {saved.level}" + return f"set radio.rxps {saved.rx_us} {saved.sleep_us}" + + +def restore_source_rxps( + args: argparse.Namespace, + saved: RxpsSettings, +) -> None: + """Restore and verify a managed source's exact saved RXPS preference.""" + current = read_source_rxps(args) + if saved.enabled: + if current != saved: + reply = mutate_source_rxps(args, rxps_restore_command(saved)) + if re.search(r"\bon\b", reply, re.IGNORECASE) is None: + raise OtaError(f"OTA source did not restore RXPS: {reply}") + elif current.enabled: + reply = mutate_source_rxps(args, "set radio.rxps off") + if re.search(r"\boff\b", reply, re.IGNORECASE) is None: + raise OtaError(f"OTA source did not restore RXPS-off: {reply}") + + verified = read_source_rxps(args) + if verified != saved: + raise OtaError("OTA source RXPS settings did not restore exactly") + print("[rxps] OTA source original RXPS settings restored") + + +def source_rxps_connection(args: argparse.Namespace) -> dict[str, object]: + """Describe the managed CLI without retaining an admin credential.""" + serial_port = getattr(args, "source_cli_serial", None) or getattr( + args, "source_serial", None + ) + tcp_console = getattr(args, "source_cli_tcp", None) + if serial_port: + return { + "kind": "serial", + "endpoint": str(serial_port), + "baud": getattr(args, "source_baud", None), + } + if tcp_console: + return { + "kind": "tcp-console", + "endpoint": str(tcp_console), + } + raise OtaError("cannot preserve source RXPS without a managed source CLI") + + +def source_rxps_recovery_payload( + args: argparse.Namespace, + saved: RxpsSettings, +) -> dict[str, object]: + return { + "connection": source_rxps_connection(args), + "rxps_enabled": saved.enabled, + "rxps_rx_us": saved.rx_us, + "rxps_sleep_us": saved.sleep_us, + "rxps_level": saved.level, + "rxps_preamble": saved.preamble, + "restore_command": ( + rxps_restore_command(saved) + if saved.enabled else "set radio.rxps off" + ), + } + + +def fsync_parent_directory(path: Path) -> None: + """Make a preceding replace/unlink durable on filesystems that support it.""" + if os.name == "nt": + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + try: + directory_fd = os.open(path.parent, flags) + except OSError as exc: + raise OtaError( + f"cannot open recovery directory for a durable update: {path.parent}: {exc}" + ) from exc + try: + os.fsync(directory_fd) + except OSError as exc: + raise OtaError( + f"cannot flush recovery directory update: {path.parent}: {exc}" + ) from exc + finally: + os.close(directory_fd) + + +def write_private_recovery_file(path: Path, contents: str) -> Path: + """Atomically install a private recovery file after flushing its contents.""" + if path.is_symlink(): + raise OtaError(f"recovery path is a symbolic link: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + temporary_fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(temporary_fd, "w", encoding="ascii", newline="\n") as output: + output.write(contents) + output.flush() + os.fsync(output.fileno()) + temporary.chmod(0o600) + os.replace(temporary, path) + fsync_parent_directory(path) + except BaseException: + try: + os.close(temporary_fd) + except OSError: + pass + try: + temporary.unlink() + except OSError: + pass + raise + return path + + +def retire_private_recovery_file(path: Path, label: str) -> None: + """Atomically remove an active private record after exact restoration.""" + if not path.exists() and not path.is_symlink(): + return + if path.is_symlink(): + raise OtaError(f"{label} path is a symbolic link: {path}") + retired = path.with_name( + f".{path.name}.restored-{os.getpid()}-{secrets.token_hex(4)}" + ) + try: + os.replace(path, retired) + fsync_parent_directory(path) + retired.unlink() + fsync_parent_directory(path) + except OSError as exc: + raise OtaError(f"cannot retire {label} {path}: {exc}") from exc + + +def retire_source_rxps_recovery(path: Path) -> None: + """Atomically remove an active source RXPS record after restoration.""" + retire_private_recovery_file(path, "source RXPS recovery record") + + +def write_source_rxps_recovery( + work_dir: Path, + args: argparse.Namespace, + saved: RxpsSettings, + *, + recovery_path: Path | None = None, +) -> Path: + """Persist a non-secret, exact source RXPS restore record atomically.""" + path = recovery_path or (work_dir / SOURCE_RXPS_RECOVERY_FILE) + if path.is_symlink(): + raise OtaError(f"source RXPS recovery path is a symbolic link: {path}") + path = path.resolve() + payload = source_rxps_recovery_payload(args, saved) + return write_private_recovery_file( + path, + json.dumps(payload, indent=2, sort_keys=True) + "\n", + ) + + +def read_source_rxps_recovery( + path: Path, + args: argparse.Namespace, +) -> RxpsSettings: + """Load an exact saved preference only for the same managed source.""" + if path.is_symlink(): + raise OtaError(f"source RXPS recovery path is a symbolic link: {path}") + path = path.resolve() + if not path.is_file(): + raise OtaError(f"source RXPS recovery file is unavailable: {path}") + if stat.S_IMODE(path.stat().st_mode) & 0o077: + raise OtaError(f"source RXPS recovery file is not private (0600): {path}") + try: + payload = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise OtaError(f"cannot read source RXPS recovery file {path}: {exc}") from exc + if not isinstance(payload, dict): + raise OtaError(f"invalid source RXPS recovery record: {path}") + if payload.get("connection") != source_rxps_connection(args): + raise OtaError( + "source RXPS recovery record belongs to a different CLI endpoint: " + f"{path}" + ) + enabled = payload.get("rxps_enabled") + rx_us = payload.get("rxps_rx_us") + sleep_us = payload.get("rxps_sleep_us") + level = payload.get("rxps_level") + preamble = payload.get("rxps_preamble") + if ( + type(enabled) is not bool + or type(rx_us) is not int + or type(sleep_us) is not int + or not RXPS_MIN_PERIOD_US <= rx_us <= RXPS_MAX_PERIOD_US + or not RXPS_MIN_PERIOD_US <= sleep_us <= RXPS_MAX_PERIOD_US + or (level is not None and (type(level) is not int or not 0 <= level <= 10)) + or (preamble is not None and preamble not in (0, 16, 32)) + ): + raise OtaError(f"invalid source RXPS values in recovery record: {path}") + saved = RxpsSettings(enabled, rx_us, sleep_us, level, preamble) + expected_command = ( + rxps_restore_command(saved) if saved.enabled else "set radio.rxps off" + ) + if payload.get("restore_command") != expected_command: + raise OtaError(f"source RXPS recovery command is inconsistent: {path}") + return saved + + def write_target_rxps_recovery( work_dir: Path, target_name: str, @@ -2487,14 +2810,10 @@ def write_target_rxps_recovery( for label, version in participant_versions.items() }, } - temporary = path.with_suffix(path.suffix + ".tmp") - temporary.write_text( + return write_private_recovery_file( + path, json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="ascii", ) - temporary.chmod(0o600) - os.replace(temporary, path) - return path def apply_remote_rxps_policy( @@ -2585,16 +2904,7 @@ def restore_remote_rxps( ) if saved.enabled: if current != saved: - if saved.level is not None and 1 <= saved.level <= 10: - if saved.preamble in (16, 32): - command = ( - f"set radio.rxps level {saved.level} " - f"preamble {saved.preamble}" - ) - else: - command = f"set radio.rxps level {saved.level}" - else: - command = f"set radio.rxps {saved.rx_us} {saved.sleep_us}" + command = rxps_restore_command(saved) reply = controller.remote_command( target_name, command, @@ -2674,14 +2984,10 @@ def write_relay_timing_recovery( } for item in settings ] - temporary = path.with_suffix(path.suffix + ".tmp") - temporary.write_text( + return write_private_recovery_file( + path, json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="ascii", ) - temporary.chmod(0o600) - os.replace(temporary, path) - return path def enforce_relay_timing( @@ -2868,6 +3174,73 @@ def remote_command_with_seeder( return retry_transmission(run_once, f"{command!r} on {target}") +def wait_for_completed_manifest_verification( + controller: Controller, + args: argparse.Namespace, + manifest_id: str, + status: str, + seeder: SeederProcess | None, +) -> str: + """Bound a retained store's transient boot-time verification state.""" + if "verifying staged blocks" not in status.lower(): + return status + + deadline = time.monotonic() + args.discovery_timeout + last_status = status + while True: + ensure_seeder_running( + seeder, "while waiting for a completed previous manifest" + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise OtaError( + f"timed out waiting for previous mOTA {manifest_id} staged " + f"block verification: {last_status}" + ) + time.sleep(min(float(args.discovery_interval), remaining)) + ensure_seeder_running( + seeder, "while waiting for a completed previous manifest" + ) + try: + last_status = controller.remote_command( + args.target, "ota status", retry=False + ) + except TransmissionError as exc: + print( + "[download] previous staged-block verification status reply " + f"was lost; retrying within the discovery window: {exc}" + ) + continue + + lowered = last_status.lower() + active_id = download_manifest_id(last_status) + if active_id is not None and active_id != manifest_id: + raise OtaError( + f"previous mOTA changed during staged-block verification: " + f"expected {manifest_id}, got {active_id}; " + f"status: {last_status}" + ) + if "no download" in lowered: + return last_status + if active_id is None: + raise OtaError( + f"previous mOTA changed during staged-block verification: " + f"expected {manifest_id}, got unknown; status: {last_status}" + ) + if "download: failed" in lowered: + raise OtaError( + f"previous mOTA {manifest_id} failed staged-block " + f"verification: {last_status}" + ) + if "ready to install" in lowered: + return last_status + if "verifying staged blocks" not in lowered: + raise OtaError( + f"previous mOTA {manifest_id} became incomplete while " + f"verifying staged blocks: {last_status}" + ) + + def find_and_start_pull( controller: Controller, args: argparse.Namespace, @@ -2891,10 +3264,14 @@ def find_and_start_pull( ) elif active_id == getattr(args, "clear_completed_manifest", None): expected_hash = args.clear_completed_on_body_hash - if "ready to install" not in status.lower(): + status = wait_for_completed_manifest_verification( + controller, args, active_id, status, seeder + ) + manager_is_idle = "no download" in status.lower() + if not manager_is_idle and "ready to install" not in status.lower(): raise OtaError( - f"refusing to clear previous mOTA {active_id} because it is " - f"not complete: {status}" + f"refusing to clear previous mOTA {active_id} because " + f"it is not complete: {status}" ) identity = remote_command_with_seeder( controller, args.target, "ota self", seeder, @@ -2909,36 +3286,46 @@ def find_and_start_pull( f"refusing to clear previous mOTA {active_id}: running body " f"hash is {running_hash or 'unknown'}, expected {expected_hash}" ) - try: - cancel_reply = controller.remote_command( - args.target, "ota cancel", retry=False + if manager_is_idle: + print( + f"[download] previous session {active_id} became idle after " + f"proving running body {expected_hash}; persistent erasure " + "is not inferred and no IDLE cancel was sent" ) - except TransmissionError as cancel_error: - resolved = remote_command_with_seeder( - controller, args.target, "ota status", seeder, - "while resolving a lost completed-manifest cancel reply", - ) - if "no download" not in resolved.lower(): - raise OtaError( - f"completed-manifest cancel outcome is ambiguous: {resolved}" - ) from cancel_error else: - if not cancel_reply.startswith("OK"): - raise OtaError( - f"could not clear completed previous mOTA: {cancel_reply}" + try: + cancel_reply = controller.remote_command( + args.target, "ota cancel", retry=False ) - status = remote_command_with_seeder( - controller, args.target, "ota status", seeder, - "after clearing a completed previous manifest", - ) - if "no download" not in status.lower(): - raise OtaError( - f"completed previous mOTA {active_id} remains: {status}" + except TransmissionError as cancel_error: + resolved = remote_command_with_seeder( + controller, args.target, "ota status", seeder, + "while resolving a lost completed-manifest cancel reply", + ) + if "no download" not in resolved.lower(): + raise OtaError( + "completed-manifest cancel outcome is ambiguous: " + f"{resolved}" + ) from cancel_error + else: + if not cancel_reply.startswith("OK"): + raise OtaError( + "could not clear completed previous mOTA: " + f"{cancel_reply}" + ) + status = remote_command_with_seeder( + controller, args.target, "ota status", seeder, + "after clearing a completed previous manifest", + ) + if "no download" not in status.lower(): + raise OtaError( + f"completed previous mOTA {active_id} remains: {status}" + ) + print( + f"[download] detached completed previous session {active_id} " + f"after proving running body {expected_hash}; persistent " + "erasure is not inferred" ) - print( - f"[download] cleared completed previous session {active_id} " - f"after proving running body {expected_hash}" - ) elif not args.replace_active_download: raise OtaError( f"destination already has mOTA {active_id} staged or downloading; " @@ -2956,9 +3343,23 @@ def find_and_start_pull( deadline = time.monotonic() + args.discovery_timeout last_reply = "" while time.monotonic() < deadline: - last_reply = remote_command_with_seeder( - controller, args.target, "ota ls", seeder, "during discovery" - ) + ensure_seeder_running(seeder, "during discovery") + try: + last_reply = controller.remote_command( + args.target, "ota ls", retry=False + ) + except TransmissionError as exc: + # `ota ls` is a discovery broadcast followed by an ordinary admin + # reply. On a busy half-duplex TempRadio channel that reply can be + # lost even though discovery was sent. Do not rebroadcast it in the + # generic retry loop; the exact pull below is safe, and its existing + # status reconciliation proves whether a lost pull reply started the + # requested manifest. + last_reply = f"reply lost: {exc}" + print( + "[download] `ota ls` reply was lost; attempting the exact " + f"manifest {package.manifest_id}" + ) def attempt_pull() -> bool: try: @@ -3140,9 +3541,16 @@ def arm_target_temp_radio( "[destination] TempRadio reply was lost; probing the declared " "temporary channel" ) - controller.set_radio(temp_radio, "probe destination TempRadio state") + shared_controller = bool( + getattr(args, "source_shares_controller", False) + ) found_on_temp = False try: + switch_controller_to_temp_radio( + controller, args, command, temp_radio + ) + if shared_controller: + time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) identity = controller.remote_command( args.target, "ota self", retry=False ) @@ -3155,9 +3563,20 @@ def arm_target_temp_radio( except (OtaError, TransmissionError): pass finally: - controller.set_radio( - normal_radio, "restore controller after TempRadio probe" - ) + # Always reassert the saved Binary tuple, even if ending the + # shared source's local live override fails. The caller owns a + # conservative source/target cleanup flag before entering this + # helper, so either failure is retried by outer cleanup. + try: + if shared_controller: + # The shared Full Companion entered TempRadio through + # its bounded local command. Leave it through that same + # path without ever persisting the temporary tuple. + shorten_source_temp_window(args) + finally: + controller.set_radio( + normal_radio, "restore controller after TempRadio probe" + ) if found_on_temp: print( "[destination] resolved lost TempRadio reply from the exact " @@ -3165,7 +3584,7 @@ def arm_target_temp_radio( ) return - # The 2-second scheduled handoff is long past by the time the + # The 1.5-second scheduled handoff is long past by the time the # temporary-channel probe times out. An exact identity reply back # on the normal channel therefore proves that the target did not # remain on TempRadio, making a bounded replay safe. @@ -3350,18 +3769,64 @@ def shorten_source_temp_window( ) -> bool: if args.source_already_temp: return True - if getattr(args, "source_shares_controller", False): - print("[source] ending shared Full Companion TempRadio before restore") - output = source_cli_command(args, "normalradio", check=check) - if not output and not check: - print( - "[warn] could not end the shared source TempRadio window; " - "end it with `normalradio` before restoring the controller", - file=sys.stderr, + shares_controller = getattr(args, "source_shares_controller", False) + label = "shared Full Companion" if shares_controller else "managed OTA source" + print(f"[source] ending {label} TempRadio before restore") + + # Current full-parser sources can cancel the lease immediately. For a + # separate legacy source, a rejected `normalradio` falls back to its + # bounded one-minute lease below. A shared source cannot use that fallback: + # its normal tuple must be proven before the Binary controller is restored. + output = source_cli_command( + args, + "normalradio", + check=check if shares_controller else False, + ) + if output: + deadline = time.monotonic() + SHARED_SOURCE_NORMAL_TIMEOUT_SECONDS + while True: + status = source_cli_command( + args, + "tempradio", + check=check if shares_controller else False, ) - return False - time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) - return True + lowered = status.strip().lower() + if lowered.startswith("tempradio inactive"): + print(f"[source] {label} TempRadio is inactive") + return True + if not lowered.startswith(("tempradio active:", "tempradio pending:")): + if not shares_controller: + break + error = OtaError( + "shared Full Companion returned an unexpected TempRadio " + f"status after normalradio: {status or 'no output'}" + ) + if check: + raise error + print(f"[warn] {error}", file=sys.stderr) + return False + remaining = deadline - time.monotonic() + if remaining <= 0: + error = OtaError( + f"{label} did not leave TempRadio within " + f"{SHARED_SOURCE_NORMAL_TIMEOUT_SECONDS} seconds" + ) + if check: + raise error + print(f"[warn] {error}", file=sys.stderr) + return False + time.sleep(min(SHARED_SOURCE_NORMAL_POLL_SECONDS, remaining)) + elif shares_controller: + print( + "[warn] could not end the shared source TempRadio window; " + "end it with `normalradio` before restoring the controller", + file=sys.stderr, + ) + return False + + # Compatibility for older separate source CLIs without `normalradio` or a + # TempRadio status query. Re-arm the same tuple for one minute and wait out + # the complete bounded lease before reporting that RXPS may be restored. command = temp_radio_command_for_minutes(args, TEMP_RADIO_RETURN_MINUTES) print( f"[source] scheduling return to the normal channel in " @@ -3375,9 +3840,29 @@ def shorten_source_temp_window( file=sys.stderr, ) return False + time.sleep( + TEMP_RADIO_RETURN_MINUTES * 60 + TEMP_RADIO_RETURN_MARGIN_SECONDS + ) + print("[source] managed OTA source bounded TempRadio lease expired") return True +def switch_controller_to_temp_radio( + controller: Controller, + args: argparse.Namespace, + temp_command: str, + temp_radio: RadioSettings, +) -> None: + """Enter TempRadio without overwriting a shared Companion's saved tuple.""" + if getattr(args, "source_shares_controller", False): + # This local command changes only the bounded live radio tuple. A Binary + # `set radio` would persist the temporary tuple, leaving `normalradio` + # with no saved normal tuple to restore. + source_cli_command(args, temp_command) + return + controller.set_radio(temp_radio, "switch controller to TempRadio") + + def verify_installed( controller: Controller, args: argparse.Namespace, @@ -3606,6 +4091,11 @@ def build_parser() -> argparse.ArgumentParser: help="destination platform for --prepare-only (detected during a live run)", ) parser.add_argument("--work-dir", type=Path) + parser.add_argument( + "--source-rxps-recovery-file", + type=Path, + help=argparse.SUPPRESS, + ) parser.add_argument("--meshcli", default="meshcli") parser.add_argument("--motatool", default="motatool") parser.add_argument( @@ -3640,7 +4130,8 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help=( "TCP source is the controller's own Full Companion; verify its " - "port-5000 identity and let the controller radio switch move both" + "port-5000 identity and use its local CLI for the bounded live " + "TempRadio override without persisting that tuple through Binary" ), ) parser.add_argument( @@ -3701,6 +4192,19 @@ def build_parser() -> argparse.ArgumentParser: return parser +def serial_paths_match(first: str, second: str) -> bool: + """Recognize two names for one serial endpoint, including /dev symlinks.""" + try: + if os.path.samefile(first, second): + return True + except OSError: + # One or both paths may not exist yet (common in dry-run tests and after + # a USB reset), but resolving existing symlink components still catches + # stable by-id aliases of the same eventual device. + pass + return Path(first).resolve(strict=False) == Path(second).resolve(strict=False) + + def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: try: args.temp_values = parse_temp_radio(args.temp_radio) @@ -3773,6 +4277,14 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> parser.error( "--source-tcp also needs --source-cli-serial, --source-cli-tcp, or --source-already-temp" ) + if args.source_already_temp and not args.source_tcp: + parser.error("--source-already-temp requires --source-tcp") + if args.source_already_temp and ( + args.source_cli_serial or args.source_cli_tcp + ): + parser.error( + "--source-already-temp cannot be combined with a managed source CLI" + ) if args.source_shares_controller and not ( args.source_tcp and args.source_cli_tcp ): @@ -3783,11 +4295,17 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> parser.error( "--source-shares-controller and --source-already-temp are mutually exclusive" ) + if args.source_shares_controller and args.leave_controller_radio: + parser.error( + "--leave-controller-radio cannot be combined with a shared " + "managed source because source RXPS must be restored only " + "after its normal radio is verified" + ) if args.controller_serial and args.source_serial: - if os.path.abspath(args.controller_serial) == os.path.abspath(args.source_serial): + if serial_paths_match(args.controller_serial, args.source_serial): parser.error("controller and source must be separate nodes/serial ports") if args.controller_serial and args.source_cli_serial: - if os.path.abspath(args.controller_serial) == os.path.abspath(args.source_cli_serial): + if serial_paths_match(args.controller_serial, args.source_cli_serial): parser.error("controller and source CLI must use separate serial ports") unsafe_text = { "TARGET_NODE": args.target, @@ -3889,6 +4407,23 @@ def make_work_dir(requested: Path | None) -> Path: return path +def validate_source_recovery_location( + recovery_path: Path | None, work_dir: Path +) -> Path | None: + """Keep caller-owned chain recovery state outside per-attempt artifacts.""" + if recovery_path is None: + return None + resolved_recovery = recovery_path.resolve(strict=False) + resolved_work = work_dir.resolve(strict=True) + if resolved_recovery == resolved_work or resolved_recovery.is_relative_to( + resolved_work + ): + raise OtaError( + "--source-rxps-recovery-file must be outside the run work directory" + ) + return resolved_recovery + + def offline_target(args: argparse.Namespace) -> TargetInfo: target_id_text = args.target_id.removeprefix("0x").removeprefix("0X") if not re.fullmatch(r"[0-9A-Fa-f]{8}", target_id_text): @@ -3932,12 +4467,49 @@ def main( seeder: SeederProcess | None = None seeder_attempted = False source_temp_owned = False + source_rxps_saved: RxpsSettings | None = None + source_rxps_changed = False + source_rxps_recovery_path: Path | None = None target_temp_owned = False target_rxps_saved: RxpsSettings | None = None target_rxps_changed = False armed_relay_values: list[tuple[str, str]] = [] relay_timing_settings: list[RelayTimingSettings] = [] password = args.password or os.environ.get("MESHCORE_ADMIN_PASSWORD", "") + temp_command = f"tempradio {args.temp_radio}" + + def restore_source_rxps_once(context: str) -> None: + """Restore the source only after its ordinary radio is proven active.""" + nonlocal source_rxps_changed + if not source_rxps_changed or source_rxps_saved is None: + return + if source_temp_owned: + raise OtaError( + f"cannot restore OTA source RXPS during {context}: its " + "TempRadio state is still active or uncertain" + ) + if args.source_shares_controller and controller_changed: + raise OtaError( + f"cannot restore OTA source RXPS during {context}: the " + "shared controller has not returned to its normal radio" + ) + # Keep this flag armed until exact readback succeeds. A transient + # restore failure remains retryable from the outer finally block; the + # restore mutation itself is deliberately idempotent. + restore_source_rxps(args, source_rxps_saved) + source_rxps_changed = False + + def retire_local_source_rxps_recovery() -> None: + """A caller-supplied chain record remains owned by that caller.""" + nonlocal source_rxps_recovery_path + if ( + source_rxps_recovery_path is None + or args.source_rxps_recovery_file is not None + ): + return + retire_source_rxps_recovery(source_rxps_recovery_path) + source_rxps_recovery_path = None + try: preflight_inputs(args) if not args.prepare_only and not password: @@ -3949,8 +4521,61 @@ def main( if any(char in password for char in "\r\n\0"): raise OtaError("admin password contains an unsupported control character") args.relay_values = [parse_relay(value, password) for value in args.relay] + # Retain recovery state before any source mutation. A hard kill after + # RXPS is disabled must not erase the only copy of its saved setting. + work_dir = make_work_dir(args.work_dir) + args.source_rxps_recovery_file = validate_source_recovery_location( + args.source_rxps_recovery_file, work_dir + ) if not args.prepare_only: preflight_source_cli(args) + if has_managed_source_cli(args): + try: + current_source_rxps = read_source_rxps(args) + except TransmissionStopped: + raise + except OtaError as exc: + raise OtaError( + "cannot safely preserve or disable OTA source RXPS " + f"because its current state is unavailable: {exc}" + ) from exc + source_rxps_recovery_path = ( + args.source_rxps_recovery_file + if args.source_rxps_recovery_file is not None + else work_dir / SOURCE_RXPS_RECOVERY_FILE + ) + if source_rxps_recovery_path.exists(): + source_rxps_saved = read_source_rxps_recovery( + source_rxps_recovery_path, args + ) + print( + "[rxps] loaded original source recovery settings: " + f"{source_rxps_recovery_path}" + ) + else: + source_rxps_saved = current_source_rxps + source_rxps_recovery_path = write_source_rxps_recovery( + work_dir, + args, + source_rxps_saved, + recovery_path=source_rxps_recovery_path, + ) + print( + "[rxps] source recovery settings: " + f"{source_rxps_recovery_path}" + ) + # Arm cleanup before the mutating command. The command may + # reach the source even if its acknowledgement is lost. A + # retained record can also differ because a killed prior run + # left the source off; restore that original at cleanup. + source_rxps_changed = ( + current_source_rxps != source_rxps_saved + or source_rxps_saved.enabled + ) + disabled_now = disable_source_rxps( + args, current_source_rxps + ) + source_rxps_changed = source_rxps_changed or disabled_now if controller is None: controller = Controller(args, password) verify_shared_source_identity(controller, args) @@ -3960,10 +4585,11 @@ def main( else: target = offline_target(args) - work_dir = make_work_dir(args.work_dir) if original_radio is not None: recovery_path = work_dir / "controller-radio.txt" - recovery_path.write_text(original_radio.meshcli_value() + "\n", encoding="ascii") + write_private_recovery_file( + recovery_path, original_radio.meshcli_value() + "\n" + ) print(f"[controller] recovery settings: {recovery_path}") package_path, package, expected_body_hash = prepare_package( args, target, work_dir @@ -4022,8 +4648,6 @@ def main( args.target_rxps_profile = target_rxps_profile confirm_update(args, target, package) freq, bandwidth, sf, cr, _minutes = args.temp_values - temp_command = f"tempradio {args.temp_radio}" - temp_radio = RadioSettings( freq, bandwidth, sf, cr, original_radio.repeat ) @@ -4053,10 +4677,15 @@ def main( # can process TempRadio even when its reply is lost, so resolve that # ambiguity by probing its exact identity on the temporary channel # instead of replaying the command from the normal channel. + target_temp_owned = True + if args.source_shares_controller: + # A lost-reply probe can schedule the shared local source onto the + # temporary tuple. Own that possible override before entering the + # helper so outer cleanup retries any failed local restore. + source_temp_owned = True arm_target_temp_radio( controller, args, temp_command, temp_radio, original_radio ) - target_temp_owned = True for relay_name, relay_password in args.relay_values: temp_reply = controller.remote_command( relay_name, temp_command, password=relay_password @@ -4064,18 +4693,23 @@ def main( require_temp_radio_reply(relay_name, temp_reply) armed_relay_values.append((relay_name, relay_password)) if not args.source_already_temp and not args.source_shares_controller: - source_cli_command(args, temp_command) + # Arm cleanup before the mutating command because the source can + # enter TempRadio even when its acknowledgement is lost. source_temp_owned = True + source_cli_command(args, temp_command) - controller_changed = True - controller.set_radio(temp_radio, "switch controller to TempRadio") if args.source_shares_controller: - # The Full Companion OTA egress gate needs its local TempRadio - # state even though the Binary API has already moved the same - # physical radio. Enter it after persisting the temporary tuple; - # cleanup runs `normalradio` before the Binary restore. - source_cli_command(args, temp_command) + # Arm cleanup before the local command. Its reply can be lost after + # the shared physical radio has already scheduled the handoff. source_temp_owned = True + switch_controller_to_temp_radio( + controller, args, temp_command, temp_radio + ) + # Arm Binary restoration only after the handoff succeeds. For a + # shared Full Companion the helper deliberately uses its bounded + # local TempRadio command so the temporary tuple is never persisted as + # the normal radio configuration. + controller_changed = True time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) for relay_name, relay_password in args.relay_values: @@ -4111,11 +4745,6 @@ def main( target_temp_owned = False shorten_relay_temp_windows(controller, args) armed_relay_values.clear() - seeder.stop() - seeder = None - if source_temp_owned: - shorten_source_temp_window(args) - source_temp_owned = False if target_rxps_changed and target_rxps_saved is not None: if args.leave_controller_radio: print( @@ -4124,10 +4753,28 @@ def main( f"{TARGET_RXPS_RECOVERY_FILE} after normalradio" ) else: + # Restore while the controller and destination still share + # the temporary tuple. The destination's one-minute return + # window remains bounded after this exact readback. restore_remote_rxps( controller, args.target, target_rxps_saved ) target_rxps_changed = False + seeder.stop() + seeder = None + if source_temp_owned and ( + not args.leave_controller_radio + or not args.source_shares_controller + ): + if shorten_source_temp_window(args): + source_temp_owned = False + if controller_changed and not args.leave_controller_radio: + controller.set_radio( + original_radio, "restore controller radio after staging" + ) + controller_changed = False + restore_source_rxps_once("stage-only cleanup") + retire_local_source_rxps_recovery() return 0 install_confirmed = request_install(controller, args, package) @@ -4153,9 +4800,12 @@ def main( # Stop seeding before returning the controller to its ordinary channel. seeder.stop() seeder = None - if source_temp_owned and not args.leave_controller_radio: - shorten_source_temp_window(args) - source_temp_owned = False + if source_temp_owned and ( + not args.leave_controller_radio + or not args.source_shares_controller + ): + if shorten_source_temp_window(args): + source_temp_owned = False controller.set_radio(original_radio, "restore controller radio for verification") controller_changed = False relay_wait = ( @@ -4190,6 +4840,8 @@ def main( temp_radio, "return controller to TempRadio after verification" ) controller_changed = True + restore_source_rxps_once("successful update cleanup") + retire_local_source_rxps_recovery() return 0 except KeyboardInterrupt: print("\nInterrupted; any partial target download remains resumable.", file=sys.stderr) @@ -4218,9 +4870,8 @@ def main( ): if not controller_changed: try: - controller.set_radio( - temp_radio, - "return controller to TempRadio for failure cleanup", + switch_controller_to_temp_radio( + controller, args, temp_command, temp_radio ) controller_changed = True time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) @@ -4286,13 +4937,20 @@ def main( f"{exc}", file=sys.stderr, ) - if source_temp_owned and not args.leave_controller_radio: - shorten_source_temp_window(args, check=False) + if source_temp_owned and ( + not args.leave_controller_radio + or not args.source_shares_controller + ): + if shorten_source_temp_window(args, check=False): + source_temp_owned = False if ( controller is not None and controller_changed and original_radio is not None and not args.leave_controller_radio + and not ( + args.source_shares_controller and source_temp_owned + ) ): try: controller.set_radio(original_radio, "restore controller radio after failure") @@ -4321,6 +4979,31 @@ def main( f"{TARGET_RXPS_RECOVERY_FILE}: {exc}", file=sys.stderr, ) + if source_rxps_changed: + try: + restore_source_rxps_once("failure cleanup") + except (OtaError, OSError) as exc: + recovery_hint = ( + str(source_rxps_recovery_path) + if source_rxps_recovery_path is not None + else SOURCE_RXPS_RECOVERY_FILE + ) + print( + "CRITICAL: could not safely restore OTA source RXPS; " + "leave RXPS off and return the source to its normal " + "radio before restoring it manually with " + f"{recovery_hint}: {exc}", + file=sys.stderr, + ) + if not source_rxps_changed: + try: + retire_local_source_rxps_recovery() + except (OtaError, OSError) as exc: + print( + "WARNING: source RXPS was restored, but its completed " + f"recovery record could not be retired: {exc}", + file=sys.stderr, + ) if work_dir is not None: print(f"[work] retained at {work_dir}") diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index 28cc133b..db190874 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -18,9 +18,9 @@ import shutil import stat import subprocess import sys +import tempfile import time from types import SimpleNamespace -import urllib.request import zipfile try: @@ -29,16 +29,28 @@ except ImportError: import lora_ota as ota -RELEASE_TAG = "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.02-e742333a" -RELEASE_URL = f"https://github.com/mikecarper/MeshCore/releases/tag/{RELEASE_TAG}" -ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip" -ASSET_URL = ( - f"https://github.com/mikecarper/MeshCore/releases/download/{RELEASE_TAG}/" - f"{ASSET_NAME}" +# These exact ten package transitions passed directly on the physical target, +# and the resulting endpoint passed independent SWD readback. The host runner +# has received cleanup/recovery fixes since that run and has not itself had a +# new clean end-to-end qualification run. Keep the artifact unpublished and +# normally gated from live use; never invent a release URL. +RELEASE_URL = "unreleased local RAK3401 10-step candidate" +ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip" +ASSET_SHA256 = "c0b33f4568985e8b2b8dc99411295907212cf2bad21764b6333d5e0ba298fd61" +CHECKSUM_LIST_SHA256 = "3f8c4af8096b96a4aa6506825c387cc8a06f74d5213a29c9387bd11689546881" +BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90" + +# Preserve the physically qualified nine-step chain for exact offline +# recognition and existing live deployments. +PHYSICALLY_QUALIFIED_9_ASSET_SHA256 = ( + "9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a" +) +PHYSICALLY_QUALIFIED_9_CHECKSUM_LIST_SHA256 = ( + "73d96e23237896a3e342fe736be12d94087a813bf09ad609fb55330bbe586055" +) +PHYSICALLY_QUALIFIED_9_ROOT_NAME = ( + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a" ) -ASSET_SHA256 = "9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a" -CHECKSUM_LIST_SHA256 = "73d96e23237896a3e342fe736be12d94087a813bf09ad609fb55330bbe586055" -BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a" # The accelerated 30-step release is the pinned reconstruction input for the # compact chain. Keep it available for offline provenance, but do not start a @@ -115,15 +127,43 @@ KNOWN_UNSAFE_ROOT_NAME = ( "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.2-c96bdd6e" ) +EXTRACTION_BINDING_FILE = ".source-archive.json" +# Release bundles are small (even the retired 30-step chain is far below +# these ceilings). Bound every untrusted archive/tree operation before it can +# consume arbitrary memory, CPU, or disk. +MAX_BUNDLE_ARCHIVE_BYTES = 64 * 1024 * 1024 +MAX_BUNDLE_UNCOMPRESSED_BYTES = 128 * 1024 * 1024 +MAX_BUNDLE_MEMBER_BYTES = 32 * 1024 * 1024 +MAX_BUNDLE_MEMBERS = 2048 +MAX_BUNDLE_TREE_ENTRIES = 4096 +MAX_CHECKSUM_LIST_BYTES = 1024 * 1024 +MAX_PROGRESS_BYTES = 1024 * 1024 +PINNED_ARCHIVE_CHECKSUMS = { + ASSET_SHA256: CHECKSUM_LIST_SHA256, + PHYSICALLY_QUALIFIED_9_ASSET_SHA256: ( + PHYSICALLY_QUALIFIED_9_CHECKSUM_LIST_SHA256 + ), + SUPERSEDED_30_ASSET_SHA256: SUPERSEDED_30_CHECKSUM_LIST_SHA256, + SUPERSEDED_29_ASSET_SHA256: SUPERSEDED_29_CHECKSUM_LIST_SHA256, + SUPERSEDED_27_ASSET_SHA256: SUPERSEDED_27_CHECKSUM_LIST_SHA256, + KNOWN_FAILED_V11701_ASSET_SHA256: KNOWN_FAILED_V11701_CHECKSUM_LIST_SHA256, + KNOWN_FAILED_V11701_STEP16_ASSET_SHA256: ( + KNOWN_FAILED_V11701_STEP16_CHECKSUM_LIST_SHA256 + ), + KNOWN_UNSAFE_ASSET_SHA256: KNOWN_UNSAFE_CHECKSUM_LIST_SHA256, +} + DEFAULT_TARGET_KEY = ( "63d8df6387eaffd2e25db7d2a8ad967a" "65202182a48d681d7e7a9260f917280d" ) EXPECTED_TARGET_ID = 0x2FA509C1 EXPECTED_HARDWARE = "RAK_3401" -EXPECTED_STEP_COUNT = 9 +EXPECTED_STEP_COUNT = 10 +PHYSICALLY_QUALIFIED_9_STEP_COUNT = 9 EXPECTED_START_VERSION = "1.16.7.0" -EXPECTED_FINAL_VERSION = "1.17.1.02" +EXPECTED_FINAL_VERSION = "1.17.1.5" +PHYSICALLY_QUALIFIED_9_FINAL_VERSION = "1.17.1.02" SUPERSEDED_FINAL_VERSION = "1.17.1.0" WATCHDOG_RESET_WAIT_SECONDS = 90 WATCHDOG_STABILITY_WAIT_SECONDS = 90 @@ -201,6 +241,23 @@ COMPACT_RELEASE_ANCHORS = ( (9, "2784e4b645bc3dc198de0b8b18d3d7369cd02eca61cd71c46a51b61854da5345"), ) +# Every target in the exact unreleased ten-package fd98bc90 sequence that +# passed direct physical transitions. These pins bind both offline verification +# and the hidden explicit live-lab override to those exact package bytes; they +# do not qualify later host-runner changes. +CURRENT_10_CANDIDATE_ANCHORS = ( + (1, "8364257a2b3a219905e870fad6fbb2040a96ca4b4bb7201b2867534cc2b45530"), + (2, "ac5f50e5028378ccfe6ea08bbf32f227f50fdcf5285a7deb866e309fbdd0a88f"), + (3, "884b5e9355b4585b7a4e079dbb44d2858ec46e2fa7bb0dba47e810db2a82e349"), + (4, "cd6fe1752f859b9e8648f2cc2b9596962d371445e39854130197d61ce1fff49f"), + (5, "e826c91480390e4eec8d49a29e8ec0a957c9668a51dffad1ff0bb1d39daf38c2"), + (6, "8e96913fbacb17f43cebba4aaa3bd99cb6953711744c9e55de5b2e09af846e27"), + (7, "47ab2282b70afeccd7fbdd0418f60a44c98639fbdd0534b43dce94c2a5af7a6d"), + (8, "74a319a8744ec3f28c0f73214dcc153960df5c439b31998be2ff05464fccf4d7"), + (9, "30aea80995def68ddff0671138b9f7269b0aa3dea7271fb7f6570637aae577a0"), + (10, "31c182c888ceb1135e5afb2376610d93cee2e807b556c838e07fd4486c79d095"), +) + # Exact anchors for every image in the accelerated 30-step release. The outer # asset and inner checksum-list pins cover every byte as well; the complete # anchor table keeps structural validation fail-closed if those layers are @@ -278,6 +335,16 @@ SUPERSEDED_30_MESSAGE = ( "which reaches the requested endpoint in nine packages without changing " "the deployed bootloader. Use --verify-only for the older bundle." ) +CURRENT_10_CANDIDATE_MESSAGE = ( + "live installation of the exact fd98bc90 ten-step candidate is disabled: " + "the ten pinned package transitions completed directly on the physical " + "RAK3401 and the endpoint passed independent SWD readback, but the current " + "host runner includes later cleanup/recovery fixes that have not had a new " + "clean end-to-end physical run. The artifact is also local and unpublished. " + "Use --verify-only with the explicit local bundle path, or the hidden " + "controlled-lab override, until the exact artifact and qualification record " + "are published." +) class KnownUnsafeReleaseError(ota.OtaError): """The pinned artifacts are intact but their live transition is unsafe.""" @@ -287,6 +354,19 @@ def require_live_release_safe( steps: list[ChainStep], ) -> None: if len(steps) == EXPECTED_STEP_COUNT: + if all( + steps[number - 1].target_sha256 == expected_sha256 + for number, expected_sha256 in CURRENT_10_CANDIDATE_ANCHORS + ): + if args.accept_test_candidate: + return + raise KnownUnsafeReleaseError(CURRENT_10_CANDIDATE_MESSAGE) + raise KnownUnsafeReleaseError( + "live installation is disabled: this is an unrecognized variant " + "of the pinned ten-step candidate" + ) + + if len(steps) == PHYSICALLY_QUALIFIED_9_STEP_COUNT: for number, expected_sha256 in COMPACT_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: raise KnownUnsafeReleaseError( @@ -397,6 +477,122 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def _open_regular_readonly(path: Path, label: str) -> tuple[int, os.stat_result]: + """Open a caller-controlled path without following a final symlink.""" + try: + path_metadata = path.lstat() + except OSError as exc: + raise ota.OtaError(f"cannot inspect {label} {path}: {exc}") from exc + if stat.S_ISLNK(path_metadata.st_mode): + raise ota.OtaError(f"{label} is a symbolic link: {path}") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise ota.OtaError(f"cannot open {label} {path}: {exc}") from exc + metadata = os.fstat(descriptor) + try: + current_path_metadata = path.lstat() + except OSError as exc: + os.close(descriptor) + raise ota.OtaError(f"{label} path changed while opening: {path}") from exc + if ( + not stat.S_ISREG(metadata.st_mode) + or not os.path.samestat(metadata, path_metadata) + or not os.path.samestat(metadata, current_path_metadata) + ): + os.close(descriptor) + raise ota.OtaError(f"{label} is not one stable regular file: {path}") + return descriptor, metadata + + +def sha256_file_limited(path: Path, maximum: int, label: str) -> str: + descriptor, metadata = _open_regular_readonly(path, label) + if metadata.st_size > maximum: + os.close(descriptor) + raise ota.OtaError( + f"{label} is {metadata.st_size} bytes; limit is {maximum}: {path}" + ) + digest = hashlib.sha256() + total = 0 + with os.fdopen(descriptor, "rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + total += len(chunk) + if total > maximum: + raise ota.OtaError( + f"{label} grew beyond its {maximum}-byte limit: {path}" + ) + digest.update(chunk) + if total != metadata.st_size: + raise ota.OtaError(f"{label} changed while it was read: {path}") + return digest.hexdigest() + + +def read_regular_bytes_limited(path: Path, maximum: int, label: str) -> bytes: + descriptor, metadata = _open_regular_readonly(path, label) + if metadata.st_size > maximum: + os.close(descriptor) + raise ota.OtaError( + f"{label} is {metadata.st_size} bytes; limit is {maximum}: {path}" + ) + with os.fdopen(descriptor, "rb") as source: + value = source.read(maximum + 1) + if len(value) > maximum or source.read(1): + raise ota.OtaError(f"{label} grew beyond its size limit: {path}") + if len(value) != metadata.st_size: + raise ota.OtaError(f"{label} changed while it was read: {path}") + return value + + +def freeze_archive(source_path: Path, frozen_path: Path) -> str: + """Copy and hash one opened archive; all later work uses this snapshot.""" + source_descriptor, metadata = _open_regular_readonly( + source_path, "release ZIP" + ) + if metadata.st_size > MAX_BUNDLE_ARCHIVE_BYTES: + os.close(source_descriptor) + raise ota.OtaError( + f"release ZIP is {metadata.st_size} bytes; limit is " + f"{MAX_BUNDLE_ARCHIVE_BYTES}: {source_path}" + ) + output_flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + ) + try: + output_descriptor = os.open(frozen_path, output_flags, 0o600) + except OSError: + os.close(source_descriptor) + raise + digest = hashlib.sha256() + total = 0 + try: + with ( + os.fdopen(source_descriptor, "rb") as source, + os.fdopen(output_descriptor, "wb") as output, + ): + for chunk in iter(lambda: source.read(1024 * 1024), b""): + total += len(chunk) + if total > MAX_BUNDLE_ARCHIVE_BYTES: + raise ota.OtaError( + "release ZIP grew beyond its bounded archive limit" + ) + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + except Exception: + # fdopen owns both descriptors once the with statement is entered. + raise + if total != metadata.st_size: + raise ota.OtaError("release ZIP changed while it was snapshotted") + frozen_path.chmod(0o600) + return digest.hexdigest() + + def safe_relative_path(value: str, label: str) -> PurePosixPath: if "\\" in value: raise ota.OtaError(f"{label} contains a backslash: {value!r}") @@ -408,7 +604,9 @@ def safe_relative_path(value: str, label: str) -> PurePosixPath: def download_release_asset(destination: Path) -> None: if destination.exists(): - actual = sha256_file(destination) + actual = sha256_file_limited( + destination, MAX_BUNDLE_ARCHIVE_BYTES, "cached release ZIP" + ) if actual != ASSET_SHA256: raise ota.OtaError( f"cached release asset has SHA-256 {actual}, expected {ASSET_SHA256}: " @@ -416,102 +614,305 @@ def download_release_asset(destination: Path) -> None: ) print(f"[bundle] using verified cached asset {destination}") return - - partial = destination.with_suffix(destination.suffix + ".part") - if partial.exists(): - partial.unlink() - print(f"[bundle] downloading {ASSET_URL}") - request = urllib.request.Request( - ASSET_URL, - headers={"User-Agent": "MeshCore-RAK3401-chain-runner/1"}, + raise ota.OtaError( + "the exact ten-step candidate is not released; pass its explicit local " + "ZIP or extracted root with --bundle" ) - digest = hashlib.sha256() - received = 0 + + +def write_extraction_binding( + destination: Path, + archive_sha256: str, + checksum_sha256: str, + root_name: str, +) -> None: + path = destination / EXTRACTION_BINDING_FILE + ota.write_private_recovery_file( + path, + json.dumps( + { + "archive_sha256": archive_sha256, + "checksum_list_sha256": checksum_sha256, + "root_name": root_name, + }, + indent=2, + sort_keys=True, + ) + + "\n", + ) + + +def read_extraction_binding(destination: Path) -> dict[str, str] | None: + path = destination / EXTRACTION_BINDING_FILE + if not path.exists(): + return None try: - with urllib.request.urlopen(request, timeout=60) as response, partial.open("xb") as output: - while True: - chunk = response.read(1024 * 1024) - if not chunk: - break - received += len(chunk) - if received > 64 * 1024 * 1024: - raise ota.OtaError("release asset exceeds the 64 MiB safety limit") - digest.update(chunk) - output.write(chunk) - actual = digest.hexdigest() - if actual != ASSET_SHA256: - raise ota.OtaError( - f"downloaded release asset has SHA-256 {actual}, expected {ASSET_SHA256}" - ) - os.replace(partial, destination) - finally: - partial.unlink(missing_ok=True) - print(f"[bundle] downloaded and verified {received} bytes") + value = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ota.OtaError(f"invalid bundle extraction binding {path}: {exc}") from exc + if not isinstance(value, dict) or any( + not isinstance(value.get(key), str) + for key in ("archive_sha256", "checksum_list_sha256", "root_name") + ): + raise ota.OtaError(f"invalid bundle extraction binding fields: {path}") + return value -def extract_bundle(archive_path: Path, destination: Path) -> Path: - root_names = ( - BUNDLE_ROOT_NAME, - SUPERSEDED_30_ROOT_NAME, - SUPERSEDED_29_ROOT_NAME, - KNOWN_FAILED_V11701_ROOT_NAME, - KNOWN_UNSAFE_ROOT_NAME, - ) - existing_roots = [destination / name for name in root_names if (destination / name).is_dir()] - if len(existing_roots) == 1: - return existing_roots[0] - if len(existing_roots) > 1: - raise ota.OtaError(f"bundle extraction contains multiple recognized roots: {destination}") - if destination.exists() and any(destination.iterdir()): +def inspect_bundle_archive(archive_path: Path) -> str: + """Validate bounded members and identify the archive's one bundle root.""" + try: + with zipfile.ZipFile(archive_path) as archive: + members = archive.infolist() + except (OSError, zipfile.BadZipFile) as exc: + raise ota.OtaError(f"cannot read release ZIP {archive_path}: {exc}") from exc + + if len(members) > MAX_BUNDLE_MEMBERS: raise ota.OtaError( - f"bundle extraction directory is incomplete or unexpected: {destination}" + f"release ZIP has {len(members)} members; limit is " + f"{MAX_BUNDLE_MEMBERS}" ) - staging = destination.with_name(destination.name + f".part-{os.getpid()}") - if staging.exists(): - shutil.rmtree(staging) - staging.mkdir(parents=True) + chain_roots: set[str] = set() + checksum_roots: set[str] = set() + member_paths: list[PurePosixPath] = [] + seen_paths: set[PurePosixPath] = set() + compressed_total = 0 + uncompressed_total = 0 + for member in members: + relative = safe_relative_path(member.filename.rstrip("/"), "ZIP member") + if len(relative.as_posix()) > 512 or len(relative.parts) > 32: + raise ota.OtaError(f"release ZIP member path is too long: {member.filename}") + if relative in seen_paths: + raise ota.OtaError(f"release ZIP contains a duplicate member: {relative}") + seen_paths.add(relative) + member_paths.append(relative) + if member.flag_bits & 0x1: + raise ota.OtaError(f"release ZIP contains an encrypted member: {relative}") + mode = (member.external_attr >> 16) & 0o170000 + if stat.S_ISLNK(mode): + raise ota.OtaError(f"release ZIP contains a symbolic link: {member.filename}") + if mode and not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): + raise ota.OtaError(f"release ZIP contains a special file: {member.filename}") + if member.file_size < 0 or member.compress_size < 0: + raise ota.OtaError(f"release ZIP has an invalid member size: {relative}") + if member.file_size > MAX_BUNDLE_MEMBER_BYTES: + raise ota.OtaError( + f"release ZIP member exceeds {MAX_BUNDLE_MEMBER_BYTES} bytes: " + f"{relative}" + ) + compressed_total += member.compress_size + uncompressed_total += member.file_size + if compressed_total > MAX_BUNDLE_ARCHIVE_BYTES: + raise ota.OtaError("release ZIP compressed members exceed the archive limit") + if uncompressed_total > MAX_BUNDLE_UNCOMPRESSED_BYTES: + raise ota.OtaError("release ZIP expands beyond the bundle size limit") + if len(relative.parts) == 2 and relative.parts[1] == "CHAIN.csv": + chain_roots.add(relative.parts[0]) + if len(relative.parts) == 2 and relative.parts[1] == "SHA256SUMS.txt": + checksum_roots.add(relative.parts[0]) + roots = chain_roots & checksum_roots + if len(roots) != 1: + raise ota.OtaError( + "release ZIP must contain exactly one top-level bundle root with " + "CHAIN.csv and SHA256SUMS.txt" + ) + root_name = next(iter(roots)) + if any(relative.parts[0] != root_name for relative in member_paths): + raise ota.OtaError( + "release ZIP contains members outside its single bundle root" + ) + return root_name + + +def extract_archive_members(archive_path: Path, staging: Path) -> None: + """Extract only inspected members with independent streamed byte caps.""" + actual_total = 0 try: with zipfile.ZipFile(archive_path) as archive: for member in archive.infolist(): - safe_relative_path(member.filename.rstrip("/"), "ZIP member") - mode = (member.external_attr >> 16) & 0o170000 - if stat.S_ISLNK(mode): - raise ota.OtaError(f"release ZIP contains a symbolic link: {member.filename}") - archive.extractall(staging) - staged_roots = [staging / name for name in root_names if (staging / name).is_dir()] - if len(staged_roots) != 1: + relative = safe_relative_path( + member.filename.rstrip("/"), "ZIP member" + ) + destination = staging.joinpath(*relative.parts) + if member.is_dir(): + destination.mkdir(mode=0o700, parents=True, exist_ok=True) + continue + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + member_total = 0 + with ( + archive.open(member, "r") as source, + destination.open("xb") as output, + ): + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + member_total += len(chunk) + actual_total += len(chunk) + if member_total > MAX_BUNDLE_MEMBER_BYTES: + raise ota.OtaError( + f"release ZIP member expanded beyond its limit: {relative}" + ) + if actual_total > MAX_BUNDLE_UNCOMPRESSED_BYTES: + raise ota.OtaError( + "release ZIP expanded beyond its aggregate limit" + ) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + destination.chmod(0o600) + if member_total != member.file_size: + raise ota.OtaError( + f"release ZIP member changed size while extracting: {relative}" + ) + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + raise ota.OtaError(f"cannot safely extract release ZIP: {exc}") from exc + + +def extract_bundle( + archive_path: Path, + destination: Path, + archive_sha256: str | None = None, +) -> Path: + if destination.is_symlink() or ( + destination.exists() and not destination.is_dir() + ): + raise ota.OtaError( + f"bundle extraction destination is not a real directory: {destination}" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + + # Hash, inspect, and extract a uniquely owned snapshot. The caller's ZIP + # pathname may be replaced after this copy without changing any byte we + # subsequently trust or extract. + with tempfile.TemporaryDirectory( + prefix=f".{destination.name}.archive-", dir=destination.parent + ) as frozen_name: + frozen_archive = Path(frozen_name) / "release.zip" + actual_archive_sha256 = freeze_archive(archive_path, frozen_archive) + if ( + archive_sha256 is not None + and actual_archive_sha256 != archive_sha256 + ): raise ota.OtaError( - f"release ZIP must contain exactly one recognized bundle root: {root_names}" + f"release ZIP changed after validation: got {actual_archive_sha256}, " + f"expected {archive_sha256}" + ) + return extract_frozen_bundle( + frozen_archive, destination, actual_archive_sha256 + ) + + +def extract_frozen_bundle( + archive_path: Path, + destination: Path, + actual_archive_sha256: str, +) -> Path: + """Extract one private, already-hashed archive snapshot.""" + expected_checksum_sha256 = PINNED_ARCHIVE_CHECKSUMS.get( + actual_archive_sha256 + ) + if expected_checksum_sha256 is None: + raise ota.OtaError( + "refusing to extract an archive whose SHA-256 is not pinned: " + f"{actual_archive_sha256}" + ) + root_name = inspect_bundle_archive(archive_path) + expected_root = destination / root_name + + if destination.exists() and any(destination.iterdir()): + allowed_names = {root_name, EXTRACTION_BINDING_FILE} + unexpected = sorted( + entry.name for entry in destination.iterdir() + if entry.name not in allowed_names + ) + if unexpected or expected_root.is_symlink() or not expected_root.is_dir(): + raise ota.OtaError( + "bundle extraction cache does not match this archive; use a " + f"different work directory (unexpected={unexpected or 'root mismatch'})" + ) + checksum_path = expected_root / "SHA256SUMS.txt" + if ( + not checksum_path.is_file() + or sha256_file_limited( + checksum_path, + MAX_CHECKSUM_LIST_BYTES, + "bundle checksum list", + ) != expected_checksum_sha256 + ): + raise ota.OtaError( + "bundle extraction cache checksum does not match this pinned " + "archive; use a different work directory" + ) + binding = read_extraction_binding(destination) + expected_binding = { + "archive_sha256": actual_archive_sha256, + "checksum_list_sha256": expected_checksum_sha256, + "root_name": root_name, + } + if binding is not None and binding != expected_binding: + raise ota.OtaError( + "bundle extraction cache is bound to a different archive; use " + "a different work directory" + ) + if binding is None: + # Adopt a legacy cache only after its pinned checksum-list digest + # proves it is the exact content paired with this archive hash. + write_extraction_binding( + destination, + actual_archive_sha256, + expected_checksum_sha256, + root_name, + ) + return expected_root + + with tempfile.TemporaryDirectory( + prefix=f".{destination.name}.part-", dir=destination.parent + ) as staging_name: + staging = Path(staging_name) + extract_archive_members(archive_path, staging) + staged_root = staging / root_name + checksum_path = staged_root / "SHA256SUMS.txt" + if ( + not staged_root.is_dir() + or not checksum_path.is_file() + or sha256_file_limited( + checksum_path, + MAX_CHECKSUM_LIST_BYTES, + "bundle checksum list", + ) != expected_checksum_sha256 + ): + raise ota.OtaError( + "release ZIP checksum-list digest does not match the value " + "paired with its pinned archive hash" ) - staged_root = staged_roots[0] if destination.exists(): destination.rmdir() os.replace(staging, destination) - finally: - if staging.exists(): - shutil.rmtree(staging) - return destination / staged_root.name + write_extraction_binding( + destination, + actual_archive_sha256, + expected_checksum_sha256, + root_name, + ) + return destination / root_name def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: if args.bundle is None: archive = work_dir / ASSET_NAME download_release_asset(archive) - return extract_bundle(archive, work_dir / "bundle") + return extract_bundle(archive, work_dir / "bundle", ASSET_SHA256) supplied = args.bundle.resolve() if supplied.is_dir(): direct = supplied / "CHAIN.csv" nested_roots = [ - supplied / name for name in ( - BUNDLE_ROOT_NAME, - SUPERSEDED_30_ROOT_NAME, - SUPERSEDED_29_ROOT_NAME, - KNOWN_FAILED_V11701_ROOT_NAME, - KNOWN_UNSAFE_ROOT_NAME, - ) - if (supplied / name / "CHAIN.csv").is_file() + child for child in supplied.iterdir() + if child.is_dir() + and not child.is_symlink() + and (child / "CHAIN.csv").is_file() + and (child / "SHA256SUMS.txt").is_file() ] if direct.is_file(): return supplied @@ -520,9 +921,12 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: raise ota.OtaError(f"bundle directory does not contain CHAIN.csv: {supplied}") if not supplied.is_file() or supplied.suffix.lower() != ".zip": raise ota.OtaError("--bundle must be the pinned release ZIP or its extracted root") - actual = sha256_file(supplied) + actual = sha256_file_limited( + supplied, MAX_BUNDLE_ARCHIVE_BYTES, "release ZIP" + ) if actual not in { ASSET_SHA256, + PHYSICALLY_QUALIFIED_9_ASSET_SHA256, SUPERSEDED_30_ASSET_SHA256, SUPERSEDED_29_ASSET_SHA256, SUPERSEDED_27_ASSET_SHA256, @@ -533,16 +937,20 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: raise ota.OtaError( f"release ZIP has SHA-256 {actual}, expected a pinned audited asset: {supplied}" ) - return extract_bundle(supplied, work_dir / "bundle") + return extract_bundle(supplied, work_dir / "bundle", actual) -def verify_checksum_list(bundle_root: Path) -> None: +def read_checksum_entries(bundle_root: Path) -> dict[PurePosixPath, str]: checksum_path = bundle_root / "SHA256SUMS.txt" if not checksum_path.is_file(): raise ota.OtaError("bundle is missing SHA256SUMS.txt") - checksum_digest = sha256_file(checksum_path) + checksum_bytes = read_regular_bytes_limited( + checksum_path, MAX_CHECKSUM_LIST_BYTES, "bundle checksum list" + ) + checksum_digest = hashlib.sha256(checksum_bytes).hexdigest() expected_lists = { CHECKSUM_LIST_SHA256, + PHYSICALLY_QUALIFIED_9_CHECKSUM_LIST_SHA256, SUPERSEDED_30_CHECKSUM_LIST_SHA256, SUPERSEDED_29_CHECKSUM_LIST_SHA256, SUPERSEDED_27_CHECKSUM_LIST_SHA256, @@ -555,9 +963,12 @@ def verify_checksum_list(bundle_root: Path) -> None: "bundle checksum list is not the one pinned by this chain runner: " f"got {checksum_digest}, expected one of {sorted(expected_lists)}" ) - checked = 0 - listed: set[PurePosixPath] = set() - for line_number, raw_line in enumerate(checksum_path.read_text(encoding="ascii").splitlines(), 1): + try: + checksum_text = checksum_bytes.decode("ascii") + except UnicodeError as exc: + raise ota.OtaError("bundle checksum list is not ASCII") from exc + listed: dict[PurePosixPath, str] = {} + for line_number, raw_line in enumerate(checksum_text.splitlines(), 1): if not raw_line.strip(): continue match = re.fullmatch(r"([0-9a-fA-F]{64}) [ *](.+)", raw_line) @@ -567,33 +978,211 @@ def verify_checksum_list(bundle_root: Path) -> None: relative = safe_relative_path(name, "checksum entry") if relative in listed: raise ota.OtaError(f"duplicate checksum entry: {relative}") - listed.add(relative) + if len(relative.as_posix()) > 512 or len(relative.parts) > 32: + raise ota.OtaError(f"checksum entry path is too long: {relative}") + listed[relative] = expected.lower() + if len(listed) > MAX_BUNDLE_MEMBERS: + raise ota.OtaError( + f"bundle checksum list exceeds {MAX_BUNDLE_MEMBERS} entries" + ) + return listed + + +def inventory_bundle_tree(bundle_root: Path) -> dict[PurePosixPath, int]: + """Inventory an extracted tree without following links or reading payloads.""" + if bundle_root.is_symlink() or not bundle_root.is_dir(): + raise ota.OtaError(f"bundle root is not a real directory: {bundle_root}") + files: dict[PurePosixPath, int] = {} + pending: list[tuple[Path, PurePosixPath]] = [(bundle_root, PurePosixPath())] + entries_seen = 0 + total_bytes = 0 + while pending: + directory, relative_directory = pending.pop() + try: + entries = list(os.scandir(directory)) + except OSError as exc: + raise ota.OtaError(f"cannot inventory bundle directory {directory}: {exc}") from exc + for entry in entries: + entries_seen += 1 + if entries_seen > MAX_BUNDLE_TREE_ENTRIES: + raise ota.OtaError( + f"bundle tree exceeds {MAX_BUNDLE_TREE_ENTRIES} entries" + ) + relative = relative_directory / entry.name + safe_relative_path(relative.as_posix(), "bundle entry") + try: + metadata = entry.stat(follow_symlinks=False) + except OSError as exc: + raise ota.OtaError(f"cannot inspect bundle entry {entry.path}: {exc}") from exc + if stat.S_ISLNK(metadata.st_mode): + raise ota.OtaError(f"bundle contains a symbolic link: {entry.path}") + if stat.S_ISDIR(metadata.st_mode): + pending.append((Path(entry.path), relative)) + continue + if not stat.S_ISREG(metadata.st_mode): + raise ota.OtaError(f"bundle contains a non-file entry: {entry.path}") + limit = ( + MAX_CHECKSUM_LIST_BYTES + if relative == PurePosixPath("SHA256SUMS.txt") + else MAX_BUNDLE_MEMBER_BYTES + ) + if metadata.st_size > limit: + raise ota.OtaError( + f"bundle entry is {metadata.st_size} bytes; limit is " + f"{limit}: {relative}" + ) + total_bytes += metadata.st_size + if total_bytes > MAX_BUNDLE_UNCOMPRESSED_BYTES: + raise ota.OtaError("bundle tree exceeds its aggregate byte limit") + files[relative] = metadata.st_size + return files + + +def verify_checksum_list(bundle_root: Path) -> None: + listed = read_checksum_entries(bundle_root) + inventory = inventory_bundle_tree(bundle_root) + checksum_relative = PurePosixPath("SHA256SUMS.txt") + actual_files = set(inventory) - {checksum_relative} + listed_files = set(listed) + if listed_files != actual_files: + missing = sorted(str(path) for path in actual_files - listed_files) + extra = sorted(str(path) for path in listed_files - actual_files) + raise ota.OtaError( + "bundle checksum coverage mismatch; " + f"unlisted={missing or 'none'}, nonexistent={extra or 'none'}" + ) + + checked = 0 + for relative, expected in listed.items(): path = bundle_root.joinpath(*relative.parts) - if path.is_symlink() or not path.is_file(): - raise ota.OtaError(f"checksum entry is missing: {relative}") - actual = sha256_file(path) + actual = sha256_file_limited( + path, MAX_BUNDLE_MEMBER_BYTES, f"bundle entry {relative}" + ) if actual.lower() != expected.lower(): raise ota.OtaError( f"checksum mismatch for {relative}: got {actual}, expected {expected}" ) checked += 1 - - actual_files: set[PurePosixPath] = set() - for path in bundle_root.rglob("*"): - if path.is_symlink(): - raise ota.OtaError(f"bundle contains a symbolic link: {path}") - if path.is_file() and path != checksum_path: - actual_files.add(PurePosixPath(path.relative_to(bundle_root).as_posix())) - if listed != actual_files: - missing = sorted(str(path) for path in actual_files - listed) - extra = sorted(str(path) for path in listed - actual_files) - raise ota.OtaError( - "bundle checksum coverage mismatch; " - f"unlisted={missing or 'none'}, nonexistent={extra or 'none'}" - ) print(f"[bundle] verified all {checked} SHA-256 entries") +def require_bundle_work_separation(bundle_root: Path, work_dir: Path) -> None: + """Keep mutable run state out of the immutable bundle input tree.""" + resolved_bundle = bundle_root.resolve(strict=True) + resolved_work = work_dir.resolve(strict=True) + if resolved_work == resolved_bundle or resolved_work.is_relative_to(resolved_bundle): + raise ota.OtaError( + "work directory must be outside the supplied bundle root so run " + "artifacts cannot change the verified input tree" + ) + + +def copy_regular_file_limited( + source: Path, + destination: Path, + maximum: int, + label: str, + *, + expected_size: int | None = None, +) -> int: + descriptor, metadata = _open_regular_readonly(source, label) + if metadata.st_size > maximum: + os.close(descriptor) + raise ota.OtaError( + f"{label} is {metadata.st_size} bytes; limit is {maximum}: {source}" + ) + if expected_size is not None and metadata.st_size != expected_size: + os.close(descriptor) + raise ota.OtaError( + f"{label} changed size after the bounded bundle inventory" + ) + copy_limit = metadata.st_size if expected_size is not None else maximum + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + total = 0 + source_name = str(source) + try: + with ( + os.fdopen(descriptor, "rb") as input_file, + destination.open("xb") as output_file, + ): + class BoundedReader: + @property + def name(self) -> str: + return source_name + + def read(self, requested: int = -1) -> bytes: + nonlocal total + remaining = copy_limit - total + if requested < 0: + requested = remaining + 1 + chunk = input_file.read(min(requested, remaining + 1)) + if len(chunk) > remaining: + raise ota.OtaError(f"{label} grew beyond its size limit") + total += len(chunk) + return chunk + + shutil.copyfileobj(BoundedReader(), output_file, 1024 * 1024) + output_file.flush() + os.fsync(output_file.fileno()) + except Exception: + raise + if total != metadata.st_size: + raise ota.OtaError(f"{label} changed while it was copied") + destination.chmod(0o600) + return total + + +def snapshot_verified_bundle(bundle_root: Path, destination_parent: Path) -> Path: + """Copy an input tree once, then verify and use only the private snapshot.""" + if destination_parent.is_symlink() or not destination_parent.is_dir(): + raise ota.OtaError( + f"bundle snapshot parent is not a real directory: {destination_parent}" + ) + resolved_bundle = bundle_root.resolve(strict=True) + resolved_parent = destination_parent.resolve(strict=True) + if resolved_parent == resolved_bundle or resolved_parent.is_relative_to( + resolved_bundle + ): + raise ota.OtaError("bundle snapshot must be outside the input bundle tree") + snapshot_root = destination_parent / bundle_root.name + snapshot_root.mkdir(mode=0o700) + try: + # Coverage and resource checks happen before the first payload byte is + # copied. Copy only the pinned set (plus the list itself), then verify + # the private result again to close mutation races during the copy. + verify_checksum_list(bundle_root) + listed = read_checksum_entries(bundle_root) + inventory = inventory_bundle_tree(bundle_root) + copy_order = [PurePosixPath("SHA256SUMS.txt"), *sorted(listed)] + copied_total = 0 + for relative in copy_order: + source = bundle_root.joinpath(*relative.parts) + destination = snapshot_root.joinpath(*relative.parts) + limit = ( + MAX_CHECKSUM_LIST_BYTES + if relative == PurePosixPath("SHA256SUMS.txt") + else MAX_BUNDLE_MEMBER_BYTES + ) + expected_size = inventory[relative] + if copied_total + expected_size > MAX_BUNDLE_UNCOMPRESSED_BYTES: + raise ota.OtaError("bundle copy exceeds its aggregate byte limit") + copied_total += copy_regular_file_limited( + source, + destination, + limit, + f"bundle entry {relative}", + expected_size=expected_size, + ) + # This post-copy verification binds every byte used by the runner. If + # the caller's extracted tree changed before or during the copy, the + # private snapshot fails closed; later caller mutations are irrelevant. + verify_checksum_list(snapshot_root) + except Exception: + shutil.rmtree(snapshot_root, ignore_errors=True) + raise + return snapshot_root + + def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: chain_path = bundle_root / "CHAIN.csv" try: @@ -601,10 +1190,12 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: rows = list(csv.DictReader(source)) except OSError as exc: raise ota.OtaError(f"cannot read {chain_path}: {exc}") from exc - if len(rows) not in (EXPECTED_STEP_COUNT, 26, 27, 29, 30): + if len(rows) not in ( + EXPECTED_STEP_COUNT, PHYSICALLY_QUALIFIED_9_STEP_COUNT, 26, 27, 29, 30 + ): raise ota.OtaError( f"chain contains {len(rows)} steps, expected {EXPECTED_STEP_COUNT}, " - "26, 27, 29, or 30" + f"{PHYSICALLY_QUALIFIED_9_STEP_COUNT}, 26, 27, 29, or 30" ) steps: list[ChainStep] = [] @@ -655,12 +1246,19 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: if steps[0].from_version != EXPECTED_START_VERSION: raise ota.OtaError("chain has an unexpected starting version") if len(steps) == EXPECTED_STEP_COUNT: + for number, expected_sha256 in CURRENT_10_CANDIDATE_ANCHORS: + if steps[number - 1].target_sha256 != expected_sha256: + raise ota.OtaError( + f"ten-step candidate step {number} does not match its audited image pin" + ) + expected_final_version = EXPECTED_FINAL_VERSION + elif len(steps) == PHYSICALLY_QUALIFIED_9_STEP_COUNT: for number, expected_sha256 in COMPACT_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: raise ota.OtaError( f"compact release step {number} does not match its audited image pin" ) - expected_final_version = EXPECTED_FINAL_VERSION + expected_final_version = PHYSICALLY_QUALIFIED_9_FINAL_VERSION elif len(steps) == 30: for number, expected_sha256 in PINNED_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: @@ -806,6 +1404,7 @@ def source_namespace(args: argparse.Namespace) -> SimpleNamespace: source_already_temp=args.source_already_temp, source_shares_controller=args.source_shares_controller, source_companion_terminal=False, + temp_values=args.temp_values, source_baud=args.source_baud, controller_baud=args.controller_baud, reply_timeout=args.reply_timeout, @@ -813,6 +1412,29 @@ def source_namespace(args: argparse.Namespace) -> SimpleNamespace: ) +def restore_persisted_source_rxps( + work_dir: Path, + source_args: SimpleNamespace, +) -> None: + """Recover the chain-start source preference after a killed nested step.""" + recovery_path = work_dir / ota.SOURCE_RXPS_RECOVERY_FILE + if not recovery_path.exists() and not recovery_path.is_symlink(): + return + saved = ota.read_source_rxps_recovery(recovery_path, source_args) + if not ota.shorten_source_temp_window(source_args): + raise ota.OtaError( + "could not prove the OTA source is back on its normal radio before " + "retiring persisted RXPS recovery" + ) + current = ota.read_source_rxps(source_args) + if current == saved: + print("[rxps] OTA source matches the persisted chain-start setting") + else: + ota.restore_source_rxps(source_args, saved) + ota.retire_source_rxps_recovery(recovery_path) + print("[rxps] retired completed chain source recovery record") + + def query_live_target( controller: ota.Controller, args: argparse.Namespace, @@ -949,12 +1571,30 @@ def load_or_capture_transfer_settings( ) -> TargetTransferSettings: path = work_dir / TRANSFER_SETTINGS_FILE if path.exists(): + if path.is_symlink() or not path.is_file(): + raise ota.OtaError( + f"saved transfer settings are not a private regular file: {path}" + ) + if stat.S_IMODE(path.stat().st_mode) & 0o077: + raise ota.OtaError( + f"saved transfer settings are not private (0600): {path}" + ) try: saved = json.loads(path.read_text(encoding="ascii")) - if saved["target_key"].lower() != target_key.lower(): + saved_target_key = saved["target_key"] + if ( + not isinstance(saved_target_key, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", saved_target_key) is None + ): + raise TypeError("saved target key must be a 64-hex string") + if saved_target_key.lower() != target_key.lower(): raise ota.OtaError( f"{path} belongs to a different destination public key" ) + if not isinstance(saved["rxdelay"], str) or not isinstance( + saved["airtime_factor"], str + ): + raise TypeError("saved delay and airtime values must be strings") if not isinstance(saved["rxps_enabled"], bool) or not isinstance( saved["powersaving_enabled"], bool ): @@ -986,7 +1626,9 @@ def load_or_capture_transfer_settings( airtime_factor=str(saved["airtime_factor"]), ota_hops=saved["ota_hops"], ) - float(settings.rxdelay) + rxdelay = float(settings.rxdelay) + if not math.isfinite(rxdelay) or rxdelay < 0.0: + raise ValueError("saved RX delay is invalid") airtime_factor = float(settings.airtime_factor) if not math.isfinite(airtime_factor) or airtime_factor < 0.0: raise ValueError("saved airtime factor is invalid") @@ -1029,10 +1671,9 @@ def load_or_capture_transfer_settings( "airtime_factor": settings.airtime_factor, "ota_hops": settings.ota_hops, } - temporary = path.with_suffix(path.suffix + ".tmp") - temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="ascii") - temporary.chmod(0o600) - os.replace(temporary, path) + ota.write_private_recovery_file( + path, json.dumps(payload, indent=2, sort_keys=True) + "\n" + ) print(f"[guardrail] saved original destination settings to {path}") return settings @@ -1186,15 +1827,55 @@ def restore_transfer_settings( ): raise ota.OtaError("destination radio transfer settings did not restore exactly") - if saved.powersaving_enabled: - reply = controller.remote_command(target_name, "powersaving on") - if re.search(r"\bon\b", reply, re.IGNORECASE) is None: + if verified.powersaving_enabled != saved.powersaving_enabled: + desired = "on" if saved.powersaving_enabled else "off" + reply = controller.remote_command(target_name, f"powersaving {desired}") + if re.search(rf"\b{desired}\b", reply, re.IGNORECASE) is None: raise ota.OtaError(f"target did not restore CPU power saving: {reply}") - elif verified.powersaving_enabled: - raise ota.OtaError("destination CPU power saving unexpectedly remained enabled") + + final = read_target_transfer_settings(controller, target_name) + if ( + final.rxps_enabled != saved.rxps_enabled + or final.powersaving_enabled != saved.powersaving_enabled + or abs(float(final.rxdelay) - float(saved.rxdelay)) > 0.0001 + or abs(float(final.airtime_factor) - float(saved.airtime_factor)) > 0.0001 + or final.ota_hops != saved.ota_hops + or ( + saved.rxps_enabled + and ( + final.rxps_rx_us != saved.rxps_rx_us + or final.rxps_sleep_us != saved.rxps_sleep_us + or ( + saved.rxps_level is not None + and ( + final.rxps_level != saved.rxps_level + or final.rxps_preamble != saved.rxps_preamble + ) + ) + ) + ) + ): + raise ota.OtaError( + "destination transfer settings changed during final restoration" + ) print("[guardrail] original destination transfer settings restored") +def restore_and_retire_transfer_settings( + controller: ota.Controller, + target_name: str, + saved: TargetTransferSettings, + work_dir: Path, +) -> None: + """Restore exactly, then disarm the persistent resume record atomically.""" + restore_transfer_settings(controller, target_name, saved) + path = work_dir / TRANSFER_SETTINGS_FILE + ota.retire_private_recovery_file( + path, "destination transfer-settings recovery record" + ) + print("[guardrail] retired completed destination settings recovery record") + + def read_ota_hops(controller: ota.Controller, target_name: str) -> int: reply = controller.remote_command(target_name, "ota config") match = re.search(r"\bhops=(\d+)\b", reply) @@ -1237,6 +1918,17 @@ def require_rescue_capability( print("[rescue] guarded no-EndF recovery command is present") +def require_rescue_capability_before_next_transition( + controller: ota.Controller, + target_name: str, + next_step_index: int, + step_count: int, +) -> None: + """Gate a bridge at a chain position before exposing it to another package.""" + if 0 < next_step_index < step_count: + require_rescue_capability(controller, target_name) + + def wait_with_label(seconds: int, label: str) -> None: deadline = time.monotonic() + seconds while True: @@ -1276,13 +1968,81 @@ def prepare_watchdog( print("[watchdog] off and target remained responsive through the stability window") +def validate_chain_state_paths(work_dir: Path) -> None: + """Reject reused state paths that could redirect or block chain writes.""" + steps_path = work_dir / "steps" + progress_path = work_dir / "progress.jsonl" + for path, expected_kind in ( + (steps_path, "directory"), + (progress_path, "regular file"), + ): + try: + metadata = path.lstat() + except FileNotFoundError: + continue + if expected_kind == "directory": + valid = stat.S_ISDIR(metadata.st_mode) + else: + valid = stat.S_ISREG(metadata.st_mode) and metadata.st_nlink == 1 + if not valid: + raise ota.OtaError( + f"chain state path must be a real {expected_kind}: {path}" + ) + if path == progress_path and metadata.st_size > MAX_PROGRESS_BYTES: + raise ota.OtaError( + f"chain progress log exceeds {MAX_PROGRESS_BYTES} bytes: {path}" + ) + + +def open_steps_directory(work_dir: Path) -> int: + parent = work_dir / "steps" + try: + parent.mkdir(mode=0o700) + except FileExistsError: + pass + try: + path_metadata = parent.lstat() + except OSError as exc: + raise ota.OtaError(f"cannot inspect chain steps path: {parent}") from exc + if not stat.S_ISDIR(path_metadata.st_mode): + raise ota.OtaError(f"chain steps path is not a real directory: {parent}") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(parent, flags) + except OSError as exc: + raise ota.OtaError(f"chain steps path is not a real directory: {parent}") from exc + metadata = os.fstat(descriptor) + try: + current_path_metadata = parent.lstat() + except OSError as exc: + os.close(descriptor) + raise ota.OtaError(f"chain steps path changed while opening: {parent}") from exc + if ( + not stat.S_ISDIR(metadata.st_mode) + or not os.path.samestat(metadata, path_metadata) + or not os.path.samestat(metadata, current_path_metadata) + ): + os.close(descriptor) + raise ota.OtaError(f"chain steps path is not one stable directory: {parent}") + return descriptor + + def next_attempt_dir(work_dir: Path, step_number: int) -> Path: parent = work_dir / "steps" - parent.mkdir(exist_ok=True) - for attempt in range(1, 1000): - candidate = parent / f"step-{step_number:02d}-attempt-{attempt:03d}" - if not candidate.exists(): - return candidate + descriptor = open_steps_directory(work_dir) + try: + for attempt in range(1, 1000): + name = f"step-{step_number:02d}-attempt-{attempt:03d}" + try: + os.mkdir(name, mode=0o700, dir_fd=descriptor) + except FileExistsError: + continue + # Reserve a private real directory first. lora_ota.main creates + # its own one-use work directory below it with exist_ok=False. + return parent / name / "work" + finally: + os.close(descriptor) raise ota.OtaError(f"too many saved attempts for step {step_number}") @@ -1298,8 +2058,55 @@ def append_progress( "to_version": step.to_version, "body_hash": body_hash.hex().upper(), } - with (work_dir / "progress.jsonl").open("a", encoding="ascii") as output: - output.write(json.dumps(record, sort_keys=True) + "\n") + path = work_dir / "progress.jsonl" + data = (json.dumps(record, sort_keys=True) + "\n").encode("ascii") + try: + path_metadata: os.stat_result | None = path.lstat() + except FileNotFoundError: + path_metadata = None + if path_metadata is not None and stat.S_ISLNK(path_metadata.st_mode): + raise ota.OtaError(f"chain progress path is a symbolic link: {path}") + flags = ( + os.O_WRONLY + | os.O_APPEND + | os.O_CREAT + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + try: + descriptor = os.open(path, flags, 0o600) + except OSError as exc: + raise ota.OtaError(f"cannot safely open chain progress log {path}: {exc}") from exc + try: + metadata = os.fstat(descriptor) + try: + current_path_metadata = path.lstat() + except OSError as exc: + raise ota.OtaError( + f"chain progress path changed while opening: {path}" + ) from exc + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_nlink != 1 + or not os.path.samestat(metadata, current_path_metadata) + or ( + path_metadata is not None + and not os.path.samestat(metadata, path_metadata) + ) + ): + raise ota.OtaError( + f"chain progress path is not a private regular file: {path}" + ) + if metadata.st_size + len(data) > MAX_PROGRESS_BYTES: + raise ota.OtaError( + f"chain progress log would exceed {MAX_PROGRESS_BYTES} bytes" + ) + if os.write(descriptor, data) != len(data): + raise ota.OtaError("short write while appending chain progress") + os.fsync(descriptor) + finally: + os.close(descriptor) def clear_completed_download( @@ -1307,7 +2114,16 @@ def clear_completed_download( target_name: str, step: ChainStep, ) -> None: - """Clear only a retained record for the exact, proven installed step.""" + """After exact body proof, detach only a visible completed manager session. + + A normal-channel ``no download`` proves that the manager is idle; it does + not prove that a persistent store is empty. In particular, the legacy + internal-flash nRF52 store's ``clear()`` reset only RAM. Do not turn an + IDLE ``ota cancel`` acknowledgement into a false durable-erasure claim. + The next transition explicitly re-adopts, proves, and cancels the previous + MID on TempRadio before beginning its new pull; a successfully consumed + final package is inert because OTAFIX has cleared its approval word. + """ status = controller.remote_command(target_name, "ota status") active_id = ota.download_manifest_id(status) if active_id is None: @@ -1316,13 +2132,17 @@ def clear_completed_download( f"post-install download state is ambiguous after step {step.number}: " f"{status}" ) + print( + f"[chain] step {step.number:02d} manager is idle; persistent " + "staging is not inferred from normal-channel status" + ) return - if active_id != step.package.manifest_id: + elif active_id != step.package.manifest_id: raise ota.OtaError( f"post-install target retained mOTA {active_id} after step " f"{step.number}; expected only {step.package.manifest_id}" ) - if "ready to install" not in status.lower(): + elif "ready to install" not in status.lower(): raise ota.OtaError( f"post-install target still has an active step-{step.number} session: " f"{status}" @@ -1337,7 +2157,10 @@ def clear_completed_download( raise ota.OtaError( f"completed step-{step.number} staging record remains: {status}" ) - print(f"[chain] cleared retained staging record for step {step.number:02d}") + print( + f"[chain] detached completed step-{step.number:02d} manager session; " + "persistent erasure is not inferred" + ) def connection_arguments(args: argparse.Namespace) -> list[str]: @@ -1390,6 +2213,8 @@ def run_step( "--transfer-timeout-minutes", str(args.transfer_timeout_minutes), "--seeder-start-wait", str(args.seeder_start_wait), "--reboot-wait", str(args.reboot_wait), + "--source-rxps-recovery-file", + str(work_dir / ota.SOURCE_RXPS_RECOVERY_FILE), "--work-dir", str(next_attempt_dir(work_dir, step.number)), "--require-system-watchdog-off", "--expected-installed-body-hash", expected_body_hash.hex().upper(), @@ -1404,6 +2229,8 @@ def run_step( "--clear-completed-manifest", previous_step.package.manifest_id, "--clear-completed-on-body-hash", step.base_hash.hex().upper(), ]) + if args.debug: + command.append("--debug") for relay in args.relay: command.extend(["--relay", relay]) result = ota.main(command, controller_override=controller) @@ -1414,14 +2241,16 @@ def run_step( def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( - "Download, verify, resume, and install the exact compact 9-step " - "RAK3401 mOTA chain from c1caa5ad to e742333a." + "Verify the exact local ten-step RAK3401 candidate from c1caa5ad " + "to fd98bc90. Its exact package transitions passed directly on " + "hardware, but this later host-runner revision has not had a clean " + "end-to-end physical rerun and normal live use remains disabled." ) ) parser.add_argument( "--bundle", type=Path, - help="pinned release ZIP or extracted bundle root (downloads it when omitted)", + help="pinned local ZIP or extracted bundle root (required until release)", ) parser.add_argument( "--work-dir", @@ -1443,8 +2272,9 @@ def build_parser() -> argparse.ArgumentParser: source = parser.add_mutually_exclusive_group() source.add_argument("--source-serial") source.add_argument("--source-tcp") - parser.add_argument("--source-cli-serial") - parser.add_argument("--source-cli-tcp") + source_cli = parser.add_mutually_exclusive_group() + source_cli.add_argument("--source-cli-serial") + source_cli.add_argument("--source-cli-tcp") parser.add_argument("--source-already-temp", action="store_true") parser.add_argument("--source-shares-controller", action="store_true") @@ -1539,6 +2369,12 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> ) if args.source_already_temp and not args.source_tcp: parser.error("--source-already-temp requires --source-tcp") + if args.source_already_temp and ( + args.source_cli_serial or args.source_cli_tcp + ): + parser.error( + "--source-already-temp cannot be combined with a managed source CLI" + ) if args.source_shares_controller and not ( args.source_tcp and args.source_cli_tcp ): @@ -1549,6 +2385,14 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> parser.error( "--source-shares-controller and --source-already-temp are mutually exclusive" ) + if args.controller_serial and args.source_serial and ota.serial_paths_match( + args.controller_serial, args.source_serial + ): + parser.error("controller and source must be separate nodes/serial ports") + if args.controller_serial and args.source_cli_serial and ota.serial_paths_match( + args.controller_serial, args.source_cli_serial + ): + parser.error("controller and source CLI must use separate serial ports") for name in ( "controller_baud", "source_baud", "reply_timeout", "discovery_timeout", "discovery_interval", "poll_seconds", "transfer_timeout_minutes", @@ -1648,15 +2492,34 @@ def main(argv: list[str] | None = None) -> int: validate_args(args, parser) work_dir = args.work_dir.resolve() work_dir.mkdir(parents=True, exist_ok=True) + validate_chain_state_paths(work_dir) + snapshot_parent: Path | None = None try: - bundle_root = locate_bundle(args, work_dir) - verify_checksum_list(bundle_root) + supplied_bundle_root = locate_bundle(args, work_dir) + require_bundle_work_separation(supplied_bundle_root, work_dir) + snapshot_parent = Path(tempfile.mkdtemp( + prefix=".verified-bundle-", dir=work_dir + )) + snapshot_parent.chmod(0o700) + bundle_root = snapshot_verified_bundle( + supplied_bundle_root, snapshot_parent + ) steps, final_body_hash = parse_chain(bundle_root) verify_motatool(args, steps) if args.verify_only: - print(f"Verified release bundle: {bundle_root}") + print(f"Verified pinned bundle: {supplied_bundle_root}") if len(steps) == EXPECTED_STEP_COUNT: + print( + "Ten-step candidate verified offline: exact archive/checksum " + "pins, all target anchors, zero-filled and erased-workspace " + "reconstruction, independent motatool verification, and " + "bootloader simulation passed. These exact package transitions " + "also passed directly on the RAK3401 and the endpoint passed " + "independent SWD readback. This later host-runner revision has " + "not had a new clean physical rerun; live use remains gated." + ) + elif len(steps) == PHYSICALLY_QUALIFIED_9_STEP_COUNT: print( "Compact release verified: all 9 transitions passed its exact " "physical RAK3401 run, zero-filled and erased-workspace " @@ -1710,18 +2573,22 @@ def main(argv: list[str] | None = None) -> int: target_name, full_key = resolve_target_by_key(controller, args.target_key) target = query_live_target(controller, args, target_name) first_index = find_resume_index(target, steps, final_body_hash) + require_rescue_capability_before_next_transition( + controller, target_name, first_index, len(steps) + ) confirm_chain(args, target_name, full_key, target, first_index, steps) if args.preflight_only: return 0 if first_index == len(steps): + restore_persisted_source_rxps(work_dir, source_args) transfer_path = work_dir / TRANSFER_SETTINGS_FILE if transfer_path.exists(): transfer_settings = load_or_capture_transfer_settings( controller, target_name, full_key, work_dir ) - restore_transfer_settings( - controller, target_name, transfer_settings + restore_and_retire_transfer_settings( + controller, target_name, transfer_settings, work_dir ) if not args.keep_watchdog_off: enabled = controller.remote_command(target_name, "set system.watchdog on") @@ -1829,7 +2696,9 @@ def main(argv: list[str] | None = None) -> int: f"runtime label {target.current_version} is historical and is " f"not the EndF chain version {step.to_version}" ) - require_rescue_capability(controller, target_name) + require_rescue_capability_before_next_transition( + controller, target_name, index + 1, len(steps) + ) clear_completed_download(controller, target_name, step) require_watchdog_state(controller, target_name, "off") append_progress(work_dir, step, target.base_hash) @@ -1838,7 +2707,10 @@ def main(argv: list[str] | None = None) -> int: final_target = query_live_target(controller, args, target_name) if find_resume_index(final_target, steps, final_body_hash) != len(steps): raise ota.OtaError("final target identity did not match the release endpoint") - restore_transfer_settings(controller, target_name, transfer_settings) + restore_persisted_source_rxps(work_dir, source_args) + restore_and_retire_transfer_settings( + controller, target_name, transfer_settings, work_dir + ) if not args.keep_watchdog_off: enabled = controller.remote_command(target_name, "set system.watchdog on") if not enabled.lower().startswith("ok - system watchdog enabled"): @@ -1870,6 +2742,9 @@ def main(argv: list[str] | None = None) -> int: file=sys.stderr, ) return 2 + finally: + if snapshot_parent is not None: + shutil.rmtree(snapshot_parent, ignore_errors=True) if __name__ == "__main__": diff --git a/tools/lora_ota/rak3401_route_search.py b/tools/lora_ota/rak3401_route_search.py new file mode 100755 index 00000000..1618e444 --- /dev/null +++ b/tools/lora_ota/rak3401_route_search.py @@ -0,0 +1,638 @@ +#!/usr/bin/env python3 +"""Exhaustively select a compact legacy-ceiling RAK3401 mOTA route. + +Cache entries are keyed by source SHA, target SHA, and workspace size rather +than inventory indices. A cache generated for an older inventory can therefore +be migrated safely when new candidate images are inserted. +""" +from __future__ import annotations + +import argparse +import csv +from concurrent.futures import ProcessPoolExecutor, as_completed +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sys +import tempfile + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent +sys.path.insert(0, str(REPO_ROOT / "tools" / "mota")) + +import motalib # noqa: E402 + +APP_BASE = 0x26000 +STAGE_CEILING = 0xD4000 +PAGE = 4096 +SEGMENT = PAGE +FIXED_MEMORY = 0x98000 +AVAILABLE_PAGES = (STAGE_CEILING - APP_BASE) // PAGE +EXPECTED_TARGET_ID = 0x2FA509C1 +EXPECTED_HARDWARE = "RAK_3401" +FIELDS = [ + "source", "target", "source_sha256", "target_sha256", "memory", + "payload", "container", "stage_start", "margin", "feasible", "error", +] + + +class RouteSearchError(RuntimeError): + """An invalid input or internally inconsistent cache.""" + + +def align_up(value: int, unit: int = PAGE) -> int: + return (value + unit - 1) // unit * unit + + +def align_down(value: int, unit: int = PAGE) -> int: + return value // unit * unit + + +def container_size(payload_size: int) -> int: + return payload_size + 210 + 4 * ((payload_size + 1023) // 1024) + + +def truth(value: object) -> bool: + return value is True or str(value).lower() == "true" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _integer(value: object, label: str) -> int: + if isinstance(value, int) and not isinstance(value, bool): + result = value + elif isinstance(value, str) and re.fullmatch(r"[0-9]+", value): + result = int(value) + else: + raise RouteSearchError(f"{label} must be an integer") + if result < 0: + raise RouteSearchError(f"{label} cannot be negative") + return result + + +def load_inventory(path: Path) -> list[dict[str, object]]: + try: + document = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RouteSearchError(f"cannot read image inventory {path}: {exc}") from exc + images = document.get("images") if isinstance(document, dict) else None + if not isinstance(images, list) or len(images) < 2: + raise RouteSearchError(f"invalid image inventory: {path}") + + result: list[dict[str, object]] = [] + seen: set[str] = set() + for index, raw in enumerate(images): + if not isinstance(raw, dict): + raise RouteSearchError(f"inventory image {index} is not an object") + record = dict(raw) + if _integer(record.get("node", index), f"inventory node {index}") != index: + raise RouteSearchError( + "inventory nodes must be ordered, contiguous, and zero-based" + ) + raw_path = record.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise RouteSearchError(f"inventory image {index} has no path") + image_path = Path(raw_path) + if not image_path.is_absolute(): + image_path = path.parent / image_path + image_path = image_path.resolve() + if not image_path.is_file(): + raise RouteSearchError(f"missing inventory image: {image_path}") + try: + image = image_path.read_bytes() + except OSError as exc: + raise RouteSearchError(f"cannot read inventory image: {image_path}") from exc + actual_sha = hashlib.sha256(image).hexdigest() + expected_sha = record.get("sha256") + if not isinstance(expected_sha, str) or len(expected_sha) != 64 or any( + character not in "0123456789abcdefABCDEF" for character in expected_sha + ): + raise RouteSearchError(f"image {index} SHA-256 is invalid") + expected_sha = expected_sha.lower() + if actual_sha != expected_sha: + raise RouteSearchError(f"image {index} SHA mismatch") + if expected_sha in seen: + raise RouteSearchError(f"duplicate image SHA at node {index}") + seen.add(expected_sha) + actual_size = len(image) + if _integer(record.get("size"), f"image {index} size") != actual_size: + raise RouteSearchError(f"image {index} size mismatch") + if not motalib.has_endf(image): + raise RouteSearchError(f"image {index} has no valid EndF") + ident = motalib.parse_endf_ident(image) + if ( + ident is None + or ident.target_id != EXPECTED_TARGET_ID + or ident.hw_id != EXPECTED_HARDWARE + ): + raise RouteSearchError(f"image {index} has the wrong RAK3401 identity") + _body, body_hash = motalib.parse_endf(image) + declared_body_hash = record.get("body_hash") + if ( + not isinstance(declared_body_hash, str) + or not re.fullmatch(r"[0-9a-f]{16}", declared_body_hash) + or declared_body_hash != body_hash.hex() + ): + raise RouteSearchError(f"image {index} body hash mismatch") + version = motalib.unpack_version(ident.fw_version) + if record.get("version") != version: + raise RouteSearchError(f"image {index} version mismatch") + source_commit = record.get("source_commit") + if source_commit is not None and ( + not isinstance(source_commit, str) + or not re.fullmatch(r"[0-9a-f]{40}", source_commit) + ): + raise RouteSearchError(f"image {index} source commit is invalid") + if index == 1 and _integer( + record.get("baseline_container_size"), + "node 1 baseline_container_size", + ) <= 0: + raise RouteSearchError("node 1 baseline_container_size must be positive") + record.update(node=index, path=str(image_path), sha256=expected_sha, size=actual_size) + result.append(record) + return result + + +def snapshot_inventory(source: Path, destination: Path) -> Path: + """Copy an inventory and its images before starting parallel patch work.""" + try: + document = json.loads(source.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RouteSearchError(f"cannot read image inventory {source}: {exc}") from exc + images = document.get("images") if isinstance(document, dict) else None + if not isinstance(images, list) or len(images) < 2: + raise RouteSearchError(f"invalid image inventory: {source}") + image_directory = destination.parent / "images" + image_directory.mkdir(parents=True, exist_ok=False) + for index, raw in enumerate(images): + if not isinstance(raw, dict): + raise RouteSearchError(f"inventory image {index} is not an object") + raw_path = raw.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise RouteSearchError(f"inventory image {index} has no path") + image_source = Path(raw_path) + if not image_source.is_absolute(): + image_source = source.parent / image_source + image_destination = image_directory / f"image-{index:03d}.bin" + try: + shutil.copy2(image_source, image_destination, follow_symlinks=True) + except OSError as exc: + raise RouteSearchError( + f"cannot snapshot inventory image {index}: {image_source}: {exc}" + ) from exc + raw["path"] = str(Path("images") / image_destination.name) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + # Validate the copied bytes against every declared content and EndF pin. + load_inventory(destination) + return destination + + +def insert_bridges_before_endpoint( + document: dict[str, object], bridges: list[dict[str, object]] +) -> dict[str, object]: + """Insert ordered bridge records immediately before the existing endpoint.""" + existing = document.get("images") + if ( + not isinstance(existing, list) + or len(existing) < 2 + or not all(isinstance(item, dict) for item in existing) + ): + raise RouteSearchError("inventory document has no valid start and endpoint") + if not all(isinstance(item, dict) for item in bridges): + raise RouteSearchError("inventory bridges must be objects") + # Copy every record, preserving the last existing record as the endpoint. + ordered = [dict(item) for item in existing[:-1]] + ordered.extend(dict(item) for item in bridges) + ordered.append(dict(existing[-1])) + images = [] + for node, record in enumerate(ordered): + record["node"] = node + images.append(record) + result = dict(document) + result["images"] = images + return result + + +def cache_key(source_sha: str, target_sha: str, memory: int) -> tuple[str, str, int]: + return source_sha.lower(), target_sha.lower(), memory + + +def migrate_csv( + inventory_path: Path, csv_path: Path +) -> dict[tuple[str, str, int], dict[str, object]]: + """Migrate numeric CSV indices through the inventory that created them.""" + images = load_inventory(inventory_path) + migrated: dict[tuple[str, str, int], dict[str, object]] = {} + try: + source = csv_path.open(newline="", encoding="ascii") + except OSError as exc: + raise RouteSearchError(f"cannot read geometry cache {csv_path}: {exc}") from exc + with source: + reader = csv.DictReader(source) + required_fields = {"source", "target", "memory"} + if reader.fieldnames is None or not required_fields.issubset(reader.fieldnames): + raise RouteSearchError(f"geometry cache has missing columns: {csv_path}") + for line, row in enumerate(reader, 2): + source_node = _integer(row["source"], f"source at {csv_path}:{line}") + target_node = _integer(row["target"], f"target at {csv_path}:{line}") + if not (0 <= source_node < target_node < len(images)): + raise RouteSearchError(f"bad nodes at {csv_path}:{line}") + source_sha = str(images[source_node]["sha256"]) + target_sha = str(images[target_node]["sha256"]) + memory = _integer(row["memory"], f"memory at {csv_path}:{line}") + for column, expected in ( + ("source_sha256", source_sha), ("target_sha256", target_sha) + ): + raw_recorded = row.get(column, "") + if raw_recorded is None: + raw_recorded = "" + if not isinstance(raw_recorded, str): + raise RouteSearchError( + f"{column} is invalid at {csv_path}:{line}" + ) + recorded = raw_recorded.lower() + if recorded and recorded != expected: + raise RouteSearchError( + f"{column} disagrees with inventory at {csv_path}:{line}" + ) + try: + payload = int(row.get("payload", "")) + total = int(row.get("container", "")) + stage_start = int(row.get("stage_start", "")) + margin = int(row.get("margin", "")) + except (TypeError, ValueError) as exc: + raise RouteSearchError( + f"invalid numeric geometry at {csv_path}:{line}" + ) from exc + if payload < 0: + raise RouteSearchError( + f"failed cached geometry cannot prove an exhaustive search at " + f"{csv_path}:{line}; regenerate it" + ) + if row.get("error", ""): + raise RouteSearchError( + f"successful cached geometry carries an error at " + f"{csv_path}:{line}; regenerate it" + ) + expected_total = container_size(payload) + expected_stage = align_down(STAGE_CEILING - expected_total) + expected_margin = expected_stage - (APP_BASE + memory) + if (total, stage_start, margin) != ( + expected_total, expected_stage, expected_margin + ): + raise RouteSearchError( + f"inconsistent cached geometry at {csv_path}:{line}" + ) + if truth(row.get("feasible", False)) != (payload >= 0 and margin >= 0): + raise RouteSearchError( + f"cached feasibility is inconsistent at {csv_path}:{line}" + ) + key = cache_key(source_sha, target_sha, memory) + normalized = {field: row.get(field, "") for field in FIELDS} + normalized.update(source=source_node, target=target_node, + source_sha256=source_sha, target_sha256=target_sha, + memory=memory) + previous = migrated.get(key) + comparable = ("payload", "container", "stage_start", "margin", "feasible", "error") + if previous is not None and any( + str(previous.get(item, "")) != str(normalized.get(item, "")) + for item in comparable + ): + raise RouteSearchError(f"conflicting cached geometry for {key}") + migrated[key] = normalized + return migrated + + +Job = tuple[int, int, int, str, str, str, str] + + +def all_jobs(images: list[dict[str, object]]) -> list[Job]: + """Enumerate every valid page-aligned workspace for every forward edge.""" + jobs: list[Job] = [] + endpoint = len(images) - 1 + # Node 0 may only use the byte-identical, physically qualified 0->1 package. + # Node 1 still runs the fixed-workspace receiver, so measure each of its + # possible forward transitions at exactly that legacy workspace. + for target in range(2, endpoint + 1): + jobs.append((1, target, FIXED_MEMORY, str(images[1]["path"]), + str(images[target]["path"]), str(images[1]["sha256"]), + str(images[target]["sha256"]))) + for source in range(2, endpoint): + source_size = int(images[source]["size"]) + for target in range(source + 1, endpoint + 1): + target_size = int(images[target]["size"]) + first_page = align_up(max(source_size + 2 * PAGE, target_size)) // PAGE + for memory_page in range(first_page, AVAILABLE_PAGES): + jobs.append((source, target, memory_page * PAGE, + str(images[source]["path"]), str(images[target]["path"]), + str(images[source]["sha256"]), str(images[target]["sha256"]))) + return jobs + + +def geometry_job(args: tuple[int, int, int, str, str, str, str, str]) -> dict[str, object]: + try: + import detools + except ImportError as exc: + raise RouteSearchError( + "detools is required to generate geometry; run this tool in the " + "detools pipx environment" + ) from exc + source, target, memory, from_raw, to_raw, source_sha, target_sha, patches_raw = args + patch = Path(patches_raw) / f"{source_sha}-{target_sha}-{memory // PAGE:03d}.patch" + try: + detools.create_patch_filenames( + from_raw, to_raw, str(patch), compression="crle", patch_type="in-place", + algorithm="bsdiff", suffix_array_algorithm="divsufsort", + memory_size=memory, segment_size=SEGMENT, use_mmap=True, + ) + payload = patch.stat().st_size + total = container_size(payload) + stage_start = align_down(STAGE_CEILING - total) + margin = stage_start - (APP_BASE + memory) + error = "" + except Exception as exc: # detools reports several backend exception types + detail = f"{type(exc).__name__}: {exc}"[-500:].replace("\n", " ") + raise RouteSearchError( + f"detools geometry failed for {source}->{target} at 0x{memory:X}: {detail}" + ) from exc + finally: + patch.unlink(missing_ok=True) + return {"source": source, "target": target, "source_sha256": source_sha, + "target_sha256": target_sha, "memory": memory, "payload": payload, + "container": total, "stage_start": stage_start, "margin": margin, + "feasible": payload >= 0 and margin >= 0, "error": error} + + +def project_cache(cache: dict[tuple[str, str, int], dict[str, object]], + images: list[dict[str, object]], + required: set[tuple[str, str, int]]) -> list[dict[str, object]]: + index = {str(record["sha256"]): int(record["node"]) for record in images} + rows: list[dict[str, object]] = [] + for key in sorted(required, key=lambda item: (index[item[0]], index[item[1]], item[2])): + if key not in cache: + raise RouteSearchError(f"required cache geometry is missing: {key}") + row = dict(cache[key]) + if int(row.get("payload", -1)) < 0 or str(row.get("error", "")): + raise RouteSearchError( + f"required cache geometry is not a successful measurement: {key}" + ) + row.update(source=index[key[0]], target=index[key[1]], + source_sha256=key[0], target_sha256=key[1], memory=key[2]) + rows.append(row) + return rows + + +def write_atomic_text(path: Path, contents: str) -> None: + """Replace one regular output without ever following its pathname symlink.""" + if path.is_symlink() or (path.exists() and not path.is_file()): + raise RouteSearchError(f"output path is not a regular file: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", dir=path.parent + ) + temporary = Path(temporary_name) + descriptor_open = True + try: + with os.fdopen( + descriptor, "w", encoding="ascii", newline="\n" + ) as output: + descriptor_open = False + output.write(contents) + output.flush() + os.fsync(output.fileno()) + temporary.chmod(0o600) + os.replace(temporary, path) + except BaseException: + if descriptor_open: + try: + os.close(descriptor) + except OSError: + pass + temporary.unlink(missing_ok=True) + raise + + +def write_csv(path: Path, rows: list[dict[str, object]]) -> None: + if path.is_symlink() or (path.exists() and not path.is_file()): + raise RouteSearchError(f"output path is not a regular file: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", dir=path.parent + ) + temporary = Path(temporary_name) + descriptor_open = True + try: + with os.fdopen(descriptor, "w", newline="", encoding="ascii") as output: + descriptor_open = False + writer = csv.DictWriter( + output, fieldnames=FIELDS, lineterminator="\n", extrasaction="ignore" + ) + writer.writeheader() + writer.writerows(rows) + output.flush() + os.fsync(output.fileno()) + temporary.chmod(0o600) + os.replace(temporary, path) + except BaseException: + if descriptor_open: + try: + os.close(descriptor) + except OSError: + pass + temporary.unlink(missing_ok=True) + raise + + +def select_route(rows: list[dict[str, object]], images: list[dict[str, object]], + baseline_size: int, output: Path, complete: bool) -> dict[str, object]: + endpoint = len(images) - 1 + baseline_stage = align_down(STAGE_CEILING - baseline_size) + baseline_margin = baseline_stage - (APP_BASE + FIXED_MEMORY) + if baseline_margin < 0: + raise RouteSearchError("pinned first package does not fit") + edges: dict[tuple[int, int], dict[str, object]] = { + (0, 1): {"source": 0, "target": 1, "memory": FIXED_MEMORY, + "container": baseline_size, "stage_start": baseline_stage, + "margin": baseline_margin, "feasible": True} + } + for row in rows: + if int(row["source"]) == 0: + # Older evidence tables measured these irrelevant shortcuts. Keep + # them countable for archive verification, but never admit them to + # the route graph; node 0 is pinned unconditionally to node 1. + continue + if not truth(row.get("feasible", False)): + continue + key = int(row["source"]), int(row["target"]) + rank = (int(row["container"]), -int(row["margin"]), int(row["memory"])) + old = edges.get(key) + if old is None or rank < (int(old["container"]), -int(old["margin"]), int(old["memory"])): + edges[key] = row + + best: dict[int, tuple[int, int, list[int]]] = {0: (0, 0, [0])} + hop_distance = [sys.maxsize] * len(images) + route_count = [0] * len(images) + hop_distance[0], route_count[0] = 0, 1 + outgoing: dict[int, list[tuple[int, dict[str, object]]]] = {} + for (source, target), row in edges.items(): + outgoing.setdefault(source, []).append((target, row)) + for source in range(endpoint): + if source not in best: + continue + for target, row in outgoing.get(source, []): + candidate = (best[source][0] + 1, + best[source][1] + int(row["container"]), + best[source][2] + [target]) + if target not in best or candidate < best[target]: + best[target] = candidate + distance = hop_distance[source] + 1 + if distance < hop_distance[target]: + hop_distance[target], route_count[target] = distance, route_count[source] + elif distance == hop_distance[target]: + route_count[target] += route_count[source] + + common = {"schema": 2, "app_base": f"0x{APP_BASE:X}", + "stage_ceiling": f"0x{STAGE_CEILING:X}", "node_count": len(images), + "search_complete": complete, "candidate_geometries": len(rows), + "feasible_edges": len(edges), + "objective": "minimum packages, then minimum total container bytes"} + if endpoint not in best: + result = {**common, "status": "unreachable", "reachable_nodes": sorted(best), + "endpoint_node": endpoint, + "endpoint_incoming_feasible": sorted( + source for source, target in edges if target == endpoint + )} + else: + nodes = best[endpoint][2] + steps = [] + for source, target in zip(nodes, nodes[1:]): + row = edges[source, target] + steps.append({"source_node": source, "target_node": target, + "inplace_memory": f"0x{int(row['memory']):X}", + "reuse_baseline_package": source == 0 and target == 1, + "expected_container_size": int(row["container"]), + "expected_staging_margin": int(row["margin"]), + "expected_target_sha256": images[target]["sha256"], + "expected_target_version": images[target].get("version", "unknown")}) + result = {**common, "status": "reachable", "nodes": nodes, + "shortest_package_count": hop_distance[endpoint], + "shortest_route_count": route_count[endpoint], + "selected_total_bytes": best[endpoint][1], "steps": steps} + write_atomic_text( + output, json.dumps(result, indent=2, sort_keys=True) + "\n" + ) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory", type=Path, required=True) + parser.add_argument("--reuse", action="append", default=[], metavar="IMAGES_JSON=GEOMETRY_CSV") + parser.add_argument("--work-dir", type=Path, required=True) + parser.add_argument("--workers", type=int, default=min(8, os.cpu_count() or 1)) + parser.add_argument("--no-generate", action="store_true") + args = parser.parse_args(argv) + if args.workers < 1: + parser.error("--workers must be positive") + args.work_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".frozen-inventory-", dir=args.work_dir + ) as frozen_raw: + frozen = Path(frozen_raw) + inventory_path = snapshot_inventory( + args.inventory, frozen / "current" / "images.json" + ) + images = load_inventory(inventory_path) + baseline_size = _integer( + images[1].get("baseline_container_size"), + "node 1 baseline_container_size", + ) + cache: dict[tuple[str, str, int], dict[str, object]] = {} + comparable = ( + "payload", "container", "stage_start", "margin", "feasible", "error" + ) + for reuse_number, value in enumerate(args.reuse, 1): + left, separator, right = value.partition("=") + if not separator: + parser.error("--reuse requires IMAGES_JSON=GEOMETRY_CSV") + reuse_root = frozen / f"reuse-{reuse_number:03d}" + reuse_inventory = snapshot_inventory( + Path(left), reuse_root / "images.json" + ) + reuse_csv = reuse_root / "geometry.csv" + try: + shutil.copy2(Path(right), reuse_csv, follow_symlinks=True) + except OSError as exc: + raise RouteSearchError( + f"cannot snapshot geometry cache {right}: {exc}" + ) from exc + for key, row in migrate_csv(reuse_inventory, reuse_csv).items(): + old = cache.get(key) + if old is not None and any( + str(old.get(field, "")) != str(row.get(field, "")) + for field in comparable + ): + raise RouteSearchError(f"conflicting reuse inputs for {key}") + cache[key] = row + + jobs = all_jobs(images) + required = {cache_key(job[5], job[6], job[2]) for job in jobs} + remaining = [ + job for job in jobs + if cache_key(job[5], job[6], job[2]) not in cache + ] + print( + f"nodes={len(images)} geometries={len(jobs)} " + f"reused={len(required & cache.keys())} remaining={len(remaining)}" + ) + if remaining and not args.no_generate: + # Patch payloads are disposable worker scratch. Keep them in a + # uniquely owned directory so cleanup can never remove a caller's + # pre-existing `work-dir/patches` tree. + with tempfile.TemporaryDirectory( + prefix=".route-patches-", dir=args.work_dir + ) as patches_raw: + patches = Path(patches_raw) + with ProcessPoolExecutor(max_workers=args.workers) as pool: + futures = [ + pool.submit(geometry_job, (*job, str(patches))) + for job in remaining + ] + for done, future in enumerate(as_completed(futures), 1): + row = future.result() + key = cache_key( + str(row["source_sha256"]), + str(row["target_sha256"]), + int(row["memory"]), + ) + cache[key] = row + if done % 100 == 0 or done == len(futures): + print(f"measured={done}/{len(remaining)}", flush=True) + completed = required & cache.keys() + rows = project_cache(cache, images, completed) + write_csv(args.work_dir / "geometry.csv", rows) + complete = completed == required + result = select_route( + rows, images, baseline_size, args.work_dir / "route.json", + complete=complete, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if complete else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/lora_ota/test_build_rak3401_compact_bundle.py b/tools/lora_ota/test_build_rak3401_compact_bundle.py new file mode 100644 index 00000000..34398671 --- /dev/null +++ b/tools/lora_ota/test_build_rak3401_compact_bundle.py @@ -0,0 +1,688 @@ +#!/usr/bin/env python3 +"""Focused tests for variable-node compact RAK3401 bundle inputs.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import struct +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import build_rak3401_compact_bundle as compact + + +def firmware(body: bytes, version: str) -> bytes: + hardware = compact.EXPECTED_HARDWARE.encode("ascii").ljust(32, b"\0") + return ( + body + + b"EndF" + + struct.pack(" bytes: + padded_size = compact.align_up(len(image), payload_size) + encoded = image + b"\xFF" * (padded_size - len(image)) + block_count = len(encoded) // payload_size + blocks = [] + for number in range(block_count): + block = bytearray(compact.UF2_BLOCK_SIZE) + struct.pack_into( + " dict[str, object]: + return { + "schema": 2, + "status": "reachable", + "search_complete": True, + "app_base": "0x26000", + "stage_ceiling": "0xD4000", + "node_count": 4, + "objective": "minimum packages, then minimum total container bytes", + "candidate_geometries": 100, + "feasible_edges": 4, + "shortest_package_count": 2, + "shortest_route_count": 1, + "selected_total_bytes": 300, + "steps": [ + { + "source_node": 0, + "target_node": 1, + "inplace_memory": "0x98000", + "reuse_baseline_package": True, + "expected_container_size": 100, + "expected_staging_margin": 1, + "expected_target_sha256": "1" * 64, + "expected_target_version": "1.16.7.9", + }, + { + "source_node": 1, + "target_node": 3, + "inplace_memory": "0x98000", + "reuse_baseline_package": False, + "expected_container_size": 200, + "expected_staging_margin": 2, + "expected_target_sha256": "2" * 64, + "expected_target_version": "1.17.1.5", + }, + ], + } + + def test_existing_schema1_route_remains_accepted(self) -> None: + route = Path(__file__).with_name("rak3401_compact_route.json") + steps, document = compact.read_route(route) + self.assertEqual(document["schema"], 1) + self.assertEqual(document["endpoint_node"], 30) + self.assertEqual(len(steps), 9) + + def test_schema2_route_is_normalized_for_bundle_provenance(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "route.json" + path.write_text(json.dumps(self.schema2_route()), encoding="ascii") + steps, document = compact.read_route(path) + self.assertEqual([step["target_node"] for step in steps], [1, 3]) + self.assertEqual(document["endpoint_node"], 3) + self.assertTrue(document["search"]["search_complete"]) + self.assertEqual(document["search"]["selected_total_bytes"], 300) + + def test_public_route_search_drops_schema1_private_fields(self) -> None: + route = Path(__file__).with_name("rak3401_compact_route.json") + _steps, document = compact.read_route(route) + document["search"].update({ + "password": "must-not-be-archived", + "local_path": "/tmp/private/route.json", + "nested": {"api_token": "must-not-be-archived"}, + }) + + public = compact.public_route_search(1, document) + + self.assertNotIn("password", public) + self.assertNotIn("local_path", public) + self.assertNotIn("nested", public) + self.assertEqual( + set(public), + { + "objective", "shortest_package_count", "selected_total_bytes", + "page_size", "candidate_pairs", "candidate_geometries", + "feasible_edges", "shortest_route_count", + }, + ) + + def test_schema2_route_must_be_a_complete_reachable_search(self) -> None: + for field, value, message in ( + ("status", "unreachable", "does not reach"), + ("search_complete", False, "incomplete"), + ): + with self.subTest(field=field), tempfile.TemporaryDirectory() as directory: + document = self.schema2_route() + document[field] = value + path = Path(directory) / "route.json" + path.write_text(json.dumps(document), encoding="ascii") + with self.assertRaisesRegex(compact.CompactBuildError, message): + compact.read_route(path) + + +class CompactImageInventoryTests(unittest.TestCase): + def test_schema2_snapshot_freezes_manifest_and_referenced_images(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + source.mkdir() + image = firmware(b"frozen body", "1.17.1.5") + image_path = source / "image.bin" + image_path.write_bytes(image) + record = { + "node": 0, + "path": image_path.name, + "size": len(image), + "sha256": hashlib.sha256(image).hexdigest(), + "body_hash": hashlib.sha256(image[:-56]).digest()[:8].hex(), + "version": "1.17.1.5", + } + manifest = source / "images.json" + manifest.write_text(json.dumps({"images": [record]}), encoding="ascii") + frozen = compact.snapshot_schema2_inventory( + manifest, root / "frozen" / "images.json" + ) + + image_path.write_bytes(b"changed") + manifest.write_text("{}", encoding="ascii") + images, _sources = compact.read_image_inventory(frozen, 1) + + self.assertEqual(images[0], image) + + def test_schema1_snapshot_freezes_manifest_sibling_zips(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + source.mkdir() + archive = source / "bridge.zip" + archive.write_bytes(b"frozen zip") + manifest = source / "bridges.json" + manifest.write_text( + json.dumps({"targets": [{"zip": archive.name}]}), + encoding="ascii", + ) + frozen = compact.snapshot_schema1_manifest( + manifest, root / "frozen" / manifest.name + ) + + archive.write_bytes(b"changed") + manifest.write_text("{}", encoding="ascii") + self.assertEqual( + json.loads(frozen.read_text(encoding="ascii")), + {"targets": [{"zip": "bridge.zip"}]}, + ) + self.assertEqual( + (frozen.parent / "bridge.zip").read_bytes(), b"frozen zip" + ) + + def test_inventory_validates_order_hash_size_identity_and_version(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records = [] + expected = {} + for node, version in enumerate(("1.16.7.0", "1.16.7.9", "1.17.1.5")): + image = firmware(bytes([node + 1]) * (128 + node), version) + image_path = root / f"image-{node:02d}.bin" + image_path.write_bytes(image) + expected[node] = image + records.append({ + "node": node, + "path": image_path.name, + "size": len(image), + "sha256": hashlib.sha256(image).hexdigest(), + "body_hash": hashlib.sha256(image[:-56]).digest()[:8].hex(), + "version": version, + }) + manifest = root / "images.json" + manifest.write_text(json.dumps({"images": records}), encoding="ascii") + images, sources = compact.read_image_inventory(manifest, 3) + self.assertEqual(images, expected) + self.assertTrue(sources[1].startswith("image-sha256:")) + + def test_inventory_rejects_a_stale_image_pin(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + image = firmware(b"body", "1.17.1.5") + (root / "image-00.bin").write_bytes(image) + record = { + "node": 0, + "path": "image-00.bin", + "size": len(image), + "sha256": "0" * 64, + "body_hash": hashlib.sha256(b"body").digest()[:8].hex(), + "version": "1.17.1.5", + } + manifest = root / "images.json" + manifest.write_text( + json.dumps({"images": [record, record, record]}), encoding="ascii" + ) + with self.assertRaisesRegex(compact.CompactBuildError, "SHA-256 mismatch"): + compact.read_image_inventory(manifest, 3) + + def test_bundled_inventory_drops_machine_local_paths(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.json" + destination = root / "bundled.json" + source.write_text( + json.dumps({ + "baseline_mota": "/tmp/private/step.mota", + "admin_password": "must-not-be-archived", + "api_token": "must-not-be-archived", + "notes": "must-not-be-archived", + "images": [{ + "node": 0, + "path": "/tmp/private/image.bin", + "size": 123, + "sha256": "1" * 64, + "body_hash": "2" * 16, + "version": "1.16.7.0", + "latitude": 47.0, + "nested": { + "source_path": "/tmp/private/source.bin", + "private_key": "must-not-be-archived", + "keep": 1, + }, + }], + }), + encoding="ascii", + ) + compact.write_inventory_provenance(source, destination) + bundled = json.loads(destination.read_text(encoding="ascii")) + self.assertNotIn("path", bundled["images"][0]) + self.assertNotIn("baseline_mota", bundled) + self.assertNotIn("admin_password", bundled) + self.assertNotIn("api_token", bundled) + self.assertNotIn("notes", bundled) + self.assertEqual( + bundled, + { + "schema": 2, + "images": [{ + "node": 0, + "size": 123, + "sha256": "1" * 64, + "body_hash": "2" * 16, + "version": "1.16.7.0", + }], + }, + ) + + def test_inventory_rejects_wrong_declared_body_hash(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records = [] + for node in range(3): + image = firmware(bytes([node + 1]) * 32, f"1.16.7.{node}") + image_path = root / f"image-{node}.bin" + image_path.write_bytes(image) + records.append({ + "node": node, "path": image_path.name, "size": len(image), + "sha256": hashlib.sha256(image).hexdigest(), + "body_hash": "0" * 16, "version": f"1.16.7.{node}", + }) + manifest = root / "images.json" + manifest.write_text(json.dumps({"images": records}), encoding="ascii") + with self.assertRaisesRegex(compact.CompactBuildError, "body hash mismatch"): + compact.read_image_inventory(manifest, 3) + + +class CompactSimulatorTests(unittest.TestCase): + def test_exact_required_simulators_are_accepted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + preview5 = Path(directory) / "preview5" + current = Path(directory) / "current" + preview5.write_bytes(b"p5") + current.write_bytes(b"current") + hashes = { + preview5: compact.REQUIRED_SIMULATORS["preview5"]["sha256"], + current: compact.REQUIRED_SIMULATORS["current"]["sha256"], + } + with mock.patch.object(compact.common, "sha256_file", side_effect=hashes.get): + result = compact.parse_simulators([ + f"preview5={preview5}", f"current={current}" + ]) + self.assertEqual([label for label, _path in result], ["preview5", "current"]) + + def test_missing_or_wrong_simulator_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + simulator = Path(directory) / "sim" + simulator.write_bytes(b"wrong") + for values, digest in (([], None), ([f"preview5={simulator}"], "0" * 64)): + with self.subTest(values=values), mock.patch.object( + compact.common, "sha256_file", return_value=digest + ): + with self.assertRaisesRegex(compact.CompactBuildError, "require exactly"): + compact.parse_simulators(values) + + +class CompactUf2Tests(unittest.TestCase): + def test_exact_app_only_uf2_is_accepted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + image = firmware(b"endpoint" * 41, "1.17.1.5") + path = Path(directory) / "endpoint.uf2" + path.write_bytes(uf2(image)) + compact.validate_uf2_firmware(path, image, "test UF2") + + def test_wrong_family_and_non_app_address_are_rejected(self) -> None: + image = firmware(b"endpoint", "1.17.1.5") + for label, offset, value, message in ( + ("family", 28, 0xDEADBEEF, "nRF52840"), + ("address", 12, compact.APP_BASE - 256, "app-only"), + ): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + raw = bytearray(uf2(image)) + struct.pack_into(" None: + image = firmware(b"x" * 600, "1.17.1.5") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + duplicate = bytearray(uf2(image)) + struct.pack_into(" None: + image = firmware(b"odd", "1.17.1.5") + with tempfile.TemporaryDirectory() as directory: + raw = bytearray(uf2(image)) + raw[compact.UF2_DATA_OFFSET + len(image)] = 0 + path = Path(directory) / "padding.uf2" + path.write_bytes(raw) + with self.assertRaisesRegex(compact.CompactBuildError, "non-erased"): + compact.validate_uf2_firmware(path, image, "test UF2") + + +class CompactToolProvenanceTests(unittest.TestCase): + def test_required_versions_and_actual_launcher_hashes_are_recorded(self) -> None: + motatool = Path("/tools/motatool") + detools = Path("/tools/detools") + hashes = {motatool: "1" * 64, detools: "2" * 64} + with mock.patch.object( + compact, "run", side_effect=["motatool 0.1.0\n", "0.53.0\n"] + ), mock.patch.object(compact.common, "sha256_file", side_effect=hashes.get): + result = compact.tool_provenance(motatool, detools) + self.assertEqual(result["motatool"]["version"], "0.1.0") + self.assertEqual( + result["motatool"]["asserted_source_commit"], compact.MOTATOOL_COMMIT + ) + self.assertEqual(result["motatool"]["executable_sha256"], "1" * 64) + self.assertEqual(result["detools"]["launcher_sha256"], "2" * 64) + + def test_wrong_tool_versions_fail_closed(self) -> None: + for outputs, message in ( + (["motatool 0.2.0\n"], "motatool version"), + (["motatool 0.1.0\n", "0.54.0\n"], "detools version"), + ): + with self.subTest(outputs=outputs), mock.patch.object( + compact, "run", side_effect=outputs + ): + with self.assertRaisesRegex(compact.CompactBuildError, message): + compact.tool_provenance(Path("motatool"), Path("detools")) + + +class CompactGeometryEvidenceTests(unittest.TestCase): + def make_evidence(self, root: Path) -> tuple[Path, Path, list[dict[str, object]]]: + records = [] + for node, version in enumerate(("1.16.7.0", "1.16.7.9", "1.17.1.3", "1.17.1.5")): + image = firmware(bytes([node + 1]) * (64 + node), version) + image_path = root / f"image-{node:02d}.bin" + image_path.write_bytes(image) + records.append({ + "node": node, "path": image_path.name, "size": len(image), + "sha256": hashlib.sha256(image).hexdigest(), + "body_hash": hashlib.sha256(image[:-56]).digest()[:8].hex(), + "version": version, + **({"baseline_container_size": 100} if node == 1 else {}), + }) + inventory_path = root / "inventory.json" + inventory_path.write_text(json.dumps({"images": records}), encoding="ascii") + inventory = compact.route_search.load_inventory(inventory_path) + rows = [] + for job in compact.route_search.all_jobs(inventory): + source, target, memory, _from, _to, source_sha, target_sha = job + if source == 1 and target == 3: + payload = 10 + else: + # A successfully generated patch may be too large for staging; + # that is valid infeasible evidence. A tool exception is not. + payload = 700_000 + container = compact.route_search.container_size(payload) + stage_start = compact.route_search.align_down( + compact.STAGE_CEILING - container + ) + margin = stage_start - (compact.APP_BASE + memory) + feasible, error = margin >= 0, "" + rows.append({ + "source": source, "target": target, "source_sha256": source_sha, + "target_sha256": target_sha, "memory": memory, "payload": payload, + "container": container, "stage_start": stage_start, "margin": margin, + "feasible": feasible, "error": error, + }) + geometry_path = root / "geometry.csv" + compact.route_search.write_csv(geometry_path, rows) + route_path = root / "route.json" + compact.route_search.select_route(rows, inventory, 100, route_path, complete=True) + return inventory_path, route_path, rows + + def test_complete_geometry_reproduces_declared_route(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, _rows = self.make_evidence(root) + result = compact.validate_geometry_results(root / "geometry.csv", inventory, route) + self.assertEqual(result["nodes"], [0, 1, 3]) + + def test_legacy_source_zero_measurement_is_accepted_but_not_routed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory_path, route_path, rows = self.make_evidence(root) + inventory = compact.route_search.load_inventory(inventory_path) + payload = 10 + container = compact.route_search.container_size(payload) + stage_start = compact.route_search.align_down( + compact.STAGE_CEILING - container + ) + margin = stage_start - (compact.APP_BASE + compact.FIXED_WORKSPACE) + rows.append({ + "source": 0, + "target": 2, + "source_sha256": inventory[0]["sha256"], + "target_sha256": inventory[2]["sha256"], + "memory": compact.FIXED_WORKSPACE, + "payload": payload, + "container": container, + "stage_start": stage_start, + "margin": margin, + "feasible": margin >= 0, + "error": "", + }) + compact.route_search.write_csv(root / "geometry.csv", rows) + compact.route_search.select_route( + rows, inventory, 100, route_path, complete=True + ) + + result = compact.validate_geometry_results( + root / "geometry.csv", inventory_path, route_path + ) + + self.assertEqual(result["nodes"], [0, 1, 3]) + self.assertEqual(result["candidate_geometries"], len(rows)) + + def test_geometry_tamper_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, rows = self.make_evidence(root) + rows[0]["container"] = 123 + compact.route_search.write_csv(root / "geometry.csv", rows) + with self.assertRaisesRegex(compact.CompactBuildError, "inconsistent"): + compact.validate_geometry_results(root / "geometry.csv", inventory, route) + + def test_missing_and_extra_geometry_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, rows = self.make_evidence(root) + for label, altered, message in ( + ("missing", rows[:-1], "missing"), + ("extra", rows + [{**rows[0], "memory": int(rows[0]["memory"]) + 1}], "extra"), + ): + with self.subTest(label=label): + compact.route_search.write_csv(root / "geometry.csv", altered) + with self.assertRaisesRegex(compact.CompactBuildError, message): + compact.validate_geometry_results(root / "geometry.csv", inventory, route) + + def test_route_metric_or_step_mismatch_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, _rows = self.make_evidence(root) + document = json.loads(route.read_text(encoding="ascii")) + document["selected_total_bytes"] += 1 + route.write_text(json.dumps(document), encoding="ascii") + with self.assertRaisesRegex(compact.CompactBuildError, "selected_total_bytes"): + compact.validate_geometry_results(root / "geometry.csv", inventory, route) + + def test_failed_patch_job_cannot_be_exhaustive_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, rows = self.make_evidence(root) + rows[0].update( + payload=-1, container=-1, stage_start=-1, margin=-1, + feasible=False, error="MemoryError: test", + ) + compact.route_search.write_csv(root / "geometry.csv", rows) + with self.assertRaisesRegex( + compact.CompactBuildError, "cannot prove an exhaustive route search" + ): + compact.validate_geometry_results(root / "geometry.csv", inventory, route) + + def test_short_geometry_row_fails_cleanly(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory, route, _rows = self.make_evidence(root) + (root / "geometry.csv").write_text( + ",".join(compact.route_search.FIELDS) + "\n1,2\n", + encoding="ascii", + ) + with self.assertRaisesRegex( + compact.CompactBuildError, "integer|malformed|invalid" + ): + compact.validate_geometry_results(root / "geometry.csv", inventory, route) + + +class CompactPhysicalValidationTests(unittest.TestCase): + def evidence(self) -> tuple[dict[str, object], dict[str, object]]: + expected = { + "chain_sha256": "1" * 64, + "start_sha256": "2" * 64, + "endpoint_sha256": "3" * 64, + "endpoint_body_hash": "a" * 16, + "endpoint_version": "1.17.1.5", + "output_rows": [{"target_body_hash": "5" * 16}], + "validation_steps": [{ + "mota_sha256": "6" * 64, + "target_sha256": "7" * 64, + }], + } + document: dict[str, object] = { + "schema": 1, + "kind": compact.PHYSICAL_VALIDATION_KIND, + "status": "passed", + "chain_sha256": expected["chain_sha256"], + "start_sha256": expected["start_sha256"], + "endpoint_sha256": expected["endpoint_sha256"], + "endpoint_body_hash": expected["endpoint_body_hash"], + "endpoint_version": expected["endpoint_version"], + "step_count": 1, + "steps": [{ + "step": 1, + "status": "passed", + "mota_sha256": "6" * 64, + "target_sha256": "7" * 64, + "target_body_hash": "5" * 16, + }], + "final_swd": {"status": "passed", "app_sha256": "3" * 64}, + } + return document, expected + + def test_exact_record_is_accepted_and_canonicalized(self) -> None: + document, expected = self.evidence() + document["endpoint_body_hash"] = str(document["endpoint_body_hash"]).upper() + result = compact.validate_physical_validation_record(document, **expected) + self.assertEqual(result["status"], "passed") + self.assertEqual(result["endpoint_body_hash"], "a" * 16) + + def test_stale_package_or_incomplete_step_list_is_rejected(self) -> None: + for label, mutate, message in ( + ( + "stale", + lambda document: document["steps"][0].__setitem__( + "mota_sha256", "8" * 64 + ), + "does not match", + ), + ( + "incomplete", + lambda document: document.__setitem__("steps", []), + "every chain step", + ), + ): + with self.subTest(label=label): + document, expected = self.evidence() + mutate(document) + with self.assertRaisesRegex(compact.CompactBuildError, message): + compact.validate_physical_validation_record(document, **expected) + + def test_extra_transcript_or_secret_fields_are_rejected(self) -> None: + document, expected = self.evidence() + document["admin_password"] = "must-not-be-archived" + with self.assertRaisesRegex(compact.CompactBuildError, "unexpected"): + compact.validate_physical_validation_record(document, **expected) + + +class CompactReproducibilityTests(unittest.TestCase): + def test_zip_metadata_does_not_depend_on_source_mtime(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archives = [] + for number in (1, 2): + tree = root / f"tree-{number}" / "same-root" + tree.mkdir(parents=True) + item = tree / "payload.bin" + item.write_bytes(b"identical") + os.utime(item, (946684800 + number, 946684800 + number)) + output = root / f"archive-{number}.zip" + compact.make_reproducible_zip(tree, output) + archives.append(output.read_bytes()) + self.assertEqual(archives[0], archives[1]) + + def test_zip_output_inside_archived_tree_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + tree = Path(directory) / "tree" + tree.mkdir() + (tree / "payload.bin").write_bytes(b"payload") + with self.assertRaisesRegex( + compact.CompactBuildError, "outside the archived tree" + ): + compact.make_reproducible_zip(tree, tree / "bundle.zip") + + def test_zip_output_symlinked_inside_archived_tree_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + tree = parent / "tree" + tree.mkdir() + (tree / "payload.bin").write_bytes(b"payload") + alias = parent / "output-alias" + alias.symlink_to(tree, target_is_directory=True) + with self.assertRaisesRegex( + compact.CompactBuildError, "outside the archived tree" + ): + compact.make_reproducible_zip(tree, alias / "bundle.zip") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index b1d52118..64b780f8 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -394,6 +394,32 @@ class FormatTests(unittest.TestCase): with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): ota.validate_args(args, parser) + def test_serial_aliases_cannot_identify_the_same_node(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + device = root / "ttyACM0" + device.touch() + alias = root / "by-id-radio" + alias.symlink_to(device) + parser = ota.build_parser() + for source_option in ("--source-serial", "--source-cli-serial"): + arguments = [ + "release.mota", "remote", + "--controller-serial", str(device), + ] + if source_option == "--source-cli-serial": + arguments.extend(( + "--source-tcp", "192.0.2.10:5001", + source_option, str(alias), + )) + else: + arguments.extend((source_option, str(alias))) + args = parser.parse_args(arguments) + with self.subTest(source_option=source_option), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + ota.validate_args(args, parser) + def test_intel_hex_rejects_an_excessive_address_span(self) -> None: def record(address: int, record_type: int, data: bytes) -> str: raw = bytes((len(data), address >> 8, address & 0xFF, record_type)) + data @@ -504,6 +530,19 @@ class DebugTests(unittest.TestCase): self.assertNotIn("top-secret", rendered) self.assertIn("", rendered) + def test_debug_stream_accepts_and_redacts_bytearray_output(self) -> None: + ota.DEBUG = True + output = io.StringIO() + with contextlib.redirect_stdout(output): + ota.debug_stream( + "pending output", + bytearray(b"reply top-secret"), + ("top-secret",), + ) + rendered = output.getvalue() + self.assertNotIn("top-secret", rendered) + self.assertIn("reply ", rendered) + def test_meshcli_debug_redacts_admin_password(self) -> None: ota.DEBUG = True args = argparse.Namespace( @@ -562,6 +601,579 @@ class DebugTests(unittest.TestCase): class SourceCliTests(unittest.TestCase): + @staticmethod + def source_args() -> argparse.Namespace: + return argparse.Namespace( + source_cli_serial=None, + source_serial="/dev/source", + source_cli_tcp=None, + ) + + def test_source_rxps_enabled_profile_is_saved_and_disabled(self) -> None: + args = self.source_args() + with mock.patch.object( + ota, + "source_cli_command", + side_effect=( + "radio.rxps.config on,level=8,preamble=16," + "rx=18205,sleep=20423", + "OK - off,18205,20423", + "radio.rxps.config off,level=8,preamble=16," + "rx=18205,sleep=20423", + ), + ) as source_command: + saved = ota.read_source_rxps(args) + changed = ota.disable_source_rxps(args, saved) + + self.assertEqual( + saved, ota.RxpsSettings(True, 18205, 20423, 8, 16) + ) + self.assertTrue(changed) + self.assertEqual( + [call.args[1] for call in source_command.call_args_list], + [ + "get radio.rxps.config", + "set radio.rxps off", + "get radio.rxps.config", + ], + ) + + def test_source_rxps_recovery_record_preserves_exact_setting(self) -> None: + args = self.source_args() + args.source_baud = 115200 + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory: + path = ota.write_source_rxps_recovery( + Path(directory), args, saved + ) + payload = json.loads(path.read_text(encoding="ascii")) + mode = path.stat().st_mode & 0o777 + recovered = ota.read_source_rxps_recovery(path, args) + + self.assertEqual(mode, 0o600) + self.assertEqual(recovered, saved) + self.assertEqual( + payload, + { + "connection": { + "baud": 115200, + "endpoint": "/dev/source", + "kind": "serial", + }, + "restore_command": "set radio.rxps level 8 preamble 16", + "rxps_enabled": True, + "rxps_level": 8, + "rxps_preamble": 16, + "rxps_rx_us": 18205, + "rxps_sleep_us": 20423, + }, + ) + + def test_source_rxps_recovery_flushes_file_and_directory(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory, mock.patch.object( + ota.os, "fsync" + ) as fsync: + ota.write_source_rxps_recovery(Path(directory), args, saved) + expected_calls = 1 if os.name == "nt" else 2 + self.assertEqual(fsync.call_count, expected_calls) + + def test_retired_source_rxps_record_is_no_longer_active(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = ota.write_source_rxps_recovery(root, args, saved) + ota.retire_source_rxps_recovery(path) + self.assertFalse(path.exists()) + self.assertEqual(list(root.glob(".*.restored-*")), []) + + def test_source_rxps_recovery_record_restores_saved_off_state(self) -> None: + args = self.source_args() + with tempfile.TemporaryDirectory() as directory: + path = ota.write_source_rxps_recovery( + Path(directory), + args, + ota.RxpsSettings(False, 18205, 20423, 8, 16), + ) + payload = json.loads(path.read_text(encoding="ascii")) + self.assertEqual(payload["restore_command"], "set radio.rxps off") + + def test_source_rxps_recovery_rejects_a_different_endpoint(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory: + path = ota.write_source_rxps_recovery(Path(directory), args, saved) + other = self.source_args() + other.source_serial = "/dev/other" + with self.assertRaisesRegex(ota.OtaError, "different CLI endpoint"): + ota.read_source_rxps_recovery(path, other) + def test_source_rxps_legacy_query_fallback(self) -> None: + args = self.source_args() + with mock.patch.object( + ota, + "source_cli_command", + side_effect=("Unknown command", "radio.rxps on,65625,60000"), + ) as source_command: + saved = ota.read_source_rxps(args) + + self.assertEqual(saved, ota.RxpsSettings(True, 65625, 60000)) + self.assertEqual( + [call.args[1] for call in source_command.call_args_list], + ["get radio.rxps.config", "get radio.rxps"], + ) + + def test_source_rxps_already_off_is_not_changed(self) -> None: + args = self.source_args() + with mock.patch.object( + ota, + "source_cli_command", + return_value=( + "radio.rxps.config off,level=8,preamble=16," + "rx=18205,sleep=20423" + ), + ) as source_command: + saved = ota.read_source_rxps(args) + changed = ota.disable_source_rxps(args, saved) + + self.assertFalse(changed) + source_command.assert_called_once_with(args, "get radio.rxps.config") + + def test_source_rxps_disable_requires_off_readback(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with ( + mock.patch.object( + ota, + "source_cli_command", + side_effect=( + "OK - off,18205,20423", + "radio.rxps.config on,level=8,preamble=16," + "rx=18205,sleep=20423", + ), + ), + self.assertRaisesRegex(ota.OtaError, "did not read back as off"), + ): + ota.disable_source_rxps(args, saved) + + def test_source_rxps_disable_retries_explicit_radio_busy(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with ( + mock.patch.object( + ota, + "source_cli_command", + side_effect=( + ota.OtaError( + "source rejected 'set radio.rxps off': " + "Error: radio busy; retry" + ), + "OK - off,18205,20423", + "radio.rxps.config off,level=8,preamble=16," + "rx=18205,sleep=20423", + ), + ) as source_command, + mock.patch.object(ota.time, "sleep") as sleep, + ): + self.assertTrue(ota.disable_source_rxps(args, saved)) + + self.assertEqual( + [call.args[1] for call in source_command.call_args_list], + [ + "set radio.rxps off", + "set radio.rxps off", + "get radio.rxps.config", + ], + ) + sleep.assert_called_once_with( + ota.source_rxps_busy_retry_delay(1) + ) + + def test_source_rxps_busy_retry_is_bounded(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + busy = ota.OtaError("Error: radio busy; retry") + with ( + mock.patch.object( + ota, + "source_cli_command", + side_effect=[busy] * (ota.SOURCE_RXPS_BUSY_RETRY_LIMIT + 1), + ) as source_command, + mock.patch.object(ota.time, "sleep") as sleep, + self.assertRaisesRegex( + ota.OtaError, + f"remained radio busy after {ota.SOURCE_RXPS_BUSY_RETRY_LIMIT} bounded", + ), + contextlib.redirect_stdout(io.StringIO()), + ): + ota.disable_source_rxps(args, saved) + + self.assertEqual( + source_command.call_count, + ota.SOURCE_RXPS_BUSY_RETRY_LIMIT + 1, + ) + expected_delays = [ + ota.source_rxps_busy_retry_delay(index) + for index in range(1, ota.SOURCE_RXPS_BUSY_RETRY_LIMIT + 1) + ] + self.assertEqual(len(set(expected_delays)), len(expected_delays)) + self.assertEqual( + [call.args[0] for call in sleep.call_args_list], expected_delays + ) + self.assertGreater(sum(expected_delays), 9.0) + self.assertLess(sum(expected_delays), 10.0) + + def test_source_rxps_disable_does_not_retry_other_rejections(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + with ( + mock.patch.object( + ota, + "source_cli_command", + side_effect=ota.OtaError("Error: unsupported"), + ) as source_command, + mock.patch.object(ota.time, "sleep") as sleep, + self.assertRaisesRegex(ota.OtaError, "unsupported"), + ): + ota.disable_source_rxps(args, saved) + + source_command.assert_called_once_with(args, "set radio.rxps off") + sleep.assert_not_called() + + def test_source_rxps_restore_uses_saved_level_and_preamble(self) -> None: + args = self.source_args() + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + disabled = ota.RxpsSettings(False, 18205, 20423, 8, 16) + with ( + mock.patch.object( + ota, "read_source_rxps", side_effect=(disabled, saved) + ) as read_source, + mock.patch.object( + ota, + "source_cli_command", + side_effect=( + ota.OtaError("Error: radio busy; retry"), + "OK - level 8,on,18205,20423,preamble=16", + ), + ) as source_command, + mock.patch.object(ota.time, "sleep") as sleep, + ): + ota.restore_source_rxps(args, saved) + + self.assertEqual(read_source.call_count, 2) + self.assertEqual( + source_command.call_args_list, + [ + mock.call(args, "set radio.rxps level 8 preamble 16"), + mock.call(args, "set radio.rxps level 8 preamble 16"), + ], + ) + sleep.assert_called_once_with( + ota.source_rxps_busy_retry_delay(1) + ) + + def test_failure_cleanup_restores_source_rxps_once(self) -> None: + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + disabled = ota.RxpsSettings(False, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) / "work" + argv = [ + "release.mota", + "remote", + "--controller-serial", "/dev/controller", + "--source-serial", "/dev/source", + "--password", "secret", + "--work-dir", str(work_dir), + "--yes", + ] + with ( + mock.patch.object(ota, "preflight_inputs"), + mock.patch.object(ota, "preflight_source_cli"), + mock.patch.object( + ota, + "read_source_rxps", + side_effect=(saved, disabled, disabled, saved), + ) as read_source, + mock.patch.object( + ota, + "source_cli_command", + side_effect=( + "OK - off,18205,20423", + "OK - level 8,on,18205,20423,preamble=16", + ), + ) as source_command, + mock.patch.object( + ota, "query_target", side_effect=ota.OtaError("synthetic failure") + ), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + result = ota.main(argv, controller_override=mock.Mock()) + recovery_exists = ( + work_dir / ota.SOURCE_RXPS_RECOVERY_FILE + ).exists() + + self.assertEqual(result, 2) + self.assertFalse(recovery_exists) + self.assertEqual(read_source.call_count, 4) + self.assertEqual( + [call.args[1] for call in source_command.call_args_list], + [ + "set radio.rxps off", + "set radio.rxps level 8 preamble 16", + ], + ) + + def test_rerun_uses_persisted_source_rxps_instead_of_temporary_off(self) -> None: + original = ota.RxpsSettings(True, 18205, 20423, 8, 16) + currently_off = ota.RxpsSettings(False, 18205, 20423, 8, 16) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + recovery_path = ota.write_source_rxps_recovery( + root, + argparse.Namespace( + source_cli_serial=None, + source_serial="/dev/source", + source_cli_tcp=None, + source_baud=115200, + ), + original, + ) + argv = [ + "release.mota", + "remote", + "--controller-serial", "/dev/controller", + "--source-serial", "/dev/source", + "--password", "secret", + "--source-rxps-recovery-file", str(recovery_path), + "--work-dir", str(root / "attempt"), + "--yes", + ] + with ( + mock.patch.object(ota, "preflight_inputs"), + mock.patch.object(ota, "preflight_source_cli"), + mock.patch.object( + ota, "read_source_rxps", return_value=currently_off + ), + mock.patch.object( + ota, "disable_source_rxps", return_value=False + ) as disable, + mock.patch.object(ota, "restore_source_rxps") as restore, + mock.patch.object( + ota, "query_target", side_effect=ota.OtaError("synthetic failure") + ), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + result = ota.main(argv, controller_override=mock.Mock()) + + self.assertEqual(result, 2) + disable.assert_called_once_with(mock.ANY, currently_off) + restore.assert_called_once_with(mock.ANY, original) + + def test_source_recovery_file_inside_attempt_is_rejected_before_mutation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) / "attempt" + argv = [ + "release.mota", "remote", + "--controller-serial", "/dev/controller", + "--source-serial", "/dev/source", + "--password", "secret", + "--work-dir", str(work_dir), + "--source-rxps-recovery-file", + str(work_dir / "controller-radio.txt"), + "--yes", + ] + with ( + mock.patch.object(ota, "preflight_inputs"), + mock.patch.object(ota, "preflight_source_cli") as source_preflight, + mock.patch.object(ota, "read_source_rxps") as read_source, + mock.patch.object(ota, "disable_source_rxps") as disable_source, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + result = ota.main(argv, controller_override=mock.Mock()) + + self.assertEqual(result, 2) + source_preflight.assert_not_called() + read_source.assert_not_called() + disable_source.assert_not_called() + + def test_success_path_source_rxps_restore_failure_is_retried_in_finally(self) -> None: + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + image = firmware(b"source-restore-retry" * 500, VERSION_NEW) + package = ota.parse_mota(mota_blob(image)) + expected_body_hash = ota.parse_endf(image).body_hash + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + controller = mock.Mock() + controller.get_radio.return_value = normal + controller.remote_command.return_value = "OK" + with tempfile.TemporaryDirectory() as directory: + argv = [ + "release.mota", + "remote", + "--controller-serial", "/dev/controller", + "--source-serial", "/dev/source", + "--password", "secret", + "--work-dir", str(Path(directory) / "work"), + "--yes", + ] + seeder = mock.Mock() + with contextlib.ExitStack() as stack: + for name in ( + "preflight_inputs", + "preflight_source_cli", + "verify_shared_source_identity", + "confirm_update", + "arm_target_temp_radio", + "switch_controller_to_temp_radio", + "find_and_start_pull", + "monitor_download", + "verify_installed", + ): + stack.enter_context(mock.patch.object(ota, name)) + stack.enter_context( + mock.patch.object(ota, "read_source_rxps", return_value=saved) + ) + stack.enter_context( + mock.patch.object(ota, "disable_source_rxps", return_value=True) + ) + restore_source = stack.enter_context( + mock.patch.object( + ota, + "restore_source_rxps", + side_effect=( + ota.OtaError("transient restore failure"), + None, + ), + ) + ) + stack.enter_context( + mock.patch.object(ota, "query_target", return_value=target()) + ) + stack.enter_context( + mock.patch.object( + ota, + "prepare_package", + return_value=( + Path("release.mota"), + package, + expected_body_hash, + ), + ) + ) + stack.enter_context( + mock.patch.object( + ota, + "read_lora_ota_participant_versions", + return_value={"destination": VERSION_NEW}, + ) + ) + stack.enter_context( + mock.patch.object( + ota, + "read_remote_rxps", + return_value=ota.RxpsSettings(False, 0, 0), + ) + ) + stack.enter_context( + mock.patch.object(ota, "request_install", return_value=True) + ) + stack.enter_context( + mock.patch.object( + ota, "shorten_source_temp_window", return_value=True + ) + ) + stack.enter_context( + mock.patch.object( + ota, + "wait_for_post_install_identity", + return_value=( + "self body=1 image=2 base_hash=" + f"{expected_body_hash.hex().upper()}" + ), + ) + ) + stack.enter_context( + mock.patch.object(ota, "SeederProcess", return_value=seeder) + ) + stack.enter_context(mock.patch.object(ota.time, "sleep")) + stack.enter_context( + mock.patch.object(ota, "source_cli_command", return_value="OK") + ) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + result = ota.main(argv, controller_override=controller) + + self.assertEqual(result, 2) + self.assertEqual(restore_source.call_count, 2) + + def test_shared_failure_cleanup_never_persists_temp_radio_tuple(self) -> None: + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + controller = mock.Mock() + controller.get_radio.return_value = normal + package = mock.Mock( + version="1.17.1.5", + kind="full", + target_id=0x1234ABCD, + manifest_id="01234567", + ) + with tempfile.TemporaryDirectory() as directory: + argv = [ + "release.mota", "remote", + "--controller-tcp", "127.0.0.1:5000", + "--source-tcp", "127.0.0.1:5001", + "--source-cli-tcp", "127.0.0.1:5002", + "--source-shares-controller", + "--password", "secret", + "--work-dir", str(Path(directory) / "work"), + "--yes", + ] + with ( + mock.patch.object(ota, "preflight_inputs"), + mock.patch.object(ota, "preflight_source_cli"), + mock.patch.object(ota, "verify_shared_source_identity"), + mock.patch.object( + ota, + "read_source_rxps", + return_value=ota.RxpsSettings(False, 0, 0), + ), + mock.patch.object(ota, "disable_source_rxps", return_value=False), + mock.patch.object(ota, "query_target", return_value=target()), + mock.patch.object( + ota, + "prepare_package", + return_value=(Path("release.mota"), package, None), + ), + mock.patch.object( + ota, "read_lora_ota_participant_versions", return_value={} + ), + mock.patch.object(ota, "read_remote_rxps", return_value=None), + mock.patch.object(ota, "confirm_update"), + mock.patch.object( + ota, + "arm_target_temp_radio", + side_effect=ota.OtaError("synthetic handoff failure"), + ), + mock.patch.object(ota, "shorten_target_temp_window"), + mock.patch.object( + ota, "shorten_source_temp_window", return_value=False + ), + mock.patch.object(ota, "source_cli_command", return_value="OK") as source_cli, + mock.patch.object(ota.time, "sleep"), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + result = ota.main(argv, controller_override=controller) + + self.assertEqual(result, 2) + self.assertIn( + mock.call(mock.ANY, "tempradio 909.950,250,5,5,120"), + source_cli.call_args_list, + ) + controller.set_radio.assert_not_called() + def test_full_companion_tcp_console_command(self) -> None: connection = mock.MagicMock() connection.__enter__.return_value = connection @@ -618,6 +1230,26 @@ class SourceCliTests(unittest.TestCase): ]) ota.validate_args(args, parser) + def test_already_temp_source_rejects_managed_cli_or_non_tcp_source(self) -> None: + parser = ota.build_parser() + cases = ( + [ + "--source-tcp", "192.0.2.10:5001", + "--source-cli-tcp", "192.0.2.10:5002", + "--source-already-temp", + ], + ["--source-serial", "/dev/source", "--source-already-temp"], + ) + for source_arguments in cases: + args = parser.parse_args([ + "release.mota", "remote", "--controller-serial", + "/dev/controller", *source_arguments, + ]) + with self.subTest(source_arguments=source_arguments), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + ota.validate_args(args, parser) + def test_source_preflight_accepts_seeder_only_status(self) -> None: args = argparse.Namespace( source_serial=None, @@ -950,6 +1582,113 @@ class DownloadSessionTests(unittest.TestCase): ], ) + def test_completed_previous_session_waits_for_staged_verification(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 2s", + "OTA | download: ready to install 9/9 id=DEADBEEF 3s", + "self body=1 image=2 base_hash=0011223344556677", + "OK dropped session", + "OTA | no download", + "Updates 1/1", + f"OK pulling mid={self.package.manifest_id} -> flash (primary traffic)", + ]) + seeder = mock.Mock() + with mock.patch.object(ota.time, "sleep"): + ota.find_and_start_pull(controller, args, self.package, seeder) + self.assertEqual( + controller.commands, + [ + "ota status", "ota status", "ota status", "ota self", + "ota cancel", "ota status", "ota ls", + f"ota pull {self.package.manifest_id} flash", + ], + ) + seeder.ensure_running.assert_any_call( + "while waiting for a completed previous manifest" + ) + + def test_completed_previous_session_does_not_cancel_idle_manager(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | no download", + "self body=1 image=2 base_hash=0011223344556677", + "Updates 1/1", + f"OK pulling mid={self.package.manifest_id} -> flash (primary traffic)", + ]) + with mock.patch.object(ota.time, "sleep"): + ota.find_and_start_pull(controller, args, self.package) + self.assertEqual( + controller.commands, + [ + "ota status", "ota status", "ota self", "ota ls", + f"ota pull {self.package.manifest_id} flash", + ], + ) + + def test_completed_previous_verification_refuses_changed_manifest(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | download: ready to install 9/9 id=CAFEBABE 2s", + ]) + with ( + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "changed during"), + ): + ota.find_and_start_pull(controller, args, self.package) + + def test_completed_previous_verification_refuses_failed_store(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | download: failed (hash) 9/9 id=DEADBEEF 2s", + ]) + with ( + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "failed staged-block"), + ): + ota.find_and_start_pull(controller, args, self.package) + + def test_completed_previous_verification_refuses_incomplete_store(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | download: downloading 8/9 id=DEADBEEF 2s", + ]) + with ( + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "became incomplete"), + ): + ota.find_and_start_pull(controller, args, self.package) + + def test_completed_previous_verification_has_bounded_timeout(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 1s", + "OTA | download: verifying staged blocks 9/9 id=DEADBEEF 2s", + ]) + with ( + mock.patch.object(ota.time, "monotonic", side_effect=[0.0, 0.0, 1.0]), + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "timed out waiting"), + ): + ota.find_and_start_pull(controller, args, self.package) + def test_replace_active_session_requires_explicit_flag(self) -> None: controller = self.Controller([ "OTA | download: downloading 3/9 id=DEADBEEF 2s", @@ -1064,6 +1803,66 @@ class DownloadSessionTests(unittest.TestCase): ], ) + def test_lost_discovery_reply_still_attempts_exact_pull_and_reconciles(self) -> None: + controller = mock.Mock() + controller.remote_command.side_effect = ( + "OTA | no download", + ota.TransmissionError("discovery reply lost"), + ota.TransmissionError("pull reply lost"), + f"OTA | download: downloading 1/9 id={self.package.manifest_id} 2s", + ) + output = io.StringIO() + with contextlib.redirect_stdout(output): + ota.find_and_start_pull(controller, self.args(), self.package) + self.assertEqual( + controller.remote_command.call_args_list, + [ + mock.call("remote", "ota status"), + mock.call("remote", "ota ls", retry=False), + mock.call( + "remote", + f"ota pull {self.package.manifest_id} flash", + retry=False, + ), + mock.call("remote", "ota status"), + ], + ) + self.assertIn("`ota ls` reply was lost", output.getvalue()) + self.assertIn("pull reply was lost, but session", output.getvalue()) + + def test_no_such_update_waits_then_repeats_one_discovery_and_pull(self) -> None: + controller = mock.Mock() + controller.remote_command.side_effect = ( + "OTA | no download", + "No updates seen yet", + "ERR no such update", + "Updates 1/1", + f"OK pulling mid={self.package.manifest_id} -> flash (primary traffic)", + ) + args = self.args() + args.discovery_timeout = 60 + with mock.patch.object(ota.time, "sleep") as sleep: + ota.find_and_start_pull(controller, args, self.package) + self.assertEqual( + controller.remote_command.call_args_list, + [ + mock.call("remote", "ota status"), + mock.call("remote", "ota ls", retry=False), + mock.call( + "remote", + f"ota pull {self.package.manifest_id} flash", + retry=False, + ), + mock.call("remote", "ota ls", retry=False), + mock.call( + "remote", + f"ota pull {self.package.manifest_id} flash", + retry=False, + ), + ], + ) + sleep.assert_called_once_with(args.discovery_interval) + class ReliabilityTests(unittest.TestCase): def test_target_version_falls_back_to_ver(self) -> None: @@ -1110,6 +1909,13 @@ class ReliabilityTests(unittest.TestCase): "OK resuming mid=1234ABCD -> flash (primary traffic)", ) ) + self.assertTrue( + ota.reply_matches_command( + "ota cancel", + "ERR dropped RAM session (was I mid=-), but persistent OTA " + "slot invalidation failed", + ) + ) def test_target_uses_bootloader_version_to_identify_nrf52(self) -> None: class Controller: @@ -1546,6 +2352,94 @@ class ReliabilityTests(unittest.TestCase): ) self.assertEqual(controller.radios, [temporary, normal]) + def test_shared_lost_target_temp_probe_preserves_saved_controller_tuple(self) -> None: + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + temporary = ota.RadioSettings(909.95, 500.0, 5, 5, False) + + class Controller: + def __init__(self) -> None: + self.commands: list[str] = [] + self.radios: list[ota.RadioSettings] = [] + + def remote_command( + self, _target: str, command: str, **_kwargs: object + ) -> str: + self.commands.append(command) + if command.startswith("tempradio "): + raise ota.TransmissionError("lost reply") + return "self body=1 image=2 base_hash=0011223344556677" + + def set_radio(self, radio: ota.RadioSettings, _label: str) -> None: + self.radios.append(radio) + + args = argparse.Namespace( + target="remote", + source_shares_controller=True, + source_already_temp=False, + ) + controller = Controller() + command = "tempradio 909.95,500,5,5,120" + with ( + mock.patch.object( + ota, + "source_cli_command", + side_effect=( + "OK - temp params for 120 mins", + "OK - normal radio restore scheduled", + "TempRadio inactive", + ), + ) as source_command, + mock.patch.object(ota.time, "sleep") as sleep, + ): + ota.arm_target_temp_radio( + controller, args, command, temporary, normal + ) + self.assertEqual(controller.commands, [command, "ota self"]) + self.assertEqual(controller.radios, [normal]) + self.assertEqual( + source_command.call_args_list, + [ + mock.call(args, command), + mock.call(args, "normalradio", check=True), + mock.call(args, "tempradio", check=True), + ], + ) + sleep.assert_called_once_with(ota.TEMP_RADIO_SWITCH_DELAY_SECONDS) + + def test_shared_probe_reasserts_binary_tuple_when_local_cleanup_fails(self) -> None: + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + temporary = ota.RadioSettings(909.95, 500.0, 5, 5, False) + controller = mock.Mock() + controller.remote_command.side_effect = ( + ota.TransmissionError("lost reply"), + "self body=1 image=2 base_hash=0011223344556677", + ) + args = argparse.Namespace( + target="remote", + source_shares_controller=True, + source_already_temp=False, + ) + with ( + mock.patch.object(ota, "switch_controller_to_temp_radio"), + mock.patch.object( + ota, + "shorten_source_temp_window", + side_effect=ota.OtaError("local cleanup failed"), + ), + mock.patch.object(ota.time, "sleep"), + self.assertRaisesRegex(ota.OtaError, "local cleanup failed"), + ): + ota.arm_target_temp_radio( + controller, + args, + "tempradio 909.95,500,5,5,120", + temporary, + normal, + ) + controller.set_radio.assert_called_once_with( + normal, "restore controller after TempRadio probe" + ) + def test_ambiguous_target_temp_probe_is_not_replayed(self) -> None: normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) temporary = ota.RadioSettings(909.95, 250.0, 5, 5, False) @@ -1822,15 +2716,15 @@ class ReliabilityTests(unittest.TestCase): 20, ) - def test_reboot_ready_probe_defaults_to_twenty_seconds(self) -> None: + def test_reboot_ready_probe_defaults_to_five_minutes(self) -> None: parser = ota.build_parser() args = parser.parse_args([ "release.mota", "remote", "--controller-serial", "/dev/controller", "--source-serial", "/dev/source", ]) - self.assertEqual(args.reboot_wait, 20) - self.assertEqual(rak_chain.build_parser().parse_args([]).reboot_wait, 20) + self.assertEqual(args.reboot_wait, 300) + self.assertEqual(rak_chain.build_parser().parse_args([]).reboot_wait, 300) def test_post_install_version_falls_back_to_runtime_ver(self) -> None: image = firmware(b"verify-fallback" * 700, VERSION_NEW) @@ -2026,11 +2920,20 @@ class ReliabilityTests(unittest.TestCase): temp_values=(909.95, 250.0, 5, 5, 120), ) with mock.patch.object( - ota, "source_cli_command", return_value="OK - temp params for 1 mins" + ota, + "source_cli_command", + side_effect=( + "OK - normal radio restore scheduled", + "TempRadio inactive", + ), ) as source_command: self.assertTrue(ota.shorten_source_temp_window(args)) - source_command.assert_called_once_with( - args, "tempradio 909.95,250,5,5,1", check=True + self.assertEqual( + source_command.call_args_list, + [ + mock.call(args, "normalradio", check=False), + mock.call(args, "tempradio", check=False), + ], ) args.source_already_temp = True @@ -2048,13 +2951,52 @@ class ReliabilityTests(unittest.TestCase): mock.patch.object( ota, "source_cli_command", - return_value="OK - normal radio restore scheduled", + side_effect=( + "OK - normal radio restore scheduled", + "TempRadio active: 909.950,250.00,5,5 5s left", + "TempRadio inactive", + ), ) as source_command, mock.patch.object(ota.time, "sleep") as sleep, ): self.assertTrue(ota.shorten_source_temp_window(args)) - source_command.assert_called_once_with(args, "normalradio", check=True) - sleep.assert_called_once_with(ota.TEMP_RADIO_SWITCH_DELAY_SECONDS) + self.assertEqual( + source_command.call_args_list, + [ + mock.call(args, "normalradio", check=True), + mock.call(args, "tempradio", check=True), + mock.call(args, "tempradio", check=True), + ], + ) + sleep.assert_called_once() + + def test_shared_controller_temp_switch_does_not_persist_binary_tuple(self) -> None: + args = argparse.Namespace(source_shares_controller=True) + controller = mock.Mock() + temporary = ota.RadioSettings(909.95, 500.0, 5, 5, False) + command = "tempradio 909.95,500,5,5,120" + with mock.patch.object(ota, "source_cli_command") as source_command: + ota.switch_controller_to_temp_radio( + controller, args, command, temporary + ) + source_command.assert_called_once_with(args, command) + controller.set_radio.assert_not_called() + + def test_separate_controller_temp_switch_uses_binary_tuple(self) -> None: + args = argparse.Namespace(source_shares_controller=False) + controller = mock.Mock() + temporary = ota.RadioSettings(909.95, 500.0, 5, 5, False) + with mock.patch.object(ota, "source_cli_command") as source_command: + ota.switch_controller_to_temp_radio( + controller, + args, + "tempradio 909.95,500,5,5,120", + temporary, + ) + source_command.assert_not_called() + controller.set_radio.assert_called_once_with( + temporary, "switch controller to TempRadio" + ) class Rak3401TransferGuardrailTests(unittest.TestCase): @@ -2156,7 +3098,19 @@ class Rak3401TransferGuardrailTests(unittest.TestCase): self.assertEqual(controller.rxdelay, "2.0") self.assertEqual(controller.airtime_factor, "1.0") self.assertEqual(controller.ota_hops, 3) - self.assertEqual(controller.commands[-1], "powersaving on") + self.assertIn("powersaving on", controller.commands) + self.assertEqual(controller.commands[-1], "ota config") + + def test_restore_converges_when_saved_power_saving_was_off(self) -> None: + controller = self.Controller() + controller.powersaving_enabled = False + saved = rak_chain.read_target_transfer_settings(controller, "remote") + controller.powersaving_enabled = True + + rak_chain.restore_transfer_settings(controller, "remote", saved) + + self.assertFalse(controller.powersaving_enabled) + self.assertIn("powersaving off", controller.commands) def test_guardrails_preserve_airtime_without_opt_in(self) -> None: controller = self.Controller() @@ -2231,6 +3185,56 @@ class Rak3401TransferGuardrailTests(unittest.TestCase): self.assertEqual(len(controller.commands), command_count) self.assertEqual(mode & 0o777, 0o600) + def test_successful_restore_retires_settings_and_next_run_recaptures(self) -> None: + controller = self.Controller() + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + saved = rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + rak_chain.enforce_transfer_guardrails( + controller, "remote", legacy_full_airtime=True + ) + rak_chain.enforce_ota_hops(controller, "remote", 0) + rak_chain.restore_and_retire_transfer_settings( + controller, "remote", saved, work_dir + ) + self.assertFalse( + (work_dir / rak_chain.TRANSFER_SETTINGS_FILE).exists() + ) + + controller.rxdelay = "4.0" + controller.airtime_factor = "0.5" + controller.ota_hops = 2 + recaptured = rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + + self.assertEqual(recaptured.rxdelay, "4.0") + self.assertEqual(recaptured.airtime_factor, "0.5") + self.assertEqual(recaptured.ota_hops, 2) + + def test_failed_restore_keeps_settings_record_armed(self) -> None: + controller = self.Controller() + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + saved = rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + path = work_dir / rak_chain.TRANSFER_SETTINGS_FILE + with ( + mock.patch.object( + rak_chain, + "restore_transfer_settings", + side_effect=ota.OtaError("restore failed"), + ), + self.assertRaisesRegex(ota.OtaError, "restore failed"), + ): + rak_chain.restore_and_retire_transfer_settings( + controller, "remote", saved, work_dir + ) + self.assertTrue(path.is_file()) + def test_saved_settings_reject_string_booleans(self) -> None: controller = self.Controller() with tempfile.TemporaryDirectory() as directory: @@ -2251,6 +3255,7 @@ class Rak3401TransferGuardrailTests(unittest.TestCase): ), encoding="ascii", ) + path.chmod(0o600) with self.assertRaisesRegex(ota.OtaError, "invalid saved transfer settings"): rak_chain.load_or_capture_transfer_settings( controller, "remote", "AA" * 32, work_dir @@ -2274,6 +3279,7 @@ class Rak3401TransferGuardrailTests(unittest.TestCase): ), encoding="ascii", ) + path.chmod(0o600) with self.assertRaisesRegex( ota.OtaError, "invalid saved transfer settings" ): @@ -2281,6 +3287,453 @@ class Rak3401TransferGuardrailTests(unittest.TestCase): controller, "remote", "AA" * 32, work_dir ) + def test_saved_settings_reject_nonfinite_delay_and_non_string_key(self) -> None: + controller = self.Controller() + mutations = ( + ("target_key", 1234), + ("rxdelay", "nan"), + ("rxdelay", "inf"), + ("rxdelay", "-1"), + ("airtime_factor", 1.0), + ) + for field, value in mutations: + with self.subTest(field=field, value=value), \ + tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + path = work_dir / rak_chain.TRANSFER_SETTINGS_FILE + document = json.loads(path.read_text(encoding="ascii")) + document[field] = value + path.write_text(json.dumps(document), encoding="ascii") + with self.assertRaisesRegex( + ota.OtaError, "invalid saved transfer settings" + ): + rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + + +class Rak3401ExtractionCacheTests(unittest.TestCase): + @staticmethod + def make_archive(path: Path, root: str, marker: str) -> tuple[str, str]: + checksum_text = "placeholder checksum list\n" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(f"{root}/CHAIN.csv", "step,from_version\n") + archive.writestr(f"{root}/SHA256SUMS.txt", checksum_text) + archive.writestr(f"{root}/{marker}.txt", marker) + return ( + rak_chain.sha256_file(path), + hashlib.sha256(checksum_text.encode("ascii")).hexdigest(), + ) + + def test_extraction_cache_is_bound_to_exact_archive_hash(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = "RAK3401-test-bundle" + first = Path(directory) / "first.zip" + second = Path(directory) / "second.zip" + destination = Path(directory) / "bundle" + first_sha, checksum_sha = self.make_archive(first, root, "first") + second_sha, second_checksum_sha = self.make_archive( + second, root, "second" + ) + self.assertEqual(checksum_sha, second_checksum_sha) + with mock.patch.dict( + rak_chain.PINNED_ARCHIVE_CHECKSUMS, + { + first_sha: checksum_sha, + second_sha: checksum_sha, + }, + clear=True, + ): + extracted = rak_chain.extract_bundle( + first, destination, first_sha + ) + self.assertEqual(extracted, destination / root) + with self.assertRaisesRegex( + ota.OtaError, "bound to a different archive" + ): + rak_chain.extract_bundle(second, destination, second_sha) + + def test_matching_legacy_cache_is_adopted_only_after_checksum_proof(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = "RAK3401-test-bundle" + archive = Path(directory) / "bundle.zip" + destination = Path(directory) / "bundle" + archive_sha, checksum_sha = self.make_archive( + archive, root, "candidate" + ) + with mock.patch.dict( + rak_chain.PINNED_ARCHIVE_CHECKSUMS, + {archive_sha: checksum_sha}, + clear=True, + ): + rak_chain.extract_bundle(archive, destination, archive_sha) + binding = destination / rak_chain.EXTRACTION_BINDING_FILE + binding.unlink() + extracted = rak_chain.extract_bundle( + archive, destination, archive_sha + ) + self.assertEqual(extracted, destination / root) + self.assertEqual( + json.loads(binding.read_text(encoding="ascii"))[ + "archive_sha256" + ], + archive_sha, + ) + + def test_extraction_never_removes_a_preexisting_part_directory(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = "RAK3401-test-bundle" + parent = Path(directory) + archive = parent / "bundle.zip" + destination = parent / "bundle" + old_fixed_scratch = parent / f"bundle.part-{os.getpid()}" + old_fixed_scratch.mkdir() + sentinel = old_fixed_scratch / "keep.txt" + sentinel.write_text("owned by caller", encoding="ascii") + archive_sha, checksum_sha = self.make_archive( + archive, root, "candidate" + ) + with mock.patch.dict( + rak_chain.PINNED_ARCHIVE_CHECKSUMS, + {archive_sha: checksum_sha}, + clear=True, + ): + rak_chain.extract_bundle(archive, destination, archive_sha) + self.assertEqual(sentinel.read_text(encoding="ascii"), "owned by caller") + + def test_extraction_uses_frozen_archive_after_caller_path_changes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = "RAK3401-test-bundle" + parent = Path(directory) + archive = parent / "bundle.zip" + replacement = parent / "replacement.zip" + destination = parent / "extracted" + archive_sha, checksum_sha = self.make_archive( + archive, root, "first" + ) + self.make_archive(replacement, root, "replacement") + original_freeze = rak_chain.freeze_archive + + def freeze_then_replace(source: Path, frozen: Path) -> str: + digest = original_freeze(source, frozen) + os.replace(replacement, source) + return digest + + with ( + mock.patch.dict( + rak_chain.PINNED_ARCHIVE_CHECKSUMS, + {archive_sha: checksum_sha}, + clear=True, + ), + mock.patch.object( + rak_chain, + "freeze_archive", + side_effect=freeze_then_replace, + ), + ): + extracted = rak_chain.extract_bundle( + archive, destination, archive_sha + ) + + self.assertTrue((extracted / "first.txt").is_file()) + self.assertFalse((extracted / "replacement.txt").exists()) + + def test_archive_member_limit_is_enforced_before_extraction(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = "RAK3401-test-bundle" + parent = Path(directory) + archive = parent / "bundle.zip" + destination = parent / "extracted" + archive_sha, checksum_sha = self.make_archive( + archive, root, "candidate" + ) + with ( + mock.patch.dict( + rak_chain.PINNED_ARCHIVE_CHECKSUMS, + {archive_sha: checksum_sha}, + clear=True, + ), + mock.patch.object(rak_chain, "MAX_BUNDLE_MEMBER_BYTES", 8), + self.assertRaisesRegex(ota.OtaError, "exceeds"), + ): + rak_chain.extract_bundle(archive, destination, archive_sha) + self.assertFalse(destination.exists()) + + @staticmethod + def make_extracted_tree(root: Path) -> tuple[Path, str]: + bundle = root / "RAK3401-test-bundle" + bundle.mkdir() + payload = bundle / "motas" / "step.mota" + payload.parent.mkdir() + payload.write_bytes(b"pinned package bytes") + checksum = hashlib.sha256(payload.read_bytes()).hexdigest() + checksum_text = f"{checksum} motas/step.mota\n" + checksum_path = bundle / "SHA256SUMS.txt" + checksum_path.write_text(checksum_text, encoding="ascii") + return bundle, hashlib.sha256(checksum_text.encode("ascii")).hexdigest() + + def test_bundle_snapshot_is_immune_to_mutation_after_verification(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bundle, checksum_digest = self.make_extracted_tree(root) + snapshot_parent = root / "snapshot" + snapshot_parent.mkdir() + with mock.patch.object( + rak_chain, "CHECKSUM_LIST_SHA256", checksum_digest + ): + snapshot = rak_chain.snapshot_verified_bundle( + bundle, snapshot_parent + ) + (bundle / "motas" / "step.mota").write_bytes(b"changed by caller") + + self.assertEqual( + (snapshot / "motas" / "step.mota").read_bytes(), + b"pinned package bytes", + ) + + def test_bundle_snapshot_detects_mutation_during_copy(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bundle, checksum_digest = self.make_extracted_tree(root) + snapshot_parent = root / "snapshot" + snapshot_parent.mkdir() + original_copy = shutil.copyfileobj + + def copy_then_mutate(source: object, destination: object, length: int) -> None: + original_copy(source, destination, length) + if Path(source.name).name == "SHA256SUMS.txt": + (bundle / "motas" / "step.mota").write_bytes(b"changed mid-copy") + + with ( + mock.patch.object( + rak_chain, "CHECKSUM_LIST_SHA256", checksum_digest + ), + mock.patch.object( + rak_chain.shutil, "copyfileobj", side_effect=copy_then_mutate + ), + self.assertRaisesRegex( + ota.OtaError, "changed size|checksum mismatch" + ), + ): + rak_chain.snapshot_verified_bundle(bundle, snapshot_parent) + + def test_bundle_snapshot_rejects_large_unlisted_file_before_copy(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bundle, checksum_digest = self.make_extracted_tree(root) + (bundle / "unlisted.bin").write_bytes(b"x" * 32) + snapshot_parent = root / "snapshot" + snapshot_parent.mkdir() + with ( + mock.patch.object( + rak_chain, "CHECKSUM_LIST_SHA256", checksum_digest + ), + mock.patch.object(rak_chain, "MAX_BUNDLE_MEMBER_BYTES", 24), + mock.patch.object(rak_chain.shutil, "copyfileobj") as copy_file, + self.assertRaisesRegex(ota.OtaError, "limit"), + ): + rak_chain.snapshot_verified_bundle(bundle, snapshot_parent) + copy_file.assert_not_called() + + def test_bundle_snapshot_rejects_listed_file_growth_before_its_copy(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bundle, checksum_digest = self.make_extracted_tree(root) + snapshot_parent = root / "snapshot" + snapshot_parent.mkdir() + payload = bundle / "motas" / "step.mota" + original_copy = rak_chain.copy_regular_file_limited + + def grow_then_copy( + source: Path, + destination: Path, + maximum: int, + label: str, + *, + expected_size: int | None = None, + ) -> int: + if source == payload: + with source.open("ab") as output: + output.write(b"growth after inventory") + return original_copy( + source, + destination, + maximum, + label, + expected_size=expected_size, + ) + + with ( + mock.patch.object( + rak_chain, "CHECKSUM_LIST_SHA256", checksum_digest + ), + mock.patch.object( + rak_chain, + "copy_regular_file_limited", + side_effect=grow_then_copy, + ), + self.assertRaisesRegex(ota.OtaError, "changed size"), + ): + rak_chain.snapshot_verified_bundle(bundle, snapshot_parent) + self.assertFalse((snapshot_parent / bundle.name).exists()) + + def test_chain_state_rejects_redirected_steps_and_progress_paths(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / "work" + work.mkdir() + external = root / "external" + external.mkdir() + (work / "steps").symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ota.OtaError, "real directory"): + rak_chain.validate_chain_state_paths(work) + + if hasattr(os, "mkfifo"): + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + os.mkfifo(work / "progress.jsonl") + with self.assertRaisesRegex(ota.OtaError, "regular file"): + rak_chain.validate_chain_state_paths(work) + + def test_progress_append_refuses_symlink_without_touching_target(self) -> None: + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + victim = work / "victim.txt" + victim.write_text("unchanged\n", encoding="ascii") + (work / "progress.jsonl").symlink_to(victim) + step = mock.Mock( + number=1, from_version="1.0", to_version="1.1" + ) + with self.assertRaisesRegex(ota.OtaError, "symbolic link"): + rak_chain.append_progress(work, step, b"12345678") + self.assertEqual(victim.read_text(encoding="ascii"), "unchanged\n") + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO test requires POSIX") + def test_progress_append_refuses_fifo_without_blocking(self) -> None: + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + os.mkfifo(work / "progress.jsonl") + step = mock.Mock( + number=1, from_version="1.0", to_version="1.1" + ) + with self.assertRaisesRegex(ota.OtaError, "safely open"): + rak_chain.append_progress(work, step, b"12345678") + + def test_attempt_work_directory_is_reserved_under_real_steps_dir(self) -> None: + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + attempt = rak_chain.next_attempt_dir(work, 3) + self.assertEqual(attempt.name, "work") + self.assertTrue(attempt.parent.is_dir()) + self.assertFalse(attempt.exists()) + + def test_work_directory_cannot_be_inside_bundle(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bundle = root / "bundle" + work = bundle / "work" + work.mkdir(parents=True) + with self.assertRaisesRegex(ota.OtaError, "outside the supplied bundle"): + rak_chain.require_bundle_work_separation(bundle, work) + + def test_chain_rejects_controller_serial_aliases_before_live_work(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + device = root / "ttyACM0" + device.touch() + alias = root / "by-id-radio" + alias.symlink_to(device) + parser = rak_chain.build_parser() + for source_arguments in ( + ["--source-serial", str(alias)], + [ + "--source-tcp", "192.0.2.10:5001", + "--source-cli-serial", str(alias), + ], + ): + args = parser.parse_args([ + "--controller-serial", str(device), *source_arguments + ]) + with self.subTest(source_arguments=source_arguments), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + rak_chain.validate_args(args, parser) + + def test_chain_source_cli_transports_are_mutually_exclusive(self) -> None: + parser = rak_chain.build_parser() + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + parser.parse_args([ + "--source-tcp", "192.0.2.10:5001", + "--source-cli-serial", "/dev/source", + "--source-cli-tcp", "192.0.2.10:5002", + ]) + + def test_chain_already_temp_source_rejects_managed_cli(self) -> None: + parser = rak_chain.build_parser() + args = parser.parse_args([ + "--source-tcp", "192.0.2.10:5001", + "--source-cli-tcp", "192.0.2.10:5002", + "--source-already-temp", + ]) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + rak_chain.validate_args(args, parser) + + def test_chain_endpoint_recovers_persisted_source_rxps(self) -> None: + saved = ota.RxpsSettings(True, 18205, 20423, 8, 16) + current = ota.RxpsSettings(False, 18205, 20423, 8, 16) + source_args = argparse.Namespace( + source_cli_serial=None, + source_serial="/dev/source", + source_cli_tcp=None, + source_baud=115200, + ) + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + ota.write_source_rxps_recovery(work_dir, source_args, saved) + with ( + mock.patch.object(ota, "read_source_rxps", return_value=current), + mock.patch.object( + ota, "shorten_source_temp_window", return_value=True + ) as shorten, + mock.patch.object(ota, "restore_source_rxps") as restore, + ): + rak_chain.restore_persisted_source_rxps( + work_dir, source_args + ) + self.assertFalse( + (work_dir / ota.SOURCE_RXPS_RECOVERY_FILE).exists() + ) + shorten.assert_called_once_with(source_args) + restore.assert_called_once_with(source_args, saved) + + def test_chain_endpoint_proves_normalradio_even_when_rxps_already_matches(self) -> None: + saved = ota.RxpsSettings(False, 18205, 20423, 8, 16) + source_args = argparse.Namespace( + source_cli_serial=None, + source_serial="/dev/source", + source_cli_tcp=None, + source_baud=115200, + ) + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + ota.write_source_rxps_recovery(work_dir, source_args, saved) + with ( + mock.patch.object(ota, "read_source_rxps", return_value=saved), + mock.patch.object( + ota, "shorten_source_temp_window", return_value=True + ) as shorten, + mock.patch.object(ota, "restore_source_rxps") as restore, + ): + rak_chain.restore_persisted_source_rxps(work_dir, source_args) + self.assertFalse( + (work_dir / ota.SOURCE_RXPS_RECOVERY_FILE).exists() + ) + shorten.assert_called_once_with(source_args) + restore.assert_not_called() + class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None: @@ -2319,6 +3772,23 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): controller.commands, ["ota status", "ota cancel", "ota status"] ) + def test_completed_chain_step_does_not_claim_idle_store_is_cleared(self) -> None: + class Controller: + def __init__(self) -> None: + self.commands: list[str] = [] + self.replies = iter([ + "OTA | no download | target:2FA509C1", + ]) + + def remote_command(self, _target: str, command: str) -> str: + self.commands.append(command) + return next(self.replies) + + controller = Controller() + step = mock.Mock(number=12, package=mock.Mock(manifest_id="1234ABCD")) + rak_chain.clear_completed_download(controller, "remote", step) + self.assertEqual(controller.commands, ["ota status"]) + def test_completed_chain_step_refuses_another_retained_manifest(self) -> None: class Controller: def remote_command(self, _target: str, _command: str) -> str: @@ -2340,6 +3810,77 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): with self.assertRaisesRegex(ota.OtaError, "refusing to expose"): rak_chain.require_rescue_capability(controller, "remote") + def test_chain_requires_rescue_for_intermediate_start_or_resume_only(self) -> None: + controller = mock.Mock() + with mock.patch.object( + rak_chain, "require_rescue_capability" + ) as require_rescue: + rak_chain.require_rescue_capability_before_next_transition( + controller, "remote", 0, 2 + ) + require_rescue.assert_not_called() + + rak_chain.require_rescue_capability_before_next_transition( + controller, "remote", 1, 2 + ) + require_rescue.assert_called_once_with(controller, "remote") + + require_rescue.reset_mock() + rak_chain.require_rescue_capability_before_next_transition( + controller, "remote", 2, 2 + ) + require_rescue.assert_not_called() + + def test_chain_step_propagates_debug_to_nested_runner(self) -> None: + args = argparse.Namespace( + controller_serial="/dev/controller", + controller_tcp=None, + controller_ble=None, + source_serial="/dev/source", + source_tcp=None, + source_cli_serial=None, + source_cli_tcp=None, + source_already_temp=False, + source_shares_controller=False, + controller_baud=115200, + source_baud=115200, + relay_txdelay=0.3, + temp_radio="909.950,500,5,5,120", + meshcli="meshcli", + motatool="motatool", + reply_timeout=45, + discovery_timeout=180, + discovery_interval=8, + poll_seconds=60, + transfer_timeout_minutes=90, + seeder_start_wait=5, + reboot_wait=90, + relay=[], + debug=False, + ) + step = mock.Mock( + number=1, + path=Path("step-01.mota"), + package=mock.Mock(manifest_id="1234ABCD"), + ) + controller = mock.Mock() + + for enabled in (False, True): + with self.subTest(debug=enabled), tempfile.TemporaryDirectory() as directory: + args.debug = enabled + with mock.patch.object(rak_chain.ota, "main", return_value=0) as nested: + rak_chain.run_step( + args, + "remote", + step, + None, + bytes.fromhex("0011223344556677"), + Path(directory), + controller, + ) + command = nested.call_args.args[0] + self.assertEqual(command.count("--debug"), int(enabled)) + def test_live_chain_is_blocked_after_failed_physical_step_6(self) -> None: self.assertEqual(rak_chain.KNOWN_UNSAFE_STEP, 6) self.assertEqual(rak_chain.KNOWN_UNSAFE_VERSION, "1.16.8.7") @@ -2436,6 +3977,42 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): argparse.Namespace(accept_test_candidate=False), steps ) + def test_exact_current_ten_step_candidate_requires_explicit_lab_ack(self) -> None: + steps = [mock.Mock(target_sha256="") for _ in range(10)] + for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS: + steps[number - 1].target_sha256 = image_sha256 + with self.assertRaisesRegex( + rak_chain.KnownUnsafeReleaseError, + "package transitions completed directly on the physical RAK3401", + ): + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=False), steps + ) + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=True), steps + ) + + def test_changed_current_ten_step_anchor_is_not_recognized(self) -> None: + steps = [mock.Mock(target_sha256="") for _ in range(10)] + for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS: + steps[number - 1].target_sha256 = image_sha256 + steps[-1].target_sha256 = "00" * 32 + with self.assertRaisesRegex( + rak_chain.KnownUnsafeReleaseError, + "unrecognized variant of the pinned ten-step candidate", + ): + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=True), steps + ) + + def test_unreleased_candidate_has_no_implicit_download(self) -> None: + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / rak_chain.ASSET_NAME + with self.assertRaisesRegex( + ota.OtaError, "not released.*--bundle" + ): + rak_chain.download_release_asset(destination) + def test_compact_9_step_release_rejects_changed_anchor(self) -> None: steps = [mock.Mock(target_sha256="") for _ in range(9)] for number, image_sha256 in rak_chain.COMPACT_RELEASE_ANCHORS: diff --git a/tools/lora_ota/test_rak3401_route_search.py b/tools/lora_ota/test_rak3401_route_search.py new file mode 100644 index 00000000..74d8b7e7 --- /dev/null +++ b/tools/lora_ota/test_rak3401_route_search.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +import csv +import hashlib +import json +from pathlib import Path +import struct +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import rak3401_route_search as search + + +def firmware(body: bytes, version: str) -> bytes: + hardware = search.EXPECTED_HARDWARE.encode("ascii").ljust(32, b"\0") + return ( + body + + b"EndF" + + struct.pack(" tuple[Path, list[dict[str, object]]]: + records = [] + for node, payload in enumerate(payloads): + version = versions[node] if versions is not None else f"1.0.0.{node}" + encoded = firmware(payload, version) + image = root / f"image-{node:02d}.bin" + image.write_bytes(encoded) + record: dict[str, object] = { + "node": node, + "path": image.name, + "size": len(encoded), + "sha256": hashlib.sha256(encoded).hexdigest(), + "body_hash": hashlib.sha256(payload).digest()[:8].hex(), + "version": version, + } + if node == 1: + record["baseline_container_size"] = baseline_size + records.append(record) + manifest = root / "images.json" + manifest.write_text(json.dumps({"images": records}), encoding="ascii") + return manifest, records + + def test_cache_migration_survives_inserted_node_and_remaps_indices(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old_dir, new_dir = root / "old", root / "new" + old_dir.mkdir() + new_dir.mkdir() + old_manifest, old = self.write_inventory( + old_dir, [b"A", b"B", b"C"], + versions=["1.0.0.0", "1.0.0.1", "1.0.0.2"], + ) + cache = old_dir / "geometry.csv" + with cache.open("w", newline="", encoding="ascii") as output: + writer = csv.DictWriter(output, fieldnames=search.FIELDS) + writer.writeheader() + writer.writerow({ + "source": 1, "target": 2, "memory": search.FIXED_MEMORY, + "payload": 7, + "container": search.container_size(7), + "stage_start": search.align_down( + search.STAGE_CEILING - search.container_size(7) + ), + "margin": search.align_down( + search.STAGE_CEILING - search.container_size(7) + ) - (search.APP_BASE + search.FIXED_MEMORY), + "feasible": True, "error": "", + }) + new_manifest, _ = self.write_inventory( + new_dir, [b"A", b"X", b"B", b"C"], + versions=["1.0.0.0", "1.0.0.99", "1.0.0.1", "1.0.0.2"], + ) + migrated = search.migrate_csv(old_manifest, cache) + key = search.cache_key(str(old[1]["sha256"]), str(old[2]["sha256"]), + search.FIXED_MEMORY) + self.assertIn(key, migrated) + projected = search.project_cache( + migrated, search.load_inventory(new_manifest), {key} + ) + self.assertEqual((projected[0]["source"], projected[0]["target"]), (2, 3)) + + def test_all_jobs_covers_every_valid_page_workspace(self) -> None: + with tempfile.TemporaryDirectory() as directory: + manifest, _ = self.write_inventory( + Path(directory), [b"a", b"b", b"c" * 10000, b"d" * 12000] + ) + jobs = search.all_jobs(search.load_inventory(manifest)) + fixed = [job for job in jobs if job[0] < 2] + dynamic = [job for job in jobs if job[0] >= 2] + self.assertEqual(len(fixed), 2) + self.assertTrue(all(job[2] == search.FIXED_MEMORY for job in fixed)) + self.assertTrue(all(job[0] == 1 for job in fixed)) + inventory = search.load_inventory(manifest) + first_page = search.align_up( + max( + int(inventory[2]["size"]) + 2 * search.PAGE, + int(inventory[3]["size"]), + ) + ) // search.PAGE + self.assertEqual( + [job[2] for job in dynamic], + [page * search.PAGE for page in range(first_page, search.AVAILABLE_PAGES)], + ) + self.assertEqual(len(jobs), 2 + search.AVAILABLE_PAGES - first_page) + + def test_route_ignores_geometry_that_bypasses_pinned_first_bridge(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory(root, [b"0", b"1", b"2"]) + images = search.load_inventory(manifest) + result = search.select_route( + [ + { + "source": 0, "target": 2, "memory": search.FIXED_MEMORY, + "container": 1, "stage_start": 0xC0000, + "margin": 10, "feasible": True, + }, + { + "source": 1, "target": 2, "memory": search.FIXED_MEMORY, + "container": 10, "stage_start": 0xC0000, + "margin": 10, "feasible": True, + }, + ], + images, 100, root / "route.json", True, + ) + self.assertEqual(result["nodes"], [0, 1, 2]) + + def test_cache_migration_rejects_inconsistent_numeric_geometry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory(root, [b"A", b"B", b"C"]) + cache = root / "geometry.csv" + with cache.open("w", newline="", encoding="ascii") as output: + writer = csv.DictWriter(output, fieldnames=search.FIELDS) + writer.writeheader() + writer.writerow({ + "source": 1, "target": 2, "memory": search.FIXED_MEMORY, + "payload": 7, "container": 999, "stage_start": 800000, + "margin": 1, "feasible": True, "error": "", + }) + with self.assertRaisesRegex( + search.RouteSearchError, "inconsistent cached geometry" + ): + search.migrate_csv(manifest, cache) + + def test_cache_migration_rejects_failed_geometry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory(root, [b"A", b"B", b"C"]) + cache = root / "geometry.csv" + with cache.open("w", newline="", encoding="ascii") as output: + writer = csv.DictWriter(output, fieldnames=search.FIELDS) + writer.writeheader() + writer.writerow({ + "source": 1, "target": 2, "memory": search.FIXED_MEMORY, + "payload": -1, "container": -1, "stage_start": -1, + "margin": -1, "feasible": False, "error": "out of memory", + }) + with self.assertRaisesRegex( + search.RouteSearchError, "cannot prove an exhaustive search" + ): + search.migrate_csv(manifest, cache) + + def test_geometry_tool_error_aborts_instead_of_becoming_infeasible(self) -> None: + class BrokenDetools: + @staticmethod + def create_patch_filenames(*_args: object, **_kwargs: object) -> None: + raise OSError("disk full") + + with tempfile.TemporaryDirectory() as directory, mock.patch.dict( + sys.modules, {"detools": BrokenDetools} + ): + with self.assertRaisesRegex(search.RouteSearchError, "disk full"): + search.geometry_job(( + 1, 2, search.FIXED_MEMORY, + str(Path(directory) / "from.bin"), + str(Path(directory) / "to.bin"), + "1" * 64, "2" * 64, directory, + )) + + def test_snapshot_inventory_is_immune_to_later_source_mutation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + source.mkdir() + manifest, _ = self.write_inventory(source, [b"A", b"B", b"C"]) + frozen = search.snapshot_inventory( + manifest, root / "frozen" / "images.json" + ) + source_image = source / "image-02.bin" + source_image.write_bytes(b"changed") + images = search.load_inventory(frozen) + self.assertNotEqual(Path(images[2]["path"]).read_bytes(), b"changed") + + def test_generation_never_removes_a_preexisting_patches_directory(self) -> None: + class ImmediateFuture: + def __init__(self, value: dict[str, object]): + self._value = value + + def result(self) -> dict[str, object]: + return self._value + + class ImmediatePool: + def __init__(self, **_kwargs: object): + pass + + def __enter__(self) -> "ImmediatePool": + return self + + def __exit__(self, *_args: object) -> None: + pass + + def submit(self, function: object, *args: object) -> ImmediateFuture: + return ImmediateFuture(function(*args)) # type: ignore[operator] + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inventory_root = root / "inventory" + inventory_root.mkdir() + manifest, records = self.write_inventory( + inventory_root, [b"start", b"bridge", b"endpoint"] + ) + work = root / "work" + existing = work / "patches" + existing.mkdir(parents=True) + sentinel = existing / "keep.txt" + sentinel.write_text("caller-owned", encoding="ascii") + source_sha = str(records[1]["sha256"]) + target_sha = str(records[2]["sha256"]) + job = ( + 1, 2, search.FIXED_MEMORY, + str(inventory_root / str(records[1]["path"])), + str(inventory_root / str(records[2]["path"])), + source_sha, target_sha, + ) + + def measured(args: tuple[object, ...]) -> dict[str, object]: + source, target, memory, *_rest = args + payload = 7 + total = search.container_size(payload) + stage = search.align_down(search.STAGE_CEILING - total) + return { + "source": source, "target": target, + "source_sha256": source_sha, + "target_sha256": target_sha, + "memory": memory, "payload": payload, + "container": total, "stage_start": stage, + "margin": stage - (search.APP_BASE + int(memory)), + "feasible": True, "error": "", + } + + with ( + mock.patch.object(search, "all_jobs", return_value=[job]), + mock.patch.object(search, "geometry_job", side_effect=measured), + mock.patch.object(search, "ProcessPoolExecutor", ImmediatePool), + mock.patch.object(search, "as_completed", side_effect=lambda values: values), + ): + self.assertEqual( + search.main([ + "--inventory", str(manifest), + "--work-dir", str(work), + "--workers", "1", + ]), + 0, + ) + self.assertEqual(sentinel.read_text(encoding="ascii"), "caller-owned") + + def test_route_minimizes_hops_then_bytes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory(root, [b"0", b"1", b"2", b"3", b"4"]) + images = search.load_inventory(manifest) + + def edge(source: int, target: int, size: int) -> dict[str, object]: + return { + "source": source, "target": target, "memory": 0x1000, + "container": size, "stage_start": 0xC0000, + "margin": 100, "feasible": True, + } + + # Both endpoint routes have three hops including the pinned 0->1. + # The path through node 3 is smaller and must win even though it is + # encountered after the path through node 2. + rows = [edge(1, 2, 100), edge(2, 4, 100), + edge(1, 3, 40), edge(3, 4, 50)] + result = search.select_route(rows, images, 100, root / "route.json", True) + self.assertEqual(result["nodes"], [0, 1, 3, 4]) + self.assertEqual(result["shortest_package_count"], 3) + self.assertEqual(result["shortest_route_count"], 2) + self.assertEqual(result["selected_total_bytes"], 190) + + # A direct 1->4 edge wins on hop count even when it costs more. + rows.append(edge(1, 4, 1000)) + result = search.select_route(rows, images, 100, root / "route2.json", True) + self.assertEqual(result["nodes"], [0, 1, 4]) + self.assertEqual(result["selected_total_bytes"], 1100) + + def test_fixed_outputs_never_follow_preexisting_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + victim = root / "victim.txt" + victim.write_text("caller-owned", encoding="ascii") + geometry = root / "geometry.csv" + geometry.symlink_to(victim) + with self.assertRaisesRegex(search.RouteSearchError, "regular file"): + search.write_csv(geometry, []) + self.assertEqual(victim.read_text(encoding="ascii"), "caller-owned") + + manifest, _ = self.write_inventory( + root, [b"start", b"bridge", b"endpoint"] + ) + route = root / "route.json" + route.symlink_to(victim) + with self.assertRaisesRegex(search.RouteSearchError, "regular file"): + search.select_route( + [], search.load_inventory(manifest), 100, route, True + ) + self.assertEqual(victim.read_text(encoding="ascii"), "caller-owned") + + def test_invalid_inventory_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, records = self.write_inventory(root, [b"one", b"two"]) + cases = [] + bad = json.loads(manifest.read_text()) + bad["images"][1]["node"] = 3 + cases.append((bad, "contiguous")) + bad = json.loads(manifest.read_text()) + bad["images"][1]["sha256"] = bad["images"][0]["sha256"] + cases.append((bad, "SHA mismatch")) + bad = json.loads(manifest.read_text()) + bad["images"][1]["path"] = "missing.bin" + cases.append((bad, "missing inventory image")) + for number, (document, message) in enumerate(cases): + candidate = root / f"bad-{number}.json" + candidate.write_text(json.dumps(document), encoding="ascii") + with self.subTest(message=message), self.assertRaisesRegex( + search.RouteSearchError, message + ): + search.load_inventory(candidate) + + def test_bridges_are_inserted_before_preserved_endpoint(self) -> None: + original = { + "schema": 1, + "images": [ + {"node": 0, "kind": "start"}, + {"node": 1, "kind": "old-bridge"}, + {"node": 2, "kind": "endpoint", "path": "endpoint.bin"}, + ], + } + result = search.insert_bridges_before_endpoint( + original, + [{"node": 99, "kind": "new-bridge-1"}, {"kind": "new-bridge-2"}], + ) + self.assertEqual( + [record["node"] for record in result["images"]], [0, 1, 2, 3, 4] + ) + self.assertEqual( + [record["kind"] for record in result["images"]], + ["start", "old-bridge", "new-bridge-1", "new-bridge-2", "endpoint"], + ) + self.assertEqual(result["images"][-1]["path"], "endpoint.bin") + self.assertEqual(len(original["images"]), 3) + + +if __name__ == "__main__": + unittest.main()