From 610eef7529a5e68c810407f8ec1b4cbcf5ba1044 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Wed, 19 Aug 2026 11:43:25 -0700 Subject: [PATCH] Avoid legacy OTA proof collisions --- docs/ota_easy.md | 21 ++++++++- docs/ota_protocol.md | 21 ++++++--- docs/rak3401_mota_chain.md | 9 ++-- src/helpers/ota/OtaManager.cpp | 28 ++++++++++++ src/helpers/ota/OtaManager.h | 17 ++++++- test/test_ota/test_ota_core.cpp | 57 ++++++++++++++++++++++-- tools/lora_ota/rebuild_rak3401_bundle.py | 2 +- 7 files changed, 136 insertions(+), 19 deletions(-) diff --git a/docs/ota_easy.md b/docs/ota_easy.md index a35bda6dc..aaed64324 100644 --- a/docs/ota_easy.md +++ b/docs/ota_easy.md @@ -240,7 +240,19 @@ check fails. ### 1. Start the temporary OTA channel -On the source node, destination node, and every intermediate repeater, run: +First save the current `ota config` output. For a direct source-to-destination link, set both OTA-enabled +endpoints to direct-only mode before starting the transfer: + +```text +ota config hops 0 +``` + +Do not use `hops 1` as extra margin on a direct link. It tells a receiving repeater to retransmit each direct +DATA/PROOF packet; that unnecessary half-duplex transmission can make it miss the source's next fragment, +especially with older single-block receivers. Use `hops 1` only when one real intermediate relay is required, +`hops 2` for two, and so on. This setting is saved, so restore the original value after the maintenance window. + +On the source node, destination node, and every intermediate repeater, then run: ```text tempradio 909.950,250,5,5,120 @@ -295,6 +307,10 @@ Monitor the transfer: ota status ``` +For a powered bench update where restart-resume is not needed, `ota config checkpoint 0` on the destination +removes periodic progress writes. It is a smaller optimization than selecting the correct hop count and it +trades away persisted mid-download resume; restore the previous checkpoint cadence afterward. + The update is ready when the status says `ready to install`. Discovery is background traffic, while an active OTA download is primary mesh traffic. At the temporary-radio settings in this guide, allow roughly **one hour** for a typical ESP32 full image over a quiet, direct link. That is a planning estimate, not an @@ -340,7 +356,8 @@ ota status - **nRF52 reports a base mismatch:** the file passed to `--base` is not the exact application running on the destination. Rebuild the delta from the correct saved `firmware.hex`. - **The download stalls:** check the source, destination, and intermediate repeaters. Restart matching, - overlapping temporary-radio windows if one expired. + overlapping temporary-radio windows if one expired. On a direct link, also require `ota config hops 0`; + a larger reach makes a repeater destination unnecessarily echo source packets. - **The serial port is busy:** close the serial terminal before starting `motatool serve`. - **The destination rejects the package:** verify the source files, their `EndF` trailers, the package's target/hardware identity, and the result of `motatool verify`. diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index e83a79984..805d85f49 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -519,10 +519,14 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up reassembly bitmap (<=16 fragments/block; 1 KB blocks = 7). `OTA_PROOF` is a single packet and needs no mask. - **Data and proof remain separate packets, without a normal extra round trip.** A server retains requested blocks in a bounded descriptor queue, admits at most one response per main-loop pass, and sends one - `OTA_PROOF` after each block's requested fragments. The receiver uses an airtime/path-aware proof grace - (never less than 500 ms) and defers `OTA_REQ_PROOF` while any flight slot still expects DATA. Thus a missing - early proof cannot make the receiver transmit into the rest of a legitimate half-duplex response train. - After the complete-flight deadline, only one slot's missing fragments/proof is requested at a time. + `OTA_PROOF` after each block's requested fragments. Before admitting that proactive proof, the source leaves + an airtime/duty-aware 100-3000 ms RX turnaround gap. It accounts for one active transmission plus the two + paced-response queue credits. A legacy receiver uses the gap to send its immediate + `OTA_REQ_PROOF`; receiving that explicit request bypasses the remaining gap, avoiding a proof/request + collision and an otherwise multi-second legacy retry. A newer receiver uses an airtime/path-aware proof + grace (never less than 500 ms) and defers `OTA_REQ_PROOF` while any flight slot still expects DATA. Thus a + missing early proof cannot make the receiver transmit into the rest of a legitimate half-duplex response + train. After the complete-flight deadline, only one slot's missing fragments/proof is requested at a time. ### 8.5 Sizing against `MAX_PACKET_PAYLOAD = 184` @@ -674,9 +678,12 @@ Manifest fragments are retained as bounded response jobs and admitted one at a time. Their source-side gap follows the active maximum packet airtime and dispatcher duty spacing, clamped to 100-1000 ms. The 100 ms floor protects fast radios' TX-to-RX turnaround; the cap keeps the receiver's one-second -manifest progress/retry observation responsive. Relay collision delay is a -separate setting: active OTA floods honor the relay role's configured -`txdelay`, and the deployment runner temporarily uses `0.3` on managed relays. +manifest progress/retry observation responsive. The source uses the same +radio-aware 100-3000 ms drain/turnaround gap before an unsolicited block proof, +but immediately serves a +legacy receiver's explicit `OTA_REQ_PROOF`. Relay collision delay is a separate +setting: active OTA floods honor the relay role's configured `txdelay`, and the +deployment runner temporarily uses `0.3` on managed relays. **What to plug into `--serial`.** Use the USB serial console of an OTA-enabled MeshCore node built with `OTA_FOLDER_SERIAL`. The node must have a working LoRa radio plus an diff --git a/docs/rak3401_mota_chain.md b/docs/rak3401_mota_chain.md index bb49f3c55..823405ae8 100644 --- a/docs/rak3401_mota_chain.md +++ b/docs/rak3401_mota_chain.md @@ -118,7 +118,7 @@ python3 tools/lora_ota/rak3401_mota_chain.py \ --source-shares-controller \ --target-key 63d8df63 \ --temp-radio 909.950,500,5,5,120 \ - --ota-hops 1 \ + --ota-hops 0 \ --motatool /path/to/motatool \ --yes ``` @@ -179,7 +179,7 @@ after every bridge reboot: powersaving off set radio.rxps off set rxdelay 0 -ota config hops 1 +ota config hops 0 tempradio 909.950,500,5,5,120 ``` @@ -187,7 +187,10 @@ tempradio 909.950,500,5,5,120 failure. The actual source failure was an oversized USB CDC reply; current firmware splits those reads. Some historical bridge builds may report RXPS as unsupported while already operating continuously. Verify the reported state -and do not substitute a normal reboot for the one-time watchdog reset. +and do not substitute a normal reboot for the one-time watchdog reset. Direct +tests use `hops 0`: `hops 1` makes the target echo each source response despite +there being no intermediate relay, increasing half-duplex loss and legacy +three-second retries. Put the source on the identical TempRadio tuple. If it is a binary-mode Full Companion, the current `motatool` can switch modes for the serving session: diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index 98eb09b2b..779ba1d9f 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -419,6 +419,7 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want ServeJob& job = _serve_jobs[i]; if (job.block != block || memcmp(job.mid, mid, 4) != 0) continue; job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask); + if (want_mask == 0) job.proof_requested = true; return true; } if (_n_serve_jobs >= OTA_SERVE_QUEUE) return false; @@ -427,6 +428,8 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want job.block = block; job.pending_mask = want_mask; job.emitted_mask = 0; + job.proof_ready_at = 0; + job.proof_requested = want_mask == 0; return true; } @@ -455,6 +458,18 @@ uint32_t OtaManager::manifestEgressGapMs() const { return (uint32_t)gap; } +uint32_t OtaManager::proofEgressGapMs() const { + if (_radio_packet_airtime_ms == 0) return OTA_PROOF_EGRESS_MIN_GAP_MS; + // The adapter may have one response transmitting and two more admitted. Start this timer when the final + // DATA is accepted, so cover all three packet-service intervals before opening the legacy-request turn. + uint64_t gap = (uint64_t)_radio_packet_airtime_ms * _tx_spacing_permille + * OTA_PROOF_EGRESS_DRAIN_PACKETS; + gap = (gap + 999u) / 1000u; + if (gap < OTA_PROOF_EGRESS_MIN_GAP_MS) gap = OTA_PROOF_EGRESS_MIN_GAP_MS; + if (gap > OTA_PROOF_EGRESS_MAX_GAP_MS) gap = OTA_PROOF_EGRESS_MAX_GAP_MS; + return (uint32_t)gap; +} + void OtaManager::clearPendingEgress() { _n_manifest_jobs = 0; _n_serve_jobs = 0; @@ -635,10 +650,23 @@ void OtaManager::serviceEgress() { const uint16_t bit = (uint16_t)(1u << fragment); job.pending_mask &= (uint16_t)~bit; job.emitted_mask |= bit; + if (job.pending_mask == 0) { + // Legacy receivers transmit REQ_PROOF as soon as the last DATA fragment arrives. Do not admit our + // proactive proof into the radio queue at the same instant: the two half-duplex transmissions would + // collide and force the receiver onto its multi-second retry tick. Include every response that the + // adapter can already have admitted ahead of this proof, then retain a bounded fast-link floor. + job.proof_ready_at = _now_ms + proofEgressGapMs(); + } } return; } + // An explicit legacy REQ_PROOF proves that the source has returned to RX and bypasses the proactive delay. + // A newer receiver sends no request unless the proof was lost, so its normal proactive path waits for the + // radio-aware gap and remains compatible with the existing proof-grace deadline. + if (!job.proof_requested && job.proof_ready_at != 0 + && (int32_t)(_now_ms - job.proof_ready_at) < 0) return; + uint8_t proof[32 * 4]; uint8_t np = merkle_gen_proof(v->m.leaves, v->m.block_count, job.block, v->scratch, proof); ProofMsg pm; diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index d81770c1d..0ca7f367b 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -16,8 +16,9 @@ // driven by a host simulation; a thin Mesh adapter wires it to PAYLOAD_TYPE_OTA on device. // // Transfer keeps 1 KB logical blocks. A server paces each block's self-describing DATA fragments through a -// bounded response queue and follows them with the Merkle PROOF. A newer fetcher waits briefly for that -// proactive proof; the existing REQ_PROOF message remains as a loss/legacy-server fallback. +// bounded response queue and follows them with the Merkle PROOF. A short radio-aware turnaround gap lets a +// legacy fetcher send its immediate REQ_PROOF without colliding with that proactive proof; newer fetchers wait +// briefly for the proactive proof and retain REQ_PROOF as a loss/legacy-server fallback. namespace mesh { namespace ota { @@ -68,6 +69,15 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #ifndef OTA_MANIFEST_EGRESS_MAX_GAP_MS #define OTA_MANIFEST_EGRESS_MAX_GAP_MS 1000 // do not outrun the receiver's one-second retry observation #endif +#ifndef OTA_PROOF_EGRESS_DRAIN_PACKETS +#define OTA_PROOF_EGRESS_DRAIN_PACKETS 3 // one active TX plus the two paced-response queue credits +#endif +#ifndef OTA_PROOF_EGRESS_MIN_GAP_MS +#define OTA_PROOF_EGRESS_MIN_GAP_MS 100 // fast-link legacy request turnaround floor +#endif +#ifndef OTA_PROOF_EGRESS_MAX_GAP_MS +#define OTA_PROOF_EGRESS_MAX_GAP_MS 3000 // bounded by the legacy receiver's normal retry cadence +#endif // Leaf-diff warm-start (motatool folder-capture only): bulk-fetch the target's leaves[], diff a seed build // locally, pull DATA only for mismatches. OTA_LEAVES is bitmap-fragmented like OTA_MANIFEST so a want_mask // retry never re-bursts (anti-deadlock). The mask is a fixed uint16, so leaves are capped at MAXFRAG frags. @@ -493,6 +503,7 @@ private: bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask); bool queueManifestJob(const uint8_t* mid, uint16_t want_mask); uint32_t manifestEgressGapMs() const; + uint32_t proofEgressGapMs() const; bool serviceManifestEgress(); void popManifestJob(); bool loadActiveServeBlock(); @@ -555,6 +566,8 @@ private: uint16_t block = 0; uint16_t pending_mask = 0; uint16_t emitted_mask = 0; + uint32_t proof_ready_at = 0; + bool proof_requested = false; }; ServeJob _serve_jobs[OTA_SERVE_QUEUE]; uint8_t _n_serve_jobs = 0; diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index db3e0b72b..be1be00f2 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -907,6 +907,9 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(data.frag_off, i * OTA_FRAG_DATA); } EXPECT_EQ(server.pendingServeJobs(), 1u); // proof is retained behind DATA + server.serviceEgress(); // reserve RX turnaround for legacy REQ_PROOF + ASSERT_EQ(sent.items.size(), fragment_count); + server.set_clock(OTA_MANIFEST_EGRESS_MIN_GAP_MS); server.serviceEgress(); ASSERT_EQ(sent.items.size(), fragment_count + 1); ProofMsg proof; @@ -916,6 +919,48 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(server.pendingServeJobs(), 0u); } +TEST(OtaTransfer, LegacyProofRequestBypassesProactiveProofTurnaround) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + + OtaManager server; + CapturedMessages sent; + server.begin(0, capture_send, &sent); + server.set_link_timing(80, 2000); // 3 admitted packet intervals = 480 ms + ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN)); + + ReqMsg request; + memcpy(request.manifest_id, manifest.merkle_root, 4); + request.block_idx = 0; + request.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + uint16_t wire_len = encode_req(wire, sizeof(wire), request); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(server.on_message(wire, wire_len)); + + const uint32_t fragment_count = + (manifest.block_size() + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA; + for (uint32_t i = 0; i < fragment_count; i++) server.serviceEgress(); + ASSERT_EQ(sent.items.size(), fragment_count); + + server.set_clock(479); + server.serviceEgress(); // no unsolicited proof before turnaround + ASSERT_EQ(sent.items.size(), fragment_count); + + ReqProofMsg proof_request; + memcpy(proof_request.manifest_id, manifest.merkle_root, 4); + proof_request.block_idx = 0; + wire_len = encode_req_proof(wire, sizeof(wire), proof_request); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(server.on_message(wire, wire_len)); + server.serviceEgress(); // explicit legacy request is served immediately + ASSERT_EQ(sent.items.size(), fragment_count + 1); + ProofMsg proof; + ASSERT_TRUE(decode_proof(sent.items.back().data(), + (uint16_t)sent.items.back().size(), proof)); + EXPECT_EQ(proof.block_idx, 0); +} + TEST(OtaTransfer, ServerPacesManifestFragmentsAndConsumesResolvedRequest) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, manifest)); @@ -994,10 +1039,14 @@ TEST(OtaTransfer, ServerQueuesEveryBlockInOneRequestFlight) { server.on_message(wire, wire_len); EXPECT_EQ(server.pendingServeJobs(), 3u); - // The bounded sender remains paced and drains the jobs in request order, including each proactive proof. - const uint32_t packets_per_full_block = - (manifest.block_size() + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA + 1; - for (uint32_t i = 0; i < packets_per_full_block; i++) server.serviceEgress(); + // The bounded sender remains paced and drains the jobs in request order, including each proactive proof + // after the legacy-receiver turnaround gap. + const uint32_t data_packets_per_full_block = + (manifest.block_size() + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA; + for (uint32_t i = 0; i < data_packets_per_full_block; i++) server.serviceEgress(); + server.set_clock(OTA_MANIFEST_EGRESS_MIN_GAP_MS); + server.serviceEgress(); + const uint32_t packets_per_full_block = data_packets_per_full_block + 1; ASSERT_EQ(sent.items.size(), packets_per_full_block); ProofMsg proof; ASSERT_TRUE(decode_proof(sent.items.back().data(), diff --git a/tools/lora_ota/rebuild_rak3401_bundle.py b/tools/lora_ota/rebuild_rak3401_bundle.py index fac27ba78..a26cae310 100755 --- a/tools/lora_ota/rebuild_rak3401_bundle.py +++ b/tools/lora_ota/rebuild_rak3401_bundle.py @@ -378,7 +378,7 @@ ZIP and a persistent work directory. The runner enforces exact target, start body hash, package order, watchdog state, post-boot identity, and normal-radio restoration. -Direct bench: `--temp-radio 909.950,500,5,5,120 --ota-hops 1`. +Direct bench: `--temp-radio 909.950,500,5,5,120 --ota-hops 0`. Live private network with two intermediate relays: use `--temp-radio 909.950,250,5,5,120 --ota-hops 3`, plus two `--relay` arguments