mirror of
https://github.com/mikecarper/MeshCore.git
synced 2026-10-23 22:42:25 +00:00
Add compressed 171-byte OTA transport
This commit is contained in:
mikecarper committed
1 parent
da774e9e75
commit
a9fae59840
20 files changed
+2273
-46
No files matched your search
+50
-9
@@ -544,8 +544,13 @@ All offsets after the 1-byte type. Encoders/decoders in `OtaProtocol.cpp`; const
|
||||
OTA_GET_MANIFEST: manifest_id[4] want_mask(uint16) # bit k = send manifest fragment k; 0xFFFF = all
|
||||
OTA_MANIFEST: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up to OTA_MF_FRAG=176 B/frag
|
||||
OTA_REQ: manifest_id[4] { block_idx(uint16) want_mask(uint16) }[1..4]
|
||||
# one or more rows; bit k = send fragment k of that block
|
||||
OTA_DATA: manifest_id[4] block_idx(uint16) frag_off(uint16) data[] # up to OTA_FRAG_DATA=160 B
|
||||
# legacy: bit k requests its 160-byte fragment
|
||||
# v2: bit15=marker, bit14=allow transport DEFLATE, bits0..12=fragment bitmap
|
||||
OTA_DATA legacy: manifest_id[4] block_idx(uint16) frag_off(uint16) data[<=160]
|
||||
OTA_DATA v2: manifest_id[4] block_idx(uint16) descriptor(uint16)
|
||||
stream_id[4] data[<=171]
|
||||
# descriptor: bit15=marker, bit14=DEFLATE, bits13..10=fragment,
|
||||
# bits9..0=complete encoded length minus one
|
||||
OTA_REQ_PROOF: manifest_id[4] block_idx(uint16)
|
||||
OTA_PROOF: manifest_id[4] block_idx(uint16) n_proof(1) proof[] # n_proof x 4 bytes
|
||||
OTA_GET_LEAVES: manifest_id[4] want_mask(uint16) # bit k = send leaves fragment k; 0xFFFF = all
|
||||
@@ -565,10 +570,30 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up
|
||||
fetch. **Normal P2P nodes never use this** - they target only the blocks they want; the only always-on part
|
||||
is answering `OTA_GET_LEAVES` with leaves the node already holds, so any node's firmware can be captured.
|
||||
|
||||
- **Block <-> fragments:** a 1 KB block remains split into self-describing `OTA_DATA` fragments. `frag_off` is the
|
||||
byte offset of `data` within the block, so the global position is `block_idx*block_size + frag_off` -
|
||||
a fragment is self-placing when returned by the source. The fetcher tracks a
|
||||
per-block slice bitmap and reassembles before requesting the proof.
|
||||
- **Negotiated 171-byte fragments:** the deployed profile remains unchanged: `frag_off` is a byte offset and
|
||||
`data[]` carries at most 160 bytes. A new fetcher marks an `OTA_REQ` row as v2 and includes all seven legacy
|
||||
bits on its first request. Old sources ignore/mask the high flags and can therefore return a complete legacy
|
||||
block immediately; receipt of an untagged `OTA_DATA` switches that fetch session to the old geometry. If no
|
||||
v2 data appears by the first adaptive deadline, the fetcher retries with an ordinary legacy mask. New
|
||||
sources answer v2 with a packed descriptor, a repeated 4-byte representation ID, and exactly 171 data bytes
|
||||
except the final fragment. A raw 1 KiB block therefore falls from seven data packets to six. Message type
|
||||
IDs do not change, so multi-hop relays continue to forward request/data/proof packets opaquely with the same
|
||||
priority.
|
||||
- **Transport-only DEFLATE:** a fetcher sets the v2 DEFLATE-permission bit only when its application includes
|
||||
and configures the decoder. A source may encode each logical block as an independent raw RFC 1951 stream
|
||||
with full stored, fixed-Huffman, and dynamic-Huffman support (`BTYPE=0/1/2`). Compression is used only when
|
||||
strictly smaller, otherwise the source returns raw v2 data. The
|
||||
receiver inflates to the manifest-derived block length, then performs the unchanged Merkle proof and writes
|
||||
the original bytes to staging. Thus existing `.mota` containers, signatures, resume markers, and legacy
|
||||
bootloader apply remain unchanged. The 4-byte `stream_id` is SHA-256:4 of the complete raw or compressed
|
||||
representation and appears in every fragment. A receiver locks `{encoding, encoded length, stream_id}` for
|
||||
a block, preventing fragments from independently encoded seeders from being mixed. After one sparse retry,
|
||||
a second stalled interval clears the full in-flight window and switches the session to legacy geometry, so
|
||||
a vanished v2 seeder cannot prevent an older source from taking over; proof verification remains unchanged.
|
||||
Every `ENABLE_OTA` MeshCore application registers the vendored tinf 1.2.1 full raw decoder; its wrapper
|
||||
bounds output to the exact logical block length and rejects truncation, malformed streams, and trailing
|
||||
whole bytes. Builds without OTA do not link the decoder. This application capability is independent of the
|
||||
bootloader because transport data is inflated before the unchanged staged container is written.
|
||||
- **Adaptive flight size is not signed block size.** The container continues to use 1 KiB Merkle leaves and
|
||||
each slot is one existing 1 KiB block. A clean link changes how many of those blocks one `OTA_REQ` names:
|
||||
1, then 2, then 3, then the compiled cap. The manifest stores `block_size_log2`, so 3 KiB is not a valid
|
||||
@@ -589,7 +614,8 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up
|
||||
half-duplex radios: re-requesting a whole multi-fragment burst let the periodic retry (a transmit) collide
|
||||
with the tail of the in-flight burst and drop the same fragment forever - a hang. Requesting only the hole
|
||||
removes the burst, so there is nothing to collide with. The block/manifest mask matches the 16-bit
|
||||
reassembly bitmap (<=16 fragments/block; 1 KB blocks = 7). `OTA_PROOF` is a single packet and needs no mask.
|
||||
reassembly bitmap (<=16 fragments/block; legacy 1 KB blocks = 7, raw v2 = 6). `OTA_PROOF` is a single packet
|
||||
and needs no mask.
|
||||
- **Data and proof remain separate packets, without a normal extra round trip.** A server retains requested
|
||||
blocks in a bounded descriptor queue, admits at most one response per main-loop pass, and sends one
|
||||
`OTA_PROOF` after each block's requested fragments. Before admitting that proactive proof, the source leaves
|
||||
@@ -605,7 +631,8 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up
|
||||
|
||||
| message | fixed overhead | payload/packet |
|
||||
|---|---|---|
|
||||
| `OTA_DATA` | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 frags per 1 KB block |
|
||||
| `OTA_DATA` legacy | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 frags per 1 KB block |
|
||||
| `OTA_DATA` v2 | 13 B (legacy header + stream ID) | `OTA_FRAG_DATA_V2 = 171` -> 6 raw frags per 1 KB block; fewer when compressed |
|
||||
| `OTA_MANIFEST` | 7 B | `OTA_MF_FRAG = 176` -> signed manifest ~ 2 frags |
|
||||
| `OTA_HAVE` | 12 B | 10 rows x 16 B per fragment |
|
||||
| `OTA_PROOF` | 8 B | up to ~44 sibling digests (>> any real tree) |
|
||||
@@ -715,12 +742,15 @@ drives USB-serial, BLE, a WiFi URL list, an NFS/samba mount, etc. - only `read()
|
||||
```cpp
|
||||
struct MotaDesc { // catalog metadata + region offsets (no whole image in RAM)
|
||||
uint8_t mid[4]; uint32_t target_id, fw_version; uint8_t codec_id, flags;
|
||||
uint8_t block_size_log2, source_caps;
|
||||
uint32_t total_size, leaves_off, block_count, payload_off, payload_size;
|
||||
};
|
||||
class MotaSource {
|
||||
virtual uint8_t count(); // # mOTAs offered
|
||||
virtual bool describe(uint8_t idx, MotaDesc& out); // metadata + offsets
|
||||
virtual bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len); // random-access bytes
|
||||
virtual bool read_deflated_block(uint8_t idx, uint16_t block,
|
||||
uint8_t* buf, uint16_t cap, uint16_t* len); // optional raw RFC 1951
|
||||
};
|
||||
```
|
||||
|
||||
@@ -745,9 +775,11 @@ response (host -> device): 'm' 's' op(1) status(1) payload... xsum(1 = XO
|
||||
OP_COUNT 0x01 args: - -> payload: count(1)
|
||||
OP_DESCRIBE 0x02 args: idx(1) -> payload: MotaDesc wire (38 B)
|
||||
OP_READ 0x03 args: idx(1) off(4) len(2) -> payload: len bytes
|
||||
OP_DEFLATE_BLOCK 0x09 args: idx(1) block(2) off(2) len(2)
|
||||
-> payload: total_encoded_len(2) + requested bytes
|
||||
MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1)
|
||||
total_size(4) leaves_off(4) block_count(4) payload_off(4) payload_size(4)
|
||||
block_size_log2(1) reserved(3)
|
||||
block_size_log2(1) source_caps(1) reserved(2)
|
||||
status: 0 = OK, non-zero = error (out of range / past EOF).
|
||||
```
|
||||
|
||||
@@ -758,6 +790,15 @@ CDC/UART receive rings even though the host successfully wrote the complete
|
||||
reply. Chunking is internal to the transport and does not change `OP_READ` or
|
||||
the `MotaSource` random-access contract.
|
||||
|
||||
`OP_DEFLATE_BLOCK` lets a host-folder source perform the optional transport compression without linking an
|
||||
encoder into the embedded seeder. `len=0, off=0` queries the exact encoded length; subsequent chunks are at
|
||||
most 190 bytes, keeping the total response payload at 192 bytes. The host independently encodes each manifest
|
||||
payload block as ordinary raw RFC 1951 at level 9. It returns an error for an invalid range, an unsupported
|
||||
operation, or a result that is not smaller than the raw block; `SerialMotaSource` then serves the ordinary raw
|
||||
v2 representation. A supporting host sets `source_caps bit 0` in every descriptor. Older hosts leave that
|
||||
formerly-reserved byte zero, so upgraded firmware skips the optional request instead of waiting for an old
|
||||
daemon that silently ignores unknown operations.
|
||||
|
||||
Manifest fragments are retained as bounded response jobs and admitted one at
|
||||
a time. Their source-side gap follows the active maximum packet airtime and
|
||||
dispatcher duty spacing, clamped to 100-1000 ms. The 100 ms floor protects
|
||||
|
||||
@@ -49,6 +49,7 @@ build_flags = -w -DNDEBUG -DRADIOLIB_STATIC_ONLY=1 -DRADIOLIB_GODMODE=1
|
||||
build_src_filter =
|
||||
+<*.cpp>
|
||||
+<helpers/*.cpp>
|
||||
+<helpers/ota/OtaTinf.c>
|
||||
; MQTT-only sources (need Timezone/NTPClient); observer envs add them back.
|
||||
-<helpers/MQTTMessageBuilder.cpp>
|
||||
-<helpers/JWTHelper.cpp>
|
||||
@@ -355,6 +356,7 @@ build_flags = -std=c++17
|
||||
-I src
|
||||
-I test/mocks
|
||||
-D OTA_FETCH_PIPELINE=4
|
||||
-D OTA_TRANSPORT_DEFLATE_TEST=1
|
||||
test_build_src = yes
|
||||
test_ignore = test_kiss_modem
|
||||
build_src_filter =
|
||||
@@ -374,6 +376,8 @@ build_src_filter =
|
||||
+<../src/helpers/ota/FolderMotaStore.cpp>
|
||||
+<../src/helpers/ota/OtaProtocol.cpp>
|
||||
+<../src/helpers/ota/OtaManager.cpp>
|
||||
+<../src/helpers/ota/OtaDeflate.cpp>
|
||||
+<../src/helpers/ota/OtaTinf.c>
|
||||
+<../src/helpers/ota/detools/detools.c>
|
||||
+<../src/helpers/UserGpio.cpp>
|
||||
+<../src/helpers/ConfigSerializer.cpp>
|
||||
|
||||
@@ -17,10 +17,17 @@
|
||||
// OP_COUNT 0x01 args: - resp payload: count(1)
|
||||
// OP_DESCRIBE 0x02 args: idx(1) resp payload: MotaDesc wire (38 B, see below) [status OK]
|
||||
// OP_READ 0x03 args: idx(1) off(4) len(2) resp payload: len bytes [status OK]
|
||||
// OP_DEFLATE_BLOCK 0x09 args: idx(1) block(2) off(2) len(2)
|
||||
// resp payload: total_encoded_len(2) + len encoded bytes [status OK]
|
||||
// `len=0` queries the size. Chunks are <=190 bytes so the full response payload remains
|
||||
// <=192. The host returns ERR unless independent raw-RFC1951 compression is smaller than
|
||||
// the original logical block; callers then use ordinary OP_READ/raw OTA_DATA v2.
|
||||
//
|
||||
// MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) total_size(4)
|
||||
// leaves_off(4) block_count(4) payload_off(4) payload_size(4)
|
||||
// block_size_log2(1) reserved(3)
|
||||
// block_size_log2(1) source_caps(1) reserved(2)
|
||||
// source_caps bit 0 = OP_DEFLATE_BLOCK is implemented. Deployed hosts send zero, so a new node never
|
||||
// waits on an unsupported operation during a rolling host upgrade.
|
||||
//
|
||||
// --- STORAGE ops (device -> host WRITE): "pull to folder". The device is fetching a `.mota` off the mesh
|
||||
// (e.g. a neighbour's self-served firmware it has no local copy of) and streams it into the host folder as
|
||||
@@ -52,14 +59,16 @@ static const uint8_t MS_OP_BEGIN = 0x05; // storage: create/truncate <mid>
|
||||
static const uint8_t MS_OP_WRITE = 0x06; // storage: write bytes at offset
|
||||
static const uint8_t MS_OP_SREAD = 0x07; // storage: read bytes back (resume: recompute missing blocks)
|
||||
static const uint8_t MS_OP_FIN = 0x08; // storage: transfer complete - validate + make servable
|
||||
static const uint8_t MS_OP_DEFLATE_BLOCK = 0x09; // source: independently compressed payload block/chunk
|
||||
|
||||
static const uint8_t MS_STATUS_OK = 0x00;
|
||||
static const uint8_t MS_STATUS_ERR = 0x01;
|
||||
|
||||
static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry block geometry; wire size is unchanged
|
||||
static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry geometry/caps; wire size is unchanged
|
||||
// Keep host-to-device replies below common 256-byte USB CDC/UART receive rings. Manifest leaves can exceed
|
||||
// that ring, and payload reads are normally 1 KB; splitting them here prevents silent overrun/checksum loss.
|
||||
static const uint16_t MOTA_SEEDER_READ_MAX = 192;
|
||||
static const uint16_t MOTA_SEEDER_DEFLATE_CHUNK_MAX = 190; // plus uint16 total length = 192-byte response payload
|
||||
static const uint16_t MOTA_SEEDER_WRITE_MAX = 512; // max data bytes per OP_WRITE/OP_SREAD (bounds frames)
|
||||
|
||||
} // namespace ota
|
||||
|
||||
@@ -86,7 +86,8 @@ bool SerialMotaSource::describe(uint8_t idx, MotaDesc& out) {
|
||||
out.payload_off = rd_u32le(w + 26);
|
||||
out.payload_size = rd_u32le(w + 30);
|
||||
out.block_size_log2 = w[34];
|
||||
// bytes [35,38) remain reserved (zero) for forward compatibility.
|
||||
out.source_caps = w[35];
|
||||
// bytes [36,38) remain reserved (zero) for forward compatibility.
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -107,5 +108,35 @@ bool SerialMotaSource::read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t le
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SerialMotaSource::read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf,
|
||||
uint16_t cap, uint16_t* len) {
|
||||
if (!buf || !len || cap == 0) return false;
|
||||
*len = 0;
|
||||
|
||||
uint8_t args[7] = { idx };
|
||||
args[1] = (uint8_t)(block & 0xFF); args[2] = (uint8_t)(block >> 8);
|
||||
// Query the independently compressed representation's exact size first. The old host returns an error for
|
||||
// the unknown operation, which is the intentional raw fallback during a rolling deployment.
|
||||
uint8_t total_wire[2];
|
||||
if (!txn(MS_OP_DEFLATE_BLOCK, args, sizeof(args), total_wire, sizeof(total_wire))) return false;
|
||||
const uint16_t total = rd_u16le(total_wire);
|
||||
if (total == 0 || total > cap) return false;
|
||||
|
||||
uint16_t done = 0;
|
||||
while (done < total) {
|
||||
const uint16_t chunk = (uint16_t)((total - done > MOTA_SEEDER_DEFLATE_CHUNK_MAX)
|
||||
? MOTA_SEEDER_DEFLATE_CHUNK_MAX : total - done);
|
||||
args[3] = (uint8_t)(done & 0xFF); args[4] = (uint8_t)(done >> 8);
|
||||
args[5] = (uint8_t)(chunk & 0xFF); args[6] = (uint8_t)(chunk >> 8);
|
||||
uint8_t response[2 + MOTA_SEEDER_DEFLATE_CHUNK_MAX];
|
||||
if (!txn(MS_OP_DEFLATE_BLOCK, args, sizeof(args), response, (uint16_t)(2 + chunk)) ||
|
||||
rd_u16le(response) != total) return false;
|
||||
memcpy(buf + done, response + 2, chunk);
|
||||
done = (uint16_t)(done + chunk);
|
||||
}
|
||||
*len = total;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace ota
|
||||
} // namespace mesh
|
||||
@@ -22,6 +22,8 @@ public:
|
||||
uint8_t count() override;
|
||||
bool describe(uint8_t idx, MotaDesc& out) override;
|
||||
bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) override;
|
||||
bool read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf,
|
||||
uint16_t cap, uint16_t* len) override;
|
||||
|
||||
private:
|
||||
// Send a request (op+args) and read its response header; on OK, `payload` (if non-null) receives
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <stdlib.h> // malloc/free (lazy ESP32 manual-stage buffer)
|
||||
#include <string.h> // strncmp/strncpy (hw_id)
|
||||
#include "OtaManager.h"
|
||||
#include "OtaDeflate.h"
|
||||
#include "OtaStore.h"
|
||||
#include "SignerAllowlist.h"
|
||||
#include "OtaApply.h"
|
||||
@@ -562,6 +563,10 @@ struct OtaContext {
|
||||
target_id = 0;
|
||||
#endif
|
||||
manager.begin(target_id, send, ctx);
|
||||
// Every OTA-capable application can receive ordinary raw RFC1951 transport blocks. OtaManager advertises
|
||||
// that permission only because this exact, bounded decoder is registered; non-OTA builds do not compile
|
||||
// OtaContext or link the decoder implementation.
|
||||
manager.set_transport_deflate_decoder(ota_transport_inflate);
|
||||
manager.set_auto_version_floor(_fi.valid ? _fi.fw_version : 0, true);
|
||||
fetch_to_folder = false;
|
||||
if (hw) { strncpy(hw_id, hw, sizeof(hw_id) - 1); hw_id[sizeof(hw_id) - 1] = 0; }
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
#include "OtaDeflate.h"
|
||||
|
||||
#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST)
|
||||
|
||||
extern "C" {
|
||||
#include "tinf/tinf.h"
|
||||
}
|
||||
|
||||
namespace mesh {
|
||||
namespace ota {
|
||||
|
||||
bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len,
|
||||
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len) {
|
||||
(void)context;
|
||||
if (dst_len) *dst_len = 0;
|
||||
if (!src || src_len == 0 || !dst || dst_cap == 0 || !dst_len) return false;
|
||||
|
||||
unsigned int produced = dst_cap;
|
||||
const int result = tinf_uncompress_exact(dst, &produced, src, src_len);
|
||||
if (result != TINF_OK || produced != dst_cap) return false;
|
||||
*dst_len = (uint16_t)produced;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace ota
|
||||
} // namespace mesh
|
||||
|
||||
#endif // ENABLE_OTA || OTA_TRANSPORT_DEFLATE_TEST
|
||||
@@ -0,0 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
namespace mesh {
|
||||
namespace ota {
|
||||
|
||||
// Full raw RFC1951 transport decoder (stored, fixed-Huffman, and dynamic-Huffman blocks).
|
||||
// `dst_cap` is the exact expected logical block length, not merely spare capacity. Success
|
||||
// requires exact output and exact whole-byte input consumption; malformed/truncated streams,
|
||||
// output overflow, and trailing bytes fail closed.
|
||||
bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len,
|
||||
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len);
|
||||
|
||||
} // namespace ota
|
||||
} // namespace mesh
|
||||
+278
-28
@@ -69,6 +69,7 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) {
|
||||
if (!mota_parse(mota, len, _view0.m)) return false;
|
||||
_view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container
|
||||
_view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload
|
||||
_view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr;
|
||||
_view0.scratch = scratch; _view0.scratch_sz = OTA_PROOFGEN_SCRATCH; // <=1024 blocks (RAM .mota is small)
|
||||
_view0.valid = true;
|
||||
clearPendingEgress();
|
||||
@@ -86,6 +87,7 @@ bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t
|
||||
_view0.m.payload = nullptr; // read on demand via `read`
|
||||
_view0.m.block_count = block_count;
|
||||
_view0.mfl = mfl; _view0.read = read; _view0.read_ctx = ctx;
|
||||
_view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr;
|
||||
_view0.scratch = proof_scratch; _view0.scratch_sz = proof_scratch_sz; // sized for our (large) image
|
||||
_view0.valid = true;
|
||||
clearPendingEgress();
|
||||
@@ -259,6 +261,9 @@ bool OtaManager::loadSource(const ServeEntry& e) {
|
||||
_srcv_rdctx.src = e.src; _srcv_rdctx.idx = e.src_idx;
|
||||
_srcv_rdctx.payload_off = d.payload_off;
|
||||
_srcv.read = srcReadTramp; _srcv.read_ctx = &_srcv_rdctx;
|
||||
_srcv.read_deflated = (d.source_caps & MOTA_SOURCE_CAP_DEFLATE_BLOCK)
|
||||
? srcDeflateReadTramp : nullptr;
|
||||
_srcv.deflate_ctx = _srcv.read_deflated ? &_srcv_rdctx : nullptr;
|
||||
_srcv.scratch = scratch; _srcv.scratch_sz = OTA_PROOFGEN_SCRATCH;
|
||||
memcpy(_srcv_mid, d.mid, 4);
|
||||
_srcv.valid = true;
|
||||
@@ -271,6 +276,12 @@ bool OtaManager::srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len)
|
||||
return x->src->read(x->idx, x->payload_off + off, buf, len);
|
||||
}
|
||||
|
||||
bool OtaManager::srcDeflateReadTramp(void* c, uint16_t block, uint8_t* buf,
|
||||
uint16_t cap, uint16_t* len) {
|
||||
SrcReadCtx* x = (SrcReadCtx*)c;
|
||||
return x->src->read_deflated_block(x->idx, block, buf, cap, len);
|
||||
}
|
||||
|
||||
// sha2-256:4 over the SORTED set of mids we serve - peers use it to tell if our offering changed. Sorting
|
||||
// makes it canonical across nodes regardless of insert order; for a single mota it is mh4(mid) (unchanged).
|
||||
void OtaManager::setDigest(uint8_t out[4]) const {
|
||||
@@ -414,10 +425,15 @@ static inline uint16_t frag_full_mask(uint32_t nf) {
|
||||
// Retain a bounded descriptor instead of allocating every DATA packet inline. Repeated requests that arrive
|
||||
// while a block is already queued merge only fragments not yet admitted to the radio queue; a later retry can
|
||||
// enqueue the block again if one of those admitted packets was actually lost over the air.
|
||||
bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask) {
|
||||
bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask,
|
||||
bool wire_v2, bool allow_deflate) {
|
||||
for (uint8_t i = 0; i < _n_serve_jobs; i++) {
|
||||
ServeJob& job = _serve_jobs[i];
|
||||
if (job.block != block || memcmp(job.mid, mid, 4) != 0) continue;
|
||||
// A proof-only request can complete whichever representation is already queued. DATA requests with
|
||||
// different geometries remain distinct because their bitmap positions do not describe the same bytes.
|
||||
if (want_mask != 0 && (job.wire_v2 != wire_v2 ||
|
||||
(wire_v2 && job.allow_deflate != allow_deflate))) continue;
|
||||
job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask);
|
||||
if (want_mask == 0) job.proof_requested = true;
|
||||
return true;
|
||||
@@ -430,6 +446,8 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want
|
||||
job.emitted_mask = 0;
|
||||
job.proof_ready_at = 0;
|
||||
job.proof_requested = want_mask == 0;
|
||||
job.wire_v2 = wire_v2;
|
||||
job.allow_deflate = allow_deflate;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -475,6 +493,10 @@ void OtaManager::clearPendingEgress() {
|
||||
_n_serve_jobs = 0;
|
||||
_serve_block_len = 0;
|
||||
_serve_block_loaded = false;
|
||||
_serve_wire_block = nullptr;
|
||||
_serve_wire_len = 0;
|
||||
_serve_wire_deflated = false;
|
||||
memset(_serve_wire_id, 0, sizeof(_serve_wire_id));
|
||||
}
|
||||
|
||||
void OtaManager::popManifestJob() {
|
||||
@@ -546,6 +568,10 @@ void OtaManager::popServeJob() {
|
||||
_n_serve_jobs--;
|
||||
_serve_block_len = 0;
|
||||
_serve_block_loaded = false;
|
||||
_serve_wire_block = nullptr;
|
||||
_serve_wire_len = 0;
|
||||
_serve_wire_deflated = false;
|
||||
memset(_serve_wire_id, 0, sizeof(_serve_wire_id));
|
||||
}
|
||||
|
||||
bool OtaManager::loadActiveServeBlock() {
|
||||
@@ -563,6 +589,30 @@ bool OtaManager::loadActiveServeBlock() {
|
||||
memcpy(_serve_block, v->m.payload + off, blen);
|
||||
}
|
||||
_serve_block_len = (uint16_t)blen;
|
||||
_serve_wire_block = _serve_block;
|
||||
_serve_wire_len = _serve_block_len;
|
||||
_serve_wire_deflated = false;
|
||||
|
||||
// A host-backed MotaSource can provide a pre-compressed block without putting an encoder in firmware.
|
||||
// Other sources may opt into a caller-supplied encoder. Both write into the proof scratch, which is safe
|
||||
// until every DATA fragment has left; proof generation reuses it only after that point.
|
||||
if (job.wire_v2 && job.allow_deflate && v->scratch && v->scratch_sz >= blen) {
|
||||
uint16_t encoded_len = 0;
|
||||
bool encoded = false;
|
||||
if (v->read_deflated) {
|
||||
encoded = v->read_deflated(v->deflate_ctx, job.block, v->scratch,
|
||||
(uint16_t)blen, &encoded_len);
|
||||
} else if (_deflate_encode) {
|
||||
encoded = _deflate_encode(_deflate_encode_ctx, _serve_block, (uint16_t)blen,
|
||||
v->scratch, (uint16_t)blen, &encoded_len);
|
||||
}
|
||||
if (encoded && encoded_len > 0 && encoded_len < blen) {
|
||||
_serve_wire_block = v->scratch;
|
||||
_serve_wire_len = encoded_len;
|
||||
_serve_wire_deflated = true;
|
||||
}
|
||||
}
|
||||
mh4(_serve_wire_id, _serve_wire_block, _serve_wire_len);
|
||||
_serve_block_loaded = true;
|
||||
return true;
|
||||
}
|
||||
@@ -580,9 +630,17 @@ bool OtaManager::handleReq(const uint8_t* m, uint16_t n) {
|
||||
uint32_t blen = block_size;
|
||||
const uint32_t off = idx * block_size;
|
||||
if (off + blen > v->m.payload_size) blen = v->m.payload_size - off;
|
||||
const uint16_t valid_mask = frag_full_mask((blen + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA);
|
||||
const uint16_t want = (uint16_t)(rq.items[i].want_mask & valid_mask);
|
||||
if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want);
|
||||
const bool wire_v2 = ota_req_is_v2(rq.items[i].want_mask);
|
||||
const bool allow_deflate = wire_v2 &&
|
||||
(rq.items[i].want_mask & OTA_REQ_V2_ALLOW_DEFLATE) != 0;
|
||||
const uint16_t valid_mask = frag_full_mask((blen +
|
||||
(wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA) - 1) /
|
||||
(wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA));
|
||||
const uint16_t requested = wire_v2
|
||||
? ota_req_v2_fragments(rq.items[i].want_mask) : rq.items[i].want_mask;
|
||||
const uint16_t want = (uint16_t)(requested & valid_mask);
|
||||
if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want,
|
||||
wire_v2, allow_deflate);
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
@@ -630,21 +688,41 @@ void OtaManager::serviceEgress() {
|
||||
popServeJob();
|
||||
return;
|
||||
}
|
||||
const uint16_t fragment_data = job.wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA;
|
||||
const uint16_t valid_mask = frag_full_mask(
|
||||
(_serve_wire_len + fragment_data - 1u) / fragment_data);
|
||||
job.pending_mask &= valid_mask;
|
||||
if (job.pending_mask == 0) {
|
||||
job.proof_ready_at = _now_ms + proofEgressGapMs();
|
||||
return;
|
||||
}
|
||||
uint8_t fragment = 0;
|
||||
while (fragment < 16 && !(job.pending_mask & (1u << fragment))) fragment++;
|
||||
uint32_t frag_off = (uint32_t)fragment * OTA_FRAG_DATA;
|
||||
if (fragment >= 16 || frag_off >= _serve_block_len) {
|
||||
uint32_t frag_off = (uint32_t)fragment * fragment_data;
|
||||
if (fragment >= 16 || frag_off >= _serve_wire_len) {
|
||||
job.pending_mask = 0;
|
||||
return;
|
||||
}
|
||||
uint32_t frag_len = _serve_block_len - frag_off;
|
||||
if (frag_len > OTA_FRAG_DATA) frag_len = OTA_FRAG_DATA;
|
||||
uint32_t frag_len = _serve_wire_len - frag_off;
|
||||
if (frag_len > fragment_data) frag_len = fragment_data;
|
||||
DataMsg dm;
|
||||
memcpy(dm.manifest_id, job.mid, 4);
|
||||
dm.block_idx = job.block;
|
||||
dm.frag_off = (uint16_t)frag_off;
|
||||
dm.data = _serve_block + frag_off;
|
||||
dm.data_len = (uint16_t)frag_len;
|
||||
uint8_t v2_data[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2];
|
||||
if (job.wire_v2) {
|
||||
if (!ota_data_v2_pack(fragment, _serve_wire_len, _serve_wire_deflated, dm.frag_off)) {
|
||||
popServeJob();
|
||||
return;
|
||||
}
|
||||
memcpy(v2_data, _serve_wire_id, OTA_DATA_V2_STREAM_ID_BYTES);
|
||||
memcpy(v2_data + OTA_DATA_V2_STREAM_ID_BYTES, _serve_wire_block + frag_off, frag_len);
|
||||
dm.data = v2_data;
|
||||
dm.data_len = (uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + frag_len);
|
||||
} else {
|
||||
dm.frag_off = (uint16_t)frag_off;
|
||||
dm.data = _serve_block + frag_off;
|
||||
dm.data_len = (uint16_t)frag_len;
|
||||
}
|
||||
uint8_t b[MAX_PACKET_PAYLOAD];
|
||||
if (emit(b, encode_data(b, sizeof(b), dm), false)) {
|
||||
const uint16_t bit = (uint16_t)(1u << fragment);
|
||||
@@ -944,6 +1022,11 @@ void OtaManager::clearReassemblySlot(uint8_t slot) {
|
||||
_reasm[slot].need = 0;
|
||||
_reasm[slot].awaiting_proof = false;
|
||||
_reasm[slot].proof_request_at = 0;
|
||||
_reasm[slot].encoded_len = 0;
|
||||
_reasm[slot].wire_v2 = false;
|
||||
_reasm[slot].deflated = false;
|
||||
memset(_reasm[slot].stream_id, 0, sizeof(_reasm[slot].stream_id));
|
||||
_reasm[slot].wire_stalls = 0;
|
||||
}
|
||||
|
||||
// Forget all blocks currently being reassembled or awaiting proofs.
|
||||
@@ -956,6 +1039,35 @@ void OtaManager::clearReassembly() {
|
||||
noteFetchActivity();
|
||||
}
|
||||
|
||||
bool OtaManager::anyWireDataReceived() const {
|
||||
for (uint8_t slot = 0; slot < OTA_FETCH_PIPELINE; slot++) {
|
||||
if (_reasm[slot].block != NO_BLOCK && _reasm[slot].mask != 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void OtaManager::downgradeWireToLegacy() {
|
||||
if (!_wire_v2_session) return;
|
||||
_wire_v2_session = false;
|
||||
_wire_v2_confirmed = false;
|
||||
_wire_allow_deflate = false;
|
||||
_wire_empty_stalls = 0;
|
||||
for (uint8_t i = 0; i < OTA_FETCH_PIPELINE; i++) {
|
||||
ReassemblySlot& slot = _reasm[i];
|
||||
if (slot.block == NO_BLOCK) continue;
|
||||
slot.mask = 0;
|
||||
slot.need = frag_full_mask((blockLen(slot.block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA);
|
||||
slot.awaiting_proof = false;
|
||||
slot.proof_request_at = 0;
|
||||
slot.encoded_len = 0;
|
||||
slot.wire_v2 = false;
|
||||
slot.deflated = false;
|
||||
memset(slot.stream_id, 0, sizeof(slot.stream_id));
|
||||
slot.wire_stalls = 0;
|
||||
}
|
||||
OTA_DBG("OTA: source uses legacy 160-byte DATA geometry\n");
|
||||
}
|
||||
|
||||
int OtaManager::findReassemblySlot(uint32_t block) const {
|
||||
for (uint8_t slot = 0; slot < OTA_FETCH_PIPELINE; slot++) {
|
||||
if (_reasm[slot].block == block) return slot;
|
||||
@@ -1007,7 +1119,8 @@ uint32_t OtaManager::fetchRetryTimeoutMs() const {
|
||||
: (uint32_t)count_mask_bits((uint16_t)(slot.need & ~slot.mask)) + 1u;
|
||||
}
|
||||
if (active == 0) {
|
||||
const uint32_t fragments = (_fbs + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA;
|
||||
const uint32_t fragment_data = _wire_v2_session ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA;
|
||||
const uint32_t fragments = (_fbs + fragment_data - 1) / fragment_data;
|
||||
packets += (fragments + 1) * _pipeline_width;
|
||||
}
|
||||
uint32_t path_transmissions = _observed_path_transmissions;
|
||||
@@ -1129,6 +1242,10 @@ OtaManager::PullResult OtaManager::startFetch(const uint8_t* mid, uint32_t targe
|
||||
if (!_fetch) return PULL_NO_STORE;
|
||||
if (fetchActive()) return PULL_BUSY;
|
||||
_fetch_error = FETCH_ERROR_NONE;
|
||||
_wire_v2_session = _wire_v2_enabled;
|
||||
_wire_v2_confirmed = false;
|
||||
_wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr;
|
||||
_wire_empty_stalls = 0;
|
||||
_fexpected_target = target;
|
||||
_validate = validate; // motatool folder-capture warm-start (seed leaf-diff)
|
||||
// A validate pull is a FRESH seed capture, not a resume: the store already holds the seed's payload (not a
|
||||
@@ -1149,6 +1266,10 @@ OtaManager::PullResult OtaManager::startFetch(const uint8_t* mid, uint32_t targe
|
||||
bool OtaManager::resumeStagedExplicit(const uint8_t* want_mid, uint32_t expected_target) {
|
||||
if (!want_mid || !_fetch || fetchActive()) return false;
|
||||
_fetch_error = FETCH_ERROR_NONE;
|
||||
_wire_v2_session = _wire_v2_enabled;
|
||||
_wire_v2_confirmed = false;
|
||||
_wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr;
|
||||
_wire_empty_stalls = 0;
|
||||
_validate = false;
|
||||
_archive_fetch = false;
|
||||
_desired_target = expected_target;
|
||||
@@ -1350,6 +1471,10 @@ bool OtaManager::resumeStaged(const uint8_t* want_mid) {
|
||||
|| _fstate == WANT_LEAVES || _fstate == VERIFYING_STAGED) {
|
||||
return false;
|
||||
}
|
||||
_wire_v2_session = _wire_v2_enabled;
|
||||
_wire_v2_confirmed = false;
|
||||
_wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr;
|
||||
_wire_empty_stalls = 0;
|
||||
if (!_fetch->reopen()) return false; // nothing persisted in the store
|
||||
uint32_t total = _fetch->staged_size();
|
||||
uint8_t hdr[8];
|
||||
@@ -1509,19 +1634,100 @@ bool OtaManager::handleData(const uint8_t* m, uint16_t n) {
|
||||
if (slot_index < 0) return false; // not one of this node's requested blocks
|
||||
ReassemblySlot& slot = _reasm[slot_index];
|
||||
uint32_t blen = blockLen(dm.block_idx);
|
||||
if (dm.frag_off % OTA_FRAG_DATA != 0) return false; // canonical FRAG_DATA-aligned slices only
|
||||
if ((uint32_t)dm.frag_off + dm.data_len > blen) return false; // slice out of the block
|
||||
uint32_t expected_len = blen - dm.frag_off;
|
||||
if (expected_len > OTA_FRAG_DATA) expected_len = OTA_FRAG_DATA;
|
||||
if (dm.data_len != expected_len) return false; // do not mark a short slice as complete
|
||||
uint32_t kf = dm.frag_off / OTA_FRAG_DATA;
|
||||
if (kf >= 16) return false;
|
||||
const uint16_t bit = (uint16_t)(1u << kf);
|
||||
if (slot.mask & bit) return true; // duplicate flood/retry copy is terminal too
|
||||
memcpy(slot.buf + dm.frag_off, dm.data, dm.data_len);
|
||||
slot.mask |= bit;
|
||||
|
||||
if ((dm.frag_off & OTA_DATA_V2_MARK) == 0) {
|
||||
if (dm.frag_off % OTA_FRAG_DATA != 0) return false; // canonical legacy-aligned slices only
|
||||
if ((uint32_t)dm.frag_off + dm.data_len > blen) return false;
|
||||
uint32_t expected_len = blen - dm.frag_off;
|
||||
if (expected_len > OTA_FRAG_DATA) expected_len = OTA_FRAG_DATA;
|
||||
if (dm.data_len != expected_len) return false; // do not mark a short slice as complete
|
||||
uint32_t kf = dm.frag_off / OTA_FRAG_DATA;
|
||||
if (kf >= 16) return false;
|
||||
// A deployed source ignores the v2 request bits, masks them against its seven-fragment geometry, and
|
||||
// responds here. Only a fully canonical legacy slice may select that profile; malformed matching-MID
|
||||
// traffic must not force a session-wide downgrade. Never mix it with already accepted v2 bytes.
|
||||
if (_wire_v2_session) {
|
||||
if (_wire_v2_confirmed || anyWireDataReceived()) return false;
|
||||
downgradeWireToLegacy();
|
||||
}
|
||||
const uint16_t bit = (uint16_t)(1u << kf);
|
||||
if (slot.mask & bit) return true; // duplicate flood/retry copy is terminal too
|
||||
memcpy(slot.buf + dm.frag_off, dm.data, dm.data_len);
|
||||
slot.mask |= bit;
|
||||
} else {
|
||||
if (!_wire_v2_session || dm.data_len <= OTA_DATA_V2_STREAM_ID_BYTES) return false;
|
||||
uint8_t fragment = 0;
|
||||
uint16_t encoded_len = 0;
|
||||
bool deflated = false;
|
||||
if (!ota_data_v2_unpack(dm.frag_off, fragment, encoded_len, deflated)) return false;
|
||||
if (encoded_len > blen || (deflated ? encoded_len >= blen : encoded_len != blen)) return false;
|
||||
const uint16_t fragment_len = (uint16_t)(dm.data_len - OTA_DATA_V2_STREAM_ID_BYTES);
|
||||
const uint32_t fragment_off = (uint32_t)fragment * OTA_FRAG_DATA_V2;
|
||||
if (fragment_off >= encoded_len || fragment_off + fragment_len > encoded_len) return false;
|
||||
uint32_t expected_len = encoded_len - fragment_off;
|
||||
if (expected_len > OTA_FRAG_DATA_V2) expected_len = OTA_FRAG_DATA_V2;
|
||||
if (fragment_len != expected_len) return false;
|
||||
_wire_v2_confirmed = true;
|
||||
|
||||
const uint8_t* stream_id = dm.data;
|
||||
if (slot.encoded_len == 0) {
|
||||
slot.encoded_len = encoded_len;
|
||||
slot.wire_v2 = true;
|
||||
slot.deflated = deflated;
|
||||
memcpy(slot.stream_id, stream_id, sizeof(slot.stream_id));
|
||||
slot.mask = 0;
|
||||
slot.need = frag_full_mask((encoded_len + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2);
|
||||
} else if (!slot.wire_v2 || slot.encoded_len != encoded_len || slot.deflated != deflated ||
|
||||
memcmp(slot.stream_id, stream_id, sizeof(slot.stream_id)) != 0) {
|
||||
return false; // another seeder's representation; never mix
|
||||
}
|
||||
_wire_empty_stalls = 0; // a valid v2 source is still serving this flight
|
||||
const uint16_t bit = (uint16_t)(1u << fragment);
|
||||
if (!(slot.need & bit)) return false;
|
||||
if (slot.mask & bit) return true;
|
||||
memcpy(slot.buf + fragment_off, dm.data + OTA_DATA_V2_STREAM_ID_BYTES, fragment_len);
|
||||
slot.mask |= bit;
|
||||
slot.wire_stalls = 0;
|
||||
}
|
||||
noteFetchActivity();
|
||||
if (slot.mask != slot.need || slot.awaiting_proof) return true;
|
||||
|
||||
if (slot.wire_v2) {
|
||||
uint8_t stream_id[4];
|
||||
mh4(stream_id, slot.buf, slot.encoded_len);
|
||||
if (memcmp(stream_id, slot.stream_id, sizeof(stream_id)) != 0) {
|
||||
slot.mask = 0; // complete bytes do not match their representation id
|
||||
slot.encoded_len = 0;
|
||||
slot.wire_v2 = false;
|
||||
slot.deflated = false;
|
||||
memset(slot.stream_id, 0, sizeof(slot.stream_id));
|
||||
notePipelineStall();
|
||||
return true;
|
||||
}
|
||||
if (slot.deflated) {
|
||||
uint16_t decoded_len = 0;
|
||||
bool decoded = _deflate_decode && _deflate_decode(
|
||||
_deflate_decode_ctx, slot.buf, slot.encoded_len,
|
||||
_serve_block, (uint16_t)blen, &decoded_len);
|
||||
// _serve_block is also the outbound server cache. Invalidate it even on failure so a simultaneous
|
||||
// serve job reloads its own authenticated block rather than transmitting decoder workspace.
|
||||
_serve_block_loaded = false;
|
||||
_serve_wire_block = nullptr;
|
||||
_serve_wire_len = 0;
|
||||
if (!decoded || decoded_len != blen) {
|
||||
_wire_allow_deflate = false; // retry this/future blocks as raw v2
|
||||
slot.mask = 0;
|
||||
slot.need = frag_full_mask((blen + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2);
|
||||
slot.encoded_len = 0;
|
||||
slot.wire_v2 = false;
|
||||
slot.deflated = false;
|
||||
memset(slot.stream_id, 0, sizeof(slot.stream_id));
|
||||
notePipelineStall();
|
||||
return true;
|
||||
}
|
||||
memcpy(slot.buf, _serve_block, blen);
|
||||
}
|
||||
}
|
||||
// The paced server sends a proof immediately after this block's requested DATA. Wait briefly so that proof
|
||||
// can arrive without another request/response turn. An older server, or a lost proof, falls back through
|
||||
// serviceEgress() to the existing REQ_PROOF wire message.
|
||||
@@ -1579,6 +1785,22 @@ bool OtaManager::handleProof(const uint8_t* m, uint16_t n) {
|
||||
return true;
|
||||
}
|
||||
|
||||
uint16_t OtaManager::wireRequestMask(uint32_t block, uint16_t need, uint16_t received,
|
||||
uint16_t encoded_len) const {
|
||||
uint16_t want = (uint16_t)(need & ~received);
|
||||
if (_wire_v2_session) {
|
||||
// Until a v2 representation answers, include every legacy fragment bit. A deployed source ignores the
|
||||
// high profile flags and can therefore return a complete seven-fragment block on this very first request.
|
||||
if (encoded_len == 0) {
|
||||
want = frag_full_mask((blockLen(block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA);
|
||||
}
|
||||
if (want == 0) want = need;
|
||||
return ota_req_make_v2(want, _wire_allow_deflate);
|
||||
}
|
||||
if (want == 0) want = need;
|
||||
return want;
|
||||
}
|
||||
|
||||
bool OtaManager::requestSlot(uint8_t slot_index) {
|
||||
if (_fstate != FETCHING || slot_index >= OTA_FETCH_PIPELINE) return false;
|
||||
ReassemblySlot& slot = _reasm[slot_index];
|
||||
@@ -1597,8 +1819,7 @@ bool OtaManager::requestSlot(uint8_t slot_index) {
|
||||
}
|
||||
return sent;
|
||||
}
|
||||
uint16_t want = (uint16_t)(slot.need & ~slot.mask);
|
||||
if (want == 0) want = slot.need; // safety: never send an empty request
|
||||
uint16_t want = wireRequestMask(slot.block, slot.need, slot.mask, slot.encoded_len);
|
||||
ReqMsg rq; memcpy(rq.manifest_id, _fid, 4);
|
||||
rq.block_idx = (uint16_t)slot.block; rq.want_mask = want;
|
||||
uint8_t b[16];
|
||||
@@ -1619,8 +1840,7 @@ bool OtaManager::requestFlight() {
|
||||
if (slot.block == NO_BLOCK || slot.awaiting_proof) continue;
|
||||
ReqItem& item = request.items[request.n_items++];
|
||||
item.block_idx = (uint16_t)slot.block;
|
||||
item.want_mask = (uint16_t)(slot.need & ~slot.mask);
|
||||
if (item.want_mask == 0) item.want_mask = slot.need;
|
||||
item.want_mask = wireRequestMask(slot.block, slot.need, slot.mask, slot.encoded_len);
|
||||
}
|
||||
if (request.n_items == 0) return false;
|
||||
_req_start = request.items[0].block_idx;
|
||||
@@ -1645,7 +1865,8 @@ bool OtaManager::fillPipeline() {
|
||||
ReassemblySlot& slot = _reasm[slot_index];
|
||||
slot.block = block;
|
||||
slot.mask = 0;
|
||||
slot.need = frag_full_mask((blockLen(block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA);
|
||||
const uint16_t fragment_data = _wire_v2_session ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA;
|
||||
slot.need = frag_full_mask((blockLen(block) + fragment_data - 1) / fragment_data);
|
||||
slot.awaiting_proof = false;
|
||||
slot.proof_request_at = 0;
|
||||
assigned++;
|
||||
@@ -1660,13 +1881,42 @@ void OtaManager::requestMissing() {
|
||||
if (_fstate != FETCHING) return;
|
||||
if (activePipelineSlots() == 0) { fillPipeline(); return; }
|
||||
|
||||
// No v2 DATA at all means the request may have reached only deployed sources which ignored its profile
|
||||
// bits (or the first response burst was lost). Retry once in the universal legacy profile instead of
|
||||
// pinning the session to a feature it has not observed on air.
|
||||
if (_wire_v2_session && !_wire_v2_confirmed && !anyWireDataReceived()) {
|
||||
downgradeWireToLegacy();
|
||||
notePipelineStall();
|
||||
requestFlight();
|
||||
return;
|
||||
}
|
||||
|
||||
// The airtime-aware deadline expired. Retry one slot's holes per deadline, round-robin, so a weak block
|
||||
// cannot trigger a simultaneous multi-block re-burst on a half-duplex channel. Recovery dirties the
|
||||
// flight; its successor is reduced (4->2, 3->2, 2->1) after all current slots finish.
|
||||
notePipelineStall();
|
||||
|
||||
// Once v2 has been confirmed, the source can still disappear cleanly between blocks (or after a bad
|
||||
// representation is discarded), leaving every active slot empty. Give the v2 flight one sparse retry,
|
||||
// then clear the whole flight and ask again in the universal legacy profile so an old seeder can take over.
|
||||
if (_wire_v2_session && _wire_v2_confirmed && !anyWireDataReceived() &&
|
||||
++_wire_empty_stalls >= 2) {
|
||||
downgradeWireToLegacy();
|
||||
requestFlight();
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint8_t offset = 0; offset < OTA_FETCH_PIPELINE; offset++) {
|
||||
uint8_t slot = (uint8_t)((_retry_slot + offset) % OTA_FETCH_PIPELINE);
|
||||
if (_reasm[slot].block == NO_BLOCK) continue;
|
||||
ReassemblySlot& reassembly = _reasm[slot];
|
||||
if (_wire_v2_session && reassembly.wire_v2 && reassembly.mask != 0 &&
|
||||
!reassembly.awaiting_proof && ++reassembly.wire_stalls >= 2) {
|
||||
// A v2 seeder may disappear after establishing a representation while only deployed legacy seeders
|
||||
// remain. Keep one sparse retry, then fall back the whole flight to the universal profile. Clearing
|
||||
// every slot prevents geometry/representation mixing and guarantees that old sources can take over.
|
||||
downgradeWireToLegacy();
|
||||
}
|
||||
if (requestSlot(slot)) _retry_slot = (uint8_t)((slot + 1) % OTA_FETCH_PIPELINE);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,22 @@ typedef bool (*OtaSend)(void* ctx, const uint8_t* msg, uint16_t len, bool flood)
|
||||
// error. nullptr means the payload is a contiguous RAM buffer (the staged `.mota`).
|
||||
typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len);
|
||||
|
||||
// Optional full raw-RFC1951 transport codec (stored, fixed-Huffman, and dynamic-Huffman blocks). Compression
|
||||
// is a wire optimization only: the receiver inflates exactly one authenticated logical block before Merkle
|
||||
// verification and writes the original `.mota` bytes to its store. Returning false, an empty result, or a
|
||||
// result no smaller than the source selects raw v2 DATA. A decoder must reject output overflow, truncated
|
||||
// streams, and trailing compressed input; OtaManager also checks that it produced exactly the manifest-derived
|
||||
// block length before accepting a proof.
|
||||
typedef bool (*OtaDeflateEncodeFn)(void* ctx, const uint8_t* src, uint16_t src_len,
|
||||
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len);
|
||||
typedef bool (*OtaDeflateDecodeFn)(void* ctx, const uint8_t* src, uint16_t src_len,
|
||||
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len);
|
||||
// Optional source-side fast path for a host/archive that already has an independently compressed logical
|
||||
// block. It avoids carrying a DEFLATE encoder on the embedded seeder. The result has the same strict bounds
|
||||
// and deterministic-per-block contract as OtaDeflateEncodeFn.
|
||||
typedef bool (*ServeDeflateReadFn)(void* ctx, uint16_t block, uint8_t* dst,
|
||||
uint16_t dst_cap, uint16_t* dst_len);
|
||||
|
||||
#ifndef OTA_PROOFGEN_SCRATCH
|
||||
#if defined(OTA_SD_STORE)
|
||||
#define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 2 MB at 1 KB/block)
|
||||
@@ -147,7 +163,10 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len)
|
||||
#define OTA_SERVE_SUPPRESS_MS 1500 // don't re-serve a block whose DATA we just overheard another holder send
|
||||
#endif // (so multiple sources of the same mota don't duplicate-broadcast it)
|
||||
#ifndef OTA_FRAG_DATA
|
||||
#define OTA_FRAG_DATA 160 // data bytes per DATA fragment (<= MAX_PACKET_PAYLOAD - 9-byte header)
|
||||
#define OTA_FRAG_DATA 160 // deployed legacy DATA geometry; never change in place
|
||||
#endif
|
||||
#ifndef OTA_FRAG_DATA_V2
|
||||
#define OTA_FRAG_DATA_V2 171 // negotiated DATA geometry (13-byte overhead => 184-byte packet payload)
|
||||
#endif
|
||||
#ifndef OTA_SERVE_QUEUE
|
||||
#define OTA_SERVE_QUEUE 4 // requested blocks retained without allocating one packet per fragment
|
||||
@@ -245,6 +264,8 @@ public:
|
||||
uint16_t mfl = 0; // manifest-minus-leaves length (the OTA_MANIFEST payload)
|
||||
ServeReadFn read = nullptr; // payload reader (nullptr => m.payload is contiguous in RAM)
|
||||
void* read_ctx = nullptr;
|
||||
ServeDeflateReadFn read_deflated = nullptr; // optional host/source-generated raw-DEFLATE block
|
||||
void* deflate_ctx = nullptr;
|
||||
uint8_t* scratch = nullptr; // proof-gen working buffer (>= block_count*4)
|
||||
uint32_t scratch_sz = 0;
|
||||
};
|
||||
@@ -302,6 +323,18 @@ public:
|
||||
|
||||
// --- fetch --- Provide the staging store; fetching starts on a matching OTA_ADV.
|
||||
void set_fetch_store(OtaStore* s) { _fetch = s; }
|
||||
// Install either side of the optional per-block transport codec. A receiver with no decoder still uses
|
||||
// negotiated 171-byte raw DATA; a source with no encoder automatically returns raw DATA. Context lifetimes
|
||||
// are caller-owned and independent because a relay/seeder may fetch and serve at the same time.
|
||||
void set_transport_deflate_encoder(OtaDeflateEncodeFn fn, void* ctx = nullptr) {
|
||||
_deflate_encode = fn; _deflate_encode_ctx = ctx;
|
||||
}
|
||||
void set_transport_deflate_decoder(OtaDeflateDecodeFn fn, void* ctx = nullptr) {
|
||||
_deflate_decode = fn; _deflate_decode_ctx = ctx;
|
||||
}
|
||||
// Defaults on for moving-forward 171-byte transfers. This switch exists for qualification and emergency
|
||||
// rollback; disabling it produces byte-for-byte legacy OTA_REQ/OTA_DATA behavior.
|
||||
void set_wire_v2_enabled(bool on) { _wire_v2_enabled = on; }
|
||||
|
||||
// Resume a fetch from a container already persisted in the store (after a reboot). want_mid=nullptr is
|
||||
// automatic adoption and rechecks current autofetch/target/version policy; a non-null MID is an explicit
|
||||
@@ -441,6 +474,7 @@ public:
|
||||
_lv_retries = 0; _loop_last_lvmask = 0;
|
||||
_resume_verify_idx = 0; _resume_invalidated = false;
|
||||
_resume_merkle.reset();
|
||||
_wire_empty_stalls = 0;
|
||||
}
|
||||
|
||||
FetchState fetchState() const { return _fstate; }
|
||||
@@ -525,7 +559,10 @@ private:
|
||||
bool loadSource(const ServeEntry& e); // load an external mota into _srcv (head+leaves)
|
||||
void registerSelfEntry(); // (re)build entry[0] from view0
|
||||
static bool srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len); // source payload reader
|
||||
bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask);
|
||||
static bool srcDeflateReadTramp(void* c, uint16_t block, uint8_t* buf,
|
||||
uint16_t cap, uint16_t* len);
|
||||
bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask,
|
||||
bool wire_v2 = false, bool allow_deflate = false);
|
||||
bool queueManifestJob(const uint8_t* mid, uint16_t want_mask);
|
||||
uint32_t manifestEgressGapMs() const;
|
||||
uint32_t proofEgressGapMs() const;
|
||||
@@ -556,6 +593,10 @@ private:
|
||||
bool requestFlight(); // send all newly assigned blocks in one OTA_REQ
|
||||
bool fillPipeline(); // assign + send a new flight only when the old one ended
|
||||
void requestMissing(); // start a flight, or recover one stalled slot
|
||||
uint16_t wireRequestMask(uint32_t block, uint16_t need, uint16_t received,
|
||||
uint16_t encoded_len) const;
|
||||
void downgradeWireToLegacy(); // old source answered/ignored the v2 request profile
|
||||
bool anyWireDataReceived() const;
|
||||
uint32_t blockLen(uint32_t i) const;
|
||||
|
||||
uint32_t _target = 0;
|
||||
@@ -593,6 +634,8 @@ private:
|
||||
uint16_t emitted_mask = 0;
|
||||
uint32_t proof_ready_at = 0;
|
||||
bool proof_requested = false;
|
||||
bool wire_v2 = false;
|
||||
bool allow_deflate = false;
|
||||
};
|
||||
ServeJob _serve_jobs[OTA_SERVE_QUEUE];
|
||||
uint8_t _n_serve_jobs = 0;
|
||||
@@ -607,6 +650,15 @@ private:
|
||||
uint8_t _serve_block[OTA_MAX_BLOCK];
|
||||
uint16_t _serve_block_len = 0;
|
||||
bool _serve_block_loaded = false;
|
||||
const uint8_t* _serve_wire_block = nullptr; // raw _serve_block or DEFLATE bytes in ServeView::scratch
|
||||
uint16_t _serve_wire_len = 0;
|
||||
bool _serve_wire_deflated = false;
|
||||
uint8_t _serve_wire_id[4] = {0}; // SHA-256:4 of this exact encoded representation
|
||||
|
||||
OtaDeflateEncodeFn _deflate_encode = nullptr;
|
||||
void* _deflate_encode_ctx = nullptr;
|
||||
OtaDeflateDecodeFn _deflate_decode = nullptr;
|
||||
void* _deflate_decode_ctx = nullptr;
|
||||
|
||||
// fetch
|
||||
OtaStore* _fetch = nullptr;
|
||||
@@ -638,6 +690,11 @@ private:
|
||||
uint16_t _advert_mins = OTA_ADVERT_INTERVAL_MINS; // beacon re-advertise cadence, minutes; 0=off (persisted)
|
||||
uint8_t _max_hops = OTA_HOP_LIMIT_DEFAULT; // OTA flood reach in hops; 0=direct only (persisted)
|
||||
uint8_t _fflags = 0; // flags of the manifest currently being fetched
|
||||
bool _wire_v2_enabled = true; // advertise v2 in OTA_REQ while retaining transparent fallback
|
||||
bool _wire_v2_session = true; // false after an old source responds or the first request stalls
|
||||
bool _wire_v2_confirmed = false; // at least one structurally valid v2 DATA packet was observed
|
||||
bool _wire_allow_deflate = false; // session may request compressed blocks only with a decoder
|
||||
uint8_t _wire_empty_stalls = 0; // confirmed v2 source may vanish between otherwise empty flights
|
||||
// Bounded adaptive multi-block client flight. Each slot independently reassembles DATA and awaits its
|
||||
// Merkle proof. One append-only OTA_REQ packet opens the whole flight; the receiver then stays quiet until
|
||||
// every slot completes or an airtime-aware deadline expires. Clean flights grow 1->2->3->4; any recovery
|
||||
@@ -648,6 +705,11 @@ private:
|
||||
uint16_t need = 0; // full bitmap for this block
|
||||
bool awaiting_proof = false;
|
||||
uint32_t proof_request_at = 0; // proactive-proof grace deadline; 0 after fallback is sent
|
||||
uint16_t encoded_len = 0; // v2 wire length; zero until the first DATA fragment establishes it
|
||||
bool wire_v2 = false;
|
||||
bool deflated = false;
|
||||
uint8_t stream_id[4] = {0}; // locks fragments to one encoded representation across seeders
|
||||
uint8_t wire_stalls = 0; // eventually unlock a vanished representation and retry cleanly
|
||||
uint8_t buf[OTA_MAX_BLOCK];
|
||||
};
|
||||
ReassemblySlot _reasm[OTA_FETCH_PIPELINE];
|
||||
|
||||
@@ -69,8 +69,11 @@ struct ManifestMsg {
|
||||
// source queues every row. This lets a fetcher open one adaptive flight with one packet, then remain silent
|
||||
// while the source and relays return the requested DATA/PROOF train.
|
||||
//
|
||||
// want_mask bit k = "send fragment k" (the slice at byte offset k*OTA_FRAG_DATA). A recovery request uses
|
||||
// only the missing bits, so one lost fragment costs one fragment to re-send rather than the whole block.
|
||||
// Legacy want_mask bit k = "send fragment k" (the slice at byte offset k*OTA_FRAG_DATA). A recovery request
|
||||
// uses only the missing bits, so one lost fragment costs one fragment to re-send rather than the whole block.
|
||||
// New receivers reserve bits 15..14 as an in-band request profile: bit 15 asks for the v2 171-byte geometry
|
||||
// and bit 14 permits per-block raw DEFLATE. An old source masks those bits away and returns the ordinary
|
||||
// 160-byte DATA train; the new receiver detects that response and downgrades without stranding old fleets.
|
||||
struct ReqMsg { uint8_t manifest_id[4]; uint16_t block_idx; uint16_t want_mask; };
|
||||
struct ReqItem { uint16_t block_idx; uint16_t want_mask; };
|
||||
#ifndef OTA_REQ_MAX_ITEMS
|
||||
@@ -84,8 +87,13 @@ struct ReqWindowMsg {
|
||||
|
||||
// ---- OTA_DATA: one self-describing fragment of a block's data (proof stays in a separate packet) ----
|
||||
// body: manifest_id(4) block_idx(2) frag_off(2) data[]
|
||||
// `frag_off` is the byte offset of `data` within block `block_idx` (global position = block_idx*block_size
|
||||
// + frag_off), so a fragment is self-placing when received from the selected source.
|
||||
// In the legacy profile, `frag_off` is the byte offset of `data` within block `block_idx` (global position =
|
||||
// block_idx*block_size + frag_off), so a fragment is self-placing when received from the selected source.
|
||||
// In the negotiated v2 profile its high bit is set and the field packs the per-block wire encoding, fragment
|
||||
// index, and total encoded size; data[] starts with SHA-256:4 of the complete encoded representation, followed
|
||||
// by up to 171 bytes. Repeating that id in every fragment prevents different seeders' valid-but-different
|
||||
// DEFLATE streams from being mixed. Keeping the existing OTA_DATA type means deployed opaque relays retain the
|
||||
// same transfer priority. Old endpoints reject the tagged offset and new endpoints only request it explicitly.
|
||||
struct DataMsg {
|
||||
uint8_t manifest_id[4];
|
||||
uint16_t block_idx;
|
||||
@@ -93,6 +101,59 @@ struct DataMsg {
|
||||
const uint8_t* data; uint16_t data_len;
|
||||
};
|
||||
|
||||
// OTA_REQ want_mask extension. The v2 marker and DEFLATE permission are deliberately outside every valid
|
||||
// fragment bit for the fixed <=1 KiB blocks used by this protocol. A v2 request includes all seven legacy
|
||||
// fragment bits on its first attempt, so a deployed source which ignores these flags can answer completely.
|
||||
static const uint16_t OTA_REQ_V2_MARK = 0x8000u;
|
||||
static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u;
|
||||
static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u;
|
||||
static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu;
|
||||
|
||||
// OTA_DATA v2 frag_off packing:
|
||||
// bit 15 v2 marker
|
||||
// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes)
|
||||
// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2)
|
||||
// bits 9..0 total encoded block length minus one (1..1024 bytes)
|
||||
static const uint16_t OTA_DATA_V2_MARK = 0x8000u;
|
||||
static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u;
|
||||
static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u;
|
||||
static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu;
|
||||
static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10;
|
||||
static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024;
|
||||
static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4;
|
||||
|
||||
inline bool ota_req_is_v2(uint16_t want_mask) {
|
||||
return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0;
|
||||
}
|
||||
|
||||
inline uint16_t ota_req_v2_fragments(uint16_t want_mask) {
|
||||
return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK);
|
||||
}
|
||||
|
||||
inline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) {
|
||||
return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK |
|
||||
(allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0));
|
||||
}
|
||||
|
||||
inline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated,
|
||||
uint16_t& packed) {
|
||||
if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;
|
||||
packed = (uint16_t)(OTA_DATA_V2_MARK |
|
||||
(deflated ? OTA_DATA_V2_DEFLATED : 0) |
|
||||
((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) |
|
||||
(encoded_len - 1u));
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool ota_data_v2_unpack(uint16_t packed, uint8_t& fragment, uint16_t& encoded_len,
|
||||
bool& deflated) {
|
||||
if ((packed & OTA_DATA_V2_MARK) == 0) return false;
|
||||
fragment = (uint8_t)((packed & OTA_DATA_V2_FRAGMENT_BITS) >> OTA_DATA_V2_FRAGMENT_SHIFT);
|
||||
encoded_len = (uint16_t)((packed & OTA_DATA_V2_LENGTH_BITS) + 1u);
|
||||
deflated = (packed & OTA_DATA_V2_DEFLATED) != 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- OTA_GET_LEAVES: request the target's merkle leaves[] in bulk (direct; motatool warm-start only) ----
|
||||
// body: manifest_id(4) want_mask(2). want_mask bit k = "send leaves fragment k"; 0xFFFF = "send all" on the
|
||||
// first ask (before frag_total is known); a retry requests only the still-missing fragments (anti-burst,
|
||||
|
||||
@@ -18,6 +18,11 @@
|
||||
namespace mesh {
|
||||
namespace ota {
|
||||
|
||||
// Optional operations exposed by a MotaSource. Serial/TCP sources carry these
|
||||
// bits in byte 35 of their unchanged 38-byte descriptor; deployed hosts leave
|
||||
// the formerly-reserved byte zero.
|
||||
static const uint8_t MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01;
|
||||
|
||||
// A parsed top-level descriptor of one `.mota` a source provides: enough to advertise it in the catalog
|
||||
// AND to locate every region for serving, WITHOUT holding the whole image in RAM. Offsets are absolute
|
||||
// byte positions within the `.mota` container (which always begins MAGIC(4) total(4) manifest...).
|
||||
@@ -28,6 +33,7 @@ struct MotaDesc {
|
||||
uint8_t codec_id = 0;
|
||||
uint8_t flags = 0;
|
||||
uint8_t block_size_log2 = 0; // serve geometry; 0 means a legacy descriptor that must be inspected
|
||||
uint8_t source_caps = 0; // MOTA_SOURCE_CAP_*; zero preserves deployed source behavior
|
||||
uint32_t total_size = 0; // full `.mota` length (bytes)
|
||||
uint32_t leaves_off = 0; // byte offset of the merkle leaves[] (manifest-minus-leaves = [8, leaves_off))
|
||||
uint32_t block_count = 0; // == number of leaves (== number of payload blocks)
|
||||
@@ -50,6 +56,15 @@ public:
|
||||
// exactly `len` bytes were produced. May block on the transport (serial round-trip); implementations
|
||||
// should keep active-transfer reads prompt because requested OTA traffic is primary traffic.
|
||||
virtual bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) = 0;
|
||||
// Optional transport-only raw-DEFLATE representation of one logical payload block. Implementations which
|
||||
// can generate/cache it off-device return a stream strictly smaller than the raw block. The default keeps
|
||||
// every existing source source-compatible and makes OtaManager fall back to raw 171-byte DATA.
|
||||
virtual bool read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf,
|
||||
uint16_t cap, uint16_t* len) {
|
||||
(void)idx; (void)block; (void)buf; (void)cap;
|
||||
if (len) *len = 0;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace ota
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Compile the vendored tinf core as C. Building the same source as C++ costs
|
||||
* several extra kilobytes with the embedded toolchains used by MeshCore.
|
||||
*/
|
||||
#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST)
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
/* nRF52's Arduino recipe defaults to -Ofast, which more than doubles this
|
||||
* decoder. Keep the size policy local to tinf instead of changing MeshCore. */
|
||||
#pragma GCC push_options
|
||||
#pragma GCC optimize ("Os")
|
||||
#endif
|
||||
#define MESHCORE_TINF_IMPLEMENTATION 1
|
||||
#include "tinf/tinflate.c"
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC pop_options
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,22 @@
|
||||
The zlib License (Zlib)
|
||||
|
||||
Copyright (c) 2003-2019 Joergen Ibsen
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must
|
||||
not claim that you wrote the original software. If you use this
|
||||
software in a product, an acknowledgment in the product
|
||||
documentation would be appreciated but is not required.
|
||||
|
||||
2. Altered source versions must be plainly marked as such, and must
|
||||
not be misrepresented as being the original software.
|
||||
|
||||
3. This notice may not be removed or altered from any source
|
||||
distribution.
|
||||
@@ -0,0 +1,23 @@
|
||||
tinf 1.2.1 raw-DEFLATE decoder
|
||||
================================
|
||||
|
||||
Upstream: https://github.com/jibsen/tinf
|
||||
Revision: 57ffa1f1d5e3dde19011b2127bd26d01689b694b
|
||||
|
||||
Vendored files:
|
||||
LICENSE
|
||||
tinf.h
|
||||
tinflate.c
|
||||
|
||||
MeshCore changes are marked in tinf.h and tinflate.c. They add
|
||||
tinf_uncompress_exact(), which accepts legal padding bits in the byte containing
|
||||
the final end-of-block code but rejects any trailing whole input byte. The C
|
||||
implementation is included by ../OtaTinf.c only for ENABLE_OTA application
|
||||
builds (and the native OTA test profile). ../OtaDeflate.cpp is the small C++
|
||||
callback adapter. Keeping the decoder in a true C translation unit avoids the
|
||||
several-kilobyte penalty produced by the embedded C++ toolchains; non-OTA
|
||||
firmware links no decoder code or persistent RAM.
|
||||
|
||||
Only the raw RFC 1951 decoder is integrated. The zlib/gzip wrappers and checksum
|
||||
sources are intentionally omitted. The original zlib license is preserved in
|
||||
LICENSE and in the source headers.
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* tinf - tiny inflate library (inflate, gzip, zlib)
|
||||
*
|
||||
* Copyright (c) 2003-2019 Joergen Ibsen
|
||||
*
|
||||
* This software is provided 'as-is', without any express or implied
|
||||
* warranty. In no event will the authors be held liable for any damages
|
||||
* arising from the use of this software.
|
||||
*
|
||||
* Permission is granted to anyone to use this software for any purpose,
|
||||
* including commercial applications, and to alter it and redistribute it
|
||||
* freely, subject to the following restrictions:
|
||||
*
|
||||
* 1. The origin of this software must not be misrepresented; you must
|
||||
* not claim that you wrote the original software. If you use this
|
||||
* software in a product, an acknowledgment in the product
|
||||
* documentation would be appreciated but is not required.
|
||||
*
|
||||
* 2. Altered source versions must be plainly marked as such, and must
|
||||
* not be misrepresented as being the original software.
|
||||
*
|
||||
* 3. This notice may not be removed or altered from any source
|
||||
* distribution.
|
||||
*/
|
||||
|
||||
/*
|
||||
* MeshCore modification (2026): declare tinf_uncompress_exact(), a raw-DEFLATE
|
||||
* entry point which rejects trailing whole input bytes. The original API and
|
||||
* behavior remain available as tinf_uncompress().
|
||||
*/
|
||||
|
||||
#ifndef TINF_H_INCLUDED
|
||||
#define TINF_H_INCLUDED
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define TINF_VER_MAJOR 1 /**< Major version number */
|
||||
#define TINF_VER_MINOR 2 /**< Minor version number */
|
||||
#define TINF_VER_PATCH 1 /**< Patch version number */
|
||||
#define TINF_VER_STRING "1.2.1" /**< Version number as a string */
|
||||
|
||||
#ifndef TINFCC
|
||||
# ifdef __WATCOMC__
|
||||
# define TINFCC __cdecl
|
||||
# else
|
||||
# define TINFCC
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Status codes returned.
|
||||
*
|
||||
* @see tinf_uncompress, tinf_gzip_uncompress, tinf_zlib_uncompress
|
||||
*/
|
||||
typedef enum {
|
||||
TINF_OK = 0, /**< Success */
|
||||
TINF_DATA_ERROR = -3, /**< Input error */
|
||||
TINF_BUF_ERROR = -5 /**< Not enough room for output */
|
||||
} tinf_error_code;
|
||||
|
||||
/**
|
||||
* Initialize global data used by tinf.
|
||||
*
|
||||
* @deprecated No longer required, may be removed in a future version.
|
||||
*/
|
||||
void TINFCC tinf_init(void);
|
||||
|
||||
/**
|
||||
* Decompress `sourceLen` bytes of deflate data from `source` to `dest`.
|
||||
*
|
||||
* The variable `destLen` points to must contain the size of `dest` on entry,
|
||||
* and will be set to the size of the decompressed data on success.
|
||||
*
|
||||
* Reads at most `sourceLen` bytes from `source`.
|
||||
* Writes at most `*destLen` bytes to `dest`.
|
||||
*
|
||||
* @param dest pointer to where to place decompressed data
|
||||
* @param destLen pointer to variable containing size of `dest`
|
||||
* @param source pointer to compressed data
|
||||
* @param sourceLen size of compressed data
|
||||
* @return `TINF_OK` on success, error code on error
|
||||
*/
|
||||
int TINFCC tinf_uncompress(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen);
|
||||
|
||||
/**
|
||||
* Decompress exactly one raw DEFLATE stream and reject trailing whole bytes.
|
||||
* Padding bits in the byte containing the final end-of-block code are allowed.
|
||||
* Output bounds and `destLen` semantics match tinf_uncompress().
|
||||
*/
|
||||
int TINFCC tinf_uncompress_exact(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen);
|
||||
|
||||
/**
|
||||
* Decompress `sourceLen` bytes of gzip data from `source` to `dest`.
|
||||
*
|
||||
* The variable `destLen` points to must contain the size of `dest` on entry,
|
||||
* and will be set to the size of the decompressed data on success.
|
||||
*
|
||||
* Reads at most `sourceLen` bytes from `source`.
|
||||
* Writes at most `*destLen` bytes to `dest`.
|
||||
*
|
||||
* @param dest pointer to where to place decompressed data
|
||||
* @param destLen pointer to variable containing size of `dest`
|
||||
* @param source pointer to compressed data
|
||||
* @param sourceLen size of compressed data
|
||||
* @return `TINF_OK` on success, error code on error
|
||||
*/
|
||||
int TINFCC tinf_gzip_uncompress(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen);
|
||||
|
||||
/**
|
||||
* Decompress `sourceLen` bytes of zlib data from `source` to `dest`.
|
||||
*
|
||||
* The variable `destLen` points to must contain the size of `dest` on entry,
|
||||
* and will be set to the size of the decompressed data on success.
|
||||
*
|
||||
* Reads at most `sourceLen` bytes from `source`.
|
||||
* Writes at most `*destLen` bytes to `dest`.
|
||||
*
|
||||
* @param dest pointer to where to place decompressed data
|
||||
* @param destLen pointer to variable containing size of `dest`
|
||||
* @param source pointer to compressed data
|
||||
* @param sourceLen size of compressed data
|
||||
* @return `TINF_OK` on success, error code on error
|
||||
*/
|
||||
int TINFCC tinf_zlib_uncompress(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen);
|
||||
|
||||
/**
|
||||
* Compute Adler-32 checksum of `length` bytes starting at `data`.
|
||||
*
|
||||
* @param data pointer to data
|
||||
* @param length size of data
|
||||
* @return Adler-32 checksum
|
||||
*/
|
||||
unsigned int TINFCC tinf_adler32(const void *data, unsigned int length);
|
||||
|
||||
/**
|
||||
* Compute CRC32 checksum of `length` bytes starting at `data`.
|
||||
*
|
||||
* @param data pointer to data
|
||||
* @param length size of data
|
||||
* @return CRC32 checksum
|
||||
*/
|
||||
unsigned int TINFCC tinf_crc32(const void *data, unsigned int length);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* TINF_H_INCLUDED */
|
||||
@@ -0,0 +1,666 @@
|
||||
/*
|
||||
* tinflate - tiny inflate
|
||||
*
|
||||
* Copyright (c) 2003-2019 Joergen Ibsen
|
||||
*
|
||||
* This software is provided 'as-is', without any express or implied
|
||||
* warranty. In no event will the authors be held liable for any damages
|
||||
* arising from the use of this software.
|
||||
*
|
||||
* Permission is granted to anyone to use this software for any purpose,
|
||||
* including commercial applications, and to alter it and redistribute it
|
||||
* freely, subject to the following restrictions:
|
||||
*
|
||||
* 1. The origin of this software must not be misrepresented; you must
|
||||
* not claim that you wrote the original software. If you use this
|
||||
* software in a product, an acknowledgment in the product
|
||||
* documentation would be appreciated but is not required.
|
||||
*
|
||||
* 2. Altered source versions must be plainly marked as such, and must
|
||||
* not be misrepresented as being the original software.
|
||||
*
|
||||
* 3. This notice may not be removed or altered from any source
|
||||
* distribution.
|
||||
*/
|
||||
|
||||
/*
|
||||
* MeshCore modifications (2026):
|
||||
* - expose tinf_uncompress_exact() to reject trailing whole input bytes;
|
||||
* - compile this vendored implementation through OtaTinf.c only, keeping
|
||||
* it out of builds which do not define ENABLE_OTA.
|
||||
* See README.meshcore.txt for the upstream revision and integration details.
|
||||
*/
|
||||
|
||||
#if defined(MESHCORE_TINF_IMPLEMENTATION)
|
||||
|
||||
#include "tinf.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <limits.h>
|
||||
|
||||
#if defined(UINT_MAX) && (UINT_MAX) < 0xFFFFFFFFUL
|
||||
# error "tinf requires unsigned int to be at least 32-bit"
|
||||
#endif
|
||||
|
||||
/* -- Internal data structures -- */
|
||||
|
||||
struct tinf_tree {
|
||||
unsigned short counts[16]; /* Number of codes with a given length */
|
||||
unsigned short symbols[288]; /* Symbols sorted by code */
|
||||
int max_sym;
|
||||
};
|
||||
|
||||
struct tinf_data {
|
||||
const unsigned char *source;
|
||||
const unsigned char *source_end;
|
||||
unsigned int tag;
|
||||
int bitcount;
|
||||
int overflow;
|
||||
|
||||
unsigned char *dest_start;
|
||||
unsigned char *dest;
|
||||
unsigned char *dest_end;
|
||||
|
||||
struct tinf_tree ltree; /* Literal/length tree */
|
||||
struct tinf_tree dtree; /* Distance tree */
|
||||
};
|
||||
|
||||
/* -- Utility functions -- */
|
||||
|
||||
static unsigned int read_le16(const unsigned char *p)
|
||||
{
|
||||
return ((unsigned int) p[0])
|
||||
| ((unsigned int) p[1] << 8);
|
||||
}
|
||||
|
||||
/* Build fixed Huffman trees */
|
||||
static void tinf_build_fixed_trees(struct tinf_tree *lt, struct tinf_tree *dt)
|
||||
{
|
||||
int i;
|
||||
|
||||
/* Build fixed literal/length tree */
|
||||
for (i = 0; i < 16; ++i) {
|
||||
lt->counts[i] = 0;
|
||||
}
|
||||
|
||||
lt->counts[7] = 24;
|
||||
lt->counts[8] = 152;
|
||||
lt->counts[9] = 112;
|
||||
|
||||
for (i = 0; i < 24; ++i) {
|
||||
lt->symbols[i] = 256 + i;
|
||||
}
|
||||
for (i = 0; i < 144; ++i) {
|
||||
lt->symbols[24 + i] = i;
|
||||
}
|
||||
for (i = 0; i < 8; ++i) {
|
||||
lt->symbols[24 + 144 + i] = 280 + i;
|
||||
}
|
||||
for (i = 0; i < 112; ++i) {
|
||||
lt->symbols[24 + 144 + 8 + i] = 144 + i;
|
||||
}
|
||||
|
||||
lt->max_sym = 285;
|
||||
|
||||
/* Build fixed distance tree */
|
||||
for (i = 0; i < 16; ++i) {
|
||||
dt->counts[i] = 0;
|
||||
}
|
||||
|
||||
dt->counts[5] = 32;
|
||||
|
||||
for (i = 0; i < 32; ++i) {
|
||||
dt->symbols[i] = i;
|
||||
}
|
||||
|
||||
dt->max_sym = 29;
|
||||
}
|
||||
|
||||
/* Given an array of code lengths, build a tree */
|
||||
static int tinf_build_tree(struct tinf_tree *t, const unsigned char *lengths,
|
||||
unsigned int num)
|
||||
{
|
||||
unsigned short offs[16];
|
||||
unsigned int i, num_codes, available;
|
||||
|
||||
assert(num <= 288);
|
||||
|
||||
for (i = 0; i < 16; ++i) {
|
||||
t->counts[i] = 0;
|
||||
}
|
||||
|
||||
t->max_sym = -1;
|
||||
|
||||
/* Count number of codes for each non-zero length */
|
||||
for (i = 0; i < num; ++i) {
|
||||
assert(lengths[i] <= 15);
|
||||
|
||||
if (lengths[i]) {
|
||||
t->max_sym = i;
|
||||
t->counts[lengths[i]]++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Compute offset table for distribution sort */
|
||||
for (available = 1, num_codes = 0, i = 0; i < 16; ++i) {
|
||||
unsigned int used = t->counts[i];
|
||||
|
||||
/* Check length contains no more codes than available */
|
||||
if (used > available) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
available = 2 * (available - used);
|
||||
|
||||
offs[i] = num_codes;
|
||||
num_codes += used;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check all codes were used, or for the special case of only one
|
||||
* code that it has length 1
|
||||
*/
|
||||
if ((num_codes > 1 && available > 0)
|
||||
|| (num_codes == 1 && t->counts[1] != 1)) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* Fill in symbols sorted by code */
|
||||
for (i = 0; i < num; ++i) {
|
||||
if (lengths[i]) {
|
||||
t->symbols[offs[lengths[i]]++] = i;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* For the special case of only one code (which will be 0) add a
|
||||
* code 1 which results in a symbol that is too large
|
||||
*/
|
||||
if (num_codes == 1) {
|
||||
t->counts[1] = 2;
|
||||
t->symbols[1] = t->max_sym + 1;
|
||||
}
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* -- Decode functions -- */
|
||||
|
||||
static void tinf_refill(struct tinf_data *d, int num)
|
||||
{
|
||||
assert(num >= 0 && num <= 32);
|
||||
|
||||
/* Read bytes until at least num bits available */
|
||||
while (d->bitcount < num) {
|
||||
if (d->source != d->source_end) {
|
||||
d->tag |= (unsigned int) *d->source++ << d->bitcount;
|
||||
}
|
||||
else {
|
||||
d->overflow = 1;
|
||||
}
|
||||
d->bitcount += 8;
|
||||
}
|
||||
|
||||
assert(d->bitcount <= 32);
|
||||
}
|
||||
|
||||
static unsigned int tinf_getbits_no_refill(struct tinf_data *d, int num)
|
||||
{
|
||||
unsigned int bits;
|
||||
|
||||
assert(num >= 0 && num <= d->bitcount);
|
||||
|
||||
/* Get bits from tag */
|
||||
bits = d->tag & ((1UL << num) - 1);
|
||||
|
||||
/* Remove bits from tag */
|
||||
d->tag >>= num;
|
||||
d->bitcount -= num;
|
||||
|
||||
return bits;
|
||||
}
|
||||
|
||||
/* Get num bits from source stream */
|
||||
static unsigned int tinf_getbits(struct tinf_data *d, int num)
|
||||
{
|
||||
tinf_refill(d, num);
|
||||
return tinf_getbits_no_refill(d, num);
|
||||
}
|
||||
|
||||
/* Read a num bit value from stream and add base */
|
||||
static unsigned int tinf_getbits_base(struct tinf_data *d, int num, int base)
|
||||
{
|
||||
return base + (num ? tinf_getbits(d, num) : 0);
|
||||
}
|
||||
|
||||
/* Given a data stream and a tree, decode a symbol */
|
||||
static int tinf_decode_symbol(struct tinf_data *d, const struct tinf_tree *t)
|
||||
{
|
||||
int base = 0, offs = 0;
|
||||
int len;
|
||||
|
||||
/*
|
||||
* Get more bits while code index is above number of codes
|
||||
*
|
||||
* Rather than the actual code, we are computing the position of the
|
||||
* code in the sorted order of codes, which is the index of the
|
||||
* corresponding symbol.
|
||||
*
|
||||
* Conceptually, for each code length (level in the tree), there are
|
||||
* counts[len] leaves on the left and internal nodes on the right.
|
||||
* The index we have decoded so far is base + offs, and if that
|
||||
* falls within the leaves we are done. Otherwise we adjust the range
|
||||
* of offs and add one more bit to it.
|
||||
*/
|
||||
for (len = 1; ; ++len) {
|
||||
offs = 2 * offs + tinf_getbits(d, 1);
|
||||
|
||||
assert(len <= 15);
|
||||
|
||||
if (offs < t->counts[len]) {
|
||||
break;
|
||||
}
|
||||
|
||||
base += t->counts[len];
|
||||
offs -= t->counts[len];
|
||||
}
|
||||
|
||||
assert(base + offs >= 0 && base + offs < 288);
|
||||
|
||||
return t->symbols[base + offs];
|
||||
}
|
||||
|
||||
/* Given a data stream, decode dynamic trees from it */
|
||||
static int tinf_decode_trees(struct tinf_data *d, struct tinf_tree *lt,
|
||||
struct tinf_tree *dt)
|
||||
{
|
||||
unsigned char lengths[288 + 32];
|
||||
|
||||
/* Special ordering of code length codes */
|
||||
static const unsigned char clcidx[19] = {
|
||||
16, 17, 18, 0, 8, 7, 9, 6, 10, 5,
|
||||
11, 4, 12, 3, 13, 2, 14, 1, 15
|
||||
};
|
||||
|
||||
unsigned int hlit, hdist, hclen;
|
||||
unsigned int i, num, length;
|
||||
int res;
|
||||
|
||||
/* Get 5 bits HLIT (257-286) */
|
||||
hlit = tinf_getbits_base(d, 5, 257);
|
||||
|
||||
/* Get 5 bits HDIST (1-32) */
|
||||
hdist = tinf_getbits_base(d, 5, 1);
|
||||
|
||||
/* Get 4 bits HCLEN (4-19) */
|
||||
hclen = tinf_getbits_base(d, 4, 4);
|
||||
|
||||
/*
|
||||
* The RFC limits the range of HLIT to 286, but lists HDIST as range
|
||||
* 1-32, even though distance codes 30 and 31 have no meaning. While
|
||||
* we could allow the full range of HLIT and HDIST to make it possible
|
||||
* to decode the fixed trees with this function, we consider it an
|
||||
* error here.
|
||||
*
|
||||
* See also: https://github.com/madler/zlib/issues/82
|
||||
*/
|
||||
if (hlit > 286 || hdist > 30) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
for (i = 0; i < 19; ++i) {
|
||||
lengths[i] = 0;
|
||||
}
|
||||
|
||||
/* Read code lengths for code length alphabet */
|
||||
for (i = 0; i < hclen; ++i) {
|
||||
/* Get 3 bits code length (0-7) */
|
||||
unsigned int clen = tinf_getbits(d, 3);
|
||||
|
||||
lengths[clcidx[i]] = clen;
|
||||
}
|
||||
|
||||
/* Build code length tree (in literal/length tree to save space) */
|
||||
res = tinf_build_tree(lt, lengths, 19);
|
||||
|
||||
if (res != TINF_OK) {
|
||||
return res;
|
||||
}
|
||||
|
||||
/* Check code length tree is not empty */
|
||||
if (lt->max_sym == -1) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* Decode code lengths for the dynamic trees */
|
||||
for (num = 0; num < hlit + hdist; ) {
|
||||
int sym = tinf_decode_symbol(d, lt);
|
||||
|
||||
if (sym > lt->max_sym) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
switch (sym) {
|
||||
case 16:
|
||||
/* Copy previous code length 3-6 times (read 2 bits) */
|
||||
if (num == 0) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
sym = lengths[num - 1];
|
||||
length = tinf_getbits_base(d, 2, 3);
|
||||
break;
|
||||
case 17:
|
||||
/* Repeat code length 0 for 3-10 times (read 3 bits) */
|
||||
sym = 0;
|
||||
length = tinf_getbits_base(d, 3, 3);
|
||||
break;
|
||||
case 18:
|
||||
/* Repeat code length 0 for 11-138 times (read 7 bits) */
|
||||
sym = 0;
|
||||
length = tinf_getbits_base(d, 7, 11);
|
||||
break;
|
||||
default:
|
||||
/* Values 0-15 represent the actual code lengths */
|
||||
length = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (length > hlit + hdist - num) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
while (length--) {
|
||||
lengths[num++] = sym;
|
||||
}
|
||||
}
|
||||
|
||||
/* Check EOB symbol is present */
|
||||
if (lengths[256] == 0) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* Build dynamic trees */
|
||||
res = tinf_build_tree(lt, lengths, hlit);
|
||||
|
||||
if (res != TINF_OK) {
|
||||
return res;
|
||||
}
|
||||
|
||||
res = tinf_build_tree(dt, lengths + hlit, hdist);
|
||||
|
||||
if (res != TINF_OK) {
|
||||
return res;
|
||||
}
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* -- Block inflate functions -- */
|
||||
|
||||
/* Given a stream and two trees, inflate a block of data */
|
||||
static int tinf_inflate_block_data(struct tinf_data *d, struct tinf_tree *lt,
|
||||
struct tinf_tree *dt)
|
||||
{
|
||||
/* Extra bits and base tables for length codes */
|
||||
static const unsigned char length_bits[30] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
|
||||
1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 5, 5, 5, 5, 0, 127
|
||||
};
|
||||
|
||||
static const unsigned short length_base[30] = {
|
||||
3, 4, 5, 6, 7, 8, 9, 10, 11, 13,
|
||||
15, 17, 19, 23, 27, 31, 35, 43, 51, 59,
|
||||
67, 83, 99, 115, 131, 163, 195, 227, 258, 0
|
||||
};
|
||||
|
||||
/* Extra bits and base tables for distance codes */
|
||||
static const unsigned char dist_bits[30] = {
|
||||
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
|
||||
9, 9, 10, 10, 11, 11, 12, 12, 13, 13
|
||||
};
|
||||
|
||||
static const unsigned short dist_base[30] = {
|
||||
1, 2, 3, 4, 5, 7, 9, 13, 17, 25,
|
||||
33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
|
||||
1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
int sym = tinf_decode_symbol(d, lt);
|
||||
|
||||
/* Check for overflow in bit reader */
|
||||
if (d->overflow) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
if (sym < 256) {
|
||||
if (d->dest == d->dest_end) {
|
||||
return TINF_BUF_ERROR;
|
||||
}
|
||||
*d->dest++ = sym;
|
||||
}
|
||||
else {
|
||||
int length, dist, offs;
|
||||
int i;
|
||||
|
||||
/* Check for end of block */
|
||||
if (sym == 256) {
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* Check sym is within range and distance tree is not empty */
|
||||
if (sym > lt->max_sym || sym - 257 > 28 || dt->max_sym == -1) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
sym -= 257;
|
||||
|
||||
/* Possibly get more bits from length code */
|
||||
length = tinf_getbits_base(d, length_bits[sym],
|
||||
length_base[sym]);
|
||||
|
||||
dist = tinf_decode_symbol(d, dt);
|
||||
|
||||
/* Check dist is within range */
|
||||
if (dist > dt->max_sym || dist > 29) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* Possibly get more bits from distance code */
|
||||
offs = tinf_getbits_base(d, dist_bits[dist],
|
||||
dist_base[dist]);
|
||||
|
||||
if (offs > d->dest - d->dest_start) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
if (d->dest_end - d->dest < length) {
|
||||
return TINF_BUF_ERROR;
|
||||
}
|
||||
|
||||
/* Copy match */
|
||||
for (i = 0; i < length; ++i) {
|
||||
d->dest[i] = d->dest[i - offs];
|
||||
}
|
||||
|
||||
d->dest += length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Inflate an uncompressed block of data */
|
||||
static int tinf_inflate_uncompressed_block(struct tinf_data *d)
|
||||
{
|
||||
unsigned int length, invlength;
|
||||
|
||||
if (d->source_end - d->source < 4) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* Get length */
|
||||
length = read_le16(d->source);
|
||||
|
||||
/* Get one's complement of length */
|
||||
invlength = read_le16(d->source + 2);
|
||||
|
||||
/* Check length */
|
||||
if (length != (~invlength & 0x0000FFFF)) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
d->source += 4;
|
||||
|
||||
if (d->source_end - d->source < length) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
if (d->dest_end - d->dest < length) {
|
||||
return TINF_BUF_ERROR;
|
||||
}
|
||||
|
||||
/* Copy block */
|
||||
while (length--) {
|
||||
*d->dest++ = *d->source++;
|
||||
}
|
||||
|
||||
/* Make sure we start next block on a byte boundary */
|
||||
d->tag = 0;
|
||||
d->bitcount = 0;
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* Inflate a block of data compressed with fixed Huffman trees */
|
||||
static int tinf_inflate_fixed_block(struct tinf_data *d)
|
||||
{
|
||||
/* Build fixed Huffman trees */
|
||||
tinf_build_fixed_trees(&d->ltree, &d->dtree);
|
||||
|
||||
/* Decode block using fixed trees */
|
||||
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
|
||||
}
|
||||
|
||||
/* Inflate a block of data compressed with dynamic Huffman trees */
|
||||
static int tinf_inflate_dynamic_block(struct tinf_data *d)
|
||||
{
|
||||
/* Decode trees from stream */
|
||||
int res = tinf_decode_trees(d, &d->ltree, &d->dtree);
|
||||
|
||||
if (res != TINF_OK) {
|
||||
return res;
|
||||
}
|
||||
|
||||
/* Decode block using decoded trees */
|
||||
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
|
||||
}
|
||||
|
||||
/* -- Public functions -- */
|
||||
|
||||
/* Initialize global (static) data */
|
||||
void tinf_init(void)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
/* Inflate stream from source to dest. Optionally require the final DEFLATE block to consume all source bytes. */
|
||||
static int tinf_uncompress_internal(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen,
|
||||
int require_exact_source)
|
||||
{
|
||||
struct tinf_data d;
|
||||
int bfinal;
|
||||
|
||||
/* Initialise data */
|
||||
d.source = (const unsigned char *) source;
|
||||
d.source_end = d.source + sourceLen;
|
||||
d.tag = 0;
|
||||
d.bitcount = 0;
|
||||
d.overflow = 0;
|
||||
|
||||
d.dest = (unsigned char *) dest;
|
||||
d.dest_start = d.dest;
|
||||
d.dest_end = d.dest + *destLen;
|
||||
|
||||
do {
|
||||
unsigned int btype;
|
||||
int res;
|
||||
|
||||
/* Read final block flag */
|
||||
bfinal = tinf_getbits(&d, 1);
|
||||
|
||||
/* Read block type (2 bits) */
|
||||
btype = tinf_getbits(&d, 2);
|
||||
|
||||
/* Decompress block */
|
||||
switch (btype) {
|
||||
case 0:
|
||||
/* Decompress uncompressed block */
|
||||
res = tinf_inflate_uncompressed_block(&d);
|
||||
break;
|
||||
case 1:
|
||||
/* Decompress block with fixed Huffman trees */
|
||||
res = tinf_inflate_fixed_block(&d);
|
||||
break;
|
||||
case 2:
|
||||
/* Decompress block with dynamic Huffman trees */
|
||||
res = tinf_inflate_dynamic_block(&d);
|
||||
break;
|
||||
default:
|
||||
res = TINF_DATA_ERROR;
|
||||
break;
|
||||
}
|
||||
|
||||
if (res != TINF_OK) {
|
||||
return res;
|
||||
}
|
||||
} while (!bfinal);
|
||||
|
||||
/* Check for overflow in bit reader */
|
||||
if (d.overflow) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
/* d.source includes the byte containing any legal final padding bits, but not a following whole byte. */
|
||||
if (require_exact_source && d.source != d.source_end) {
|
||||
return TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
*destLen = d.dest - d.dest_start;
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
int tinf_uncompress(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen)
|
||||
{
|
||||
return tinf_uncompress_internal(dest, destLen, source, sourceLen, 0);
|
||||
}
|
||||
|
||||
int tinf_uncompress_exact(void *dest, unsigned int *destLen,
|
||||
const void *source, unsigned int sourceLen)
|
||||
{
|
||||
return tinf_uncompress_internal(dest, destLen, source, sourceLen, 1);
|
||||
}
|
||||
|
||||
/* clang -g -O1 -fsanitize=fuzzer,address -DTINF_FUZZING tinflate.c */
|
||||
#if defined(TINF_FUZZING)
|
||||
#include <limits.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
unsigned char depacked[64 * 1024];
|
||||
|
||||
extern int
|
||||
LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
|
||||
{
|
||||
if (size > UINT_MAX / 2) { return 0; }
|
||||
unsigned int destLen = sizeof(depacked);
|
||||
tinf_uncompress(depacked, &destLen, data, size);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MESHCORE_TINF_IMPLEMENTATION */
|
||||
@@ -1,4 +1,5 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
@@ -16,6 +17,7 @@
|
||||
#include "helpers/ota/OtaStore.h"
|
||||
#include "helpers/ota/OtaProtocol.h"
|
||||
#include "helpers/ota/OtaManager.h"
|
||||
#include "helpers/ota/OtaDeflate.h"
|
||||
#include "helpers/ota/OtaApply.h"
|
||||
#include "helpers/ota/OtaBlInfo.h"
|
||||
#include "helpers/ota/OtaFlashLayout_nrf52.h"
|
||||
@@ -773,6 +775,71 @@ private:
|
||||
size_t response_pos = 0;
|
||||
};
|
||||
|
||||
class FakeDeflateMotaSeederStream : public Stream {
|
||||
public:
|
||||
using Stream::write;
|
||||
|
||||
FakeDeflateMotaSeederStream() {
|
||||
encoded.resize(421);
|
||||
for (size_t i = 0; i < encoded.size(); i++) encoded[i] = (uint8_t)(i * 17u + 3u);
|
||||
}
|
||||
|
||||
size_t write(const uint8_t* request, size_t len) override {
|
||||
request_valid = len == 11 && request[0] == MOTA_SEEDER_REQ_MAGIC0
|
||||
&& request[1] == MOTA_SEEDER_REQ_MAGIC1 && request[2] == MS_OP_DEFLATE_BLOCK;
|
||||
uint8_t checksum = request_valid ? request[2] : 0;
|
||||
for (size_t i = 3; request_valid && i + 1 < len; i++) checksum ^= request[i];
|
||||
request_valid = request_valid && checksum == request[len - 1];
|
||||
if (!request_valid) return len;
|
||||
|
||||
const uint16_t block = rd_u16le(request + 4);
|
||||
const uint16_t offset = rd_u16le(request + 6);
|
||||
const uint16_t read_len = rd_u16le(request + 8);
|
||||
blocks.push_back(block);
|
||||
offsets.push_back(offset);
|
||||
lengths.push_back(read_len);
|
||||
|
||||
response.clear();
|
||||
response_pos = 0;
|
||||
response.push_back(MOTA_SEEDER_RSP_MAGIC0);
|
||||
response.push_back(MOTA_SEEDER_RSP_MAGIC1);
|
||||
response.push_back(MS_OP_DEFLATE_BLOCK);
|
||||
const bool valid_range = offset <= encoded.size() && read_len <= encoded.size() - offset;
|
||||
const uint8_t status = reject || !valid_range ? MS_STATUS_ERR : MS_STATUS_OK;
|
||||
response.push_back(status);
|
||||
if (status == MS_STATUS_OK) {
|
||||
response.push_back((uint8_t)(encoded.size() & 0xFF));
|
||||
response.push_back((uint8_t)(encoded.size() >> 8));
|
||||
response.insert(response.end(), encoded.begin() + offset,
|
||||
encoded.begin() + offset + read_len);
|
||||
}
|
||||
uint8_t response_checksum = 0;
|
||||
for (uint8_t byte : response) response_checksum ^= byte;
|
||||
response.push_back(response_checksum);
|
||||
return len;
|
||||
}
|
||||
|
||||
int read() override {
|
||||
if (response_pos < response.size()) return response[response_pos++];
|
||||
++g_mock_millis;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void flush() override { ++flush_calls; }
|
||||
|
||||
std::vector<uint8_t> encoded;
|
||||
std::vector<uint16_t> blocks;
|
||||
std::vector<uint16_t> offsets;
|
||||
std::vector<uint16_t> lengths;
|
||||
bool request_valid = true;
|
||||
bool reject = false;
|
||||
size_t flush_calls = 0;
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> response;
|
||||
size_t response_pos = 0;
|
||||
};
|
||||
|
||||
static_assert(!std::is_constructible<SerialMotaSource, Stream&>::value,
|
||||
"mOTA source callers must choose a stream write policy");
|
||||
static_assert(!std::is_constructible<SerialMotaSource, Stream&, uint32_t>::value,
|
||||
@@ -837,6 +904,37 @@ TEST(MotaSourceSerial, WrongFlushPolicyDemonstratesNetworkReplyLoss) {
|
||||
EXPECT_EQ(stream.flush_calls, 1U);
|
||||
}
|
||||
|
||||
TEST(MotaSourceSerial, ReadsIndependentDeflateBlockInBoundedChunks) {
|
||||
resetArduinoMock();
|
||||
FakeDeflateMotaSeederStream stream;
|
||||
SerialMotaSource source(stream, MotaStreamWritePolicy::FlushTransmit);
|
||||
std::array<uint8_t, 1024> data{};
|
||||
uint16_t encoded_len = 0;
|
||||
|
||||
ASSERT_TRUE(source.read_deflated_block(2, 37, data.data(), data.size(), &encoded_len));
|
||||
EXPECT_TRUE(stream.request_valid);
|
||||
EXPECT_EQ(encoded_len, stream.encoded.size());
|
||||
EXPECT_EQ(stream.blocks, (std::vector<uint16_t>{37, 37, 37, 37}));
|
||||
EXPECT_EQ(stream.offsets, (std::vector<uint16_t>{0, 0, 190, 380}));
|
||||
EXPECT_EQ(stream.lengths, (std::vector<uint16_t>{0, 190, 190, 41}));
|
||||
EXPECT_EQ(stream.flush_calls, 4u);
|
||||
EXPECT_EQ(0, memcmp(data.data(), stream.encoded.data(), encoded_len));
|
||||
}
|
||||
|
||||
TEST(MotaSourceSerial, DeflateExtensionFailureLeavesRawFallbackAvailable) {
|
||||
resetArduinoMock();
|
||||
FakeDeflateMotaSeederStream stream;
|
||||
stream.reject = true; // deployed host does not implement operation 0x09
|
||||
SerialMotaSource source(stream, MotaStreamWritePolicy::FlushTransmit);
|
||||
std::array<uint8_t, 1024> data{};
|
||||
uint16_t encoded_len = 999;
|
||||
|
||||
EXPECT_FALSE(source.read_deflated_block(0, 0, data.data(), data.size(), &encoded_len));
|
||||
EXPECT_EQ(encoded_len, 0u);
|
||||
EXPECT_EQ(stream.offsets, (std::vector<uint16_t>{0}));
|
||||
EXPECT_EQ(stream.lengths, (std::vector<uint16_t>{0}));
|
||||
}
|
||||
|
||||
class FakeFolderMotaSeederStream : public Stream {
|
||||
public:
|
||||
using Stream::write;
|
||||
@@ -1329,6 +1427,104 @@ TEST(OtaStoreDiscardTest, ConservativeDefaultCannotClaimDurableInvalidation) {
|
||||
EXPECT_TRUE(store.cleared);
|
||||
}
|
||||
|
||||
TEST(OtaDeflate, AcceptsStoredFixedDynamicAndMultiBlockRawStreams) {
|
||||
static const uint8_t stored[] = {
|
||||
0x01,0x40,0x00,0xBF,0xFF,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,
|
||||
0x0A,0x0B,0x0C,0x0D,0x0E,0x0F,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
|
||||
0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,
|
||||
0x28,0x29,0x2A,0x2B,0x2C,0x2D,0x2E,0x2F,0x30,0x31,0x32,0x33,0x34,0x35,0x36,
|
||||
0x37,0x38,0x39,0x3A,0x3B,0x3C,0x3D,0x3E,0x3F,
|
||||
};
|
||||
static const uint8_t fixed[] = {
|
||||
0xF3,0x4D,0x2D,0xCE,0x70,0xCE,0x2F,0x4A,0x55,0x28,0x29,0x4A,0xCC,0x2B,0x2E,0xC8,
|
||||
0x2F,0x2A,0x51,0x48,0xCB,0xAC,0x48,0x4D,0x51,0xF0,0x28,0x4D,0x4B,0xCB,0x4D,0xCC,
|
||||
0x53,0x28,0x49,0x2D,0x2E,0xD1,0x53,0xF0,0x1D,0x55,0x46,0x1B,0x65,0x00,
|
||||
};
|
||||
static const uint8_t dynamic[] = {
|
||||
0xED,0xCF,0xD9,0x0D,0x83,0x30,0x14,0x44,0xD1,0x0B,0xC1,0x2C,0xC1,0x2C,0xC1,0x2C,
|
||||
0xB1,0x81,0xF4,0x5F,0x66,0x9E,0x26,0x05,0xA4,0x01,0x7F,0x5F,0x1D,0x69,0x86,0xA2,
|
||||
0x7C,0x54,0xAE,0xA6,0xED,0x9E,0xBD,0x1F,0x98,0xE6,0xD7,0x12,0x56,0xF6,0xE3,0x1D,
|
||||
0xD3,0xC9,0xFD,0xC1,0x32,0xAE,0x6E,0x2C,0xE3,0x87,0xD1,0x32,0x61,0xDD,0x2C,0x93,
|
||||
0xCE,0xCB,0x32,0xD2,0x0D,0xD2,0x23,0xD2,0x1B,0xD2,0x17,0xD2,0x15,0xD2,0x3D,0xD2,
|
||||
0x0B,0xD2,0x11,0xE9,0x02,0xE9,0x16,0xE9,0x09,0xE9,0x1D,0xE9,0x1B,0x69,0x87,0xB4,
|
||||
0x47,0x3A,0x20,0x9D,0x90,0x2E,0x91,0xEE,0x90,0x9E,0x91,0x3E,0x88,0xBF,0x69,0xF9,
|
||||
0x58,0x3E,0x96,0x8F,0xFD,0x3B,0xF6,0x05,
|
||||
};
|
||||
static const uint8_t multi_block[] = {
|
||||
0x4A,0xCB,0x2C,0x2A,0x2E,0xD1,0x4D,0xCA,0xC9,0x4F,0xCE,0xD6,0x4D,0x1B,0x66,0x6C,
|
||||
0x00,0x00,0x00,0x00,0xFF,0xFF,0x2B,0x4E,0x4D,0xCE,0xCF,0x4B,0x81,0x0A,0x8C,0x72,
|
||||
0xF0,0x72,0x00,
|
||||
};
|
||||
|
||||
EXPECT_EQ((stored[0] >> 1) & 3u, 0u);
|
||||
EXPECT_EQ((fixed[0] >> 1) & 3u, 1u);
|
||||
EXPECT_EQ((dynamic[0] >> 1) & 3u, 2u);
|
||||
|
||||
std::array<uint8_t, 1024> output{};
|
||||
uint16_t output_len = 0;
|
||||
ASSERT_TRUE(ota_transport_inflate(nullptr, stored, sizeof(stored), output.data(), 64, &output_len));
|
||||
ASSERT_EQ(output_len, 64u);
|
||||
for (uint16_t i = 0; i < output_len; i++) EXPECT_EQ(output[i], (uint8_t)i);
|
||||
|
||||
static const char fixed_phrase[] = "MeshCore transport fixed Huffman test. ";
|
||||
ASSERT_TRUE(ota_transport_inflate(nullptr, fixed, sizeof(fixed), output.data(), 384, &output_len));
|
||||
ASSERT_EQ(output_len, 384u);
|
||||
for (uint16_t i = 0; i < output_len; i++)
|
||||
EXPECT_EQ(output[i], (uint8_t)fixed_phrase[i % (sizeof(fixed_phrase) - 1)]);
|
||||
|
||||
ASSERT_TRUE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic), output.data(), 1024, &output_len));
|
||||
ASSERT_EQ(output_len, 1024u);
|
||||
for (uint16_t i = 0; i < output_len; i++)
|
||||
EXPECT_EQ(output[i], (uint8_t)((i % 7u) ? i % 31u : 0u));
|
||||
|
||||
ASSERT_TRUE(ota_transport_inflate(nullptr, multi_block, sizeof(multi_block), output.data(), 520,
|
||||
&output_len));
|
||||
ASSERT_EQ(output_len, 520u);
|
||||
static const char first_phrase[] = "first-block-";
|
||||
static const char second_phrase[] = "second-block-";
|
||||
for (uint16_t i = 0; i < 220; i++)
|
||||
EXPECT_EQ(output[i], (uint8_t)first_phrase[i % (sizeof(first_phrase) - 1)]);
|
||||
for (uint16_t i = 220; i < output_len; i++)
|
||||
EXPECT_EQ(output[i], (uint8_t)second_phrase[(i - 220) % (sizeof(second_phrase) - 1)]);
|
||||
}
|
||||
|
||||
TEST(OtaDeflate, RejectsTruncationBoundsWrongLengthTrailingBytesAndMalformedTrees) {
|
||||
static const uint8_t dynamic[] = {
|
||||
0xED,0xCF,0xD9,0x0D,0x83,0x30,0x14,0x44,0xD1,0x0B,0xC1,0x2C,0xC1,0x2C,0xC1,0x2C,
|
||||
0xB1,0x81,0xF4,0x5F,0x66,0x9E,0x26,0x05,0xA4,0x01,0x7F,0x5F,0x1D,0x69,0x86,0xA2,
|
||||
0x7C,0x54,0xAE,0xA6,0xED,0x9E,0xBD,0x1F,0x98,0xE6,0xD7,0x12,0x56,0xF6,0xE3,0x1D,
|
||||
0xD3,0xC9,0xFD,0xC1,0x32,0xAE,0x6E,0x2C,0xE3,0x87,0xD1,0x32,0x61,0xDD,0x2C,0x93,
|
||||
0xCE,0xCB,0x32,0xD2,0x0D,0xD2,0x23,0xD2,0x1B,0xD2,0x17,0xD2,0x15,0xD2,0x3D,0xD2,
|
||||
0x0B,0xD2,0x11,0xE9,0x02,0xE9,0x16,0xE9,0x09,0xE9,0x1D,0xE9,0x1B,0x69,0x87,0xB4,
|
||||
0x47,0x3A,0x20,0x9D,0x90,0x2E,0x91,0xEE,0x90,0x9E,0x91,0x3E,0x88,0xBF,0x69,0xF9,
|
||||
0x58,0x3E,0x96,0x8F,0xFD,0x3B,0xF6,0x05,
|
||||
};
|
||||
static const uint8_t invalid_distance[] = {0x63,0x00,0x3E,0x00};
|
||||
static const uint8_t invalid_tree[] = {
|
||||
0x0D,0xCE,0x81,0x00,0x00,0x00,0x00,0x80,0xA0,0xFD,0xA9,0xBB,0x1F,0xA0,0x01,
|
||||
};
|
||||
std::array<uint8_t, 1025> output{};
|
||||
uint16_t output_len = 999;
|
||||
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic) - 1,
|
||||
output.data(), 1024, &output_len));
|
||||
EXPECT_EQ(output_len, 0u);
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic),
|
||||
output.data(), 1023, &output_len));
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic),
|
||||
output.data(), 1025, &output_len));
|
||||
|
||||
std::array<uint8_t, sizeof(dynamic) + 1> trailing{};
|
||||
memcpy(trailing.data(), dynamic, sizeof(dynamic));
|
||||
trailing.back() = 0x00;
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, trailing.data(), trailing.size(),
|
||||
output.data(), 1024, &output_len));
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, invalid_distance, sizeof(invalid_distance),
|
||||
output.data(), 4, &output_len));
|
||||
EXPECT_FALSE(ota_transport_inflate(nullptr, invalid_tree, sizeof(invalid_tree),
|
||||
output.data(), 4, &output_len));
|
||||
}
|
||||
|
||||
// --- merkle proof GENERATION (server side) matches the Python oracle ---------------------------
|
||||
|
||||
TEST(OtaMerkle, GenProofMatchesPythonAndVerifies) {
|
||||
@@ -1484,6 +1680,40 @@ TEST(OtaProtocol, CodecRoundTrips) {
|
||||
EXPECT_EQ(0, memcmp(pm2.proof, proof, 12));
|
||||
}
|
||||
|
||||
TEST(OtaProtocol, V2ProfilePacks171ByteFragmentsInsideExistingMessages) {
|
||||
const uint16_t request = ota_req_make_v2(0x007Fu, true);
|
||||
EXPECT_TRUE(ota_req_is_v2(request));
|
||||
EXPECT_EQ(ota_req_v2_fragments(request), 0x007Fu);
|
||||
EXPECT_NE(request & OTA_REQ_V2_ALLOW_DEFLATE, 0u);
|
||||
EXPECT_FALSE(ota_req_is_v2((uint16_t)(request | OTA_REQ_V2_RESERVED)));
|
||||
EXPECT_FALSE(ota_req_is_v2(0x007Fu));
|
||||
|
||||
uint16_t descriptor = 0;
|
||||
ASSERT_TRUE(ota_data_v2_pack(5, 1024, true, descriptor));
|
||||
uint8_t fragment = 0;
|
||||
uint16_t encoded_len = 0;
|
||||
bool deflated = false;
|
||||
ASSERT_TRUE(ota_data_v2_unpack(descriptor, fragment, encoded_len, deflated));
|
||||
EXPECT_EQ(fragment, 5u);
|
||||
EXPECT_EQ(encoded_len, 1024u);
|
||||
EXPECT_TRUE(deflated);
|
||||
EXPECT_FALSE(ota_data_v2_pack(16, 1024, false, descriptor));
|
||||
EXPECT_FALSE(ota_data_v2_pack(0, 0, false, descriptor));
|
||||
EXPECT_FALSE(ota_data_v2_pack(0, 1025, false, descriptor));
|
||||
|
||||
std::array<uint8_t, OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2> payload{};
|
||||
ASSERT_TRUE(ota_data_v2_pack(0, 1024, false, descriptor));
|
||||
DataMsg message{{1, 2, 3, 4}, 9, descriptor, payload.data(),
|
||||
(uint16_t)payload.size()};
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
const uint16_t wire_len = encode_data(wire, sizeof(wire), message);
|
||||
EXPECT_EQ(wire_len, MAX_PACKET_PAYLOAD);
|
||||
DataMsg decoded{};
|
||||
ASSERT_TRUE(decode_data(wire, wire_len, decoded));
|
||||
EXPECT_EQ(decoded.data_len,
|
||||
(uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2));
|
||||
}
|
||||
|
||||
// --- full transfer simulation between two OtaManagers (P4b) ------------------------------------
|
||||
|
||||
namespace {
|
||||
@@ -1510,6 +1740,60 @@ static bool gated_capture_send(void* ctx, const uint8_t* msg, uint16_t len, bool
|
||||
return true;
|
||||
}
|
||||
|
||||
// Tiny reversible stand-in for the separately tested RFC1951 implementation. These callbacks isolate the
|
||||
// OtaManager negotiation/reassembly path: each payload block is represented by its authenticated block index,
|
||||
// then expanded from the immutable test manifest before the normal Merkle proof is checked.
|
||||
struct TestWireCodec {
|
||||
const MotaManifest* manifest = nullptr;
|
||||
uint32_t calls = 0;
|
||||
};
|
||||
|
||||
static bool test_wire_encode(void* context, const uint8_t* source, uint16_t source_len,
|
||||
uint8_t* destination, uint16_t capacity, uint16_t* encoded_len) {
|
||||
TestWireCodec* codec = (TestWireCodec*)context;
|
||||
if (!codec || !codec->manifest || !source || !destination || !encoded_len || capacity < 2) return false;
|
||||
const MotaManifest& manifest = *codec->manifest;
|
||||
const uint32_t block_size = manifest.block_size();
|
||||
for (uint32_t block = 0; block < manifest.block_count && block <= 0xFFu; block++) {
|
||||
const uint32_t offset = block * block_size;
|
||||
const uint32_t length = std::min<uint32_t>(block_size, manifest.payload_size - offset);
|
||||
if (length == source_len && memcmp(source, manifest.payload + offset, length) == 0) {
|
||||
destination[0] = 0xD1;
|
||||
destination[1] = (uint8_t)block;
|
||||
*encoded_len = 2;
|
||||
codec->calls++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool test_wire_decode(void* context, const uint8_t* source, uint16_t source_len,
|
||||
uint8_t* destination, uint16_t capacity, uint16_t* decoded_len) {
|
||||
TestWireCodec* codec = (TestWireCodec*)context;
|
||||
if (!codec || !codec->manifest || !source || !destination || !decoded_len || source_len != 2 ||
|
||||
source[0] != 0xD1 || source[1] >= codec->manifest->block_count) return false;
|
||||
const MotaManifest& manifest = *codec->manifest;
|
||||
const uint32_t offset = (uint32_t)source[1] * manifest.block_size();
|
||||
const uint32_t length = std::min<uint32_t>(manifest.block_size(), manifest.payload_size - offset);
|
||||
if (length != capacity) return false;
|
||||
memcpy(destination, manifest.payload + offset, length);
|
||||
*decoded_len = (uint16_t)length;
|
||||
codec->calls++;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool test_representation_decode(void* context, const uint8_t* source, uint16_t source_len,
|
||||
uint8_t* destination, uint16_t capacity,
|
||||
uint16_t* decoded_len) {
|
||||
const MotaManifest* manifest = (const MotaManifest*)context;
|
||||
if (!manifest || !source || !destination || !decoded_len || source_len != 300 ||
|
||||
(source[0] != 0xA1 && source[0] != 0xB2) || capacity != manifest->block_size()) return false;
|
||||
memcpy(destination, manifest->payload, capacity);
|
||||
*decoded_len = capacity;
|
||||
return true;
|
||||
}
|
||||
|
||||
TEST(OtaMetrics, CountsOnlyPacketsAcceptedByTheRadioAdapter) {
|
||||
OtaManager manager;
|
||||
GatedCapture sent;
|
||||
@@ -1585,6 +1869,8 @@ private:
|
||||
class RamMotaSource : public mesh::ota::MotaSource {
|
||||
public:
|
||||
void add(const uint8_t* buf, uint32_t len) { if (_n < 8) { _buf[_n] = buf; _len[_n] = len; _n++; } }
|
||||
void setSourceCaps(uint8_t caps) { _source_caps = caps; }
|
||||
uint32_t deflateCalls() const { return _deflate_calls; }
|
||||
uint8_t count() override { return _n; }
|
||||
bool describe(uint8_t idx, mesh::ota::MotaDesc& d) override {
|
||||
if (idx >= _n) return false;
|
||||
@@ -1594,6 +1880,7 @@ public:
|
||||
d.target_id = m.target_id; d.fw_version = m.fw_version;
|
||||
d.codec_id = m.codec_id; d.flags = m.flags;
|
||||
d.block_size_log2 = m.block_size_log2;
|
||||
d.source_caps = _source_caps;
|
||||
d.total_size = _len[idx];
|
||||
d.leaves_off = (uint32_t)(m.leaves - _buf[idx]);
|
||||
d.block_count = m.block_count;
|
||||
@@ -1606,8 +1893,15 @@ public:
|
||||
std::memcpy(out, _buf[idx] + off, len);
|
||||
return true;
|
||||
}
|
||||
bool read_deflated_block(uint8_t, uint16_t, uint8_t*, uint16_t, uint16_t* len) override {
|
||||
_deflate_calls++;
|
||||
if (len) *len = 0;
|
||||
return false;
|
||||
}
|
||||
private:
|
||||
const uint8_t* _buf[8] = {nullptr}; uint32_t _len[8] = {0}; uint8_t _n = 0;
|
||||
uint8_t _source_caps = 0;
|
||||
uint32_t _deflate_calls = 0;
|
||||
};
|
||||
|
||||
class SyntheticCatalogSource : public mesh::ota::MotaSource {
|
||||
@@ -1706,6 +2000,36 @@ TEST(OtaTransfer, TwoManagersFullTransfer) {
|
||||
EXPECT_TRUE(mota_check_image_hash_full(m));
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, TransportCodecInflatesBeforeProofAndStagesOriginalContainer) {
|
||||
g_q.clear();
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
OtaManager server, client;
|
||||
OtaStoreRam<4096> store;
|
||||
SendTo to_client{&client}, to_server{&server};
|
||||
TestWireCodec encoder;
|
||||
TestWireCodec decoder;
|
||||
encoder.manifest = &manifest;
|
||||
decoder.manifest = &manifest;
|
||||
|
||||
server.begin(0, sim_send, &to_client);
|
||||
server.set_transport_deflate_encoder(test_wire_encode, &encoder);
|
||||
client.begin(SIM_TARGET_ID, sim_send, &to_server);
|
||||
client.set_transport_deflate_decoder(test_wire_decode, &decoder);
|
||||
client.set_fetch_store(&store);
|
||||
client.set_autofetch(OtaManager::AUTOFETCH_ANY);
|
||||
ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN));
|
||||
|
||||
server.announce();
|
||||
pump(client, &server);
|
||||
|
||||
EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE);
|
||||
EXPECT_GT(encoder.calls, 0u);
|
||||
EXPECT_EQ(decoder.calls, manifest.block_count);
|
||||
ASSERT_EQ(store.staged_size(), SIM_MOTA_1K_LEN);
|
||||
EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_1K, SIM_MOTA_1K_LEN));
|
||||
}
|
||||
|
||||
static void deliver_manifest_fragment(OtaManager& client, const uint8_t mid[4], uint8_t frag_idx,
|
||||
const uint8_t* bytes, uint16_t len) {
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
@@ -1758,6 +2082,48 @@ static void deliver_verified_block(OtaManager& client, const MotaManifest& manif
|
||||
ASSERT_TRUE(client.on_message(wire, wire_len));
|
||||
}
|
||||
|
||||
static void deliver_verified_v2_raw_block(OtaManager& client, const MotaManifest& manifest,
|
||||
uint32_t block) {
|
||||
ASSERT_LT(block, manifest.block_count);
|
||||
const uint32_t block_size = manifest.block_size();
|
||||
const uint32_t block_off = block * block_size;
|
||||
const uint16_t block_len = (uint16_t)(block_off + block_size <= manifest.payload_size
|
||||
? block_size : manifest.payload_size - block_off);
|
||||
const uint32_t fragments = (block_len + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2;
|
||||
uint8_t stream_id[OTA_DATA_V2_STREAM_ID_BYTES];
|
||||
mh4(stream_id, manifest.payload + block_off, block_len);
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
for (uint32_t fragment = 0; fragment < fragments; fragment++) {
|
||||
const uint32_t offset = fragment * OTA_FRAG_DATA_V2;
|
||||
uint16_t length = (uint16_t)(block_len - offset);
|
||||
if (length > OTA_FRAG_DATA_V2) length = OTA_FRAG_DATA_V2;
|
||||
uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2];
|
||||
memcpy(body, stream_id, sizeof(stream_id));
|
||||
memcpy(body + sizeof(stream_id), manifest.payload + block_off + offset, length);
|
||||
DataMsg data{};
|
||||
memcpy(data.manifest_id, manifest.merkle_root, 4);
|
||||
data.block_idx = (uint16_t)block;
|
||||
ASSERT_TRUE(ota_data_v2_pack((uint8_t)fragment, block_len, false, data.frag_off));
|
||||
data.data = body;
|
||||
data.data_len = (uint16_t)(sizeof(stream_id) + length);
|
||||
const uint16_t wire_len = encode_data(wire, sizeof(wire), data);
|
||||
ASSERT_GT(wire_len, 0);
|
||||
ASSERT_TRUE(client.on_message(wire, wire_len));
|
||||
}
|
||||
std::vector<uint8_t> scratch(manifest.block_count * 4);
|
||||
uint8_t siblings[32 * 4];
|
||||
const uint8_t sibling_count = merkle_gen_proof(
|
||||
manifest.leaves, manifest.block_count, block, scratch.data(), siblings);
|
||||
ProofMsg proof{};
|
||||
memcpy(proof.manifest_id, manifest.merkle_root, 4);
|
||||
proof.block_idx = (uint16_t)block;
|
||||
proof.n_proof = sibling_count;
|
||||
proof.proof = siblings;
|
||||
const uint16_t wire_len = encode_proof(wire, sizeof(wire), proof);
|
||||
ASSERT_GT(wire_len, 0);
|
||||
ASSERT_TRUE(client.on_message(wire, wire_len));
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
@@ -1811,6 +2177,58 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure)
|
||||
EXPECT_EQ(server.pendingServeJobs(), 0u);
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, NegotiatedV2Carries171RawBytesAndRepeatsRepresentationId) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
ASSERT_EQ(manifest.block_size(), 1024u);
|
||||
|
||||
OtaManager server;
|
||||
CapturedMessages sent;
|
||||
server.begin(0, capture_send, &sent);
|
||||
ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN));
|
||||
|
||||
ReqMsg request;
|
||||
memcpy(request.manifest_id, manifest.merkle_root, 4);
|
||||
request.block_idx = 0;
|
||||
request.want_mask = ota_req_make_v2(0x007Fu, true); // no encoder installed: source must fall back to raw
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
const uint16_t request_len = encode_req(wire, sizeof(wire), request);
|
||||
ASSERT_GT(request_len, 0);
|
||||
ASSERT_TRUE(server.on_message(wire, request_len));
|
||||
|
||||
const uint32_t fragment_count =
|
||||
(manifest.block_size() + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2;
|
||||
ASSERT_EQ(fragment_count, 6u);
|
||||
std::array<uint8_t, 1024> reassembled{};
|
||||
uint8_t expected_stream_id[4];
|
||||
mh4(expected_stream_id, manifest.payload, manifest.block_size());
|
||||
|
||||
for (uint32_t i = 0; i < fragment_count; i++) {
|
||||
server.serviceEgress();
|
||||
ASSERT_EQ(sent.items.size(), i + 1);
|
||||
DataMsg data{};
|
||||
ASSERT_TRUE(decode_data(sent.items.back().data(),
|
||||
(uint16_t)sent.items.back().size(), data));
|
||||
uint8_t fragment = 0;
|
||||
uint16_t encoded_len = 0;
|
||||
bool deflated = true;
|
||||
ASSERT_TRUE(ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated));
|
||||
EXPECT_EQ(fragment, i);
|
||||
EXPECT_EQ(encoded_len, manifest.block_size());
|
||||
EXPECT_FALSE(deflated);
|
||||
ASSERT_GE(data.data_len, OTA_DATA_V2_STREAM_ID_BYTES);
|
||||
EXPECT_EQ(0, memcmp(data.data, expected_stream_id, sizeof(expected_stream_id)));
|
||||
const uint16_t slice_len =
|
||||
(uint16_t)(data.data_len - OTA_DATA_V2_STREAM_ID_BYTES);
|
||||
const uint32_t offset = i * OTA_FRAG_DATA_V2;
|
||||
EXPECT_EQ(slice_len, (uint16_t)std::min<uint32_t>(OTA_FRAG_DATA_V2,
|
||||
manifest.block_size() - offset));
|
||||
if (i + 1 < fragment_count) EXPECT_EQ(sent.items.back().size(), MAX_PACKET_PAYLOAD);
|
||||
memcpy(reassembled.data() + offset, data.data + OTA_DATA_V2_STREAM_ID_BYTES, slice_len);
|
||||
}
|
||||
EXPECT_EQ(0, memcmp(reassembled.data(), manifest.payload, manifest.block_size()));
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, LegacyProofRequestBypassesProactiveProofTurnaround) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
@@ -2046,6 +2464,265 @@ TEST(OtaTransfer, ClientUsesQuietBatchedFlightsAndAcceptsOutOfOrderBlocks) {
|
||||
EXPECT_EQ(sent.items.size(), 2u); // no continuous-refill request was emitted
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, NewClientRequestsV2ThenRetriesLegacyWhenNoV2DataArrives) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
|
||||
OtaManager client;
|
||||
OtaStoreRam<4096> store;
|
||||
CapturedMessages sent;
|
||||
client.begin(SIM_TARGET_ID, capture_send, &sent);
|
||||
client.set_fetch_store(&store);
|
||||
client.set_clock(100);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
sent.items.clear();
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 0,
|
||||
manifest.manifest_start, OTA_MF_FRAG);
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 1,
|
||||
manifest.manifest_start + OTA_MF_FRAG,
|
||||
(uint16_t)(MOTA_MFL - OTA_MF_FRAG));
|
||||
ASSERT_EQ(sent.items.size(), 1u);
|
||||
ReqMsg first{};
|
||||
ASSERT_TRUE(decode_req(sent.items[0].data(), (uint16_t)sent.items[0].size(), first));
|
||||
EXPECT_TRUE(ota_req_is_v2(first.want_mask));
|
||||
EXPECT_EQ(ota_req_v2_fragments(first.want_mask), 0x007Fu); // remains a complete old-source request
|
||||
EXPECT_EQ(first.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); // no decoder was linked/configured
|
||||
|
||||
const uint32_t timeout = client.fetchRetryTimeoutMs();
|
||||
client.set_clock(100 + timeout);
|
||||
client.loop();
|
||||
ASSERT_EQ(sent.items.size(), 2u);
|
||||
ReqMsg fallback{};
|
||||
ASSERT_TRUE(decode_req(sent.items[1].data(), (uint16_t)sent.items[1].size(), fallback));
|
||||
EXPECT_FALSE(ota_req_is_v2(fallback.want_mask));
|
||||
EXPECT_EQ(fallback.want_mask, 0x007Fu);
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, MalformedLegacyDataCannotForceV2SessionDowngrade) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
OtaManager client;
|
||||
OtaStoreRam<4096> store;
|
||||
CapturedMessages sent;
|
||||
client.begin(SIM_TARGET_ID, capture_send, &sent);
|
||||
client.set_fetch_store(&store);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
sent.items.clear();
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 0,
|
||||
manifest.manifest_start, OTA_MF_FRAG);
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 1,
|
||||
manifest.manifest_start + OTA_MF_FRAG,
|
||||
(uint16_t)(MOTA_MFL - OTA_MF_FRAG));
|
||||
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
DataMsg malformed;
|
||||
memcpy(malformed.manifest_id, manifest.merkle_root, 4);
|
||||
malformed.block_idx = 0;
|
||||
malformed.frag_off = 1; // not canonical 160-byte legacy alignment
|
||||
malformed.data = manifest.payload;
|
||||
malformed.data_len = OTA_FRAG_DATA;
|
||||
uint16_t wire_len = encode_data(wire, sizeof(wire), malformed);
|
||||
ASSERT_GT(wire_len, 0);
|
||||
EXPECT_FALSE(client.on_message(wire, wire_len));
|
||||
|
||||
uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2];
|
||||
mh4(body, manifest.payload, manifest.block_size());
|
||||
memcpy(body + OTA_DATA_V2_STREAM_ID_BYTES, manifest.payload, OTA_FRAG_DATA_V2);
|
||||
DataMsg valid;
|
||||
memcpy(valid.manifest_id, manifest.merkle_root, 4);
|
||||
valid.block_idx = 0;
|
||||
ASSERT_TRUE(ota_data_v2_pack(0, (uint16_t)manifest.block_size(), false, valid.frag_off));
|
||||
valid.data = body;
|
||||
valid.data_len = sizeof(body);
|
||||
wire_len = encode_data(wire, sizeof(wire), valid);
|
||||
ASSERT_EQ(wire_len, MAX_PACKET_PAYLOAD);
|
||||
EXPECT_TRUE(client.on_message(wire, wire_len)); // malformed legacy packet left v2 negotiation intact
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, PartialV2SeederCanDisappearAndLegacySeederTakesOver) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
OtaManager client;
|
||||
OtaStoreRam<4096> store;
|
||||
CapturedMessages sent;
|
||||
client.begin(SIM_TARGET_ID, capture_send, &sent);
|
||||
client.set_transport_deflate_decoder(ota_transport_inflate);
|
||||
client.set_fetch_store(&store);
|
||||
client.set_clock(100);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
sent.items.clear();
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 0,
|
||||
manifest.manifest_start, OTA_MF_FRAG);
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 1,
|
||||
manifest.manifest_start + OTA_MF_FRAG,
|
||||
(uint16_t)(MOTA_MFL - OTA_MF_FRAG));
|
||||
|
||||
uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2];
|
||||
mh4(body, manifest.payload, manifest.block_size());
|
||||
memcpy(body + OTA_DATA_V2_STREAM_ID_BYTES, manifest.payload, OTA_FRAG_DATA_V2);
|
||||
DataMsg partial{};
|
||||
memcpy(partial.manifest_id, manifest.merkle_root, 4);
|
||||
partial.block_idx = 0;
|
||||
ASSERT_TRUE(ota_data_v2_pack(0, (uint16_t)manifest.block_size(), false, partial.frag_off));
|
||||
partial.data = body;
|
||||
partial.data_len = sizeof(body);
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
uint16_t wire_len = encode_data(wire, sizeof(wire), partial);
|
||||
ASSERT_EQ(wire_len, MAX_PACKET_PAYLOAD);
|
||||
ASSERT_TRUE(client.on_message(wire, wire_len));
|
||||
|
||||
sent.items.clear();
|
||||
uint32_t now = 100;
|
||||
uint32_t timeout = client.fetchRetryTimeoutMs();
|
||||
now += timeout;
|
||||
client.set_clock(now);
|
||||
client.loop(); // one sparse v2 retry
|
||||
ASSERT_FALSE(sent.items.empty());
|
||||
ReqMsg retry{};
|
||||
ASSERT_TRUE(decode_req(sent.items.back().data(), (uint16_t)sent.items.back().size(), retry));
|
||||
EXPECT_TRUE(ota_req_is_v2(retry.want_mask));
|
||||
|
||||
sent.items.clear();
|
||||
timeout = client.fetchRetryTimeoutMs();
|
||||
now += timeout;
|
||||
client.set_clock(now);
|
||||
client.loop(); // bounded whole-session legacy fallback
|
||||
ASSERT_FALSE(sent.items.empty());
|
||||
ReqMsg fallback{};
|
||||
ASSERT_TRUE(decode_req(sent.items.back().data(), (uint16_t)sent.items.back().size(), fallback));
|
||||
EXPECT_FALSE(ota_req_is_v2(fallback.want_mask));
|
||||
EXPECT_EQ(fallback.want_mask, 0x007Fu);
|
||||
|
||||
deliver_verified_block(client, manifest, 0); // an old seeder can now make forward progress
|
||||
EXPECT_EQ(client.blocksHave(), 1u);
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, ConfirmedV2SeederCanDisappearBetweenBlocksAndLegacySeederTakesOver) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
ASSERT_GE(manifest.block_count, 2u);
|
||||
OtaManager client;
|
||||
OtaStoreRam<4096> store;
|
||||
CapturedMessages sent;
|
||||
client.begin(SIM_TARGET_ID, capture_send, &sent);
|
||||
client.set_fetch_store(&store);
|
||||
client.set_clock(100);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
sent.items.clear();
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 0,
|
||||
manifest.manifest_start, OTA_MF_FRAG);
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 1,
|
||||
manifest.manifest_start + OTA_MF_FRAG,
|
||||
(uint16_t)(MOTA_MFL - OTA_MF_FRAG));
|
||||
|
||||
deliver_verified_v2_raw_block(client, manifest, 0);
|
||||
ASSERT_EQ(client.blocksHave(), 1u);
|
||||
sent.items.clear(); // v2 source vanishes before the next flight answers
|
||||
|
||||
uint32_t now = 100;
|
||||
uint32_t timeout = client.fetchRetryTimeoutMs();
|
||||
now += timeout;
|
||||
client.set_clock(now);
|
||||
client.loop(); // one sparse v2 retry
|
||||
ASSERT_EQ(sent.items.size(), 1u);
|
||||
ReqWindowMsg retry{};
|
||||
ASSERT_TRUE(decode_req_window(sent.items.back().data(),
|
||||
(uint16_t)sent.items.back().size(), retry));
|
||||
ASSERT_EQ(retry.n_items, 1u);
|
||||
EXPECT_TRUE(ota_req_is_v2(retry.items[0].want_mask));
|
||||
|
||||
sent.items.clear();
|
||||
timeout = client.fetchRetryTimeoutMs();
|
||||
now += timeout;
|
||||
client.set_clock(now);
|
||||
client.loop(); // whole empty flight falls back to legacy
|
||||
ASSERT_EQ(sent.items.size(), 1u);
|
||||
ReqWindowMsg fallback{};
|
||||
ASSERT_TRUE(decode_req_window(sent.items.back().data(),
|
||||
(uint16_t)sent.items.back().size(), fallback));
|
||||
ASSERT_GE(fallback.n_items, 1u);
|
||||
for (uint8_t i = 0; i < fallback.n_items; i++) {
|
||||
EXPECT_FALSE(ota_req_is_v2(fallback.items[i].want_mask));
|
||||
}
|
||||
|
||||
deliver_verified_block(client, manifest, 1); // deployed source resumes the same session
|
||||
EXPECT_EQ(client.blocksHave(), 2u);
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, V2NeverMixesDifferentCompressedRepresentationsAcrossSeeders) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
|
||||
OtaManager client;
|
||||
OtaStoreRam<4096> store;
|
||||
CapturedMessages sent;
|
||||
client.begin(SIM_TARGET_ID, capture_send, &sent);
|
||||
client.set_transport_deflate_decoder(test_representation_decode, &manifest);
|
||||
client.set_fetch_store(&store);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
sent.items.clear();
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 0,
|
||||
manifest.manifest_start, OTA_MF_FRAG);
|
||||
deliver_manifest_fragment(client, manifest.merkle_root, 1,
|
||||
manifest.manifest_start + OTA_MF_FRAG,
|
||||
(uint16_t)(MOTA_MFL - OTA_MF_FRAG));
|
||||
ASSERT_EQ(sent.items.size(), 1u);
|
||||
ReqMsg request{};
|
||||
ASSERT_TRUE(decode_req(sent.items[0].data(), (uint16_t)sent.items[0].size(), request));
|
||||
EXPECT_NE(request.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u);
|
||||
|
||||
std::array<uint8_t, 300> representation_a{};
|
||||
std::array<uint8_t, 300> representation_b{};
|
||||
representation_a.fill(0xA1);
|
||||
representation_b.fill(0xB2);
|
||||
uint8_t id_a[4], id_b[4];
|
||||
mh4(id_a, representation_a.data(), representation_a.size());
|
||||
mh4(id_b, representation_b.data(), representation_b.size());
|
||||
ASSERT_NE(0, memcmp(id_a, id_b, sizeof(id_a)));
|
||||
|
||||
auto deliver_fragment = [&](const std::array<uint8_t, 300>& representation,
|
||||
const uint8_t id[4], uint8_t fragment) {
|
||||
const uint32_t offset = (uint32_t)fragment * OTA_FRAG_DATA_V2;
|
||||
const uint16_t length = (uint16_t)std::min<uint32_t>(OTA_FRAG_DATA_V2,
|
||||
representation.size() - offset);
|
||||
std::array<uint8_t, OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2> body{};
|
||||
memcpy(body.data(), id, OTA_DATA_V2_STREAM_ID_BYTES);
|
||||
memcpy(body.data() + OTA_DATA_V2_STREAM_ID_BYTES, representation.data() + offset, length);
|
||||
uint16_t descriptor = 0;
|
||||
if (!ota_data_v2_pack(fragment, (uint16_t)representation.size(), true, descriptor)) return false;
|
||||
DataMsg data;
|
||||
memcpy(data.manifest_id, manifest.merkle_root, 4);
|
||||
data.block_idx = 0;
|
||||
data.frag_off = descriptor;
|
||||
data.data = body.data();
|
||||
data.data_len = (uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + length);
|
||||
uint8_t wire[MAX_PACKET_PAYLOAD];
|
||||
const uint16_t wire_len = encode_data(wire, sizeof(wire), data);
|
||||
return wire_len != 0 && client.on_message(wire, wire_len);
|
||||
};
|
||||
|
||||
ASSERT_TRUE(deliver_fragment(representation_a, id_a, 0));
|
||||
EXPECT_FALSE(deliver_fragment(representation_b, id_b, 1)); // valid stream, wrong representation id
|
||||
ASSERT_TRUE(deliver_fragment(representation_a, id_a, 1));
|
||||
|
||||
std::vector<uint8_t> scratch(manifest.block_count * 4);
|
||||
uint8_t siblings[32 * 4];
|
||||
const uint8_t sibling_count = merkle_gen_proof(
|
||||
manifest.leaves, manifest.block_count, 0, scratch.data(), siblings);
|
||||
ProofMsg proof;
|
||||
memcpy(proof.manifest_id, manifest.merkle_root, 4);
|
||||
proof.block_idx = 0;
|
||||
proof.n_proof = sibling_count;
|
||||
proof.proof = siblings;
|
||||
uint8_t proof_wire[MAX_PACKET_PAYLOAD];
|
||||
const uint16_t proof_len = encode_proof(proof_wire, sizeof(proof_wire), proof);
|
||||
ASSERT_GT(proof_len, 0);
|
||||
ASSERT_TRUE(client.on_message(proof_wire, proof_len));
|
||||
EXPECT_EQ(client.blocksHave(), 1u);
|
||||
const uint32_t payload_offset = (uint32_t)(manifest.payload - SIM_MOTA_1K);
|
||||
EXPECT_EQ(0, memcmp(store.data() + payload_offset, manifest.payload, manifest.block_size()));
|
||||
}
|
||||
|
||||
TEST(OtaTransfer, NewClientFallsBackWhenLegacySourceServesOnlyFirstWindowRow) {
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
@@ -2430,6 +3107,34 @@ TEST(OtaFolder, ServesSelfPlusFolderAndFetchesExternal) {
|
||||
EXPECT_TRUE(mota_check_image_hash_full(got));
|
||||
}
|
||||
|
||||
TEST(OtaFolder, OldHostCapabilityZeroSkipsUnsupportedDeflateOperation) {
|
||||
g_q.clear();
|
||||
OtaManager server, client;
|
||||
OtaStoreRam<4096> store;
|
||||
SendTo to_client{&client}, to_server{&server};
|
||||
server.begin(0, sim_send, &to_client);
|
||||
client.begin(SIM_TARGET_ID, sim_send, &to_server);
|
||||
client.set_transport_deflate_decoder(ota_transport_inflate);
|
||||
client.set_fetch_store(&store);
|
||||
|
||||
RamMotaSource old_host;
|
||||
old_host.add(SIM_MOTA_1K, SIM_MOTA_1K_LEN);
|
||||
old_host.setSourceCaps(0); // deployed descriptor's reserved byte
|
||||
ASSERT_TRUE(server.add_source(&old_host));
|
||||
MotaManifest manifest;
|
||||
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest));
|
||||
|
||||
server.announce();
|
||||
pump(client, &server);
|
||||
client.pull(manifest.merkle_root, manifest.target_id);
|
||||
pump(client, &server);
|
||||
|
||||
EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE);
|
||||
EXPECT_EQ(old_host.deflateCalls(), 0u); // no per-block timeout on unknown op 0x09
|
||||
ASSERT_EQ(store.staged_size(), SIM_MOTA_1K_LEN);
|
||||
EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_1K, SIM_MOTA_1K_LEN));
|
||||
}
|
||||
|
||||
TEST(OtaCatalog, QueryCanRequestOnlyMissingHaveFragment) {
|
||||
OtaManager server;
|
||||
CapturedMessages sent;
|
||||
|
||||
@@ -10,11 +10,13 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import shutil
|
||||
import signal
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
BLEAK_IMPORT_ERROR: ImportError | None = None
|
||||
@@ -50,10 +52,13 @@ MOTA_FLAG_ANOTHER_LINK_ACTIVE = 0x04
|
||||
OP_COUNT = 0x01
|
||||
OP_DESCRIBE = 0x02
|
||||
OP_READ = 0x03
|
||||
OP_DEFLATE_BLOCK = 0x09
|
||||
STATUS_OK = 0
|
||||
STATUS_ERR = 1
|
||||
MOTA_READ_MAX = 192
|
||||
MOTA_DEFLATE_CHUNK_MAX = 190
|
||||
MOTA_DESC_WIRE = 38
|
||||
MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01
|
||||
MOTA_HEADER_LEN = 8
|
||||
MOTA_MANIFEST_LEN = 197
|
||||
MOTA_TRAILER = b"vk496"
|
||||
@@ -72,6 +77,28 @@ class MotaFile:
|
||||
size: int
|
||||
descriptor: bytes
|
||||
|
||||
@functools.lru_cache(maxsize=16)
|
||||
def deflated_block(self, block_index: int) -> bytes | None:
|
||||
"""Return one smaller raw-DEFLATE payload block, cached for chunk reads."""
|
||||
block_count = struct.unpack_from("<I", self.descriptor, 22)[0]
|
||||
payload_offset = struct.unpack_from("<I", self.descriptor, 26)[0]
|
||||
payload_size = struct.unpack_from("<I", self.descriptor, 30)[0]
|
||||
block_size = 1 << self.descriptor[34]
|
||||
if block_index < 0 or block_index >= block_count:
|
||||
return None
|
||||
raw_offset = block_index * block_size
|
||||
raw_length = min(block_size, payload_size - raw_offset)
|
||||
if raw_length <= 0 or raw_length > 1024 or self.path.stat().st_size != self.size:
|
||||
return None
|
||||
with self.path.open("rb") as stream:
|
||||
stream.seek(payload_offset + raw_offset)
|
||||
raw = stream.read(raw_length)
|
||||
if len(raw) != raw_length:
|
||||
return None
|
||||
compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-10)
|
||||
encoded = compressor.compress(raw) + compressor.flush()
|
||||
return encoded if 0 < len(encoded) < len(raw) else None
|
||||
|
||||
@staticmethod
|
||||
def load(path: Path) -> "MotaFile":
|
||||
size = path.stat().st_size
|
||||
@@ -117,6 +144,7 @@ class MotaFile:
|
||||
struct.pack_into("<I", descriptor, 26, payload_offset)
|
||||
struct.pack_into("<I", descriptor, 30, payload_size)
|
||||
descriptor[34] = block_size_log2
|
||||
descriptor[35] = MOTA_SOURCE_CAP_DEFLATE_BLOCK
|
||||
return MotaFile(path=path, size=size, descriptor=bytes(descriptor))
|
||||
|
||||
|
||||
@@ -195,6 +223,21 @@ class Catalog:
|
||||
status = STATUS_OK
|
||||
except OSError:
|
||||
payload = b""
|
||||
elif op == OP_DEFLATE_BLOCK and len(args) == 7:
|
||||
index = args[0]
|
||||
block = struct.unpack_from("<H", args, 1)[0]
|
||||
offset = struct.unpack_from("<H", args, 3)[0]
|
||||
length = struct.unpack_from("<H", args, 5)[0]
|
||||
if index < len(self.files) and length <= MOTA_DEFLATE_CHUNK_MAX:
|
||||
try:
|
||||
encoded = self.files[index].deflated_block(block)
|
||||
end = offset + length
|
||||
if (encoded is not None and end >= offset and end <= len(encoded)
|
||||
and (length != 0 or offset == 0)):
|
||||
payload = struct.pack("<H", len(encoded)) + encoded[offset:end]
|
||||
status = STATUS_OK
|
||||
except (OSError, zlib.error):
|
||||
payload = b""
|
||||
|
||||
response = bytearray(b"ms")
|
||||
response.extend((op, status))
|
||||
@@ -207,6 +250,11 @@ class Catalog:
|
||||
detail = f"index={args[0]}"
|
||||
elif op == OP_READ and len(args) == 7:
|
||||
detail = f"index={args[0]} offset={struct.unpack_from('<I', args, 1)[0]}"
|
||||
elif op == OP_DEFLATE_BLOCK and len(args) == 7:
|
||||
detail = (
|
||||
f"index={args[0]} block={struct.unpack_from('<H', args, 1)[0]} "
|
||||
f"offset={struct.unpack_from('<H', args, 3)[0]}"
|
||||
)
|
||||
else:
|
||||
detail = "invalid"
|
||||
print(f"mOTA op=0x{op:02x} {detail} status={status}")
|
||||
|
||||
@@ -1,10 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import ble_mota_seeder as seeder
|
||||
|
||||
|
||||
def request_frame(op: int, args: bytes) -> bytes:
|
||||
return b"MS" + bytes((op,)) + args + bytes((seeder.xor_bytes(args, op),))
|
||||
|
||||
|
||||
def response_payload(response: bytes, op: int) -> tuple[int, bytes]:
|
||||
assert response[:2] == b"ms"
|
||||
assert response[2] == op
|
||||
assert response[-1] == seeder.xor_bytes(response[:-1])
|
||||
return response[3], response[4:-1]
|
||||
|
||||
|
||||
class TransportDeflateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.directory.name) / "test.mota"
|
||||
# This distribution makes zlib level 9 choose a dynamic-Huffman block, proving the
|
||||
# host is not silently constrained to the bootloader's separate fixed-only profile.
|
||||
self.raw = bytes(i % 31 if i % 7 else 0 for i in range(1024))
|
||||
payload_offset = 256
|
||||
self.path.write_bytes(bytes(payload_offset) + self.raw)
|
||||
descriptor = bytearray(seeder.MOTA_DESC_WIRE)
|
||||
struct.pack_into("<I", descriptor, 22, 1) # block_count
|
||||
struct.pack_into("<I", descriptor, 26, payload_offset)
|
||||
struct.pack_into("<I", descriptor, 30, len(self.raw))
|
||||
descriptor[34] = 10
|
||||
mota = seeder.MotaFile(self.path, self.path.stat().st_size, bytes(descriptor))
|
||||
self.catalog = seeder.Catalog([mota], False)
|
||||
|
||||
def tearDown(self):
|
||||
seeder.MotaFile.deflated_block.cache_clear()
|
||||
self.directory.cleanup()
|
||||
|
||||
def exchange(self, block: int, offset: int, length: int) -> tuple[int, bytes]:
|
||||
args = struct.pack("<BHHH", 0, block, offset, length)
|
||||
response = self.catalog.handle_request(
|
||||
request_frame(seeder.OP_DEFLATE_BLOCK, args)
|
||||
)
|
||||
self.assertIsNotNone(response)
|
||||
return response_payload(response, seeder.OP_DEFLATE_BLOCK)
|
||||
|
||||
def test_serves_independent_raw_deflate_in_bounded_chunks(self):
|
||||
status, payload = self.exchange(0, 0, 0)
|
||||
self.assertEqual(status, seeder.STATUS_OK)
|
||||
total = struct.unpack("<H", payload)[0]
|
||||
self.assertLess(total, len(self.raw))
|
||||
|
||||
encoded = bytearray()
|
||||
for offset in range(0, total, seeder.MOTA_DEFLATE_CHUNK_MAX):
|
||||
length = min(seeder.MOTA_DEFLATE_CHUNK_MAX, total - offset)
|
||||
status, payload = self.exchange(0, offset, length)
|
||||
self.assertEqual(status, seeder.STATUS_OK)
|
||||
self.assertEqual(struct.unpack_from("<H", payload)[0], total)
|
||||
encoded.extend(payload[2:])
|
||||
self.assertEqual((encoded[0] >> 1) & 0x03, 2) # dynamic Huffman (BTYPE=2)
|
||||
self.assertEqual(zlib.decompress(bytes(encoded), wbits=-10), self.raw)
|
||||
|
||||
def test_rejects_bad_block_range_and_oversized_chunk(self):
|
||||
self.assertEqual(self.exchange(1, 0, 0)[0], seeder.STATUS_ERR)
|
||||
self.assertEqual(
|
||||
self.exchange(0, 0, seeder.MOTA_DEFLATE_CHUNK_MAX + 1)[0],
|
||||
seeder.STATUS_ERR,
|
||||
)
|
||||
|
||||
|
||||
class SourceStatusTests(unittest.TestCase):
|
||||
def test_parses_current_status_with_packet_count(self):
|
||||
response = bytes.fromhex("0001030200020078563412")
|
||||
|
||||
Reference in new issue
Block a user