From 10fa434412bb75bc5e100c84b767f45929f21349 Mon Sep 17 00:00:00 2001 From: Valentin Kivachuk Burda Date: Thu, 2 Jul 2026 08:44:10 +0200 Subject: [PATCH] =?UTF-8?q?ota:=20pull=20a=20fetched=20.mota=20to=20a=20ho?= =?UTF-8?q?st=20folder=20(`ota=20pull=20<#>=20folder`)=20=E2=80=94=20firmw?= =?UTF-8?q?are=20side?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FolderMotaStore: an OtaStore that streams an in-transit .mota straight to the host folder over the seeder link (OP_BEGIN/WRITE/SREAD/STAT/FIN) instead of RAM/flash — the device holds no local staging for it. Wired as a selectable pull destination: - OtaContext gains a folder_dest (registered by the app while a motatool `serve` link is up) + its human id (e.g. "tcp 192.168.4.5"). - `ota pull <#> ` now takes a MANDATORY destination; `ota pull <#>` with none lists the choices (flash always; folder + its link id when connected). - The ESP32 WiFi seeder registers/clears the folder destination on connect/close — the same connection both serves the folder and accepts pulls into it. Captures an exact copy of a device's firmware over the mesh to build a delta against. Pause/resume on a mid-pull disconnect follows. --- examples/companion_radio/main.cpp | 14 +++- src/helpers/ota/FolderMotaStore.cpp | 113 ++++++++++++++++++++++++++++ src/helpers/ota/FolderMotaStore.h | 57 ++++++++++++++ src/helpers/ota/OtaCli.cpp | 49 +++++++++--- src/helpers/ota/OtaContext.h | 15 ++++ 5 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 src/helpers/ota/FolderMotaStore.cpp create mode 100644 src/helpers/ota/FolderMotaStore.h diff --git a/examples/companion_radio/main.cpp b/examples/companion_radio/main.cpp index 6844724b..3744a439 100644 --- a/examples/companion_radio/main.cpp +++ b/examples/companion_radio/main.cpp @@ -116,20 +116,24 @@ void halt() { #if defined(ESP32) && defined(WIFI_SSID) && defined(ENABLE_OTA) #include #include + #include // `ota pull <#> folder` destination over this same connection #ifndef OTA_SEEDER_TCP_PORT #define OTA_SEEDER_TCP_PORT 5001 #endif static WiFiServer ota_seeder_server(OTA_SEEDER_TCP_PORT); static WiFiClient ota_seeder_client; // the live seeder connection (reused) - static mesh::ota::SerialMotaSource ota_seeder_source(ota_seeder_client, 3000); + static mesh::ota::SerialMotaSource ota_seeder_source(ota_seeder_client, 3000); // SERVE: read folder -> relay + static mesh::ota::FolderMotaStore ota_folder_store(ota_seeder_client, 3000); // PULL: capture .mota -> folder static bool ota_seeder_attached = false; - // Accept one motatool connection at a time; while connected, register its folder as a serve source so - // the node advertises + relays it over LoRa. Drop the source the moment the connection closes. + // Accept one motatool connection at a time. While connected, the same link both SERVES the host folder + // over LoRa (register it as a source) and is offered as a `folder` PULL destination (`ota pull <#> folder` + // captures a fetched .mota back to that folder). Drop both the moment the connection closes. static void ota_seeder_loop() { if (ota_seeder_client && ota_seeder_client.connected()) return; // still serving the current client if (ota_seeder_attached) { // previous client just disconnected mesh::ota::ota_ctx().detach_folder(); + mesh::ota::ota_ctx().clear_folder_dest(); // the `folder` pull destination is gone too mesh::ota::ota_ctx().manager.announce(); // served set shrank back to our own fw -> re-advertise ota_seeder_attached = false; WIFI_DEBUG_PRINTLN("OTA seeder: client disconnected, relay stopped"); @@ -139,8 +143,10 @@ void halt() { ota_seeder_client = c; // rebind the persistent Stream to it if (mesh::ota::ota_ctx().manager.add_source(&ota_seeder_source)) { ota_seeder_attached = true; + char di[24]; snprintf(di, sizeof di, "tcp %s", ota_seeder_client.remoteIP().toString().c_str()); + mesh::ota::ota_ctx().set_folder_dest(&ota_folder_store, di); // offer `ota pull <#> folder` mesh::ota::ota_ctx().manager.announce(); // new served set -> advertise the folder's fw to peers - WIFI_DEBUG_PRINTLN("OTA seeder: client connected, relaying its folder"); + WIFI_DEBUG_PRINTLN("OTA seeder: client connected (%s) — relay + folder pull-dest ready", di); } else { ota_seeder_client.stop(); // no free source slot } diff --git a/src/helpers/ota/FolderMotaStore.cpp b/src/helpers/ota/FolderMotaStore.cpp new file mode 100644 index 00000000..bc84a03d --- /dev/null +++ b/src/helpers/ota/FolderMotaStore.cpp @@ -0,0 +1,113 @@ +#include "FolderMotaStore.h" +#include "MotaSeederProto.h" +#include "OtaByteIO.h" +#include + +namespace mesh { +namespace ota { + +bool FolderMotaStore::readByteT(uint8_t& b) const { + uint32_t t0 = millis(); + while ((millis() - t0) < _to) { + int c = _io.read(); + if (c >= 0) { b = (uint8_t)c; return true; } + } + return false; +} + +bool FolderMotaStore::readExact(uint8_t* b, uint16_t n) const { + for (uint16_t i = 0; i < n; i++) if (!readByteT(b[i])) return false; + return true; +} + +// Resync-safe request/response (same framing as SerialMotaSource): drain stale input, frame op+args with an +// XOR checksum, scan for the response magic, validate op+status+checksum, deliver payload. +bool FolderMotaStore::txn(uint8_t op, const uint8_t* args, uint16_t arglen, + uint8_t* payload, uint16_t payload_len) const { + while (_io.read() >= 0) {} // drop any stale/partial bytes before a fresh request + uint8_t xs = op; + for (uint16_t i = 0; i < arglen; i++) xs ^= args[i]; + _io.write(MOTA_SEEDER_REQ_MAGIC0); _io.write(MOTA_SEEDER_REQ_MAGIC1); + _io.write(op); + if (arglen) _io.write(args, arglen); + _io.write(xs); + _io.flush(); + + uint32_t t0 = millis(); bool got = false; uint8_t prev = 0; + while ((millis() - t0) < _to) { // scan for response magic 'm' 's' (tolerate noise) + int c = _io.read(); + if (c < 0) continue; + if (prev == MOTA_SEEDER_RSP_MAGIC0 && (uint8_t)c == MOTA_SEEDER_RSP_MAGIC1) { got = true; break; } + prev = (uint8_t)c; + } + if (!got) return false; + + uint8_t hdr[2]; + if (!readExact(hdr, 2)) return false; // op, status + if (hdr[0] != op) return false; + uint8_t rxs = (uint8_t)(MOTA_SEEDER_RSP_MAGIC0 ^ MOTA_SEEDER_RSP_MAGIC1) ^ hdr[0] ^ hdr[1]; + bool ok = (hdr[1] == MS_STATUS_OK); + if (ok && payload_len) { + if (!readExact(payload, payload_len)) return false; + for (uint16_t i = 0; i < payload_len; i++) rxs ^= payload[i]; + } + uint8_t xsum; + if (!readByteT(xsum)) return false; + if (xsum != rxs) return false; // corrupt frame -> caller retries + return ok; +} + +bool FolderMotaStore::begin(uint32_t total_size) { + uint8_t args[8]; + memcpy(args, _mid, 4); + wr_u32le(args + 4, total_size); + if (!txn(MS_OP_BEGIN, args, 8, nullptr, 0)) return false; + _total = total_size; + return true; +} + +bool FolderMotaStore::write(uint32_t off, const uint8_t* data, uint32_t len) { + uint8_t req[10 + MOTA_SEEDER_WRITE_MAX]; + memcpy(req, _mid, 4); + uint32_t done = 0; + while (done < len) { + uint16_t chunk = (len - done > MOTA_SEEDER_WRITE_MAX) ? MOTA_SEEDER_WRITE_MAX : (uint16_t)(len - done); + wr_u32le(req + 4, off + done); + req[8] = (uint8_t)(chunk & 0xFF); req[9] = (uint8_t)(chunk >> 8); + memcpy(req + 10, data + done, chunk); + if (!txn(MS_OP_WRITE, req, (uint16_t)(10 + chunk), nullptr, 0)) return false; + done += chunk; + } + return true; +} + +bool FolderMotaStore::read(uint32_t off, uint8_t* buf, uint32_t len) const { + uint8_t args[10]; + memcpy(args, _mid, 4); + uint32_t done = 0; + while (done < len) { + uint16_t chunk = (len - done > MOTA_SEEDER_WRITE_MAX) ? MOTA_SEEDER_WRITE_MAX : (uint16_t)(len - done); + wr_u32le(args + 4, off + done); + args[8] = (uint8_t)(chunk & 0xFF); args[9] = (uint8_t)(chunk >> 8); + if (!txn(MS_OP_SREAD, args, 10, buf + done, chunk)) return false; + done += chunk; + } + return true; +} + +void FolderMotaStore::finalize() { + txn(MS_OP_FIN, _mid, 4, nullptr, 0); // publish .mota.part as .mota (best-effort) +} + +bool FolderMotaStore::reopen() { + uint8_t pl[5]; + if (!txn(MS_OP_STAT, _mid, 4, pl, 5)) return false; + if (pl[0] == 0) return false; // host has no partial/complete file -> start fresh + uint32_t total = rd_u32le(pl + 1); + if (total < 13) return false; // implausible container + _total = total; + return true; +} + +} // namespace ota +} // namespace mesh diff --git a/src/helpers/ota/FolderMotaStore.h b/src/helpers/ota/FolderMotaStore.h new file mode 100644 index 00000000..63823af4 --- /dev/null +++ b/src/helpers/ota/FolderMotaStore.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include "OtaStore.h" + +// An OtaStore that captures an in-transit `.mota` onto a HOST folder over the mota-seeder link (the WRITE +// half of MotaSeederProto: OP_STAT/BEGIN/WRITE/SREAD/FIN). This is the destination for `ota pull <#> folder`: +// blocks stream straight to the host `.mota` — the device holds NO RAM/flash staging for it. +// +// begin(total) -> OP_BEGIN (host creates a 0xFF-filled .mota.part) +// write(off,data) -> OP_WRITE (split into <= MOTA_SEEDER_WRITE_MAX chunks) +// read(off,buf) -> OP_SREAD (read back; unwritten regions are 0xFF) +// reopen() -> OP_STAT (adopt an existing partial/complete file for resume) +// finalize() -> OP_FIN (publish .mota.part as .mota) +// +// If the link drops, every op returns false: the OtaManager PAUSES and keeps its progress (it does NOT fall +// back to RAM/flash). On reconnect the host still has the partial, and the manager re-reopen()s so only the +// missing blocks are refetched. All ops are keyed by `mid` (set via set_mid() before the fetch begins). + +namespace mesh { +namespace ota { + +class FolderMotaStore : public OtaStore { +public: + explicit FolderMotaStore(Stream& io, uint32_t timeout_ms = 3000) : _io(io), _to(timeout_ms) {} + + // The container being pulled — set from the chosen `ota pull` mid before begin()/reopen(). + void set_mid(const uint8_t mid[4]) { memcpy(_mid, mid, 4); } + + bool begin(uint32_t total_size) override; + bool write(uint32_t off, const uint8_t* data, uint32_t len) override; + bool read(uint32_t off, uint8_t* buf, uint32_t len) const override; + uint32_t capacity() const override { return 0xF0000000u; } // host disk — effectively unbounded + uint32_t staged_size() const override { return _total; } + void clear() override { _total = 0; } + void finalize() override; + bool reopen() override; // OP_STAT: adopt an existing file for resume + + // A host-backed store is fully random-access, so it needs no pinned-meta RAM page and no layout planning. + bool set_meta_size(uint32_t) override { return true; } + bool plan_layout(bool, uint32_t, uint32_t, uint32_t) override { return true; } + +private: + // One request/response transaction over the seeder link (mirrors SerialMotaSource). `arglen` is uint16 so + // OP_WRITE can carry its data inline. Returns true iff a well-formed OK response for `op` arrived in time. + bool txn(uint8_t op, const uint8_t* args, uint16_t arglen, uint8_t* payload, uint16_t payload_len) const; + bool readByteT(uint8_t& b) const; + bool readExact(uint8_t* b, uint16_t n) const; + + Stream& _io; + uint32_t _to; + uint8_t _mid[4] = {0, 0, 0, 0}; + uint32_t _total = 0; +}; + +} // namespace ota +} // namespace mesh diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index f514dea2..abe91872 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -1,5 +1,6 @@ #include "OtaCli.h" #include "OtaContext.h" +#include "FolderMotaStore.h" // `ota pull <#> folder` destination (set_mid on the connected folder store) #include "OtaVerify.h" #include "OtaSelf.h" #include "OtaTargets.h" // ota_target_env_name(): human-readable name for a target_id (no string on the wire) @@ -161,23 +162,50 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board // ---- start fetching a specific catalogued mOTA (by list index or manifest_id) ---- } else if (is_cmd(a, "pull|get|download", &rest)) { - const char* p = rest; - if (*p == 0) { strcpy(reply, "usage: ota get <#> (see the numbers in `ota ls`)"); return true; } + const char* p = rest; while (*p == ' ') p++; + // split into " [destination]": selector = #N / N (catalogue index) or mid hex; destination = + // flash | folder (MANDATORY — `folder` captures the .mota onto the connected motatool folder as .mota). + char selstr[24]; int i = 0; + while (p[i] && p[i] != ' ' && i < (int)sizeof(selstr) - 1) { selstr[i] = p[i]; i++; } + selstr[i] = 0; + const char* dst = p + i; while (*dst == ' ') dst++; + if (selstr[0] == 0) { strcpy(reply, "usage: ota pull <#> (see `ota ls`)"); return true; } + // resolve the catalogue row (index or explicit manifest_id) const OtaManager::CatRow* sel = nullptr; uint8_t mid[4]; - if (*p == '#' || (p[0] >= '1' && p[0] <= '9' && (p[1] == 0 || p[1] == ' '))) { // index among catalogue - int idx = atoi(*p == '#' ? p + 1 : p); + bool isnum = (selstr[0] == '#'); + if (!isnum) { isnum = true; for (const char* x = selstr; *x; x++) if (*x < '0' || *x > '9') { isnum = false; break; } } + if (isnum) { + int idx = atoi(selstr[0] == '#' ? selstr + 1 : selstr); if (idx >= 1 && idx <= c.manager.catalogCount()) sel = c.manager.catalogRow((uint8_t)(idx - 1)); - } else if (mesh::Utils::fromHex(mid, 4, p)) { // explicit manifest_id - for (uint8_t i = 0; i < c.manager.catalogCount(); i++) - if (memcmp(c.manager.catalogRow(i)->mid, mid, 4) == 0) { sel = c.manager.catalogRow(i); break; } + } else if (mesh::Utils::fromHex(mid, 4, selstr)) { + for (uint8_t k = 0; k < c.manager.catalogCount(); k++) + if (memcmp(c.manager.catalogRow(k)->mid, mid, 4) == 0) { sel = c.manager.catalogRow(k); break; } } if (!sel) { strcpy(reply, "ERR no such update (see the numbers in `ota ls`)"); return true; } + // destination is MANDATORY: with none given, show the choices (flash always; folder iff a link is up). + if (*dst == 0) { + if (c.folder_dest) + snprintf(reply, 160, "choose a destination: `ota pull %s flash` | `ota pull %s folder` (folder: %s)", + selstr, selstr, c.folder_dest_info); + else + snprintf(reply, 160, "choose a destination: `ota pull %s flash` (folder: none connected — motatool serve)", + selstr); + return true; + } if (c.apply_pending) { strcpy(reply, "ERR busy applying"); return true; } uint8_t selmid[4]; uint32_t seltgt = sel->target_id; memcpy(selmid, sel->mid, 4); // sel may move on reset - c.manager.reset_session(); c.fetch_store.clear(); + OtaStore* store; const char* dname; + if (strncmp(dst, "flash", 5) == 0) { + store = &c.fetch_store; c.fetch_store.clear(); dname = "flash"; + } else if (strncmp(dst, "folder", 6) == 0) { + if (!c.folder_dest) { strcpy(reply, "ERR no folder connected (run motatool serve --tcp/--serial)"); return true; } + c.folder_dest->set_mid(selmid); store = c.folder_dest; dname = "folder"; + } else { strcpy(reply, "ERR destination must be `flash` or `folder`"); return true; } + c.manager.reset_session(); + c.manager.set_fetch_store(store); // stage this pull to the chosen destination c.manager.pull(selmid, seltgt); // sets want + begins the manifest fetch now char midhx[9]; mesh::Utils::toHex(midhx, selmid, 4); - sprintf(reply, "OK pulling mid=%s target=%08X (low priority)", midhx, (unsigned)seltgt); + snprintf(reply, 160, "OK pulling mid=%s -> %s (low priority)", midhx, dname); // ---- discard the current session (e.g. a stalled old fetch) to free the slot ---- } else if (is_cmd(a, "drop|cancel|stop", &rest)) { @@ -185,8 +213,9 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board char midhx[9]; strcpy(midhx, "-"); if (fs != OtaManager::IDLE) mesh::Utils::toHex(midhx, c.manager.fetchManifestId(), 4); c.manager.reset_session(); c.manager.want(0); c.manager.want_mid(nullptr); + c.manager.set_fetch_store(&c.fetch_store); // revert to the default flash store (a folder pull switched it) c.fetch_store.clear(); c.serving = false; c.serve_expected = 0; c.session_started_ms = 0; - sprintf(reply, "OK dropped session (was %c mid=%s); slot free for a new pull", fstate_char(fs), midhx); + snprintf(reply, 160, "OK dropped session (was %c mid=%s); slot free for a new pull", fstate_char(fs), midhx); // ---- broadcast our tiny beacon so peers discover us. If not already serving, set up flash-backed // self-serve first (so we're a real, fetchable source of our own running firmware). ---- diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index 2a67c238..e598e546 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -37,6 +37,8 @@ namespace mesh { namespace ota { +class FolderMotaStore; // pull destination over the seeder link (full type only where instantiated/used) + #ifndef OTA_SERVE_BUF_SIZE #define OTA_SERVE_BUF_SIZE 16384 #endif @@ -73,6 +75,19 @@ struct OtaContext { bool config_dirty = false; // CLI set a policy/key -> CommonCLI persists + clears char hw_id[33] = {0}; // this device's hardware tag (from board.getOtaHwId(), set in begin) + // ---- Pull destinations (`ota pull <#> `): where a fetched .mota is staged. `flash` (fetch_store, + // always present) or `folder` — an external host folder over the seeder link, registered by the app + // while a motatool `serve` connection is attached (else no `folder` dest is offered). Captures the + // container to the host as .mota so an exact firmware copy can be pulled for delta-building. ---- + FolderMotaStore* folder_dest = nullptr; // non-null == a folder destination is currently connected + char folder_dest_info[24] = {0}; // human id of the link (e.g. "tcp 192.168.4.5", "serial") + void set_folder_dest(FolderMotaStore* fs, const char* info) { + folder_dest = fs; + strncpy(folder_dest_info, info ? info : "?", sizeof(folder_dest_info) - 1); + folder_dest_info[sizeof(folder_dest_info) - 1] = 0; + } + void clear_folder_dest() { folder_dest = nullptr; folder_dest_info[0] = 0; } + // True if the staged .mota's hw_id is compatible with this device: equal tags, or either side empty // ("unknown" -> can't enforce -> permissive). Brick-safety gate for apply (esp. manual cross-target). bool hwMatches(const uint8_t* mhw /*32B, may be null*/) const {