ota: pull a fetched .mota to a host folder (ota pull <#> folder) — firmware side

FolderMotaStore: an OtaStore that streams an in-transit .mota straight to the host
folder over the seeder link (OP_BEGIN/WRITE/SREAD/STAT/FIN) instead of RAM/flash —
the device holds no local staging for it. Wired as a selectable pull destination:

- OtaContext gains a folder_dest (registered by the app while a motatool `serve`
  link is up) + its human id (e.g. "tcp 192.168.4.5").
- `ota pull <#> <dest>` now takes a MANDATORY destination; `ota pull <#>` with none
  lists the choices (flash always; folder + its link id when connected).
- The ESP32 WiFi seeder registers/clears the folder destination on connect/close —
  the same connection both serves the folder and accepts pulls into it.

Captures an exact copy of a device's firmware over the mesh to build a delta
against. Pause/resume on a mid-pull disconnect follows.

examples/companion_radio/main.cpp

@@ -116,20 +116,24 @@ void halt() {
 #if defined(ESP32) && defined(WIFI_SSID) && defined(ENABLE_OTA)
   #include <helpers/ota/OtaContext.h>
   #include <helpers/ota/MotaSourceSerial.h>
+  #include <helpers/ota/FolderMotaStore.h>   // `ota pull <#> folder` destination over this same connection
   #ifndef OTA_SEEDER_TCP_PORT
     #define OTA_SEEDER_TCP_PORT 5001
   #endif
   static WiFiServer ota_seeder_server(OTA_SEEDER_TCP_PORT);
   static WiFiClient ota_seeder_client;                                  // the live seeder connection (reused)
-  static mesh::ota::SerialMotaSource ota_seeder_source(ota_seeder_client, 3000);
+  static mesh::ota::SerialMotaSource ota_seeder_source(ota_seeder_client, 3000);   // SERVE: read folder -> relay
+  static mesh::ota::FolderMotaStore  ota_folder_store(ota_seeder_client, 3000);    // PULL: capture .mota -> folder
   static bool ota_seeder_attached = false;
 
-  // Accept one motatool connection at a time; while connected, register its folder as a serve source so
-  // the node advertises + relays it over LoRa. Drop the source the moment the connection closes.
+  // Accept one motatool connection at a time. While connected, the same link both SERVES the host folder
+  // over LoRa (register it as a source) and is offered as a `folder` PULL destination (`ota pull <#> folder`
+  // captures a fetched .mota back to that folder). Drop both the moment the connection closes.
   static void ota_seeder_loop() {
     if (ota_seeder_client && ota_seeder_client.connected()) return;     // still serving the current client
     if (ota_seeder_attached) {                                          // previous client just disconnected
       mesh::ota::ota_ctx().detach_folder();
+      mesh::ota::ota_ctx().clear_folder_dest();  // the `folder` pull destination is gone too
       mesh::ota::ota_ctx().manager.announce();   // served set shrank back to our own fw -> re-advertise
       ota_seeder_attached = false;
       WIFI_DEBUG_PRINTLN("OTA seeder: client disconnected, relay stopped");
@@ -139,8 +143,10 @@ void halt() {
       ota_seeder_client = c;                                            // rebind the persistent Stream to it
       if (mesh::ota::ota_ctx().manager.add_source(&ota_seeder_source)) {
         ota_seeder_attached = true;
+        char di[24]; snprintf(di, sizeof di, "tcp %s", ota_seeder_client.remoteIP().toString().c_str());
+        mesh::ota::ota_ctx().set_folder_dest(&ota_folder_store, di);   // offer `ota pull <#> folder`
         mesh::ota::ota_ctx().manager.announce();   // new served set -> advertise the folder's fw to peers
-        WIFI_DEBUG_PRINTLN("OTA seeder: client connected, relaying its folder");
+        WIFI_DEBUG_PRINTLN("OTA seeder: client connected (%s) — relay + folder pull-dest ready", di);
       } else {
         ota_seeder_client.stop();                                      // no free source slot
       }

src/helpers/ota/FolderMotaStore.cpp

@@ -0,0 +1,113 @@
+#include "FolderMotaStore.h"
+#include "MotaSeederProto.h"
+#include "OtaByteIO.h"
+#include <string.h>
+
+namespace mesh {
+namespace ota {
+
+bool FolderMotaStore::readByteT(uint8_t& b) const {
+  uint32_t t0 = millis();
+  while ((millis() - t0) < _to) {
+    int c = _io.read();
+    if (c >= 0) { b = (uint8_t)c; return true; }
+  }
+  return false;
+}
+
+bool FolderMotaStore::readExact(uint8_t* b, uint16_t n) const {
+  for (uint16_t i = 0; i < n; i++) if (!readByteT(b[i])) return false;
+  return true;
+}
+
+// Resync-safe request/response (same framing as SerialMotaSource): drain stale input, frame op+args with an
+// XOR checksum, scan for the response magic, validate op+status+checksum, deliver payload.
+bool FolderMotaStore::txn(uint8_t op, const uint8_t* args, uint16_t arglen,
+                          uint8_t* payload, uint16_t payload_len) const {
+  while (_io.read() >= 0) {}                        // drop any stale/partial bytes before a fresh request
+  uint8_t xs = op;
+  for (uint16_t i = 0; i < arglen; i++) xs ^= args[i];
+  _io.write(MOTA_SEEDER_REQ_MAGIC0); _io.write(MOTA_SEEDER_REQ_MAGIC1);
+  _io.write(op);
+  if (arglen) _io.write(args, arglen);
+  _io.write(xs);
+  _io.flush();
+
+  uint32_t t0 = millis(); bool got = false; uint8_t prev = 0;
+  while ((millis() - t0) < _to) {                   // scan for response magic 'm' 's' (tolerate noise)
+    int c = _io.read();
+    if (c < 0) continue;
+    if (prev == MOTA_SEEDER_RSP_MAGIC0 && (uint8_t)c == MOTA_SEEDER_RSP_MAGIC1) { got = true; break; }
+    prev = (uint8_t)c;
+  }
+  if (!got) return false;
+
+  uint8_t hdr[2];
+  if (!readExact(hdr, 2)) return false;             // op, status
+  if (hdr[0] != op) return false;
+  uint8_t rxs = (uint8_t)(MOTA_SEEDER_RSP_MAGIC0 ^ MOTA_SEEDER_RSP_MAGIC1) ^ hdr[0] ^ hdr[1];
+  bool ok = (hdr[1] == MS_STATUS_OK);
+  if (ok && payload_len) {
+    if (!readExact(payload, payload_len)) return false;
+    for (uint16_t i = 0; i < payload_len; i++) rxs ^= payload[i];
+  }
+  uint8_t xsum;
+  if (!readByteT(xsum)) return false;
+  if (xsum != rxs) return false;                    // corrupt frame -> caller retries
+  return ok;
+}
+
+bool FolderMotaStore::begin(uint32_t total_size) {
+  uint8_t args[8];
+  memcpy(args, _mid, 4);
+  wr_u32le(args + 4, total_size);
+  if (!txn(MS_OP_BEGIN, args, 8, nullptr, 0)) return false;
+  _total = total_size;
+  return true;
+}
+
+bool FolderMotaStore::write(uint32_t off, const uint8_t* data, uint32_t len) {
+  uint8_t req[10 + MOTA_SEEDER_WRITE_MAX];
+  memcpy(req, _mid, 4);
+  uint32_t done = 0;
+  while (done < len) {
+    uint16_t chunk = (len - done > MOTA_SEEDER_WRITE_MAX) ? MOTA_SEEDER_WRITE_MAX : (uint16_t)(len - done);
+    wr_u32le(req + 4, off + done);
+    req[8] = (uint8_t)(chunk & 0xFF); req[9] = (uint8_t)(chunk >> 8);
+    memcpy(req + 10, data + done, chunk);
+    if (!txn(MS_OP_WRITE, req, (uint16_t)(10 + chunk), nullptr, 0)) return false;
+    done += chunk;
+  }
+  return true;
+}
+
+bool FolderMotaStore::read(uint32_t off, uint8_t* buf, uint32_t len) const {
+  uint8_t args[10];
+  memcpy(args, _mid, 4);
+  uint32_t done = 0;
+  while (done < len) {
+    uint16_t chunk = (len - done > MOTA_SEEDER_WRITE_MAX) ? MOTA_SEEDER_WRITE_MAX : (uint16_t)(len - done);
+    wr_u32le(args + 4, off + done);
+    args[8] = (uint8_t)(chunk & 0xFF); args[9] = (uint8_t)(chunk >> 8);
+    if (!txn(MS_OP_SREAD, args, 10, buf + done, chunk)) return false;
+    done += chunk;
+  }
+  return true;
+}
+
+void FolderMotaStore::finalize() {
+  txn(MS_OP_FIN, _mid, 4, nullptr, 0);   // publish <midhex>.mota.part as <midhex>.mota (best-effort)
+}
+
+bool FolderMotaStore::reopen() {
+  uint8_t pl[5];
+  if (!txn(MS_OP_STAT, _mid, 4, pl, 5)) return false;
+  if (pl[0] == 0) return false;                     // host has no partial/complete file -> start fresh
+  uint32_t total = rd_u32le(pl + 1);
+  if (total < 13) return false;                     // implausible container
+  _total = total;
+  return true;
+}
+
+} // namespace ota
+} // namespace mesh

src/helpers/ota/FolderMotaStore.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include <Arduino.h>
+#include "OtaStore.h"
+
+// An OtaStore that captures an in-transit `.mota` onto a HOST folder over the mota-seeder link (the WRITE
+// half of MotaSeederProto: OP_STAT/BEGIN/WRITE/SREAD/FIN). This is the destination for `ota pull <#> folder`:
+// blocks stream straight to the host `<mid>.mota` — the device holds NO RAM/flash staging for it.
+//
+//   begin(total)      -> OP_BEGIN  (host creates a 0xFF-filled <midhex>.mota.part)
+//   write(off,data)   -> OP_WRITE  (split into <= MOTA_SEEDER_WRITE_MAX chunks)
+//   read(off,buf)     -> OP_SREAD  (read back; unwritten regions are 0xFF)
+//   reopen()          -> OP_STAT   (adopt an existing partial/complete file for resume)
+//   finalize()        -> OP_FIN    (publish <midhex>.mota.part as <midhex>.mota)
+//
+// If the link drops, every op returns false: the OtaManager PAUSES and keeps its progress (it does NOT fall
+// back to RAM/flash). On reconnect the host still has the partial, and the manager re-reopen()s so only the
+// missing blocks are refetched. All ops are keyed by `mid` (set via set_mid() before the fetch begins).
+
+namespace mesh {
+namespace ota {
+
+class FolderMotaStore : public OtaStore {
+public:
+  explicit FolderMotaStore(Stream& io, uint32_t timeout_ms = 3000) : _io(io), _to(timeout_ms) {}
+
+  // The container being pulled — set from the chosen `ota pull` mid before begin()/reopen().
+  void set_mid(const uint8_t mid[4]) { memcpy(_mid, mid, 4); }
+
+  bool begin(uint32_t total_size) override;
+  bool write(uint32_t off, const uint8_t* data, uint32_t len) override;
+  bool read(uint32_t off, uint8_t* buf, uint32_t len) const override;
+  uint32_t capacity() const override { return 0xF0000000u; }   // host disk — effectively unbounded
+  uint32_t staged_size() const override { return _total; }
+  void clear() override { _total = 0; }
+  void finalize() override;
+  bool reopen() override;                                       // OP_STAT: adopt an existing file for resume
+
+  // A host-backed store is fully random-access, so it needs no pinned-meta RAM page and no layout planning.
+  bool set_meta_size(uint32_t) override { return true; }
+  bool plan_layout(bool, uint32_t, uint32_t, uint32_t) override { return true; }
+
+private:
+  // One request/response transaction over the seeder link (mirrors SerialMotaSource). `arglen` is uint16 so
+  // OP_WRITE can carry its data inline. Returns true iff a well-formed OK response for `op` arrived in time.
+  bool txn(uint8_t op, const uint8_t* args, uint16_t arglen, uint8_t* payload, uint16_t payload_len) const;
+  bool readByteT(uint8_t& b) const;
+  bool readExact(uint8_t* b, uint16_t n) const;
+
+  Stream&  _io;
+  uint32_t _to;
+  uint8_t  _mid[4] = {0, 0, 0, 0};
+  uint32_t _total = 0;
+};
+
+} // namespace ota
+} // namespace mesh

src/helpers/ota/OtaCli.cpp

@@ -1,5 +1,6 @@
 #include "OtaCli.h"
 #include "OtaContext.h"
+#include "FolderMotaStore.h"   // `ota pull <#> folder` destination (set_mid on the connected folder store)
 #include "OtaVerify.h"
 #include "OtaSelf.h"
 #include "OtaTargets.h"   // ota_target_env_name(): human-readable name for a target_id (no string on the wire)
@@ -161,23 +162,50 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board
 
   // ---- start fetching a specific catalogued mOTA (by list index or manifest_id) ----
   } else if (is_cmd(a, "pull|get|download", &rest)) {
-    const char* p = rest;
-    if (*p == 0) { strcpy(reply, "usage: ota get <#>   (see the numbers in `ota ls`)"); return true; }
+    const char* p = rest; while (*p == ' ') p++;
+    // split into "<selector> [destination]": selector = #N / N (catalogue index) or mid hex; destination =
+    // flash | folder (MANDATORY — `folder` captures the .mota onto the connected motatool folder as <mid>.mota).
+    char selstr[24]; int i = 0;
+    while (p[i] && p[i] != ' ' && i < (int)sizeof(selstr) - 1) { selstr[i] = p[i]; i++; }
+    selstr[i] = 0;
+    const char* dst = p + i; while (*dst == ' ') dst++;
+    if (selstr[0] == 0) { strcpy(reply, "usage: ota pull <#> <flash|folder>   (see `ota ls`)"); return true; }
+    // resolve the catalogue row (index or explicit manifest_id)
     const OtaManager::CatRow* sel = nullptr; uint8_t mid[4];
-    if (*p == '#' || (p[0] >= '1' && p[0] <= '9' && (p[1] == 0 || p[1] == ' '))) {   // index among catalogue
-      int idx = atoi(*p == '#' ? p + 1 : p);
+    bool isnum = (selstr[0] == '#');
+    if (!isnum) { isnum = true; for (const char* x = selstr; *x; x++) if (*x < '0' || *x > '9') { isnum = false; break; } }
+    if (isnum) {
+      int idx = atoi(selstr[0] == '#' ? selstr + 1 : selstr);
       if (idx >= 1 && idx <= c.manager.catalogCount()) sel = c.manager.catalogRow((uint8_t)(idx - 1));
-    } else if (mesh::Utils::fromHex(mid, 4, p)) {                                    // explicit manifest_id
-      for (uint8_t i = 0; i < c.manager.catalogCount(); i++)
-        if (memcmp(c.manager.catalogRow(i)->mid, mid, 4) == 0) { sel = c.manager.catalogRow(i); break; }
+    } else if (mesh::Utils::fromHex(mid, 4, selstr)) {
+      for (uint8_t k = 0; k < c.manager.catalogCount(); k++)
+        if (memcmp(c.manager.catalogRow(k)->mid, mid, 4) == 0) { sel = c.manager.catalogRow(k); break; }
     }
     if (!sel) { strcpy(reply, "ERR no such update (see the numbers in `ota ls`)"); return true; }
+    // destination is MANDATORY: with none given, show the choices (flash always; folder iff a link is up).
+    if (*dst == 0) {
+      if (c.folder_dest)
+        snprintf(reply, 160, "choose a destination: `ota pull %s flash` | `ota pull %s folder`  (folder: %s)",
+                 selstr, selstr, c.folder_dest_info);
+      else
+        snprintf(reply, 160, "choose a destination: `ota pull %s flash`  (folder: none connected — motatool serve)",
+                 selstr);
+      return true;
+    }
     if (c.apply_pending) { strcpy(reply, "ERR busy applying"); return true; }
     uint8_t selmid[4]; uint32_t seltgt = sel->target_id; memcpy(selmid, sel->mid, 4);   // sel may move on reset
-    c.manager.reset_session(); c.fetch_store.clear();
+    OtaStore* store; const char* dname;
+    if (strncmp(dst, "flash", 5) == 0) {
+      store = &c.fetch_store; c.fetch_store.clear(); dname = "flash";
+    } else if (strncmp(dst, "folder", 6) == 0) {
+      if (!c.folder_dest) { strcpy(reply, "ERR no folder connected (run motatool serve --tcp/--serial)"); return true; }
+      c.folder_dest->set_mid(selmid); store = c.folder_dest; dname = "folder";
+    } else { strcpy(reply, "ERR destination must be `flash` or `folder`"); return true; }
+    c.manager.reset_session();
+    c.manager.set_fetch_store(store);                        // stage this pull to the chosen destination
     c.manager.pull(selmid, seltgt);                          // sets want + begins the manifest fetch now
     char midhx[9]; mesh::Utils::toHex(midhx, selmid, 4);
-    sprintf(reply, "OK pulling mid=%s target=%08X (low priority)", midhx, (unsigned)seltgt);
+    snprintf(reply, 160, "OK pulling mid=%s -> %s (low priority)", midhx, dname);
 
   // ---- discard the current session (e.g. a stalled old fetch) to free the slot ----
   } else if (is_cmd(a, "drop|cancel|stop", &rest)) {
@@ -185,8 +213,9 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board
     char midhx[9]; strcpy(midhx, "-");
     if (fs != OtaManager::IDLE) mesh::Utils::toHex(midhx, c.manager.fetchManifestId(), 4);
     c.manager.reset_session(); c.manager.want(0); c.manager.want_mid(nullptr);
+    c.manager.set_fetch_store(&c.fetch_store);   // revert to the default flash store (a folder pull switched it)
     c.fetch_store.clear(); c.serving = false; c.serve_expected = 0; c.session_started_ms = 0;
-    sprintf(reply, "OK dropped session (was %c mid=%s); slot free for a new pull", fstate_char(fs), midhx);
+    snprintf(reply, 160, "OK dropped session (was %c mid=%s); slot free for a new pull", fstate_char(fs), midhx);
 
   // ---- broadcast our tiny beacon so peers discover us. If not already serving, set up flash-backed
   //      self-serve first (so we're a real, fetchable source of our own running firmware). ----

src/helpers/ota/OtaContext.h

@@ -37,6 +37,8 @@
 namespace mesh {
 namespace ota {
 
+class FolderMotaStore;   // pull destination over the seeder link (full type only where instantiated/used)
+
 #ifndef OTA_SERVE_BUF_SIZE
 #define OTA_SERVE_BUF_SIZE 16384
 #endif
@@ -73,6 +75,19 @@ struct OtaContext {
   bool     config_dirty = false;            // CLI set a policy/key -> CommonCLI persists + clears
   char     hw_id[33] = {0};                 // this device's hardware tag (from board.getOtaHwId(), set in begin)
 
+  // ---- Pull destinations (`ota pull <#> <dest>`): where a fetched .mota is staged. `flash` (fetch_store,
+  //      always present) or `folder` — an external host folder over the seeder link, registered by the app
+  //      while a motatool `serve` connection is attached (else no `folder` dest is offered). Captures the
+  //      container to the host as <mid>.mota so an exact firmware copy can be pulled for delta-building. ----
+  FolderMotaStore* folder_dest = nullptr;   // non-null == a folder destination is currently connected
+  char     folder_dest_info[24] = {0};      // human id of the link (e.g. "tcp 192.168.4.5", "serial")
+  void set_folder_dest(FolderMotaStore* fs, const char* info) {
+    folder_dest = fs;
+    strncpy(folder_dest_info, info ? info : "?", sizeof(folder_dest_info) - 1);
+    folder_dest_info[sizeof(folder_dest_info) - 1] = 0;
+  }
+  void clear_folder_dest() { folder_dest = nullptr; folder_dest_info[0] = 0; }
+
   // True if the staged .mota's hw_id is compatible with this device: equal tags, or either side empty
   // ("unknown" -> can't enforce -> permissive). Brick-safety gate for apply (esp. manual cross-target).
   bool hwMatches(const uint8_t* mhw /*32B, may be null*/) const {