diff --git a/deploy/site/sdk.html b/deploy/site/sdk.html index 0f3dce07..e24277c7 100644 --- a/deploy/site/sdk.html +++ b/deploy/site/sdk.html @@ -402,6 +402,7 @@ return app

Drop a bare .lua file into /apps/ on the SD card and it shows up in the drawer — no manifest needed. The filename becomes the name. That is the fast loop while you are writing something.

Apps you side-load appear in the Store under Your own apps, where you can remove them again. A long press on the drawer tile also offers to remove.

On boards without an SD card the same path lives on internal storage.

+

No card slot you can reach (the ThinkNode M9's card is soldered on)? Push the files over the USB serial console instead: scripts/sideload_app.py --port /dev/cu.wchusbserial10 --reboot deploy/apps/<id>/<ver> sends the .lua + .json through the firmware's fput/fadd/fend commands into the same /apps/ the Store uses (--dest /lang for a language file). Needs pyserial.

diff --git a/scripts/sideload_app.py b/scripts/sideload_app.py new file mode 100644 index 00000000..a690498b --- /dev/null +++ b/scripts/sideload_app.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Sideload a Lua Store app (or any /apps or /lang file) onto a device over its +serial console. + +Boards whose storage is soldered on (the ThinkNode M9) cannot take the +"drop it on the SD card" route from the SDK page, and the firmware's Store can +only download from firmware.wadamesh.com. This pushes the files through the +serial CLI instead, using three commands the firmware offers for exactly this: + + fput /apps/ open (truncate) the file + fadd append a chunk; the device checks the + offset, decoded length and byte sum and + answers "Error: ..." so we can re-send + fend close it + +Lines are kept under 128 bytes on purpose: that is the ESP32's UART hardware +FIFO, and it is all that buffers the console while the firmware is inside a +flash-cache pause (the UART interrupt is not IRAM-resident). Longer lines +lose their middle there; shorter ones just wait. + +Usage: + scripts/sideload_app.py --port /dev/cu.wchusbserial10 deploy/apps/gpscompass/1.0 + scripts/sideload_app.py --port ... --reboot deploy/apps/gpscompass/1.0/gpscompass.lua + scripts/sideload_app.py --port ... --dest /lang deploy/apps/lang/11/de.lang + +Pass an app VERSION directory to send its .lua + .json, or individual +files. --reboot restarts the device afterwards so the drawer/Store rescan +picks the new app up (opening the Store page also rescans). + +Needs pyserial: python3 -m pip install pyserial +(or: uvx --from platformio --with pyserial python scripts/sideload_app.py ...) +""" +import argparse +import base64 +import os +import sys +import time + +try: + import serial +except ImportError: + sys.exit("Missing dependency: pyserial (python3 -m pip install pyserial)") + +CHUNK = 72 # bytes per fadd line -> 96 base64 chars; whole line < 128 bytes +RETRIES = 6 # per chunk, on an "Error:" reply or a silent (lost) line + + +def open_port(port, baud): + s = serial.Serial() + s.port, s.baudrate, s.timeout = port, baud, 0.02 + s.dtr = False + s.rts = False # try not to reset the board; some bridges do it anyway + s.open() + # The CH34x bridge on the M9 resets the board on open regardless of DTR/RTS. + # If a boot log starts streaming, wait for the UI to come up ("[BOOT] ui + # ready" — the console is not serviced before that, and the UI init has + # multi-second quiet gaps that a plain silence wait mistakes for "done"). + # No output within 2 s = the board did not reset and is ready now. + t0, seen = time.time(), b"" + while time.time() - t0 < 40: + chunk = s.read(4096) + if chunk: + seen += chunk + if b"[BOOT] ui ready" in seen: + time.sleep(1.5) + break + elif not seen and time.time() - t0 > 2.0: + break + s.reset_input_buffer() + return s + + +class DeviceError(RuntimeError): + pass + + +def command(s, line, expect, timeout=4.0): # the loop stalls for ~0.5 s on GPS work + """Send one CLI line, return the reply line that starts with `expect`. + The console echoes what it receives, so skip the echo and any log noise. + Raises DeviceError on an "Error:" reply, TimeoutError when nothing came.""" + s.write((line + "\n").encode()) + s.flush() + buf = b"" + t0 = time.time() + while time.time() - t0 < timeout: + buf += s.read(4096) + for raw in buf.split(b"\n"): + text = raw.decode("utf-8", "replace").strip("\r ") + if text.startswith(expect): + return text + if text.startswith("Error:"): + raise DeviceError(text) + raise TimeoutError("no reply to: %s" % line[:40]) + + +def push(s, local_path, remote_path): + data = open(local_path, "rb").read() + command(s, "fput " + remote_path, "ok fput") + off, retries = 0, 0 + while off < len(data): + piece = data[off:off + CHUNK] + line = "fadd %d %d %d %s" % (off, len(piece), sum(piece) & 0xFFFF, base64.b64encode(piece).decode()) + try: + reply = command(s, line, "ok fadd") + except DeviceError as e: + msg = str(e) + if msg.startswith("Error: offset "): + off = int(msg.split()[-1]) # device tells us where it really is + retries += 1 + if retries > RETRIES: + raise RuntimeError("giving up at offset %d: %s" % (off, msg)) + continue + except TimeoutError: + # a lost line leaves the device waiting for its terminator; send one + # (an empty line is ignored by the console) and ask where it is + s.write(b"\n") + retries += 1 + if retries > RETRIES: + raise RuntimeError("no reply from the device at offset %d" % off) + continue + retries = 0 + off = int(reply.split()[-1]) + if sys.stdout.isatty(): + sys.stdout.write("\r %s: %d / %d bytes" % (remote_path, off, len(data))) + sys.stdout.flush() + reply = command(s, "fend", "ok fend") + total = int(reply.split()[2]) + if sys.stdout.isatty(): + print() + if total != len(data): + raise RuntimeError("device wrote %d bytes, expected %d" % (total, len(data))) + print(" done: %s (%d bytes)" % (remote_path, total)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("paths", nargs="+", help="app version directory, or individual files") + ap.add_argument("--port", required=True) + ap.add_argument("--baud", type=int, default=115200) + ap.add_argument("--dest", default="/apps", help="/apps (default) or /lang") + ap.add_argument("--reboot", action="store_true", help="reboot the device afterwards") + args = ap.parse_args() + + files = [] + for p in args.paths: + if os.path.isdir(p): + app_id = os.path.basename(os.path.dirname(os.path.abspath(p).rstrip("/"))) + for ext in ("lua", "json"): + f = os.path.join(p, "%s.%s" % (app_id, ext)) + if os.path.isfile(f): + files.append(f) + if not files: + sys.exit("no %s.lua in %s (pass deploy/apps//)" % (app_id, p)) + elif os.path.isfile(p): + files.append(p) + else: + sys.exit("not found: " + p) + + s = open_port(args.port, args.baud) + try: + for f in files: + push(s, f, "%s/%s" % (args.dest.rstrip("/"), os.path.basename(f))) + if args.reboot: + s.write(b"reboot\n") + s.flush() + print("rebooting the device") + finally: + s.close() + + +if __name__ == "__main__": + main() diff --git a/src/DataStore.cpp b/src/DataStore.cpp index 1c51ed3c..4919f10e 100644 --- a/src/DataStore.cpp +++ b/src/DataStore.cpp @@ -225,6 +225,11 @@ bool DataStore::fileExists(FILESYSTEM* fs, const char* filename) { return fs->exists(_rp(filename)); } +bool DataStore::mkdirRooted(FILESYSTEM* fs, const char* dir) { + const char* p = _rp(dir); + return fs->exists(p) || fs->mkdir(p); +} + bool DataStore::removeRooted(FILESYSTEM* fs, const char* filename) { return fs->remove(_rp(filename)); } diff --git a/src/DataStore.h b/src/DataStore.h index b999bb84..0a547c35 100644 --- a/src/DataStore.h +++ b/src/DataStore.h @@ -108,6 +108,7 @@ public: FILESYSTEM* getHotDataFS() const { return _getContactsChannelsFS(); } File openAppend(FILESYSTEM* fs, const char* filename); // create if missing bool fileExists(FILESYSTEM* fs, const char* filename); + bool mkdirRooted(FILESYSTEM* fs, const char* dir); // true if it exists afterwards bool removeRooted(FILESYSTEM* fs, const char* filename); bool renameFile(FILESYSTEM* fs, const char* from, const char* to); #endif diff --git a/src/MyMesh.cpp b/src/MyMesh.cpp index 25391828..9e698f2d 100644 --- a/src/MyMesh.cpp +++ b/src/MyMesh.cpp @@ -8,6 +8,9 @@ #include // derive a region's flood-scope key from its #hashtag name #include // gmtime_r for the "clock" CLI command #ifdef ESP32 +#include // "fadd" serial-sideload chunks +#endif +#ifdef ESP32 #include // esp_restart for the "bootloader" CLI command #if !defined(HAS_TANMATSU) && !defined(HAS_TDISPLAY_P4) #include // RTC_CNTL_OPTION1_REG / FORCE_DOWNLOAD_BOOT (S3-only; both P4 boards lack it) @@ -5357,6 +5360,21 @@ void MyMesh::checkCLIRescueCmd() { } +#if defined(ESP32) + } else if (memcmp(cli_command, "fput ", 5) == 0) { + // Serial sideload (scripts/sideload_app.py). Boards with a soldered card + // (the ThinkNode M9) have no other way to get a Lua app or a .lang file + // onto the device than the Store's own download, so: "fput /apps/" + // opens the file, "fadd " lines append to it + // (self-checking, see cliPutChunk), "fend" closes it. + // Same physical-access trust level as "rm" and "erase" above, with a + // NARROWER scope: only the two sideload directories the Store writes. + cliPutBegin(&cli_command[5]); + } else if (memcmp(cli_command, "fadd ", 5) == 0) { + cliPutChunk(&cli_command[5]); + } else if (strcmp(cli_command, "fend") == 0) { + cliPutEnd(); +#endif } else if (strcmp(cli_command, "reboot") == 0) { board.reboot(); // doesn't return } else { @@ -5367,6 +5385,77 @@ void MyMesh::checkCLIRescueCmd() { } } +#if defined(ESP32) +// ---- serial sideload: fput / fadd / fend ----------------------------------- +// Writes go to the same storage the Store installs into (the hot-data FS, +// root-aware: the SD card's /meshcomod on an adopted card, the internal FS +// otherwise). Replies are single lines the host script keys on: "ok ..." or +// "Error: ...". +void MyMesh::cliPutBegin(const char* path) { + if (_cli_put) { _cli_put.close(); _cli_put_len = 0; } + const char* dir = nullptr; + if (memcmp(path, "/apps/", 6) == 0) dir = "/apps"; + else if (memcmp(path, "/lang/", 6) == 0) dir = "/lang"; + if (!dir) { Serial.println("Error: path must be /apps/ or /lang/"); return; } + const char* name = path + 6; + const size_t nlen = strlen(name); + if (nlen == 0 || nlen > 40) { Serial.println("Error: bad file name length"); return; } + for (size_t i = 0; i < nlen; i++) { + const char c = name[i]; + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '.' || c == '_' || c == '-'; + if (!ok) { Serial.println("Error: file name may only use A-Z a-z 0-9 . _ -"); return; } + } + if (name[0] == '.') { Serial.println("Error: file name may not start with '.'"); return; } + FILESYSTEM* fs = _store->getHotDataFS(); + if (!fs) { Serial.println("Error: storage not ready"); return; } + if (!_store->mkdirRooted(fs, dir)) { Serial.printf("Error: cannot create %s\n", dir); return; } + _cli_put = _store->openWrite(fs, path); + if (!_cli_put) { Serial.printf("Error: cannot open %s for writing\n", path); return; } + _cli_put_len = 0; + Serial.printf("ok fput %s\n", path); +} + +// "fadd ". Every field is checked before a byte +// is written: the UART ISR is not IRAM-resident, so whenever the loop is +// inside a flash-cache pause the hardware FIFO (128 bytes) is all that +// buffers the line and the middle of a long one simply vanishes — a bare +// base64 payload would then decode to plausible garbage. Offset catches a +// lost or duplicated line, len catches truncation, the byte sum catches +// corruption; the host re-sends on any "Error:" reply. +void MyMesh::cliPutChunk(const char* args) { + if (!_cli_put) { Serial.println("Error: no file open (fput first)"); return; } + unsigned off = 0, len = 0, sum = 0; int consumed = 0; + if (sscanf(args, "%u %u %u %n", &off, &len, &sum, &consumed) != 3 || consumed == 0) { + Serial.println("Error: usage fadd "); + return; + } + if (off != _cli_put_len) { Serial.printf("Error: offset %u\n", (unsigned)_cli_put_len); return; } + const char* b64 = args + consumed; + uint8_t buf[160]; + size_t olen = 0; + const int rc = mbedtls_base64_decode(buf, sizeof buf, &olen, (const unsigned char*)b64, strlen(b64)); + if (rc != 0 || olen != len) { Serial.printf("Error: chunk damaged at %u\n", (unsigned)_cli_put_len); return; } + unsigned s = 0; + for (size_t i = 0; i < olen; i++) s += buf[i]; + if ((s & 0xFFFF) != (sum & 0xFFFF)) { Serial.printf("Error: checksum at %u\n", (unsigned)_cli_put_len); return; } + if (olen && _cli_put.write(buf, olen) != olen) { + Serial.println("Error: write failed (card full?)"); + _cli_put.close(); _cli_put_len = 0; + return; + } + _cli_put_len += olen; + Serial.printf("ok fadd %u\n", (unsigned)_cli_put_len); +} + +void MyMesh::cliPutEnd() { + if (!_cli_put) { Serial.println("Error: no file open (fput first)"); return; } + _cli_put.close(); + Serial.printf("ok fend %u bytes\n", (unsigned)_cli_put_len); + _cli_put_len = 0; +} +#endif // ESP32 + void MyMesh::checkSerialInterface() { bool handled_cmd = false; // Drain a small burst of inbound frames each loop to reduce sync latency under load. diff --git a/src/MyMesh.h b/src/MyMesh.h index 3f8dc57b..6e0365ee 100644 --- a/src/MyMesh.h +++ b/src/MyMesh.h @@ -1214,7 +1214,17 @@ private: bool _cli_rescue; bool send_unscoped; // force un-scoped flood (instead of using send_scope) bool scope_direct_floods = false; // opt-in (#64): tag direct/login/admin floods with the default region scope - char cli_command[80]; + // 200 not 80: the serial sideload ("fadd ...", see cliPutChunk) carries up + // to ~130 chars per line; every other command is far shorter. + char cli_command[200]; +#if defined(ESP32) + // Serial sideload state ("fput" / "fadd" / "fend"): the file being written. + File _cli_put; + uint32_t _cli_put_len = 0; + void cliPutBegin(const char* path); + void cliPutChunk(const char* args); + void cliPutEnd(); +#endif uint8_t app_target_ver; uint8_t *sign_data; uint32_t sign_data_len; diff --git a/variants/thinknode_m9/M9_PORT.md b/variants/thinknode_m9/M9_PORT.md index 5537df4f..0c062f52 100644 --- a/variants/thinknode_m9/M9_PORT.md +++ b/variants/thinknode_m9/M9_PORT.md @@ -553,12 +553,23 @@ but nothing ever talked to it; the QMI8658 IMU still has no driver. 10, 4000 LSB/G) buys 4× finer counts; not worth it unless `|B|` is noisy. **Hardware-verify recipe:** flash; boot log shows the `M9 compass:` line; -copy `gpscompass.lua` + `gpscompass.json` to the SD card at -`/meshcomod/apps/` (the Store's "Your own apps" section picks them up; the -catalog copy arrives once `deploy/apps` is published); open the app; `C`, -turn the device through every orientation for 20 s; check `|B|`; set O/F -against a known north; walk with GPS on and confirm `Speed`/`Course` populate -above ~1 km/h; pick a contact and sanity-check the bearing against the map. +push the app over the console — the M9's card is soldered on, so +`scripts/sideload_app.py --port /dev/cu.wchusbserial10 --reboot +deploy/apps/gpscompass/1.0` (the `fput`/`fadd`/`fend` CLI commands write into +the Store's own `/apps/` on the card; "Your own apps" lists it, the catalog +copy arrives once `deploy/apps` is published); open the app; `C`, turn the +device through every orientation for 20 s; check `|B|`; set O/F against a +known north; walk with GPS on and confirm `Speed`/`Course` populate above +~1 km/h; pick a contact and sanity-check the bearing against the map. + +Serial-console notes learned doing this: the CH34x bridge resets the board +whenever the port is opened (DTR/RTS, regardless of what pyserial asks), and +the console is not serviced until `[BOOT] ui ready`; the UART interrupt is +not IRAM-resident, so while the loop sits in a flash-cache pause only the +128-byte hardware FIFO buffers input and the middle of a longer line is +lost — hence the sideload's short, self-checking lines. Done on 2026-08-22: +`M9 compass: QMC6309 ok (id=0x90, 50 Hz, +/-32 G, OSR 8/8)` on the first +flash (0x7C answers, config sticks); app files pushed and listed by `ls /apps`. ## Deferred — hardware-verify list