From 8e4ca507475972dcdf8b6bf01dc2dc02b84ffe76 Mon Sep 17 00:00:00 2001 From: liquidraver <504870+liquidraver@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:45:58 +0200 Subject: [PATCH] meshamerica flasher v1 --- .github/workflows/build.yml | 76 ++++++++- PROVIDER_CATALOG.md | 78 +++++++++ gen_provider_catalog.py | 320 ++++++++++++++++++++++++++++++++++++ 3 files changed, 473 insertions(+), 1 deletion(-) create mode 100644 PROVIDER_CATALOG.md create mode 100644 gen_provider_catalog.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6eb3f6f..ff039f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -222,6 +222,8 @@ jobs: create-release: needs: [build-nrf, build-esp32-companions, build-esp32-repeaters, build-linux] runs-on: ubuntu-latest + outputs: + release_tag: ${{ steps.tag.outputs.release_tag }} steps: - name: generate release tag @@ -265,4 +267,76 @@ jobs: name: Firmware ${{ steps.tag.outputs.release_tag }} body: "" draft: false - files: firmware/* \ No newline at end of file + files: firmware/* + + # Publish firmware + provider catalog for the Mesh America configurator + # (apps.meshamerica.com). GitHub Release assets have no CORS header, so the + # browser flasher cannot fetch them; instead we force-push the firmware and a + # generated catalog.json to the `firmware-dist` orphan branch, served from + # CORS-clean hosts (firmware via jsDelivr, catalog via raw.githubusercontent). + # The stable catalog URL registered with Mesh America is: + # https://raw.githubusercontent.com///firmware-dist/catalog.json + publish-catalog: + needs: create-release + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.python_version }}' + + # Pull every board's firmware back into a single firmware/ directory. + - uses: actions/cache/restore@v5 + with: + path: firmware + key: cache-firmware-nrf-${{ github.sha }} + - uses: actions/cache/restore@v5 + with: + path: firmware + key: cache-firmware-esp32-companions-${{ github.sha }} + - uses: actions/cache/restore@v5 + with: + path: firmware + key: cache-firmware-esp32-repeaters-${{ github.sha }} + - uses: actions/cache/restore@v5 + with: + path: firmware + key: cache-firmware-linux-${{ github.sha }} + + - name: generate provider catalog + run: | + python gen_provider_catalog.py --firmware-dir firmware \ + --url-base "https://cdn.jsdelivr.net/gh/${{ github.repository }}@firmware-dist" \ + --version "${{ needs.create-release.outputs.release_tag }}" \ + --out catalog.json + + - name: publish firmware + catalog to firmware-dist branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tag="${{ needs.create-release.outputs.release_tag }}" + work="$(mktemp -d)" + cp catalog.json "$work/" + cp firmware/* "$work/" + { + echo "# ZephCore firmware-dist" + echo + echo "Auto-generated distribution branch for the Mesh America device" + echo "configurator. Force-pushed by the \`build firmware\` workflow each" + echo "release -- do not edit by hand." + echo + echo "- Catalog (register this URL): https://raw.githubusercontent.com/${{ github.repository }}/firmware-dist/catalog.json" + echo "- Firmware (jsDelivr): https://cdn.jsdelivr.net/gh/${{ github.repository }}@firmware-dist/" + echo "- Built from release: $tag" + } > "$work/README.md" + cd "$work" + git init -q -b firmware-dist + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -q -m "firmware-dist: $tag" + git push -f "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" firmware-dist \ No newline at end of file diff --git a/PROVIDER_CATALOG.md b/PROVIDER_CATALOG.md new file mode 100644 index 0000000..a606db4 --- /dev/null +++ b/PROVIDER_CATALOG.md @@ -0,0 +1,78 @@ +# Mesh America configurator — provider catalog + +ZephCore firmware is published to the [Mesh America Device Configurator](https://apps.meshamerica.com) +as a third-party provider catalog. Users pick a board in the app and flash ZephCore +directly from the browser. + +## What to send Mesh America + +``` +Provider name: ZephCore +Catalog URL: https://raw.githubusercontent.com/liquidraver/ZephCore/firmware-dist/catalog.json +``` + +(Send this once, after the first `master` build has run and created the `firmware-dist` +branch. Validate first at .) + +## How it works + +`gen_provider_catalog.py` turns a release's firmware assets into `catalog.json` +(the provider-spec schema: `version -> {notes, files:[{type,name,url}]}` with +absolute HTTPS URLs). The `build firmware` workflow runs it in the +`publish-catalog` job and force-pushes the firmware + catalog to the +`firmware-dist` orphan branch on every `master` build. + +**Hosting / CORS.** The configurator runs in the browser and `fetch()`es both the +catalog and each firmware binary, so every response needs +`Access-Control-Allow-Origin: *`. GitHub **Release assets do not** send it (they +redirect to `release-assets.githubusercontent.com`, which omits the header), so we +cannot serve firmware from the normal release. Instead: + +| Artifact | Host | Why | +|----------|------|-----| +| `catalog.json` | `raw.githubusercontent.com/.../firmware-dist/catalog.json` | CORS `*`, always fresh (raw doesn't cache branch refs) | +| firmware binaries | `cdn.jsdelivr.net/gh/liquidraver/ZephCore@firmware-dist/` | CORS `*`, CDN-backed; filenames carry the git hash so they're effectively immutable | + +The normal timestamped GitHub Release (with all the same binaries) is still created +for manual downloads and OTA — the configurator pipeline is additive. + +## Device mapping + +Boards whose `name` matches an official MeshCore device **fold** into that device +(our firmware shows as extra options under the existing tile). Boards MeshCore +doesn't have become **new** ZephCore-badged tiles. The mapping table lives in +`BOARDS` in `gen_provider_catalog.py`; adjust it there when boards are added or +renamed. Names must match MeshCore's canonical `config.json` +(`meshcore-dev/flasher.meshcore.io`) **exactly** to fold — the validator flags +mismatches. + +Notes on the mapping: +- **ESP32 offers `flash-wipe` only — because of an offset mismatch, not flashability.** + Our `-update.bin` (the MCUboot-signed app) *is* serial-flashable, but only at ZephCore's + app-slot offset `0x20000` (`slot0_partition: partition@20000` in every ESP32 overlay; + MCUboot itself owns `0x0`–`0x20000`). Mesh America's `flash-update` hardcodes `0x10000` + (MeshCore's slot layout) and the catalog schema has no offset override, so it would write + the app inside our MCUboot partition and the board wouldn't boot. `flash-wipe` (full merged + image at `0x0`) is offset-independent and always works. Enabling `flash-update` would mean + moving our MCUboot partition to `0x10000` to match MeshCore — out of scope. +- **Companion firmware** is one image serving BLE + USB; it's listed under both + `companionBle` and `companionUsb` (same file). Native-Linux companions use a + custom `companionTcp` role. +- **Heltec T114** lists screen and "No screen" variants (subTitle) under one tile. +- **Native-Linux** boards (femtofox, rak6421) are `noflash` download-only tiles. +- **Thumbnails:** folded devices inherit MeshCore's image automatically (device-level + fields come from the official entry on merge), so only the new tiles set a `tooltip`. + Those reuse the configurator's own image set via jsDelivr (`IMG_BASE` in the script); + boards without dedicated art fall back to a neutral LoRa icon. Swap in ZephCore-branded + art by pointing `img` at your own absolute HTTPS URLs. + +## Local test + +``` +python gen_provider_catalog.py --firmware-dir firmware \ + --url-base "https://cdn.jsdelivr.net/gh/liquidraver/ZephCore@firmware-dist" \ + --version v0.0.0 --out catalog.json +``` + +It validates the output against the spec's hard requirements and prints a +fold/new summary; a non-zero exit means a schema error. diff --git a/gen_provider_catalog.py b/gen_provider_catalog.py new file mode 100644 index 0000000..bf47b3c --- /dev/null +++ b/gen_provider_catalog.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Generate the Mesh America configurator provider catalog for ZephCore. + +The Mesh America Device Configurator (apps.meshamerica.com) merges third-party +"provider catalogs" with the official MeshCore catalog. This script emits our +catalog.json from a release's firmware assets, following the provider API spec +(the `version -> {notes, files:[{type,name,url}]}` shape with absolute HTTPS +URLs -- NOT MeshCore's internal `github`+regex+staticPath mechanism). + +Hosting note: GitHub *Release assets* do NOT return Access-Control-Allow-Origin +(they redirect to release-assets.githubusercontent.com, which omits the header), +so the browser-based configurator cannot fetch them. Firmware + this catalog are +instead published to a force-pushed `firmware-dist` orphan branch and served from +CORS-clean hosts: firmware via jsDelivr (`--url-base`), the catalog itself via +raw.githubusercontent.com. `--url-base` is prepended to each firmware filename. + +Usage (CI, scanning the built firmware/ dir): + python gen_provider_catalog.py --firmware-dir firmware \ + --url-base "https://cdn.jsdelivr.net/gh/liquidraver/ZephCore@firmware-dist" \ + --version v20260704.220543 --out catalog.json + +Usage (local test, from a GitHub release API json): + python gen_provider_catalog.py --assets-json rel.json \ + --url-base "https://cdn.jsdelivr.net/gh/liquidraver/ZephCore@firmware-dist" \ + --version v20260704.220543 --out catalog.json +""" + +import argparse +import io +import json +import os +import re +import sys + +# --------------------------------------------------------------------------- +# Provider identity +# --------------------------------------------------------------------------- + +MAKER_KEY = "zephcore" +MAKER_NAME = "ZephCore" +DEVICE_CLASS = "zephcore" + +DESCRIPTION = ( + "ZephCore is an independent, ground-up implementation of the MeshCore " + "protocol on the Zephyr RTOS. It is wire-compatible with stock MeshCore " + "and adds features such as an adaptive contention window, LoRa RX " + "duty-cycling, and support for additional boards. Companion (BLE + USB), " + "Repeater, and native-Linux builds are provided." +) + +# Device thumbnails for our NEW tiles only (folded devices inherit MeshCore's on +# merge). Reused from the configurator's own image set via jsDelivr -- a stable, +# CORS-clean absolute URL. `` display isn't CORS-restricted, but the spec +# asks for absolute URLs. Boards without dedicated art use the neutral lora icon. +IMG_BASE = "https://cdn.jsdelivr.net/gh/meshcore-dev/flasher.meshcore.io@main/img" + +# A single custom role for the native-Linux companion, which speaks TCP rather +# than BLE/USB. Custom roles render with a plain label and never map to the +# in-app Configure step -- that is fine for these download-only entries. +CUSTOM_ROLES = { + "companionTcp": { + "icon": "lan", + "title": "Companion", + "subTitle": "TCP", + "tooltip": "Native-Linux companion radio -- connect over TCP.", + } +} + +# --------------------------------------------------------------------------- +# Board -> device mapping +# +# `stem` : the exact filename stem build.sh emits (board_clean_for_path). +# `kind` : nrf | esp32 | linux -- drives device type, file types, extensions. +# `device` : the MeshCore-canonical device name to FOLD into, or a new tile name. +# `new` : True if this device is not in the official catalog (its own tile). +# `variant`: optional filename infix (e.g. "noscreen") + shown as subTitle. +# `subtitle`: optional role-row subTitle to disambiguate hardware variants. +# --------------------------------------------------------------------------- + +BOARDS = [ + # --- nRF52: fold into existing MeshCore devices ----------------------- + dict(stem="rak4631", kind="nrf", device="RAK WisBlock / WisMesh (RAK 4631)"), + dict(stem="rak3401_1watt", kind="nrf", device="RAK WisMesh 1W Booster (3401 + 13302)"), + dict(stem="wio_tracker_l1", kind="nrf", device="Seeed Studio Wio Tracker L1 Pro"), + dict(stem="t1000_e", kind="nrf", device="Seeed Studio SenseCAP T1000-E"), + dict(stem="thinknode_m1", kind="nrf", device="Elecrow ThinkNode M1"), + dict(stem="thinknode_m3", kind="nrf", device="Elecrow ThinkNode M3"), + dict(stem="thinknode_m6", kind="nrf", device="Elecrow ThinkNode M6"), + dict(stem="rak_wismesh_tag", kind="nrf", device="RAK WisMesh Tag"), + dict(stem="ikoka_nano_30dbm", kind="nrf", device="Ikoka Nano", subtitle="30 dBm"), + dict(stem="sensecap_solar", kind="nrf", device="Seeed Studio SenseCAP Solar"), + dict(stem="xiao_nrf52840", kind="nrf", device="Seeed Studio Xiao nRF52 WIO"), + dict(stem="lilygo_techo", kind="nrf", device="LilyGo T-Echo"), + dict(stem="promicro_sx1262", kind="nrf", device="ProMicro nrf52 (faketec)"), + dict(stem="heltec_t114", kind="nrf", device="Heltec T114"), + dict(stem="heltec_t114", kind="nrf", device="Heltec T114", + variant="noscreen", subtitle="No screen"), + dict(stem="gat562_30s", kind="nrf", device="GAT-IoT GAT562 30s"), + + # --- nRF52: new ZephCore-only hardware (own tile) --------------------- + dict(stem="lilygo_timpulse_plus", kind="nrf", device="LilyGo T-Impulse Plus", new=True, img="lora.svg"), + dict(stem="heltec_t096", kind="nrf", device="Heltec T96", new=True, img="lora.svg"), + + # --- ESP32 (sysbuild/MCUboot, -merged.bin): fold --------------------- + dict(stem="xiao_esp32c3", kind="esp32", device="Seeed Studio Xiao C3"), + dict(stem="xiao_esp32s3-esp32s3-procpu", kind="esp32", device="Seeed Studio Xiao S3 WIO"), + dict(stem="station_g2-esp32s3-procpu", kind="esp32", device="UnitEng Station G2"), + dict(stem="heltec_wifi_lora32_v3-esp32s3-procpu", kind="esp32", device="Heltec v3"), + dict(stem="heltec_wifi_lora32_v4-esp32s3-procpu", kind="esp32", device="Heltec v4"), + dict(stem="heltec_wireless_tracker-esp32s3-procpu", kind="esp32", device="Heltec Wireless Tracker"), + dict(stem="heltec_wireless_tracker_v2-esp32s3-procpu", kind="esp32", device="Heltec Wireless Tracker v2"), + # Classic ESP32 T-Beam: ships -merged.bin (full-flash, 0x0) too. + dict(stem="ttgo_tbeam-esp32-procpu", kind="esp32", device="LilyGo T-Beam (SX1262)"), + + # --- ESP32: new ZephCore-only hardware (own tile) -------------------- + dict(stem="xiao_esp32c6-esp32c6-hpcore", kind="esp32", device="Seeed Studio Xiao ESP32-C6", new=True, img="xiao_esp32c6.svg"), + dict(stem="lilygo_tlora_c6-esp32c6-hpcore", kind="esp32", device="LilyGo T-Lora C6", new=True, img="lilygo_tlora_c6.svg"), + dict(stem="heltec_wifi_lora32_v43-esp32s3-procpu", kind="esp32", device="Heltec v4.3", new=True, img="heltec_v4.svg"), + + # --- Native Linux (noflash, download only): new tiles ---------------- + dict(stem="zephcore_linux_femtofox", kind="linux", device="Femtofox (Luckfox Pico Mini)", new=True, img="lora.svg"), + dict(stem="zephcore_linux_rak6421", kind="linux", device="RAK6421 WisMesh (Raspberry Pi)", new=True, img="rpi.svg"), + dict(stem="zephcore_linux_rak6421_pi5", kind="linux", device="RAK6421 WisMesh (Raspberry Pi 5)", new=True, img="rpi.svg"), +] + +DEVICE_TYPE = {"nrf": "nrf52", "esp32": "esp32", "linux": "noflash"} + +# Companion firmware is a single image that serves both transports (and, on +# Linux, TCP). Expose it under each applicable role, all pointing at one file. +COMPANION_ROLES = { + "nrf": ["companionBle", "companionUsb"], + "esp32": ["companionBle", "companionUsb"], + "linux": ["companionTcp"], +} + +HASH_RE = r"[0-9a-f]{7,40}" + + +def find_file(assets, stem, token, variant, suffix): + """Return the single asset filename for (stem, role token, variant, suffix). + + `suffix` is the trailing part after the git hash, e.g. r"\\.zip", + r"-merged\\.bin", or "" (Linux, no extension). Returns None if absent. + """ + mid = f"-{variant}" if variant else "" + pat = re.compile(rf"^{re.escape(stem)}-{token}{mid}-{HASH_RE}{suffix}$") + matches = sorted(n for n in assets if pat.match(n)) + return matches[0] if matches else None + + +def files_for(assets, board, token): + """Build the spec `files[]` list for one role of one board, or [] if none.""" + stem, kind, variant = board["stem"], board["kind"], board.get("variant") + out = [] + if kind == "nrf": + zip_ = find_file(assets, stem, token, variant, r"\.zip") + uf2 = find_file(assets, stem, token, variant, r"\.uf2") + if zip_: + out.append(("flash", zip_, "DFU package (install or update over USB)")) + if uf2: + out.append(("download", uf2, "UF2 -- drag-and-drop onto the bootloader drive")) + elif kind == "esp32": + merged = find_file(assets, stem, token, variant, r"-merged\.bin") + if merged: + out.append(("flash-wipe", merged, "Full install (bootloader + firmware)")) + elif kind == "linux": + elf = find_file(assets, stem, token, variant, r"") + if elf: + out.append(("download", elf, "Linux executable (download and run)")) + return out + + +def build(assets, url_base, version): + base = url_base.rstrip("/") + devices = {} # name -> device object (first-seen order preserved) + order = [] + used_roles = set() + stats = {"fold": 0, "new": 0, "skipped": []} + + def spec_files(triples): + return [ + {"type": t, "name": name, "url": f"{base}/{name}", "title": title} + for (t, name, title) in triples + ] + + def add_option(dev, role, subtitle, version, triples): + opt = {"role": role, "version": {version: {"notes": "", "files": spec_files(triples)}}} + if subtitle: + opt["subTitle"] = subtitle + dev["firmware"].append(opt) + used_roles.add(role) + + for board in BOARDS: + name = board["device"] + subtitle = board.get("subtitle") + comp = files_for(assets, board, "companion") + repe = files_for(assets, board, "repeater") + if not comp and not repe: + stats["skipped"].append(f"{board['stem']}" + + (f" ({board['variant']})" if board.get("variant") else "")) + continue + + if name not in devices: + dev = { + "maker": MAKER_KEY, + "class": DEVICE_CLASS, + "name": name, + "type": DEVICE_TYPE[board["kind"]], + "firmware": [], + } + if board.get("img"): + dev["tooltip"] = f"" + devices[name] = dev + order.append(name) + stats["new" if board.get("new") else "fold"] += 1 + dev = devices[name] + + if comp: + for role in COMPANION_ROLES[board["kind"]]: + add_option(dev, role, subtitle, version, comp) + if repe: + add_option(dev, "repeater", subtitle, version, repe) + + catalog = { + "description": DESCRIPTION, + "maker": {MAKER_KEY: {"name": MAKER_NAME}}, + "device": [devices[n] for n in order], + } + role_map = {k: v for k, v in CUSTOM_ROLES.items() if k in used_roles} + if role_map: + catalog["role"] = role_map + return catalog, stats + + +def validate(catalog): + """Enforce the spec's hard requirements; return a list of error strings.""" + errs = [] + file_types = {"flash-wipe", "flash-update", "flash", "download"} + dev_types = {"esp32", "nrf52", "noflash"} + makers = set(catalog.get("maker", {})) + for dev in catalog.get("device", []): + tag = f"device {dev.get('name')!r}" + if not dev.get("name"): + errs.append("a device is missing 'name'") + if dev.get("maker") not in makers: + errs.append(f"{tag}: maker {dev.get('maker')!r} not in maker map") + if dev.get("type") not in dev_types: + errs.append(f"{tag}: bad type {dev.get('type')!r}") + fw = dev.get("firmware", []) + if not fw: + errs.append(f"{tag}: no firmware options") + for opt in fw: + if not opt.get("role"): + errs.append(f"{tag}: firmware option missing 'role'") + ver = opt.get("version", {}) + if not ver: + errs.append(f"{tag}: role {opt.get('role')} has no version") + for vname, v in ver.items(): + files = v.get("files", []) + if not files: + errs.append(f"{tag}: version {vname} has no files") + for f in files: + if f.get("type") not in file_types: + errs.append(f"{tag}: file bad type {f.get('type')!r}") + if not f.get("name"): + errs.append(f"{tag}: a file is missing 'name'") + url = f.get("url", "") + if not url.startswith("https://"): + errs.append(f"{tag}: file url not absolute https: {url!r}") + return errs + + +def load_assets(args): + if args.firmware_dir: + return [n for n in os.listdir(args.firmware_dir) + if os.path.isfile(os.path.join(args.firmware_dir, n))] + if args.assets_json: + with io.open(args.assets_json, encoding="utf-8") as fh: + data = json.load(fh) + return [a["name"] for a in data.get("assets", [])] + raise SystemExit("provide --firmware-dir or --assets-json") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--firmware-dir", help="directory of built firmware assets") + ap.add_argument("--assets-json", help="GitHub release API json (for testing)") + ap.add_argument("--url-base", required=True, + help="prepended to each firmware filename, e.g. " + "https://cdn.jsdelivr.net/gh/liquidraver/ZephCore@firmware-dist") + ap.add_argument("--version", required=True, + help="version label shown in the role picker, e.g. the release tag") + ap.add_argument("--out", default="catalog.json") + args = ap.parse_args() + + assets = load_assets(args) + catalog, stats = build(assets, args.url_base, args.version) + errs = validate(catalog) + + devcount = len(catalog["device"]) + optcount = sum(len(d["firmware"]) for d in catalog["device"]) + print(f"assets seen: {len(assets)}") + print(f"devices: {devcount} (fold={stats['fold']}, new={stats['new']})") + print(f"firmware options: {optcount}") + if stats["skipped"]: + print("skipped (no assets found): " + ", ".join(stats["skipped"])) + if errs: + print("\nVALIDATION ERRORS:", file=sys.stderr) + for e in errs: + print(" - " + e, file=sys.stderr) + raise SystemExit(1) + + with io.open(args.out, "w", encoding="utf-8", newline="\n") as fh: + json.dump(catalog, fh, indent=2, ensure_ascii=False) + fh.write("\n") + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main()