diff --git a/RELEASE.md b/RELEASE.md index a4a19495a..967334385 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,5 +1,10 @@ # Releasing Firmware +For the USA Cascade 1.17.1.5 matrix, use +[the option 3 release instructions](docs/releases/1.17.1.5.md). +The [Full Companion feature guide](docs/full_companion_features.md) is intended +to accompany the release assets. + GitHub Actions is set up to automatically build and release firmware. It will automatically build firmware when one of the following tag formats are pushed. diff --git a/build.sh b/build.sh index 7377d6bad..0d2df280d 100755 --- a/build.sh +++ b/build.sh @@ -56,6 +56,8 @@ PIO_BUILD_JOBS_OVERRIDE="" PIO_BUILD_DIR_OVERRIDE="" RESOLVED_BUILD_TARGETS=() RESUME_BUILD_OUTPUT="${RESUME_BUILD_OUTPUT:-0}" +REQUIRE_OTA_UPDATES="${REQUIRE_OTA_UPDATES:-}" +OTA_EXCLUDED_TARGETS=() LOGGING_MATRIX_FAILURES=() LOGGING_MATRIX_DEFERRED_TARGETS=() RADIO_PRESET_SELECTION="" @@ -140,6 +142,10 @@ Options: --auto|--standard|--full: Short forms of --build-profile. --skip-kiss|--include-kiss: Exclude (default) or include KISS modem targets in bulk builds. --clean|--resume: Clean output or resume existing Option 3/FULL-only artifacts. + --require-ota: Require a verified wireless self-update path for infrastructure. + Default for Option 3; Companion USB updates remain supported. + Cable-only infrastructure platforms are reported and omitted. + --allow-no-ota: Include cable-only development targets instead. --background: Run the selected build as a persistent detached job. Its log and status live outside OUTPUT_DIR so --clean cannot erase them. @@ -1000,6 +1006,14 @@ parse_cli_options() { BUILD_BACKGROUND_EXPLICIT=1 shift ;; + --require-ota) + REQUIRE_OTA_UPDATES=1 + shift + ;; + --allow-no-ota) + REQUIRE_OTA_UPDATES=0 + shift + ;; --) shift positional+=("$@") @@ -2841,6 +2855,26 @@ declare_build_capability_contract() { record_build_capability "profile.${BUILD_PROFILE_FOR_TARGET}" + if [ "$env_platform" = "NRF52_PLATFORM" ] && ! is_kiss_modem_target "$env_name"; then + # Prove the real DFU service is linked, not merely a generic OTA CLI stub. + record_build_expectation "ota.update.bluetooth" "_ZN6BLEDfu5beginEv" + fi + + if [ "$env_platform" = "ESP32_PLATFORM" ]; then + if is_companion_radio_full_target "$env_name" \ + || { [ "$BUILD_PROFILE_FOR_TARGET" = "standard" ] \ + && requires_esp32_field_browser_ota "$env_name"; }; then + record_build_expectation "ota.update.wifi" "MeshCore firmware update" + elif ! is_companion_build "$env_name" \ + && [ "${PIO_ENV_FULL_WIFI_OTA_BY_NAME[$env_name]:-0}" = "1" ] \ + && [ "$BUILD_PROFILE_FOR_TARGET" = "full" ]; then + record_build_expectation "ota.update.wifi" "Started: http://%s/update" + fi + if is_lora_ota_build "$env_name" && ! is_companion_radio_full_target "$env_name"; then + record_build_expectation "ota.update.lora" "image_hash MISMATCH after decode" + fi + fi + if is_repeater_role_target "$env_name" \ || is_room_server_role_target "$env_name"; then record_build_expectation "cli.retry_preset" "retry.preset" @@ -2905,6 +2939,8 @@ declare_build_capability_contract() { "Companion: starting Bluetooth" record_build_expectation "companion.usb_logging" "get usb.logging" record_build_expectation "companion.usb_mota_source" "ota folder on" + record_build_expectation "companion.mota_sender" \ + "_ZN4mesh3ota16SerialMotaSource4read" if is_nrf52_companion_radio_full_target "$env_name"; then record_build_expectation "companion.dedicated_usb_logging" \ "get usb.logging" @@ -3429,6 +3465,8 @@ apply_companion_radio_full_profile() { # layout explicitly so Full-only runtime policies, including the bounded # first-boot WebConfig window, cannot silently compile with legacy defaults. export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DWIFI_OTA_SEEDER=1 -DCOMPANION_FEATURE_USB_MOTA_SOURCE=1 -DCOMPANION_FEATURE_NETWORK_TERMINAL=1 -DCOMPANION_FEATURE_MEMORY_DIAGNOSTICS=1 -DMESHCORE_EXPANDED_PARTITION_PROFILE=1" + append_platformio_build_unflags "-DDISABLE_WIFI_OTA=1 -fexceptions" + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -UDISABLE_WIFI_OTA -DLIGHTWEIGHT_WIFI_OTA=1 -fno-exceptions" # Both Indicator radio layouts share the same original NVS/SPIFFS data # placement. The LoRa WiFi base names that map directly; the ESP-NOW USB @@ -3626,6 +3664,16 @@ write_build_capability_manifest() { ) local item + if [ "${REQUIRE_OTA_UPDATES:-0}" = "1" ] && ! is_companion_build "$env_name"; then + checker_args+=(--require-ota) + fi + if [ "$env_platform" = "ESP32_PLATFORM" ]; then + checker_args+=(--firmware-bin "${build_output_dir}/firmware.bin" + --partitions "${build_output_dir}/partitions.bin") + elif [ "$env_platform" = "NRF52_PLATFORM" ]; then + checker_args+=(--dfu-package "${build_output_dir}/firmware.zip") + fi + for item in "${BUILD_CAPABILITIES[@]}"; do checker_args+=(--capability "$item") done @@ -3722,6 +3770,16 @@ build_artifacts_exist() { output_artifact_exists "${firmware_filename}.capabilities.json" || return 1 grep -q '"verified": true' \ "${OUTPUT_DIR}/${firmware_filename}.capabilities.json" || return 1 + if [ "${REQUIRE_OTA_UPDATES:-0}" = "1" ]; then + python3 - "${OUTPUT_DIR}/${firmware_filename}.capabilities.json" <<'PY' || return 1 +import json, sys +manifest = json.load(open(sys.argv[1])) +target = manifest.get("target", "").lower() +companion = "companion" in target or "comp_radio" in target +sys.exit(0 if manifest.get("schema_version", 0) >= 2 and + (companion or manifest.get("ota_update_verified")) else 1) +PY + fi case "$env_platform" in ESP32_PLATFORM) @@ -3729,7 +3787,9 @@ build_artifacts_exist() { && output_artifact_exists "${firmware_filename}-merged.bin" ;; NRF52_PLATFORM) - output_artifact_exists "${firmware_filename}.uf2" + output_artifact_exists "${firmware_filename}.uf2" \ + && { [ "${REQUIRE_OTA_UPDATES:-0}" != "1" ] \ + || output_artifact_exists "${firmware_filename}.zip"; } ;; STM32_PLATFORM) output_artifact_exists "${firmware_filename}.bin" \ @@ -4665,6 +4725,33 @@ resolve_command_targets() { fi fi + OTA_EXCLUDED_TARGETS=() + if [ -z "$REQUIRE_OTA_UPDATES" ]; then + REQUIRE_OTA_UPDATES=0 + if is_logging_matrix_command "$1"; then REQUIRE_OTA_UPDATES=1; fi + fi + if [ "$REQUIRE_OTA_UPDATES" = "1" ]; then + local ota_targets=() + for target in "${RESOLVED_BUILD_TARGETS[@]}"; do + if is_companion_build "$target"; then + ota_targets+=("$target") + continue + fi + case "${PIO_ENV_PLATFORM_BY_NAME[$target]:-}" in + ESP32_PLATFORM|NRF52_PLATFORM) ota_targets+=("$target") ;; + *) OTA_EXCLUDED_TARGETS+=("$target") ;; + esac + done + if [ ${#OTA_EXCLUDED_TARGETS[@]} -gt 0 ]; then + echo "No wireless updater exists for these cable-only targets:" + printf ' %s\n' "${OTA_EXCLUDED_TARGETS[@]}" + if [ "$1" = "build-firmware" ]; then return 1; fi + fi + RESOLVED_BUILD_TARGETS=("${ota_targets[@]}") + [ ${#RESOLVED_BUILD_TARGETS[@]} -gt 0 ] || return 1 + echo "OTA required: infrastructure must prove a WiFi, Bluetooth, or LoRa self-update path; Companions may update over USB." + fi + # Keep one queue so parallel workers stay saturated. The scheduler may pull # a later target forward when a generated alias shares an active PlatformIO # base environment, so this is a best-effort start order rather than a phase @@ -4673,7 +4760,7 @@ resolve_command_targets() { mapfile -t RESOLVED_BUILD_TARGETS < <( sort_build_targets_by_platform_and_name "${RESOLVED_BUILD_TARGETS[@]}" ) - echo "Bulk target start order: nRF52, ESP32, RP2040, STM32; alphabetical within each platform (best effort with parallel workers)." + echo "Bulk target order: nRF52, ESP32, RP2040, STM32; alphabetical within each platform, one PlatformIO process at a time." fi } @@ -5247,6 +5334,9 @@ run_logging_matrix_build_targets() { fi LOGGING_MATRIX_FAILURES=() LOGGING_MATRIX_DEFERRED_TARGETS=() + if [ "${REQUIRE_OTA_UPDATES:-0}" = "1" ]; then + printf '%s\n' "${OTA_EXCLUDED_TARGETS[@]}" > "${OUTPUT_DIR}/ota-excluded-targets.txt" + fi PROFILE_BUILD_WORKERS=$OPTION3_BUILD_WORKERS echo "Option 3 PlatformIO policy: one target build at a time, ${OPTION3_PIO_JOBS} compiler job(s) inside that process." @@ -5531,6 +5621,14 @@ write_background_build_status() { printf 'exit_code=%s\n' "$exit_code" fi printf 'working_directory=%s\n' "$PWD" + printf 'output_directory=%s\n' "$OUTPUT_DIR" + printf 'source_commit=%s\n' "$(git rev-parse HEAD)" + printf 'firmware_version=%s\n' "${FIRMWARE_VERSION:-}" + printf 'firmware_profile=%s\n' "$FIRMWARE_PROFILE_OVERRIDE" + printf 'radio_frequency=%s\n' "$RADIO_FREQ_OVERRIDE" + printf 'radio_bandwidth=%s\n' "$RADIO_BW_OVERRIDE" + printf 'radio_sf=%s\n' "$RADIO_SF_OVERRIDE" + printf 'radio_cr=%s\n' "$RADIO_CR_OVERRIDE" printf 'log=%s\n' "$BUILD_BACKGROUND_LOG_FILE" } > "$status_tmp" mv -f -- "$status_tmp" "$BUILD_BACKGROUND_STATUS_FILE" @@ -5632,7 +5730,7 @@ launch_background_build() { MESHDEBUG_OVERRIDE PACKET_LOGGING_OVERRIDE MQTT_BRIDGE_OVERRIDE MQTT_DEBUG_OVERRIDE RADIO_SETTING_TITLE RADIO_FREQ_OVERRIDE RADIO_BW_OVERRIDE RADIO_SF_OVERRIDE RADIO_CR_OVERRIDE - RESUME_BUILD_OUTPUT KISS_MODE_OVERRIDE + RESUME_BUILD_OUTPUT KISS_MODE_OVERRIDE REQUIRE_OTA_UPDATES SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE GIT_CONFIG_GLOBAL ) diff --git a/docs/companion_radio_full.md b/docs/companion_radio_full.md index e8a8659e2..8d5dd61f0 100644 --- a/docs/companion_radio_full.md +++ b/docs/companion_radio_full.md @@ -1,5 +1,8 @@ # Full Companion +For practical on/off commands and update directions, see +[Full Companion feature switches](full_companion_features.md). + `companion_radio_full` combines every qualified Companion transport for its platform into one artifact and acts as a host-backed LoRa mOTA source for updating other nodes. Most targets can run those transports simultaneously; @@ -21,6 +24,7 @@ firmware as an mOTA image. | Hardware serial Companion | On targets with assigned serial pins | On targets with assigned serial pins | | Ethernet Companion | On targets with an Ethernet module | On RAK4631 with RAK13800 | | LoRa self-update | No | No | +| Optional self-update | WiFi on existing dual-app layouts; USB on single-app layouts | Bluetooth DFU with a compatible bootloader; USB always supported | ### SenseCAP Indicator transport and rendering exception diff --git a/docs/full_companion_features.md b/docs/full_companion_features.md new file mode 100644 index 000000000..9fca32f90 --- /dev/null +++ b/docs/full_companion_features.md @@ -0,0 +1,163 @@ +# Full Companion: turn features on and off + +Use the exact `companion_radio_full` image for your board. Full includes its +supported transports and MOTA sending in one firmware; ordinary settings do +not require rebuilding it. USB remains the normal way to update the Companion +itself. WiFi/Bluetooth updates are optional, and Full does not install LoRa +MOTA packages on itself. + +## Open the text terminal + +Close the app or serial monitor using the USB port, then open a terminal at +115200 baud. A fresh boot starts in text mode. If it is already in Binary +Companion mode, send this line once and wait for the terminal banner: + +```text ++++MESHCORE-TERM-START +``` + +Run `help`, `board`, and `version` to inspect the device. On ESP32 you can +alternatively connect to its text terminal with `nc DEVICE_IP 5002`. +When finished, send `+++MESHCORE-TERM-STOP`, close the terminal, and reconnect +your Companion app. These are local text commands; an app's custom-variable +editor does not necessarily forward them. + +## Common switches + +| Feature | Turn on | Turn off | When it takes effect | +| --- | --- | --- | --- | +| Device power saving | `powersaving on` | `powersaving off` | Immediately; saved | +| LoRa receive power saving | `set radio.rxps on` | `set radio.rxps off` | Saved; radio applies when available | +| Radio chip RX boost | `set radio.rxgain on` | `set radio.rxgain off` | Immediately; saved; supported radios only | +| External FEM RX boost | `set radio.fem.rxgain on` | `set radio.fem.rxgain off` | Immediately; saved; controllable FEM only | +| External FEM TX gain | `set radio.fem.txgain on` | `set radio.fem.txgain off` | Immediately; saved; controllable PA only | +| ESP32 USB packet/debug logging | `set usb.logging on` | `set usb.logging off` | Immediately; saved; logging owns the USB terminal | +| nRF52 separate USB logging port | `set usb.logging on reboot` | `set usb.logging off reboot` | Saves and reboots to add/remove the second USB port | +| ESP32 browser settings | `set webui on` | `set webui off` | Saved; starts/stops WebConfig | +| ESP32 temporary setup portal | `start webconfig ap` | `stop webconfig` | This session; opens a setup network/QR where available | +| ESP32 WiFi firmware uploader | `start ota` or `start ota ap` | `stop ota` | This session; only usable with two application slots | +| Temporary MOTA radio window | `tempradio 910.525,250,5,5,120` | `normalradio` | Bounded window; saved normal settings return afterward | + +Check a saved switch with the corresponding `get` command, for example +`get usb.logging`, `get radio.rxps`, or `get webui`. Use `powersaving` to +inspect device power saving. Fresh installations enable device power saving +and leave USB logging off. Existing saved preferences take precedence. + +ESP32 logging and Binary Companion share one USB port. After turning logging +off, return to Binary Companion with `+++MESHCORE-TERM-STOP`. nRF52 keeps its +optional logging port separate from its primary USB port and BLE connection. +See the [USB switching guide](full_companion_usb_switcher.md). + +## WiFi, Bluetooth, GPS, and MQTT + +On ESP32, configure WiFi through the setup portal or these commands: + +```text +set wifi.ssid MyNetwork +set wifi.pwd my-password +get wifi.status +``` + +Credentials are saved. A credential change schedules a reconnect; a TCP +terminal will disconnect. The WiFi switch on supported device displays or +their assigned WiFi button controls the Companion WiFi services. Stopping +WebConfig alone stops its browser service, while the Companion's TCP and +MOTA services can remain available. + +WiFi modem sleep is independent of device power saving. Use +`set wifi.powersave min` for normal BLE/WiFi coexistence; inspect the effective +state with `get wifi.powersave`. `max` is available only where the firmware's +radio coexistence policy permits it. + +Most Full Companions provide BLE alongside their other transports. There is +no universal `set bluetooth off` command. SenseCAP Indicator selects one +secondary wireless transport per boot: + +```text +set companion.transport wifi +reboot +``` + +Use `set companion.transport ble` followed by `reboot` to select Bluetooth +instead. USB stays available. The ESP-NOW layout keeps its primary ESP-NOW +mesh active in either mode. + +For GPS-equipped boards, use the Companion app's GPS/custom sensor setting: +`gps=1` enables GPS and `gps=0` disables it. Only boards with a compiled GPS +provider expose this setting. This is separate from sharing location with +contacts. + +When the exact Full image includes MQTT, use WebConfig's MQTT cards. Select +and configure a broker slot to enable it; choose the `none` preset for every +slot to disable broker connections. The MQTT status, packets, raw, receive, +and transmit switches control their individual functions. Save the settings +and follow any reboot instruction shown. Builds without MQTT omit these +controls. There is no need for a separate logging or WiFi-MQTT Companion +image when that feature is included in Full. + +## Send MOTA from any Full Companion + +Place valid, destination-specific `.mota` files in a host directory. Set the +source, destination, controller, and required relays to the same bounded +temporary radio tuple. For USA Cascade on LoRa hardware: + +```text +tempradio 910.525,250,5,5,120 +``` + +On the computer connected to the Full Companion, run: + +```bash +motatool serve --serial /dev/ttyACM0 --dir ./motas -v +``` + +This works on both ESP32 and nRF52 Full. Close the text terminal first. +`motatool` automatically sends `ota folder on` and owns USB while serving. +Stop the host tool to detach the source; use `normalradio` afterward to +return early, or let the window expire. A `.bin`, `.uf2`, or DFU `.zip` is +not itself a `.mota`; use [motatool](../tools/mota/README.md) to prepare the +package for the exact destination firmware and hardware. + +ESP32 also accepts a host over WiFi: + +```bash +motatool serve --tcp DEVICE_IP:5001 --dir ./motas -v +``` + +nRF52 also accepts an authenticated Bluetooth host: + +```bash +python3 tools/ble_mota/ble_mota_seeder.py \ + --device MeshCore-MyCompanion --dir ./motas \ + --local 'tempradio 910.525,250,5,5,120' +``` + +Use one MOTA source transport at a time. See the +[complete MOTA instructions](companion_radio_full.md#serve-mota-images-manually) +for pairing, remote destination commands, and automation. ESP-NOW hardware +uses its ESP-NOW primary radio rather than transmitting LoRa. + +## Update the Companion itself + +USB is supported for every Full Companion. Use the exact board's UF2/DFU +package or ESP32 image and the normal flasher. Preserve the target's existing +partition layout unless deliberately performing a documented USB migration. + +For ESP32 with two application slots, `start ota` returns the WiFi uploader +URL, normally `http://DEVICE_IP:8080/update`. `start ota ap` explicitly opens +`MeshCore-OTA`; join it and use the returned URL. Upload the exact board's +application `.bin`. The device reboots when the upload succeeds. Use +`stop ota` to close an unused uploader. Port 8080 keeps WebConfig on port 80 +available. Use a trusted local network or a temporary setup network. + +The 4 MB Full layouts and T-Beam 1W Full use a single application slot and +require USB; the command reports that limitation. The artifact's +`.capabilities.json` lists verified `ota_update_methods` when a wireless +self-update path is usable. + +nRF52 Full exposes Bluetooth DFU to paired clients. Use the matching +application DFU `.zip` with a compatible Nordic/Adafruit Legacy DFU client +and the board's matching BLE-DFU-capable bootloader/SoftDevice. USB remains +the fallback when the installed bootloader does not support that procedure. +Bluetooth DFU updates the Companion; the separate Bluetooth MOTA source +service feeds packages to other mesh nodes. diff --git a/docs/releases/1.17.1.5.md b/docs/releases/1.17.1.5.md new file mode 100644 index 000000000..43c357983 --- /dev/null +++ b/docs/releases/1.17.1.5.md @@ -0,0 +1,57 @@ +# MeshCore 1.17.1.5 — USA Cascade + +This release uses the USA/Canada radio preset and Cascade runtime defaults. +The release build records the resolved frequency, bandwidth, spreading factor, +and coding rate. The offline USA Cascadia fallback is 910.525 MHz, BW62.5, +SF7, CR5. Saved node settings survive ordinary firmware updates. + +Use the exact board's Full Companion image when available. Every Full +Companion must pass linked-firmware checks for MOTA sending, its text terminal, +and the platform's source transports. ESP32 supplies USB and WiFi sources; +nRF52 supplies USB and encrypted Bluetooth sources. Full Companions remain +USB-updateable and do not require LoRa self-installation. Optional ESP32 +WiFi self-update uses port 8080 where the existing partition layout has two +application slots; nRF52 retains Bluetooth DFU with a compatible bootloader. + +Read [Full Companion feature switches](../full_companion_features.md) for +on/off commands, USB logging, power saving, WiFi/Bluetooth selection, GPS, +MQTT, MOTA sending, and update directions. + +Infrastructure artifacts must have a verified wireless self-update path. +Option 3 reports and omits infrastructure on platforms without a wireless +updater. The report is `ota-excluded-targets.txt`. Source/seeder capability +alone does not count as self-update support. ESP32 qualification checks both +application slots and OTA metadata; nRF52 Bluetooth qualification checks the +linked DFU service and its application ZIP. LoRa installation on nRF52 still +requires the exact matching OTAFIX bootloader and storage arrangement. + +Since 1.17.1.4, this branch also includes USB session and contact-persistence +fixes, automatic nRF52 ExtraFS recovery, improved Indicator setup/display +handling, compressed MOTA transport, and measured ESP32 memory limits. + +## Build using option 3 + +The equivalent reproducible command is: + +```bash +OUTPUT_DIR=.releases/1.17.1.5-usa-cascade \ +bash build.sh build-firmwares-logging-matrix \ + --firmware-version v1.17.1.5 \ + --radio-preset usa-cascadia --profile cascade \ + --require-ota --skip-kiss --resume --background +``` + +Or run `bash build.sh`, select option 3, set version `v1.17.1.5`, and retain +USA Cascadia/Cascade. Option 3 enables the infrastructure OTA requirement by +default. It merges supported runtime USB logging into canonical artifacts and +builds the expanded ESP32 profiles. It runs one PlatformIO process at a time. + +The background launcher prints its persistent log and status paths. Each +target also gets a log in `OUTPUT_DIR/build-logs/`. Resume accepts only +artifacts with the current capability schema and a successful qualification; +infrastructure also needs a verified update method. Any failed target leaves +the matrix unsuccessful and must be resolved before publication. + +`--allow-no-ota` is an explicit development override for cable-only +infrastructure. Those outputs do not belong in this OTA-required release. +Use a separate output directory when changing the radio or settings profile. diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index 0c428c932..358135c0f 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -2184,6 +2184,38 @@ bool MyMesh::handleLocalControlCommand(const char* command, char* reply, if (!command || !reply || reply_size == 0) return false; while (*command == ' ') command++; +#if defined(ESP32_PLATFORM) && defined(COMPANION_RADIO_FULL) + if (strcmp(command, "start ota") == 0 + || strcmp(command, "start ota ap") == 0) { +#if defined(COMPANION_EXCLUSIVE_WIFI_BLE) + if (!isCompanionWiFiEnabled()) { + snprintf(reply, reply_size, + "ERR: select WiFi transport and reboot before starting WiFi OTA"); + return true; + } +#endif + // Board implementations use the common 160-byte CLI reply contract. + // Binary Companion callers may supply a shorter buffer. + char ota_reply[160] = {0}; + if (!board.startOTAUpdate(_prefs.node_name, ota_reply, + strcmp(command, "start ota ap") == 0)) { + snprintf(reply, reply_size, "%s", + ota_reply[0] ? ota_reply : "ERR: WiFi OTA unavailable"); + } else { + snprintf(reply, reply_size, "%s", ota_reply); + } + return true; + } + if (strcmp(command, "stop ota") == 0) { + char ota_reply[160] = {0}; + if (!board.stopOTAUpdate(ota_reply) && !ota_reply[0]) { + snprintf(ota_reply, sizeof(ota_reply), "ERR: WiFi OTA unavailable"); + } + snprintf(reply, reply_size, "%s", ota_reply); + return true; + } +#endif + if (strcmp(command, "board") == 0) { const char* hardware_name = board.getManufacturerName(); snprintf(reply, reply_size, "%s", @@ -7150,6 +7182,10 @@ void MyMesh::handleTerminalCommand(char* command) { terminalOutput().print(" set usb.logging [reboot]\r\n"); #endif #if defined(ESP32) && defined(WIFI_SSID) +#if defined(COMPANION_RADIO_FULL) + terminalOutput().print(" start ota [ap] (WiFi firmware update on port 8080)\r\n"); + terminalOutput().print(" stop ota\r\n"); +#endif #if defined(COMPANION_EXCLUSIVE_WIFI_BLE) terminalOutput().print(" get companion.transport\r\n"); terminalOutput().print(" set companion.transport \r\n"); diff --git a/scripts/check_firmware_capabilities.py b/scripts/check_firmware_capabilities.py index c3aabd3bd..39c6a4d1c 100644 --- a/scripts/check_firmware_capabilities.py +++ b/scripts/check_firmware_capabilities.py @@ -8,6 +8,8 @@ import json import os from pathlib import Path import sys +import struct +import zipfile def parse_args() -> argparse.Namespace: @@ -21,6 +23,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--build-profile", required=True) parser.add_argument("--capability", action="append", default=[]) parser.add_argument("--reduction", action="append", default=[]) + parser.add_argument("--require-ota", action="store_true") + parser.add_argument("--firmware-bin", type=Path) + parser.add_argument("--partitions", type=Path) + parser.add_argument("--dfu-package", type=Path) parser.add_argument( "--expect", action="append", @@ -35,6 +41,50 @@ def stable_unique(values: list[str]) -> list[str]: return list(dict.fromkeys(value for value in values if value)) +def verify_ota_artifacts(args, methods): + """Check the actual update medium; source/seeder support cannot qualify.""" + if not methods: + return False, "no linked wireless self-update implementation" + try: + if args.platform == "ESP32_PLATFORM": + if args.partitions is None or args.firmware_bin is None: + return False, "ESP32 OTA requires firmware.bin and partitions.bin" + table = args.partitions.read_bytes() + slots = {} + otadata = False + for offset in range(0, len(table) - 31, 32): + magic, kind, subtype, address, size = struct.unpack_from("= 0x2000: + otadata = True + if len(slots) != 2 or not otadata: + return False, "ESP32 OTA requires ota_0, ota_1, and otadata" + image_size = args.firmware_bin.stat().st_size + ordered = sorted(slots.values()) + if ordered[0][0] + ordered[0][1] > ordered[1][0]: + return False, "ESP32 OTA application slots overlap" + if image_size <= 0 or any(image_size > size for _, size in ordered): + return False, "ESP32 firmware does not fit both OTA application slots" + return True, "firmware fits both OTA application slots; otadata present" + if args.platform == "NRF52_PLATFORM" and "bluetooth" in methods: + if args.dfu_package is None: + return False, "Bluetooth DFU requires its application ZIP" + with zipfile.ZipFile(args.dfu_package) as package: + manifest = json.loads(package.read("manifest.json"))["manifest"] + application = manifest["application"] + if not package.read(application["bin_file"]) or not package.read(application["dat_file"]): + return False, "Bluetooth DFU application ZIP is empty" + if package.testzip() is not None: + return False, "Bluetooth DFU application ZIP is corrupt" + return True, "application DFU ZIP verified; matching BLE DFU bootloader required" + except (OSError, ValueError, KeyError, zipfile.BadZipFile) as exc: + return False, str(exc) + return False, "no qualified wireless update artifact for this platform" + + def main() -> int: args = parse_args() try: @@ -67,8 +117,16 @@ def main() -> int: args.capability + [check["capability"] for check in checks] ) missing = [check for check in checks if not check["present"]] + ota_methods = stable_unique([ + check["capability"].removeprefix("ota.update.") + for check in checks + if check["present"] and check["capability"].startswith("ota.update.") + ]) + ota_verified, ota_evidence = verify_ota_artifacts(args, ota_methods) + if not ota_verified: + capabilities = [value for value in capabilities if not value.startswith("ota.update.")] manifest = { - "schema_version": 1, + "schema_version": 2, "target": args.target, "artifact_target": args.artifact_target or args.target, "platformio_env": args.platformio_env or args.target, @@ -77,7 +135,10 @@ def main() -> int: "capabilities": capabilities, "reductions": stable_unique(args.reduction), "verification": checks, - "verified": not malformed and not missing, + "ota_update_methods": ota_methods if ota_verified else [], + "ota_update_verified": ota_verified, + "ota_update_evidence": ota_evidence, + "verified": not malformed and not missing and (not args.require_ota or ota_verified), } args.output.parent.mkdir(parents=True, exist_ok=True) @@ -87,6 +148,9 @@ def main() -> int: if malformed: return 2 + if args.require_ota and not ota_verified: + print(f"capability check failed: wireless self-update: {ota_evidence}", file=sys.stderr) + return 1 if missing: for check in missing: print( diff --git a/scripts/package_cascade_release.py b/scripts/package_cascade_release.py new file mode 100644 index 000000000..dde0b0e99 --- /dev/null +++ b/scripts/package_cascade_release.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Stage a completed Option 3 matrix as audited, directly downloadable releases.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +from pathlib import Path +import shutil +from urllib.parse import quote + + +ROOT = Path(__file__).resolve().parents[1] +FIRMWARE_SUFFIXES = {".bin", ".uf2", ".hex", ".zip"} + + +def is_companion(target): + return "companion" in target.lower() or "comp_radio" in target.lower() + + +def validate_manifest(manifest): + target = manifest["target"] + if manifest.get("schema_version", 0) < 2 or not manifest.get("verified"): + raise ValueError(f"{target}: firmware capability qualification failed") + if not is_companion(target) and not manifest.get("ota_update_verified"): + raise ValueError(f"{target}: infrastructure has no verified wireless updater") + if "companion_radio_full" in target.lower(): + required = {"companion.usb_mota_source", "companion.mota_sender", + "companion.temp_radio", "companion.ota_cli"} + required.add("companion.wifi_ota_seeder" if manifest["platform"] == "ESP32_PLATFORM" + else "companion.ble_mota_source") + verified = {item["capability"] for item in manifest["verification"] if item["present"]} + if not required <= verified: + raise ValueError(f"{target}: Full Companion lacks verified MOTA sending") + + +def collect_artifacts(directory, version): + records = [] + accounted = set() + for path in sorted(directory.glob("*.capabilities.json")): + stem = path.name.removesuffix(".capabilities.json") + files = [candidate for candidate in directory.glob(stem + ".*") + if candidate.suffix in FIRMWARE_SUFFIXES] + merged = directory / (stem + "-merged.bin") + if merged.is_file(): + files.append(merged) + if not files: + continue # A measured rejected attempt may leave only a manifest. + if not stem.endswith("-" + version): + raise ValueError(f"stale or mixed-version artifact: {stem}") + manifest = json.loads(path.read_text()) + validate_manifest(manifest) + extensions = {item.suffix for item in files} + platform = manifest["platform"] + if platform == "ESP32_PLATFORM" and (not merged.is_file() or not (directory / (stem + ".bin")).is_file()): + raise ValueError(f"{stem}: ESP32 application/merged pair incomplete") + if platform == "NRF52_PLATFORM": + if ".uf2" not in extensions or (manifest.get("ota_update_verified") and ".zip" not in extensions): + raise ValueError(f"{stem}: nRF52 UF2/DFU artifacts incomplete") + if any(item.stat().st_size == 0 for item in files): + raise ValueError(f"{stem}: empty firmware artifact") + accounted.update(files) + records.append({"manifest": manifest, "files": sorted(files) + [path]}) + unaccounted = {path for path in directory.iterdir() if path.suffix in FIRMWARE_SUFFIXES} - accounted + if unaccounted: + raise ValueError("firmware without qualification: " + ", ".join(sorted(p.name for p in unaccounted))) + if not records: + raise ValueError("no qualified firmware artifacts") + return records + + +def category(record): + manifest = record["manifest"] + if is_companion(manifest["target"]): + return "companion" + if manifest["build_profile"] == "full": + return "full-profiles" + if "lora_ota" in manifest["target"].lower(): + return "lora-ota" + return "infrastructure" + + +def sha256(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--build-status", required=True, type=Path) + parser.add_argument("--version", default="1.17.1.5") + parser.add_argument("--commit", required=True) + parser.add_argument("--repo", default="mikecarper/MeshCore") + args = parser.parse_args() + status = dict(line.split("=", 1) for line in args.build_status.read_text().splitlines() if "=" in line) + if status.get("state") != "completed" or status.get("exit_code") != "0": + raise ValueError("the Option 3 matrix has not completed successfully") + output_directory = Path(status["working_directory"]) / status["output_directory"] + if output_directory.resolve() != args.input.resolve(): + raise ValueError("build status belongs to another output directory") + if status.get("source_commit") != args.commit or status.get("firmware_version") != "v" + args.version: + raise ValueError("build status belongs to another source revision or version") + if status.get("firmware_profile") != "cascade": + raise ValueError("matrix did not use Cascade runtime defaults") + radio = {key: status["radio_" + key] for key in ("frequency", "bandwidth", "sf", "cr")} + if args.output.exists() and any(args.output.iterdir()): + raise ValueError("staging directory must be empty; existing releases are never overwritten") + version = f"v{args.version}-{args.commit[:8]}" + records = collect_artifacts(args.input, version) + base_tag = f"v{args.version}-usa-cascade-{args.commit[:8]}" + groups = [] + for name in ("companion", "infrastructure", "lora-ota", "full-profiles"): + current = [] + count = 0 + chunks = [] + for record in (r for r in records if category(r) == name): + if count + len(record["files"]) > 900: + chunks.append(current) + current, count = [], 0 + current.append(record) + count += len(record["files"]) + if current: + chunks.append(current) + for index, chunk in enumerate(chunks): + key = name + (f"-{index + 1}" if index else "") + tag = base_tag if key == "companion" else f"{key}-{base_tag}" + groups.append({"key": key, "tag": tag, "records": chunk}) + links = "\n".join(f"- [{g['key']}](https://github.com/{args.repo}/releases/tag/{g['tag']})" for g in groups) + source_url = f"https://github.com/{args.repo}/blob/{args.commit}" + rows = [] + for group in groups: + destination = args.output / group["key"] + destination.mkdir(parents=True) + summaries = [] + for record in group["records"]: + manifest = record["manifest"] + file_links = [] + for path in record["files"]: + shutil.copy2(path, destination / path.name) + if path.suffix in FIRMWARE_SUFFIXES: + url = f"https://github.com/{args.repo}/releases/download/{group['tag']}/{quote(path.name)}" + label = "merged.bin (USB)" if path.name.endswith("-merged.bin") else path.suffix[1:] + file_links.append(f'{html.escape(label)}') + methods = ", ".join(manifest.get("ota_update_methods", [])) or "USB" + summaries.append({**manifest, "files": [path.name for path in record["files"]]}) + rows.append(f"{html.escape(manifest['artifact_target'])}{html.escape(manifest['build_profile'])}{html.escape(methods)}{' · '.join(file_links)}") + (destination / "TARGET-MANIFEST.json").write_text(json.dumps(summaries, indent=2) + "\n") + shutil.copy2(ROOT / "docs/full_companion_features.md", destination / "FULL-COMPANION-FEATURES.md") + exclusions = args.input / "ota-excluded-targets.txt" + if exclusions.is_file(): + shutil.copy2(exclusions, destination / exclusions.name) + body = (f"# MeshCore {args.version} — USA Cascade\n\n" + f"Source: `{args.commit}`. USA/Canada: **{radio['frequency']} MHz, BW{radio['bandwidth']}, SF{radio['sf']}, CR{radio['cr']}**; Cascade defaults.\n\n" + f"This page contains **{len(summaries)} qualified firmware profiles**. Full Companions send MOTA and normally update over USB. Infrastructure has a verified wireless update path; see each capability manifest.\n\n" + f"[Feature on/off and update directions]({source_url}/docs/full_companion_features.md) · " + f"[Release details]({source_url}/docs/releases/{args.version}.md)\n\n" + "Match the exact board, radio, display, and storage variant. ESP32 WiFi updates use the application `.bin`; the merged image installs boot/partition data over USB. nRF52 `.zip` files are native application DFU packages. LoRa MOTA installation requires an exact destination package; nRF52 also requires its matching OTAFIX bootloader. Full Companions do not LoRa-install onto themselves.\n\n" + "Firmware and capability checks passed in the build matrix. Hardware update testing across every board was not performed.\n\n" + + links + "\n") + (args.output / (group["key"] + "-notes.md")).write_text(body) + picker = ("" + f"MeshCore {html.escape(args.version)} USA Cascade" + "" + f"

MeshCore {html.escape(args.version)} USA Cascade

" + "

Match your exact hardware. Full Companions can send MOTA; USB is their normal update method.

" + f"

Feature switches and update instructions

" + "" + "" + + "\n".join(rows) + "
TargetProfileSelf-updateDownloads
" + "") + for group in groups: + destination = args.output / group["key"] + (destination / f"FIRMWARE-PICKER-{args.version}.html").write_text(picker) + files = sorted(destination.iterdir()) + (destination / "SHA256SUMS.txt").write_text("".join(f"{sha256(path)} {path.name}\n" for path in files)) + group["asset_count"] = len(files) + 1 + group["target_count"] = len(group.pop("records")) + (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups}, indent=2) + "\n") + print(json.dumps({"targets": len(records), "groups": groups}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 7f6814719..b58847da5 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -55,9 +55,11 @@ bool ESP32Board::isUserGpioAvailable(uint8_t pin) const { return !UserGpioPinPolicy::isFirmwareReserved(pin); } -#if defined(ADMIN_PASSWORD) && defined(LIGHTWEIGHT_WIFI_OTA) +#if defined(LIGHTWEIGHT_WIFI_OTA) && \ + (defined(ADMIN_PASSWORD) || defined(COMPANION_RADIO_FULL)) #include #include +#include #include #include #include @@ -65,6 +67,14 @@ bool ESP32Board::isUserGpioAvailable(uint8_t pin) const { static bool lightweight_ota_started_ap; +// Full Companion keeps WebConfig on port 80 while an explicitly started +// updater uses 8080. Infrastructure retains its established /update URL. +#if defined(COMPANION_RADIO_FULL) +static constexpr uint16_t LIGHTWEIGHT_OTA_PORT = 8080; +#else +static constexpr uint16_t LIGHTWEIGHT_OTA_PORT = 80; +#endif + static const char LIGHTWEIGHT_OTA_PAGE[] = R"HTML( MeshCore OTA @@ -78,7 +88,7 @@ x.onload=()=>{s.textContent=x.responseText;b.disabled=false};x.onerror=()=>{s.te )HTML"; class LightweightOTAServer { - WiFiServer server{80}; + WiFiServer server{LIGHTWEIGHT_OTA_PORT}; TaskHandle_t task = nullptr; volatile bool running = false; ESP32Board* board = nullptr; @@ -265,6 +275,14 @@ static LightweightOTAServer lightweight_ota_server; bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { (void)id; +#if defined(COMPANION_RADIO_FULL) + const esp_partition_t* running = esp_ota_get_running_partition(); + const esp_partition_t* next = esp_ota_get_next_update_partition(nullptr); + if (!running || !next || running->address == next->address) { + strcpy(reply, "ERR: this partition layout requires a USB firmware update"); + return false; + } +#endif inhibit_sleep = true; IPAddress ip; @@ -296,7 +314,12 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { ota_server = &lightweight_ota_server; } - snprintf(reply, 160, "Started: http://%s/update", ip.toString().c_str()); + if (LIGHTWEIGHT_OTA_PORT == 80) { + snprintf(reply, 160, "Started: http://%s/update", ip.toString().c_str()); + } else { + snprintf(reply, 160, "Started: http://%s:%u/update", ip.toString().c_str(), + static_cast(LIGHTWEIGHT_OTA_PORT)); + } MESH_DEBUG_PRINTLN("startOTAUpdate: %s", reply); return true; } diff --git a/test/test_cascade_release_package.py b/test/test_cascade_release_package.py new file mode 100644 index 000000000..ef8509fb0 --- /dev/null +++ b/test/test_cascade_release_package.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("package_cascade_release", ROOT / "scripts/package_cascade_release.py") +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) + + +class ReleaseQualificationTest(unittest.TestCase): + def manifest(self, target="test_repeater", **changes): + return {"target": target, "platform": "ESP32_PLATFORM", + "schema_version": 2, "verified": True, + "ota_update_verified": True, "verification": [], **changes} + + def test_source_capability_cannot_qualify_infrastructure(self): + with self.assertRaisesRegex(ValueError, "wireless updater"): + package.validate_manifest(self.manifest(ota_update_verified=False)) + + def test_usb_companion_is_accepted(self): + package.validate_manifest(self.manifest("test_companion_radio_usb", ota_update_verified=False)) + + def test_full_companion_requires_linked_source_implementation(self): + manifest = self.manifest("test_companion_radio_full", ota_update_verified=False) + with self.assertRaisesRegex(ValueError, "MOTA sending"): + package.validate_manifest(manifest) + manifest["verification"] = [{"capability": name, "present": True} for name in ( + "companion.usb_mota_source", "companion.mota_sender", "companion.temp_radio", + "companion.ota_cli", "companion.wifi_ota_seeder")] + package.validate_manifest(manifest) + manifest["verification"][1]["present"] = False + with self.assertRaisesRegex(ValueError, "MOTA sending"): + package.validate_manifest(manifest) + + def test_missing_qualification_and_mixed_versions_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + stem = "test_companion_radio_usb-v1.0-deadbeef" + (directory / (stem + ".bin")).write_bytes(b"firmware") + with self.assertRaisesRegex(ValueError, "without qualification"): + package.collect_artifacts(directory, "v1.0-deadbeef") + (directory / (stem + ".capabilities.json")).write_text(json.dumps(self.manifest("test_companion_radio_usb"))) + with self.assertRaisesRegex(ValueError, "mixed-version"): + package.collect_artifacts(directory, "v2.0-deadbeef") + with self.assertRaisesRegex(ValueError, "pair incomplete"): + package.collect_artifacts(directory, "v1.0-deadbeef") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_firmware_capabilities.py b/test/test_firmware_capabilities.py index 2037098c1..261ff192e 100644 --- a/test/test_firmware_capabilities.py +++ b/test/test_firmware_capabilities.py @@ -6,6 +6,9 @@ import subprocess import sys import tempfile import unittest +import struct +import io +import zipfile ROOT = Path(__file__).resolve().parents[1] @@ -13,13 +16,18 @@ CHECKER = ROOT / "scripts" / "check_firmware_capabilities.py" class FirmwareCapabilityCheckerTest(unittest.TestCase): - def run_checker(self, image_bytes, *extra_args): + def run_checker(self, image_bytes, *extra_args, artifacts=None): temp_dir = tempfile.TemporaryDirectory() self.addCleanup(temp_dir.cleanup) directory = Path(temp_dir.name) image = directory / "firmware.elf" manifest = directory / "firmware.capabilities.json" image.write_bytes(image_bytes) + artifact_args = [] + for option, data in (artifacts or {}).items(): + path = directory / option + path.write_bytes(data) + artifact_args.extend(["--" + option, str(path)]) result = subprocess.run( [ sys.executable, @@ -35,6 +43,7 @@ class FirmwareCapabilityCheckerTest(unittest.TestCase): "--build-profile", "auto", *extra_args, + *artifact_args, ], capture_output=True, text=True, @@ -64,6 +73,69 @@ class FirmwareCapabilityCheckerTest(unittest.TestCase): self.assertFalse(manifest["verified"]) self.assertIn("promised", result.stderr) + def ota_artifacts(self, *, second_slot=True, image_size=48): + def entry(kind, subtype, address, size): + return struct.pack("&2; exit 1; } + +# Exercise release resolution without starting PlatformIO. This inventory +# includes a personal USB Companion and infrastructure on the same platform. +PIO_ENV_PLATFORM_BY_NAME[nrf_repeater]=NRF52_PLATFORM +PIO_ENV_PLATFORM_BY_NAME[esp_companion_radio_full]=ESP32_PLATFORM +PIO_ENV_PLATFORM_BY_NAME[rp_companion_radio_usb]=RP2040_PLATFORM +PIO_ENV_PLATFORM_BY_NAME[rp_repeater]=RP2040_PLATFORM +PIO_ENV_PLATFORM_BY_NAME[stm_sensor]=STM32_PLATFORM +resolve_bulk_command_targets() { + RESOLVED_BUILD_TARGETS=(nrf_repeater esp_companion_radio_full + rp_companion_radio_usb rp_repeater stm_sensor) +} +sort_build_targets_by_platform_and_name() { printf '%s\n' "$@"; } +validate_build_target() { return 0; } + +REQUIRE_OTA_UPDATES="" +resolve_command_targets build-firmwares-logging-matrix >/dev/null +[ "$REQUIRE_OTA_UPDATES" = 1 ] || fail "option 3 did not require infrastructure OTA" +[ "${#RESOLVED_BUILD_TARGETS[@]}" = 3 ] || fail "wrong release inventory" +[[ " ${RESOLVED_BUILD_TARGETS[*]} " == *" rp_companion_radio_usb "* ]] \ + || fail "USB-updated Companion was excluded" +[ "${#OTA_EXCLUDED_TARGETS[@]}" = 2 ] || fail "missing exclusion report" + +parse_cli_options build-firmwares-logging-matrix --allow-no-ota +resolve_command_targets "${PARSED_COMMAND_ARGS[@]}" >/dev/null +[ "${#RESOLVED_BUILD_TARGETS[@]}" = 5 ] || fail "development override lost targets" + +parse_cli_options build-firmware rp_repeater --require-ota +if resolve_command_targets "${PARSED_COMMAND_ARGS[@]}" >/dev/null; then + fail "explicit cable-only infrastructure build passed OTA requirement" +fi +resolve_command_targets build-firmware rp_companion_radio_usb >/dev/null +[ "${#RESOLVED_BUILD_TARGETS[@]}" = 1 ] || fail "explicit USB Companion was rejected" + +echo "test_release_ota_policy: OK"