Fix constrained logging matrix builds

build.sh

@@ -57,6 +57,7 @@ PIO_BUILD_DIR_OVERRIDE=""
 RESOLVED_BUILD_TARGETS=()
 RESUME_BUILD_OUTPUT="${RESUME_BUILD_OUTPUT:-0}"
 LOGGING_MATRIX_FAILURES=()
+LOGGING_MATRIX_DEFERRED_TARGETS=()
 RADIO_PRESET_SELECTION=""
 KISS_MODE_OVERRIDE="${KISS_MODE_OVERRIDE-}"
 PARSED_COMMAND_ARGS=()
@@ -3291,7 +3292,7 @@ apply_nrf52_lora_ota_build_recipe() {
   # recipe, but a new board may not. Add only the missing pieces so ENABLE_OTA
   # never reaches the linker without its implementation or the EndF
   # trailer/zip post-build step.
-  if ! pio_env_option_contains "$pio_env_name" build_src_filter "helpers/ota/"; then
+  if ! pio_env_option_contains "$pio_env_name" build_src_filter "helpers/ota/*.cpp"; then
     append_platformio_build_src_filter "+<helpers/ota/*.cpp>"
   fi
   if ! pio_env_option_contains "$pio_env_name" extra_scripts "tools/mota/pio_endf.py"; then
@@ -3409,7 +3410,7 @@ apply_companion_radio_full_profile() {
     append_platformio_build_unflags "-UOTA_FOLDER_SERIAL"
     export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DOTA_FOLDER_SERIAL=1 -DCOMPANION_FEATURE_USB_MOTA_SOURCE=1 -DCOMPANION_FEATURE_BLE_MOTA_SOURCE=1 -DCOMPANION_FEATURE_DEDICATED_USB_LOGGING=1 -DCFG_TUD_CDC=2 -DMESH_DUAL_CDC_LOGGING=1 -DMESH_DEBUG=1 -DMESH_PACKET_LOGGING=1"
 
-    if ! pio_env_option_contains "$pio_env_name" build_src_filter "helpers/ota/"; then
+    if ! pio_env_option_contains "$pio_env_name" build_src_filter "helpers/ota/*.cpp"; then
       append_platformio_build_src_filter "+<helpers/ota/*.cpp>"
     fi
     if ! pio_env_option_contains "$pio_env_name" build_src_filter "helpers/nrf52/*.cpp" \
@@ -4903,7 +4904,14 @@ run_logged_build_targets() {
       else
         mv -f -- "$log_tmp" "$log_path"
       fi
-      if [ "$build_status" -ne 0 ]; then
+      if [ "$build_status" -eq 42 ] \
+          && [ "${DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL:-0}" = "1" ] \
+          && [ "$profile" = "standard" ] \
+          && supports_esp32_full_build "$env"; then
+        LOGGING_MATRIX_DEFERRED_TARGETS+=("$env")
+        echo "DEFERRED: ${env} (${profile}) exceeds the portable OTA slot; the expanded FULL pass is required."
+        echo "DEFERRED: ${env} (${profile}) exceeds the portable OTA slot; the expanded FULL pass is required." >> "$log_path"
+      elif [ "$build_status" -ne 0 ]; then
         overall_status=1
         LOGGING_MATRIX_FAILURES+=("${env} (${profile}) -> ${log_path}")
         echo "FAILED: ${env} (${profile}), status ${build_status}"
@@ -5015,7 +5023,14 @@ run_logged_build_targets() {
     else
       mv -f -- "$log_tmp" "$log_path"
     fi
-    if [ "$build_status" -ne 0 ]; then
+    if [ "$build_status" -eq 42 ] \
+        && [ "${DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL:-0}" = "1" ] \
+        && [ "$profile" = "standard" ] \
+        && supports_esp32_full_build "$env"; then
+      LOGGING_MATRIX_DEFERRED_TARGETS+=("$env")
+      echo "DEFERRED: ${env} (${profile}) exceeds the portable OTA slot; the expanded FULL pass is required."
+      echo "DEFERRED: ${env} (${profile}) exceeds the portable OTA slot; the expanded FULL pass is required." >> "$log_path"
+    elif [ "$build_status" -ne 0 ]; then
       overall_status=1
       LOGGING_MATRIX_FAILURES+=("${env} (${profile}) -> ${log_path}")
       echo "FAILED: ${env} (${profile}), status ${build_status}"
@@ -5199,12 +5214,14 @@ run_logging_matrix_build_targets() {
   local constrained_merged_logging_count=0
   local build_status=0
   local pass_status=0
+  local DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL=0
 
   if [ ${#targets[@]} -eq 0 ]; then
     echo "No build targets resolved."
     return 1
   fi
   LOGGING_MATRIX_FAILURES=()
+  LOGGING_MATRIX_DEFERRED_TARGETS=()
   PROFILE_BUILD_WORKERS=$OPTION3_BUILD_WORKERS
   echo "Option 3 PlatformIO policy: one target build at a time, ${OPTION3_PIO_JOBS} compiler job(s) inside that process."
 
@@ -5238,8 +5255,10 @@ run_logging_matrix_build_targets() {
   MQTT_BRIDGE_OVERRIDE="off"
   FIRMWARE_FILENAME_INFIX=""
   if [ ${#standard_targets[@]} -gt 0 ]; then
+    DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL=1
     run_logged_build_targets "${standard_targets[@]}"
     pass_status=$?
+    DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL=0
     if [ "$pass_status" -eq 130 ]; then return 130; fi
     if [ "$pass_status" -ne 0 ]; then build_status=1; fi
   fi
@@ -5262,6 +5281,10 @@ run_logging_matrix_build_targets() {
   ESP32_FULL_BUILD=$original_esp32_full_build
   PROFILE_BUILD_WORKERS=$original_profile_build_workers
 
+  if [ ${#LOGGING_MATRIX_DEFERRED_TARGETS[@]} -gt 0 ]; then
+    echo "${#LOGGING_MATRIX_DEFERRED_TARGETS[@]} standard ESP32 target(s) exceeded the portable OTA slot and were deferred to the expanded FULL pass:"
+    printf '  %s\n' "${LOGGING_MATRIX_DEFERRED_TARGETS[@]}"
+  fi
   if [ ${#LOGGING_MATRIX_FAILURES[@]} -gt 0 ]; then
     echo "Logging matrix completed with ${#LOGGING_MATRIX_FAILURES[@]} failed build(s):"
     printf '  %s\n' "${LOGGING_MATRIX_FAILURES[@]}"

lib/ed25519/ge.c

@@ -340,6 +340,7 @@ static unsigned char equal(signed char b, signed char c) {
     return (unsigned char) y;
 }
 
+#ifndef ED25519_COMPACT_BASE
 static unsigned char negative(signed char b) {
     uint64_t x = b; /* 18446744073709551361..18446744073709551615: yes; 0..255: no */
     x >>= 63; /* 1: yes; 0: no */
@@ -432,6 +433,83 @@ void ge_scalarmult_base(ge_p3 *h, const unsigned char *a) {
         ge_p1p1_to_p3(h, &r);
     }
 }
+#else
+/*
+ * Space-constrained fixed-base multiplication.
+ *
+ * The usual ref10 implementation uses a 30 KiB table containing shifted
+ * multiples of the public Ed25519 base point.  Small STM32 targets cannot
+ * afford that table alongside OTA, packet logging, the CLI, and their 16 KiB
+ * filesystem.  Build the much smaller 0B..7B table in RAM and consume the
+ * scalar in fixed three-bit windows instead.  The loop counts and selections
+ * do not depend on secret scalar bits.
+ */
+static void cached_cmov(ge_cached *t, const ge_cached *u, unsigned char b) {
+    fe_cmov(t->YplusX, u->YplusX, b);
+    fe_cmov(t->YminusX, u->YminusX, b);
+    fe_cmov(t->Z, u->Z, b);
+    fe_cmov(t->T2d, u->T2d, b);
+}
+
+static unsigned char scalar_window3(const unsigned char *a, int window) {
+    const int bit = window * 3;
+    const int byte = bit >> 3;
+    const int shift = bit & 7;
+    unsigned int value = a[byte] >> shift;
+
+    if (shift > 5 && byte < 31) {
+        value |= (unsigned int)a[byte + 1] << (8 - shift);
+    }
+    return (unsigned char)(value & 7U);
+}
+
+void ge_scalarmult_base(ge_p3 *h, const unsigned char *a) {
+    ge_cached multiples[8];
+    ge_cached base_cached;
+    ge_cached selected;
+    ge_p1p1 r;
+    ge_p3 base_point;
+    ge_p3 current;
+    unsigned char digit;
+    int i;
+    int j;
+
+    /* Bi[0] is B in precomputed form; add it to the identity to recover B. */
+    ge_p3_0(&base_point);
+    ge_madd(&r, &base_point, &Bi[0]);
+    ge_p1p1_to_p3(&base_point, &r);
+    ge_p3_to_cached(&base_cached, &base_point);
+
+    ge_p3_0(&current);
+    ge_p3_to_cached(&multiples[0], &current);
+    multiples[1] = base_cached;
+    current = base_point;
+    for (i = 2; i < 8; ++i) {
+        ge_add(&r, &current, &base_cached);
+        ge_p1p1_to_p3(&current, &r);
+        ge_p3_to_cached(&multiples[i], &current);
+    }
+
+    ge_p3_0(h);
+    /* a[31] <= 127, so 85 windows cover all possible scalar bits. */
+    for (i = 84; i >= 0; --i) {
+        ge_p3_dbl(&r, h);
+        ge_p1p1_to_p3(h, &r);
+        ge_p3_dbl(&r, h);
+        ge_p1p1_to_p3(h, &r);
+        ge_p3_dbl(&r, h);
+        ge_p1p1_to_p3(h, &r);
+
+        selected = multiples[0];
+        digit = scalar_window3(a, i);
+        for (j = 1; j < 8; ++j) {
+            cached_cmov(&selected, &multiples[j], equal((signed char)digit, (signed char)j));
+        }
+        ge_add(&r, h, &selected);
+        ge_p1p1_to_p3(h, &r);
+    }
+}
+#endif
 
 
 /*

lib/ed25519/precomp_data.h

@@ -43,6 +43,7 @@ static const ge_precomp Bi[8] = {
 
 
 /* base[i][j] = (j+1)*256^i*B */
+#ifndef ED25519_COMPACT_BASE
 static const ge_precomp base[32][8] = {
     {
         {
@@ -1389,3 +1390,4 @@ static const ge_precomp base[32][8] = {
         },
     },
 };
+#endif

platformio.ini

@@ -248,6 +248,7 @@ platform = ststm32
 extra_scripts = post:arch/stm32/build_hex.py
 build_flags = ${arduino_base.build_flags}
   -D STM32_PLATFORM
+  -D ED25519_COMPACT_BASE=1
   -I src/helpers/stm32
   -I $PROJECT_PACKAGES_DIR/framework-arduinoststm32/libraries/SubGhz/src
 build_src_filter = ${arduino_base.build_src_filter}

src/helpers/StorageLayout.cpp

@@ -9,6 +9,7 @@
 #include <nrf.h>
 #include "ota/OtaFlashLayout_nrf52.h"
 #elif defined(STM32_PLATFORM)
+#include <Arduino.h>
 #include <InternalFileSystem.h>
 #endif
 

test/fixtures/ed25519_compact/test_ed25519_compact.c

@@ -0,0 +1,107 @@
+#include <stdio.h>
+#include <string.h>
+
+#include "ed_25519.h"
+
+typedef struct {
+    const char *seed;
+    const char *public_key;
+    const char *message;
+    const char *signature;
+} rfc8032_vector;
+
+static int hex_nibble(char c) {
+    if (c >= '0' && c <= '9') return c - '0';
+    if (c >= 'a' && c <= 'f') return c - 'a' + 10;
+    if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+    return -1;
+}
+
+static int decode_hex(unsigned char *out, size_t out_len, const char *hex) {
+    size_t i;
+
+    if (strlen(hex) != out_len * 2) return 0;
+    for (i = 0; i < out_len; ++i) {
+        const int high = hex_nibble(hex[i * 2]);
+        const int low = hex_nibble(hex[i * 2 + 1]);
+        if (high < 0 || low < 0) return 0;
+        out[i] = (unsigned char)((high << 4) | low);
+    }
+    return 1;
+}
+
+static int run_vector(const rfc8032_vector *vector, int number) {
+    unsigned char seed[32];
+    unsigned char expected_public_key[32];
+    unsigned char expected_signature[64];
+    unsigned char public_key[32];
+    unsigned char private_key[64];
+    unsigned char signature[64];
+    unsigned char message[256];
+    const size_t message_len = strlen(vector->message) / 2;
+
+    if (message_len > sizeof(message) ||
+        !decode_hex(seed, sizeof(seed), vector->seed) ||
+        !decode_hex(expected_public_key, sizeof(expected_public_key), vector->public_key) ||
+        !decode_hex(message, message_len, vector->message) ||
+        !decode_hex(expected_signature, sizeof(expected_signature), vector->signature)) {
+        fprintf(stderr, "RFC 8032 vector %d is malformed\n", number);
+        return 0;
+    }
+
+    ed25519_create_keypair(public_key, private_key, seed);
+    if (memcmp(public_key, expected_public_key, sizeof(public_key)) != 0) {
+        fprintf(stderr, "RFC 8032 vector %d public key mismatch\n", number);
+        return 0;
+    }
+
+    ed25519_sign(signature, message, message_len, public_key, private_key);
+    if (memcmp(signature, expected_signature, sizeof(signature)) != 0) {
+        fprintf(stderr, "RFC 8032 vector %d signature mismatch\n", number);
+        return 0;
+    }
+    if (!ed25519_verify(signature, message, message_len, public_key)) {
+        fprintf(stderr, "RFC 8032 vector %d signature did not verify\n", number);
+        return 0;
+    }
+
+    signature[17] ^= 0x01;
+    if (ed25519_verify(signature, message, message_len, public_key)) {
+        fprintf(stderr, "RFC 8032 vector %d accepted a damaged signature\n", number);
+        return 0;
+    }
+    return 1;
+}
+
+int main(void) {
+    static const rfc8032_vector vectors[] = {
+        {
+            "9d61b19deffd5a60ba844af492ec2cc4"
+            "4449c5697b326919703bac031cae7f60",
+            "d75a980182b10ab7d54bfed3c964073a"
+            "0ee172f3daa62325af021a68f707511a",
+            "",
+            "e5564300c360ac729086e2cc806e828a"
+            "84877f1eb8e5d974d873e06522490155"
+            "5fb8821590a33bacc61e39701cf9b46b"
+            "d25bf5f0595bbe24655141438e7a100b"
+        },
+        {
+            "4ccd089b28ff96da9db6c346ec114e0f"
+            "5b8a319f35aba624da8cf6ed4fb8a6fb",
+            "3d4017c3e843895a92b70aa74d1b7ebc"
+            "9c982ccf2ec4968cc0cd55f12af4660c",
+            "72",
+            "92a009a9f0d4cab8720e820b5f642540"
+            "a2b27b5416503f8fb3762223ebdb69da"
+            "085ac1e43e15996e458f3613d0f11d8c"
+            "387b2eaeb4302aeeb00d291612bb0c00"
+        }
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(vectors) / sizeof(vectors[0]); ++i) {
+        if (!run_vector(&vectors[i], (int)i + 1)) return 1;
+    }
+    return 0;
+}

test/test_build_profiles.sh

@@ -500,10 +500,27 @@ for full_env in "${SUPPORTED_PIO_ENVS[@]}"; do
     || fail "$full_env Full Companion omitted USB folder seeding"
   [[ "$PLATFORMIO_BUILD_FLAGS" == *"COMPANION_FEATURE_BLE_MOTA_SOURCE=1"* ]] \
     || fail "$full_env Full Companion omitted BLE folder seeding"
+  if ! pio_env_option_contains "$pio_env" build_src_filter "helpers/ota/*.cpp" \
+      && [[ "$PLATFORMIO_BUILD_SRC_FILTER" != *"helpers/ota/*.cpp"* ]]; then
+    fail "$full_env Full Companion omitted the OTA C++ implementation"
+  fi
 done
 unset PLATFORMIO_BUILD_FLAGS PLATFORMIO_BUILD_UNFLAGS \
   PLATFORMIO_BUILD_SRC_FILTER
 
+# An environment that includes only the tiny OTA inflater still needs the C++
+# protocol implementation when build.sh overlays LoRa OTA.
+reduced_nrf_ota_env=RAK_3401_repeater_lora_ota_no_external_sensors
+reduced_nrf_pio_env=$(get_pio_build_env "$reduced_nrf_ota_env")
+PLATFORMIO_BUILD_SRC_FILTER=""
+PLATFORMIO_EXTRA_SCRIPTS=""
+apply_nrf52_lora_ota_build_recipe "$reduced_nrf_ota_env" "$reduced_nrf_pio_env"
+if ! pio_env_option_contains "$reduced_nrf_pio_env" build_src_filter "helpers/ota/*.cpp" \
+    && [[ "$PLATFORMIO_BUILD_SRC_FILTER" != *"helpers/ota/*.cpp"* ]]; then
+  fail "$reduced_nrf_ota_env omitted the OTA C++ implementation"
+fi
+unset PLATFORMIO_BUILD_SRC_FILTER PLATFORMIO_EXTRA_SCRIPTS
+
 # The bounded AsyncTCP task stack belongs to every ESP32 Full profile, including
 # expanded non-Companion builds, but must not leak into ordinary or nRF52
 # recipes.
@@ -764,6 +781,29 @@ if declare -f run_logging_matrix_build_targets | grep -q 'Profile 2'; then
   fail "logging matrix still labels a separate Profile 2"
 fi
 
+# Option 3 must not publish an oversized standard ESP32 image, but its expected
+# portable-slot status is satisfied by the mandatory expanded FULL pass.
+saved_supports_esp32_full_build=$(declare -f supports_esp32_full_build)
+supports_esp32_full_build() { [ "$1" = esp32_portable_overflow ]; }
+build_firmware() { return 42; }
+saved_output_dir=$OUTPUT_DIR
+OUTPUT_DIR=$version_test_dir
+LOGGING_MATRIX_FAILURES=()
+LOGGING_MATRIX_DEFERRED_TARGETS=()
+DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL=1
+FIRMWARE_FILENAME_INFIX=""
+MQTT_BRIDGE_OVERRIDE="off"
+if ! run_logged_build_targets esp32_portable_overflow >/dev/null; then
+  fail "portable ESP32 overflow made the logging pass fail"
+fi
+[ ${#LOGGING_MATRIX_FAILURES[@]} -eq 0 ] \
+  || fail "portable ESP32 overflow remained a logging-matrix failure"
+[ "${LOGGING_MATRIX_DEFERRED_TARGETS[*]}" = esp32_portable_overflow ] \
+  || fail "portable ESP32 overflow was not deferred to FULL"
+OUTPUT_DIR=$saved_output_dir
+unset DEFER_ESP32_PORTABLE_OVERFLOW_TO_FULL
+eval "$saved_supports_esp32_full_build"
+
 calls=()
 build_firmware() {
   calls+=("$1:$BUILD_PROFILE_EFFECTIVE:$SKIP_DECLARED_REDUCTIONS:$FIRMWARE_OUTPUT_ENV_NAME")

test/test_ed25519_compact.py

@@ -0,0 +1,60 @@
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ED25519 = ROOT / "lib" / "ed25519"
+HARNESS = ROOT / "test" / "fixtures" / "ed25519_compact" / "test_ed25519_compact.c"
+SOURCES = [
+    ED25519 / name
+    for name in (
+        "fe.c",
+        "ge.c",
+        "keypair.c",
+        "sc.c",
+        "sha512.c",
+        "sign.c",
+        "verify.c",
+    )
+]
+
+
+class Ed25519CompactTest(unittest.TestCase):
+    def test_stm32_builds_enable_compact_base_table(self):
+        platformio = (ROOT / "platformio.ini").read_text()
+        stm32_base = platformio.split("[stm32_base]", 1)[1].split("\n[", 1)[0]
+        self.assertIn("-D ED25519_COMPACT_BASE=1", stm32_base)
+
+    def test_standard_and_compact_paths_match_rfc8032(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for label, extra_flags in (
+                ("standard", []),
+                ("compact", ["-DED25519_COMPACT_BASE=1"]),
+            ):
+                executable = Path(temp_dir) / label
+                subprocess.run(
+                    [
+                        "cc",
+                        "-std=c99",
+                        "-O2",
+                        "-Wall",
+                        "-Wextra",
+                        "-Werror",
+                        "-DED25519_NO_SEED=1",
+                        *extra_flags,
+                        f"-I{ED25519}",
+                        str(HARNESS),
+                        *(str(source) for source in SOURCES),
+                        "-o",
+                        str(executable),
+                    ],
+                    check=True,
+                    cwd=ROOT,
+                )
+                subprocess.run([str(executable)], check=True, cwd=ROOT)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_storage_layout_cli.py

@@ -108,6 +108,14 @@ int main() {
         companion = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text()
         formatter = (ROOT / "src/helpers/StorageLayout.cpp").read_text()
 
+        stm32_include = formatter.index("#elif defined(STM32_PLATFORM)")
+        stm32_include_end = formatter.index("#endif", stm32_include)
+        stm32_headers = formatter[stm32_include:stm32_include_end]
+        self.assertLess(
+            stm32_headers.index("#include <Arduino.h>"),
+            stm32_headers.index("#include <InternalFileSystem.h>"),
+        )
+
         self.assertIn('#include "StorageLayout.h"', common)
         self.assertIn(
             "mesh::cli::handleStorageLayoutGet(config, *_board, reply, 160)",