diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 553493e78..82c882a28 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -16,6 +16,8 @@ jobs: - uses: actions/checkout@v4 - name: Setup Build Environment uses: ./.github/actions/setup-build-environment + - name: Verify nRF52 HEX packaging with installed build dependencies + run: python3 -B test/test_nrf52_endf_hex.py -v - name: Compile GPS Sensor tracker on nRF52 boards run: pio run -e t1000e_sensor -e RAK_WisMesh_Tag_sensor @@ -56,6 +58,11 @@ jobs: - name: Setup Build Environment uses: ./.github/actions/setup-build-environment + - name: Verify compact STM32 signature arithmetic + run: | + python3 -B test/test_ed25519_compact_scalar.py -v + python3 -B test/test_ed25519_compact.py -v + - name: Compile ordinary STM32 USB Companion with pinned core and RadioLib run: pio run -e RAK_3x72_companion_radio_usb diff --git a/README.md b/README.md index c400282a5..ccfdb8503 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ For developers: - Install [PlatformIO](https://docs.platformio.org) in [Visual Studio Code](https://code.visualstudio.com). - Clone and open the MeshCore repository in Visual Studio Code. -- Install the pinned build-time compressors with `python -m pip install -r requirements-build.txt`. +- Install the pinned build-time dependencies in PlatformIO's Python environment with `python -m pip install -r requirements-build.txt`. - See the example applications you can modify and run: - [Companion Radio](./examples/companion_radio) - For use with an external chat app, over BLE, USB or Wi-Fi. - [KISS Modem](./examples/kiss_modem) - Serial KISS protocol bridge for host applications. ([protocol docs](./docs/kiss_modem_protocol.md)) diff --git a/lib/ed25519/sc.c b/lib/ed25519/sc.c index ca5bad2ca..f90077064 100644 --- a/lib/ed25519/sc.c +++ b/lib/ed25519/sc.c @@ -1,6 +1,10 @@ #include "fixedint.h" #include "sc.h" +#if defined(ED25519_COMPACT_SCALAR) && ED25519_COMPACT_SCALAR +#include "sc_compact.h" +#else + static uint64_t load_3(const unsigned char *in) { uint64_t result; @@ -807,3 +811,5 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[30] = (unsigned char) (s11 >> 9); s[31] = (unsigned char) (s11 >> 17); } + +#endif diff --git a/lib/ed25519/sc_compact.h b/lib/ed25519/sc_compact.h new file mode 100644 index 000000000..10fc95d5d --- /dev/null +++ b/lib/ed25519/sc_compact.h @@ -0,0 +1,88 @@ +/* + * Loop form of the ref10 scalar arithmetic in sc.c. The 21-bit limbs, + * folding coefficients, and carry order are unchanged. Small STM32 + * Companion images cannot afford the unrolled reduction and multiply-add. + * All loop bounds and memory addresses depend only on public limb indices; + * scalar values never select a branch or an array index. + */ + +static void sc_unpack(int64_t *limbs, const unsigned char *bytes, int count) { + const int size = count == 24 ? 64 : 32; + int i; + for (i = 0; i < count; ++i) { + const int start = (i * 21) >> 3; + const int shift = (i * 21) & 7; + uint32_t word = 0; + int j; + for (j = 0; j < 4 && start + j < size; ++j) + word |= (uint32_t)bytes[start + j] << (j * 8); + word >>= shift; + /* The last limb retains every remaining input bit, as in ref10. */ + limbs[i] = (int64_t)(i == count - 1 ? word : word & 2097151); + } +} + +static void sc_carry(int64_t *limbs, int first, int last, int step, int rounded) { + int i; + for (i = first; i <= last; i += step) { + const int64_t value = (limbs[i] + (rounded ? 1048576 : 0)) >> 21; + limbs[i + 1] += value; + /* A carry can be negative; multiplication avoids a signed left shift. */ + limbs[i] -= value * 2097152; + } +} + +static void sc_fold(int64_t *limbs, int first, int last) { + static const int32_t coefficients[6] = {666643, 470296, 654183, -997805, 136657, -683901}; + int i; + for (i = first; i >= last; --i) { + int j; + for (j = 0; j < 6; ++j) + limbs[i - 12 + j] += limbs[i] * coefficients[j]; + limbs[i] = 0; + } +} + +static void sc_reduce_limbs(unsigned char *out, int64_t limbs[24]) { + int i; + sc_fold(limbs, 23, 18); + sc_carry(limbs, 6, 16, 2, 1); + sc_carry(limbs, 7, 15, 2, 1); + sc_fold(limbs, 17, 12); + sc_carry(limbs, 0, 10, 2, 1); + sc_carry(limbs, 1, 11, 2, 1); + sc_fold(limbs, 12, 12); + sc_carry(limbs, 0, 11, 1, 0); + sc_fold(limbs, 12, 12); + sc_carry(limbs, 0, 10, 1, 0); + /* Encode the same 256 low bits as ref10's byte assignments. */ + for (i = 0; i < 32; ++i) { + const int index = (i * 8) / 21; + const int shift = (i * 8) % 21; + uint64_t word = (uint64_t)limbs[index] >> shift; + if (shift > 13 && index < 11) + word |= (uint64_t)limbs[index + 1] << (21 - shift); + out[i] = (unsigned char)word; + } +} + +void sc_reduce(unsigned char *s) { + int64_t limbs[24]; + sc_unpack(limbs, s, 24); + sc_reduce_limbs(s, limbs); +} + +void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, const unsigned char *c) { + int64_t aa[12], bb[12], limbs[24] = {0}; + int i, j; + /* Read all inputs before writing the result; s may alias a, b, or c. */ + sc_unpack(aa, a, 12); + sc_unpack(bb, b, 12); + sc_unpack(limbs, c, 12); + for (i = 0; i < 12; ++i) + for (j = 0; j < 12; ++j) + limbs[i + j] += aa[i] * bb[j]; + sc_carry(limbs, 0, 22, 2, 1); + sc_carry(limbs, 1, 21, 2, 1); + sc_reduce_limbs(s, limbs); +} diff --git a/requirements-build.txt b/requirements-build.txt index 231294402..6a8685f4b 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1,2 +1,5 @@ +# nRF52 HEX post-build trailer generation. +intelhex==2.3.0 + # Reproducible compressors for firmware-resident and OTA assets. zopfli==0.4.3 diff --git a/test/test_cli_settings_contract.py b/test/test_cli_settings_contract.py index ad6915ae1..4e92ef441 100644 --- a/test/test_cli_settings_contract.py +++ b/test/test_cli_settings_contract.py @@ -182,7 +182,10 @@ class CLISettingsContractTest(unittest.TestCase): 'recent.repeaters', # Discovery/cache query }), ('simple_room_server', 'MyMesh.cpp', {'acl'}), - ('simple_sensor', 'SensorMesh.cpp', {'acl'}), + ('simple_sensor', 'SensorMesh.cpp', { + 'acl', + 'tracker', # Aggregate status; set tracker. changes settings + }), ): text = (ROOT/'examples'/role/filename).read_text(encoding='utf-8') sets, gets = full_command_keys(text) diff --git a/test/test_ed25519_compact.py b/test/test_ed25519_compact.py index aed61e2c0..1f910ef00 100644 --- a/test/test_ed25519_compact.py +++ b/test/test_ed25519_compact.py @@ -116,6 +116,10 @@ class Ed25519CompactTest(unittest.TestCase): ("compact_sha512", ["-DED25519_COMPACT_SHA512=1"]), ("compact_base_sha512", ["-DED25519_COMPACT_BASE=1", "-DED25519_COMPACT_SHA512=1"]), + ("compact_scalar", ["-DED25519_COMPACT_SCALAR=1"]), + ("compact_all", ["-DED25519_COMPACT_BASE=1", + "-DED25519_COMPACT_SHA512=1", + "-DED25519_COMPACT_SCALAR=1"]), ): executable = Path(temp_dir) / label subprocess.run( diff --git a/test/test_ed25519_compact_scalar.py b/test/test_ed25519_compact_scalar.py new file mode 100644 index 000000000..0a32591bf --- /dev/null +++ b/test/test_ed25519_compact_scalar.py @@ -0,0 +1,147 @@ +"""Check compact ref10 scalars against independent integer arithmetic.""" + +import ctypes +import itertools +from pathlib import Path +import random +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +ED25519 = ROOT / "lib/ed25519" +ORDER = 2**252 + 27742317777372353535851937790883648493 +EDGES = (0, 1, ORDER - 1, ORDER, ORDER + 1, 2**256 - 1) + + +def octets(value, size): + return value.to_bytes(size, "little") + + +class Ed25519CompactScalarTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + cls.libraries = [] + for compact in (0, 1): + library_path = Path(cls.directory.name) / f"scalar-{compact}.so" + subprocess.run(["cc", "-std=c99", "-O2", "-Wall", "-Wextra", "-Werror", + "-shared", "-fPIC", f"-DED25519_COMPACT_SCALAR={compact}", + str(ED25519 / "sc.c"), "-o", str(library_path)], check=True) + library = ctypes.CDLL(str(library_path)) + library.sc_reduce.argtypes = [ctypes.c_void_p] + library.sc_reduce.restype = None + library.sc_muladd.argtypes = [ctypes.c_void_p] * 4 + library.sc_muladd.restype = None + cls.libraries.append(library) + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + def test_reduction_matches_integer_oracle_and_standard(self): + rng = random.Random(0xED25519) + values = list(EDGES) + [2**512 - 1] + values += [ORDER * factor + delta for factor in (2, 7, 1000000) + for delta in (-1, 0, 1)] + values += [1 << bit for bit in range(512)] + values += [rng.getrandbits(512) for _ in range(4000)] + for value in values: + original = octets(value, 64) + expected = octets(value % ORDER, 32) + for compact, library in enumerate(self.libraries): + with self.subTest(value=value, compact=compact): + data = ctypes.create_string_buffer(original, 64) + library.sc_reduce(data) + self.assertEqual(data.raw[:32], expected) + self.assertEqual(data.raw[32:], original[32:]) + + def test_multiply_add_matches_integer_oracle_and_aliases(self): + rng = random.Random(0x25519) + triples = list(itertools.product(EDGES, repeat=3)) + triples += [tuple(rng.getrandbits(256) for _ in range(3)) for _ in range(4000)] + for a, b, c in triples: + expected = octets((a * b + c) % ORDER, 32) + for compact, library in enumerate(self.libraries): + for alias in range(4): + with self.subTest(a=a, b=b, c=c, compact=compact, alias=alias): + values = [ctypes.create_string_buffer(octets(value, 32), 32) + for value in (a, b, c)] + output = ctypes.create_string_buffer(32) if alias == 3 else values[alias] + library.sc_muladd(output, *values) + self.assertEqual(output.raw, expected) + + def test_compact_scalars_under_address_and_undefined_sanitizers(self): + rng = random.Random(0x8032) + reductions = [0, ORDER - 1, ORDER, ORDER + 1, 2**512 - 1] + reductions += [rng.getrandbits(512) for _ in range(64)] + triples = list(itertools.product(EDGES, repeat=3)) + triples += [tuple(rng.getrandbits(256) for _ in range(3)) for _ in range(64)] + + def array(value, size): + return "{" + ",".join(str(byte) for byte in octets(value, size)) + "}" + + reduce_vectors = ",\n".join( + "{" + array(value, 64) + "," + array(value % ORDER, 32) + "}" + for value in reductions) + muladd_vectors = ",\n".join( + "{" + ",".join(array(value, 32) for value in (a, b, c, (a * b + c) % ORDER)) + "}" + for a, b, c in triples) + source = r''' +#include +#include +#include "sc.h" +struct ReduceVector { unsigned char input[64], expected[32]; }; +struct MuladdVector { unsigned char a[32], b[32], c[32], expected[32]; }; +static const struct ReduceVector reductions[] = { @REDUCTIONS@ }; +static const struct MuladdVector triples[] = { @TRIPLES@ }; +int main(void) { + unsigned char data[64], a[32], b[32], c[32], separate[32]; + unsigned int i, alias; + for (i = 0; i < sizeof(reductions) / sizeof(reductions[0]); ++i) { + memcpy(data, reductions[i].input, sizeof(data)); + sc_reduce(data); + assert(memcmp(data, reductions[i].expected, 32) == 0); + assert(memcmp(data + 32, reductions[i].input + 32, 32) == 0); + } + for (i = 0; i < sizeof(triples) / sizeof(triples[0]); ++i) { + for (alias = 0; alias < 4; ++alias) { + unsigned char *out; + memcpy(a, triples[i].a, 32); + memcpy(b, triples[i].b, 32); + memcpy(c, triples[i].c, 32); + out = alias == 0 ? a : alias == 1 ? b : alias == 2 ? c : separate; + sc_muladd(out, a, b, c); + assert(memcmp(out, triples[i].expected, 32) == 0); + } + } + return 0; +} +'''.replace("@REDUCTIONS@", reduce_vectors).replace("@TRIPLES@", muladd_vectors) + with tempfile.TemporaryDirectory() as directory: + harness = Path(directory) / "scalars.c" + harness.write_text(source) + executable = Path(directory) / "scalars" + subprocess.run(["cc", "-std=c99", "-O1", "-g", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-no-pie", + "-DED25519_COMPACT_SCALAR=1", "-I", str(ED25519), + str(harness), str(ED25519 / "sc.c"), "-o", str(executable)], check=True) + subprocess.run([str(executable)], check=True) + + def test_only_stm32_companion_profiles_enable_compact_scalars(self): + profiles = [] + for ini in (ROOT / "variants").glob("*/platformio.ini"): + section = "" + for line in ini.read_text().splitlines(): + if line.startswith("["): + section = line + if "ED25519_COMPACT_SCALAR" in line: + profiles.append((ini.parent.name, section)) + self.assertEqual(sorted(profiles), [ + ("rak3x72", "[env:RAK_3x72_companion_radio_usb]"), + ("wio-e5-mini", "[env:wio-e5-mini_companion_radio_usb]"), + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_fleet_location.py b/test/test_fleet_location.py index 601ba5f82..ac6b52e92 100644 --- a/test/test_fleet_location.py +++ b/test/test_fleet_location.py @@ -107,6 +107,8 @@ int main(int argc, char** argv) { sensors.seed(47.6062f, -122.3321f); sensors.block(true); // Cached reading must not reclaim a bridge-owned UART. const unsigned starts = sensors.starts, stops = sensors.stops; + // UART ownership setup may consult the provider; cache reads must not. + const unsigned providers = sensors.providers, queries = sensors.queries; const bool active = sensors.active, receiving = sensors.receiving(); for (unsigned i = 0; i < 100; ++i) { assert(sensors.getCachedGpsPosition(latitude, longitude)); @@ -115,7 +117,7 @@ int main(int argc, char** argv) { assert(mesh::readFleetLocation(prefs, sensors, lat, lon)); } assert(sensors.starts == starts && sensors.stops == stops && sensors.active == active); - assert(sensors.receiving() == receiving && sensors.providers == 0 && sensors.queries == 0); + assert(sensors.receiving() == receiving && sensors.providers == providers && sensors.queries == queries); assert(sensors.powersaving_enabled && !sensors.active && !sensors.receiving()); } else if (scenario == "cache_freshness" || scenario == "cache_wrap") { const uint32_t start = scenario == "cache_wrap" ? UINT32_MAX - 1000 : 0; diff --git a/test/test_nrf52_endf_hex.py b/test/test_nrf52_endf_hex.py new file mode 100644 index 000000000..5802509f1 --- /dev/null +++ b/test/test_nrf52_endf_hex.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Exercise the nRF52 HEX post-build hook with the installed build dependencies.""" + +from contextlib import redirect_stdout +import importlib.util +import io +from pathlib import Path +import sys +import tempfile +import unittest + +from intelhex import IntelHex + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools" / "mota")) +import motalib as ml # noqa: E402 + + +class BuildEnvironment(dict): + def __init__(self, app_base): + super().__init__( + PROJECT_DIR=str(ROOT), + PIOENV="nrf52_sensor_hex_test", + CPPDEFINES=[ + "ENABLE_OTA", "NRF52_PLATFORM", + ("FIRMWARE_VERSION", "v1.17.1.9"), + ("MOTA_TARGET_ID", "0x12345678"), + ("MOTA_HW_ID", "NRF52_TEST"), + ], + ) + self.app_base = app_base + self.post_actions = {} + + def AddPostAction(self, target, action): + self.post_actions[target] = action + + def BoardConfig(self): + return {"upload.maximum_size": ml.NRF52_APP_END - self.app_base} + + def GetProjectOption(self, name, default=""): + if name in ("build_src_filter", "src_filter"): + return "+" + return default + + +class Nrf52EndfHexTests(unittest.TestCase): + def test_hex_hook_appends_valid_identity_and_preserves_other_segments(self): + for app_base in (ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_BASE_S140_V7): + with self.subTest(app_base=hex(app_base)), tempfile.TemporaryDirectory() as tmp: + env = BuildEnvironment(app_base) + spec = importlib.util.spec_from_file_location( + "nrf52_endf_hex_test", ROOT / "tools" / "mota" / "pio_endf.py") + module = importlib.util.module_from_spec(spec) + module.Import = lambda _name: None + module.env = env + spec.loader.exec_module(module) + action = env.post_actions["$BUILD_DIR/${PROGNAME}.hex"] + + path = Path(tmp) / "firmware.hex" + firmware = bytes(range(256)) + uicr_base = 0x10001080 + uicr = b"\x12\x34\x56\x78" + original = IntelHex() + original.frombytes(firmware, offset=app_base) + original.frombytes(uicr, offset=uicr_base) + original.write_hex_file(str(path)) + + with redirect_stdout(io.StringIO()): + action([], [path], env) + result = IntelHex(str(path)) + segments = result.segments() + self.assertEqual(len(segments), 2) + self.assertEqual(segments[0], ( + app_base, app_base + len(firmware) + ml.NRF52_LAYOUT_LEN + ml.ENDF_LEN)) + image = bytes(result.tobinarray(start=app_base, end=segments[0][1] - 1)) + self.assertEqual(image[:len(firmware)], firmware) + self.assertEqual(bytes(result.tobinarray( + start=uicr_base, size=len(uicr))), uicr) + self.assertTrue(ml.has_endf(image)) + self.assertEqual(ml.parse_endf_ident(image), ml.FwIdent( + ml.pack_version("1.17.1.9"), 0x12345678, "NRF52_TEST")) + self.assertEqual(ml.parse_nrf52_layout(image), ml.Nrf52Layout( + app_base, ml.NRF52_APP_END, ml.NRF52_APP_END)) + + first_pass = path.read_bytes() + with redirect_stdout(io.StringIO()): + action([], [path], env) + self.assertEqual(path.read_bytes(), first_pass) + + +if __name__ == "__main__": + unittest.main() diff --git a/variants/rak3x72/platformio.ini b/variants/rak3x72/platformio.ini index 7950677c1..792f5c80e 100644 --- a/variants/rak3x72/platformio.ini +++ b/variants/rak3x72/platformio.ini @@ -67,6 +67,7 @@ platform_packages = platformio/toolchain-gccarmnoneeabi@^1.140201.0 build_unflags = -Os build_flags = ${rak3x72.build_flags} -D ED25519_COMPACT_SHA512=1 ; same hash rounds, compact loop preserves the filesystem boundary + -D ED25519_COMPACT_SCALAR=1 ; same ref10 arithmetic with fixed loops instead of unrolled code -Oz -flto -fno-inline-small-functions diff --git a/variants/wio-e5-mini/platformio.ini b/variants/wio-e5-mini/platformio.ini index d979b0169..5005c1b9a 100644 --- a/variants/wio-e5-mini/platformio.ini +++ b/variants/wio-e5-mini/platformio.ini @@ -80,6 +80,7 @@ build_unflags = -Os build_flags = ${lora_e5_mini.build_flags} -D ED25519_COMPACT_SHA512=1 ; same hash rounds, compact loop preserves the filesystem boundary + -D ED25519_COMPACT_SCALAR=1 ; same ref10 arithmetic with fixed loops instead of unrolled code -Oz -flto -fno-inline-small-functions