dandri/HaloKeymind
1
0
Fork 0
mirror of https://github.com/mikecarper/MeshCore.git synced 2026-09-15 01:16:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Build inflater-capable RAK3401 bridge inventory

Browse Source
This commit is contained in:
mikecarper
2026-09-02 22:48:22 -07:00
parent 3f6eddd5c2
commit ecbe02c28c
9 changed files with 1987 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/lora_ota/build_rak3401_inflate_bridges.py
+724
View File
@@ -0,0 +1,724 @@
#!/usr/bin/env python3
"""Build receive-DEFLATE RAK3401 bridge images for the historical mOTA chain.
The input commits are pinned because each image is a deliberate binary bridge.
The script uses one temporary Git worktree, applies the reviewed backport
patches plus the pinned receive-only transport shim without committing them,
and emits a machine-readable image manifest.
"""
from __future__ import annotations
import argparse
from dataclasses import dataclass
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import zipfile
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parent.parent
sys.path.insert(0, str(REPO_ROOT / "tools" / "mota"))
import motalib # noqa: E402
ENV_NAME = "RAK_3401_repeater_lora_ota_no_external_sensors"
VERSION_SUFFIX = "halo-keymind-cascade-mota-inflate"
EXPECTED_TARGET_ID = 0x2FA509C1
EXPECTED_HARDWARE = "RAK_3401"
COMMON_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_backport.patch"
LEGACY_MESH_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_mesh_legacy.patch"
GUARDED_MESH_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_mesh_guarded.patch"
VERSION_PATCH = SCRIPT_DIR / "rak3401_four_component_endf_backport.patch"
WORKSPACE_PATCH = SCRIPT_DIR / "rak3401_dynamic_workspace_backport.patch"
SELECTIVE_OS_HOOK = SCRIPT_DIR / "rak3401_selective_os.py"
INFLATE_ASSET_ROOT = SCRIPT_DIR / "rak3401_inflate_assets"
INFLATE_SNAPSHOT_COMMIT = "add51bf00c46c15ef54318ca766a6daf08a147ee"
INFLATE_ASSETS = {
"src/helpers/ota/OtaDeflate.cpp": (
"OtaDeflate.cpp",
"7cc464fc3c304cc65f52973593068c8d85a783853da121476c4bc2e196798e8e",
),
"src/helpers/ota/OtaDeflate.h": (
"OtaDeflate.h",
"f6cd5b0fe8b2164178881d93664df37722d6434f8ef61c37d08255bce12f842c",
),
"src/helpers/ota/OtaTinf.c": (
"OtaTinf.c",
"e1344c73f3ef7ea9b19ab1358990f7aed70b9011aa2b370b5e0872c6699d60e8",
),
"src/helpers/ota/tinf/LICENSE": (
"tinf/LICENSE",
"bf5d1b9b3eb2917e32e030d64dd963356b76751b41a7a432f1187bb199cb0b00",
),
"src/helpers/ota/tinf/README.meshcore.txt": (
"tinf/README.meshcore.txt",
"8d1f4ae316ff71c58efaf4f85ed8d0ef6d10729919c5db37565c50ccc64ea041",
),
"src/helpers/ota/tinf/tinf.h": (
"tinf/tinf.h",
"831b59414b3ddf3ecb2e5082a7d8e03b56b7eecc05385c868c0e6537e9c57941",
),
"src/helpers/ota/tinf/tinflate.c": (
"tinf/tinflate.c",
"f5bc68a32a50f7e34c63928a8c176e65a5dc496256338b758ed8b714e24a2982",
),
}
TRACKED_BACKPORT_FILES = (
"platformio.ini",
"src/Mesh.cpp",
"src/helpers/ota/OtaContext.h",
"src/helpers/ota/OtaFlashLayout_nrf52.h",
"src/helpers/ota/OtaManager.cpp",
"src/helpers/ota/OtaManager.h",
"src/helpers/ota/OtaProtocol.h",
"tools/mota/pio_endf.py",
)
@dataclass(frozen=True)
class Target:
version: str
source_commit: str
bridge_stage: int = 0
os_stage: int = 0
os_stage_part: int = 0
os_stage_parts: int = 0
os_stage_subpart: int = 0
os_stage_subparts: int = 0
split_stage5: str = ""
mymesh_opt: str = ""
flood_rule_engine: bool = True
TARGETS = (
Target("1.16.7.10", "90abbd110ef4fa7c96fca30205bbc566bf5c966c", bridge_stage=6),
Target("1.16.7.11", "90abbd110ef4fa7c96fca30205bbc566bf5c966c", bridge_stage=7),
Target("1.16.7.12", "90abbd110ef4fa7c96fca30205bbc566bf5c966c", bridge_stage=8),
Target("1.16.7.13", "804f45303ac615362ab56f44da0789301bb3de11"),
Target("1.16.8.0", "cd89b5400cbc16b31a87079a793bc88600369c80"),
Target("1.16.8.7", "1fa940aa3a5d98d41a7320a25d60798edeae4921"),
Target("1.16.8.8", "bbc2c905a1befb83365cccec583c527a24ebb104"),
Target("1.16.8.9", "bec98977389bae5116d675107fa6c11eeab1a3f7"),
Target("1.16.9.0", "1f7f2a8034f0fc74b378dfb44126ef869c2a9344"),
Target("1.16.9.102", "63ab53e4445d726945ce32d530252e34ecf6d1b1"),
Target("1.16.9.104", "262542cf3411bfd093978c502ebb94ba3ba3cc0c", flood_rule_engine=False),
Target("1.16.9.105", "352d70465276d66cda5cbda955aa3dfffee39bbb", flood_rule_engine=False),
Target("1.16.9.108", "352d70465276d66cda5cbda955aa3dfffee39bbb"),
Target("1.16.9.109", "03edc027b19086918f567561cab673fe3724b9b8"),
Target("1.16.9.110", "07a624af00cea922ae495612b728e4abf51f9f87"),
Target("1.16.9.111", "4efd561dcdbafd345641f2a07689d4657b103eed", os_stage=1),
Target("1.16.9.112", "217b78aac01f538cac49afe79886371a8b000528", os_stage=1),
Target("1.16.9.113", "217b78aac01f538cac49afe79886371a8b000528", os_stage=2),
Target("1.16.9.114", "217b78aac01f538cac49afe79886371a8b000528", os_stage=3),
Target("1.16.9.115", "217b78aac01f538cac49afe79886371a8b000528", os_stage=4),
Target(
"1.16.9.116", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=5, split_stage5="first",
),
Target("1.16.9.117", "217b78aac01f538cac49afe79886371a8b000528", os_stage=5),
Target("1.16.9.118", "217b78aac01f538cac49afe79886371a8b000528", os_stage=7),
Target(
"1.16.9.119", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=8, mymesh_opt="O2_SIZE_1_INLINE_HOIST",
),
Target(
"1.16.9.120", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=8, mymesh_opt="O2_SIZE_1_HOIST",
),
Target(
"1.16.9.121", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=8, mymesh_opt="-Os",
),
Target(
"1.16.9.122", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=12, mymesh_opt="-Os",
),
Target(
"1.16.10.0", "217b78aac01f538cac49afe79886371a8b000528",
os_stage=13, mymesh_opt="-Os",
),
)
LEGACY_MESH_COMMITS = {
"90abbd110ef4fa7c96fca30205bbc566bf5c966c",
"804f45303ac615362ab56f44da0789301bb3de11",
}
class BuildError(RuntimeError):
pass
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def replace_exact(path: Path, old: bytes, new: bytes, label: str) -> None:
data = path.read_bytes()
count = data.count(old)
if count != 1:
raise BuildError(
f"{label}: expected one exact anchor in {path}, found {count}"
)
path.write_bytes(data.replace(old, new, 1))
def inflate_asset_metadata() -> dict[str, dict[str, str]]:
metadata: dict[str, dict[str, str]] = {}
for destination, (relative_source, expected_sha256) in INFLATE_ASSETS.items():
source = INFLATE_ASSET_ROOT / relative_source
if not source.is_file():
raise BuildError(f"missing receive-inflate asset: {source}")
actual_sha256 = sha256_file(source)
if actual_sha256 != expected_sha256:
raise BuildError(
f"receive-inflate asset hash mismatch for {relative_source}: "
f"{actual_sha256}, expected {expected_sha256}"
)
metadata[destination] = {
"asset": relative_source,
"sha256": actual_sha256,
}
return metadata
def install_inflate_assets(source: Path) -> None:
for destination, (relative_source, _) in INFLATE_ASSETS.items():
output = source / destination
if output.exists():
raise BuildError(
f"historical source unexpectedly already contains {destination}"
)
output.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(INFLATE_ASSET_ROOT / relative_source, output)
def remove_inflate_assets(source: Path) -> None:
for destination in reversed(tuple(INFLATE_ASSETS)):
output = source / destination
if output.is_file() or output.is_symlink():
output.unlink()
elif output.exists():
raise BuildError(f"receive-inflate asset is not a file: {output}")
tinf_dir = source / "src/helpers/ota/tinf"
if tinf_dir.exists():
try:
tinf_dir.rmdir()
except OSError as exc:
raise BuildError(f"receive-inflate asset directory is not empty: {tinf_dir}") from exc
def restore_backport(source: Path) -> None:
run(
[
"git", "restore", "--source=HEAD", "--staged", "--worktree", "--",
*TRACKED_BACKPORT_FILES,
],
"restore prior backport",
source,
)
remove_inflate_assets(source)
def validate_inflate_source(source: Path) -> None:
manager_h = (source / "src/helpers/ota/OtaManager.h").read_text(encoding="utf-8")
manager_cpp = (source / "src/helpers/ota/OtaManager.cpp").read_text(encoding="utf-8")
context_h = (source / "src/helpers/ota/OtaContext.h").read_text(encoding="utf-8")
protocol_h = (source / "src/helpers/ota/OtaProtocol.h").read_text(encoding="utf-8")
platformio = (source / "platformio.ini").read_text(encoding="utf-8")
required = {
"void OTA send ABI": "typedef void (*OtaSend)",
"void manager receive ABI": "void on_message(const uint8_t* msg, uint16_t len)",
"requested block hook declaration": "uint16_t requestedBlockLength(",
"requested block hook implementation": "uint16_t OtaManager::requestedBlockLength(",
"inflater context member": "OtaTransportInflateReceiver transport_inflate;",
"inflater context begin": "transport_inflate.begin(manager, target_id, send, ctx);",
"v2 request marker": "OTA_REQ_V2_MARK = 0x8000u",
"v2 fragment size": "#define OTA_FRAG_DATA_V2 171",
"tinf build filter": "+<helpers/ota/OtaTinf.c>",
}
haystacks = {
"void OTA send ABI": manager_h,
"void manager receive ABI": manager_h,
"requested block hook declaration": manager_h,
"requested block hook implementation": manager_cpp,
"inflater context member": context_h,
"inflater context begin": context_h,
"v2 request marker": protocol_h,
"v2 fragment size": manager_h,
"tinf build filter": platformio,
}
for label, needle in required.items():
if needle not in haystacks[label]:
raise BuildError(f"receive-inflate validation lacks {label}: {needle}")
for destination, (_, expected_sha256) in INFLATE_ASSETS.items():
actual_sha256 = sha256_file(source / destination)
if actual_sha256 != expected_sha256:
raise BuildError(
f"installed receive-inflate asset mismatch for {destination}: {actual_sha256}"
)
run(["git", "diff", "--check"], "backport whitespace check", source)
def apply_inflate_transforms(source: Path) -> None:
"""Apply the add51bf0 receive shim through exact, fail-closed anchors."""
manager_h = source / "src/helpers/ota/OtaManager.h"
replace_exact(
manager_h,
b"#ifndef OTA_FRAG_DATA\n"
b"#define OTA_FRAG_DATA 160 // data bytes per DATA fragment (<= MAX_PACKET_PAYLOAD - 9-byte header)\n"
b"#endif\n",
b"#ifndef OTA_FRAG_DATA\n"
b"#define OTA_FRAG_DATA 160 // deployed legacy DATA geometry; never change in place\n"
b"#endif\n"
b"#ifndef OTA_FRAG_DATA_V2\n"
b"#define OTA_FRAG_DATA_V2 171 // negotiated DATA geometry (13-byte overhead => 184-byte packet payload)\n"
b"#endif\n",
"OTA fragment constants",
)
replace_exact(
manager_h,
b" bool terminallyConsumes(const uint8_t* msg, uint16_t len);\n"
b" void on_message(const uint8_t* msg, uint16_t len); // feed one received OTA message\n",
b" bool terminallyConsumes(const uint8_t* msg, uint16_t len);\n"
b" // Exact logical length of a block this receiver currently requested, or zero for unsolicited DATA.\n"
b" // Used by the historical transport shim without exposing or changing the manager's reassembly buffers.\n"
b" uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const;\n"
b" void on_message(const uint8_t* msg, uint16_t len); // feed one received OTA message\n",
"OtaManager receive shim declaration",
)
manager_cpp = source / "src/helpers/ota/OtaManager.cpp"
replace_exact(
manager_cpp,
b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n",
b"uint16_t OtaManager::requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const {\n"
b" if (!manifest_id || !_fetch || _fstate != FETCHING || block >= _fbc ||\n"
b" memcmp(manifest_id, _fid, sizeof(_fid)) != 0 || findReassemblySlot(block) < 0) return 0;\n"
b" const uint32_t len = blockLen(block);\n"
b" return len <= OTA_MAX_BLOCK ? (uint16_t)len : 0;\n"
b"}\n\n"
b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n",
"OtaManager receive shim implementation",
)
protocol_h = source / "src/helpers/ota/OtaProtocol.h"
data_message = (
b"struct DataMsg {\n"
b" uint8_t manifest_id[4];\n"
b" uint16_t block_idx;\n"
b" uint16_t frag_off;\n"
b" const uint8_t* data; uint16_t data_len;\n"
b"};\n"
)
v2_protocol = data_message + (
b"\n// OTA_REQ want_mask extension. These bits sit outside every valid fragment bit for <=1 KiB blocks. A first\n"
b"// v2 request deliberately includes all seven legacy fragment bits so an old source can answer it completely.\n"
b"static const uint16_t OTA_REQ_V2_MARK = 0x8000u;\n"
b"static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u;\n"
b"static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u;\n"
b"static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu;\n"
b"\n// OTA_DATA v2 frag_off packing:\n"
b"// bit 15 v2 marker\n"
b"// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes)\n"
b"// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2)\n"
b"// bits 9..0 total encoded block length minus one (1..1024 bytes)\n"
b"static const uint16_t OTA_DATA_V2_MARK = 0x8000u;\n"
b"static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u;\n"
b"static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u;\n"
b"static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu;\n"
b"static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10;\n"
b"static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024;\n"
b"static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4;\n"
b"\ninline bool ota_req_is_v2(uint16_t want_mask) {\n"
b" return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0;\n"
b"}\n"
b"\ninline uint16_t ota_req_v2_fragments(uint16_t want_mask) {\n"
b" return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK);\n"
b"}\n"
b"\ninline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) {\n"
b" return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK |\n"
b" (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0));\n"
b"}\n"
b"\ninline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated,\n"
b" uint16_t& packed) {\n"
b" if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;\n"
b" packed = (uint16_t)(OTA_DATA_V2_MARK |\n"
b" (deflated ? OTA_DATA_V2_DEFLATED : 0) |\n"
b" ((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) |\n"
b" (encoded_len - 1u));\n"
b" return true;\n"
b"}\n"
b"\ninline bool ota_data_v2_unpack(uint16_t packed, uint8_t& fragment, uint16_t& encoded_len,\n"
b" bool& deflated) {\n"
b" if ((packed & OTA_DATA_V2_MARK) == 0) return false;\n"
b" fragment = (uint8_t)((packed & OTA_DATA_V2_FRAGMENT_BITS) >> OTA_DATA_V2_FRAGMENT_SHIFT);\n"
b" encoded_len = (uint16_t)((packed & OTA_DATA_V2_LENGTH_BITS) + 1u);\n"
b" deflated = (packed & OTA_DATA_V2_DEFLATED) != 0;\n"
b" return true;\n"
b"}\n"
)
replace_exact(protocol_h, data_message, v2_protocol, "OTA v2 protocol helpers")
context_h = source / "src/helpers/ota/OtaContext.h"
replace_exact(
context_h,
b'#include "OtaManager.h"\n#include "OtaStore.h"\n',
b'#include "OtaManager.h"\n#include "OtaDeflate.h"\n#include "OtaStore.h"\n',
"OtaContext inflater include",
)
replace_exact(
context_h,
b"struct OtaContext {\n OtaManager manager;\n",
b"struct OtaContext {\n OtaManager manager;\n OtaTransportInflateReceiver transport_inflate;\n",
"OtaContext inflater member",
)
replace_exact(
context_h,
b" void begin(uint32_t target_id, OtaSend send, void* ctx, const char* hw = nullptr) {\n",
b" void on_message(const uint8_t* msg, uint16_t len) {\n"
b" transport_inflate.on_message(msg, len);\n"
b" }\n\n"
b" void begin(uint32_t target_id, OtaSend send, void* ctx, const char* hw = nullptr) {\n",
"OtaContext inflater receive wrapper",
)
replace_exact(
context_h,
b" manager.begin(target_id, send, ctx);\n",
b" transport_inflate.begin(manager, target_id, send, ctx);\n",
"OtaContext inflater begin wrapper",
)
mesh_cpp = source / "src/Mesh.cpp"
replace_exact(
mesh_cpp,
b"ota::ota_ctx().manager.on_message(pkt->payload, pkt->payload_len);",
b"ota::ota_ctx().on_message(pkt->payload, pkt->payload_len); ",
"Mesh inflater receive wrapper",
)
platformio = source / "platformio.ini"
replace_exact(
platformio,
b" +<helpers/*.cpp>\n ; MQTT-only sources",
b" +<helpers/*.cpp>\n +<helpers/ota/OtaTinf.c>\n ; MQTT-only sources",
"PlatformIO tinf C source",
)
install_inflate_assets(source)
def run(
command: list[str],
label: str,
cwd: Path = REPO_ROOT,
timeout: int = 900,
env: dict[str, str] | None = None,
) -> str:
try:
result = subprocess.run(
command,
cwd=cwd,
env=env,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=timeout,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise BuildError(f"{label} failed: {exc}") from exc
if result.returncode != 0:
raise BuildError(
f"{label} exited {result.returncode}:\n{result.stdout[-6000:]}"
)
return result.stdout
def read_image(path: Path) -> bytes:
with zipfile.ZipFile(path) as archive:
members = [item for item in archive.infolist() if item.filename == "firmware.bin"]
if len(members) != 1:
raise BuildError(f"{path} does not contain one root firmware.bin")
return archive.read(members[0])
def verify_image(path: Path, target: Target) -> dict[str, object]:
image = read_image(path)
if not motalib.has_endf(image):
raise BuildError(f"{path} has no valid EndF")
ident = motalib.parse_endf_ident(image)
assert ident is not None
version = motalib.unpack_version(ident.fw_version)
if version != target.version:
raise BuildError(f"{path} version is {version}, expected {target.version}")
if ident.target_id != EXPECTED_TARGET_ID or ident.hw_id != EXPECTED_HARDWARE:
raise BuildError(
f"{path} identity is target={ident.target_id:08X} hw={ident.hw_id!r}"
)
body, body_hash = motalib.parse_endf(image)
return {
"version": target.version,
"source_commit": target.source_commit,
"zip": path.name,
"firmware_size": len(image),
"firmware_sha256": hashlib.sha256(image).hexdigest(),
"body_size": len(body),
"body_hash": body_hash.hex(),
}
def apply_backport(source: Path, commit: str) -> None:
required = (
COMMON_PATCH,
LEGACY_MESH_PATCH if commit in LEGACY_MESH_COMMITS else GUARDED_MESH_PATCH,
WORKSPACE_PATCH,
)
for patch in required:
run(["git", "apply", "--check", str(patch)], f"check {patch.name}", source)
run(["git", "apply", str(patch)], f"apply {patch.name}", source)
version_check = subprocess.run(
["git", "apply", "--check", str(VERSION_PATCH)],
cwd=source,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
if version_check.returncode == 0:
run(["git", "apply", str(VERSION_PATCH)], f"apply {VERSION_PATCH.name}", source)
else:
run(
["git", "apply", "--reverse", "--check", str(VERSION_PATCH)],
f"confirm existing {VERSION_PATCH.name}",
source,
)
apply_inflate_transforms(source)
validate_inflate_source(source)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--work-dir", type=Path, required=True)
parser.add_argument("--output-dir", type=Path)
parser.add_argument(
"--validate-backports-only",
action="store_true",
help="apply and verify every unique pinned source without running PlatformIO",
)
parser.add_argument("--keep-worktree", action="store_true")
args = parser.parse_args()
for patch in (
COMMON_PATCH, LEGACY_MESH_PATCH, GUARDED_MESH_PATCH, VERSION_PATCH,
WORKSPACE_PATCH,
):
if not patch.is_file():
raise BuildError(f"missing backport patch: {patch}")
if not SELECTIVE_OS_HOOK.is_file():
raise BuildError(f"missing selective optimizer hook: {SELECTIVE_OS_HOOK}")
asset_metadata = inflate_asset_metadata()
if not args.validate_backports_only and args.output_dir is None:
raise BuildError("--output-dir is required unless --validate-backports-only is used")
if args.work_dir.exists():
raise BuildError(f"work directory already exists: {args.work_dir}")
if args.output_dir is not None and args.output_dir.exists():
raise BuildError(f"output directory already exists: {args.output_dir}")
args.work_dir.mkdir(parents=True)
if args.output_dir is not None:
args.output_dir.mkdir(parents=True)
log_dir = args.work_dir / "logs"
if not args.validate_backports_only:
log_dir.mkdir()
source = args.work_dir / "source"
run(
["git", "worktree", "add", "--detach", str(source), TARGETS[0].source_commit],
"create build worktree",
)
current_commit = ""
records: list[dict[str, object]] = []
succeeded = False
try:
targets = TARGETS
if args.validate_backports_only:
unique: dict[str, Target] = {}
for target in TARGETS:
unique.setdefault(target.source_commit, target)
targets = tuple(unique.values())
for index, target in enumerate(targets, 1):
if target.source_commit != current_commit:
if current_commit:
restore_backport(source)
run(
["git", "checkout", "--detach", target.source_commit],
f"checkout {target.source_commit}",
source,
)
apply_backport(source, target.source_commit)
current_commit = target.source_commit
if args.validate_backports_only:
print(
f"[validate] {index:02d}/{len(targets)} "
f"{target.source_commit} receive-inflate backport applies",
flush=True,
)
continue
firmware_version = f"v{target.version}-{VERSION_SUFFIX}"
build_env = dict(os.environ)
for name in (
"MOTA_BRIDGE_OS_STAGE", "MOTA_BRIDGE_OS_STAGE_PART",
"MOTA_BRIDGE_OS_STAGE_PARTS", "MOTA_BRIDGE_OS_STAGE_SUBPART",
"MOTA_BRIDGE_OS_STAGE_SUBPARTS", "MOTA_BRIDGE_SPLIT_STAGE5",
"MOTA_BRIDGE_MY_MESH_OPT", "MOTA_BRIDGE_OS_VERBOSE",
"PLATFORMIO_BUILD_FLAGS", "PLATFORMIO_EXTRA_SCRIPTS",
):
build_env.pop(name, None)
build_env["DISABLE_DEBUG"] = "1"
flags = [
"-DLORA_FREQ=910.525", "-DLORA_BW=62.5", "-DLORA_SF=7",
"-DLORA_CR=5", "-DOTA_FETCH_PIPELINE=1", "-DOTA_TX_PRIORITY=0",
]
if not target.flood_rule_engine:
flags.append("-DMESH_ENABLE_FLOOD_RULE_ENGINE=0")
if target.bridge_stage:
flags.append(f"-DMOTA_BRIDGE_586_STAGE={target.bridge_stage}")
if target.os_stage:
build_env["MOTA_BRIDGE_OS_STAGE"] = str(target.os_stage)
build_env["PLATFORMIO_EXTRA_SCRIPTS"] = f"pre:{SELECTIVE_OS_HOOK}"
if target.os_stage_parts:
build_env["MOTA_BRIDGE_OS_STAGE_PART"] = str(target.os_stage_part)
build_env["MOTA_BRIDGE_OS_STAGE_PARTS"] = str(target.os_stage_parts)
if target.os_stage_subparts:
build_env["MOTA_BRIDGE_OS_STAGE_SUBPART"] = str(target.os_stage_subpart)
build_env["MOTA_BRIDGE_OS_STAGE_SUBPARTS"] = str(target.os_stage_subparts)
if target.split_stage5:
build_env["MOTA_BRIDGE_SPLIT_STAGE5"] = target.split_stage5
if target.mymesh_opt:
build_env["MOTA_BRIDGE_MY_MESH_OPT"] = target.mymesh_opt
build_env["PLATFORMIO_BUILD_FLAGS"] = " ".join(flags)
run(
["pio", "run", "-e", ENV_NAME, "-t", "clean"],
f"clean {target.version}",
source,
)
output = run(
[
"bash", "build.sh", "build-firmware", ENV_NAME,
"--profile", "cascade", "--firmware-version", firmware_version,
],
f"build {target.version}",
source,
env=build_env,
)
(log_dir / f"build-{index:02d}-v{target.version}.log").write_text(
output, encoding="utf-8"
)
for required in (*flags, "-DMOTA_TARGET_ID=0x2fa509c1", "SUCCESS"):
if required not in output:
raise BuildError(f"build {target.version} log lacks {required}")
if target.os_stage and f"selective optimizer stage {target.os_stage}/13" not in output:
raise BuildError(f"build {target.version} did not run the selective optimizer")
stem = f"{ENV_NAME}-ota-{firmware_version}-{target.source_commit[:8]}"
built_zip = source / "out" / f"{stem}.zip"
built_uf2 = source / "out" / f"{stem}.uf2"
if not built_zip.is_file() or not built_uf2.is_file():
raise BuildError(f"build {target.version} did not emit the expected ZIP and UF2")
assert args.output_dir is not None
output_zip = args.output_dir / built_zip.name
output_uf2 = args.output_dir / built_uf2.name
shutil.copy2(built_zip, output_zip)
shutil.copy2(built_uf2, output_uf2)
record = verify_image(output_zip, target)
record["uf2"] = output_uf2.name
record["zip_sha256"] = sha256_file(output_zip)
record["uf2_sha256"] = sha256_file(output_uf2)
record["build_flags"] = flags
record["bridge_stage"] = target.bridge_stage
record["os_stage"] = target.os_stage
record["os_stage_part"] = target.os_stage_part
record["os_stage_parts"] = target.os_stage_parts
record["os_stage_subpart"] = target.os_stage_subpart
record["os_stage_subparts"] = target.os_stage_subparts
record["split_stage5"] = target.split_stage5
record["mymesh_opt"] = target.mymesh_opt
record["flood_rule_engine"] = target.flood_rule_engine
records.append(record)
print(
f"[build] {index:02d}/{len(TARGETS)} v{target.version} "
f"image={record['firmware_size']} sha={str(record['firmware_sha256'])[:16]}",
flush=True,
)
if args.validate_backports_only:
succeeded = True
print(
f"[validate] all {len(targets)} unique pinned source commits passed",
flush=True,
)
return 0
assert args.output_dir is not None
manifest = {
"environment": ENV_NAME,
"target_id": f"{EXPECTED_TARGET_ID:08X}",
"hardware": EXPECTED_HARDWARE,
"version_suffix": VERSION_SUFFIX,
"inflate_snapshot_commit": INFLATE_SNAPSHOT_COMMIT,
"inflate_assets": asset_metadata,
"builder_script": Path(__file__).name,
"builder_script_sha256": sha256_file(Path(__file__)),
"common_patch": COMMON_PATCH.name,
"common_patch_sha256": sha256_file(COMMON_PATCH),
"legacy_mesh_patch": LEGACY_MESH_PATCH.name,
"legacy_mesh_patch_sha256": sha256_file(LEGACY_MESH_PATCH),
"guarded_mesh_patch": GUARDED_MESH_PATCH.name,
"guarded_mesh_patch_sha256": sha256_file(GUARDED_MESH_PATCH),
"version_patch": VERSION_PATCH.name,
"version_patch_sha256": sha256_file(VERSION_PATCH),
"workspace_patch": WORKSPACE_PATCH.name,
"workspace_patch_sha256": sha256_file(WORKSPACE_PATCH),
"selective_os_hook": SELECTIVE_OS_HOOK.name,
"selective_os_hook_sha256": sha256_file(SELECTIVE_OS_HOOK),
"transport_fragment_bytes": 171,
"transport_codec": "raw-deflate-receive-only",
"fetch_pipeline": 1,
"targets": records,
}
(args.output_dir / "images.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
succeeded = True
print(f"[manifest] {args.output_dir / 'images.json'}", flush=True)
finally:
if succeeded and not args.keep_worktree:
run(
["git", "worktree", "remove", "--force", str(source)],
"remove build worktree",
)
if args.validate_backports_only:
args.work_dir.rmdir()
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except BuildError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
raise SystemExit(2)
tools/lora_ota/rak3401_inflate_assets/OtaDeflate.cpp
+250
View File
@@ -0,0 +1,250 @@
#include "OtaDeflate.h"
#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST)
#include "Multihash.h"
#include <string.h>
extern "C" {
#include "tinf/tinf.h"
}
namespace mesh {
namespace ota {
static_assert(OTA_MAX_BLOCK <= OTA_DATA_V2_MAX_ENCODED,
"transport descriptor must represent every OTA logical block");
static_assert(OTA_FETCH_PIPELINE == 1,
"historical transport shim requires OTA_FETCH_PIPELINE=1");
static uint16_t transport_fragment_mask(uint16_t bytes, uint16_t fragment_bytes) {
const uint16_t count = (uint16_t)((bytes + fragment_bytes - 1u) / fragment_bytes);
return count >= 16 ? 0xFFFFu : (uint16_t)((1u << count) - 1u);
}
bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len,
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len) {
(void)context;
if (dst_len) *dst_len = 0;
if (!src || src_len == 0 || !dst || dst_cap == 0 || !dst_len) return false;
unsigned int produced = dst_cap;
const int result = tinf_uncompress_exact(dst, &produced, src, src_len);
if (result != TINF_OK || produced != dst_cap) return false;
*dst_len = (uint16_t)produced;
return true;
}
void OtaTransportInflateReceiver::begin(OtaManager& manager, uint32_t target_id,
OtaSend send_fn, void* send_ctx) {
_manager = &manager;
_send = send_fn;
_send_ctx = send_ctx;
reset_session();
manager.begin(target_id, send_trampoline, this);
}
void OtaTransportInflateReceiver::reset_representation() {
_encoded_len = 0;
_mask = 0;
_need = 0;
_partial_retries = 0;
_request_sent = false;
_complete = false;
_deflated = false;
memset(_stream_id, 0, sizeof(_stream_id));
}
void OtaTransportInflateReceiver::reset_session() {
reset_representation();
_active = false;
_block = 0;
memset(_mid, 0, sizeof(_mid));
_allow_deflate = true;
_legacy_session = false;
}
bool OtaTransportInflateReceiver::is_active(const uint8_t* mid, uint16_t block) const {
return _active && mid && block == _block && memcmp(mid, _mid, sizeof(_mid)) == 0;
}
void OtaTransportInflateReceiver::send_trampoline(void* ctx, const uint8_t* msg,
uint16_t len, bool flood) {
if (ctx) static_cast<OtaTransportInflateReceiver*>(ctx)->send(msg, len, flood);
}
void OtaTransportInflateReceiver::send(const uint8_t* msg, uint16_t len, bool flood) {
if (!_send || !msg || len == 0) return;
// A new manifest request starts a new negotiation. A retry while still waiting for that manifest is also
// safe to reset because no payload block can yet be active.
if (ota_msg_type(msg, len) == OTA_GET_MANIFEST) reset_session();
ReqMsg request;
if (ota_msg_type(msg, len) == OTA_REQ && decode_req(msg, len, request)) {
if (!is_active(request.manifest_id, request.block_idx)) {
// resumeStaged() can enter FETCHING without sending GET_MANIFEST first. A different MID is therefore
// an independent negotiation even when this shim retained conservative fallback state from an earlier
// transfer in the same boot. Preserve that state only while moving between blocks of the same MID.
if (!_active || memcmp(request.manifest_id, _mid, sizeof(_mid)) != 0) reset_session();
else reset_representation();
_active = true;
_block = request.block_idx;
memcpy(_mid, request.manifest_id, sizeof(_mid));
} else if (_complete) {
// The unchanged manager asks for this block again only after discarding it (for example, a bad proof).
reset_representation();
}
if (!_legacy_session) {
uint16_t fragments = request.want_mask;
if (_encoded_len != 0 && _mask != _need) {
// Preserve one sparse retry for a partially received representation. A second retry without a new
// fragment falls back to the universally deployed profile so an old source can take over.
if (++_partial_retries >= 2) {
_legacy_session = true;
reset_representation();
} else {
fragments = (uint16_t)(_need & ~_mask);
}
} else if (_request_sent && _encoded_len == 0) {
// No valid v2 DATA answered the first request. Retry in the literal legacy profile; this also covers
// sources which reject, rather than mask, unknown want_mask bits.
_legacy_session = true;
reset_representation();
}
if (!_legacy_session) {
request.want_mask = ota_req_make_v2(fragments, _allow_deflate);
uint8_t wire[16];
const uint16_t wire_len = encode_req(wire, sizeof(wire), request);
if (wire_len != 0) {
_request_sent = true;
_send(_send_ctx, wire, wire_len, flood);
return;
}
}
}
}
_send(_send_ctx, msg, len, flood);
}
void OtaTransportInflateReceiver::on_message(const uint8_t* msg, uint16_t len) {
if (!_manager || !msg || len == 0) return;
DataMsg data;
if (ota_msg_type(msg, len) == OTA_DATA && decode_data(msg, len, data)) {
if ((data.frag_off & OTA_DATA_V2_MARK) != 0) {
(void)handle_v2_data(msg, len); // malformed/unsolicited v2 DATA fails closed
return;
}
// A fully canonical legacy fragment is an actual old-source response, not a reason to discard a valid
// v2 representation merely because an attacker sent a malformed matching-MID packet.
const uint16_t expected = _manager->requestedBlockLength(data.manifest_id, data.block_idx);
if (expected != 0 && is_active(data.manifest_id, data.block_idx) &&
data.frag_off % OTA_FRAG_DATA == 0 && data.frag_off < expected) {
uint16_t expected_slice = (uint16_t)(expected - data.frag_off);
if (expected_slice > OTA_FRAG_DATA) expected_slice = OTA_FRAG_DATA;
if (data.data_len == expected_slice) {
_legacy_session = true;
reset_representation();
}
}
}
_manager->on_message(msg, len);
}
bool OtaTransportInflateReceiver::handle_v2_data(const uint8_t* msg, uint16_t len) {
DataMsg data;
if (!decode_data(msg, len, data) || !is_active(data.manifest_id, data.block_idx) ||
data.data_len <= OTA_DATA_V2_STREAM_ID_BYTES || _legacy_session) return false;
const uint16_t block_len = _manager->requestedBlockLength(data.manifest_id, data.block_idx);
if (block_len == 0) return false;
uint8_t fragment = 0;
uint16_t encoded_len = 0;
bool deflated = false;
if (!ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated)) return false;
if (encoded_len > block_len || (deflated ? encoded_len >= block_len : encoded_len != block_len) ||
(deflated && !_allow_deflate)) return false;
const uint16_t fragment_len = (uint16_t)(data.data_len - OTA_DATA_V2_STREAM_ID_BYTES);
const uint32_t fragment_off = (uint32_t)fragment * OTA_FRAG_DATA_V2;
if (fragment_off >= encoded_len || fragment_off + fragment_len > encoded_len) return false;
uint16_t expected_slice = (uint16_t)(encoded_len - fragment_off);
if (expected_slice > OTA_FRAG_DATA_V2) expected_slice = OTA_FRAG_DATA_V2;
if (fragment_len != expected_slice) return false;
const uint8_t* stream_id = data.data;
if (_encoded_len == 0) {
_encoded_len = encoded_len;
_deflated = deflated;
memcpy(_stream_id, stream_id, sizeof(_stream_id));
_mask = 0;
_need = transport_fragment_mask(encoded_len, OTA_FRAG_DATA_V2);
} else if (_encoded_len != encoded_len || _deflated != deflated ||
memcmp(_stream_id, stream_id, sizeof(_stream_id)) != 0) {
return false; // never mix different wire representations
}
const uint16_t bit = (uint16_t)(1u << fragment);
if (!(_need & bit)) return false;
if (!(_mask & bit)) {
memcpy(_encoded + fragment_off, data.data + OTA_DATA_V2_STREAM_ID_BYTES, fragment_len);
_mask |= bit;
_partial_retries = 0;
}
if (_mask != _need) return true;
uint8_t actual_id[4];
mh4(actual_id, _encoded, _encoded_len);
if (memcmp(actual_id, _stream_id, sizeof(actual_id)) != 0) {
reset_representation();
return true;
}
const uint8_t* logical = _encoded;
if (_deflated) {
uint16_t decoded_len = 0;
if (!ota_transport_inflate(nullptr, _encoded, _encoded_len,
_decoded, block_len, &decoded_len) || decoded_len != block_len) {
// Keep the 171-byte profile but stop offering DEFLATE for this session. The next manager retry asks the
// source for raw v2 DATA, which still stages the exact same authenticated `.mota` block.
_allow_deflate = false;
reset_representation();
return true;
}
logical = _decoded;
}
// Feed the historical manager only the canonical representation it already knows. Its existing final
// fragment transition requests/accepts the ordinary proof; Merkle verification happens before any write.
for (uint16_t off = 0; off < block_len; off = (uint16_t)(off + OTA_FRAG_DATA)) {
uint16_t slice = (uint16_t)(block_len - off);
if (slice > OTA_FRAG_DATA) slice = OTA_FRAG_DATA;
DataMsg canonical;
memcpy(canonical.manifest_id, _mid, sizeof(canonical.manifest_id));
canonical.block_idx = _block;
canonical.frag_off = off;
canonical.data = logical + off;
canonical.data_len = slice;
uint8_t wire[9 + OTA_FRAG_DATA];
const uint16_t wire_len = encode_data(wire, sizeof(wire), canonical);
if (wire_len == 0) {
reset_representation();
return false;
}
_manager->on_message(wire, wire_len);
}
_complete = true;
return true;
}
} // namespace ota
} // namespace mesh
#endif // ENABLE_OTA || OTA_TRANSPORT_DEFLATE_TEST
tools/lora_ota/rak3401_inflate_assets/OtaDeflate.h
+56
View File
@@ -0,0 +1,56 @@
#pragma once
#include <stdint.h>
#include "OtaManager.h"
#include "OtaProtocol.h"
namespace mesh {
namespace ota {
// Full raw RFC1951 transport decoder (stored, fixed-Huffman, and dynamic-Huffman blocks).
// `dst_cap` is the exact expected logical block length, not merely spare capacity. Success
// requires exact output and exact whole-byte input consumption; malformed/truncated streams,
// output overflow, and trailing bytes fail closed.
bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len,
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len);
// Receive-only compatibility shim for historical OTA managers. The wrapped manager continues to reassemble
// canonical 160-byte DATA, verify the Merkle proof, and stage the original `.mota`. This shim only negotiates
// the v2 wire profile, locks and reassembles one encoded representation, exact-inflates it, and injects the
// resulting logical block back into that unchanged path. Bridge builds use OTA_FETCH_PIPELINE=1, so one fixed
// encoded block and one fixed decoded block cover every in-flight request without heap allocation.
class OtaTransportInflateReceiver {
public:
void begin(OtaManager& manager, uint32_t target_id, OtaSend send, void* send_ctx);
void on_message(const uint8_t* msg, uint16_t len);
static void send_trampoline(void* ctx, const uint8_t* msg, uint16_t len, bool flood);
private:
void send(const uint8_t* msg, uint16_t len, bool flood);
bool handle_v2_data(const uint8_t* msg, uint16_t len);
void reset_session();
void reset_representation();
bool is_active(const uint8_t* mid, uint16_t block) const;
OtaManager* _manager = nullptr;
OtaSend _send = nullptr;
void* _send_ctx = nullptr;
uint8_t _mid[4] = {0};
uint8_t _stream_id[4] = {0};
uint16_t _block = 0;
uint16_t _encoded_len = 0;
uint16_t _mask = 0;
uint16_t _need = 0;
uint8_t _partial_retries = 0;
bool _active = false;
bool _request_sent = false;
bool _complete = false;
bool _deflated = false;
bool _allow_deflate = true;
bool _legacy_session = false;
uint8_t _encoded[OTA_MAX_BLOCK];
uint8_t _decoded[OTA_MAX_BLOCK];
};
} // namespace ota
} // namespace mesh
tools/lora_ota/rak3401_inflate_assets/OtaTinf.c
+18
View File
@@ -0,0 +1,18 @@
/*
* Compile the vendored tinf core as C. Building the same source as C++ costs
* several extra kilobytes with the embedded toolchains used by MeshCore.
*/
#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST)
#if defined(__GNUC__) && !defined(__clang__)
/* nRF52's Arduino recipe defaults to -Ofast, which more than doubles this
* decoder. Keep the size policy local to tinf instead of changing MeshCore. */
#pragma GCC push_options
#pragma GCC optimize ("Os")
#endif
#define MESHCORE_TINF_IMPLEMENTATION 1
#include "tinf/tinflate.c"
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC pop_options
#endif
#endif
tools/lora_ota/rak3401_inflate_assets/tinf/LICENSE
+23
View File
@@ -0,0 +1,23 @@
The zlib License (Zlib)
Copyright (c) 2003-2019 Joergen Ibsen
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product
documentation would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must
not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
tools/lora_ota/rak3401_inflate_assets/tinf/README.meshcore.txt
+24
View File
@@ -0,0 +1,24 @@
tinf 1.2.1 raw-DEFLATE decoder
================================
Upstream: https://github.com/jibsen/tinf
Revision: 57ffa1f1d5e3dde19011b2127bd26d01689b694b
Vendored files:
LICENSE
tinf.h
tinflate.c
MeshCore changes are marked in tinf.h and tinflate.c. They add
tinf_uncompress_exact(), which accepts legal padding bits in the byte containing
the final end-of-block code but rejects any trailing whole input byte. The C
implementation is included by ../OtaTinf.c only for ENABLE_OTA application
builds (and the native OTA test profile). ../OtaDeflate.cpp is the small C++
callback adapter. Keeping the decoder in a true C translation unit avoids the
several-kilobyte penalty produced by the embedded C++ toolchains; non-OTA
firmware links no decoder code or persistent RAM.
Only the raw RFC 1951 decoder is integrated. The zlib/gzip wrappers and checksum
sources are intentionally omitted. The original zlib license is preserved in
LICENSE and in the source headers.
tools/lora_ota/rak3401_inflate_assets/tinf/tinf.h
+155
View File
@@ -0,0 +1,155 @@
/*
* tinf - tiny inflate library (inflate, gzip, zlib)
*
* Copyright (c) 2003-2019 Joergen Ibsen
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must
* not claim that you wrote the original software. If you use this
* software in a product, an acknowledgment in the product
* documentation would be appreciated but is not required.
*
* 2. Altered source versions must be plainly marked as such, and must
* not be misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source
* distribution.
*/
/*
* MeshCore modification (2026): declare tinf_uncompress_exact(), a raw-DEFLATE
* entry point which rejects trailing whole input bytes. The original API and
* behavior remain available as tinf_uncompress().
*/
#ifndef TINF_H_INCLUDED
#define TINF_H_INCLUDED
#ifdef __cplusplus
extern "C" {
#endif
#define TINF_VER_MAJOR 1 /**< Major version number */
#define TINF_VER_MINOR 2 /**< Minor version number */
#define TINF_VER_PATCH 1 /**< Patch version number */
#define TINF_VER_STRING "1.2.1" /**< Version number as a string */
#ifndef TINFCC
# ifdef __WATCOMC__
# define TINFCC __cdecl
# else
# define TINFCC
# endif
#endif
/**
* Status codes returned.
*
* @see tinf_uncompress, tinf_gzip_uncompress, tinf_zlib_uncompress
*/
typedef enum {
TINF_OK = 0, /**< Success */
TINF_DATA_ERROR = -3, /**< Input error */
TINF_BUF_ERROR = -5 /**< Not enough room for output */
} tinf_error_code;
/**
* Initialize global data used by tinf.
*
* @deprecated No longer required, may be removed in a future version.
*/
void TINFCC tinf_init(void);
/**
* Decompress `sourceLen` bytes of deflate data from `source` to `dest`.
*
* The variable `destLen` points to must contain the size of `dest` on entry,
* and will be set to the size of the decompressed data on success.
*
* Reads at most `sourceLen` bytes from `source`.
* Writes at most `*destLen` bytes to `dest`.
*
* @param dest pointer to where to place decompressed data
* @param destLen pointer to variable containing size of `dest`
* @param source pointer to compressed data
* @param sourceLen size of compressed data
* @return `TINF_OK` on success, error code on error
*/
int TINFCC tinf_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
/**
* Decompress exactly one raw DEFLATE stream and reject trailing whole bytes.
* Padding bits in the byte containing the final end-of-block code are allowed.
* Output bounds and `destLen` semantics match tinf_uncompress().
*/
int TINFCC tinf_uncompress_exact(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
/**
* Decompress `sourceLen` bytes of gzip data from `source` to `dest`.
*
* The variable `destLen` points to must contain the size of `dest` on entry,
* and will be set to the size of the decompressed data on success.
*
* Reads at most `sourceLen` bytes from `source`.
* Writes at most `*destLen` bytes to `dest`.
*
* @param dest pointer to where to place decompressed data
* @param destLen pointer to variable containing size of `dest`
* @param source pointer to compressed data
* @param sourceLen size of compressed data
* @return `TINF_OK` on success, error code on error
*/
int TINFCC tinf_gzip_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
/**
* Decompress `sourceLen` bytes of zlib data from `source` to `dest`.
*
* The variable `destLen` points to must contain the size of `dest` on entry,
* and will be set to the size of the decompressed data on success.
*
* Reads at most `sourceLen` bytes from `source`.
* Writes at most `*destLen` bytes to `dest`.
*
* @param dest pointer to where to place decompressed data
* @param destLen pointer to variable containing size of `dest`
* @param source pointer to compressed data
* @param sourceLen size of compressed data
* @return `TINF_OK` on success, error code on error
*/
int TINFCC tinf_zlib_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
/**
* Compute Adler-32 checksum of `length` bytes starting at `data`.
*
* @param data pointer to data
* @param length size of data
* @return Adler-32 checksum
*/
unsigned int TINFCC tinf_adler32(const void *data, unsigned int length);
/**
* Compute CRC32 checksum of `length` bytes starting at `data`.
*
* @param data pointer to data
* @param length size of data
* @return CRC32 checksum
*/
unsigned int TINFCC tinf_crc32(const void *data, unsigned int length);
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* TINF_H_INCLUDED */
tools/lora_ota/rak3401_inflate_assets/tinf/tinflate.c
+667
View File
@@ -0,0 +1,667 @@
/*
* tinflate - tiny inflate
*
* Copyright (c) 2003-2019 Joergen Ibsen
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must
* not claim that you wrote the original software. If you use this
* software in a product, an acknowledgment in the product
* documentation would be appreciated but is not required.
*
* 2. Altered source versions must be plainly marked as such, and must
* not be misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source
* distribution.
*/
/*
* MeshCore modifications (2026):
* - expose tinf_uncompress_exact() to reject trailing whole input bytes;
* - compile this vendored implementation through OtaTinf.c only, keeping
* it out of builds which do not define ENABLE_OTA.
* See README.meshcore.txt for the upstream revision and integration details.
*/
#if defined(MESHCORE_TINF_IMPLEMENTATION)
#include "tinf.h"
#include <assert.h>
#include <limits.h>
#if defined(UINT_MAX) && (UINT_MAX) < 0xFFFFFFFFUL
# error "tinf requires unsigned int to be at least 32-bit"
#endif
/* -- Internal data structures -- */
struct tinf_tree {
unsigned short counts[16]; /* Number of codes with a given length */
unsigned short symbols[288]; /* Symbols sorted by code */
int max_sym;
};
struct tinf_data {
const unsigned char *source;
const unsigned char *source_end;
unsigned int tag;
int bitcount;
int overflow;
unsigned char *dest_start;
unsigned char *dest;
unsigned char *dest_end;
struct tinf_tree ltree; /* Literal/length tree */
struct tinf_tree dtree; /* Distance tree */
};
/* -- Utility functions -- */
static unsigned int read_le16(const unsigned char *p)
{
return ((unsigned int) p[0])
| ((unsigned int) p[1] << 8);
}
/* Build fixed Huffman trees */
static void tinf_build_fixed_trees(struct tinf_tree *lt, struct tinf_tree *dt)
{
int i;
/* Build fixed literal/length tree */
for (i = 0; i < 16; ++i) {
lt->counts[i] = 0;
}
lt->counts[7] = 24;
lt->counts[8] = 152;
lt->counts[9] = 112;
for (i = 0; i < 24; ++i) {
lt->symbols[i] = 256 + i;
}
for (i = 0; i < 144; ++i) {
lt->symbols[24 + i] = i;
}
for (i = 0; i < 8; ++i) {
lt->symbols[24 + 144 + i] = 280 + i;
}
for (i = 0; i < 112; ++i) {
lt->symbols[24 + 144 + 8 + i] = 144 + i;
}
lt->max_sym = 285;
/* Build fixed distance tree */
for (i = 0; i < 16; ++i) {
dt->counts[i] = 0;
}
dt->counts[5] = 32;
for (i = 0; i < 32; ++i) {
dt->symbols[i] = i;
}
dt->max_sym = 29;
}
/* Given an array of code lengths, build a tree */
static int tinf_build_tree(struct tinf_tree *t, const unsigned char *lengths,
unsigned int num)
{
unsigned short offs[16];
unsigned int i, num_codes, available;
assert(num <= 288);
for (i = 0; i < 16; ++i) {
t->counts[i] = 0;
}
t->max_sym = -1;
/* Count number of codes for each non-zero length */
for (i = 0; i < num; ++i) {
assert(lengths[i] <= 15);
if (lengths[i]) {
t->max_sym = i;
t->counts[lengths[i]]++;
}
}
/* Compute offset table for distribution sort */
for (available = 1, num_codes = 0, i = 0; i < 16; ++i) {
unsigned int used = t->counts[i];
/* Check length contains no more codes than available */
if (used > available) {
return TINF_DATA_ERROR;
}
available = 2 * (available - used);
offs[i] = num_codes;
num_codes += used;
}
/*
* Check all codes were used, or for the special case of only one
* code that it has length 1
*/
if ((num_codes > 1 && available > 0)
|| (num_codes == 1 && t->counts[1] != 1)) {
return TINF_DATA_ERROR;
}
/* Fill in symbols sorted by code */
for (i = 0; i < num; ++i) {
if (lengths[i]) {
t->symbols[offs[lengths[i]]++] = i;
}
}
/*
* For the special case of only one code (which will be 0) add a
* code 1 which results in a symbol that is too large
*/
if (num_codes == 1) {
t->counts[1] = 2;
t->symbols[1] = t->max_sym + 1;
}
return TINF_OK;
}
/* -- Decode functions -- */
static void tinf_refill(struct tinf_data *d, int num)
{
assert(num >= 0 && num <= 32);
/* Read bytes until at least num bits available */
while (d->bitcount < num) {
if (d->source != d->source_end) {
d->tag |= (unsigned int) *d->source++ << d->bitcount;
}
else {
d->overflow = 1;
}
d->bitcount += 8;
}
assert(d->bitcount <= 32);
}
static unsigned int tinf_getbits_no_refill(struct tinf_data *d, int num)
{
unsigned int bits;
assert(num >= 0 && num <= d->bitcount);
/* Get bits from tag */
bits = d->tag & ((1UL << num) - 1);
/* Remove bits from tag */
d->tag >>= num;
d->bitcount -= num;
return bits;
}
/* Get num bits from source stream */
static unsigned int tinf_getbits(struct tinf_data *d, int num)
{
tinf_refill(d, num);
return tinf_getbits_no_refill(d, num);
}
/* Read a num bit value from stream and add base */
static unsigned int tinf_getbits_base(struct tinf_data *d, int num, int base)
{
return base + (num ? tinf_getbits(d, num) : 0);
}
/* Given a data stream and a tree, decode a symbol */
static int tinf_decode_symbol(struct tinf_data *d, const struct tinf_tree *t)
{
int base = 0, offs = 0;
int len;
/*
* Get more bits while code index is above number of codes
*
* Rather than the actual code, we are computing the position of the
* code in the sorted order of codes, which is the index of the
* corresponding symbol.
*
* Conceptually, for each code length (level in the tree), there are
* counts[len] leaves on the left and internal nodes on the right.
* The index we have decoded so far is base + offs, and if that
* falls within the leaves we are done. Otherwise we adjust the range
* of offs and add one more bit to it.
*/
for (len = 1; ; ++len) {
offs = 2 * offs + tinf_getbits(d, 1);
assert(len <= 15);
if (offs < t->counts[len]) {
break;
}
base += t->counts[len];
offs -= t->counts[len];
}
assert(base + offs >= 0 && base + offs < 288);
return t->symbols[base + offs];
}
/* Given a data stream, decode dynamic trees from it */
static int tinf_decode_trees(struct tinf_data *d, struct tinf_tree *lt,
struct tinf_tree *dt)
{
unsigned char lengths[288 + 32];
/* Special ordering of code length codes */
static const unsigned char clcidx[19] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5,
11, 4, 12, 3, 13, 2, 14, 1, 15
};
unsigned int hlit, hdist, hclen;
unsigned int i, num, length;
int res;
/* Get 5 bits HLIT (257-286) */
hlit = tinf_getbits_base(d, 5, 257);
/* Get 5 bits HDIST (1-32) */
hdist = tinf_getbits_base(d, 5, 1);
/* Get 4 bits HCLEN (4-19) */
hclen = tinf_getbits_base(d, 4, 4);
/*
* The RFC limits the range of HLIT to 286, but lists HDIST as range
* 1-32, even though distance codes 30 and 31 have no meaning. While
* we could allow the full range of HLIT and HDIST to make it possible
* to decode the fixed trees with this function, we consider it an
* error here.
*
* See also: https://github.com/madler/zlib/issues/82
*/
if (hlit > 286 || hdist > 30) {
return TINF_DATA_ERROR;
}
for (i = 0; i < 19; ++i) {
lengths[i] = 0;
}
/* Read code lengths for code length alphabet */
for (i = 0; i < hclen; ++i) {
/* Get 3 bits code length (0-7) */
unsigned int clen = tinf_getbits(d, 3);
lengths[clcidx[i]] = clen;
}
/* Build code length tree (in literal/length tree to save space) */
res = tinf_build_tree(lt, lengths, 19);
if (res != TINF_OK) {
return res;
}
/* Check code length tree is not empty */
if (lt->max_sym == -1) {
return TINF_DATA_ERROR;
}
/* Decode code lengths for the dynamic trees */
for (num = 0; num < hlit + hdist; ) {
int sym = tinf_decode_symbol(d, lt);
if (sym > lt->max_sym) {
return TINF_DATA_ERROR;
}
switch (sym) {
case 16:
/* Copy previous code length 3-6 times (read 2 bits) */
if (num == 0) {
return TINF_DATA_ERROR;
}
sym = lengths[num - 1];
length = tinf_getbits_base(d, 2, 3);
break;
case 17:
/* Repeat code length 0 for 3-10 times (read 3 bits) */
sym = 0;
length = tinf_getbits_base(d, 3, 3);
break;
case 18:
/* Repeat code length 0 for 11-138 times (read 7 bits) */
sym = 0;
length = tinf_getbits_base(d, 7, 11);
break;
default:
/* Values 0-15 represent the actual code lengths */
length = 1;
break;
}
if (length > hlit + hdist - num) {
return TINF_DATA_ERROR;
}
while (length--) {
lengths[num++] = sym;
}
}
/* Check EOB symbol is present */
if (lengths[256] == 0) {
return TINF_DATA_ERROR;
}
/* Build dynamic trees */
res = tinf_build_tree(lt, lengths, hlit);
if (res != TINF_OK) {
return res;
}
res = tinf_build_tree(dt, lengths + hlit, hdist);
if (res != TINF_OK) {
return res;
}
return TINF_OK;
}
/* -- Block inflate functions -- */
/* Given a stream and two trees, inflate a block of data */
static int tinf_inflate_block_data(struct tinf_data *d, struct tinf_tree *lt,
struct tinf_tree *dt)
{
/* Extra bits and base tables for length codes */
static const unsigned char length_bits[30] = {
0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 0, 127
};
static const unsigned short length_base[30] = {
3, 4, 5, 6, 7, 8, 9, 10, 11, 13,
15, 17, 19, 23, 27, 31, 35, 43, 51, 59,
67, 83, 99, 115, 131, 163, 195, 227, 258, 0
};
/* Extra bits and base tables for distance codes */
static const unsigned char dist_bits[30] = {
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13
};
static const unsigned short dist_base[30] = {
1, 2, 3, 4, 5, 7, 9, 13, 17, 25,
33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577
};
for (;;) {
int sym = tinf_decode_symbol(d, lt);
/* Check for overflow in bit reader */
if (d->overflow) {
return TINF_DATA_ERROR;
}
if (sym < 256) {
if (d->dest == d->dest_end) {
return TINF_BUF_ERROR;
}
*d->dest++ = sym;
}
else {
int length, dist, offs;
int i;
/* Check for end of block */
if (sym == 256) {
return TINF_OK;
}
/* Check sym is within range and distance tree is not empty */
if (sym > lt->max_sym || sym - 257 > 28 || dt->max_sym == -1) {
return TINF_DATA_ERROR;
}
sym -= 257;
/* Possibly get more bits from length code */
length = tinf_getbits_base(d, length_bits[sym],
length_base[sym]);
dist = tinf_decode_symbol(d, dt);
/* Check dist is within range */
if (dist > dt->max_sym || dist > 29) {
return TINF_DATA_ERROR;
}
/* Possibly get more bits from distance code */
offs = tinf_getbits_base(d, dist_bits[dist],
dist_base[dist]);
if (offs > d->dest - d->dest_start) {
return TINF_DATA_ERROR;
}
if (d->dest_end - d->dest < length) {
return TINF_BUF_ERROR;
}
/* Copy match */
for (i = 0; i < length; ++i) {
d->dest[i] = d->dest[i - offs];
}
d->dest += length;
}
}
}
/* Inflate an uncompressed block of data */
static int tinf_inflate_uncompressed_block(struct tinf_data *d)
{
unsigned int length, invlength;
if (d->source_end - d->source < 4) {
return TINF_DATA_ERROR;
}
/* Get length */
length = read_le16(d->source);
/* Get one's complement of length */
invlength = read_le16(d->source + 2);
/* Check length */
if (length != (~invlength & 0x0000FFFF)) {
return TINF_DATA_ERROR;
}
d->source += 4;
if (d->source_end - d->source < length) {
return TINF_DATA_ERROR;
}
if (d->dest_end - d->dest < length) {
return TINF_BUF_ERROR;
}
/* Copy block */
while (length--) {
*d->dest++ = *d->source++;
}
/* Make sure we start next block on a byte boundary */
d->tag = 0;
d->bitcount = 0;
return TINF_OK;
}
/* Inflate a block of data compressed with fixed Huffman trees */
static int tinf_inflate_fixed_block(struct tinf_data *d)
{
/* Build fixed Huffman trees */
tinf_build_fixed_trees(&d->ltree, &d->dtree);
/* Decode block using fixed trees */
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
}
/* Inflate a block of data compressed with dynamic Huffman trees */
static int tinf_inflate_dynamic_block(struct tinf_data *d)
{
/* Decode trees from stream */
int res = tinf_decode_trees(d, &d->ltree, &d->dtree);
if (res != TINF_OK) {
return res;
}
/* Decode block using decoded trees */
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
}
/* -- Public functions -- */
/* Initialize global (static) data */
void tinf_init(void)
{
return;
}
/* Inflate stream from source to dest. Optionally require the final DEFLATE block to consume all source bytes. */
static int tinf_uncompress_internal(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen,
int require_exact_source)
{
struct tinf_data d;
int bfinal;
/* Initialise data */
d.source = (const unsigned char *) source;
d.source_end = d.source + sourceLen;
d.tag = 0;
d.bitcount = 0;
d.overflow = 0;
d.dest = (unsigned char *) dest;
d.dest_start = d.dest;
d.dest_end = d.dest + *destLen;
do {
unsigned int btype;
int res;
/* Read final block flag */
bfinal = tinf_getbits(&d, 1);
/* Read block type (2 bits) */
btype = tinf_getbits(&d, 2);
/* Decompress block */
switch (btype) {
case 0:
/* Decompress uncompressed block */
res = tinf_inflate_uncompressed_block(&d);
break;
case 1:
/* Decompress block with fixed Huffman trees */
res = tinf_inflate_fixed_block(&d);
break;
case 2:
/* Decompress block with dynamic Huffman trees */
res = tinf_inflate_dynamic_block(&d);
break;
default:
res = TINF_DATA_ERROR;
break;
}
if (res != TINF_OK) {
return res;
}
} while (!bfinal);
/* Check for overflow in bit reader */
if (d.overflow) {
return TINF_DATA_ERROR;
}
/* d.source includes the byte containing any legal final padding bits, but not a following whole byte. */
if (require_exact_source && d.source != d.source_end) {
return TINF_DATA_ERROR;
}
*destLen = d.dest - d.dest_start;
return TINF_OK;
}
int tinf_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen)
{
return tinf_uncompress_internal(dest, destLen, source, sourceLen, 0);
}
int tinf_uncompress_exact(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen)
{
return tinf_uncompress_internal(dest, destLen, source, sourceLen, 1);
}
/* clang -g -O1 -fsanitize=fuzzer,address -DTINF_FUZZING tinflate.c */
#if defined(TINF_FUZZING)
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
unsigned char depacked[64 * 1024];
extern int
LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
if (size > UINT_MAX / 2) { return 0; }
unsigned int destLen = sizeof(depacked);
tinf_uncompress(depacked, &destLen, data, size);
return 0;
}
#endif
#endif /* MESHCORE_TINF_IMPLEMENTATION */
tools/lora_ota/test_build_rak3401_inflate_bridges.py
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env python3
"""Focused tests for the reproducible RAK3401 receive-inflate builder."""
from __future__ import annotations
from pathlib import Path
import sys
import tempfile
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_rak3401_inflate_bridges as inflate
class InflateAssetTests(unittest.TestCase):
def test_vendored_snapshot_assets_match_their_pins(self) -> None:
metadata = inflate.inflate_asset_metadata()
self.assertEqual(set(metadata), set(inflate.INFLATE_ASSETS))
for destination, (_asset, expected_hash) in inflate.INFLATE_ASSETS.items():
self.assertEqual(metadata[destination]["sha256"], expected_hash)
def test_install_and_remove_are_exact_inverses(self) -> None:
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)
inflate.install_inflate_assets(source)
for destination, (_asset, expected_hash) in inflate.INFLATE_ASSETS.items():
self.assertEqual(
inflate.sha256_file(source / destination), expected_hash
)
inflate.remove_inflate_assets(source)
self.assertFalse((source / "src/helpers/ota/tinf").exists())
for destination in inflate.INFLATE_ASSETS:
self.assertFalse((source / destination).exists())
class ExactTransformTests(unittest.TestCase):
def test_replace_exact_replaces_one_anchor(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "source.txt"
path.write_bytes(b"before anchor after")
inflate.replace_exact(path, b"anchor", b"replacement", "test")
self.assertEqual(path.read_bytes(), b"before replacement after")
def test_replace_exact_rejects_missing_or_ambiguous_anchor(self) -> None:
for contents, count in ((b"none", 0), (b"anchor anchor", 2)):
with self.subTest(count=count), tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "source.txt"
path.write_bytes(contents)
with self.assertRaisesRegex(
inflate.BuildError, f"found {count}"
):
inflate.replace_exact(path, b"anchor", b"replacement", "test")
class BuilderConfigurationTests(unittest.TestCase):
def test_inflate_builder_has_distinct_reproducibility_identity(self) -> None:
self.assertEqual(
inflate.VERSION_SUFFIX, "halo-keymind-cascade-mota-inflate"
)
self.assertEqual(
inflate.INFLATE_SNAPSHOT_COMMIT,
"add51bf00c46c15ef54318ca766a6daf08a147ee",
)
self.assertNotIn("6f03eae8", {target.source_commit[:8] for target in inflate.TARGETS})
if __name__ == "__main__":
unittest.main()