From dd64eae4f97dcd3edcabe545bea7dccc113d8c4d Mon Sep 17 00:00:00 2001 From: mikecarper Date: Sun, 13 Sep 2026 09:19:17 -0700 Subject: [PATCH] Publish qualified partial release matrices with explicit failure reports --- docs/releases/1.17.1.6.md | 10 +++-- scripts/package_cascade_release.py | 54 +++++++++++++++++++++++++-- test/test_cascade_release_package.py | 56 ++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/docs/releases/1.17.1.6.md b/docs/releases/1.17.1.6.md index 67133a285..56a1415c4 100644 --- a/docs/releases/1.17.1.6.md +++ b/docs/releases/1.17.1.6.md @@ -105,6 +105,10 @@ bash build.sh build-firmwares-logging-matrix \ --require-ota --skip-kiss --resume --background ``` -Run only one PlatformIO process in that checkout. Publish only after the full -matrix completes successfully and `scripts/package_cascade_release.py` accepts -the completed build status, source commit and `--version 1.17.1.6`. +Run only one PlatformIO process in that checkout. Let the matrix finish even +when individual targets fail. The packager normally requires a successful full +matrix. For an initial release with gaps, `scripts/package_cascade_release.py` +accepts `--allow-partial` with `--version 1.17.1.6`, the original completed build +status and source commit. It lists every failed attempt in `BUILD-FAILURES.md` +and `BUILD-FAILURES.json`; each published file must still pass all qualification +and memory checks. Repair the failures and add the corrected profiles afterward. diff --git a/scripts/package_cascade_release.py b/scripts/package_cascade_release.py index 59fd8c044..5219fbb26 100644 --- a/scripts/package_cascade_release.py +++ b/scripts/package_cascade_release.py @@ -99,6 +99,35 @@ def sha256(path): return hashlib.file_digest(stream, "sha256").hexdigest() +def completed_matrix_failures(status, allow_partial=False): + if status.get("state") == "completed" and status.get("exit_code") == "0": + return [] + if not allow_partial: + raise ValueError("the Option 3 matrix has not completed successfully") + if status.get("state") != "failed" or status.get("exit_code") != "1": + raise ValueError("partial publication requires a finished matrix, not a running or interrupted build") + log = Path(status.get("log", "")) + if not log.is_file(): + raise ValueError("partial publication requires the matrix failure summary") + text = log.read_text(errors="replace") + summaries = list(re.finditer(r"^Logging matrix completed with (\d+) failed build\(s\):\s*$", text, re.M)) + if not summaries: + raise ValueError("partial publication requires the matrix failure summary") + summary = summaries[-1] + count = int(summary[1]) + lines = text[summary.end():].lstrip("\r\n").splitlines() + failures = [] + for line in lines[:count]: + match = re.fullmatch(r" ([\w.+-]+) \(([^()]+)\) -> (.+)", line) + if not match: + raise ValueError("matrix failure summary is incomplete") + failures.append({"target": match[1], "profile": match[2], + "log_file": Path(match[3]).name}) + if not count or len(failures) != count: + raise ValueError("matrix failure summary is incomplete") + return failures + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", required=True, type=Path) @@ -107,10 +136,11 @@ def main(): parser.add_argument("--version", default="1.17.1.5") parser.add_argument("--commit", required=True) parser.add_argument("--repo", default="mikecarper/MeshCore") + parser.add_argument("--allow-partial", action="store_true", + help="Publish qualified outputs from a finished matrix with failures, listing every failed attempt") args = parser.parse_args() status = dict(line.split("=", 1) for line in args.build_status.read_text().splitlines() if "=" in line) - if status.get("state") != "completed" or status.get("exit_code") != "0": - raise ValueError("the Option 3 matrix has not completed successfully") + failures = completed_matrix_failures(status, args.allow_partial) output_directory = Path(status["working_directory"]) / status["output_directory"] if output_directory.resolve() != args.input.resolve(): raise ValueError("build status belongs to another output directory") @@ -182,6 +212,21 @@ def main(): exclusions = args.input / "ota-excluded-targets.txt" if exclusions.is_file(): shutil.copy2(exclusions, destination / exclusions.name) + partial_note = "" + if failures: + report = {"source": args.commit, "version": args.version, "matrix_state": status["state"], + "matrix_exit_code": int(status["exit_code"]), "failures": failures} + (destination / "BUILD-FAILURES.json").write_text(json.dumps(report, indent=2) + "\n") + (destination / "BUILD-FAILURES.md").write_text( + "# Build attempts awaiting repair\n\n" + f"Source: `{args.commit}`. The matrix finished with exit code {status['exit_code']}.\n\n" + "Published files passed their individual capability, size and memory checks. " + "These failed attempts are omitted pending repair; another profile for the same board may be available.\n\n" + "| Target | Profile | Build log |\n| --- | --- | --- |\n" + + "".join(f"| `{f['target']}` | `{f['profile']}` | `{f['log_file']}` |\n" for f in failures)) + report_url = f"https://github.com/{args.repo}/releases/download/{group['tag']}/BUILD-FAILURES.md" + partial_note = (f"**Partial matrix:** {len(failures)} build attempt(s) failed and await repair. " + f"Only individually qualified firmware is published. [Missing builds]({report_url}).\n\n") body = (f"> **Development prerelease.** Firmware identifier: **{base_tag}**.\n\n" f"# MeshCore {args.version} Dev — USA Cascade\n\n" f"Source: `{args.commit}`. USA/Canada: **{radio['frequency']} MHz, BW{radio['bandwidth']}, SF{radio['sf']}, CR{radio['cr']}**; Cascade defaults.\n\n" @@ -190,7 +235,7 @@ def main(): f"[Release details]({source_url}/docs/releases/{args.version}.md)\n\n" "Match the exact board, radio, display, and storage variant. ESP32 WiFi updates use the application `.bin`; the merged image installs boot/partition data over USB. nRF52 `.zip` files are native application DFU packages. LoRa MOTA installation requires an exact destination package; nRF52 also requires its matching OTAFIX bootloader. Full Companions do not LoRa-install onto themselves.\n\n" "Firmware and capability checks passed in the build matrix. Hardware update testing across every board was not performed.\n\n" - + links + "\n") + + partial_note + links + "\n") (args.output / (group["key"] + "-notes.md")).write_text(body) (destination / "BUILD-NOTES.txt").write_text(body) picker = ("" @@ -219,7 +264,8 @@ def main(): (destination / "SHA256SUMS.txt").write_text("".join(f"{sha256(path)} {path.name}\n" for path in files)) group["asset_count"] = len(files) + 1 group["target_count"] = len(group.pop("records")) - (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups}, indent=2) + "\n") + (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups, + "matrix": {"state": status["state"], "exit_code": int(status["exit_code"]), "failures": failures}}, indent=2) + "\n") print(json.dumps({"targets": len(records), "groups": groups}, indent=2)) diff --git a/test/test_cascade_release_package.py b/test/test_cascade_release_package.py index 39d01d986..f60039f8e 100644 --- a/test/test_cascade_release_package.py +++ b/test/test_cascade_release_package.py @@ -15,6 +15,28 @@ spec.loader.exec_module(package) class ReleaseQualificationTest(unittest.TestCase): + def test_partial_matrix_requires_completion_and_an_explicit_opt_in(self): + self.assertEqual(package.completed_matrix_failures({"state": "completed", "exit_code": "0"}), []) + for state, code in (("running", "0"), ("starting", "0"), ("failed", "143")): + with self.subTest(state=state, code=code), self.assertRaisesRegex(ValueError, "finished matrix"): + package.completed_matrix_failures({"state": state, "exit_code": code}, True) + with tempfile.TemporaryDirectory() as temp: + log = Path(temp) / "matrix.log" + status = {"state": "failed", "exit_code": "1", "log": str(log)} + with self.assertRaisesRegex(ValueError, "not completed successfully"): + package.completed_matrix_failures(status) + for body in ("", "Building a target\n", "Logging matrix completed with 2 failed build(s):\n one (standard) -> /tmp/one.log\n"): + log.write_text(body) + with self.assertRaisesRegex(ValueError, "summary"): + package.completed_matrix_failures(status, True) + log.write_text("Logging matrix completed with 2 failed build(s):\n" + " one (standard) -> /tmp/one.log\n" + " two (full-usb-wifi-ota) -> /tmp/two.log\n") + failures = package.completed_matrix_failures(status, True) + self.assertEqual([f["target"] for f in failures], ["one", "two"]) + self.assertEqual(failures[1]["profile"], "full-usb-wifi-ota") + self.assertEqual(failures[1]["log_file"], "two.log") + def manifest(self, target="test_repeater", **changes): return {"target": target, "platform": "ESP32_PLATFORM", "schema_version": 2, "verified": True, @@ -99,6 +121,40 @@ class ReleaseQualificationTest(unittest.TestCase): digest, name = line.split(" ", 1) self.assertEqual(hashlib.sha256((assets / name).read_bytes()).hexdigest(), digest) + # A finished matrix can publish good files while explicitly + # retaining the failed attempts and the real nonzero exit code. + log = directory / "matrix.log" + log.write_text("Logging matrix completed with 1 failed build(s):\n" + " missing_repeater (standard) -> /tmp/missing.log\n") + status.write_text("state=failed\n" + settings.replace("exit_code=0", "exit_code=1") + + f"log={log}\n") + partial_command = command.copy() + partial_command[partial_command.index("--output") + 1] = str(directory / "partial") + result = subprocess.run(partial_command, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((directory / "partial").exists()) + result = subprocess.run(partial_command + ["--allow-partial"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads((directory / "partial/release-plan.json").read_text()) + self.assertEqual(plan["matrix"]["exit_code"], 1) + self.assertEqual(plan["matrix"]["failures"][0]["target"], "missing_repeater") + assets = directory / "partial/companion" + self.assertIn("Partial matrix", (assets / "BUILD-NOTES.txt").read_text()) + self.assertIn("missing_repeater", (assets / "BUILD-FAILURES.md").read_text()) + self.assertEqual(len(list(assets.iterdir())), plan["groups"][0]["asset_count"]) + checksums = (assets / "SHA256SUMS.txt").read_text() + self.assertIn("BUILD-FAILURES.json", checksums) + for line in checksums.splitlines(): + digest, name = line.split(" ", 1) + self.assertEqual(hashlib.sha256((assets / name).read_bytes()).hexdigest(), digest) + + # Opting in to holes never authorizes publishing bad firmware. + (inputs / (stem + ".bin")).write_bytes(b"corrupted") + partial_command[partial_command.index("--output") + 1] = str(directory / "bad") + result = subprocess.run(partial_command + ["--allow-partial"], capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((directory / "bad").exists()) + if __name__ == "__main__": unittest.main()