diff --git a/docs/releases/1.17.1.6.md b/docs/releases/1.17.1.6.md
index 67133a285..56a1415c4 100644
--- a/docs/releases/1.17.1.6.md
+++ b/docs/releases/1.17.1.6.md
@@ -105,6 +105,10 @@ bash build.sh build-firmwares-logging-matrix \
--require-ota --skip-kiss --resume --background
```
-Run only one PlatformIO process in that checkout. Publish only after the full
-matrix completes successfully and `scripts/package_cascade_release.py` accepts
-the completed build status, source commit and `--version 1.17.1.6`.
+Run only one PlatformIO process in that checkout. Let the matrix finish even
+when individual targets fail. The packager normally requires a successful full
+matrix. For an initial release with gaps, `scripts/package_cascade_release.py`
+accepts `--allow-partial` with `--version 1.17.1.6`, the original completed build
+status and source commit. It lists every failed attempt in `BUILD-FAILURES.md`
+and `BUILD-FAILURES.json`; each published file must still pass all qualification
+and memory checks. Repair the failures and add the corrected profiles afterward.
diff --git a/scripts/package_cascade_release.py b/scripts/package_cascade_release.py
index 59fd8c044..5219fbb26 100644
--- a/scripts/package_cascade_release.py
+++ b/scripts/package_cascade_release.py
@@ -99,6 +99,35 @@ def sha256(path):
return hashlib.file_digest(stream, "sha256").hexdigest()
+def completed_matrix_failures(status, allow_partial=False):
+ if status.get("state") == "completed" and status.get("exit_code") == "0":
+ return []
+ if not allow_partial:
+ raise ValueError("the Option 3 matrix has not completed successfully")
+ if status.get("state") != "failed" or status.get("exit_code") != "1":
+ raise ValueError("partial publication requires a finished matrix, not a running or interrupted build")
+ log = Path(status.get("log", ""))
+ if not log.is_file():
+ raise ValueError("partial publication requires the matrix failure summary")
+ text = log.read_text(errors="replace")
+ summaries = list(re.finditer(r"^Logging matrix completed with (\d+) failed build\(s\):\s*$", text, re.M))
+ if not summaries:
+ raise ValueError("partial publication requires the matrix failure summary")
+ summary = summaries[-1]
+ count = int(summary[1])
+ lines = text[summary.end():].lstrip("\r\n").splitlines()
+ failures = []
+ for line in lines[:count]:
+ match = re.fullmatch(r" ([\w.+-]+) \(([^()]+)\) -> (.+)", line)
+ if not match:
+ raise ValueError("matrix failure summary is incomplete")
+ failures.append({"target": match[1], "profile": match[2],
+ "log_file": Path(match[3]).name})
+ if not count or len(failures) != count:
+ raise ValueError("matrix failure summary is incomplete")
+ return failures
+
+
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", required=True, type=Path)
@@ -107,10 +136,11 @@ def main():
parser.add_argument("--version", default="1.17.1.5")
parser.add_argument("--commit", required=True)
parser.add_argument("--repo", default="mikecarper/MeshCore")
+ parser.add_argument("--allow-partial", action="store_true",
+ help="Publish qualified outputs from a finished matrix with failures, listing every failed attempt")
args = parser.parse_args()
status = dict(line.split("=", 1) for line in args.build_status.read_text().splitlines() if "=" in line)
- if status.get("state") != "completed" or status.get("exit_code") != "0":
- raise ValueError("the Option 3 matrix has not completed successfully")
+ failures = completed_matrix_failures(status, args.allow_partial)
output_directory = Path(status["working_directory"]) / status["output_directory"]
if output_directory.resolve() != args.input.resolve():
raise ValueError("build status belongs to another output directory")
@@ -182,6 +212,21 @@ def main():
exclusions = args.input / "ota-excluded-targets.txt"
if exclusions.is_file():
shutil.copy2(exclusions, destination / exclusions.name)
+ partial_note = ""
+ if failures:
+ report = {"source": args.commit, "version": args.version, "matrix_state": status["state"],
+ "matrix_exit_code": int(status["exit_code"]), "failures": failures}
+ (destination / "BUILD-FAILURES.json").write_text(json.dumps(report, indent=2) + "\n")
+ (destination / "BUILD-FAILURES.md").write_text(
+ "# Build attempts awaiting repair\n\n"
+ f"Source: `{args.commit}`. The matrix finished with exit code {status['exit_code']}.\n\n"
+ "Published files passed their individual capability, size and memory checks. "
+ "These failed attempts are omitted pending repair; another profile for the same board may be available.\n\n"
+ "| Target | Profile | Build log |\n| --- | --- | --- |\n"
+ + "".join(f"| `{f['target']}` | `{f['profile']}` | `{f['log_file']}` |\n" for f in failures))
+ report_url = f"https://github.com/{args.repo}/releases/download/{group['tag']}/BUILD-FAILURES.md"
+ partial_note = (f"**Partial matrix:** {len(failures)} build attempt(s) failed and await repair. "
+ f"Only individually qualified firmware is published. [Missing builds]({report_url}).\n\n")
body = (f"> **Development prerelease.** Firmware identifier: **{base_tag}**.\n\n"
f"# MeshCore {args.version} Dev — USA Cascade\n\n"
f"Source: `{args.commit}`. USA/Canada: **{radio['frequency']} MHz, BW{radio['bandwidth']}, SF{radio['sf']}, CR{radio['cr']}**; Cascade defaults.\n\n"
@@ -190,7 +235,7 @@ def main():
f"[Release details]({source_url}/docs/releases/{args.version}.md)\n\n"
"Match the exact board, radio, display, and storage variant. ESP32 WiFi updates use the application `.bin`; the merged image installs boot/partition data over USB. nRF52 `.zip` files are native application DFU packages. LoRa MOTA installation requires an exact destination package; nRF52 also requires its matching OTAFIX bootloader. Full Companions do not LoRa-install onto themselves.\n\n"
"Firmware and capability checks passed in the build matrix. Hardware update testing across every board was not performed.\n\n"
- + links + "\n")
+ + partial_note + links + "\n")
(args.output / (group["key"] + "-notes.md")).write_text(body)
(destination / "BUILD-NOTES.txt").write_text(body)
picker = (""
@@ -219,7 +264,8 @@ def main():
(destination / "SHA256SUMS.txt").write_text("".join(f"{sha256(path)} {path.name}\n" for path in files))
group["asset_count"] = len(files) + 1
group["target_count"] = len(group.pop("records"))
- (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups}, indent=2) + "\n")
+ (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups,
+ "matrix": {"state": status["state"], "exit_code": int(status["exit_code"]), "failures": failures}}, indent=2) + "\n")
print(json.dumps({"targets": len(records), "groups": groups}, indent=2))
diff --git a/test/test_cascade_release_package.py b/test/test_cascade_release_package.py
index 39d01d986..f60039f8e 100644
--- a/test/test_cascade_release_package.py
+++ b/test/test_cascade_release_package.py
@@ -15,6 +15,28 @@ spec.loader.exec_module(package)
class ReleaseQualificationTest(unittest.TestCase):
+ def test_partial_matrix_requires_completion_and_an_explicit_opt_in(self):
+ self.assertEqual(package.completed_matrix_failures({"state": "completed", "exit_code": "0"}), [])
+ for state, code in (("running", "0"), ("starting", "0"), ("failed", "143")):
+ with self.subTest(state=state, code=code), self.assertRaisesRegex(ValueError, "finished matrix"):
+ package.completed_matrix_failures({"state": state, "exit_code": code}, True)
+ with tempfile.TemporaryDirectory() as temp:
+ log = Path(temp) / "matrix.log"
+ status = {"state": "failed", "exit_code": "1", "log": str(log)}
+ with self.assertRaisesRegex(ValueError, "not completed successfully"):
+ package.completed_matrix_failures(status)
+ for body in ("", "Building a target\n", "Logging matrix completed with 2 failed build(s):\n one (standard) -> /tmp/one.log\n"):
+ log.write_text(body)
+ with self.assertRaisesRegex(ValueError, "summary"):
+ package.completed_matrix_failures(status, True)
+ log.write_text("Logging matrix completed with 2 failed build(s):\n"
+ " one (standard) -> /tmp/one.log\n"
+ " two (full-usb-wifi-ota) -> /tmp/two.log\n")
+ failures = package.completed_matrix_failures(status, True)
+ self.assertEqual([f["target"] for f in failures], ["one", "two"])
+ self.assertEqual(failures[1]["profile"], "full-usb-wifi-ota")
+ self.assertEqual(failures[1]["log_file"], "two.log")
+
def manifest(self, target="test_repeater", **changes):
return {"target": target, "platform": "ESP32_PLATFORM",
"schema_version": 2, "verified": True,
@@ -99,6 +121,40 @@ class ReleaseQualificationTest(unittest.TestCase):
digest, name = line.split(" ", 1)
self.assertEqual(hashlib.sha256((assets / name).read_bytes()).hexdigest(), digest)
+ # A finished matrix can publish good files while explicitly
+ # retaining the failed attempts and the real nonzero exit code.
+ log = directory / "matrix.log"
+ log.write_text("Logging matrix completed with 1 failed build(s):\n"
+ " missing_repeater (standard) -> /tmp/missing.log\n")
+ status.write_text("state=failed\n" + settings.replace("exit_code=0", "exit_code=1")
+ + f"log={log}\n")
+ partial_command = command.copy()
+ partial_command[partial_command.index("--output") + 1] = str(directory / "partial")
+ result = subprocess.run(partial_command, capture_output=True, text=True)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertFalse((directory / "partial").exists())
+ result = subprocess.run(partial_command + ["--allow-partial"], capture_output=True, text=True)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ plan = json.loads((directory / "partial/release-plan.json").read_text())
+ self.assertEqual(plan["matrix"]["exit_code"], 1)
+ self.assertEqual(plan["matrix"]["failures"][0]["target"], "missing_repeater")
+ assets = directory / "partial/companion"
+ self.assertIn("Partial matrix", (assets / "BUILD-NOTES.txt").read_text())
+ self.assertIn("missing_repeater", (assets / "BUILD-FAILURES.md").read_text())
+ self.assertEqual(len(list(assets.iterdir())), plan["groups"][0]["asset_count"])
+ checksums = (assets / "SHA256SUMS.txt").read_text()
+ self.assertIn("BUILD-FAILURES.json", checksums)
+ for line in checksums.splitlines():
+ digest, name = line.split(" ", 1)
+ self.assertEqual(hashlib.sha256((assets / name).read_bytes()).hexdigest(), digest)
+
+ # Opting in to holes never authorizes publishing bad firmware.
+ (inputs / (stem + ".bin")).write_bytes(b"corrupted")
+ partial_command[partial_command.index("--output") + 1] = str(directory / "bad")
+ result = subprocess.run(partial_command + ["--allow-partial"], capture_output=True, text=True)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertFalse((directory / "bad").exists())
+
if __name__ == "__main__":
unittest.main()