Preserve qualified release artifacts across resumed builds

scripts/build_local_release.py

@@ -51,6 +51,145 @@ def run_logged(command: list[str], log: Path, environment: dict[str, str]) -> No
             raise RuntimeError(f"build failed; see {log}")
 
 
+def capacity_rejected_attempt(manifest_path: Path, version: str, source: str) -> dict:
+    """Recognize only the successful qualification followed by portable-size rejection."""
+    work = manifest_path.parent
+    if manifest_path.is_symlink() or not manifest_path.is_file():
+        raise ValueError("manifest is not a regular non-symlink file")
+    raw = manifest_path.read_bytes()
+    manifest = json.loads(raw)
+    target = manifest.get("target")
+    if not isinstance(target, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]*", target):
+        raise ValueError("invalid target identity")
+    artifact_version = f"{version}-{source[:8]}"
+    stems = (f"{target}-{artifact_version}", f"{target}-ota-{artifact_version}")
+    if manifest_path.name not in {stem + ".capabilities.json" for stem in stems}:
+        raise ValueError("manifest filename does not match target/version/source")
+    stem = manifest_path.name.removesuffix(".capabilities.json")
+    # Even an unfamiliar sidecar can contain firmware or other failed-build
+    # evidence. Never consume such a package merely because its bin is absent.
+    siblings = list(work.glob(stem + "*"))
+    if siblings != [manifest_path] or list(work.rglob(stem + "*")) != [manifest_path]:
+        raise ValueError("manifest is not the sole same-stem output")
+    recipe = manifest.get("build_recipe")
+    checks = manifest.get("verification")
+    if (type(manifest.get("schema_version")) is not int or manifest["schema_version"] != 2
+            or manifest.get("platform") != "ESP32_PLATFORM"
+            or manifest.get("build_profile") != "standard"
+            or manifest.get("artifact_target") != target
+            or manifest.get("verified") is not True
+            or manifest.get("ota_update_verified") is not True
+            or manifest.get("ota_update_evidence") !=
+               "firmware fits both OTA application slots; otadata present"
+            or not isinstance(checks, list) or not checks
+            or any(not isinstance(check, dict) or check.get("present") is not True
+                   for check in checks)
+            or not isinstance(recipe, dict) or set(recipe) != {"schema_version", "sha256"}
+            or type(recipe.get("schema_version")) is not int or recipe["schema_version"] != 1
+            or not isinstance(recipe.get("sha256"), str)
+            or not re.fullmatch(r"[0-9a-f]{64}", recipe["sha256"])):
+        raise ValueError("qualification or original recipe binding is incomplete")
+    log = work / "build-logs" / f"{target}-standard.log"
+    if log.parent.is_symlink() or log.is_symlink() or not log.is_file():
+        raise ValueError("original standard log is missing or unsafe")
+    log_raw = log.read_bytes()
+    lines = log_raw.decode("utf-8").splitlines()
+    overflow = re.fullmatch(
+        r"ESP32 app image is (\d+) bytes, exceeding portable LoRa-OTA "
+        r"slot 0x10000\.\.0x150000 \((\d+) bytes\) by (\d+) bytes",
+        lines[-2] if len(lines) >= 2 else "")
+    marker = (f"DEFERRED: {target} (standard) exceeds the portable OTA slot; "
+              "the expanded FULL pass is required.")
+    version_line = f'    -DFIRMWARE_VERSION="{artifact_version}"'
+    qualification = re.compile(
+        r"Verified \d+ capability marker\(s\); manifest: " + re.escape(str(manifest_path)))
+    if (not overflow or lines[-1] != marker or version_line not in lines
+            or not any(qualification.fullmatch(line) for line in lines[:-2])):
+        raise ValueError("original log does not prove this exact portable-size rejection")
+    size, limit, excess = map(int, overflow.groups())
+    if limit != 0x140000 or size <= limit or excess != size - limit:
+        raise ValueError("portable overflow measurements are inconsistent")
+    return {"target": target, "manifest": manifest_path.name,
+            "manifest_sha256": hashlib.sha256(raw).hexdigest(),
+            "original_log": str(log.relative_to(work)),
+            "original_log_sha256": hashlib.sha256(log_raw).hexdigest(),
+            "build_recipe": recipe, "image_bytes": size,
+            "portable_limit_bytes": limit, "excess_bytes": excess,
+            "reason": "portable_slot_overflow"}
+
+
+def archive_capacity_rejected_attempts(work: Path, version: str, source: str,
+                                      *, project_root: Path = ROOT,
+                                      archive_parent: Path | None = None) -> dict:
+    """Preserve proven manifest-only rejections outside recursive package discovery."""
+    if not re.fullmatch(r"[0-9a-f]{40}", source):
+        raise ValueError("invalid release source commit")
+    if not re.fullmatch(r"v[0-9]+(?:\.[0-9]+){3}-halo-keymind-cascade-dev", version):
+        raise ValueError("invalid release firmware version")
+    work = Path(os.path.abspath(work))
+    parent = Path(os.path.abspath(archive_parent or work.parent))
+    if (work.resolve() != work or not work.is_dir() or parent.resolve() != parent
+            or parent == work or work in parent.parents):
+        raise ValueError("unsafe work/archive location")
+    lock_path = project_root / ".pio" / "build-sh.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    if lock_path.parent.resolve() != lock_path.parent or lock_path.is_symlink():
+        raise ValueError("unsafe build lock location")
+    with lock_path.open("a+") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError as error:
+            raise RuntimeError("active build owns the checkout; no attempts were archived") from error
+        candidates = []
+        preserved = []
+        for manifest in sorted(work.glob("*.capabilities.json")):
+            try:
+                candidates.append(capacity_rejected_attempt(manifest, version, source))
+            except (ValueError, OSError, TypeError, AttributeError) as error:
+                preserved.append({"manifest": manifest.name, "reason": str(error)})
+        if not candidates:
+            return {"archived": [], "preserved": preserved, "archive": None}
+        # Complete the scan and recheck every candidate before any file moves.
+        for record in candidates:
+            if capacity_rejected_attempt(work / record["manifest"], version, source) != record:
+                raise ValueError("capacity attempt changed during archival scan")
+        archive = Path(tempfile.mkdtemp(
+            prefix=f".capacity-rejected-{version}-{source[:8]}-", dir=parent))
+        (archive / "manifests").mkdir()
+        (archive / "logs").mkdir()
+        for record in candidates:
+            original = work / record["original_log"]
+            copy = archive / "logs" / original.name
+            shutil.copy2(original, copy)
+            if digest(copy) != record["original_log_sha256"]:
+                raise ValueError("original log changed while preserving rejection evidence")
+        index = {"format": "meshcore-capacity-rejected-attempts-v1",
+                 "source_commit": source, "firmware_version": version,
+                 "work_directory": str(work), "state": "prepared",
+                 "attempts": candidates, "moved_manifests": []}
+        def save_index() -> None:
+            temporary = archive / ".index.json.tmp"
+            temporary.write_text(json.dumps(index, indent=2, sort_keys=True,
+                                             ensure_ascii=True) + "\n", encoding="ascii")
+            temporary.replace(archive / "index.json")
+        save_index()
+        try:
+            for record in candidates:
+                manifest = work / record["manifest"]
+                if capacity_rejected_attempt(manifest, version, source) != record:
+                    raise ValueError("capacity attempt changed before archival move")
+                manifest.rename(archive / "manifests" / manifest.name)
+                index["moved_manifests"].append(manifest.name)
+                save_index()
+        except (ValueError, OSError):
+            index["state"] = "interrupted"
+            save_index()
+            raise
+        index["state"] = "archived"
+        save_index()
+        return {"archived": candidates, "preserved": preserved, "archive": str(archive)}
+
+
 def full_image_pattern(target: str, version: str, short_source: str) -> str:
     return f"{target}-full-*-{version}-{short_source}.bin"
 
@@ -400,6 +539,15 @@ def main() -> None:
     environment["PLATFORMIO_BUILD_DIR"] = str(utility_build_dir)
     work.mkdir(parents=True, exist_ok=True)
     environment["OUTPUT_DIR"] = str(work)
+    if args.resume:
+        archived = archive_capacity_rejected_attempts(work, args.firmware_version, source)
+        if archived["archived"]:
+            print(f'Archived {len(archived["archived"])} proven portable-capacity rejections: '
+                  f'{archived["archive"]}', flush=True)
+        print(f'Preserved {len(archived["preserved"])} other package manifests; '
+              'ordinary resume qualification still applies.', flush=True)
+        # All other packages retain the existing recipe/qualification checks
+        # in build_legacy.sh; this helper never makes them resumable.
     run_logged([
         "bash", "build_legacy.sh", "build-firmwares-logging-matrix",
         "--firmware-version", args.firmware_version,

scripts/generate_mesh_america_release.py

@@ -81,10 +81,9 @@ def read_stage(stage):
         if len(manifests) != group["target_count"]:
             raise ValueError("release target count differs from manifest")
         for row in manifests:
-            key = row["artifact_target"]
-            if key in seen or row.get("verified") is not True:
-                raise ValueError("duplicate or unqualified release profile")
-            seen.add(key)
+            logical_target = row["artifact_target"]
+            if row.get("verified") is not True or row.get("source_commit", source) != source:
+                raise ValueError("unqualified or wrong-source release profile")
             files = []
             for name in row["files"]:
                 if Path(name).name != name or name not in checksums:
@@ -93,12 +92,35 @@ def read_stage(stage):
                 if hashlib.sha256(path.read_bytes()).hexdigest() != checksums[name]:
                     raise ValueError(f"release asset checksum failed: {name}")
                 if path.suffix in SUFFIXES:
-                    if identity(name) != key or family not in name:
-                        raise ValueError("firmware filename differs from qualified profile")
+                    if family not in name:
+                        raise ValueError("firmware filename belongs to a different family")
                     files.append(path)
             if not files:
                 raise ValueError("qualified profile contains no firmware")
-            records.append((row, group["tag"], files))
+            identities = {identity(path.name) for path in files}
+            if len(identities) != 1:
+                raise ValueError("firmware filenames disagree on publication profile")
+            published_target = identities.pop()
+            # Full images keep their OTA identity while the filename adds its
+            # qualified build profile. Do not relabel the logical target.
+            infix = published_target.removeprefix(logical_target)
+            valid_infix = (published_target == logical_target
+                or (published_target.startswith(logical_target + "-") and (
+                    (infix == "-ota" and "lora" in row.get("ota_update_methods", []))
+                    or (row["build_profile"] == "full" and
+                        re.fullmatch(r"-full(?:-usb-wifi|-logging)?(?:-ota)?", infix))
+                    or (row["build_profile"] == "logging" and
+                        re.fullmatch(r"-logging(?:-ota)?", infix)))))
+            qualified_stem = published_target + "-" + family
+            if any(path.stem.removesuffix("-merged") != qualified_stem for path in files):
+                raise ValueError("firmware filename differs from exact release source/version")
+            if (not valid_infix or any(qualified_stem + suffix not in row["files"]
+                                      for suffix in (".capabilities.json", ".memory.json"))):
+                raise ValueError("firmware filename differs from qualified profile")
+            if published_target in seen:
+                raise ValueError("duplicate release publication profile")
+            seen.add(published_target)
+            records.append(({**row, "publication_target": published_target}, group["tag"], files))
     return plan, family, records
 
 
@@ -116,7 +138,7 @@ def generate(catalog, plan, family, records, profiles, repo):
                     by_hardware.setdefault((device["type"], hardware), set()).add(index)
         result["device"][index]["firmware"] = []
     for row, tag, files in records:
-        target = row["artifact_target"]
+        target = row.get("publication_target", row["artifact_target"])
         parsed = profiles[target]
         platform = PLATFORMS[row["platform"]]
         indexes = by_identity.get(target) or by_identity.get(row["target"])
@@ -201,7 +223,7 @@ def main():
     args = parser.parse_args()
     catalog = json.loads(args.catalog.read_text())
     plan, family, records = read_stage(args.stage)
-    targets = {row["artifact_target"] for row, _, _ in records}
+    targets = {row.get("publication_target", row["artifact_target"]) for row, _, _ in records}
     targets.update(identity(file["name"]) for device in catalog["device"]
                    for firmware in device["firmware"] for version in firmware["version"].values()
                    for file in version["files"])

scripts/package_cascade_release.py

@@ -290,18 +290,24 @@ def collect_local_release(directory, version, source):
         if any(category(record) != group.name for record in group_records):
             raise ValueError("local release firmware group disagrees with qualification")
         records.extend(group_records)
-    indexed = {record["manifest"]["artifact_target"]: record for record in records}
+    # Standard and expanded Full images may retain the same OTA identity.
+    # Match each independently qualified package by its exact file inventory;
+    # the logical target alone cannot distinguish those publication profiles.
+    indexed = {frozenset(str(p.relative_to(directory)) for p in record["files"]): record
+               for record in records}
     entries = manifest["firmware"]
     if (len(indexed) != len(records) or len(entries) != len(records)
             or len(records) != manifest["firmware_target_count"]
-            or len({entry["artifact_target"] for entry in entries}) != len(entries)):
+            or any(len(entry["files"]) != len(set(entry["files"])) for entry in entries)
+            or len({frozenset(entry["files"]) for entry in entries}) != len(entries)):
         raise ValueError("local release target inventory disagrees with qualification")
     for entry in entries:
-        record = indexed.get(entry["artifact_target"])
+        record = indexed.get(frozenset(entry["files"]))
         if record is None:
             raise ValueError("local release contains an unqualified target")
         qualified = record["manifest"]
         if (qualified["target"] != entry["target"]
+                or qualified["artifact_target"] != entry["artifact_target"]
                 or qualified.get("source_commit", source) != source
                 or qualified["platform"] != entry["platform"]
                 or qualified["build_profile"] != entry["profile"]
@@ -404,6 +410,8 @@ def main():
         summaries = []
         for record in group["records"]:
             manifest = record["manifest"]
+            if manifest.get("source_commit", args.commit) != args.commit:
+                raise ValueError("qualified firmware belongs to another source")
             file_links = []
             for path in record["files"]:
                 shutil.copy2(path, destination / path.name)
@@ -412,7 +420,10 @@ def main():
                     label = "merged.bin (USB)" if path.name.endswith("-merged.bin") else path.suffix[1:]
                     file_links.append(f'<a href="{html.escape(url)}">{html.escape(label)}</a>')
             methods = ", ".join(manifest.get("ota_update_methods", [])) or "USB"
-            summaries.append({**manifest, "sensor_profile": nrf52_sensor_profile(manifest),
+            # Older capability manifests inherit source provenance from the
+            # validated completed release inventory. Preserve their raw bytes.
+            summaries.append({**manifest, "source_commit": args.commit,
+                              "sensor_profile": nrf52_sensor_profile(manifest),
                               "files": [path.name for path in record["files"]]})
             rows.append(f"<tr><td>{html.escape(manifest['artifact_target'])}</td><td>{html.escape(release_profile_label(manifest))}</td><td>{html.escape(methods)}</td><td>{' | '.join(file_links)}</td></tr>")
         if group['key'] == 'utility':