Collect bounded Darwin GHC crash reports after CI failure

scripts/nix/README.md

@@ -348,6 +348,20 @@ optional pinned GDB derivation via `--arg debugger`. Omitting `debugger` retains
 the phase-only trace. GDB leaves ASLR enabled; its host/runtime effects must be
 reported when comparing diagnostics.
 
+After a failed Intel Darwin GitHub CI build, `build.sh` also prints technical
+fields from recent Nix GHC `.ips` crash reports, if macOS publishes readable
+reports. The collector reads the user and system `DiagnosticReports` directories
+without a debugger or process replay. It selects reports modified since the
+failed target's build started, then reports the exception, faulting thread's
+registers and up to 80 frames with their original binary-image indexes. These
+are recent compiler reports; their PID and executable path must be correlated
+with the failed component before attributing a cause. At most three reports
+are printed, each limited to 2 MiB input and 64 KiB technical output, with a
+15-second overall deadline. Missing, inaccessible or incomplete reports do not
+change the original build's exit status. Local builds and successful CI builds
+do not invoke this collector. Run its focused regressions with
+`python3 -B scripts/nix/test-darwin-ghc-crash.py`.
+
 When the pinned Intel Darwin `ghc-boot` component fails, CI captures a separate
 Setup diagnostic. It selects one exact failed `ghc-boot` derivation from the
 canonical log and checks it against the selected compiler project; a rebuilt

scripts/nix/build.sh

@@ -78,8 +78,23 @@ for package in $packages; do
     printf '%s\n' "$attribute"
   else
     mkdir -p "$out_dir/$system"
-    nix build --print-build-logs --timeout "${NIX_BUILD_TIMEOUT:-7200}" \
+    crash_since=
+    if [ "$system" = x86_64-darwin ] && [ "${GITHUB_ACTIONS:-}" = true ]; then
+      crash_since=$(date +%s) || crash_since=
+    fi
+    if nix build --print-build-logs --timeout "${NIX_BUILD_TIMEOUT:-7200}" \
       --out-link "$out_dir/$system/$package" \
-      "$repo_dir#$attribute" "$@"
+      "$repo_dir#$attribute" "$@"; then
+      :
+    else
+      status=$?
+      if [ -n "$crash_since" ] && command -v python3 >/dev/null 2>&1; then
+        # ReportCrash can provide a compiler fault's registers and stack even
+        # when the failure is outside the separate ghc-boot Setup diagnostic.
+        python3 "$repo_dir/scripts/nix/collect-darwin-ghc-crash.py" \
+          --since "$crash_since" || printf '%s\n' 'Darwin crash-report collection failed.' >&2
+      fi
+      exit "$status"
+    fi
   fi
 done

scripts/nix/collect-darwin-ghc-crash.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Print bounded technical fields from recent Nix GHC crash reports."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import signal
+import stat
+import time
+
+
+MAX_INPUT = 2 * 1024 * 1024
+MAX_OUTPUT = 64 * 1024
+MAX_REPORTS = 3
+PROCESS = re.compile(r"(?:ghc(?:-[\w.+-]+)?|iserv(?:-(?:prof|dyn))?)\Z")
+
+
+class CollectionDeadline(Exception):
+    """Keep the deadline distinct from recoverable filesystem errors."""
+
+
+def select_fields(value, keys):
+    return {key: value[key] for key in keys if key in value}
+
+
+def crash_fields(data):
+    # Apple documents two JSON objects: a metadata line, then the report.
+    # https://developer.apple.com/documentation/xcode/interpreting-the-json-format-of-a-crash-report
+    header, body = data.decode("utf-8").split("\n", 1)
+    if str(json.loads(header).get("bug_type")) != "309":
+        return None
+    report = json.loads(body)
+    executable = report.get("procPath", "")
+    if not executable.startswith("/nix/store/") or not PROCESS.fullmatch(Path(executable).name):
+        return None
+    threads = report.get("threads", [])
+    index = report.get("faultingThread")
+    if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < len(threads):
+        return None
+    thread = threads[index]
+    frames = [select_fields(frame, ("imageIndex", "imageOffset", "symbol",
+                                   "symbolLocation", "sourceFile", "sourceLine"))
+              for frame in thread.get("frames", [])[:80]]
+    images = report.get("usedImages", [])
+    image_indexes = sorted({frame["imageIndex"] for frame in frames
+                            if isinstance(frame.get("imageIndex"), int)
+                            and 0 <= frame["imageIndex"] < len(images)})
+    # Keep indexes so each frame remains attributable after filtering images.
+    return {
+        "procPath": executable,
+        "pid": report.get("pid"),
+        "cpuType": report.get("cpuType"),
+        "exception": select_fields(report.get("exception", {}),
+                                   ("type", "signal", "subtype", "codes", "rawCodes")),
+        "termination": select_fields(report.get("termination", {}),
+                                     ("namespace", "code", "indicator", "flags")),
+        "faultingThread": index,
+        "frames": frames,
+        "framesTruncated": len(thread.get("frames", [])) > len(frames),
+        "threadState": thread.get("threadState"),
+        "images": [{"index": i, **{k: images[i].get(k) for k in
+                    ("name", "path", "base", "size", "arch", "uuid")}}
+                   for i in image_indexes],
+    }
+
+
+def recent_reports(directories, since):
+    candidates = []
+    for directory in directories:
+        try:
+            with os.scandir(directory) as entries:
+                # Do not recurse into archives or inspect unrelated filenames.
+                for count, entry in enumerate(entries):
+                    if count >= 4096:
+                        break
+                    if entry.name.endswith(".ips") and entry.name.startswith(("ghc", "iserv")):
+                        info = entry.stat(follow_symlinks=False)
+                        if stat.S_ISREG(info.st_mode) and info.st_mtime >= since:
+                            candidates.append((info.st_mtime, Path(entry.path)))
+        except OSError:
+            continue
+    return [path for _, path in sorted(candidates, reverse=True)[:MAX_REPORTS * 4]]
+
+
+def collect(directories, since):
+    found = 0
+    for path in recent_reports(directories, since):
+        try:
+            # Reject symlinks and non-regular files again at open time.
+            fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+            with os.fdopen(fd, "rb") as source:
+                info = os.fstat(source.fileno())
+                if not stat.S_ISREG(info.st_mode) or info.st_mtime < since or info.st_size > MAX_INPUT:
+                    continue
+                data = source.read(MAX_INPUT + 1)
+            if len(data) > MAX_INPUT:
+                continue
+            fields = crash_fields(data)
+            if fields is None:
+                continue
+            encoded = json.dumps(fields, ensure_ascii=True)
+            if len(encoded) > MAX_OUTPUT:
+                print("Darwin GHC crash report exceeds the technical-output bound", flush=True)
+                continue
+            print("Darwin GHC crash report: " + encoded, flush=True)
+            found += 1
+            if found == MAX_REPORTS:
+                break
+        except (OSError, ValueError, TypeError, AttributeError, KeyError):
+            # ReportCrash can still be writing; a later bounded scan may succeed.
+            continue
+    return found
+
+
+def deadline_expired(_signum, _frame):
+    raise CollectionDeadline("Darwin crash-report collection reached its 15-second bound")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--since", type=float, required=True, help="Build start as Unix seconds")
+    parser.add_argument("--directory", type=Path, action="append", help="Override report directories")
+    args = parser.parse_args()
+    directories = args.directory or [Path.home() / "Library/Logs/DiagnosticReports",
+                                     Path("/Library/Logs/DiagnosticReports")]
+    signal.signal(signal.SIGALRM, deadline_expired)
+    signal.alarm(15)
+    try:
+        for attempt in range(3):
+            if collect(directories, args.since):
+                return
+            if attempt < 2:
+                time.sleep(2)
+        print("No recent readable Nix GHC .ips crash report was published", flush=True)
+    except CollectionDeadline as error:
+        print(str(error), flush=True)
+    finally:
+        signal.alarm(0)
+
+
+if __name__ == "__main__":
+    main()

scripts/nix/test-darwin-ghc-crash.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Focused crash-report and canonical build-status regressions."""
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location("crash_reports", ROOT / "collect-darwin-ghc-crash.py")
+reports = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(reports)
+
+
+def ips(path="/nix/store/abc-ghc914-sh-compiler/lib/bin/ghc-9.14", **extra):
+    body = {
+        "procPath": path, "pid": 1234, "cpuType": "X86-64",
+        "faultingThread": 1, "exception": {"type": "EXC_BAD_ACCESS", "signal": "SIGBUS"},
+        "termination": {"namespace": "SIGNAL", "code": 10},
+        "threads": [{"frames": [{"symbol": "other_thread"}]}, {
+            "triggered": True, "threadState": {"flavor": "x86_THREAD_STATE", "rip": {"value": 4160}},
+            "frames": [{"imageIndex": 2, "imageOffset": 64, "symbol": "crashed_here"}]}],
+        "usedImages": [{"path": "unused-one"}, {"path": "unused-two"}, {
+            "name": "ghc-9.14", "path": path, "base": 4096, "size": 1024,
+            "arch": "x86_64", "uuid": "fixture-uuid"}],
+        "crashReporterKey": "PRIVATE-HEADER", "environment": "PRIVATE-ENVIRONMENT",
+    }
+    body.update(extra)
+    return (json.dumps({"bug_type": "309", "incident_id": "PRIVATE-INCIDENT"})
+            + "\n" + json.dumps(body)).encode()
+
+
+class CrashReports(unittest.TestCase):
+    def test_faulting_thread_and_original_image_index(self):
+        result = reports.crash_fields(ips())
+        self.assertEqual(result["frames"][0]["symbol"], "crashed_here")
+        self.assertEqual(result["threadState"]["rip"]["value"], 4160)
+        self.assertEqual(result["images"][0]["index"], 2)
+        self.assertEqual(result["images"][0]["base"] + result["frames"][0]["imageOffset"], 4160)
+        self.assertNotIn("PRIVATE", json.dumps(result))
+        self.assertNotIn("other_thread", json.dumps(result))
+
+    def test_rejects_unrelated_process_and_non_crash_reports(self):
+        self.assertIsNone(reports.crash_fields(ips("/Applications/Other.app/Other")))
+        self.assertIsNone(reports.crash_fields(ips().replace(b'"309"', b'"288"')))
+        self.assertIsNone(reports.crash_fields(ips(faultingThread=99)))
+
+    def test_only_recent_regular_bounded_files(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "ghc-current.ips").write_bytes(ips())
+            old = root / "ghc-old.ips"
+            old.write_bytes(ips())
+            os.utime(old, (1, 1))
+            (root / "ghc-link.ips").symlink_to(root / "ghc-current.ips")
+            (root / "unrelated.ips").write_bytes(ips())
+            (root / "ghc-incomplete.ips").write_text('{"bug_type":"309"}\n{')
+            (root / "ghc-large.ips").write_bytes(b"x" * (reports.MAX_INPUT + 1))
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                self.assertEqual(reports.collect([root], time.time() - 30), 1)
+            self.assertEqual(output.getvalue().count("Darwin GHC crash report:"), 1)
+
+    def test_report_frame_and_output_bounds(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            for index in range(5):
+                (root / f"ghc-{index}.ips").write_bytes(ips())
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(reports.collect([root], 0), reports.MAX_REPORTS)
+            frames = [{"symbol": "x" * 1000} for _ in range(100)]
+            (root / "ghc-huge-fields.ips").write_bytes(ips(threads=[{"frames": frames}], faultingThread=0))
+            self.assertEqual(len(reports.crash_fields(ips(threads=[{"frames": frames}], faultingThread=0))["frames"]), 80)
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "ghc-huge-fields.ips").write_bytes(ips(threads=[{"frames": frames}], faultingThread=0))
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                self.assertEqual(reports.collect([root], 0), 0)
+            self.assertIn("technical-output bound", output.getvalue())
+            self.assertLess(len(output.getvalue()), reports.MAX_OUTPUT)
+
+    def test_deadline_cannot_be_swallowed_as_filesystem_error(self):
+        with patch.object(reports, "recent_reports", side_effect=reports.CollectionDeadline("bound")):
+            with self.assertRaises(reports.CollectionDeadline):
+                reports.collect([], 0)
+
+
+class BuildStatus(unittest.TestCase):
+    def run_driver(self, status, system="x86_64-darwin", github="true", arguments=None):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            script = root / "scripts/nix/build.sh"
+            script.parent.mkdir(parents=True)
+            shutil.copy2(ROOT / "build.sh", script)
+            bin_dir = root / "bin"
+            bin_dir.mkdir()
+            for name, body in {
+                "nix": 'printf "%s\\n" "$*" >> "$NIX_TEST_LOG"\nexit "$NIX_TEST_STATUS"\n',
+                "python3": 'printf "COLLECTOR %s\\n" "$*"\nexit 99\n',
+            }.items():
+                executable = bin_dir / name
+                executable.write_text("#!/bin/sh\n" + body)
+                executable.chmod(0o755)
+            env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ["PATH"],
+                       NIX_SYSTEM=system, GITHUB_ACTIONS=github, NIX_TEST_STATUS=str(status),
+                       NIX_TEST_LOG=str(root / "calls"))
+            result = subprocess.run(["sh", str(script), *(arguments or ["native"])],
+                                    env=env, capture_output=True, text=True, timeout=5)
+            calls = (root / "calls").read_text().splitlines() if (root / "calls").exists() else []
+            return result, calls
+
+    def test_failed_intel_ci_build_keeps_exact_status_when_collector_fails(self):
+        result, calls = self.run_driver(42)
+        self.assertEqual(result.returncode, 42)
+        self.assertEqual(len(calls), 1)
+        self.assertIn("COLLECTOR", result.stdout)
+        self.assertIn("--since", result.stdout)
+
+    def test_success_builds_both_original_goals_without_collection(self):
+        result, calls = self.run_driver(0)
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(len(calls), 2)
+        self.assertIn("hydraJobs.x86_64-darwin.lib:simplex-chat", calls[0])
+        self.assertIn("hydraJobs.x86_64-darwin.exe:simplex-chat", calls[1])
+        self.assertNotIn("COLLECTOR", result.stdout)
+
+    def test_non_intel_and_non_ci_failures_keep_status_without_collection(self):
+        for system, github in [("x86_64-linux", "true"), ("aarch64-darwin", "true"),
+                               ("x86_64-darwin", "false")]:
+            with self.subTest(system=system, github=github):
+                result, _ = self.run_driver(42, system, github)
+                self.assertEqual(result.returncode, 42)
+                self.assertNotIn("COLLECTOR", result.stdout)
+
+    def test_list_does_not_build_or_collect(self):
+        result, calls = self.run_driver(42, arguments=["--list", "all"])
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(len(result.stdout.splitlines()), 4)
+        self.assertEqual(calls, [])
+        self.assertNotIn("COLLECTOR", result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()