diff --git a/scripts/nix/README.md b/scripts/nix/README.md index 692311ad4e..21252c4997 100644 --- a/scripts/nix/README.md +++ b/scripts/nix/README.md @@ -348,6 +348,20 @@ optional pinned GDB derivation via `--arg debugger`. Omitting `debugger` retains the phase-only trace. GDB leaves ASLR enabled; its host/runtime effects must be reported when comparing diagnostics. +After a failed Intel Darwin GitHub CI build, `build.sh` also prints technical +fields from recent Nix GHC `.ips` crash reports, if macOS publishes readable +reports. The collector reads the user and system `DiagnosticReports` directories +without a debugger or process replay. It selects reports modified since the +failed target's build started, then reports the exception, faulting thread's +registers and up to 80 frames with their original binary-image indexes. These +are recent compiler reports; their PID and executable path must be correlated +with the failed component before attributing a cause. At most three reports +are printed, each limited to 2 MiB input and 64 KiB technical output, with a +15-second overall deadline. Missing, inaccessible or incomplete reports do not +change the original build's exit status. Local builds and successful CI builds +do not invoke this collector. Run its focused regressions with +`python3 -B scripts/nix/test-darwin-ghc-crash.py`. + When the pinned Intel Darwin `ghc-boot` component fails, CI captures a separate Setup diagnostic. It selects one exact failed `ghc-boot` derivation from the canonical log and checks it against the selected compiler project; a rebuilt diff --git a/scripts/nix/build.sh b/scripts/nix/build.sh index b846f62635..45ac831dbf 100755 --- a/scripts/nix/build.sh +++ b/scripts/nix/build.sh @@ -78,8 +78,23 @@ for package in $packages; do printf '%s\n' "$attribute" else mkdir -p "$out_dir/$system" - nix build --print-build-logs --timeout "${NIX_BUILD_TIMEOUT:-7200}" \ + crash_since= + if [ "$system" = x86_64-darwin ] && [ "${GITHUB_ACTIONS:-}" = true ]; then + crash_since=$(date +%s) || crash_since= + fi + if nix build --print-build-logs --timeout "${NIX_BUILD_TIMEOUT:-7200}" \ --out-link "$out_dir/$system/$package" \ - "$repo_dir#$attribute" "$@" + "$repo_dir#$attribute" "$@"; then + : + else + status=$? + if [ -n "$crash_since" ] && command -v python3 >/dev/null 2>&1; then + # ReportCrash can provide a compiler fault's registers and stack even + # when the failure is outside the separate ghc-boot Setup diagnostic. + python3 "$repo_dir/scripts/nix/collect-darwin-ghc-crash.py" \ + --since "$crash_since" || printf '%s\n' 'Darwin crash-report collection failed.' >&2 + fi + exit "$status" + fi fi done diff --git a/scripts/nix/collect-darwin-ghc-crash.py b/scripts/nix/collect-darwin-ghc-crash.py new file mode 100644 index 0000000000..5b636ff19e --- /dev/null +++ b/scripts/nix/collect-darwin-ghc-crash.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Print bounded technical fields from recent Nix GHC crash reports.""" + +import argparse +import json +import os +from pathlib import Path +import re +import signal +import stat +import time + + +MAX_INPUT = 2 * 1024 * 1024 +MAX_OUTPUT = 64 * 1024 +MAX_REPORTS = 3 +PROCESS = re.compile(r"(?:ghc(?:-[\w.+-]+)?|iserv(?:-(?:prof|dyn))?)\Z") + + +class CollectionDeadline(Exception): + """Keep the deadline distinct from recoverable filesystem errors.""" + + +def select_fields(value, keys): + return {key: value[key] for key in keys if key in value} + + +def crash_fields(data): + # Apple documents two JSON objects: a metadata line, then the report. + # https://developer.apple.com/documentation/xcode/interpreting-the-json-format-of-a-crash-report + header, body = data.decode("utf-8").split("\n", 1) + if str(json.loads(header).get("bug_type")) != "309": + return None + report = json.loads(body) + executable = report.get("procPath", "") + if not executable.startswith("/nix/store/") or not PROCESS.fullmatch(Path(executable).name): + return None + threads = report.get("threads", []) + index = report.get("faultingThread") + if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < len(threads): + return None + thread = threads[index] + frames = [select_fields(frame, ("imageIndex", "imageOffset", "symbol", + "symbolLocation", "sourceFile", "sourceLine")) + for frame in thread.get("frames", [])[:80]] + images = report.get("usedImages", []) + image_indexes = sorted({frame["imageIndex"] for frame in frames + if isinstance(frame.get("imageIndex"), int) + and 0 <= frame["imageIndex"] < len(images)}) + # Keep indexes so each frame remains attributable after filtering images. + return { + "procPath": executable, + "pid": report.get("pid"), + "cpuType": report.get("cpuType"), + "exception": select_fields(report.get("exception", {}), + ("type", "signal", "subtype", "codes", "rawCodes")), + "termination": select_fields(report.get("termination", {}), + ("namespace", "code", "indicator", "flags")), + "faultingThread": index, + "frames": frames, + "framesTruncated": len(thread.get("frames", [])) > len(frames), + "threadState": thread.get("threadState"), + "images": [{"index": i, **{k: images[i].get(k) for k in + ("name", "path", "base", "size", "arch", "uuid")}} + for i in image_indexes], + } + + +def recent_reports(directories, since): + candidates = [] + for directory in directories: + try: + with os.scandir(directory) as entries: + # Do not recurse into archives or inspect unrelated filenames. + for count, entry in enumerate(entries): + if count >= 4096: + break + if entry.name.endswith(".ips") and entry.name.startswith(("ghc", "iserv")): + info = entry.stat(follow_symlinks=False) + if stat.S_ISREG(info.st_mode) and info.st_mtime >= since: + candidates.append((info.st_mtime, Path(entry.path))) + except OSError: + continue + return [path for _, path in sorted(candidates, reverse=True)[:MAX_REPORTS * 4]] + + +def collect(directories, since): + found = 0 + for path in recent_reports(directories, since): + try: + # Reject symlinks and non-regular files again at open time. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_mtime < since or info.st_size > MAX_INPUT: + continue + data = source.read(MAX_INPUT + 1) + if len(data) > MAX_INPUT: + continue + fields = crash_fields(data) + if fields is None: + continue + encoded = json.dumps(fields, ensure_ascii=True) + if len(encoded) > MAX_OUTPUT: + print("Darwin GHC crash report exceeds the technical-output bound", flush=True) + continue + print("Darwin GHC crash report: " + encoded, flush=True) + found += 1 + if found == MAX_REPORTS: + break + except (OSError, ValueError, TypeError, AttributeError, KeyError): + # ReportCrash can still be writing; a later bounded scan may succeed. + continue + return found + + +def deadline_expired(_signum, _frame): + raise CollectionDeadline("Darwin crash-report collection reached its 15-second bound") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--since", type=float, required=True, help="Build start as Unix seconds") + parser.add_argument("--directory", type=Path, action="append", help="Override report directories") + args = parser.parse_args() + directories = args.directory or [Path.home() / "Library/Logs/DiagnosticReports", + Path("/Library/Logs/DiagnosticReports")] + signal.signal(signal.SIGALRM, deadline_expired) + signal.alarm(15) + try: + for attempt in range(3): + if collect(directories, args.since): + return + if attempt < 2: + time.sleep(2) + print("No recent readable Nix GHC .ips crash report was published", flush=True) + except CollectionDeadline as error: + print(str(error), flush=True) + finally: + signal.alarm(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/nix/test-darwin-ghc-crash.py b/scripts/nix/test-darwin-ghc-crash.py new file mode 100644 index 0000000000..05b018f64f --- /dev/null +++ b/scripts/nix/test-darwin-ghc-crash.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Focused crash-report and canonical build-status regressions.""" + +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import time +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("crash_reports", ROOT / "collect-darwin-ghc-crash.py") +reports = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reports) + + +def ips(path="/nix/store/abc-ghc914-sh-compiler/lib/bin/ghc-9.14", **extra): + body = { + "procPath": path, "pid": 1234, "cpuType": "X86-64", + "faultingThread": 1, "exception": {"type": "EXC_BAD_ACCESS", "signal": "SIGBUS"}, + "termination": {"namespace": "SIGNAL", "code": 10}, + "threads": [{"frames": [{"symbol": "other_thread"}]}, { + "triggered": True, "threadState": {"flavor": "x86_THREAD_STATE", "rip": {"value": 4160}}, + "frames": [{"imageIndex": 2, "imageOffset": 64, "symbol": "crashed_here"}]}], + "usedImages": [{"path": "unused-one"}, {"path": "unused-two"}, { + "name": "ghc-9.14", "path": path, "base": 4096, "size": 1024, + "arch": "x86_64", "uuid": "fixture-uuid"}], + "crashReporterKey": "PRIVATE-HEADER", "environment": "PRIVATE-ENVIRONMENT", + } + body.update(extra) + return (json.dumps({"bug_type": "309", "incident_id": "PRIVATE-INCIDENT"}) + + "\n" + json.dumps(body)).encode() + + +class CrashReports(unittest.TestCase): + def test_faulting_thread_and_original_image_index(self): + result = reports.crash_fields(ips()) + self.assertEqual(result["frames"][0]["symbol"], "crashed_here") + self.assertEqual(result["threadState"]["rip"]["value"], 4160) + self.assertEqual(result["images"][0]["index"], 2) + self.assertEqual(result["images"][0]["base"] + result["frames"][0]["imageOffset"], 4160) + self.assertNotIn("PRIVATE", json.dumps(result)) + self.assertNotIn("other_thread", json.dumps(result)) + + def test_rejects_unrelated_process_and_non_crash_reports(self): + self.assertIsNone(reports.crash_fields(ips("/Applications/Other.app/Other"))) + self.assertIsNone(reports.crash_fields(ips().replace(b'"309"', b'"288"'))) + self.assertIsNone(reports.crash_fields(ips(faultingThread=99))) + + def test_only_recent_regular_bounded_files(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "ghc-current.ips").write_bytes(ips()) + old = root / "ghc-old.ips" + old.write_bytes(ips()) + os.utime(old, (1, 1)) + (root / "ghc-link.ips").symlink_to(root / "ghc-current.ips") + (root / "unrelated.ips").write_bytes(ips()) + (root / "ghc-incomplete.ips").write_text('{"bug_type":"309"}\n{') + (root / "ghc-large.ips").write_bytes(b"x" * (reports.MAX_INPUT + 1)) + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(reports.collect([root], time.time() - 30), 1) + self.assertEqual(output.getvalue().count("Darwin GHC crash report:"), 1) + + def test_report_frame_and_output_bounds(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index in range(5): + (root / f"ghc-{index}.ips").write_bytes(ips()) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(reports.collect([root], 0), reports.MAX_REPORTS) + frames = [{"symbol": "x" * 1000} for _ in range(100)] + (root / "ghc-huge-fields.ips").write_bytes(ips(threads=[{"frames": frames}], faultingThread=0)) + self.assertEqual(len(reports.crash_fields(ips(threads=[{"frames": frames}], faultingThread=0))["frames"]), 80) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "ghc-huge-fields.ips").write_bytes(ips(threads=[{"frames": frames}], faultingThread=0)) + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(reports.collect([root], 0), 0) + self.assertIn("technical-output bound", output.getvalue()) + self.assertLess(len(output.getvalue()), reports.MAX_OUTPUT) + + def test_deadline_cannot_be_swallowed_as_filesystem_error(self): + with patch.object(reports, "recent_reports", side_effect=reports.CollectionDeadline("bound")): + with self.assertRaises(reports.CollectionDeadline): + reports.collect([], 0) + + +class BuildStatus(unittest.TestCase): + def run_driver(self, status, system="x86_64-darwin", github="true", arguments=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / "scripts/nix/build.sh" + script.parent.mkdir(parents=True) + shutil.copy2(ROOT / "build.sh", script) + bin_dir = root / "bin" + bin_dir.mkdir() + for name, body in { + "nix": 'printf "%s\\n" "$*" >> "$NIX_TEST_LOG"\nexit "$NIX_TEST_STATUS"\n', + "python3": 'printf "COLLECTOR %s\\n" "$*"\nexit 99\n', + }.items(): + executable = bin_dir / name + executable.write_text("#!/bin/sh\n" + body) + executable.chmod(0o755) + env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + NIX_SYSTEM=system, GITHUB_ACTIONS=github, NIX_TEST_STATUS=str(status), + NIX_TEST_LOG=str(root / "calls")) + result = subprocess.run(["sh", str(script), *(arguments or ["native"])], + env=env, capture_output=True, text=True, timeout=5) + calls = (root / "calls").read_text().splitlines() if (root / "calls").exists() else [] + return result, calls + + def test_failed_intel_ci_build_keeps_exact_status_when_collector_fails(self): + result, calls = self.run_driver(42) + self.assertEqual(result.returncode, 42) + self.assertEqual(len(calls), 1) + self.assertIn("COLLECTOR", result.stdout) + self.assertIn("--since", result.stdout) + + def test_success_builds_both_original_goals_without_collection(self): + result, calls = self.run_driver(0) + self.assertEqual(result.returncode, 0) + self.assertEqual(len(calls), 2) + self.assertIn("hydraJobs.x86_64-darwin.lib:simplex-chat", calls[0]) + self.assertIn("hydraJobs.x86_64-darwin.exe:simplex-chat", calls[1]) + self.assertNotIn("COLLECTOR", result.stdout) + + def test_non_intel_and_non_ci_failures_keep_status_without_collection(self): + for system, github in [("x86_64-linux", "true"), ("aarch64-darwin", "true"), + ("x86_64-darwin", "false")]: + with self.subTest(system=system, github=github): + result, _ = self.run_driver(42, system, github) + self.assertEqual(result.returncode, 42) + self.assertNotIn("COLLECTOR", result.stdout) + + def test_list_does_not_build_or_collect(self): + result, calls = self.run_driver(42, arguments=["--list", "all"]) + self.assertEqual(result.returncode, 0) + self.assertEqual(len(result.stdout.splitlines()), 4) + self.assertEqual(calls, []) + self.assertNotIn("COLLECTOR", result.stdout) + + +if __name__ == "__main__": + unittest.main()