ci: trace Intel Setup aborts without debugger access

.github/workflows/nix.yml

@@ -154,6 +154,15 @@ jobs:
           lldb_path=$(xcrun --find lldb)
           printf 'Intel Setup host debugger: %s\n' "$lldb_path"
           "$lldb_path" --version
+          clang_path=$(xcrun --find clang)
+          sdk_path=$(xcrun --sdk macosx --show-sdk-path)
+          printf 'Intel Setup abort tracer compiler: %s\n' "$clang_path"
+          "$clang_path" --version
+          abort_tracer="$GITHUB_WORKSPACE/intel-setup-abort.dylib"
+          "$clang_path" -isysroot "$sdk_path" -arch x86_64 \
+            -std=c11 -Werror=vla -pedantic -Wall -Wextra -Werror \
+            -dynamiclib -O0 -g -fno-omit-frame-pointer \
+            scripts/nix/trace_intel_setup_abort.c -o "$abort_tracer"
           flake_ref="git+file://$GITHUB_WORKSPACE?rev=$(git rev-parse HEAD)"
           original=$(cat <<NIX
           let
@@ -170,6 +179,7 @@ jobs:
           # Only this diagnostic permits host LLDB access outside the Nix sandbox.
           exec nix build --file scripts/nix/trace-intel-setup.nix \
             --arg original "$original" --argstr expectedDrv "$expected_drv" \
+            --argstr abortTracerPath "$abort_tracer" \
             --argstr lldbPath "$lldb_path" --option sandbox false \
             --no-link -L --keep-failed --max-jobs 1 --timeout 7200
           TRACE

scripts/nix/README.md

@@ -172,10 +172,22 @@ runs its actual configure arguments under the runner's Apple LLDB. The probe
 records bounded thread backtraces, registers, instructions and loaded images;
 compiler queries and other Setup commands pass through. Child failures remain
 failures, and the canonical job stays failed. The diagnostic has a two-hour
-step/per-derivation limit and a five-minute limit for LLDB itself. Its log is
+step/per-derivation limit and one five-minute deadline for LLDB plus any fallback.
+Its log is
 uploaded as `intel-setup-lldb-trace.log`, including after failure. Only this probe
 disables the Nix sandbox to reach the host debugger; it records that runtime
-difference and the debugger version. A successful probe is diagnostic evidence.
+difference and the debugger version. If LLDB explicitly denies permission before
+launching a child, the probe records that denial and runs the same Setup bytes
+with a diagnostic abort interposer built by the runner's existing Apple Clang.
+It reports up to 80 current-thread unwind frames, available callee-saved
+registers and image addresses. It symbolicates mapped register addresses without
+dereferencing them, then forwards to real `abort`. Its constructor
+restores the original loader environment before Setup starts compiler children
+and disables core output for the fallback process tree. This adds a loaded image
+to Setup; it changes no debugger permissions or security settings. Child
+exits/signals remain failures. The Intel fallback runtime is
+unverified until CI executes it; native ARM64 C controls cover the helper.
+A successful probe is diagnostic evidence.
 
 Hydra continues to consume
 the same `hydraJobs` attributes. The older application release workflow still

scripts/nix/trace-intel-setup.nix

@@ -1,10 +1,12 @@
 # Failure-only diagnostic for the pinned Intel Darwin compiler's ghc-boot Setup.
 # Resolve lldbPath with xcrun on the runner before evaluation. Only this
 # diagnostic build may use `--option sandbox false` to reach Apple's debugger.
-{ original, lldbPath
+{ original, lldbPath, abortTracerPath
 , expectedDrv ? "/nix/store/vghjgkr64qjigddh3gnf3c103wbvpxmz-ghc-boot-lib-ghc-boot-9.14.drv"
 }:
 let
+  # Track the runner-compiled dylib's bytes, not a mutable host-path reference.
+  abortTracer = builtins.path { path = abortTracerPath; name = "intel-setup-abort.dylib"; };
   shellQuote = value: "'" + builtins.replaceStrings [ "'" ] [ "'\\''" ] value + "'";
   needle = "    printf '#!/bin/sh\\nexec %s \"$@\"\\n' \"$g\" > \"$wrapBin/$gName\"";
   replacement = ''
@@ -56,7 +58,7 @@ let
     ${shellQuote lldbPath} --version
     exec timeout --kill-after=10s 300s ${shellQuote lldbPath} --batch --no-lldbinit \
       -o "command script import \"$driverDir/trace_intel_setup_lldb.py\"" \
-      -o "intel-setup-run \"$argvFile\"" -- "$realSetup"
+      -o "intel-setup-run \"$argvFile\" ${shellQuote (toString abortTracer)}" -- "$realSetup"
     SIMPLEX_SETUP_EXECUTION
           chmod +x "$setupOutput"
         fi
@@ -74,6 +76,7 @@ assert original.system == "x86_64-darwin";
 assert original.identifier.name == "ghc-boot";
 assert original.drvPath == expectedDrv;
 assert builtins.isString lldbPath && builtins.substring 0 1 lldbPath == "/";
+assert builtins.isString abortTracerPath && builtins.substring 0 1 abortTracerPath == "/";
 original.overrideAttrs (old:
   assert occurrences old.buildPhase == 1; {
     buildPhase = builtins.replaceStrings [ needle ] [ replacement ] old.buildPhase + ''

scripts/nix/trace_intel_setup_abort.c

@@ -0,0 +1,196 @@
+/* Diagnostic-only abort interposer. Never loaded by the canonical build. */
+#define _DARWIN_C_SOURCE 1
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <libunwind.h>
+#include <mach-o/dyld.h>
+#include <stdarg.h>
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+static void line(const char *format, ...)
+{
+    char buffer[2048];
+    va_list arguments;
+    va_start(arguments, format);
+    int count = vsnprintf(buffer, sizeof(buffer), format, arguments);
+    va_end(arguments);
+    if (count < 0) {
+        return;
+    }
+    size_t length = (size_t)count;
+    if (length >= sizeof(buffer)) {
+        length = sizeof(buffer) - 1;
+        buffer[length - 1] = '\n';
+    }
+    size_t written = 0;
+    while (written < length) {
+        ssize_t result = write(STDERR_FILENO, buffer + written, length - written);
+        if (result > 0) {
+            written += (size_t)result;
+        } else if (result < 0 && errno == EINTR) {
+            continue;
+        } else {
+            break;
+        }
+    }
+}
+
+__attribute__((constructor)) static void restore_child_environment(void)
+{
+    const char *present = getenv("SIMPLEX_SETUP_ABORT_DYLD_PRESENT");
+    const char *previous = getenv("SIMPLEX_SETUP_ABORT_DYLD_VALUE");
+    bool valid = present != NULL &&
+        (strcmp(present, "0") == 0 || strcmp(present, "1") == 0);
+    if (!valid || (strcmp(present, "1") == 0 && previous == NULL)) {
+        line("Intel Setup abort tracer: invalid private loader metadata\n");
+        _exit(125);
+    }
+    int restored = strcmp(present, "1") == 0
+        ? setenv("DYLD_INSERT_LIBRARIES", previous, 1)
+        : unsetenv("DYLD_INSERT_LIBRARIES");
+    if (restored != 0 || unsetenv("SIMPLEX_SETUP_ABORT_DYLD_PRESENT") != 0 ||
+        unsetenv("SIMPLEX_SETUP_ABORT_DYLD_VALUE") != 0) {
+        line("Intel Setup abort tracer: cannot restore child loader environment\n");
+        _exit(125);
+    }
+    /* The fallback reports addresses, never a core or arbitrary memory. */
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_CORE, &limit) != 0) {
+        line("Intel Setup abort tracer: cannot read core limit\n");
+        _exit(125);
+    }
+    limit.rlim_cur = 0;
+    if (setrlimit(RLIMIT_CORE, &limit) != 0) {
+        line("Intel Setup abort tracer: cannot disable diagnostic core output\n");
+        _exit(125);
+    }
+    line("Intel Setup abort tracer loaded; child loader environment restored; core output disabled\n");
+}
+
+static void address_details(uintptr_t value)
+{
+    /* Resolve an address without reading any pointed-to closure data. */
+    Dl_info address = {0};
+    if (dladdr((const void *)value, &address) != 0) {
+        uintptr_t start = (uintptr_t)address.dli_saddr;
+        bool named = address.dli_sname != NULL && address.dli_saddr != NULL && value >= start;
+        line("    address image=%s base=%p symbol=%s+0x%" PRIxPTR "\n",
+             address.dli_fname != NULL ? address.dli_fname : "<unavailable>",
+             address.dli_fbase, named ? address.dli_sname : "<unavailable>",
+             named ? value - start : 0);
+    }
+}
+
+static void saved_registers(unw_cursor_t *cursor)
+{
+#if defined(__x86_64__)
+    static const int registers[] = {
+        UNW_X86_64_RBX, UNW_X86_64_RBP, UNW_X86_64_R12,
+        UNW_X86_64_R13, UNW_X86_64_R14, UNW_X86_64_R15
+    };
+    static const char *const names[] = { "rbx", "rbp", "r12", "r13", "r14", "r15" };
+#elif defined(__aarch64__) || defined(__arm64__)
+    static const int registers[] = {
+        UNW_ARM64_X19, UNW_ARM64_X20, UNW_ARM64_X21, UNW_ARM64_X22,
+        UNW_ARM64_X23, UNW_ARM64_X24, UNW_ARM64_X25, UNW_ARM64_X26,
+        UNW_ARM64_X27, UNW_ARM64_X28, UNW_ARM64_FP, UNW_ARM64_LR
+    };
+    static const char *const names[] = {
+        "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26",
+        "x27", "x28", "fp", "lr"
+    };
+#else
+#error "Intel Setup abort tracer requires a 64-bit Darwin target"
+#endif
+    for (size_t index = 0; index < sizeof(registers) / sizeof(registers[0]); ++index) {
+        unw_word_t value = 0;
+        int result = unw_get_reg(cursor, registers[index], &value);
+        if (result == 0) {
+            line("  saved %s=0x%" PRIxPTR "\n", names[index], (uintptr_t)value);
+            address_details((uintptr_t)value);
+        } else {
+            line("  saved %s unavailable (%d)\n", names[index], result);
+        }
+    }
+}
+
+static void trace_frames(void)
+{
+    unw_context_t context;
+    unw_cursor_t cursor;
+    if (unw_getcontext(&context) != 0 || unw_init_local(&cursor, &context) != 0) {
+        line("Intel Setup abort tracer: local unwind initialization failed\n");
+        return;
+    }
+    for (unsigned int frame = 0; frame < 80; ++frame) {
+        unw_word_t ip = 0;
+        unw_word_t sp = 0;
+        unw_word_t offset = 0;
+        char symbol[512] = "<unavailable>";
+        int ip_result = unw_get_reg(&cursor, UNW_REG_IP, &ip);
+        int sp_result = unw_get_reg(&cursor, UNW_REG_SP, &sp);
+        int symbol_result = unw_get_proc_name(&cursor, symbol, sizeof(symbol), &offset);
+        symbol[sizeof(symbol) - 1] = '\0';
+        Dl_info image = {0};
+        int image_result = ip_result == 0 ? dladdr((const void *)(uintptr_t)ip, &image) : 0;
+        line("Intel Setup abort frame %u ip=0x%" PRIxPTR " (%d) sp=0x%" PRIxPTR
+             " (%d) symbol=%s+0x%" PRIxPTR " (%d) image=%s base=%p\n",
+             frame, (uintptr_t)ip, ip_result, (uintptr_t)sp, sp_result,
+             symbol, (uintptr_t)offset, symbol_result,
+             image_result != 0 && image.dli_fname != NULL ? image.dli_fname : "<unavailable>",
+             image_result != 0 ? image.dli_fbase : NULL);
+        saved_registers(&cursor);
+        int result = unw_step(&cursor);
+        if (result <= 0) {
+            line("Intel Setup abort unwind finished (%d)\n", result);
+            return;
+        }
+    }
+    line("Intel Setup abort backtrace truncated after 80 frames\n");
+}
+
+static void trace_images(void)
+{
+    uint32_t count = _dyld_image_count();
+    uint32_t bound = count < 256 ? count : 256;
+    for (uint32_t index = 0; index < bound; ++index) {
+        const char *name = _dyld_get_image_name(index);
+        line("Intel Setup abort image %u base=%p slide=0x%" PRIxPTR " path=%s\n",
+             index, (const void *)_dyld_get_image_header(index),
+             (uintptr_t)_dyld_get_image_vmaddr_slide(index),
+             name != NULL ? name : "<unavailable>");
+    }
+    if (bound != count) {
+        line("Intel Setup abort images truncated at 256 of %u\n", count);
+    }
+}
+
+/* Avoid overlapping or recursive logging if a failed unwind itself aborts. */
+static atomic_flag tracing = ATOMIC_FLAG_INIT;
+
+_Noreturn static void trace_abort(void)
+{
+    if (!atomic_flag_test_and_set_explicit(&tracing, memory_order_relaxed)) {
+        line("Intel Setup abort tracer: abort called; current-thread unwind is best-effort\n");
+        trace_frames();
+        trace_images();
+    }
+    /* dyld excludes the interposing image's own reference, calling real abort. */
+    abort();
+}
+
+/* Darwin's __interpose ABI is two pointer-sized words; no function/data cast. */
+__attribute__((used, section("__DATA,__interpose")))
+static const struct {
+    void (*replacement)(void);
+    void (*original)(void);
+} abort_interpose = { trace_abort, abort };

scripts/nix/trace_intel_setup_lldb.py

@@ -5,10 +5,49 @@ import json
 import os
 from pathlib import Path
 import shlex
+import subprocess
+import time
 
 import lldb
 
 
+def permission_denied_before_launch(error, process):
+    # Never repeat a process that LLDB launched, even if it already exited.
+    denied = "this is a non-interactive debug session, cannot get permission to debug processes."
+    pid = process.GetProcessID() if process.IsValid() else 0
+    return error.Fail() and denied in str(error) and pid == 0
+
+
+def abort_fallback(executable, argv, environment, tracer, deadline):
+    keys = ("SIMPLEX_SETUP_ABORT_DYLD_PRESENT", "SIMPLEX_SETUP_ABORT_DYLD_VALUE")
+    child_environment = dict(item.split("=", 1) for item in environment)
+    if any(key in child_environment for key in keys):
+        raise ValueError("abort tracer private environment key already exists")
+    original_dyld = child_environment.get("DYLD_INSERT_LIBRARIES")
+    child_environment[keys[0]] = "0" if original_dyld is None else "1"
+    if original_dyld is not None:
+        child_environment[keys[1]] = original_dyld
+    child_environment["DYLD_INSERT_LIBRARIES"] = str(tracer) + (
+        ":" + original_dyld if original_dyld else "")
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError("Intel Setup overall diagnostic deadline expired")
+    digest = hashlib.sha256(executable.read_bytes()).hexdigest()
+    print("Intel Setup fallback: original LLDB permission denial retained; "
+          "diagnostic-only abort dylib inserted into unchanged Setup; "
+          "constructor restores child loader environment; no core output", flush=True)
+    result = subprocess.run(argv, executable=str(executable), env=child_environment,
+                            timeout=remaining, check=False)
+    if hashlib.sha256(executable.read_bytes()).hexdigest() != digest:
+        raise RuntimeError("compiler-produced Setup bytes changed during fallback")
+    code = 128 - result.returncode if result.returncode < 0 else result.returncode
+    print("Intel Setup fallback result: " + json.dumps({
+        "returncode": result.returncode, "shell_status": code,
+        "executable_sha256": digest, "unchanged": True,
+    }), flush=True)
+    return code
+
+
 def nul_entries(path):
     data = path.read_bytes()
     if not data.endswith(b"\0"):
@@ -76,11 +115,13 @@ def stopped_signals(process):
 def run(debugger, command, _result, _internal_dict):
     process = None
     code = 1
+    deadline = time.monotonic() + 300
     try:
         arguments = shlex.split(command)
-        if len(arguments) != 1:
-            raise ValueError("expected one argv-capture pathname")
+        if len(arguments) not in (1, 2):
+            raise ValueError("expected argv-capture pathname and optional abort dylib")
         capture = Path(arguments[0])
+        tracer = Path(arguments[1]) if len(arguments) == 2 else None
         argv = nul_entries(capture)
         environment_file = Path(str(capture) + ".env")
         environment_bytes = environment_file.read_bytes()
@@ -113,6 +154,14 @@ def run(debugger, command, _result, _internal_dict):
         error = lldb.SBError()
         process = target.Launch(launch, error)
         if error.Fail() or not process.IsValid():
+            print("Intel Setup LLDB launch failure: " + json.dumps({
+                "error": str(error), "valid": process.IsValid(),
+                "pid": process.GetProcessID() if process.IsValid() else 0,
+                "state": process.GetState() if process.IsValid() else None,
+            }), flush=True)
+            if tracer is not None and permission_denied_before_launch(error, process):
+                code = abort_fallback(executable_path, argv, environment, tracer, deadline)
+                return
             raise RuntimeError("LLDB launch failed: " + str(error))
 
         unix_signals = process.GetUnixSignals()