fix(nix): resolve selected Apple diagnostic tools directly [skip ci]

scripts/nix/README.md

@@ -405,6 +405,17 @@ deadline and kills only that group on expiry, with at most five seconds to reap
 the command; GNU `timeout` and Homebrew aliases are not required. Missing Apple
 or Nix tools are named explicitly.
 
+For the default Apple toolchain, the launcher reads `DEVELOPER_DIR` or makes
+one bounded `xcode-select --print-path` query, then validates the existing
+Clang executable and macOS SDK in that selected Xcode or Command Line Tools
+installation. This avoids `xcrun --find clang`, which reached its ten-second
+limit on hosted Intel before the recorder could start. An invalid selection
+does not fall back to another installation. Explicit custom `TOOLCHAINS`
+selection retains the original bounded `xcrun` queries. The Intel policy tests
+also check the real selected Apple paths and Clang version on hosted Darwin
+before the production build starts; this metadata check has a twenty-second
+overall limit and does not compile or run GHC.
+
 The recorder loads only into the selected normal SimpleX iOS compiler process.
 It reports its main executable's UUID, image base and slide, then records SIGBUS
 fault address, signal code and registers, including PC and SP, if the process

scripts/nix/test-select-intel-setup.py

@@ -1,5 +1,8 @@
 import importlib.util
+import os
 import pathlib
+import sys
+import time
 import unittest
 
 
@@ -45,5 +48,24 @@ class FailedComponentSelection(unittest.TestCase):
             )
 
 
+class AppleToolDiscovery(unittest.TestCase):
+    @unittest.skipUnless(sys.platform == "darwin" and os.environ.get("GITHUB_ACTIONS") == "true",
+                         "real Apple metadata check runs on hosted Darwin CI")
+    def test_selected_tools_before_production_build(self):
+        # This suite already runs in the Intel policy step before the long
+        # build. Exercise the actual resolver early, without invoking GHC.
+        spec = importlib.util.spec_from_file_location(
+            "intel_trace", pathlib.Path(__file__).with_name("trace-intel-ghc.py")
+        )
+        trace = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(trace)
+        deadline = time.monotonic() + 20
+        compiler, sdk = trace.apple_tool_paths(deadline)
+        self.assertTrue(compiler.is_file())
+        self.assertTrue(sdk.is_dir())
+        version = trace.run_step([compiler, "--version"], deadline, 10, True, phase="check-apple-clang")
+        self.assertIn("Apple clang", version)
+
+
 if __name__ == "__main__":
     unittest.main()

scripts/nix/test-trace-intel-ghc.py

@@ -26,6 +26,18 @@ FAILURE = ("Failed to build simplex-chat-7.1.0.11-unit. The build process termin
 
 
 class LauncherTests(unittest.TestCase):
+    def apple_installation(self, root, *, command_line=False):
+        developer = Path(root) / ("CommandLineTools" if command_line else "Xcode.app/Contents/Developer")
+        compiler = developer / ("usr/bin/clang" if command_line
+                                else "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang")
+        sdk = developer / ("SDKs/MacOSX.sdk" if command_line
+                           else "Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk")
+        compiler.parent.mkdir(parents=True, exist_ok=True)
+        compiler.write_text("fixture Apple tool; never executed\n")
+        compiler.chmod(0o755)
+        sdk.mkdir(parents=True, exist_ok=True)
+        return developer, compiler, sdk
+
     def test_unique_observed_failure(self):
         self.assertEqual(TRACE.select_failure(FAILURE + FAILURE), DRV)
         self.assertIsNone(TRACE.select_failure(FAILURE.replace("exit code -10", "exit code 1")))
@@ -34,7 +46,8 @@ class LauncherTests(unittest.TestCase):
             TRACE.select_failure(FAILURE + FAILURE.replace("a" * 32, "b" * 32))
 
     def invoke(self, folder, *, actions="true", machine="x86_64", target=None, failure=FAILURE,
-               reject_build=False, reject_cache=False):
+               reject_build=False, reject_cache=False, reject_xcrun=False):
+        developer, compiler, sdk = self.apple_installation(folder)
         log = Path(folder) / "nix-build.log"
         log.write_text(failure)
         calls = []
@@ -43,15 +56,18 @@ class LauncherTests(unittest.TestCase):
             calls.append((values, seconds, capture))
             if values[0] == "git":
                 return "c" * 40
+            if values[0] == "xcrun" and reject_xcrun:
+                raise TimeoutError("Diagnostic command reached its wall-clock limit")
             if values[:3] == ["xcrun", "--find", "clang"]:
-                return "/usr/bin/clang"
+                return str(compiler)
             if values[0] == "xcrun":
-                return "/existing/SDK"
+                return str(sdk)
             if (reject_build and "--file" in values) or (reject_cache and "--expr" in values):
                 raise RuntimeError("Diagnostic command returned 1")
             return None
         argv = ["trace-intel-ghc.py", "--repo", folder, "--attribute", target or TRACE.ATTRIBUTE, "--log", str(log)]
-        with patch.dict(os.environ, {"GITHUB_ACTIONS": actions}), patch.object(TRACE.platform, "system", return_value="Darwin"), \
+        with patch.dict(os.environ, {"GITHUB_ACTIONS": actions, "DEVELOPER_DIR": str(developer), "TOOLCHAINS": ""}), \
+                patch.object(TRACE.platform, "system", return_value="Darwin"), \
                 patch.object(TRACE.platform, "machine", return_value=machine), \
                 patch.object(TRACE.shutil, "which", side_effect=lambda tool: None if tool in ("timeout", "gtimeout") else "/existing/tool"), \
                 patch.object(TRACE, "run_step", side_effect=step), patch("sys.argv", argv), \
@@ -88,7 +104,81 @@ class LauncherTests(unittest.TestCase):
             original = arguments[arguments.index("original") + 1]
             self.assertIn('f.hydraJobs.x86_64-darwin."x86_64-darwin-ios:lib:simplex-chat"', original)
             self.assertNotIn(".project.hsPkgs", original)
-            self.assertEqual(sum(call[0][:1] == ["/usr/bin/clang"] for call in calls), 1)
+            compiler = str(Path(folder) / "Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang")
+            self.assertEqual(sum(call[0][:1] == [compiler] for call in calls), 1)
+
+    def test_observed_xcrun_timeout_does_not_block_selected_tools(self):
+        with tempfile.TemporaryDirectory() as folder:
+            result, calls = self.invoke(folder, reject_xcrun=True)
+        self.assertEqual(result, 0)
+        self.assertEqual(sum("--file" in call[0] for call in calls), 1)
+        self.assertFalse(any(call[0][0] == "xcrun" for call in calls))
+
+    def test_selected_xcode_and_command_line_tools_need_no_discovery_process(self):
+        with tempfile.TemporaryDirectory() as folder:
+            for command_line in (False, True):
+                developer, compiler, sdk = self.apple_installation(folder, command_line=command_line)
+                with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": "", "SDKROOT": "/unused/iPhoneOS.sdk"}), \
+                        patch.object(TRACE, "run_step") as step, contextlib.redirect_stdout(io.StringIO()):
+                    self.assertEqual(TRACE.apple_tool_paths(time.monotonic() + 20), (compiler, sdk))
+                step.assert_not_called()
+
+    def test_xcode_application_selection_is_supported(self):
+        with tempfile.TemporaryDirectory() as folder:
+            developer, compiler, sdk = self.apple_installation(folder)
+            with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer.parent.parent), "TOOLCHAINS": ""}), \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(TRACE.apple_tool_paths(time.monotonic() + 20), (compiler, sdk))
+
+    def test_system_developer_selection_is_bounded(self):
+        with tempfile.TemporaryDirectory() as folder:
+            developer, compiler, sdk = self.apple_installation(folder)
+            deadline = time.monotonic() + 20
+            with patch.dict(os.environ, {}, clear=True), \
+                    patch.object(TRACE.shutil, "which", return_value="/usr/bin/xcode-select"), \
+                    patch.object(TRACE, "run_step", return_value=str(developer)) as step, \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(TRACE.apple_tool_paths(deadline), (compiler, sdk))
+            step.assert_called_once_with(["xcode-select", "--print-path"], deadline, 10, True,
+                                         phase="apple-developer-path")
+
+    def test_invalid_developer_selection_never_uses_another_installation(self):
+        with tempfile.TemporaryDirectory() as folder:
+            for selection in ("", "relative", "bad\npath", str(Path(folder) / "missing")):
+                with self.subTest(selection=selection), \
+                        patch.dict(os.environ, {"DEVELOPER_DIR": selection, "TOOLCHAINS": ""}), \
+                        patch.object(TRACE, "run_step") as step:
+                    with self.assertRaisesRegex(ValueError, "developer directory"):
+                        TRACE.apple_tool_paths(time.monotonic() + 20)
+                    step.assert_not_called()
+
+    def test_missing_selected_clang_or_sdk_is_named(self):
+        with tempfile.TemporaryDirectory() as folder:
+            developer, compiler, sdk = self.apple_installation(folder)
+            with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": ""}):
+                compiler.chmod(0o644)
+                with self.assertRaisesRegex(ValueError, "Clang.*not executable"):
+                    TRACE.apple_tool_paths(time.monotonic() + 20)
+                compiler.chmod(0o755)
+                sdk.rmdir()
+                with self.assertRaisesRegex(ValueError, "macOS SDK is unavailable"):
+                    TRACE.apple_tool_paths(time.monotonic() + 20)
+
+    def test_custom_toolchain_uses_original_bounded_discovery(self):
+        with tempfile.TemporaryDirectory() as folder:
+            developer, compiler, sdk = self.apple_installation(folder)
+            custom = Path(folder) / "Selected.xctoolchain/usr/bin/clang"
+            custom.parent.mkdir(parents=True)
+            custom.write_text("fixture custom tool; never executed\n")
+            custom.chmod(0o755)
+            deadline = time.monotonic() + 30
+            with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": "selected.custom.toolchain"}), \
+                    patch.object(TRACE.shutil, "which", return_value="/usr/bin/xcrun"), \
+                    patch.object(TRACE, "run_step", side_effect=[str(custom), str(sdk)]) as step, \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(TRACE.apple_tool_paths(deadline), (custom, sdk))
+            self.assertEqual(step.call_args_list[0].args, (["xcrun", "--find", "clang"], deadline, 10, True))
+            self.assertEqual(step.call_args_list[1].args, (["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True))
 
     def test_failed_diagnostic_returns_without_replay_loop(self):
         with tempfile.TemporaryDirectory() as folder:

scripts/nix/trace-intel-ghc.py

@@ -42,6 +42,51 @@ def python_expression(flake):
             + ' { haskellNix = f.inputs.haskellNix; } "x86_64-darwin").python3')
 
 
+def apple_tool_paths(deadline):
+    developer = os.environ.get("DEVELOPER_DIR")
+    if developer is None:
+        if not shutil.which("xcode-select"):
+            raise ValueError("Required existing diagnostic tool unavailable: xcode-select")
+        developer = run_step(["xcode-select", "--print-path"], deadline, 10, True,
+                             phase="apple-developer-path")
+    if not developer or any(char in developer for char in "\r\n\0"):
+        raise ValueError("Invalid selected Apple developer directory")
+    directory = Path(developer)
+    if not directory.is_absolute():
+        raise ValueError("Expected an absolute selected Apple developer directory")
+    # xcode-select accepts either the application or its Developer directory.
+    if directory.suffix == ".app":
+        directory = directory / "Contents/Developer"
+    if not directory.is_dir():
+        raise ValueError("Selected Apple developer directory is unavailable: " + str(directory))
+
+    toolchain = os.environ.get("TOOLCHAINS", "")
+    if toolchain not in ("", "default", "XcodeDefault", "com.apple.dt.toolchain.XcodeDefault"):
+        # Preserve xcrun's explicit custom-toolchain selection semantics. Do
+        # not silently replace it with the default toolchain after a failure.
+        if not shutil.which("xcrun"):
+            raise ValueError("Required existing diagnostic tool unavailable: xcrun")
+        compiler = Path(run_step(["xcrun", "--find", "clang"], deadline, 10, True,
+                                 phase="apple-clang-path"))
+        sdk = Path(run_step(["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True,
+                            phase="macos-sdk-path"))
+    elif (directory / "Toolchains").is_dir():
+        # Use the existing selected installation directly. xcrun discovery
+        # itself timed out on hosted Intel before the recorder could start.
+        compiler = directory / "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
+        sdk = directory / "Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
+    else:
+        compiler = directory / "usr/bin/clang"
+        sdk = directory / "SDKs/MacOSX.sdk"
+    if not compiler.is_absolute() or not compiler.is_file() or not os.access(compiler, os.X_OK):
+        raise ValueError("Selected Apple Clang is unavailable or not executable: " + str(compiler))
+    if not sdk.is_absolute() or not sdk.is_dir():
+        raise ValueError("Selected macOS SDK is unavailable: " + str(sdk))
+    print(json.dumps({"intelGHCAppleTools": {"developer": str(directory),
+                      "clang": str(compiler), "macosSDK": str(sdk)}}), flush=True)
+    return compiler, sdk
+
+
 def command_status(arguments, seconds, **streams):
     # A fresh session gives this command and its descendants a group owned by
     # this launcher. macOS need not provide GNU timeout (or a Homebrew alias).
@@ -85,7 +130,7 @@ def run_step(arguments, deadline, seconds, capture=False, *, phase="command"):
 def run_command(arguments, seconds, capture):
     if capture:
         # Keep subprocess output out of memory until its size is known. These
-        # three metadata queries have tiny successful outputs; errors remain
+        # metadata queries have tiny successful outputs; errors remain
         # visible through a bounded stderr suffix.
         with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors:
             status = command_status(arguments, seconds, stdout=output, stderr=errors)
@@ -127,7 +172,7 @@ def main():
         if expected is None:
             print("Skipping Intel GHC diagnostic: no unique observed SimpleX SIGBUS component.", flush=True)
             return 0
-        missing = [tool for tool in ("xcrun", "git", "nix") if not shutil.which(tool)]
+        missing = [tool for tool in ("git", "nix") if not shutil.which(tool)]
         if missing:
             raise ValueError("Required existing diagnostic tools unavailable: " + ", ".join(missing))
         repo = args.repo.resolve()
@@ -135,12 +180,7 @@ def main():
                             phase="checkout-revision")
         if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
             raise ValueError("Invalid checkout revision")
-        compiler = run_step(["xcrun", "--find", "clang"], deadline, 10, True,
-                            phase="apple-clang-path")
-        sdk = run_step(["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True,
-                       phase="macos-sdk-path")
-        if not Path(compiler).is_absolute() or not Path(sdk).is_absolute():
-            raise ValueError("Expected absolute existing Apple tool paths")
+        compiler, sdk = apple_tool_paths(deadline)
         scripts = repo / "scripts/nix"
         with tempfile.TemporaryDirectory(prefix="simplex-intel-ghc-") as folder:
             tracer = Path(folder) / "ghc-sigbus-recorder.dylib"