From acc42acf53fd467bf45824d3d02e8486ac8c6df8 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sun, 18 Oct 2026 15:14:10 +0900 Subject: [PATCH] fix(nix): resolve selected Apple diagnostic tools directly [skip ci] --- scripts/nix/README.md | 11 +++ scripts/nix/test-select-intel-setup.py | 22 ++++++ scripts/nix/test-trace-intel-ghc.py | 100 +++++++++++++++++++++++-- scripts/nix/trace-intel-ghc.py | 56 ++++++++++++-- 4 files changed, 176 insertions(+), 13 deletions(-) diff --git a/scripts/nix/README.md b/scripts/nix/README.md index ad9d5fa8f7..bc06835c32 100644 --- a/scripts/nix/README.md +++ b/scripts/nix/README.md @@ -405,6 +405,17 @@ deadline and kills only that group on expiry, with at most five seconds to reap the command; GNU `timeout` and Homebrew aliases are not required. Missing Apple or Nix tools are named explicitly. +For the default Apple toolchain, the launcher reads `DEVELOPER_DIR` or makes +one bounded `xcode-select --print-path` query, then validates the existing +Clang executable and macOS SDK in that selected Xcode or Command Line Tools +installation. This avoids `xcrun --find clang`, which reached its ten-second +limit on hosted Intel before the recorder could start. An invalid selection +does not fall back to another installation. Explicit custom `TOOLCHAINS` +selection retains the original bounded `xcrun` queries. The Intel policy tests +also check the real selected Apple paths and Clang version on hosted Darwin +before the production build starts; this metadata check has a twenty-second +overall limit and does not compile or run GHC. + The recorder loads only into the selected normal SimpleX iOS compiler process. It reports its main executable's UUID, image base and slide, then records SIGBUS fault address, signal code and registers, including PC and SP, if the process diff --git a/scripts/nix/test-select-intel-setup.py b/scripts/nix/test-select-intel-setup.py index 261b10605a..051f302a48 100644 --- a/scripts/nix/test-select-intel-setup.py +++ b/scripts/nix/test-select-intel-setup.py @@ -1,5 +1,8 @@ import importlib.util +import os import pathlib +import sys +import time import unittest @@ -45,5 +48,24 @@ class FailedComponentSelection(unittest.TestCase): ) +class AppleToolDiscovery(unittest.TestCase): + @unittest.skipUnless(sys.platform == "darwin" and os.environ.get("GITHUB_ACTIONS") == "true", + "real Apple metadata check runs on hosted Darwin CI") + def test_selected_tools_before_production_build(self): + # This suite already runs in the Intel policy step before the long + # build. Exercise the actual resolver early, without invoking GHC. + spec = importlib.util.spec_from_file_location( + "intel_trace", pathlib.Path(__file__).with_name("trace-intel-ghc.py") + ) + trace = importlib.util.module_from_spec(spec) + spec.loader.exec_module(trace) + deadline = time.monotonic() + 20 + compiler, sdk = trace.apple_tool_paths(deadline) + self.assertTrue(compiler.is_file()) + self.assertTrue(sdk.is_dir()) + version = trace.run_step([compiler, "--version"], deadline, 10, True, phase="check-apple-clang") + self.assertIn("Apple clang", version) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/nix/test-trace-intel-ghc.py b/scripts/nix/test-trace-intel-ghc.py index 67b332ac9e..9e8144d600 100644 --- a/scripts/nix/test-trace-intel-ghc.py +++ b/scripts/nix/test-trace-intel-ghc.py @@ -26,6 +26,18 @@ FAILURE = ("Failed to build simplex-chat-7.1.0.11-unit. The build process termin class LauncherTests(unittest.TestCase): + def apple_installation(self, root, *, command_line=False): + developer = Path(root) / ("CommandLineTools" if command_line else "Xcode.app/Contents/Developer") + compiler = developer / ("usr/bin/clang" if command_line + else "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang") + sdk = developer / ("SDKs/MacOSX.sdk" if command_line + else "Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + compiler.parent.mkdir(parents=True, exist_ok=True) + compiler.write_text("fixture Apple tool; never executed\n") + compiler.chmod(0o755) + sdk.mkdir(parents=True, exist_ok=True) + return developer, compiler, sdk + def test_unique_observed_failure(self): self.assertEqual(TRACE.select_failure(FAILURE + FAILURE), DRV) self.assertIsNone(TRACE.select_failure(FAILURE.replace("exit code -10", "exit code 1"))) @@ -34,7 +46,8 @@ class LauncherTests(unittest.TestCase): TRACE.select_failure(FAILURE + FAILURE.replace("a" * 32, "b" * 32)) def invoke(self, folder, *, actions="true", machine="x86_64", target=None, failure=FAILURE, - reject_build=False, reject_cache=False): + reject_build=False, reject_cache=False, reject_xcrun=False): + developer, compiler, sdk = self.apple_installation(folder) log = Path(folder) / "nix-build.log" log.write_text(failure) calls = [] @@ -43,15 +56,18 @@ class LauncherTests(unittest.TestCase): calls.append((values, seconds, capture)) if values[0] == "git": return "c" * 40 + if values[0] == "xcrun" and reject_xcrun: + raise TimeoutError("Diagnostic command reached its wall-clock limit") if values[:3] == ["xcrun", "--find", "clang"]: - return "/usr/bin/clang" + return str(compiler) if values[0] == "xcrun": - return "/existing/SDK" + return str(sdk) if (reject_build and "--file" in values) or (reject_cache and "--expr" in values): raise RuntimeError("Diagnostic command returned 1") return None argv = ["trace-intel-ghc.py", "--repo", folder, "--attribute", target or TRACE.ATTRIBUTE, "--log", str(log)] - with patch.dict(os.environ, {"GITHUB_ACTIONS": actions}), patch.object(TRACE.platform, "system", return_value="Darwin"), \ + with patch.dict(os.environ, {"GITHUB_ACTIONS": actions, "DEVELOPER_DIR": str(developer), "TOOLCHAINS": ""}), \ + patch.object(TRACE.platform, "system", return_value="Darwin"), \ patch.object(TRACE.platform, "machine", return_value=machine), \ patch.object(TRACE.shutil, "which", side_effect=lambda tool: None if tool in ("timeout", "gtimeout") else "/existing/tool"), \ patch.object(TRACE, "run_step", side_effect=step), patch("sys.argv", argv), \ @@ -88,7 +104,81 @@ class LauncherTests(unittest.TestCase): original = arguments[arguments.index("original") + 1] self.assertIn('f.hydraJobs.x86_64-darwin."x86_64-darwin-ios:lib:simplex-chat"', original) self.assertNotIn(".project.hsPkgs", original) - self.assertEqual(sum(call[0][:1] == ["/usr/bin/clang"] for call in calls), 1) + compiler = str(Path(folder) / "Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang") + self.assertEqual(sum(call[0][:1] == [compiler] for call in calls), 1) + + def test_observed_xcrun_timeout_does_not_block_selected_tools(self): + with tempfile.TemporaryDirectory() as folder: + result, calls = self.invoke(folder, reject_xcrun=True) + self.assertEqual(result, 0) + self.assertEqual(sum("--file" in call[0] for call in calls), 1) + self.assertFalse(any(call[0][0] == "xcrun" for call in calls)) + + def test_selected_xcode_and_command_line_tools_need_no_discovery_process(self): + with tempfile.TemporaryDirectory() as folder: + for command_line in (False, True): + developer, compiler, sdk = self.apple_installation(folder, command_line=command_line) + with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": "", "SDKROOT": "/unused/iPhoneOS.sdk"}), \ + patch.object(TRACE, "run_step") as step, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(TRACE.apple_tool_paths(time.monotonic() + 20), (compiler, sdk)) + step.assert_not_called() + + def test_xcode_application_selection_is_supported(self): + with tempfile.TemporaryDirectory() as folder: + developer, compiler, sdk = self.apple_installation(folder) + with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer.parent.parent), "TOOLCHAINS": ""}), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(TRACE.apple_tool_paths(time.monotonic() + 20), (compiler, sdk)) + + def test_system_developer_selection_is_bounded(self): + with tempfile.TemporaryDirectory() as folder: + developer, compiler, sdk = self.apple_installation(folder) + deadline = time.monotonic() + 20 + with patch.dict(os.environ, {}, clear=True), \ + patch.object(TRACE.shutil, "which", return_value="/usr/bin/xcode-select"), \ + patch.object(TRACE, "run_step", return_value=str(developer)) as step, \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(TRACE.apple_tool_paths(deadline), (compiler, sdk)) + step.assert_called_once_with(["xcode-select", "--print-path"], deadline, 10, True, + phase="apple-developer-path") + + def test_invalid_developer_selection_never_uses_another_installation(self): + with tempfile.TemporaryDirectory() as folder: + for selection in ("", "relative", "bad\npath", str(Path(folder) / "missing")): + with self.subTest(selection=selection), \ + patch.dict(os.environ, {"DEVELOPER_DIR": selection, "TOOLCHAINS": ""}), \ + patch.object(TRACE, "run_step") as step: + with self.assertRaisesRegex(ValueError, "developer directory"): + TRACE.apple_tool_paths(time.monotonic() + 20) + step.assert_not_called() + + def test_missing_selected_clang_or_sdk_is_named(self): + with tempfile.TemporaryDirectory() as folder: + developer, compiler, sdk = self.apple_installation(folder) + with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": ""}): + compiler.chmod(0o644) + with self.assertRaisesRegex(ValueError, "Clang.*not executable"): + TRACE.apple_tool_paths(time.monotonic() + 20) + compiler.chmod(0o755) + sdk.rmdir() + with self.assertRaisesRegex(ValueError, "macOS SDK is unavailable"): + TRACE.apple_tool_paths(time.monotonic() + 20) + + def test_custom_toolchain_uses_original_bounded_discovery(self): + with tempfile.TemporaryDirectory() as folder: + developer, compiler, sdk = self.apple_installation(folder) + custom = Path(folder) / "Selected.xctoolchain/usr/bin/clang" + custom.parent.mkdir(parents=True) + custom.write_text("fixture custom tool; never executed\n") + custom.chmod(0o755) + deadline = time.monotonic() + 30 + with patch.dict(os.environ, {"DEVELOPER_DIR": str(developer), "TOOLCHAINS": "selected.custom.toolchain"}), \ + patch.object(TRACE.shutil, "which", return_value="/usr/bin/xcrun"), \ + patch.object(TRACE, "run_step", side_effect=[str(custom), str(sdk)]) as step, \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(TRACE.apple_tool_paths(deadline), (custom, sdk)) + self.assertEqual(step.call_args_list[0].args, (["xcrun", "--find", "clang"], deadline, 10, True)) + self.assertEqual(step.call_args_list[1].args, (["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True)) def test_failed_diagnostic_returns_without_replay_loop(self): with tempfile.TemporaryDirectory() as folder: diff --git a/scripts/nix/trace-intel-ghc.py b/scripts/nix/trace-intel-ghc.py index d71c541383..e1f4101695 100644 --- a/scripts/nix/trace-intel-ghc.py +++ b/scripts/nix/trace-intel-ghc.py @@ -42,6 +42,51 @@ def python_expression(flake): + ' { haskellNix = f.inputs.haskellNix; } "x86_64-darwin").python3') +def apple_tool_paths(deadline): + developer = os.environ.get("DEVELOPER_DIR") + if developer is None: + if not shutil.which("xcode-select"): + raise ValueError("Required existing diagnostic tool unavailable: xcode-select") + developer = run_step(["xcode-select", "--print-path"], deadline, 10, True, + phase="apple-developer-path") + if not developer or any(char in developer for char in "\r\n\0"): + raise ValueError("Invalid selected Apple developer directory") + directory = Path(developer) + if not directory.is_absolute(): + raise ValueError("Expected an absolute selected Apple developer directory") + # xcode-select accepts either the application or its Developer directory. + if directory.suffix == ".app": + directory = directory / "Contents/Developer" + if not directory.is_dir(): + raise ValueError("Selected Apple developer directory is unavailable: " + str(directory)) + + toolchain = os.environ.get("TOOLCHAINS", "") + if toolchain not in ("", "default", "XcodeDefault", "com.apple.dt.toolchain.XcodeDefault"): + # Preserve xcrun's explicit custom-toolchain selection semantics. Do + # not silently replace it with the default toolchain after a failure. + if not shutil.which("xcrun"): + raise ValueError("Required existing diagnostic tool unavailable: xcrun") + compiler = Path(run_step(["xcrun", "--find", "clang"], deadline, 10, True, + phase="apple-clang-path")) + sdk = Path(run_step(["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True, + phase="macos-sdk-path")) + elif (directory / "Toolchains").is_dir(): + # Use the existing selected installation directly. xcrun discovery + # itself timed out on hosted Intel before the recorder could start. + compiler = directory / "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" + sdk = directory / "Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" + else: + compiler = directory / "usr/bin/clang" + sdk = directory / "SDKs/MacOSX.sdk" + if not compiler.is_absolute() or not compiler.is_file() or not os.access(compiler, os.X_OK): + raise ValueError("Selected Apple Clang is unavailable or not executable: " + str(compiler)) + if not sdk.is_absolute() or not sdk.is_dir(): + raise ValueError("Selected macOS SDK is unavailable: " + str(sdk)) + print(json.dumps({"intelGHCAppleTools": {"developer": str(directory), + "clang": str(compiler), "macosSDK": str(sdk)}}), flush=True) + return compiler, sdk + + def command_status(arguments, seconds, **streams): # A fresh session gives this command and its descendants a group owned by # this launcher. macOS need not provide GNU timeout (or a Homebrew alias). @@ -85,7 +130,7 @@ def run_step(arguments, deadline, seconds, capture=False, *, phase="command"): def run_command(arguments, seconds, capture): if capture: # Keep subprocess output out of memory until its size is known. These - # three metadata queries have tiny successful outputs; errors remain + # metadata queries have tiny successful outputs; errors remain # visible through a bounded stderr suffix. with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: status = command_status(arguments, seconds, stdout=output, stderr=errors) @@ -127,7 +172,7 @@ def main(): if expected is None: print("Skipping Intel GHC diagnostic: no unique observed SimpleX SIGBUS component.", flush=True) return 0 - missing = [tool for tool in ("xcrun", "git", "nix") if not shutil.which(tool)] + missing = [tool for tool in ("git", "nix") if not shutil.which(tool)] if missing: raise ValueError("Required existing diagnostic tools unavailable: " + ", ".join(missing)) repo = args.repo.resolve() @@ -135,12 +180,7 @@ def main(): phase="checkout-revision") if re.fullmatch(r"[0-9a-f]{40}", revision) is None: raise ValueError("Invalid checkout revision") - compiler = run_step(["xcrun", "--find", "clang"], deadline, 10, True, - phase="apple-clang-path") - sdk = run_step(["xcrun", "--sdk", "macosx", "--show-sdk-path"], deadline, 10, True, - phase="macos-sdk-path") - if not Path(compiler).is_absolute() or not Path(sdk).is_absolute(): - raise ValueError("Expected absolute existing Apple tool paths") + compiler, sdk = apple_tool_paths(deadline) scripts = repo / "scripts/nix" with tempfile.TemporaryDirectory(prefix="simplex-intel-ghc-") as folder: tracer = Path(folder) / "ghc-sigbus-recorder.dylib"