fix(ci): split Intel native and mobile Nix jobs [skip ci]

scripts/nix/README.md

@@ -294,17 +294,26 @@ make nix-native NIX_ARGS=--dry-run
 ```
 
 `.github/workflows/nix.yml` runs these same outputs on Linux ARM64/x86_64 and
-Darwin ARM64/x86_64, with a six-hour limit per build job. Its seven build jobs cover all 21
+Darwin ARM64/x86_64, with a six-hour limit per build job. Its eight build jobs cover all 21
 outputs once: x86_64 Linux runs `native`, `android`, `windows`, and `linux-static`
-separately; ARM64 Linux runs `native`; both Darwin hosts run `all`. A failed
+separately; ARM64 Linux runs `native`; ARM64 Darwin runs `all`; Intel Darwin runs
+`native` and `darwin-libs` separately. On a cold Intel runner the combined job
+finished the native library and CLI only after about five and a half hours,
+then hit GitHub's six-hour cap while building mobile dependencies. The split
+gives each group its own existing six-hour budget and artifact upload; both use
+the same checkout, lock file, and `ghc914-sh`. Intel policy, info-table and
+failure diagnostic checks apply to both groups; CLI smoke runs in `native`. A failed
 native Linux build therefore leaves the cross-build jobs running. PRs, pushes,
 and default manual runs retain that complete matrix. A manual dispatch can
 select the existing three `linux-static` outputs with
 `gh workflow run nix.yml --ref BRANCH -f group=linux-static`, or the four Android
 outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`, or the two
 Windows outputs with `gh workflow run nix.yml --ref BRANCH -f group=windows`, or
-all four Intel Darwin outputs with
-`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin`. The registered
+all four Intel Darwin outputs in both jobs with
+`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin`. Select only the two
+Intel mobile library outputs with
+`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin-libs`; its distinct
+manual queue can run while an earlier full Intel trial remains active. The registered
 workflow can run on a branch containing this workflow file; it does not require
 that file on the default branch. Runs for the same workflow/ref queue without
 canceling an active run; manual groups have separate queues. Legacy PR build jobs also preserve running

scripts/nix/ci-matrix.py

@@ -10,17 +10,22 @@ ROWS = (
     ("x86_64-linux", "ubuntu-24.04", "linux-static"),
     ("aarch64-linux", "ubuntu-24.04-arm", "native"),
     ("aarch64-darwin", "macos-15", "all"),
-    ("x86_64-darwin", "macos-15-intel", "all"),
+    # Cold native compiler/application builds left too little of GitHub's
+    # six-hour job limit for the two mobile libraries. Keep their budgets separate.
+    ("x86_64-darwin", "macos-15-intel", "native"),
+    ("x86_64-darwin", "macos-15-intel", "darwin-libs"),
 )
 
 
 def select(group):
-    if group not in ("all", "linux-static", "android", "windows", "x86_64-darwin"):
+    if group not in ("all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"):
         raise ValueError(f"unknown Nix CI group: {group}")
     if group == "all":
         rows = ROWS
     elif group == "x86_64-darwin":
         rows = tuple(row for row in ROWS if row[0] == group)
+    elif group == "x86_64-darwin-libs":
+        rows = tuple(row for row in ROWS if row[0] == "x86_64-darwin" and row[2] == "darwin-libs")
     else:
         rows = tuple(row for row in ROWS if row[2] == group)
     return {"include": [dict(system=system, runner=runner, target=target)
@@ -29,5 +34,5 @@ def select(group):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--group", choices=["all", "linux-static", "android", "windows", "x86_64-darwin"], default="all")
+    parser.add_argument("--group", choices=["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"], default="all")
     print(json.dumps(select(parser.parse_args().group), separators=(",", ":")))

scripts/nix/test-ci-matrix.py

@@ -14,21 +14,47 @@ matrix = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(matrix)
 
 
+def listed_outputs(system, target):
+    env = dict(os.environ, NIX_SYSTEM=system, NIX_TARGET=target)
+    return subprocess.check_output(["make", "--no-print-directory", "nix-list"],
+                                   cwd=HERE.parent.parent, env=env, text=True).splitlines()
+
+
 def outputs(group):
-    result = []
-    for row in matrix.select(group)["include"]:
-        env = dict(os.environ, NIX_SYSTEM=row["system"], NIX_TARGET=row["target"])
-        result += subprocess.check_output(["make", "--no-print-directory", "nix-list"],
-                                          cwd=HERE.parent.parent, env=env, text=True).splitlines()
-    return result
+    return [output for row in matrix.select(group)["include"]
+            for output in listed_outputs(row["system"], row["target"])]
+
+
+def workflow_step(name):
+    source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text()
+    match = re.search(r"^      - name: " + re.escape(name) + r"\n(.*?)(?=^      - |\Z)",
+                      source, re.MULTILINE | re.DOTALL)
+    if match is None:
+        raise AssertionError(f"missing workflow step: {name}")
+    return match.group(1)
+
+
+def step_enabled(name, row, failed=False, build_outcome="success"):
+    match = re.search(r"^        if: \$\{\{ (.*) \}\}$", workflow_step(name), re.MULTILINE)
+    if match is None:
+        raise AssertionError(f"missing workflow condition: {name}")
+    expression = match.group(1).replace("matrix.system", "system").replace("matrix.target", "target")
+    expression = expression.replace("steps.haskell_build.outcome", "build_outcome")
+    expression = expression.replace("failure()", "failed").replace("&&", "and").replace("||", "or")
+    return bool(eval(expression, {"__builtins__": {}}, {
+        **row, "failed": failed, "build_outcome": build_outcome,
+    }))
 
 
 class Matrix(unittest.TestCase):
     def test_default_covers_all_21_outputs_once(self):
         selected = outputs("all")
-        self.assertEqual(len(matrix.select("all")["include"]), 7)
+        self.assertEqual(len(matrix.select("all")["include"]), 8)
         self.assertEqual(len(selected), 21)
         self.assertEqual(len(set(selected)), 21)
+        self.assertEqual(selected, [output for system in [
+            "x86_64-linux", "aarch64-linux", "aarch64-darwin", "x86_64-darwin",
+        ] for output in listed_outputs(system, "all")])
 
     def test_static_dispatch_is_exactly_the_existing_three_outputs(self):
         self.assertEqual(outputs("linux-static"), [
@@ -51,24 +77,99 @@ class Matrix(unittest.TestCase):
             "hydraJobs.x86_64-linux.x86_64-windows:lib:simplex-chat",
         ])
 
-    def test_intel_dispatch_retains_all_four_existing_outputs(self):
+    def test_intel_dispatch_splits_all_four_existing_outputs_in_order(self):
         selected = matrix.select("x86_64-darwin")
-        self.assertEqual(selected, {"include": [dict(
-            system="x86_64-darwin", runner="macos-15-intel", target="all")]})
-        self.assertEqual(outputs("x86_64-darwin"), [
+        self.assertEqual(selected, {"include": [
+            dict(system="x86_64-darwin", runner="macos-15-intel", target="native"),
+            dict(system="x86_64-darwin", runner="macos-15-intel", target="darwin-libs"),
+        ]})
+        native = [
             "hydraJobs.x86_64-darwin.lib:simplex-chat",
             "hydraJobs.x86_64-darwin.exe:simplex-chat",
+        ]
+        mobile = [
+            "hydraJobs.x86_64-darwin.x86_64-darwin-ios:lib:simplex-chat",
+            "hydraJobs.x86_64-darwin.x86_64-darwin:lib:simplex-chat",
+        ]
+        self.assertEqual(listed_outputs("x86_64-darwin", "native"), native)
+        self.assertEqual(listed_outputs("x86_64-darwin", "darwin-libs"), mobile)
+        self.assertEqual(outputs("x86_64-darwin"), native + mobile)
+        cli = subprocess.check_output([
+            "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin"], text=True)
+        self.assertEqual(json.loads(cli), selected)
+
+    def test_intel_mobile_dispatch_is_only_the_two_existing_library_outputs(self):
+        selected = matrix.select("x86_64-darwin-libs")
+        self.assertEqual(selected, {"include": [dict(
+            system="x86_64-darwin", runner="macos-15-intel", target="darwin-libs")]})
+        self.assertEqual(outputs("x86_64-darwin-libs"), [
             "hydraJobs.x86_64-darwin.x86_64-darwin-ios:lib:simplex-chat",
             "hydraJobs.x86_64-darwin.x86_64-darwin:lib:simplex-chat",
         ])
         cli = subprocess.check_output([
-            "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin"], text=True)
+            "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin-libs"], text=True)
         self.assertEqual(json.loads(cli), selected)
 
+    def test_workflow_manual_choices_match_the_selector(self):
+        source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text()
+        choices = re.search(r"^        options: \[(.*)\]$", source, re.MULTILINE)
+        self.assertIsNotNone(choices)
+        expected = ["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"]
+        self.assertEqual(choices.group(1).split(", "), expected)
+        for group in expected:
+            with self.subTest(group=group):
+                cli = subprocess.check_output([
+                    "python3", str(HERE / "ci-matrix.py"), "--group", group], text=True)
+                self.assertEqual(json.loads(cli), matrix.select(group))
+
+    def test_intel_policy_and_info_table_checks_cover_both_groups(self):
+        for row in matrix.select("all")["include"]:
+            for step in ["Check Intel Darwin build policies",
+                         "Check native Intel Darwin GHC info-table placement"]:
+                with self.subTest(row=row, step=step):
+                    self.assertEqual(step_enabled(step, row), row["system"] == "x86_64-darwin")
+        for target in ["native", "darwin-libs"]:
+            row = dict(system="x86_64-darwin", target=target)
+            self.assertTrue(step_enabled("Check Intel Darwin build policies", row))
+            self.assertTrue(step_enabled("Check native Intel Darwin GHC info-table placement", row))
+
+    def test_intel_setup_diagnostic_keeps_exact_failure_gate_in_both_groups(self):
+        for row in matrix.select("all")["include"]:
+            for failed in [False, True]:
+                for outcome in ["success", "failure", "cancelled", "skipped"]:
+                    with self.subTest(row=row, failed=failed, outcome=outcome):
+                        self.assertEqual(step_enabled("Capture Intel Darwin ghc-boot Setup failure", row,
+                                                      failed, outcome),
+                                         row["system"] == "x86_64-darwin" and failed and outcome == "failure")
+
+    def test_native_cli_smoke_matches_the_actual_output_group(self):
+        for row in matrix.select("all")["include"]:
+            native_cli = f'hydraJobs.{row["system"]}.exe:simplex-chat'
+            with self.subTest(row=row):
+                self.assertEqual(step_enabled("Check native CLI encryption and profile passwords", row),
+                                 native_cli in listed_outputs(row["system"], row["target"]))
+
+    def test_split_artifacts_and_existing_execution_bounds(self):
+        source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text()
+        template = "nix-${{ matrix.system }}-${{ matrix.target }}-${{ github.sha }}"
+        self.assertEqual(source.count("name: " + template), 1)
+        names = [template.replace("${{ matrix.system }}", row["system"])
+                 .replace("${{ matrix.target }}", row["target"])
+                 .replace("${{ github.sha }}", "same-source-sha")
+                 for row in matrix.select("all")["include"]]
+        self.assertEqual(len(set(names)), len(names))
+        self.assertRegex(source, r"(?m)^    timeout-minutes: 360$")
+        self.assertRegex(source, r"(?m)^      NIX_BUILD_TIMEOUT: 7200$")
+        self.assertRegex(source, r"(?m)^      fail-fast: false$")
+        self.assertIn("timeout-minutes: 1", workflow_step("Check Intel Darwin build policies"))
+        self.assertIn("timeout-minutes: 10", workflow_step("Check native Intel Darwin GHC info-table placement"))
+        self.assertIn("timeout-minutes: 120", workflow_step("Capture Intel Darwin ghc-boot Setup failure"))
+
     def test_manual_groups_have_distinct_queues_without_changing_pr_or_push_keys(self):
         source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text()
         groups = re.findall(r"^  group: (.*)$", source, re.MULTILINE)
         self.assertEqual(len(groups), 1)
+        self.assertRegex(source, r"(?m)^  cancel-in-progress: false$")
         prefix = "nix-${{ github.workflow }}-${{ github.ref }}"
         self.assertTrue(groups[0].startswith(prefix))
         suffix = groups[0][len(prefix):]
@@ -85,11 +186,11 @@ class Matrix(unittest.TestCase):
             return "nix-Nix Haskell builds-refs/heads/branch" + suffix_value
 
         for event in ["pull_request", "push"]:
-            for group in [None, "all", "linux-static", "android", "windows", "x86_64-darwin"]:
+            for group in [None, "all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"]:
                 self.assertEqual(queue_key(event, group), "nix-Nix Haskell builds-refs/heads/branch")
         keys = [queue_key("workflow_dispatch", group)
-                for group in ["all", "linux-static", "android", "windows", "x86_64-darwin"]]
-        self.assertEqual(len(set(keys)), 5)
+                for group in ["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"]]
+        self.assertEqual(len(set(keys)), 6)
         self.assertEqual(queue_key("workflow_dispatch", ""), keys[0])
         self.assertEqual(queue_key("workflow_dispatch", None), keys[0])