Capture Wine exceptions in failed Windows TH diagnostics

.github/workflows/nix.yml

@@ -8,7 +8,7 @@ on:
         type: choice
         required: true
         default: all
-        options: [all, linux-static, android]
+        options: [all, linux-static, android, windows]
   push:
     branches: [master, stable, build/stable-ghc-9.14]
     paths:

scripts/nix/README.md

@@ -219,7 +219,8 @@ native Linux build therefore leaves the cross-build jobs running. PRs, pushes,
 and default manual runs retain that complete matrix. A manual dispatch can
 select the existing three `linux-static` outputs with
 `gh workflow run nix.yml --ref BRANCH -f group=linux-static`, or the four Android
-outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`. The registered
+outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`, or the two
+Windows outputs with `gh workflow run nix.yml --ref BRANCH -f group=windows`. The registered
 workflow can run on a branch containing this workflow file; it does not require
 that file on the default branch. Runs for the same workflow/ref queue without
 canceling an active run; manual groups have separate queues. Legacy PR build jobs also preserve running
@@ -237,15 +238,20 @@ the normal 7,200-second per-derivation limit. It then builds that exact componen
 with a 300-second limit; success is reused by the full Windows build. A failure
 keeps the job red and skips the full outputs. CI replays only that failed
 component in a separate diagnostic derivation, with `ISERV_ARGS=-v` and
-`strace -f -tt -T -s 80` around compiler calls. Compiler queries, original GHC
-arguments, the Wine/DLL wrapper and its normal/profile selection stay unchanged.
+`strace -f -tt -T -s 80` around compiler calls. It copies the exact selected Wine
+scripts and appends `+seh,+loaddll` to their pinned `WINEDEBUG` value, with an
+assertion that no other script bytes change. Only compile calls receive the
+copied `-pgmi` path. Compiler queries, source/plan, original arguments, DLL
+collection, Wine prefix and normal/profile selection stay unchanged.
 The trace has a 15-minute step bound and a 300-second per-derivation limit;
 strace is fetched from caches only. Logs and the selected derivation/input
 metadata are uploaded even on failure. Repeat the canonical gate on Linux with
 `python3 scripts/nix/windows-th-gate.py prepare` followed by
 `python3 scripts/nix/windows-th-gate.py build`; use `trace` only after a failed
 canonical attempt. These commands use the committed checkout and preserve the
-canonical Nix core setting. The diagnostic changes its derivation and process
+canonical Nix core setting. A fresh Wine prefix and full syscall tracing can
+consume most of the diagnostic budget; a timeout does not identify a deadlock.
+The diagnostic changes its derivation and process
 environment; it does not qualify a Windows artifact or change the failed result.
 
 After a failed native x86_64 Linux library build, CI tries a separate derivation

scripts/nix/ci-matrix.py

@@ -15,7 +15,7 @@ ROWS = (
 
 
 def select(group):
-    if group not in ("all", "linux-static", "android"):
+    if group not in ("all", "linux-static", "android", "windows"):
         raise ValueError(f"unknown Nix CI group: {group}")
     rows = ROWS if group == "all" else tuple(row for row in ROWS if row[2] == group)
     return {"include": [dict(system=system, runner=runner, target=target)
@@ -24,5 +24,5 @@ def select(group):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--group", choices=["all", "linux-static", "android"], default="all")
+    parser.add_argument("--group", choices=["all", "linux-static", "android", "windows"], default="all")
     print(json.dumps(select(parser.parse_args().group), separators=(",", ":")))

scripts/nix/test-ci-matrix.py

@@ -44,6 +44,12 @@ class Matrix(unittest.TestCase):
             "hydraJobs.x86_64-linux.armv7a-android:lib:support",
         ])
 
+    def test_windows_dispatch_is_exactly_the_existing_two_outputs(self):
+        self.assertEqual(outputs("windows"), [
+            "hydraJobs.x86_64-linux.x86_64-windows:exe:simplex-chat",
+            "hydraJobs.x86_64-linux.x86_64-windows:lib:simplex-chat",
+        ])
+
     def test_manual_groups_have_distinct_queues_without_changing_pr_or_push_keys(self):
         source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text()
         groups = re.findall(r"^  group: (.*)$", source, re.MULTILINE)
@@ -64,10 +70,10 @@ class Matrix(unittest.TestCase):
             return "nix-Nix Haskell builds-refs/heads/branch" + suffix_value
 
         for event in ["pull_request", "push"]:
-            for group in [None, "all", "linux-static", "android"]:
+            for group in [None, "all", "linux-static", "android", "windows"]:
                 self.assertEqual(queue_key(event, group), "nix-Nix Haskell builds-refs/heads/branch")
-        keys = [queue_key("workflow_dispatch", group) for group in ["all", "linux-static", "android"]]
-        self.assertEqual(len(set(keys)), 3)
+        keys = [queue_key("workflow_dispatch", group) for group in ["all", "linux-static", "android", "windows"]]
+        self.assertEqual(len(set(keys)), 4)
         self.assertEqual(queue_key("workflow_dispatch", ""), keys[0])
         self.assertEqual(queue_key("workflow_dispatch", None), keys[0])
 
@@ -99,8 +105,8 @@ class Matrix(unittest.TestCase):
 
     def test_unknown_group_rejected(self):
         with self.assertRaises(ValueError):
-            matrix.select("windows")
-        result = subprocess.run(["python3", str(HERE / "ci-matrix.py"), "--group", "windows"],
+            matrix.select("unknown")
+        result = subprocess.run(["python3", str(HERE / "ci-matrix.py"), "--group", "unknown"],
                                 capture_output=True, text=True)
         self.assertNotEqual(result.returncode, 0)
         self.assertEqual(result.stdout, "")

scripts/nix/test-windows-th-gate.py

@@ -16,6 +16,18 @@ spec.loader.exec_module(gate)
 SELECTED = "/nix/store/" + "0" * 32 + "-" + gate.COMPONENT + ".drv"
 DEP = "/nix/store/" + "1" * 32 + "-dependency.drv"
 NEEDLE = '    printf \'#!/bin/sh\\nexec %s "$@"\\n\' "$g" > "$wrapBin/$gName"'
+WINEDEBUG = "WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag"
+
+
+def store_iserv(temp, *, normal=None, profiled=None):
+    """Import tiny script inputs, not a derivation or a compiler."""
+    source = Path(temp) / "iserv-wrapper"
+    (source / "bin").mkdir(parents=True)
+    for suffix, content in [("", normal), ("-prof", profiled)]:
+        path = source / ("bin/iserv-wrapper" + suffix)
+        path.write_text(content if content is not None else "#!/bin/sh\n" + WINEDEBUG + " true \"$@\"\n")
+        path.chmod(0o755)
+    return subprocess.check_output(["nix-store", "--add", str(source)], text=True).strip() + "/bin/iserv-wrapper"
 
 
 def document(modern=True):
@@ -135,6 +147,7 @@ class Wrapper(unittest.TestCase):
             temp = Path(temp)
             (temp / "trace/bin").mkdir(parents=True)
             (temp / "wrappers").mkdir()
+            selected_iserv = store_iserv(temp)
             fake = '''#!/usr/bin/env python3
 import json, os, sys
 kind = os.path.basename(sys.argv[0])
@@ -147,7 +160,7 @@ sys.exit(int(os.environ.get("GHC_EXIT", "0")))
 '''
             for name in ["ghc", "timeout", "trace/bin/strace"]:
                 path = temp / name
-                path.write_text(fake)
+                path.write_text(fake.replace("import json, os, sys", f"# -pgmi {selected_iserv}\nimport json, os, sys"))
                 path.chmod(0o755)
             phase = f'g={temp}/ghc\nwrapBin={temp}/wrappers\ngName=ghc\n' + NEEDLE + '\nchmod +x "$wrapBin/$gName"\nexit 0\n'
             rendered = self.render(phase, temp)
@@ -171,12 +184,78 @@ sys.exit(int(os.environ.get("GHC_EXIT", "0")))
                     self.assertEqual(actual.returncode, status)
                     records = [json.loads(line) for line in capture.read_text().splitlines()]
                     self.assertEqual([r["kind"] for r in records], ["timeout", "strace", "ghc"] if compile_call else ["ghc"])
-                    self.assertEqual(records[-1]["args"], args)
+                    ghc_args = records[-1]["args"]
+                    if compile_call:
+                        self.assertEqual(ghc_args[:-2], args)
+                        self.assertEqual(ghc_args[-2], "-pgmi")
+                        selected_debug = Path(ghc_args[-1])
+                        self.assertEqual(selected_debug.name, "iserv-wrapper")
+                        for suffix in ["", "-prof"]:
+                            copied = Path(str(selected_debug) + suffix).read_text()
+                            original = Path(selected_iserv + suffix).read_text()
+                            self.assertEqual(copied.replace(WINEDEBUG + ",+seh,+loaddll", WINEDEBUG), original)
+                    else:
+                        self.assertEqual(ghc_args, args)
                     self.assertEqual(records[-1]["iserv"], "-v" if compile_call else "ambient")
                     self.assertEqual(actual.stdout, "query-control\n")
                     if compile_call:
                         self.assertEqual(records[1]["args"][:5], ["-f", "-tt", "-T", "-s", "80"])
 
 
+class WineDebug(unittest.TestCase):
+    def run_copy(self, ghc, destination):
+        return subprocess.run(["sh", str(HERE / "windows-th-wine-debug.sh"), str(ghc), str(destination)],
+                              capture_output=True, text=True)
+
+    def test_copies_only_debug_value_for_both_interpreters(self):
+        with tempfile.TemporaryDirectory() as temp:
+            temp = Path(temp)
+            script = """#!/bin/sh
+unset configureFlags
+REMOTE_ISERV=unchanged
+WINEDLLOVERRIDES=unchanged WINEDEBUG_VALUE WINEPREFIX="$TMP" python3 -c 'import json,os,sys; print(json.dumps([os.environ["WINEDEBUG"],os.environ["WINEPREFIX"],sys.argv[1:]]))' "$@"
+""".replace("WINEDEBUG_VALUE", WINEDEBUG)
+            original = store_iserv(temp, normal=script, profiled=script.replace("unchanged", "profiled"))
+            ghc = temp / "ghc"
+            ghc.write_text("#!/bin/sh\n# -pgmi " + original + "\n")
+            copied = self.run_copy(ghc, temp / "debug")
+            self.assertEqual(copied.returncode, 0, copied.stderr)
+            self.assertEqual(copied.stdout.strip(), str(temp / "debug/iserv-wrapper"))
+            for suffix in ["", "-prof"]:
+                target = Path(copied.stdout.strip() + suffix)
+                subprocess.run(["sh", "-n", str(target)], check=True)
+                self.assertEqual(target.read_text().replace(WINEDEBUG + ",+seh,+loaddll", WINEDEBUG),
+                                 Path(original + suffix).read_text())
+                env = dict(os.environ, TMP="same-prefix")
+                result = subprocess.run([str(target), "a b", "quote'and\"", "-v"], env=env,
+                                        capture_output=True, text=True, check=True)
+                debug, prefix, args = json.loads(result.stdout)
+                self.assertEqual(debug, WINEDEBUG.removeprefix("WINEDEBUG=") + ",+seh,+loaddll")
+                self.assertEqual(prefix, "same-prefix")
+                self.assertEqual(args, ["a b", "quote'and\"", "-v"])
+
+    def test_rejects_changed_or_ambiguous_originals(self):
+        for case in ["missing-path", "two-paths", "missing-assignment", "two-assignments", "missing-profile"]:
+            with self.subTest(case=case), tempfile.TemporaryDirectory() as temp:
+                temp = Path(temp)
+                script = "#!/bin/sh\n" + WINEDEBUG + " true\n"
+                if case == "missing-assignment": script = "#!/bin/sh\ntrue\n"
+                if case == "two-assignments": script += WINEDEBUG + " true\n"
+                selected = store_iserv(temp, normal=script)
+                if case == "missing-profile":
+                    # A valid store path without the paired script must fail,
+                    # without changing any existing store object.
+                    source = temp / "unpaired/iserv-wrapper/bin"
+                    source.mkdir(parents=True)
+                    path = source / "iserv-wrapper"
+                    path.write_text(script)
+                    path.chmod(0o755)
+                    selected = subprocess.check_output(["nix-store", "--add", str(source.parent)], text=True).strip() + "/bin/iserv-wrapper"
+                ghc = temp / "ghc"
+                ghc.write_text("none" if case == "missing-path" else selected + ("\n" + selected if case == "two-paths" else ""))
+                result = self.run_copy(ghc, temp / "debug")
+                self.assertEqual(result.returncode, 1, result.stderr)
+
+
 if __name__ == "__main__":
     unittest.main()

scripts/nix/trace-windows-th.nix

@@ -1,5 +1,6 @@
 # Failure-only replay of the exact canonical Windows TH component. Source,
-# plan, compiler arguments, Wine wrapper and normal/profile selection stay put.
+# plan and normal/profile selection stay put. A copied Wine wrapper changes
+# only its debug-channel value; the canonical component stays untouched.
 { original, strace, expectedDrv }:
 let
   needle = "    printf '#!/bin/sh\\nexec %s \"$@\"\\n' \"$g\" > \"$wrapBin/$gName\"";
@@ -21,8 +22,10 @@ let
       # PROXY_ARGS change: their positions differ from this protocol's --pipe.
       export ISERV_ARGS=-v
       : "''${NIX_BUILD_TOP:?Windows TH diagnostic requires NIX_BUILD_TOP}"
+      debugDir=$(mktemp -d "$NIX_BUILD_TOP/windows-th-wine-debug.XXXXXX") || exit 1
+      debugPgmi=$(sh ${./windows-th-wine-debug.sh} "$realGhc" "$debugDir") || exit 1
       touch "$NIX_BUILD_TOP/windows-th-trace-attempted"
-      exec timeout --kill-after=30s 300s ${strace}/bin/strace -f -tt -T -s 80 "$realGhc" "$@"
+      exec timeout --kill-after=30s 300s ${strace}/bin/strace -f -tt -T -s 80 "$realGhc" "$@" -pgmi "$debugPgmi"
     fi
     exec "$realGhc" "$@"
     SIMPLEX_WINDOWS_TH_TRACE

scripts/nix/windows-th-wine-debug.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# Copy the selected Wine iserv scripts, changing only diagnostic channels.
+set -eu
+
+real_ghc=$1
+destination=$2
+needle='WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag'
+# Wine parses these left to right. Explicit channel flags follow the global
+# class exclusions, so exception and module traces remain enabled.
+replacement="$needle,+seh,+loaddll"
+
+paths=$(grep -oE '/nix/store/[0-9abcdfghijklmnpqrsvwxyz]{32}-iserv-wrapper/bin/iserv-wrapper' "$real_ghc" || true)
+count=$(printf '%s\n' "$paths" | grep -c . || true)
+if [ "$count" -ne 1 ]; then
+  echo 'ERROR: expected one original Wine iserv path in the selected GHC wrapper' >&2
+  exit 1
+fi
+original=$paths
+mkdir -p "$destination"
+for suffix in '' '-prof'; do
+  source="$original$suffix"
+  target="$destination/iserv-wrapper$suffix"
+  if [ ! -x "$source" ]; then
+    echo "ERROR: missing selected Wine iserv script: $source" >&2
+    exit 1
+  fi
+  count=$(grep -oF "$needle" "$source" | wc -l)
+  if [ "$count" -ne 1 ]; then
+    echo "ERROR: expected one pinned WINEDEBUG assignment in $source" >&2
+    exit 1
+  fi
+  sed "s/$needle/$replacement/g" "$source" > "$target"
+  # Refuse any transformation beyond the one debug value.
+  if ! sed "s/$replacement/$needle/g" "$target" | cmp -s "$source" -; then
+    echo "ERROR: unexpected change to the selected Wine iserv script: $source" >&2
+    exit 1
+  fi
+  chmod 700 "$target"
+done
+printf 'Windows TH diagnostic: %s; WINEDEBUG=%s\n' "$original" "${replacement#WINEDEBUG=}" >&2
+printf '%s\n' "$destination/iserv-wrapper"