diff --git a/flake.lock b/flake.lock index 699f5b4a84..d56c358397 100644 --- a/flake.lock +++ b/flake.lock @@ -250,17 +250,17 @@ "stackage": "stackage" }, "locked": { - "lastModified": 1791711104, - "narHash": "sha256-DxVmDfED9S2V2eg9W6xn4EetcI8dt8KnTP/QGExVr/c=", + "lastModified": 1792223581, + "narHash": "sha256-HRGkrnfD6Fn2DyAw2yOv5kpyC7uaBkxNo9Apw1EesVo=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "54c49b7badca29e8b746fe8681d9bd70086e308a", + "rev": "fd96dff55bcf6f81ccb96872f16a2ec3970edab0", "type": "github" }, "original": { "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "54c49b7badca29e8b746fe8681d9bd70086e308a", + "rev": "fd96dff55bcf6f81ccb96872f16a2ec3970edab0", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 2c3f6eb01c..8ac983a38b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "nix flake for simplex-chat"; # hkm/stable-haskell: use the same compiler and boot libraries on every target. - inputs.haskellNix.url = "github:input-output-hk/haskell.nix/54c49b7badca29e8b746fe8681d9bd70086e308a"; + inputs.haskellNix.url = "github:input-output-hk/haskell.nix/fd96dff55bcf6f81ccb96872f16a2ec3970edab0"; inputs.nixpkgs.follows = "haskellNix/nixpkgs"; inputs.mac2ios.url = "github:zw3rk/mobile-core-tools"; inputs.hackage = { @@ -60,8 +60,8 @@ else pkgs'; in projectPkgs.haskell-nix.project ({ config, ... }: let - # v2 restores Hackage description overrides after package patches. - # Plan Windows x509 from its patched local source to retain crypt32. + # Keep the patched Windows x509 metadata in the original plan too; + # post-plan Cabal edits can change native package unit IDs. androidPthreads = import ./scripts/nix/android-pthread-libraries.nix { pkgs = config.evalPackages; hostPlatform = pkgs'.stdenv.hostPlatform; @@ -683,6 +683,7 @@ ghc-elf-comdat-functions = import ./scripts/nix/check-elf-comdat-functions.nix { inherit pkgs; compiler = (compilerSelection pkgs).ghc914-sh; + patch = haskellNix + "/overlays/patches/ghc914-rts-elf-comdat-functions.patch"; }; ghc-elf-alignment = import ./scripts/nix/check-elf-alignment.nix { inherit pkgs; diff --git a/scripts/nix/README.md b/scripts/nix/README.md index ba7121113c..7a6f816460 100644 --- a/scripts/nix/README.md +++ b/scripts/nix/README.md @@ -4,6 +4,9 @@ The compiler and all cross-compilers come from the haskell.nix input pinned in `flake.lock`. The `ghc914-sh` compiler is the stable-haskell fork, not the stock GHC 9.14 release from GHCup. Use the same source checkout and lock file for every platform; Android ARMv7 does not require a separate branch or tag. +The pinned haskell.nix includes the merged ARM32 personality, v2 Cabal metadata, +Windows patch-selector, and i386 COMDAT fixes. The raw GHC source pin is unchanged; +changes to v2 build scripts can still change compiler and component derivations. `compiler-selection.nix` and `compiler-cache-overlay.nix` match Hydra's compiler build context, including auxiliary iserv projects, so the pinned compiler can use @@ -89,6 +92,8 @@ does not persist weak-symbol status or suppress ordinary strong conflicts. Multi-member groups remain outside this patch's scope. The source patch applies to application and interpreter projects, including profiling. Native compilers and other cross targets keep their existing RTS sources. +This production patch now comes from haskell.nix's project module. The scope and +native runtime regressions consume that same pinned upstream module and patch. Run the native x86_64 Linux regression with `nix build .#checks.x86_64-linux.ghc-elf-comdat-functions --no-link -L --max-jobs 1 --timeout 7200`. @@ -105,8 +110,8 @@ The generated constant converts through Haskell `Int` with `Prelude.fromIntegral Basement's `RebindableSyntax` has no `HasNegation IntPtr` instance. Windows also plans the already selected `crypton-x509-system-1.6.8` as a local patched source, so its Cabal metadata names `crypt32` before source packaging. -The pinned v2 builder otherwise replaces a package's patched Cabal file with its -original description override and copies that override into the slice index. +This keeps planning and packaged Cabal metadata aligned, including the metadata +used for native package unit IDs. The source producer applies the existing haskell.nix patch once; its later package patch list is empty to prevent a second application. @@ -212,6 +217,9 @@ still go through the host, and queries and setters retain the previous guest value. This QEMU patch applies only to the Android ARM32 emulator; the other emulator targets stay unchanged. Android's libc initializes this state again at startup after an exec. +The shared helper passes the actual Android guest platform to the pinned +haskell.nix selector, which applies the personality patch once. The explicit +local Linux scheduling opt-out remains restricted to supported ARM32 emulators. A standalone C regression checks startup, full personality values, the host's `UNAME26` effect, and independent thread/fork state. Run it from this checkout @@ -376,13 +384,14 @@ establish an out-of-memory cause. Hydra needs builders advertising `aarch64-darwin` and `x86_64-darwin` to run the corresponding Mac/iOS jobs. Check its [builder pool](https://ci.zw3rk.com/machines) -when jobs remain queued. For cross builds, the pinned haskell.nix uses -`recursive-nix` as a scheduling marker; these builds do not need recursive Nix -operations. This checkout opts `ghc914-sh` Android ARM32 out of that restriction -on Linux with patched QEMU 9.1–11, after passing normal target TH under emulation. -Other target/compiler package sets retain the upstream guard, which also covers -cross-compiler stalls without QEMU. Advertise the marker on a native x86_64 Linux -builder for those jobs; keep it off Rosetta slots. Nix also requires the +when jobs remain queued. The pinned haskell.nix now defaults +`emulatorSystemFeatures` to `[]`; its own CI opts back into `recursive-nix` as a +native-builder scheduling marker. This changes scheduling defaults for cross +targets independently of this checkout's explicit `ghc914-sh` Android ARM32 +opt-out on Linux with patched QEMU 9.1–11, where normal target TH passed under +emulation. The marker also covers cross-compiler stalls without QEMU. If a job +requires it, advertise it on a native x86_64 Linux builder and keep it off +Rosetta slots. Nix also requires the [`recursive-nix` experimental feature](https://nix.dev/manual/nix/2.35/development/experimental-features.html#xp-feature-recursive-nix) when building derivations with this reserved marker. Enable both `extra-experimental-features = recursive-nix` and diff --git a/scripts/nix/android-arm32-qemu.nix b/scripts/nix/android-arm32-qemu.nix index 41aef4d7a4..40a124f053 100644 --- a/scripts/nix/android-arm32-qemu.nix +++ b/scripts/nix/android-arm32-qemu.nix @@ -1,12 +1,11 @@ -{ haskellNix, pkgs }: -(import (haskellNix + "/overlays/qemu-linux-user.nix") { +{ haskellNix, pkgs, hostPlatform }: +import (haskellNix + "/overlays/qemu-linux-user.nix") { qemu = pkgs.qemu.override { userOnly = true; hostCpuTargets = [ "arm-linux-user" ]; }; inherit (pkgs) lib; buildPlatform = pkgs.stdenv.hostPlatform; + inherit hostPlatform; qemuSuffix = "arm"; -}).overrideAttrs (old: { - patches = (old.patches or []) ++ [ ./simplex-qemu-arm-personality.patch ]; -}) +} diff --git a/scripts/nix/android-emulator-overlay.nix b/scripts/nix/android-emulator-overlay.nix index 0f9ae00fd1..71a87ac6e5 100644 --- a/scripts/nix/android-emulator-overlay.nix +++ b/scripts/nix/android-emulator-overlay.nix @@ -15,6 +15,7 @@ let emulator = import ./android-arm32-qemu.nix { inherit haskellNix; pkgs = prev.pkgsBuildBuild; + inherit (prev.stdenv) hostPlatform; }; exes = prev.haskell-nix.iserv-proxy-exes.ghc914-sh; mkTemplateHaskell = interpreterExes: @@ -39,8 +40,8 @@ in lib.optionalAttrs androidArm { ); } // lib.optionalAttrs supportedEmulator { # Use the patched ARM32 emulator on local Linux builders too. This is the - # supported haskell.nix opt-out from its native-builder scheduling marker; - # other target package sets keep their existing restrictions. + # explicit supported haskell.nix scheduling opt-out. The pinned upstream + # default is also empty; its CI opts back into the native-builder marker. emulatorSystemFeatures = []; templateHaskell = prev.haskell-nix.templateHaskell // { ghc914-sh = mkTemplateHaskell exes // { diff --git a/scripts/nix/check-android-personality.nix b/scripts/nix/check-android-personality.nix index 9d8100d23b..d2793977dc 100644 --- a/scripts/nix/check-android-personality.nix +++ b/scripts/nix/check-android-personality.nix @@ -14,7 +14,10 @@ let crossSystem = buildPkgs.lib.systems.examples.armv7a-android-prebuilt // { androidSdkVersion = "26"; }; }; - emulator = import ./android-arm32-qemu.nix { inherit haskellNix pkgs; }; + emulator = import ./android-arm32-qemu.nix { + inherit haskellNix pkgs; + hostPlatform = target.stdenv.hostPlatform; + }; payload = buildPkgs.runCommand "android-arm32-personality-payload" {} '' mkdir -p "$out" # API 26's crtbegin predates Bionic's aligned TLS input. Add the same diff --git a/scripts/nix/check-elf-comdat-functions.nix b/scripts/nix/check-elf-comdat-functions.nix index 056f438819..bf616e7803 100644 --- a/scripts/nix/check-elf-comdat-functions.nix +++ b/scripts/nix/check-elf-comdat-functions.nix @@ -1,10 +1,10 @@ -{ pkgs, compiler }: +{ pkgs, compiler, patch }: assert pkgs.stdenv.hostPlatform.isLinux && pkgs.stdenv.hostPlatform.isx86_64; let source = pkgs.applyPatches { name = "ghc914-rts-single-function-comdat"; src = compiler.configured-src + "/rts"; - patches = [ ./ghc914-rts-elf-comdat-functions.patch ]; + patches = [ patch ]; }; in pkgs.runCommandCC "ghc914-elf-comdat-functions-check" {} '' set -euo pipefail diff --git a/scripts/nix/check-musl-rts-scope.nix b/scripts/nix/check-musl-rts-scope.nix index 0dac70c5f4..a902be0a78 100644 --- a/scripts/nix/check-musl-rts-scope.nix +++ b/scripts/nix/check-musl-rts-scope.nix @@ -1,5 +1,9 @@ # Use the pinned lib and the real project module; no compiler or project IFD. -{ lib, projectModule ? import ./musl-rts-project.nix }: +{ lib +, haskellNix ? builtins.fetchTree + (builtins.fromJSON (builtins.readFile ../../flake.lock)).nodes.haskellNix.locked +, projectModule ? import (haskellNix + "/modules/stable-haskell-rts-comdat.nix") +}: let apply = { musl ? true, i386 ? true, linux ? true, compiler ? "ghc914-sh", builderVersion ? 2, @@ -37,9 +41,9 @@ let config.packages.rts.patches = [ ./musl-iserv-public-libc.patch ]; } ] ++ project.config.modules; }; - in package.config.packages.rts.patches; - unchanged = [ ./musl-iserv-public-libc.patch ]; - selected = unchanged ++ [ ./ghc914-rts-elf-comdat-functions.patch ]; + in map toString package.config.packages.rts.patches; + unchanged = map toString [ ./musl-iserv-public-libc.patch ]; + selected = unchanged ++ [ (toString (haskellNix + "/overlays/patches/ghc914-rts-elf-comdat-functions.patch")) ]; in assert apply {} == selected; assert apply { name = "iserv-proxy"; } == selected; diff --git a/scripts/nix/ghc914-rts-elf-comdat-functions.patch b/scripts/nix/ghc914-rts-elf-comdat-functions.patch deleted file mode 100644 index 1ad465dc6c..0000000000 --- a/scripts/nix/ghc914-rts-elf-comdat-functions.patch +++ /dev/null @@ -1,149 +0,0 @@ ---- a/linker/Elf.c -+++ b/linker/Elf.c -@@ -796,6 +796,101 @@ - } - #endif -- -+ -+/* A zero st_size means unknown size, as used by GCC's i386 PIC thunks. -+ * Only local labels and section metadata may accompany that signature. -+ */ -+static bool -+isOnlyFunctionDefinition_ELF(ObjectCode *oc, const Elf_Sym *function) -+{ -+ for (ElfSymbolTable *table = oc->info->symbolTables; -+ table != NULL; table = table->next) { -+ for (size_t i = 0; i < table->n_symbols; i++) { -+ const Elf_Sym *other = table->symbols[i].elf_sym; -+ if (other == function || other->st_shndx != function->st_shndx) { -+ continue; -+ } -+ if (ELF_ST_BIND(other->st_info) != STB_LOCAL -+ || (ELF_ST_TYPE(other->st_info) != STT_NOTYPE -+ && ELF_ST_TYPE(other->st_info) != STT_SECTION)) { -+ return false; -+ } -+ } -+ } -+ return true; -+} -+ -+/* Recognize only a COMDAT group containing one hidden function section. -+ * GCC's i386 PIC thunks use this form with a GLOBAL, not WEAK, signature. -+ * This is deliberately not general multi-section COMDAT selection: retaining -+ * duplicate initializer, TLS, data, or partial-function sections is unsafe. -+ */ -+static bool -+isSingleFunctionComdat_ELF(ObjectCode *oc, const SymbolName *name) -+{ -+ if (oc == NULL || oc->type != STATIC_OBJECT || oc->info == NULL -+ || oc->info->elfHeader == NULL || oc->image == NULL -+ || oc->fileSize <= 0) { -+ return false; -+ } -+ -+ const Elf_Shdr *shdr = oc->info->sectionHeader; -+ const Elf_Word shnum = elf_shnum(oc->info->elfHeader); -+ const size_t imageSize = (size_t)oc->fileSize; -+ for (Elf_Word i = 0; i < shnum; i++) { -+ const Elf_Shdr *group = &shdr[i]; -+ if (group->sh_type != SHT_GROUP -+ || group->sh_size != 2 * sizeof(Elf_Word) -+ || group->sh_offset > imageSize -+ || group->sh_size > imageSize - group->sh_offset) { -+ continue; -+ } -+ Elf_Word entries[2]; -+ memcpy(entries, oc->image + group->sh_offset, sizeof(entries)); -+ if (entries[0] != GRP_COMDAT || entries[1] == SHN_UNDEF -+ || entries[1] >= shnum) { -+ continue; -+ } -+ const Elf_Shdr *member = &shdr[entries[1]]; -+ const Elf_Word required = SHF_ALLOC | SHF_EXECINSTR | SHF_GROUP; -+ if (member->sh_type != SHT_PROGBITS || member->sh_size == 0 -+ || (member->sh_flags & required) != required -+ || (member->sh_flags & (SHF_WRITE | SHF_TLS)) != 0) { -+ continue; -+ } -+ for (ElfSymbolTable *table = oc->info->symbolTables; -+ table != NULL; table = table->next) { -+ if (table->index != group->sh_link -+ || group->sh_info >= table->n_symbols) { -+ continue; -+ } -+ const ElfSymbol *signature = &table->symbols[group->sh_info]; -+ const Elf_Sym *symbol = signature->elf_sym; -+ if (signature->name != NULL && strcmp(signature->name, name) == 0 -+ && ELF_ST_BIND(symbol->st_info) == STB_GLOBAL -+ && ELF_ST_TYPE(symbol->st_info) == STT_FUNC -+ && ELF_ST_VISIBILITY(symbol->st_other) == STV_HIDDEN -+ && symbol->st_shndx < SHN_LORESERVE -+ && symbol->st_shndx == entries[1] -+ && symbol->st_value == 0 -+ && (symbol->st_size == member->sh_size -+ || (symbol->st_size == 0 -+ && isOnlyFunctionDefinition_ELF(oc, symbol)))) { -+ return true; -+ } -+ } -+ } -+ return false; -+} -+ -+bool -+ocIsComdatDuplicate_ELF(ObjectCode *oc, const SymbolName *name) -+{ -+ RtsSymbolInfo *existing = lookupStrHashTable(symhash, name); -+ return existing != NULL && existing->owner != oc -+ && isSingleFunctionComdat_ELF(oc, name) -+ && isSingleFunctionComdat_ELF(existing->owner, name); -+} -+ - int - ocGetNames_ELF ( ObjectCode* oc ) - { -@@ -1234,7 +1329,9 @@ - setWeakSymbol(oc, nm); - } - if (!ghciInsertSymbolTable(oc->fileName, symhash, -- nm, symbol->addr, isWeak, sym_type, symbol->elf_sym->st_size, oc) -+ nm, symbol->addr, -+ isWeak || ocIsComdatDuplicate_ELF(oc, nm), -+ sym_type, symbol->elf_sym->st_size, oc) - ) { - goto fail; - } ---- a/linker/Elf.h -+++ b/linker/Elf.h -@@ -10,6 +10,7 @@ - void ocDeinit_ELF ( ObjectCode* oc ); - int ocVerifyImage_ELF ( ObjectCode* oc ); - int ocGetNames_ELF ( ObjectCode* oc ); -+bool ocIsComdatDuplicate_ELF(ObjectCode *oc, const SymbolName *name); - int ocResolve_ELF ( ObjectCode* oc ); - int ocRunInit_ELF ( ObjectCode* oc ); - int ocRunFini_ELF ( ObjectCode* oc ); ---- a/Linker.c -+++ b/Linker.c -@@ -1645,10 +1645,19 @@ - Symbol_t symbol; - for (x = 0; x < oc->n_symbols; x++) { - symbol = oc->symbols[x]; -+ bool isWeak = isSymbolWeak(oc, symbol.name); -+#if defined(OBJFORMAT_ELF) -+ /* Archive discovery can change the selected owner. Recheck COMDAT -+ * eligibility now; never persist conditional acceptance as weakness. -+ */ -+ if (symbol.name != NULL && !isWeak) { -+ isWeak = ocIsComdatDuplicate_ELF(oc, symbol.name); -+ } -+#endif - if ( symbol.name - && !ghciInsertSymbolTable(oc->fileName, symhash, symbol.name, - symbol.addr, -- isSymbolWeak(oc, symbol.name), -+ isWeak, - symbol.type, 0, oc)) { - return 0; - } diff --git a/scripts/nix/musl-iserv-overlay.nix b/scripts/nix/musl-iserv-overlay.nix index 62359d5bd2..8ab81129f4 100644 --- a/scripts/nix/musl-iserv-overlay.nix +++ b/scripts/nix/musl-iserv-overlay.nix @@ -4,7 +4,7 @@ prev.lib.optionalAttrs ( ) { haskell-nix = prev.haskell-nix // { cabalProject' = projectModule: prev.haskell-nix.cabalProject' ( - [ ./musl-rts-project.nix ({ config, lib, pkgs, ... }: + [ ({ config, lib, pkgs, ... }: lib.mkIf ( pkgs.stdenv.hostPlatform.isMusl && pkgs.stdenv.buildPlatform.isLinux diff --git a/scripts/nix/musl-rts-project.nix b/scripts/nix/musl-rts-project.nix deleted file mode 100644 index f81bd5d373..0000000000 --- a/scripts/nix/musl-rts-project.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ config, lib, pkgs, ... }: -lib.mkIf ( - pkgs.stdenv.hostPlatform.isMusl - && pkgs.stdenv.hostPlatform.isx86_32 - && pkgs.stdenv.buildPlatform.isLinux - && config.compiler-nix-name == "ghc914-sh" - && config.builderVersion == 2 -) { - # Cross projects build their target RTS alongside the boot libraries. - # Cover both application and auxiliary interpreter projects, including - # profiling, while keeping the native compiler and its bootstrap intact. - modules = [ - ({ lib, ... }: { - packages.rts.patches = lib.mkAfter [ ./ghc914-rts-elf-comdat-functions.patch ]; - }) - ]; -} diff --git a/scripts/nix/simplex-qemu-arm-personality.patch b/scripts/nix/simplex-qemu-arm-personality.patch deleted file mode 100644 index 3b75457d47..0000000000 --- a/scripts/nix/simplex-qemu-arm-personality.patch +++ /dev/null @@ -1,84 +0,0 @@ ---- a/linux-user/qemu.h -+++ b/linux-user/qemu.h -@@ -100,6 +100,11 @@ - - struct TaskState { - pid_t ts_tid; /* tid (or pid) of this task */ -+#if defined(TARGET_ARM) && !defined(TARGET_AARCH64) -+ /* Guest execution domain when the host cannot accept PER_LINUX32. */ -+ uint32_t guest_personality; -+ bool guest_personality_virtualized; -+#endif - #ifdef TARGET_ARM - # ifdef TARGET_ABI32 - /* FPA state */ ---- a/linux-user/syscall.c -+++ b/linux-user/syscall.c -@@ -6969,6 +6969,11 @@ - ts->bprm = parent_ts->bprm; - ts->info = parent_ts->info; - ts->signal_mask = parent_ts->signal_mask; -+#if defined(TARGET_ARM) && !defined(TARGET_AARCH64) -+ ts->guest_personality = parent_ts->guest_personality; -+ ts->guest_personality_virtualized = -+ parent_ts->guest_personality_virtualized; -+#endif - - #ifdef TARGET_AARCH64 - ts->gcs_el0_locked = parent_ts->gcs_el0_locked; -@@ -9653,6 +9658,43 @@ - int, __to_dfd, const char *, __to_pathname, unsigned int, flag) - #endif - -+#if defined(TARGET_ARM) && !defined(TARGET_AARCH64) -+/* -+ * Android's ARM32 libc requests PER_LINUX32 during startup. An ARM64 host -+ * without AArch32 EL0 rejects that execution domain even though TCG can run -+ * the guest. QEMU already provides the guest architecture for uname(). -+ * -+ * Only substitute the execution domain after the host rejects it with EINVAL. -+ * Keep every upper flag host-backed, and retain the full guest value so query -+ * and setter return values do not expose the substituted host domain. This -+ * per-task state is copied by clone; a new Android exec initializes it again -+ * when libc sets its personality. -+ */ -+static abi_long do_arm32_personality(CPUState *cpu, uint32_t value) -+{ -+ TaskState *ts = get_task_state(cpu); -+ uint32_t previous = ts->guest_personality; -+ bool virtualized = ts->guest_personality_virtualized; -+ int ret = personality(value); -+ -+ if (ret == -1 && errno == EINVAL && (value & PER_MASK) == PER_LINUX32) { -+ ret = personality((value & ~PER_MASK) | PER_LINUX); -+ if (ret != -1) { -+ ts->guest_personality = value; -+ ts->guest_personality_virtualized = true; -+ return virtualized ? previous : (uint32_t)ret; -+ } -+ } -+ if (ret == -1) { -+ return get_errno(ret); -+ } -+ if (value != UINT32_MAX) { -+ ts->guest_personality_virtualized = false; -+ } -+ return virtualized ? previous : (uint32_t)ret; -+} -+#endif -+ - /* This is an internal helper for do_syscall so that it is easier - * to have a single return point, so that actions, such as logging - * of syscall results, can be performed. -@@ -11732,7 +11774,11 @@ - case TARGET_NR_fchdir: - return get_errno(fchdir(arg1)); - case TARGET_NR_personality: -+#if defined(TARGET_ARM) && !defined(TARGET_AARCH64) -+ return do_arm32_personality(cpu, (uint32_t)arg1); -+#else - return get_errno(personality(arg1)); -+#endif - #ifdef TARGET_NR__llseek /* Not on alpha */ - case TARGET_NR__llseek: - {