"""The Windows CLI, msi and core libraries, built in an MSYS2 UCRT64 shell, and their comparison with signed releases.""" import array import contextlib import dataclasses import os import shutil import struct import sys import uuid import zipfile from pathlib import Path, PureWindowsPath from .buildroot import MIN_MACHINE_PATH_PARTS, BuildRoot, check_machine_paths, find_machine_paths, fixed_build_root from .common import ( REPO, BuildError, checkout, compare_release, download_release, edit_lines, fetch_pinned, log, log_group, remove_tree, run, sha256, single_dir, single_file, touch_tree, ) from .desktop import ( DESKTOP_ENV, DESKTOP_RELEASE_DIR, SOURCE_DATE_EPOCH, VLC_RELEASE, VLC_SHA256, WINDOWS_DESKTOP, Database, build_host_cli, check_exports, client_library_constraints, desktop_libs, link_desktop_libs, remove_skiko_runtime, run_cabal, zip_desktop_libs, ) from .openssl import OpenSsl, check_static_openssl, host_openssl, openssl_work, static_libcrypto # The system libraries that the static libcrypto of a MinGW build needs (ex_libs of OpenSSL's mingw-common) OPENSSL_WINDOWS_STATIC_LINK = "-optl-lws2_32 -optl-lgdi32 -optl-lcrypt32" # the linker writes the time of the link into each executable and DLL unless told not to LINKER_NO_TIMESTAMP = "-Wl,--no-insert-timestamp" WINDOWS_NO_TIMESTAMP = f"-optl{LINKER_NO_TIMESTAMP}" # Compose's packageMsi makes an app image of its own; this makes it pack the one of createDistributable, as on macOS MSI_FROM_APP_IMAGE_INIT = """gradle.allprojects { project -> project.tasks.configureEach { task -> if (task.name == "packageMsi") { def app = project.tasks.named("createDistributable") task.dependsOn(app) task.appImage.set(app.flatMap { it.destinationDir.dir(it.packageName) }) } } } """ # WiX gives each msi a random package code, which the build replaces with one derived from the msi's content MSI_PLACEHOLDER_PACKAGE_CODE = "{00000000-0000-0000-0000-000000000000}" MSI_PACKAGE_CODE_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, REPO) # The compound file header and the root storage's directory entry (MS-CFB 2.2, 2.6.1) CFB_SIGNATURE = bytes.fromhex("d0cf11e0a1b11ae1") CFB_HEADER_SIZE = 512 CFB_SECTOR_SHIFT_OFFSET = 0x1E CFB_FIRST_DIRECTORY_SECTOR_OFFSET = 0x30 CFB_MODIFIED_TIME_OFFSET = 0x6C # FILETIME counts 100 ns intervals from 1601 FILETIME_TICKS_PER_SECOND = 10_000_000 FILETIME_AT_UNIX_EPOCH = 11_644_473_600 * FILETIME_TICKS_PER_SECOND # Windows Installer's automation interface has no type information for PowerShell, which reaches it with InvokeMember MSI_SUMMARY_SCRIPT = """param([string]$Msi, [string]$PackageCode, [long]$Epoch) $ErrorActionPreference = "Stop" $RevisionNumber, $CreateTime, $LastSaveTime = 9, 12, 13 $time = [DateTimeOffset]::FromUnixTimeSeconds($Epoch).UtcDateTime $properties = @(@($RevisionNumber, $PackageCode), @($CreateTime, $time), @($LastSaveTime, $time)) $installer = New-Object -ComObject WindowsInstaller.Installer $summary = $installer.GetType().InvokeMember("SummaryInformation", "GetProperty", $null, $installer, @($Msi, $properties.Count)) foreach ($property in $properties) { [void]$summary.GetType().InvokeMember("Property", "SetProperty", $null, $summary, $property) } [void]$summary.GetType().InvokeMember("Persist", "InvokeMethod", $null, $summary, $null) """ def disable_ghc_lld() -> None: """With GHC's lld, linking libsimplex.dll can't find the exported symbols; a missing linker makes GHC use ld.""" settings = Path(run(["ghc", "--print-libdir"], capture=True).stdout.strip()) / "settings" edit_lines(settings, lambda line: line.replace("ld.lld.exe", "abracadabra.exe", 1)) def build_windows_lib(source: Path, openssl: OpenSsl) -> Path: """libsimplex.dll and the libcrypto it loads, in the desktop app's libraries directory, which it returns.""" check_exports(source) for old in source.glob("dist-newstyle/build/x86_64-windows/ghc-*/simplex-chat-*"): remove_tree(old) options = ( f"-shared -threaded {WINDOWS_NO_TIMESTAMP} -optl-L{openssl.lib.as_posix()} -optl-lcrypto-3-x64 " "-o libsimplex.dll libsimplex.dll.def" ) (source / "cabal.project.local").write_text(openssl.cabal_local(options)) disable_ghc_lld() run_cabal(source, ["build", "lib:simplex-chat", *client_library_constraints(Database.SQLITE)]) libs = desktop_libs(source, WINDOWS_DESKTOP.libs_dir) shutil.rmtree(libs, ignore_errors=True) shutil.rmtree(source / "apps/multiplatform/desktop/build/cmake", ignore_errors=True) libs.mkdir(parents=True) shutil.copy(openssl.lib / "libcrypto-3-x64.dll", libs) shutil.copy(source / "libsimplex.dll", libs) return libs def add_vlc_windows(libs: Path, work: Path) -> None: vlc = libs / "vlc" shutil.rmtree(vlc, ignore_errors=True) (vlc / "vlc").mkdir(parents=True) name = "vlc-win-x86_64.zip" extracted = work / "vlc" with zipfile.ZipFile(fetch_pinned(f"{VLC_RELEASE}/{name}", VLC_SHA256[name], work / name)) as z: z.extractall(extracted) root = single_dir(extracted, "vlc-*") # the plugin cache must hold the time the plugins get when the app extracts them from its jar for plugin in (p for p in (root / "plugins").rglob("*") if ".dll" in p.name): os.utime(plugin, (plugin.stat().st_atime, 0)) run([root / "vlc-cache-gen.exe", "plugins"], cwd=root) for dll in root.glob("*.dll"): shutil.copy(dll, vlc) shutil.copytree(root / "plugins", vlc / "vlc/plugins", dirs_exist_ok=True) def release_windows_libs(source: Path, openssl: OpenSsl, out: Path) -> Path: """The library release, with the import library of libsimplex.dll; returns the libraries directory of the app.""" libs = build_windows_lib(source, openssl) run(["dlltool", "-d", "libsimplex.dll.def", "-l", "libsimplex.lib", "-D", "libsimplex.dll"], cwd=source) zip_desktop_libs(libs, [source / "libsimplex.lib"], out / WINDOWS_DESKTOP.libs) return libs # The release builds run in a copy of the checkout here, with cabal's directory and the temporary directory in it: # cabal hashes the paths of OpenSSL's build in the checkout into the package ids that the binaries contain. Any user # can make a folder in the root of the system drive. WINDOWS_BUILD_ROOT = Path("C:/simplex-chat-build") # what the paths of the machine look like in the files: written by Windows, by MSYS2 or by GHC, in UTF-16 or not WINDOWS_PATH_ENCODINGS = ("utf-8", "utf-16-le") def windows_path(path: Path) -> str: # MSYS2's Python writes paths with /, which cabal's hackage-security joins to the drive as a relative path: C:dir return str(path).replace("/", "\\") def windows_build_env(root: BuildRoot) -> dict[str, str]: tmp = windows_path(root.tmp) return {"CABAL_DIR": windows_path(root.directory / "cabal"), "TMP": tmp, "TEMP": tmp} def windows_build_root(source: Path) -> contextlib.AbstractContextManager[BuildRoot]: root = BuildRoot.at(WINDOWS_BUILD_ROOT) return fixed_build_root(source, root, windows_build_env(root)) def windows_machine_paths(source: Path) -> list[str]: """The checkout, home and temporary directories, which release files built on other machines can't contain, with either slash, either case of the drive letter and as MSYS2 writes them, and with short names expanded.""" paths = [source, Path.home(), *(Path(os.environ[name]) for name in ("TMP", "TEMP") if os.environ.get(name))] forms = set() for path in {*paths, *(path.resolve() for path in paths)}: windows = PureWindowsPath(path) drive, rest = windows.drive, windows.as_posix()[len(windows.drive) :] for written in (str(windows), windows.as_posix()): forms |= {written, written.replace(drive, drive.swapcase(), 1)} forms.add(f"/{drive.rstrip(':').lower()}{rest}") return sorted(form for form in forms if len(PureWindowsPath(form).parts) >= MIN_MACHINE_PATH_PARTS) # a file of the checkout that is committed with LF line endings LINE_ENDINGS_PROBE = "simplex-chat.cabal" def check_line_endings(source: Path) -> None: """Fails unless source has the line endings it is committed with: the binaries contain text of the source, such as the SQL of the database migrations.""" if b"\r\n" in (source / LINE_ENDINGS_PROBE).read_bytes(): raise BuildError(f"{source} is checked out with CRLF line endings: check it out with git -c core.autocrlf=false") def build_windows_libs(source: Path, out: Path) -> None: check_line_endings(source) paths = windows_machine_paths(source) with windows_build_root(source) as root: libs = release_windows_libs(root.source, host_openssl(openssl_work(root.source)), out) found = find_machine_paths([libs], paths, encodings=WINDOWS_PATH_ENCODINGS) check_machine_paths(found) def set_msi_summary(msi: Path, package_code: str, work: Path) -> None: """The msi's package code, and its creation and save times at SOURCE_DATE_EPOCH.""" script = work / "msi-summary.ps1" script.write_text(MSI_SUMMARY_SCRIPT) msi_path = str(PureWindowsPath(msi)) run(["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script, msi_path, package_code, SOURCE_DATE_EPOCH]) script.unlink() def msi_package_code(msi: Path) -> str: """Windows Installer treats msi files with one package code as copies of one file, so the code is a UUID of the content.""" return "{" + str(uuid.uuid5(MSI_PACKAGE_CODE_NAMESPACE, sha256(msi))).upper() + "}" def set_msi_storage_time(msi: Path) -> None: """The modification time of the msi's root storage, which each commit sets to its own time, at SOURCE_DATE_EPOCH.""" with msi.open("r+b") as f: header = f.read(CFB_HEADER_SIZE) if not header.startswith(CFB_SIGNATURE): raise BuildError(f"{msi} is not a compound file") sector_shift = struct.unpack_from(" None: set_msi_summary(msi, MSI_PLACEHOLDER_PACKAGE_CODE, work) set_msi_storage_time(msi) set_msi_summary(msi, msi_package_code(msi), work) set_msi_storage_time(msi) MEBIBYTE = 1 << 20 # where Compose keeps what jpackage and WiX print, which Gradle doesn't show when they fail JPACKAGE_MSI_LOGS = "desktop/build/compose/logs/packageMsi" def print_jpackage_logs(logs: Path) -> None: """What jpackage printed, and the free space of the drive that it wrote to.""" for log_file in sorted(logs.glob("*.txt")): with log_group(log_file.name): print(log_file.read_text(errors="replace"), flush=True) log(f"{shutil.disk_usage(logs).free // MEBIBYTE} MiB free on the drive of {logs}") def package_msi(source: Path, work: Path) -> Path: """jpackage's msi of the app image without the skiko runtime jar and with fixed times, which the msi's cabinet keeps.""" multiplatform = source / "apps/multiplatform" # gradlew.bat, as Windows finds its WSL bash.exe before MSYS2's; a daemon would keep files of the build directory # open after the build, which Windows then can't delete gradle: list[str | Path] = [multiplatform / "gradlew.bat", "--no-daemon", f"-Psimplex.assets.dir={DESKTOP_ENV['ASSETS_DIR']}"] # CMake links libapp-lib.dll with LDFLAGS env = {"LDFLAGS": LINKER_NO_TIMESTAMP} run([*gradle, "createDistributable"], cwd=multiplatform, env=env) app = single_dir(source / DESKTOP_RELEASE_DIR / "app", "*imple*") remove_skiko_runtime(app / "app") touch_tree(app, SOURCE_DATE_EPOCH) init = work / "msi-from-app-image.gradle" init.write_text(MSI_FROM_APP_IMAGE_INIT) # packageMsi would otherwise create the edited app image again try: run([*gradle, "--init-script", init, "packageMsi", "-x", "createDistributable"], cwd=multiplatform, env=env) except BuildError: print_jpackage_logs(multiplatform / JPACKAGE_MSI_LOGS) raise init.unlink() msi = single_file(source / DESKTOP_RELEASE_DIR / "msi", "*imple*.msi") normalize_msi_summary(msi, work) return msi def build_desktop_windows(source: Path, out: Path) -> None: host = WINDOWS_DESKTOP check_line_endings(source) paths = windows_machine_paths(source) with windows_build_root(source) as root: work = root.work openssl_dir = openssl_work(root.source) openssl = host_openssl(openssl_dir) static = OpenSsl(openssl.include, static_libcrypto(openssl.lib, openssl_dir)) cli = build_host_cli(root.source, static.cabal_local(f"{OPENSSL_WINDOWS_STATIC_LINK} {WINDOWS_NO_TIMESTAMP}")) check_static_openssl(cli, run(["objdump", "-p", cli], capture=True).stdout) shutil.copy(cli, out / host.cli) libs = release_windows_libs(root.source, openssl, out) add_vlc_windows(libs, work) link_desktop_libs(root.source, host.libs_dir, "windows-x64") msi = package_msi(root.source, work) shutil.copy(msi, out / host.desktop) app = single_dir(root.source / DESKTOP_RELEASE_DIR / "app", "*imple*") found = find_machine_paths([out / host.cli, app, msi], paths, encodings=WINDOWS_PATH_ENCODINGS) check_machine_paths(found) PE_DOS_MAGIC = b"MZ" PE_POINTER_OFFSET = 0x3C PE_SIGNATURE = b"PE\0\0" COFF_HEADER_SIZE = 20 PE32_MAGIC = 0x10B PE32_PLUS_MAGIC = 0x20B OPTIONAL_HEADER_CHECKSUM = 64 # the data directories of the optional header, of which the fifth is the certificate table (PE format, "Optional Header # Data Directories") DATA_DIRECTORIES = {PE32_MAGIC: 96, PE32_PLUS_MAGIC: 112} CERTIFICATE_TABLE = 4 * 8 CHECKSUM_FOLD = 0xFFFF PE_DIFFERS = "it differs by more than its Authenticode signature" @dataclasses.dataclass(frozen=True, slots=True) class PeHeader: """The offsets of the checksum and of the certificate table's directory entry in a PE file.""" checksum: int certificate_table: int def pe_header(data: bytes) -> PeHeader | None: if len(data) < PE_POINTER_OFFSET + 4 or not data.startswith(PE_DOS_MAGIC): return None (pe,) = struct.unpack_from(" int: """The checksum of the PE file as CheckSumMappedFile computes it: its 16-bit words, without the checksum field, added with end-around carry, plus its length.""" padded = bytearray(data) padded[checksum : checksum + 4] = bytes(4) if len(padded) % 2: padded.append(0) words = array.array("H", padded) if sys.byteorder == "big": words.byteswap() total = sum(words) while total > CHECKSUM_FOLD: total = (total & CHECKSUM_FOLD) + (total >> 16) return total + len(data) def copy_authenticode(released: bytes, built: bytes) -> bytes: """built with released's certificate table appended as signtool appends it, and the directory entry and checksum that it updates; built as it is when released isn't a signed PE file or built isn't an unsigned one.""" theirs, ours = pe_header(released), pe_header(built) if theirs is None or ours is None: return built offset, size = struct.unpack_from(" bool: """Whether released is built, or built with released's Authenticode signature.""" if released == built: return True try: return copy_authenticode(released, built) == released except struct.error: return False # The compound file binary format (MS-CFB): the header's fields, the directory entries and the special sector numbers CFB_HEADER = "<8s16sHHHHH6sIIIIIIIII" CFB_HEADER_DIFAT = 0x4C CFB_HEADER_DIFAT_COUNT = 109 CFB_DIRECTORY_ENTRY = "<64sHBBIII16sIQQIQ" CFB_DIRECTORY_ENTRY_SIZE = 128 CFB_NO_STREAM = 0xFFFFFFFF CFB_END_OF_CHAIN = 0xFFFFFFFE CFB_STORAGE = 1 CFB_STREAM = 2 CFB_ROOT = 5 # The streams that signtool adds to an msi: the signature, and the hash of the storages' metadata it may cover MSI_SIGNATURE_STREAMS = ("\x05DigitalSignature", "\x05MsiDigitalSignatureEx") @dataclasses.dataclass(frozen=True, slots=True) class CompoundEntry: """A storage or stream of a compound file, with what its directory entry records and a stream's data.""" kind: int clsid: bytes state: int created: int modified: int data: bytes class CompoundFileError(Exception): pass def compound_entries(data: bytes) -> dict[str, CompoundEntry]: """The storages and streams of a compound file by their paths, "/" for the root storage.""" if not data.startswith(CFB_SIGNATURE): raise CompoundFileError("not a compound file") header = struct.unpack_from(CFB_HEADER, data) sector_shift, mini_shift = header[5], header[6] first_directory, cutoff, first_mini_fat, first_difat, difat_sectors = header[10], header[12], header[13], header[15], header[16] sector_size = 1 << sector_shift def sector(number: int) -> bytes: start = (number + 1) << sector_shift if start >= len(data): raise CompoundFileError(f"sector {number} is past the end of the file") return data[start : start + sector_size] def numbers(chunk: bytes) -> list[int]: return list(struct.unpack(f"<{len(chunk) // 4}I", chunk)) fat_sectors = numbers(data[CFB_HEADER_DIFAT : CFB_HEADER_DIFAT + 4 * CFB_HEADER_DIFAT_COUNT]) difat = first_difat for _ in range(difat_sectors): listed = numbers(sector(difat)) fat_sectors += listed[:-1] difat = listed[-1] fat = [entry for number in fat_sectors if number < CFB_END_OF_CHAIN for entry in numbers(sector(number))] def chain(start: int, table: list[int]) -> list[int]: found: list[int] = [] while start < CFB_END_OF_CHAIN: if start >= len(table) or len(found) > len(table): raise CompoundFileError(f"broken sector chain at {start}") found.append(start) start = table[start] return found def read(start: int) -> bytes: return b"".join(sector(number) for number in chain(start, fat)) directory = read(first_directory) records = [struct.unpack_from(CFB_DIRECTORY_ENTRY, directory, i) for i in range(0, len(directory), CFB_DIRECTORY_ENTRY_SIZE)] root = records[0] mini_stream = read(root[11])[: root[12]] mini_fat = numbers(read(first_mini_fat)) if first_mini_fat < CFB_END_OF_CHAIN else [] mini_size = 1 << mini_shift def stream(record: tuple[int, ...]) -> bytes: start, size = record[11], record[12] if size < cutoff: return b"".join(mini_stream[n * mini_size : (n + 1) * mini_size] for n in chain(start, mini_fat))[:size] return read(start)[:size] entries: dict[str, CompoundEntry] = {} def visit(number: int, parent: str) -> None: if number == CFB_NO_STREAM: return name_bytes, name_length, kind, _, left, right, child, clsid, state, created, modified, _, _ = records[number] name = name_bytes[: max(0, name_length - 2)].decode("utf-16-le") path = f"{parent}{name}" if path in entries: raise CompoundFileError(f"{path} is listed twice") entries[path] = CompoundEntry(kind, clsid, state, created, modified, stream(records[number]) if kind == CFB_STREAM else b"") visit(left, parent) visit(right, parent) if kind == CFB_STORAGE: visit(child, f"{path}/") entries["/"] = CompoundEntry(CFB_ROOT, root[7], root[8], root[9], root[10], b"") visit(root[6], "/") return entries def msi_differences(released: bytes, built: bytes) -> list[str]: """The storages and streams of the released msi, other than its signature and the time signtool saved it at, that the built one doesn't have as they are, and the ones that only the built one has.""" try: theirs, ours = compound_entries(released), compound_entries(built) except (CompoundFileError, struct.error, UnicodeDecodeError) as e: return [f"not a readable msi: {e}"] signed = [theirs.pop(f"/{name}") for name in MSI_SIGNATURE_STREAMS if f"/{name}" in theirs] if signed and "/" in ours: # signtool saves the root storage with the time it signs at theirs["/"] = dataclasses.replace(theirs["/"], modified=ours["/"].modified) return sorted(path for path in theirs.keys() | ours.keys() if theirs.get(path) != ours.get(path)) def windows_differences(released: Path, built: Path) -> list[str]: """What differs other than the Authenticode signature: the msi's streams and storages, or the PE file's bytes.""" data, built_data = released.read_bytes(), built.read_bytes() if data.startswith(CFB_SIGNATURE): return [f"{path} differs" for path in msi_differences(data, built_data)] return [] if same_pe(data, built_data) else [PE_DIFFERS] def reproduce_windows(tag: str, repo: str, source: Path, results: Path) -> list[str]: out = results / "windows" names = (WINDOWS_DESKTOP.cli, WINDOWS_DESKTOP.desktop) download_release(tag, repo, names, out / "prebuilt") checkout(source, tag) (out / "from-source").mkdir(exist_ok=True) build_desktop_windows(source, out / "from-source") return compare_release(names, out, windows_differences)