"""The Linux CLI, debs, AppImage and core libraries, and their reproduction.""" import contextlib import platform import re import shutil import tempfile from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from .common import ( GHC, BuildError, checkout, compare_dirs, download, edit_lines, extract_flat, fetch_pinned, log, normalize_permissions, release_url, single_dir, single_file, tag_version, touch_tree, write_sha256sums, ) from .desktop import ( CLI_BUILD_STEPS, COMPOSE_ARCHES, DESKTOP_ENV, DESKTOP_RELEASE_DIR, SOURCE_DATE_EPOCH, VLC_RELEASE, VLC_SHA256, Database, check_exports, client_library_constraints, desktop_libs, link_desktop_libs, remove_skiko_runtime, zip_desktop_libs, ) from .docker import OUT, PROJECT, Container, HostRunner, Runner, Step @dataclass(frozen=True, slots=True) class UbuntuImage: version: str arch: str digest: str @property def suffix(self) -> str: return self.version.replace(".", "_") # The Ubuntu images of build.yml's release builds, for each version and machine DESKTOP_IMAGES = ( UbuntuImage("22.04", "x86_64", "sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5"), UbuntuImage("24.04", "x86_64", "sha256:98ff7968124952e719a8a69bb3cccdd217f5fe758108ac4f21ad22e1df44d237"), UbuntuImage("22.04", "aarch64", "sha256:6a62a4157b8775eaf4959cb629e757d32d39d1f4c8ac1b0ddc2510b555cf72f3"), UbuntuImage("24.04", "aarch64", "sha256:68434214381cb38287104e629fe8ee720167dd98cbb36ab1cbbab342515fa6ab"), ) APPIMAGE_IMAGE = "22.04" DESKTOP_CABAL_LOCAL = "ignore-project: False\npackage direct-sqlcipher\n flags: +openssl" CABAL_DIR = "/home/builder/.cabal" RELEASE_WORKFLOW = ".github/workflows/build.yml" def install_linux_libs(source: Path, package: Path) -> None: arch = platform.machine() extract_flat(package, desktop_libs(source, f"linux-{arch}"), "libs/") link_desktop_libs(source, f"linux-{arch}", f"linux-{COMPOSE_ARCHES[arch]}") CLI_DEB_SOURCE_DATE_EPOCH = "1764547200" DEB_ARCHES = {"x86_64": "amd64", "aarch64": "arm64"} LINUX_LIB_GHC_OPTIONS = "-optl-Wl,-rpath,$ORIGIN -optl-Wl,-soname,libsimplex.so -flink-rts -threaded" # The only directory where the released libraries and programs look for libraries: their own. GHC and cabal link each # library with the absolute directories, in the build container, of the libraries it loads. ORIGIN = "$ORIGIN" ELF_MAGIC = b"\x7fELF" # readelf -d names each file before its dynamic section when it is given more than one READELF_FILE = "File: " READELF_SEARCH_PATHS = ("(RPATH)", "(RUNPATH)") # The builds of the continuous releases of simplex-chat/appimagetool and simplex-chat/type2-runtime of 2025-06-04 APPIMAGE_TOOLS = { "x86_64": ( ("appimagetool", "appimagetool-x86_64.AppImage", "ad30156bd2c0a364e742b13bd205021e2408399a9090bc2d76048dd504b23f66"), ("type2-runtime", "runtime-x86_64", "df90eadfa40b83a4183bf7a7f6fdf9c293eebea83a614fe48ccd2f849a75ad58"), ), "aarch64": ( ("appimagetool", "appimagetool-aarch64.AppImage", "1401a0557ed979b32ca82fb136b2f328ff2fafbb024b2b61e155980605cb3ccb"), ("type2-runtime", "runtime-aarch64", "75ad8a177eef47943328e541a57911a79a88446c3a1a0c7a172f2046eabebb9e"), ), } # jpackage's desktop entry passes no URL and handles no MIME type; this registers the simplexchat: and simplex: schemes DESKTOP_MIME_TYPES = "MimeType=x-scheme-handler/simplexchat;x-scheme-handler/simplex;" def build_linux_lib(runner: Runner, source: Path, arch: str, database: Database, dist_dir: str = "dist-newstyle") -> Path: """libsimplex.so and the GHC libraries it loads, in the desktop app's libraries directory, which it returns.""" check_exports(source) cabal = ("cabal", "build", f"--builddir={dist_dir}", "lib:simplex-chat", f"--ghc-options={LINUX_LIB_GHC_OPTIONS}") runner.run(Step((*cabal, *client_library_constraints(database)), runner.path(source))) build = single_dir(source / dist_dir / "build" / f"{arch}-linux" / f"ghc-{GHC}", "simplex-chat-*") / "build" for lib in build.glob(f"libHSsimplex-chat-*-inplace-ghc{GHC}.so"): lib.rename(build / "libsimplex.so") # Ubuntu 20.04 runners linked libffi.so.7, which users' systems may lack; libffi.so.8 ships with the libraries runner.run(Step(("patchelf", "--replace-needed", "libffi.so.7", "libffi.so.8", "libsimplex.so"), runner.path(build))) ldd = runner.run(Step(("ldd", "libsimplex.so"), runner.path(build)), capture=True) ghc_libs = [line.split()[2] for line in ldd.splitlines() if "ghc" in line] deps = build / "deps" deps.mkdir(exist_ok=True) if ghc_libs: runner.run(Step(("cp", *ghc_libs, runner.path(deps)), runner.path(build))) libs = desktop_libs(source, f"linux-{arch}") shutil.rmtree(libs, ignore_errors=True) shutil.rmtree(source / "apps/multiplatform/desktop/build/cmake", ignore_errors=True) libs.mkdir(parents=True) for path in [*deps.iterdir(), build / "libsimplex.so"]: shutil.copy(path, libs) set_origin_runpaths(runner, libs) return libs def elf_files(roots: Sequence[Path]) -> list[Path]: """The ELF files that are roots or in them, without symbolic links.""" candidates = [] for root in roots: candidates += [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file() and not p.is_symlink()) found = [] for candidate in candidates: with candidate.open("rb") as f: if f.read(len(ELF_MAGIC)) == ELF_MAGIC: found.append(candidate) return found def library_search_paths(runner: Runner, files: Sequence[Path]) -> dict[Path, list[str]]: """The RPATH and RUNPATH entries of each of files, as readelf -d prints them.""" if not files: return {} named = {runner.path(file): file for file in files} found: dict[Path, list[str]] = {file: [] for file in files} current = files[0] for line in runner.run(Step(("readelf", "-d", *named), runner.path(files[0].parent)), capture=True).splitlines(): if line.startswith(READELF_FILE): current = named[line.removeprefix(READELF_FILE).strip()] elif any(kind in line for kind in READELF_SEARCH_PATHS): found[current] += [entry for entry in line.split("[", 1)[1].rstrip().rstrip("]").split(":") if entry] return found def set_origin_runpaths(runner: Runner, libs: Path) -> None: """Each library in libs that looks for libraries elsewhere than in its own directory looks only there, where the release puts the libraries it loads.""" for lib, entries in library_search_paths(runner, elf_files([libs])).items(): if entries and entries != [ORIGIN]: runner.run(Step(("patchelf", "--set-rpath", ORIGIN, runner.path(lib)), runner.path(libs))) def check_origin_runpaths(runner: Runner, roots: Sequence[Path]) -> None: """Fails unless each ELF file in roots looks for libraries only in directories relative to its own.""" outside = [ f"{file} ({entry})" for file, entries in library_search_paths(runner, elf_files(roots)).items() for entry in entries if entry != ORIGIN and not entry.startswith(f"{ORIGIN}/") ] if outside: raise BuildError("these files look for libraries outside their own directory:\n" + "\n".join(outside)) def add_vlc_linux(runner: Runner, libs: Path, arch: str, work: Path) -> None: vlc = libs / "vlc" if vlc.exists(): return name = f"vlc-linux-{arch}.appimage" appimage = fetch_pinned(f"{VLC_RELEASE}/{name}", VLC_SHA256[name], work / name) appimage.chmod(0o755) runner.run(Step((runner.path(appimage), "--appimage-extract"), runner.path(work))) shutil.copytree(work / "squashfs-root/usr/lib", vlc, symlinks=True) def build_cli_deb(runner: Runner, binary: Path, version: str, arch: str, work: Path) -> Path: """The CLI's deb, with the release's control file and fixed timestamps.""" build = work / "deb-build" package = build / "simplex-chat" for directory in (build, package, package / "DEBIAN", package / "usr", package / "usr/bin"): directory.mkdir(mode=0o755) directory.chmod(0o755) size = f"{(binary.stat().st_size + 1023) / 1024:.0f}" control = package / "DEBIAN/control" control.write_text( "Package: simplex-chat\n" f"Version: {version}\n" "Section: Messenger\n" "Priority: optional\n" f"Architecture: {DEB_ARCHES[arch]}\n" "Maintainer: SimpleX Chat \n" "Description: SimpleX - the first messaging platform that has no user identifiers of any kind - 100% private by design! (CLI)\n" f"Installed-Size: {size}\n" ) control.chmod(0o644) shutil.copy(binary, package / "usr/bin/simplex-chat") (package / "usr/bin/simplex-chat").chmod(0o755) touch_tree(package, CLI_DEB_SOURCE_DATE_EPOCH) epoch = {"SOURCE_DATE_EPOCH": CLI_DEB_SOURCE_DATE_EPOCH} runner.run(Step(("dpkg-deb", "--build", "--root-owner-group", "--uniform-compression", "./simplex-chat"), runner.path(build), epoch)) runner.run(Step(("strip-nondeterminism", "./simplex-chat.deb"), runner.path(build), epoch)) return build / "simplex-chat.deb" def register_url_schemes(line: str) -> str: if line.startswith("Exec=") and "%u" not in line and "%U" not in line: return f"{line} %u" return DESKTOP_MIME_TYPES if line.startswith("MimeType=") else line def package_desktop_deb(runner: Runner, source: Path, arch: str, codename: str) -> Path: """jpackage's deb without the skiko runtime jar, versioned for the Ubuntu release and with fixed timestamps.""" multiplatform = source / "apps/multiplatform" runner.run(Step(("./gradlew", f"-Psimplex.assets.dir={DESKTOP_ENV['ASSETS_DIR']}", "packageDeb"), runner.path(multiplatform))) release = multiplatform / "release/main/deb" final = release / f"simplex_{arch}.deb" jpackage_debs = [p for p in sorted(release.glob("simplex*.deb")) if p != final] if len(jpackage_debs) != 1: raise BuildError(f"expected one deb of jpackage in {release}, found {len(jpackage_debs)}") extracted = multiplatform / "extracted" shutil.rmtree(extracted, ignore_errors=True) runner.run(Step(("dpkg-deb", "-R", runner.path(jpackage_debs[0]), "./extracted"), runner.path(multiplatform))) app = single_dir(extracted / "opt", "*imple*") remove_skiko_runtime(app / "lib/app") edit_lines(extracted / "DEBIAN/control", lambda line: f"{line}~{codename}" if "Version" in line else line) for entry in app.glob("lib/*.desktop"): edit_lines(entry, register_url_schemes) if DESKTOP_MIME_TYPES not in entry.read_text().splitlines(): raise BuildError(f"{entry.name} does not register simplexchat: and simplex:") check_origin_runpaths(runner, [extracted]) touch_tree(extracted, SOURCE_DATE_EPOCH) epoch = {"SOURCE_DATE_EPOCH": SOURCE_DATE_EPOCH} build = ("dpkg-deb", "--build", "--root-owner-group", "--uniform-compression", "./extracted", runner.path(final)) runner.run(Step(build, runner.path(multiplatform), epoch)) runner.run(Step(("strip-nondeterminism", runner.path(final)), runner.path(multiplatform), epoch)) shutil.rmtree(extracted) return final def ubuntu_codename(runner: Runner) -> str: release = runner.run(Step(("cat", "/etc/os-release")), capture=True) for line in release.splitlines(): if line.startswith("VERSION_CODENAME="): return line.split("=", 1)[1].strip('"') raise BuildError("VERSION_CODENAME is not in /etc/os-release") def system_libcrypto(runner: Runner, libs: Path, work: Path) -> Path: """A copy in work of the system's libcrypto that libsimplex.so loads, which the AppImage bundles.""" ldd = runner.run(Step(("ldd", "libsimplex.so"), runner.path(libs)), capture=True) found = [fields[2] for fields in map(str.split, ldd.splitlines()) if fields and fields[0].startswith("libcrypto.")] if len(found) != 1: raise BuildError(f"libsimplex.so loads {len(found)} libcrypto libraries, not one: {', '.join(found)}") copy = work / Path(found[0]).name runner.run(Step(("cp", "-L", found[0], runner.path(copy)), runner.path(libs))) return copy def package_appimage(runner: Runner, source: Path, arch: str, libcrypto: Path, work: Path) -> Path: """The AppImage of the desktop app, with libcrypto bundled and fixed timestamps.""" multiplatform = source / "apps/multiplatform" libs = desktop_libs(source, f"linux-{arch}") bundled = libs / libcrypto.name shutil.copy(libcrypto, bundled) try: runner.run( Step(("./gradlew", f"-Psimplex.assets.dir={DESKTOP_ENV['ASSETS_DIR']}", "createDistributable"), runner.path(multiplatform)) ) finally: bundled.unlink() app = multiplatform / "release/main/app" appdir = app / "AppDir" shutil.rmtree(appdir, ignore_errors=True) (appdir / "usr").mkdir(parents=True) distributable = single_dir(app, "*imple*") for part in ("bin", "lib"): shutil.copytree(distributable / part, appdir / "usr" / part, symlinks=True) shutil.copy(appdir / "usr/lib/simplex.png", appdir) for share in ("icons", "metainfo", "applications"): (appdir / "usr/share" / share).mkdir(parents=True) # AppImageLauncher shows this icon shutil.copy(appdir / "usr/lib/simplex.png", appdir / "usr/share/icons") (appdir / "AppRun").symlink_to(Path("usr/bin") / single_file(appdir / "usr/bin", "*imple*").name) distribute = multiplatform / "desktop/src/jvmMain/resources/distribute" entry = appdir / "chat.simplex.app.desktop" shutil.copy(single_file(distribute, "*imple*.desktop"), entry) edit_lines(entry, lambda line: re.sub(r"^Exec=[^ ]*", "Exec=simplex", re.sub(r"Icon=.*", "Icon=simplex", line))) shutil.copy(entry, appdir / "usr/share/applications") for appdata in distribute.glob("*.appdata.xml"): shutil.copy(appdata, appdir / "usr/share/metainfo") tool, runtime = ( fetch_pinned(f"https://github.com/simplex-chat/{repo}/releases/download/continuous/{name}", digest, work / name) for repo, name, digest in APPIMAGE_TOOLS[arch] ) tool.chmod(0o755) remove_skiko_runtime(appdir / "usr/lib/app") touch_tree(appdir, SOURCE_DATE_EPOCH) epoch = {"SOURCE_DATE_EPOCH": SOURCE_DATE_EPOCH} check_origin_runpaths(runner, [appdir]) appimagetool = (runner.path(tool), "--verbose", "--no-appstream", "--runtime-file", runner.path(runtime), ".") runner.run(Step(appimagetool, runner.path(appdir), epoch)) image = shutil.move(single_file(appdir, "*imple*.AppImage"), multiplatform / "release/main") runner.run(Step(("strip-nondeterminism", runner.path(Path(image))), runner.path(multiplatform), epoch)) return Path(image) # Releases made before this script built their Linux files with these scripts of their own, which reproduce runs TAG_CLI_DEB_SCRIPT = "scripts/desktop/build-cli-deb.sh" # a checkout without it is such a release BUILD_SCRIPT = "scripts/simplex-chat-build.py" TAG_DESKTOP_DEB_SCRIPT = "scripts/desktop/make-deb-linux.sh" TAG_APPIMAGE_SCRIPT = "scripts/desktop/make-appimage-linux.sh" def desktop_files(image: UbuntuImage) -> list[str]: cli = f"simplex-chat-ubuntu-{image.suffix}-{image.arch}" files = [cli, f"{cli}.deb", f"simplex-desktop-ubuntu-{image.suffix}-{image.arch}.deb"] if image.version == APPIMAGE_IMAGE: files.append(f"simplex-desktop-{image.arch}.AppImage") return files def linux_libs_zip(arch: str, database: Database) -> str: return f"simplex-chat-libs-linux-{arch}{'-postgres' if database == Database.POSTGRES else ''}.zip" def linux_image(version: str, machine: str) -> UbuntuImage: """The image of an Ubuntu version for this machine: Docker runs images of the host's architecture.""" for image in DESKTOP_IMAGES: if (image.version, image.arch) == (version, machine): return image raise BuildError(f"no Ubuntu {version} image for {machine}") def cabal_dir_args(source: Path) -> list[str]: """The cabal directory of source's release builds: CABAL_DIR since v7.1.0-beta.7, before it cabal's default ~/.local/state/cabal; the binaries contain the path of its package store.""" if f"CABAL_DIR={CABAL_DIR}" in (source / RELEASE_WORKFLOW).read_text(): return ["-e", f"CABAL_DIR={CABAL_DIR}"] return [] def linux_container(source: Path, image: UbuntuImage, work: Path) -> Container: return Container( source, {source: PROJECT, work: OUT}, build_args={"TAG": image.version, "HASH": image.digest, "GHC": GHC}, run_args=["--device", "/dev/fuse", "--cap-add", "SYS_ADMIN", "--security-opt", "apparmor:unconfined", *cabal_dir_args(source)], ) def build_desktop_linux(source: Path, image: UbuntuImage, version: str, out: Path) -> None: """The Linux release files of an Ubuntu image, with their released names, and on APPIMAGE_IMAGE the libraries.""" (source / "cabal.project.local").write_text(DESKTOP_CABAL_LOCAL) normalize_permissions(source) log(f"building Ubuntu {image.version} {image.arch} CLI and desktop apps") with tempfile.TemporaryDirectory() as tmp: work = Path(tmp).resolve() with linux_container(source, image, work) as container: for step in CLI_BUILD_STEPS: container.run(Step(step.argv, container.path(source))) cli = container.run(Step(("cabal", "list-bin", "exe:simplex-chat"), container.path(source)), capture=True).strip() container.run(Step(("cp", cli, container.path(work / "simplex-chat")), container.path(source))) container.run(Step(("strip", container.path(work / "simplex-chat")), container.path(source))) check_origin_runpaths(container, [work / "simplex-chat"]) shutil.copy(work / "simplex-chat", out / desktop_files(image)[0]) if not (source / BUILD_SCRIPT).exists(): build_desktop_linux_with_tag_scripts(container, source, image, version, work, out) return shutil.copy(build_cli_deb(container, work / "simplex-chat", version, image.arch, work), out / desktop_files(image)[1]) libs = build_linux_lib(container, source, image.arch, Database.SQLITE) add_vlc_linux(container, libs, image.arch, work) check_origin_runpaths(container, [libs]) link_desktop_libs(source, f"linux-{image.arch}", f"linux-{COMPOSE_ARCHES[image.arch]}") deb = package_desktop_deb(container, source, image.arch, ubuntu_codename(container)) shutil.copy(deb, out / desktop_files(image)[2]) if image.version == APPIMAGE_IMAGE: zip_desktop_libs(libs, [], out / linux_libs_zip(image.arch, Database.SQLITE)) appimage = package_appimage(container, source, image.arch, system_libcrypto(container, libs, work), work) shutil.copy(appimage, out / desktop_files(image)[3]) def build_desktop_linux_with_tag_scripts( container: Container, source: Path, image: UbuntuImage, version: str, work: Path, out: Path ) -> None: """The CLI deb, the desktop deb and the AppImage of a release made before this script, by its own scripts.""" files = desktop_files(image) deb_build = work / "deb-build" cli_deb = (TAG_CLI_DEB_SCRIPT, version, container.path(work / "simplex-chat"), container.path(deb_build)) container.run(Step(cli_deb, container.path(source))) shutil.copy(deb_build / "simplex-chat.deb", out / files[1]) container.run(Step((TAG_DESKTOP_DEB_SCRIPT,), container.path(source), DESKTOP_ENV)) release = source / DESKTOP_RELEASE_DIR shutil.copy(release / f"deb/simplex_{image.arch}.deb", out / files[2]) if image.version == APPIMAGE_IMAGE: container.run(Step((TAG_APPIMAGE_SCRIPT,), container.path(source), DESKTOP_ENV)) shutil.copy(single_file(release, "*imple*.AppImage"), out / files[3]) @dataclass(frozen=True, slots=True) class LibsBuilder: """Where the library release is built: this machine, a running container, or else a new one of the image.""" on_host: bool = False container: str | None = None def build_linux_libs(source: Path, image: UbuntuImage, database: Database, dist_dir: str, builder: LibsBuilder, out: Path) -> None: """The library release: the desktop libraries without VLC.""" (source / "cabal.project.local").write_text(DESKTOP_CABAL_LOCAL) with contextlib.ExitStack() as stack: runner: Runner if builder.on_host: runner = HostRunner() elif builder.container: runner = Container.running(builder.container, source) else: normalize_permissions(source) work = Path(stack.enter_context(tempfile.TemporaryDirectory())).resolve() runner = stack.enter_context(linux_container(source, image, work)) runner.run(Step(("cabal", "update"), runner.path(source))) libs = build_linux_lib(runner, source, image.arch, database, dist_dir) check_origin_runpaths(runner, [libs]) zip_desktop_libs(libs, [], out / linux_libs_zip(image.arch, database)) def reproduce_desktop(tag: str, repo: str, source: Path, results: Path) -> list[str]: built = results / "from-source" released = results / "prebuilt" built.mkdir(parents=True, exist_ok=True) released.mkdir(parents=True, exist_ok=True) for image in (image for image in DESKTOP_IMAGES if image.arch == platform.machine()): checkout(source, tag) build_desktop_linux(source, image, tag_version(tag), built) for name in desktop_files(image): download(release_url(tag, name, repo), released / name) mismatched = compare_dirs(built, released) for name in mismatched: log(f"{name}: sha256 doesn't match") if not mismatched: write_sha256sums(released, results / "_sha256sums") log(f"checksums: {results / '_sha256sums'}") return mismatched