diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a86d4790a8..f73bfa7927 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,6 +147,12 @@ jobs: with: swap-size-gb: 30 + - name: Get UID and GID + id: ids + run: | + echo "uid=$(id -u)" >> $GITHUB_OUTPUT + echo "gid=$(id -g)" >> $GITHUB_OUTPUT + # Otherwise we run out of disk space with Docker build - name: Free disk space if: matrix.should_run == true @@ -177,6 +183,8 @@ jobs: build-args: | TAG=${{ matrix.os }} GHC=${{ matrix.ghc }} + USER_UID=${{ steps.ids.outputs.uid }} + USER_GID=${{ steps.ids.outputs.gid }} # Docker needs these flags for AppImage build: # --device /dev/fuse @@ -209,7 +217,6 @@ jobs: if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | - chmod -R 777 dist-newstyle ~/.cabal && git config --global --add safe.directory '*' cabal clean cabal update cabal build -j --enable-tests diff --git a/Dockerfile.build b/Dockerfile.build index fddc96b6c2..3ddff59d12 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -13,6 +13,10 @@ ARG JAVA_HASH_ARM64=2b460859b681757b33a7591b6238ecaf51569d05d2684984e5f0a89c6514 ENV TZ=Etc/UTC \ DEBIAN_FRONTEND=noninteractive +ARG USER_UID=1000 +ARG USER_GID=1000 +ARG USER_NAME=builder + # Install curl, git and and simplex-chat dependencies RUN apt-get update && \ apt-get install -y curl \ @@ -38,6 +42,11 @@ RUN apt-get update && \ file \ appstream \ gpg \ + zipalign \ + apksigner \ + python3 \ + python3-venv \ + xz-utils \ unzip &&\ ln -s /bin/fusermount /bin/fusermount3 || : @@ -67,6 +76,12 @@ RUN export JAVA_FILENAME='java-corretto.deb' \ echo "Checksum mismatch" && exit 1; \ fi +RUN userdel -r ubuntu || : +RUN groupadd -g ${USER_GID} ${USER_NAME} || :; useradd -u ${USER_UID} -g ${USER_GID} --create-home --shell /bin/bash ${USER_NAME} || : +RUN mkdir /nix /out && chown ${USER_NAME}:${USER_NAME} /nix /out +USER ${USER_NAME} +WORKDIR /home/${USER_NAME} + # Specify bootstrap Haskell versions ENV BOOTSTRAP_HASKELL_GHC_VERSION=${GHC} ENV BOOTSTRAP_HASKELL_CABAL_VERSION=${CABAL} @@ -78,8 +93,10 @@ ENV BOOTSTRAP_HASKELL_INSTALL_NO_STACK_HOOK=true # Install ghcup RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 sh +# Setup basic env variables (required) +ENV HOME="/home/${USER_NAME}" USER="${USER_NAME}" # Adjust PATH -ENV PATH="/root/.cabal/bin:/root/.ghcup/bin:$PATH" +ENV PATH="$HOME/.cabal/bin:$HOME/.ghcup/bin:$PATH" # Set both as default RUN ghcup set ghc "${GHC}" && \ @@ -90,8 +107,8 @@ RUN ghcup set ghc "${GHC}" && \ #===================== ARG SDK_VERSION=13114758 -ENV SDK_VERSION=$SDK_VERSION \ - ANDROID_HOME=/root +ENV SDK_VERSION="$SDK_VERSION" \ + ANDROID_HOME="$HOME" RUN curl -L -o tools.zip "https://dl.google.com/android/repository/commandlinetools-linux-${SDK_VERSION}_latest.zip" && \ unzip tools.zip && rm tools.zip && \ @@ -101,11 +118,17 @@ RUN curl -L -o tools.zip "https://dl.google.com/android/repository/commandlineto ENV PATH="$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/tools/bin" # https://askubuntu.com/questions/885658/android-sdk-repositories-cfg-could-not-be-loaded -RUN mkdir -p ~/.android ~/.gradle && \ - touch ~/.android/repositories.cfg && \ - echo 'org.gradle.console=plain' > ~/.gradle/gradle.properties &&\ +RUN mkdir -p "$HOME/.android" "$HOME/.gradle" && \ + touch "$HOME/.android/repositories.cfg" && \ + echo 'org.gradle.console=plain' > "$HOME/.gradle/gradle.properties" &&\ yes | sdkmanager --licenses >/dev/null -ENV PATH=$PATH:$ANDROID_HOME/platform-tools:$ANDROID_HOME/build-tools +ENV PATH="$PATH:$ANDROID_HOME/platform-tools:$ANDROID_HOME/build-tools" + +# Android reproducibility scripts +RUN python3 -m venv "$HOME/.venv" +RUN "$HOME/.venv/bin/pip" install apksigcopier repro-apk + +ENV PATH="$HOME/.venv/bin:$PATH" WORKDIR /project diff --git a/scripts/android/build-android.sh b/scripts/android/build-android.sh index 90d6092385..afd13011c9 100755 --- a/scripts/android/build-android.sh +++ b/scripts/android/build-android.sh @@ -102,6 +102,7 @@ build() { sed -i.bak 's/jniLibs.useLegacyPackaging =.*/jniLibs.useLegacyPackaging = true/' "$folder/apps/multiplatform/android/build.gradle.kts" sed -i.bak '/android {/a lint {abortOnError = false}' "$folder/apps/multiplatform/android/build.gradle.kts" sed -i.bak '/tasks/Q' "$folder/apps/multiplatform/android/build.gradle.kts" + sed -i.bak "s/android.version_code=.*/android.version_code=${vercode}/" "$folder/apps/multiplatform/gradle.properties" for arch in $arches; do if [ "$arch" = "armv7a" ]; then @@ -169,8 +170,10 @@ pre() { done shift $(( $OPTIND - 1 )) - - commit="${1:-HEAD}" + + vercode="${1}" + + commit="${2:-HEAD}" } main() { diff --git a/scripts/simplex-chat-reproduce-builds-android.sh b/scripts/simplex-chat-reproduce-builds-android.sh new file mode 100755 index 0000000000..ac026f95cf --- /dev/null +++ b/scripts/simplex-chat-reproduce-builds-android.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env sh +set -eu + +SIMPLEX_KEY='3C:52:C4:FD:3C:AD:1C:07:C9:B0:0A:70:80:E3:58:FA:B9:FE:FC:B8:AF:5A:EC:14:77:65:F1:6D:0F:21:AD:85' + +REPO_NAME="simplex-chat" +REPO="https://github.com/simplex-chat/${REPO_NAME}" + +IMAGE_NAME='sx-local-android' +CONTAINER_NAME='sx-builder-android' +DOCKER_PATH_PROJECT='/project' +DOCKER_PATH_VERIFY='/verify' + +export DOCKER_BUILDKIT=1 + +SIMPLEX_REPO='simplex-chat/simplex-chat' +CMDS="curl git docker" + +INIT_DIR="$PWD" +TEMPDIR="$(mktemp -d)" + +ARCHES="${ARCHES:-aarch64 armv7a}" + +COLOR_CYAN="\033[36m" +COLOR_RESET="\033[0m" + +SUFFIX_BUILT='built' +SUFFIX_DOWNLOADED='downloaded' +SUFFIX_BUILT_WITH_SIGNATURE='built-with-downloaded-signature' + +cleanup() { + rm -rf -- "${TEMPDIR}" + docker rm --force "${CONTAINER_NAME}" 2>/dev/null || : + docker image rm "${IMAGE_NAME}" 2>/dev/null || : +} +trap 'cleanup' EXIT INT + +check() { + set +u + + for i in $commands; do + case $i in + *) + if ! command -v "$i" > /dev/null 2>&1; then + commands_failed="$i $commands_failed" + fi + ;; + esac + done + + if [ -n "$commands_failed" ]; then + commands_failed=${commands_failed% *} + printf "%s is not found in your \$PATH. Please install them and re-run the script.\n" "$commands_failed" + exit 1 + fi + + set -u +} + +download_apk() { + tag="$1" + filename="$2" + file_out="$3" + + curl -L "${REPO}/releases/download/${tag}/${filename}" -o "$file_out" +} + +setup_git() { + workdir="$1" + name="$2" + + git -C "$workdir" clone "${REPO}.git" "$name" +} + +checkout_git() { + git_dir="$1" + tag="$2" + + git -C "$git_dir" reset --hard + git -C "$git_dir" clean -dfx + git -C "$git_dir" checkout "$tag" +} + +check_apk() { + apk_name="$1" + expected="$2" + + actual=$(docker exec "${CONTAINER_NAME}" apksigner verify --print-certs "${DOCKER_PATH_VERIFY}/${apk_name}" | grep 'SHA-256' | awk '{print $NF}' | fold -w2 | paste -sd: | tr '[:lower:]' '[:upper:]') + + if [ "$expected" = "$actual" ]; then + return 0 + else + return 1 + fi +} + +verify_apk() { + apk_name="$1" + + # https://github.com/obfusk/apksigcopier?tab=readme-ov-file#what-about-signatures-made-by-apksigner-from-build-tools--3500-rc1 + docker exec "${CONTAINER_NAME}" repro-apk zipalign --page-size 16 --pad-like-apksigner --replace "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \ + "${DOCKER_PATH_VERIFY}/${apk_name}.aligned" + docker exec "${CONTAINER_NAME}" mv "${DOCKER_PATH_VERIFY}/${apk_name}.aligned" \ + "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" + + docker exec "${CONTAINER_NAME}" apksigcopier copy "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" \ + "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT}" \ + "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}" + + downloaded_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_DOWNLOADED}" | awk '{print $1}') + built_apk_hash=$(docker exec "${CONTAINER_NAME}" sha256sum "${DOCKER_PATH_VERIFY}/${apk_name}.${SUFFIX_BUILT_WITH_SIGNATURE}" | awk '{print $1}') + + if [ "$downloaded_apk_hash" = "$built_apk_hash" ]; then + return 0 + else + return 1 + fi +} + +print_vercode() { + build_dir="$1" + awk -F'=' '/android.version_code=/ {print $2}' "${build_dir}/apps/multiplatform/gradle.properties" +} + +setup_container() { + dir_git="$1" + dir_apk="$2" + + docker build \ + --no-cache \ + -f "${dir_git}/Dockerfile.build" \ + -t "${IMAGE_NAME}" \ + --build-arg=USER_UID="$(id -u)" \ + --build-arg=USER_GID="$(id -g)" \ + . + + # Run container in background + docker run -t -d \ + --name "${CONTAINER_NAME}" \ + --device /dev/fuse \ + --cap-add SYS_ADMIN \ + --security-opt apparmor:unconfined \ + --security-opt seccomp:unconfined \ + -v "${dir_git}:${DOCKER_PATH_PROJECT}" \ + -v "${dir_apk}:${DOCKER_PATH_VERIFY}" \ + "${IMAGE_NAME}" +} + +build_apk() { + arch="$1" + vercode="$2" + + apk_out="simplex-${arch}.apk.${SUFFIX_BUILT}" + + # Gradle setup + docker exec -i "${CONTAINER_NAME}" sh << EOF +cd $DOCKER_PATH_PROJECT/apps/multiplatform +./gradlew +EOF + + docker exec -i "${CONTAINER_NAME}" sh << EOF +GRADLE_BIN=\$(find \$HOME/.gradle/wrapper/dists -name "gradle" -type f -executable 2>/dev/null | head -1) +GRADLE_DIR=\$(dirname "\$GRADLE_BIN") +export PATH="\$GRADLE_DIR:\$PATH" + +ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode" || ARCHES="$arch" ./scripts/android/build-android.sh -gs "$vercode" + +APK_FILE=\$(find . -maxdepth 1 -type f -name '*.apk') + +mv "\$APK_FILE" $DOCKER_PATH_VERIFY/$apk_out +EOF +} + +main() { + tag="$1" + + build_directory="${TEMPDIR}/${REPO_NAME}" + final_directory="$INIT_DIR/${tag}-${REPO_NAME}" + apk_directory="${final_directory}/android" + + printf 'This script will: +1) build docker container. +2) download APK from GitHub and validate signatures. +3) build core library with nix (12-24 hours). +4) build APK and compare with downloaded one + +Continue?' + + read _ + + check + + mkdir -p "${apk_directory}" + + # Setup initial git for Dockerfile.build + setup_git "$TEMPDIR" "$REPO_NAME" + checkout_git "$build_directory" "$tag" + + printf "${COLOR_CYAN}Building Docker container...${COLOR_RESET}\n" + setup_container "$build_directory" "$apk_directory" + + # Check phase + for arch in $ARCHES; do + filename="simplex-${arch}.apk" + + download_apk "$tag" "$filename" "${apk_directory}/${filename}.${SUFFIX_DOWNLOADED}" + + if check_apk "${filename}.${SUFFIX_DOWNLOADED}" "$SIMPLEX_KEY"; then + printf "${COLOR_CYAN}APK for %s is signed by valid key.${COLOR_RESET}\n" "$arch" + else + printf "${COLOR_CYAN}Signature of APK for %s is invalid., aborting the script.${COLOR_RESET}\n" "$arch" + exit 1 + fi + done + + # Build phase + for arch in $ARCHES; do + case "$arch" in + armv7a) + build_tag="${tag}-armv7a" + ;; + aarch64) + build_tag="${tag}" + ;; + *) + printf "${COLOR_CYAN}Unknown architecture: %s! Skipping the build...${COLOR_RESET}\n" "$arch" + continue + esac + + # Setup the code + checkout_git "$build_directory" "$build_tag" + vercode=$(print_vercode "$build_directory") + + printf "${COLOR_CYAN}Building APK for for %s...${COLOR_RESET}\n" "$arch" + build_apk "$arch" "$vercode" + done + + # Verification phase + for arch in $ARCHES; do + filename="simplex-${arch}.apk" + + if ! verify_apk "$filename"; then + printf "${COLOR_CYAN}Failed to verify %s! Aborting.\n${COLOR_RESET}" "$filename" + exit 1 + fi + done + + printf "${COLOR_CYAN}%s is reproducible.${COLOR_RESET}\n" "$tag" + + cleanup +} + +main "$@" diff --git a/scripts/simplex-chat-reproduce-builds.sh b/scripts/simplex-chat-reproduce-builds.sh index e1a62dc73a..0ca2522fa0 100755 --- a/scripts/simplex-chat-reproduce-builds.sh +++ b/scripts/simplex-chat-reproduce-builds.sh @@ -50,6 +50,8 @@ for os in '22.04' '24.04'; do --no-cache \ --build-arg TAG="${os}" \ --build-arg GHC="${ghc}" \ + --build-arg=USER_UID="$(id -u)" \ + --build-arg=USER_GID="$(id -g)" \ -f "${tempdir}/${repo_name}/Dockerfile.build" \ -t "${image_name}" \ .