diff --git a/.github/workflows/android-emu.yml b/.github/workflows/android-emu.yml index 3bc26a5a50..f8512aeda9 100644 --- a/.github/workflows/android-emu.yml +++ b/.github/workflows/android-emu.yml @@ -250,13 +250,13 @@ jobs: unzip -l "$RUNNER_TEMP/apk/$APK" | grep -E ' lib/' - name: Upload app - if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.plan.outputs.run-claude == 'true' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' }} uses: actions/upload-artifact@v7 with: name: android-app path: ${{ runner.temp }}/apk/${{ env.APK }} compression-level: 0 - retention-days: 1 + retention-days: 14 if-no-files-found: error - name: Install emulator and system image diff --git a/.github/workflows/badge-web.yml b/.github/workflows/badge-web.yml deleted file mode 100644 index 4961c79fd1..0000000000 --- a/.github/workflows/badge-web.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: badge web - -on: - push: - branches: - - master - - stable - paths: - - "apps/simplex-badge-service/web/**" - - ".github/workflows/badge-web.yml" - pull_request: - paths: - - "apps/simplex-badge-service/web/**" - - ".github/workflows/badge-web.yml" - -jobs: - test: - runs-on: ubuntu-latest - defaults: - run: - working-directory: apps/simplex-badge-service/web - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - cache-dependency-path: apps/simplex-badge-service/web/package-lock.json - - - run: npm ci - - - run: npm test diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b828a6800a..1f28d05d74 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,14 +29,32 @@ on: - "scripts/ci/ios-stub-libs.sh" - "scripts/desktop/**" - ".github/**" + - "packages/simplex-chat-nodejs/**" + - "packages/simplex-chat-client/types/typescript/**" + - "packages/simplex-chat-python/**" + - "scripts/ci/package-tests.sh" + - "apps/simplex-badge-service/web/**" + - "scripts/ci/packages_release.py" + - "scripts/ci/test_packages_release.py" + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + tests: + description: "hspec match patterns, one per line; / runs all tests, empty runs none" + type: string + default: "" + base: + description: the branch that the changes are compared with + type: string + default: master env: - opt: ${{ github.event_name == 'pull_request' && 'O0' || 'O2' }} - ghc_options: ${{ github.event_name == 'pull_request' && '--ghc-options=-O0' || '' }} + opt: ${{ github.event_name == 'push' && 'O2' || 'O0' }} + ghc_options: ${{ github.event_name != 'push' && '--ghc-options=-O0' || '' }} release: ${{ startsWith(github.ref, 'refs/tags/v') }} branch: ${{ github.base_ref || github.ref_name }} cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }} - release_branch: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') && github.ref_name || '' }} + release_branch: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') && github.repository == 'simplex-chat/simplex-chat' && github.ref_name || '' }} ZSTD_CLEVEL: 9 # This workflow uses custom actions (prepare-build and prepare-release) defined in: @@ -104,30 +122,46 @@ jobs: publish_nodejs: ${{ steps.publish.outputs.nodejs_version }} publish_python: ${{ steps.publish.outputs.python_version }} publish: ${{ steps.publish.outputs.nodejs_version || steps.publish.outputs.python_version }} + packages: ${{ steps.changes.outputs.packages }} + scopes: ${{ steps.changes.outputs.scopes }} + reduced: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == true || (github.event_name == 'push' && github.ref_type == 'branch' && github.repository != 'simplex-chat/simplex-chat') }} steps: + - name: Reject a manual run on a tag + if: github.event_name == 'workflow_dispatch' && github.ref_type == 'tag' + run: | + echo "::error::A manual run starts on a branch; to retry a release, re-run the jobs of its tag push." + exit 1 + - name: Clone project - if: github.event_name == 'push' || github.event_name == 'pull_request' uses: actions/checkout@v3 with: fetch-depth: 2 - name: Check changes id: changes - if: github.event_name == 'push' || github.event_name == 'pull_request' shell: bash env: - BASE: ${{ github.event.pull_request.base.sha }} + BASE_BRANCH: ${{ inputs.base }} + GH_TOKEN: ${{ github.token }} run: | - if [ "$GITHUB_EVENT_NAME" = pull_request ]; then - git fetch -q --depth=1 origin "$BASE" - python3 scripts/ci/packages_release.py changes "$BASE" - else - echo "haskell=true" - if [ "$GITHUB_REF_TYPE" = branch ]; then - echo "kotlin=true" - echo "ios=true" - fi - fi >> "$GITHUB_OUTPUT" + case "$GITHUB_EVENT_NAME" in + pull_request) + python3 scripts/ci/packages_release.py changes HEAD^1 + ;; + workflow_dispatch) + base=$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_BRANCH...$GITHUB_REF_NAME" --jq .merge_base_commit.sha) + git fetch -q --depth=1 origin "$base" + python3 scripts/ci/packages_release.py changes "$base" + ;; + *) + echo "haskell=true" + if [ -n "$release_branch" ]; then + echo "kotlin=true" + echo "ios=true" + python3 scripts/ci/packages_release.py scopes + fi + ;; + esac >> "$GITHUB_OUTPUT" - name: Build changelog id: build_changelog @@ -223,16 +257,17 @@ jobs: build-linux: name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" needs: [maybe-release, variables] + if: needs.maybe-release.outputs.haskell == 'true' || needs.maybe-release.outputs.scopes != '' || (needs.maybe-release.outputs.reduced == 'true' && (needs.maybe-release.outputs.kotlin == 'true' || needs.maybe-release.outputs.packages == 'true')) runs-on: ${{ matrix.runner }} permissions: contents: write packages: write env: - image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + image_cache: ghcr.io/${{ github.repository_owner }}/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} source_hashes: dist-newstyle/sources.sha256 concurrency: &linux-concurrency group: ${{ github.workflow }}-${{ github.ref }}-linux-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ghc }}-${{ needs.maybe-release.outputs.libs_tag }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: ${{ github.event_name != 'push' }} strategy: fail-fast: false matrix: @@ -245,17 +280,12 @@ jobs: ghc: ${{ needs.variables.outputs.GHC_VER }} hash: 'sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5' libs: true - - os: 24.04 - os_underscore: 24_04 - arch: x86_64 - runner: "ubuntu-24.04" - should_run: ${{ needs.maybe-release.outputs.haskell == 'true' }} - ghc: ${{ needs.variables.outputs.GHC_VER }} - hash: 'sha256:98ff7968124952e719a8a69bb3cccdd217f5fe758108ac4f21ad22e1df44d237' - kotlin: ${{ needs.maybe-release.outputs.kotlin == 'true' }} + kotlin: ${{ needs.maybe-release.outputs.reduced == 'true' && needs.maybe-release.outputs.kotlin == 'true' }} + packages: ${{ needs.maybe-release.outputs.reduced == 'true' && needs.maybe-release.outputs.packages == 'true' }} + scope_tests: ${{ needs.maybe-release.outputs.scopes != '' }} steps: &linux-steps - name: Checkout Code - if: matrix.should_run == true || matrix.kotlin == true + if: matrix.should_run == true || matrix.kotlin == true || matrix.packages == true || matrix.scope_tests == true uses: actions/checkout@v3 with: submodules: recursive @@ -274,7 +304,7 @@ jobs: # Otherwise we run out of disk space with Docker build - name: Free disk space - if: matrix.should_run == true || matrix.kotlin == true + if: (matrix.should_run == true || matrix.kotlin == true) && needs.maybe-release.outputs.reduced != 'true' shell: bash run: ./scripts/ci/linux_util_free_space.sh @@ -388,7 +418,7 @@ jobs: - name: Restore library build directory from cache id: restore-build-lib - if: env.release != 'true' && matrix.libs && needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64' + if: env.release != 'true' && matrix.libs && needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64' && (needs.maybe-release.outputs.reduced != 'true' || matrix.packages == true) uses: actions/cache/restore@v4 with: path: | @@ -434,7 +464,7 @@ jobs: # Also for pull requests that change the chat core, for the package tests. - name: Build library id: build-lib - if: matrix.libs && (needs.maybe-release.outputs.libs_tag || (needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64')) + if: matrix.libs && (needs.maybe-release.outputs.libs_tag || (needs.maybe-release.outputs.core_changed && matrix.arch == 'x86_64' && (needs.maybe-release.outputs.reduced != 'true' || matrix.packages == true))) shell: docker exec -t builder sh -eu {0} run: | scripts/ci/prune-old-builds.sh dist-lib @@ -618,23 +648,12 @@ jobs: if: matrix.should_run == true && matrix.arch == 'x86_64' timeout-minutes: 120 shell: bash - run: | - i=1 - attempts=1 - ${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} && attempts=3 - while [ "$i" -le "$attempts" ]; do - if ./simplex-chat-test; then - break - else - echo "Attempt $i failed, retrying..." - i=$((i + 1)) - sleep 1 - fi - done - if [ "$i" -gt "$attempts" ]; then - echo "All "$attempts" attempts failed." - exit 1 - fi + env: + REDUCED: ${{ needs.maybe-release.outputs.reduced }} + TESTS: ${{ github.event_name == 'workflow_dispatch' && inputs.tests || '' }} + PR_BODY: ${{ github.event.pull_request.draft == true && github.event.pull_request.body || '' }} + ATTEMPTS: ${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) && 3 || 1 }} + run: scripts/ci/run-tests.sh ./simplex-chat-test # Compiling the desktop and Android targets of :common in parallel exceeds the 2 GB Kotlin daemon heap. # cmakeBuildAndCopy links the JNI library against libsimplex, which this job does not build. @@ -652,7 +671,7 @@ jobs: - name: Setup Node.js id: setup-node - if: ${{ !cancelled() && matrix.should_run == true && matrix.libs && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} + if: ${{ !cancelled() && (matrix.packages == true || matrix.scope_tests == true || (matrix.should_run == true && matrix.libs && ((github.event_name == 'pull_request' && needs.maybe-release.outputs.reduced != 'true') || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish))) }} uses: actions/setup-node@v7 with: node-version: 22 @@ -665,7 +684,7 @@ jobs: python-version: "3.12" - name: Package tests (pull request) - if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' && steps.build-lib.outcome != 'failure' }} + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name != 'push' && steps.build-lib.outcome != 'failure' && ((needs.maybe-release.outputs.reduced != 'true' && matrix.should_run == true) || matrix.packages == true) }} timeout-minutes: 30 shell: bash env: @@ -701,6 +720,17 @@ jobs: name: package-tests-linux-${{ matrix.arch }} path: ${{ runner.temp }}/package-tests + - name: Test changed scopes + if: ${{ !cancelled() && matrix.scope_tests == true && steps.setup-python.outcome == 'success' }} + timeout-minutes: 30 + shell: bash + env: + SCOPES: ${{ needs.maybe-release.outputs.scopes }} + run: | + for scope in $SCOPES; do + scripts/ci/scope-tests.sh "$scope" + done + - name: Remove -O2 build directory id: remove-o2 if: env.cache_o0 == 'true' && !cancelled() && steps.build-cli.outcome == 'success' @@ -746,16 +776,42 @@ jobs: !dist-newstyle/src key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }} - build-linux-push: + build-linux-full: name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" needs: [maybe-release, variables] - if: github.event_name != 'pull_request' + if: needs.maybe-release.outputs.reduced != 'true' && (needs.maybe-release.outputs.haskell == 'true' || needs.maybe-release.outputs.kotlin == 'true') runs-on: ${{ matrix.runner }} permissions: contents: write packages: write env: - image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + image_cache: ghcr.io/${{ github.repository_owner }}/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + source_hashes: dist-newstyle/sources.sha256 + concurrency: *linux-concurrency + strategy: + fail-fast: false + matrix: + include: + - os: 24.04 + os_underscore: 24_04 + arch: x86_64 + runner: "ubuntu-24.04" + should_run: ${{ needs.maybe-release.outputs.haskell == 'true' }} + ghc: ${{ needs.variables.outputs.GHC_VER }} + hash: 'sha256:98ff7968124952e719a8a69bb3cccdd217f5fe758108ac4f21ad22e1df44d237' + kotlin: ${{ needs.maybe-release.outputs.kotlin == 'true' }} + steps: *linux-steps + + build-linux-push: + name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" + needs: [maybe-release, variables] + if: needs.maybe-release.outputs.reduced != 'true' && github.event_name == 'push' + runs-on: ${{ matrix.runner }} + permissions: + contents: write + packages: write + env: + image_cache: ghcr.io/${{ github.repository_owner }}/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} source_hashes: dist-newstyle/sources.sha256 cache_o0: false concurrency: *linux-concurrency @@ -785,13 +841,13 @@ jobs: build-linux-ghc8: name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" needs: [maybe-release, variables] - if: ${{ needs.maybe-release.outputs.haskell == 'true' && !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} + if: ${{ needs.maybe-release.outputs.reduced != 'true' && needs.maybe-release.outputs.haskell == 'true' && !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} runs-on: ${{ matrix.runner }} permissions: contents: write packages: write env: - image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + image_cache: ghcr.io/${{ github.repository_owner }}/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} source_hashes: dist-newstyle/sources.sha256 concurrency: *linux-concurrency strategy: @@ -930,6 +986,7 @@ jobs: build-macos: name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" needs: [maybe-release, variables] + if: needs.maybe-release.outputs.reduced != 'true' && needs.maybe-release.outputs.haskell == 'true' runs-on: ${{ matrix.os }} concurrency: group: ${{ github.workflow }}-${{ github.ref }}-macos-${{ matrix.os }}-${{ needs.maybe-release.outputs.libs_tag }} @@ -945,7 +1002,6 @@ jobs: desktop_asset_name: simplex-desktop-macos-aarch64.dmg openssl_dir: "/opt/homebrew/opt" should_run: ${{ needs.maybe-release.outputs.haskell == 'true' }} - ios: ${{ needs.maybe-release.outputs.ios == 'true' }} - os: macos-15-intel arch: x86_64 ghc: ${{ needs.variables.outputs.GHC_VER }} @@ -955,13 +1011,13 @@ jobs: should_run: ${{ needs.maybe-release.outputs.haskell == 'true' }} steps: - name: Checkout Code - if: matrix.should_run == true || matrix.ios == true + if: matrix.should_run == true uses: actions/checkout@v3 with: submodules: recursive - name: Point Command Line Tools SDK to Xcode SDK - if: matrix.should_run == true || matrix.ios == true + if: matrix.should_run == true shell: bash run: sudo ln -sfn "$(xcrun --sdk macosx --show-sdk-path)" /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk @@ -1183,23 +1239,6 @@ jobs: exit 1 fi - - name: Build iOS app - if: ${{ !cancelled() && matrix.ios == true }} - timeout-minutes: 60 - shell: bash - run: | - scripts/ci/ios-stub-libs.sh - xcodebuild build -quiet \ - -project apps/ios/SimpleX.xcodeproj \ - -scheme "SimpleX (iOS)" \ - -configuration Debug \ - -destination "generic/platform=iOS Simulator" \ - -onlyUsePackageVersionsFromResolvedFile \ - -skipPackagePluginValidation \ - -skipMacroValidation \ - ARCHS=arm64 \ - CODE_SIGNING_ALLOWED=NO - - name: Build library id: mac_lib_build if: ${{ !cancelled() && matrix.should_run == true && needs.maybe-release.outputs.libs_tag && steps.mac_cli_build.outcome == 'success' }} @@ -1309,6 +1348,38 @@ jobs: !dist-newstyle/src key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }} + build-ios: + name: "macos-latest (iOS)" + needs: maybe-release + if: needs.maybe-release.outputs.ios == 'true' + runs-on: macos-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-ios + cancel-in-progress: ${{ github.event_name != 'push' }} + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Point Command Line Tools SDK to Xcode SDK + shell: bash + run: sudo ln -sfn "$(xcrun --sdk macosx --show-sdk-path)" /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk + + - name: Build iOS app + timeout-minutes: 60 + shell: bash + run: | + scripts/ci/ios-stub-libs.sh + xcodebuild build -quiet \ + -project apps/ios/SimpleX.xcodeproj \ + -scheme "SimpleX (iOS)" \ + -configuration Debug \ + -destination "generic/platform=iOS Simulator" \ + -onlyUsePackageVersionsFromResolvedFile \ + -skipPackagePluginValidation \ + -skipMacroValidation \ + ARCHS=arm64 \ + CODE_SIGNING_ALLOWED=NO + # ========================= # Windows Build # ========================= @@ -1316,7 +1387,7 @@ jobs: build-windows: name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" needs: [maybe-release, variables] - if: needs.maybe-release.outputs.haskell == 'true' + if: needs.maybe-release.outputs.reduced != 'true' && needs.maybe-release.outputs.haskell == 'true' runs-on: ${{ matrix.os }} concurrency: group: ${{ github.workflow }}-${{ github.ref }}-windows-${{ matrix.os }}-${{ needs.maybe-release.outputs.libs_tag }} @@ -1622,7 +1693,7 @@ jobs: # ========================= releases: - needs: [maybe-release, build-linux, build-linux-push, build-linux-postgres, build-macos, build-windows] + needs: [maybe-release, build-linux, build-linux-full, build-linux-push, build-linux-postgres, build-macos, build-ios, build-windows] if: ${{ !cancelled() && (needs.maybe-release.outputs.libs_tag || needs.maybe-release.outputs.addon_draft) }} runs-on: ubuntu-latest env: @@ -1646,7 +1717,7 @@ jobs: # publish under other names. publish: - needs: [maybe-release, build-linux, build-linux-push, build-macos, build-windows, releases] + needs: [maybe-release, build-linux, build-linux-full, build-linux-push, build-macos, build-ios, build-windows, releases] if: ${{ !cancelled() && (needs.maybe-release.outputs.publish || needs.maybe-release.outputs.publish_types) }} runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/desktop-linux.yml b/.github/workflows/desktop-linux.yml index 82725791f5..ce5a1c7922 100644 --- a/.github/workflows/desktop-linux.yml +++ b/.github/workflows/desktop-linux.yml @@ -253,19 +253,19 @@ jobs: du -sh "$app" LD_LIBRARY_PATH="$app/lib/app/resources" ldd "$app/lib/app/resources/libapp-lib.so" - # for the Claude job; a tarball keeps the app's file modes, which an artifact drops + # a tarball keeps the app's file modes, which an artifact drops - name: Pack app - if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.plan.outputs.run-claude == 'true' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' }} run: tar -czf "$RUNNER_TEMP/desktop-app.tar.gz" -C apps/multiplatform/release/main/app simplex - name: Upload app - if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.plan.outputs.run-claude == 'true' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' }} uses: actions/upload-artifact@v7 with: name: desktop-app path: ${{ runner.temp }}/desktop-app.tar.gz compression-level: 0 - retention-days: 1 + retention-days: 14 if-no-files-found: error - name: Run desktop tests diff --git a/.github/workflows/ios-sim.yml b/.github/workflows/ios-sim.yml index 08502674d9..b6477018e9 100644 --- a/.github/workflows/ios-sim.yml +++ b/.github/workflows/ios-sim.yml @@ -249,19 +249,17 @@ jobs: DEVELOPMENT_TEAM= \ PROVISIONING_PROFILE_SPECIFIER= - # for the Claude job; a tarball keeps the app's file modes, which an artifact drops + # a tarball keeps the app's file modes, which an artifact drops - name: Pack app - if: needs.plan.outputs.run-claude == 'true' run: tar -czf "$RUNNER_TEMP/SimpleX.app.tar.gz" -C "$RUNNER_TEMP/DerivedData/Build/Products/Debug-iphonesimulator" SimpleX.app - name: Upload app - if: needs.plan.outputs.run-claude == 'true' uses: actions/upload-artifact@v7 with: name: ios-sim-app path: ${{ runner.temp }}/SimpleX.app.tar.gz compression-level: 0 - retention-days: 1 + retention-days: 14 if-no-files-found: error - name: Install AXe diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ca22698096..12fc5f5862 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -7,6 +7,7 @@ on: - "packages/simplex-chat-client/types/typescript/**" - "packages/simplex-chat-python/**" - ".github/workflows/packages.yml" + types: [opened, synchronize, reopened, ready_for_review] concurrency: # build.yml and this workflow both run for a pull request that changes the core and the packages @@ -23,20 +24,19 @@ env: jobs: addon: + if: github.event.pull_request.draft == false runs-on: ubuntu-latest outputs: platforms: ${{ steps.plan.outputs.platforms }} steps: - name: Checkout Code uses: actions/checkout@v6 + with: + fetch-depth: 2 - name: Build plan id: plan - env: - BASE: ${{ github.event.pull_request.base.sha }} - run: | - git fetch -q --depth=1 origin "$BASE" - python3 scripts/ci/packages_release.py build-plan --base "$BASE" >> "$GITHUB_OUTPUT" + run: python3 scripts/ci/packages_release.py build-plan --base HEAD^1 >> "$GITHUB_OUTPUT" build: name: "build ${{ matrix.platform }}" diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 5fbe8293bc..7628eb1f9a 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -33,6 +33,7 @@ jobs: ./website/web.sh - name: Deploy + if: github.repository == 'simplex-chat/simplex-chat' uses: simplex-chat/actions-gh-pages@v3 with: publish_dir: ./website/_site diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..2c3cceced7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,4 @@ +@docs/CONTRIBUTING.md +@docs/contributing/PROJECT.md +@docs/contributing/CODE.md +@scripts/ci/README.md diff --git a/apps/ios/CLAUDE.md b/apps/ios/CLAUDE.md new file mode 100644 index 0000000000..c2722350fa --- /dev/null +++ b/apps/ios/CLAUDE.md @@ -0,0 +1,12 @@ +@README.md + +## Swift code + +- Functions and computed properties are declared in the body of the type they belong to, never at file scope or in an extension of a type declared in this project. +- A member of a SimpleXChat type is declared in that type when everything it uses is available in SimpleXChat. A member that depends on the app is a static member of the app type that uses it; when several app types use it, of the type that names its concept. +- An extension is written only where Swift has no other form: + - a SwiftUI environment value, which SwiftUI requires as a property of EnvironmentValues; + - a conformance, required by an API, of a type declared in another module. +- A view modifier is a ViewModifier type applied with `.modifier(_:)`. +- A custom alignment is a static constant of a non-generic type that uses it. +- A new member is declared in its type even when the file already extends that type. Existing extensions and file-scope functions are moved only in a task that concerns them. diff --git a/apps/multiplatform/CLAUDE.md b/apps/multiplatform/CLAUDE.md new file mode 100644 index 0000000000..99ee350f47 --- /dev/null +++ b/apps/multiplatform/CLAUDE.md @@ -0,0 +1 @@ +@README.md diff --git a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml index 565554f7e8..4d56cccf7b 100644 --- a/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml +++ b/apps/multiplatform/common/src/commonMain/moko-resources/base/strings.xml @@ -697,6 +697,7 @@ File will be received when your contact completes uploading it. File will be received when your contact is online, please wait or check later! File saved + File already exists. Replace it? File not found Error saving file Formatted view diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Files.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Files.desktop.kt index b779954efc..9720a78435 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Files.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Files.desktop.kt @@ -75,9 +75,10 @@ actual class FileChooserLauncher actual constructor() { ) res = simplexWindowState.openDialog.awaitResult(params) } else { - res = simplexWindowState.saveDialog.awaitResult(DialogParams(filename = input)) + val filename = saveDialogFileName(input) + res = simplexWindowState.saveDialog.awaitResult(DialogParams(filename = filename)) if (res != null && res.isDirectory) { - res = File(res, input) + res = File(res, filename) } } onResult(res?.toURI()) @@ -115,5 +116,7 @@ private fun fileFilterDescription(input: String): String = when(input) { else -> "" } +internal fun saveDialogFileName(name: String): String = name.trimStart('.').ifEmpty { name } + actual fun URI.inputStream(): InputStream? = toFile().inputStream() actual fun URI.outputStream(): OutputStream = toFile().outputStream() diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/helpers/DefaultDialog.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/helpers/DefaultDialog.desktop.kt index 4f26f1d5fe..7b5d0ae371 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/helpers/DefaultDialog.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/helpers/DefaultDialog.desktop.kt @@ -17,6 +17,7 @@ import java.awt.FileDialog import java.awt.event.ActionListener import java.io.File import javax.swing.JFileChooser +import javax.swing.JOptionPane import javax.swing.filechooser.FileFilter import javax.swing.filechooser.FileNameExtensionFilter import javax.swing.plaf.basic.BasicFileChooserUI @@ -75,7 +76,21 @@ fun FrameWindowScope.FileDialogChooserMultiple( val scope = rememberCoroutineScope() DisposableEffect(Unit) { val job = scope.launch(Dispatchers.Main) { - val fileChooser = JFileChooser() + val fileChooser = object: JFileChooser() { + override fun approveSelection() { + if (!isLoad && selectedFile?.isFile == true) { + val answer = JOptionPane.showConfirmDialog( + this, + generalGetString(MR.strings.file_already_exists_replace_question), + dialogTitle, + JOptionPane.YES_NO_OPTION, + JOptionPane.WARNING_MESSAGE + ) + if (answer != JOptionPane.YES_OPTION) return + } + super.approveSelection() + } + } fileChooser.dialogTitle = title fileChooser.isMultiSelectionEnabled = allowMultiple && isLoad fileChooser.isAcceptAllFileFilterUsed = fileFilter == null diff --git a/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/SaveDialogFileNameTest.kt b/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/SaveDialogFileNameTest.kt new file mode 100644 index 0000000000..21b3d2e341 --- /dev/null +++ b/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/SaveDialogFileNameTest.kt @@ -0,0 +1,20 @@ +package chat.simplex.app + +import chat.simplex.common.platform.saveDialogFileName +import kotlin.test.Test +import kotlin.test.assertEquals + +class SaveDialogFileNameTest { + @Test + fun testLeadingDotsRemoved() { + assertEquals("bashrc", saveDialogFileName(".bashrc")) + assertEquals("hidden.txt", saveDialogFileName("..hidden.txt")) + } + + @Test + fun testOtherNamesUnchanged() { + assertEquals("report.pdf", saveDialogFileName("report.pdf")) + assertEquals("archive.tar.gz", saveDialogFileName("archive.tar.gz")) + assertEquals("...", saveDialogFileName("...")) + } +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 7dbf4510e9..9fe9079403 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -21,16 +21,7 @@ Please discuss the problem you want to solve and your detailed implementation pl ./contributing/CODE.md has details about general requirements common for `simplexmq` and `simplex-chat` repositories. -These files can be used with LLM prompts, e.g. if you use Claude Code you can create CLAUDE.md file in project root importing content from these files: - -```markdown -@README.md -@docs/CONTRIBUTING.md -@docs/contributing/PROJECT.md -@docs/contributing/CODE.md -``` - -For Android/Desktop and iOS apps you can additionally import `apps/multiplatform/README.md` and `apps/ios/README.md`. +CLAUDE.md in the repository root imports these files, this guide and ../scripts/ci/README.md for Claude Code; apps/ios/CLAUDE.md and apps/multiplatform/CLAUDE.md import the README of each app. ## Compiling with SQLCipher encryption enabled @@ -94,6 +85,15 @@ We squash PRs, do not rewrite branch history after the review. For some complex features we create feature branches that will be merged once ready - do not make commits directly to them, make PRs to feature branches. +## Pull requests and CI + +1. Open a pull request to `master` as a draft. On a draft, CI runs the reduced build: one Linux job that builds and tests the changed scopes, and one macOS job only when iOS code changed (CI guide: scripts/ci/README.md). +2. Select the core tests in a `ci-tests` block in the description, one hspec match pattern per line. `/` runs all tests; without a block, the reduced build runs no core tests. +3. Push the changes for review comments to the same branch; each push runs the reduced build. +4. Reviewers approve the draft. A draft cannot be merged. +5. After the approval, mark the pull request ready for review. CI then runs the full matrix, and the pull request is merged by squash once it passes. +6. A change to a scope without CI tests adds its build and tests to CI in the same pull request. + ## Differences between GHC 8.10.7 and GHC 9.6.3 1. The main difference is related to `DuplicateRecordFields` extension. diff --git a/docs/contributing/CODE.md b/docs/contributing/CODE.md index 1dcf795c00..fcdbeb4081 100644 --- a/docs/contributing/CODE.md +++ b/docs/contributing/CODE.md @@ -4,6 +4,12 @@ This file provides guidance on coding style and approaches and on building the c ## Code Security +Zero trust: providers and server owners cannot access users' data - design every feature with this constraint. Every code and design decision considers two threat vectors: +1. users attacking providers; +2. providers or server owners attacking users. + +Default to end-to-end encryption, minimal metadata exposure, and cryptographic verification over trust assumptions. Review the documented threat models and monitor compliance. + When designing code and planning implementations: - Apply adversarial thinking, and consider what may happen if one of the communicating parties is malicious. - Formulate an explicit threat model for each change - who can do which undesirable things and under which circumstances. diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 7034ea8e7e..e3096b71fe 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -1,14 +1,158 @@ -# App tests on GitHub's runners +# CI on GitHub's runners Three workflows build the apps from a branch's own code and test them, only when asked: `ios-sim` (the iOS app in a simulator on a macOS runner), `desktop-linux` (the desktop app on a Linux runner) and `android-emu` (the Android app in an emulator on a Linux runner). Each runs a scripted smoke test and, on request, a Claude session that follows a test plan, and a test backend (relay, directory, names server, peers). Details: [iOS](ios-sim/README.md), [desktop](desktop-linux/README.md), [Android](android/README.md). +## Reduced build + +GitHub runs the reduced build of `build.yml` on a dispatch (workflow_dispatch), on each push to a draft pull request and on a push to master or stable in a fork. Once the pull request is marked ready for review, GitHub runs the full matrix, as on a push to master or stable in simplex-chat/simplex-chat. + +On a push to master or stable in a fork, the release steps and tests are skipped; build-linux saves the -O0 build caches, and on master the Docker image cache, that the fork's dispatches restore. + +On a draft pull request, the tests are selected by a fenced block with the info string `ci-tests` in the description, one hspec match pattern per line; the block is read at each push. `/` runs the whole suite; without a block, the reduced build runs no tests. + +### Dispatch + +```sh +curl -fsS -X POST \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos//simplex-chat/actions/workflows/build.yml/dispatches \ + -d '{"ref": "", "inputs": {"tests": "/SimpleX chat client/group tests/\n/Message batching/"}}' +``` + +Endpoint: https://docs.github.com/en/rest/actions/workflows#create-a-workflow-dispatch-event. Inputs: + +| Input | Default | Value | +|---|---|---| +| `tests` | empty | hspec match patterns, one per line (`\n` in JSON); `/` runs the whole suite, and an empty value builds without tests | +| `base` | `master` | the branch that the changes are compared with:
`stable` for a branch cut from stable
the parent branch for a stacked branch | + +When a newer dispatch on the same branch starts build-linux or build-ios, GitHub cancels the unfinished job of the older one (`cancel-in-progress`). + +Results: +- Run and jobs: `GET /repos//simplex-chat/actions/runs?head_sha=&event=workflow_dispatch` returns the run id; `GET /repos//simplex-chat/actions/runs//jobs` returns each step with its start, end and conclusion. +- Job log: `GET /repos//simplex-chat/actions/jobs//logs` returns a redirect to a text file, valid for 1 minute (https://docs.github.com/en/rest/actions/workflow-jobs#download-job-logs-for-a-workflow-run). The test summary is printed as "Finished in N seconds" and "N examples, N failures". + +### Scopes + +The reduced build runs one Linux job, build-linux (ubuntu-22.04 x86_64), for every scope except ios, and one macOS job, build-ios (macos-latest), for ios. + +#### core + +- Globs: `BUILD_CHANGES["haskell"]` in packages_release.py. +- CI, in build-linux: + - Docker image (step "Build and cache Docker image"); + - caches (the steps "Restore ... from cache"); + - `cabal build -j --enable-tests --ghc-options=-O0` (step "Build CLI"); + - simplex-chat-test with the selected tests (step "Run tests"). +- Local: + ```sh + cp scripts/cabal.project.local.linux cabal.project.local + cabal build --enable-tests --ghc-options=-O0 + mkdir -p tests/tmp + cabal test simplex-chat-test --ghc-options=-O0 --test-option=-m --test-option='/SimpleX chat client/group tests/' + ``` + The selection runs as in CI with scripts/ci/run-tests.sh, which appends the patterns of `TESTS` to the test binary: + ```sh + TESTS='/SimpleX chat client/group tests/' scripts/ci/run-tests.sh "$(cabal list-bin simplex-chat-test --ghc-options=-O0)" + ``` + `--ghc-options=-O0` matches the reduced build; the library and test-suite stanzas of simplex-chat.cabal set -O2. Test directories are created under tests/tmp (`tmpBracket` in tests/Test.hs). + +#### kotlin + +- Globs: `BUILD_CHANGES["kotlin"]` in packages_release.py. +- CI: Gradle in the Docker container of build-linux (step "Build and test Kotlin apps", matrix field `kotlin`). +- Local: + ```sh + cd apps/multiplatform + ./gradlew --no-daemon --max-workers=1 :common:desktopTest :desktop:compileKotlinJvm -x :desktop:cmakeBuildAndCopy :android:compileGoogleDebugKotlin :android:testFossDebugUnitTest + ``` + JDK 17 and an Android SDK in ANDROID_HOME are required, as installed by Dockerfile.build. + +#### ios + +- Globs: + - apps/ios/** + - scripts/ci/ios-stub-libs.sh +- CI: build-ios: stub libraries, then xcodebuild for the iOS Simulator without signing. +- Local, on macOS: `scripts/ci/ios-stub-libs.sh`, then the xcodebuild command of the step "Build iOS app" of build-ios. + +#### packages + +- Globs: + - packages/simplex-chat-nodejs/** + - packages/simplex-chat-client/types/typescript/** + - packages/simplex-chat-python/** + - scripts/ci/package-tests.sh +- CI, in build-linux: + - Node 22 and Python 3.12 (steps "Setup Node.js" and "Setup Python"); + - libsimplex, when `CORE_SOURCES` of packages_release.py also changed (step "Build library"); + - `package-tests.sh build linux-x86_64` and `package-tests.sh test` (step "Package tests (pull request)"). +- With libsimplex built, SIMPLEX_LIBS_DIR is set to it for the nodejs tests; the python tests run with it unset (package-tests.sh). Without SIMPLEX_LIBS_DIR, the pinned release libraries are downloaded (packages/simplex-chat-nodejs/src/libs.ts). +- Local: + ```sh + scripts/ci/package-tests.sh build linux-x86_64 + scripts/ci/package-tests.sh test + ``` + +#### badge-web + +- Globs: apps/simplex-badge-service/web/**. These paths are also in the core globs (apps/simplex-badge-service/**), and web/src/catalog.ts is read by core tests (tests/Bots/BadgeService/CatalogTests.hs, WebTests.hs). +- CI, in build-linux: core build and tests, then `scope-tests.sh badge-web` (step "Test changed scopes"). +- Selected core tests: `/SimpleX badge service/` and `/SimpleX badge service e2e/`. +- Local: + ```sh + cd apps/simplex-badge-service/web + npm ci + npm test + ``` + +#### ci-scripts + +- Globs: + - scripts/ci/packages_release.py + - scripts/ci/test_packages_release.py +- These paths are also in the core globs (scripts/ci/**). +- CI, in build-linux: `scope-tests.sh ci-scripts` (step "Test changed scopes"). +- Local: `python3 scripts/ci/test_packages_release.py`; Python 3.11 or newer is required for tomllib (packages_release.py). + +#### Scopes lacking CI tests + +| Scope | Paths | CI | +|---|---|---| +| ios | apps/ios/** | compile only | +| types package | packages/simplex-chat-client/types/typescript/** | compile only | +| website | website/**
images/**
blog/**
docs/**
PRIVACY.md | build and deploy on master pushes (web.yml) | +| ts-client | packages/simplex-chat-client/typescript/** | - | +| webrtc | packages/simplex-chat-webrtc/** | - | +| mcp | packages/simplex-chat-mcp/** | - | +| calculator-bot | apps/simplex-calculator-bot/** | - | +| support-bot | apps/simplex-support-bot/** | - | +| support-bot-light | apps/simplex-support-bot-light/** | - | +| prices | scripts/prices/** | - | +| nft | eth/nft/** | - | + +CI coverage of such a scope is added in the pull request that changes it (Adding CI coverage). + +### Adding CI coverage + +For a scope lacking CI tests, in the pull request that changes it: +1. scripts/ci/packages_release.py: an entry `"": ["", ...]` in SCOPE_CHANGES. +2. build.yml, `on.pull_request.paths`: the same globs; the two are compared in `test_build_yml_runs_for_the_paths_of_its_builds` of test_packages_release.py. +3. scripts/ci/scope-tests.sh: a `)` case with the build and test commands, and the scope name in the usage line. +4. scripts/ci/test_packages_release.py, ChangesTest: a case asserting that the scope is printed in `scopes` for a change to one of its files. +5. This file, "Scopes": a subsection for the scope, in place of its row in "Scopes lacking CI tests". +6. Checks: `python3 scripts/ci/test_packages_release.py`, the new case of scope-tests.sh locally, then a dispatch of the reduced build. + +The scope commands run in build-linux after the core tests, with Node 22 and Python 3.12 (steps "Setup Node.js" and "Setup Python"). For a scope that needs other versions, a setup step is added before "Test changed scopes", conditioned on the scope name in `needs.maybe-release.outputs.scopes`. A scope that needs macOS is a BUILD_CHANGES entry with a separate maybe-release output, as ios is, and a step in build-ios; the `if` of build-ios includes that output. + ## Starting a run -Pushes start nothing: a run is started by hand, on any branch, with inputs that are off by default and combine. +Pushes start none of the app workflows: a run is started by hand, on any branch, with inputs that are off by default and combine. | Input | iOS | Desktop | Android | |---|---|---|---| -| no input | scripted test | scripted test | scripted test | +| no input | scripted test, the app uploaded | scripted test, the app uploaded | scripted test, the app uploaded | | `claude` | scripted test, then Claude | scripted test, then Claude | scripted test, then Claude | | `backend` | also the test backend | also the test backend | also the test backend | | `release` | — | also the latest release's onboarding, to tell a branch's failure from the runner's | also the latest release's onboarding, to tell a branch's failure from the runner's | @@ -24,6 +168,8 @@ gh workflow run android-emu.yml --ref -f claude=true A newer run of a workflow on the same branch cancels the older one. +Each run uploads the app it built, for manual tests, as an artifact kept 14 days: `ios-sim-app` (SimpleX.app.tar.gz, for the iOS simulator on an Apple-silicon Mac), `desktop-app` (desktop-app.tar.gz, for Linux x86-64) and `android-app` (the debug APK, arm64-v8a only). The run's page lists it under Artifacts; `gh run download -n ` downloads it. + ## How long a run takes | Run | iOS | Desktop | Android | diff --git a/scripts/ci/android/README.md b/scripts/ci/android/README.md index b18e79a619..ceac2a1b08 100644 --- a/scripts/ci/android/README.md +++ b/scripts/ci/android/README.md @@ -4,7 +4,7 @@ ## Running it -The workflow runs only when started by hand, on any branch: in the Actions tab, `android-emu` → Run workflow, or `gh workflow run android-emu.yml --ref ` with `-f claude=true`, `-f backend=true` or `-f release=true`. [App tests on GitHub's runners](../README.md) lists the inputs. A newer run on the same branch cancels the older one. +The workflow runs only when started by hand, on any branch: in the Actions tab, `android-emu` → Run workflow, or `gh workflow run android-emu.yml --ref ` with `-f claude=true`, `-f backend=true` or `-f release=true`. [CI on GitHub's runners](../README.md) lists the inputs. A newer run on the same branch cancels the older one. Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged, as an artifact of the same branch named after a hash of its sources (30 days): for the core `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md`, the Nix files (`flake.nix`, `flake.lock`, `scripts/nix/**`) and `build-core.sh`; the backend is desktop-linux's, under its name and hash, so a backend either workflow built is reused by both. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`. @@ -19,7 +19,7 @@ Every run builds the core library, and the backend when it's asked for, from the | `backend` (opt-in) | desktop-linux's `build-backend.sh` | about 30 min, skipped when reused | | `release` (opt-in) | the latest release's arm64 APK, its signer checked against SimpleX's key, onboarding only | about 4 min | -Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for the Claude job, is kept a day. The job summaries list the checks and the Claude usage. +Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. ## The emulator diff --git a/scripts/ci/desktop-linux/README.md b/scripts/ci/desktop-linux/README.md index 2363068a63..90143005e8 100644 --- a/scripts/ci/desktop-linux/README.md +++ b/scripts/ci/desktop-linux/README.md @@ -4,7 +4,7 @@ ## Running it -The workflow runs only when started by hand, on any branch, from the Actions tab or with `gh workflow run`: [App tests on GitHub's runners](../README.md) says how and lists the inputs. +The workflow runs only when started by hand, on any branch, from the Actions tab or with `gh workflow run`: [CI on GitHub's runners](../README.md) says how and lists the inputs. Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged, as an artifact of the same branch named after a hash of its sources (30 days): for the core `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md` and the build scripts, for the backend `src/**`, the CLI's and the directory's sources, `cabal.project`, `simplex-chat.cabal` and its build script. So a branch that changes core and app code together tests both. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`. @@ -19,7 +19,7 @@ Every run builds the core library, and the backend when it's asked for, from the | `backend` (opt-in) | `build-backend.sh`: `simplex-chat`, `simplex-directory-service`, and `smp-server` from the simplexmq commit `cabal.project` pins | about 30 min, skipped when reused | | `release` (opt-in) | the latest release's .deb for this Ubuntu, onboarding only, to tell a branch's failure from the runner's | 1.5 min | -Queueing for a runner adds from under a minute to 20 minutes. Results are artifacts: `desktop-smoke-test` (numbered screenshots, the text OCR read from each, `steps.log`, `checks.txt`, `app.log`), `desktop-test-report`, `desktop-claude-test`, and the backend's logs; `desktop-app`, the distributable for the Claude job, is kept a day. The job summaries list the checks, the desktopTest results and the Claude usage. +Queueing for a runner adds from under a minute to 20 minutes. Results are artifacts: `desktop-smoke-test` (numbered screenshots, the text OCR read from each, `steps.log`, `checks.txt`, `app.log`), `desktop-test-report`, `desktop-claude-test`, and the backend's logs; `desktop-app`, the distributable for manual tests and the Claude job, is kept 14 days. The job summaries list the checks, the desktopTest results and the Claude usage. ## Scripts diff --git a/scripts/ci/ios-sim/README.md b/scripts/ci/ios-sim/README.md index f72c6cb0b2..6bc7d673e4 100644 --- a/scripts/ci/ios-sim/README.md +++ b/scripts/ci/ios-sim/README.md @@ -4,7 +4,7 @@ ## Running it -The workflow runs only when started by hand, on any branch, from the Actions tab or with `gh workflow run`: [App tests on GitHub's runners](../README.md) says how and lists the inputs. +The workflow runs only when started by hand, on any branch, from the Actions tab or with `gh workflow run`: [CI on GitHub's runners](../README.md) says how and lists the inputs. Results: the job logs, the Claude job's summary (Claude's usage) and the artifacts `ios-sim-smoke-test` (a screenshot, a 1000-px view and an accessibility tree per step, the app log), `ios-sim-backend-logs` and `ios-sim-claude-test` (the Claude job's smoke test in `smoke-test/`, and per session: `prompt.md`, `report.md`, `shots/`, `transcript.jsonl`, `usage.json`). Read them with `gh run view --log` and `gh run download -n `. @@ -22,7 +22,7 @@ The macOS jobs run on `macos-26`, the image with Xcode 26 and the iPhone 17 simu ## The Claude session -It runs in its own job, `claude`, once the app job has passed: the job installs that job's build (the artifact `ios-sim-app`, kept a day) in a new simulator, runs the smoke test to reach the sessions' starting state, then one session per layout listed in `claude-scope`, with Claude Code installed from npm at the version the desktop workflow pins too. Only this job references the environment `app-sim-claude`, which holds the secret `CLAUDE_CODE_OAUTH_TOKEN` (a Claude subscription token) and admits the branches its rule allows; one runs per branch at a time. Only the Claude step gets the token, and `claude-test.sh` passes it to the `claude` processes alone. The job checks out without persisting its GitHub token and installs a pinned AXe (`install-axe.sh`), since it holds the Claude token. It starts the test backend when `claude-scope` says `backend: yes`; a run that asks for Claude must then ask for the backend too, or the plan job fails. On master it says `no`, since the smoke plan doesn't use the backend. +It runs in its own job, `claude`, once the app job has passed: the job installs that job's build (the artifact `ios-sim-app`, kept 14 days) in a new simulator, runs the smoke test to reach the sessions' starting state, then one session per layout listed in `claude-scope`, with Claude Code installed from npm at the version the desktop workflow pins too. Only this job references the environment `app-sim-claude`, which holds the secret `CLAUDE_CODE_OAUTH_TOKEN` (a Claude subscription token) and admits the branches its rule allows; one runs per branch at a time. Only the Claude step gets the token, and `claude-test.sh` passes it to the `claude` processes alone. The job checks out without persisting its GitHub token and installs a pinned AXe (`install-axe.sh`), since it holds the Claude token. It starts the test backend when `claude-scope` says `backend: yes`; a run that asks for Claude must then ask for the backend too, or the plan job fails. On master it says `no`, since the smoke plan doesn't use the backend. Each session's final message ends with `CLAUDE-TEST: PASS`, or `CLAUDE-TEST: FAIL` and the failed cases; the step fails unless every session passes. diff --git a/scripts/ci/packages_release.py b/scripts/ci/packages_release.py index 534ab6241f..393fc85c5d 100644 --- a/scripts/ci/packages_release.py +++ b/scripts/ci/packages_release.py @@ -10,7 +10,8 @@ publish-release publish a draft release that has every asset check-pins check the pins of the packages against the published releases download download a release asset and check its digest download-pinned download a pinned asset and check its pin -changes print whether the changes since a commit touch the chat core, the add-on and each build of build.yml +changes print whether the changes since a commit touch the chat core, the add-on, each build of build.yml and each scope of scope-tests.sh +scopes print every scope of scope-tests.sh build-plan print the platforms to test the packages on, none when build.yml builds the core for the pull request prepare update pins and versions of the packages publish-plan print the package versions that are not published yet @@ -47,7 +48,7 @@ RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts" PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py" ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"] CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"] -# The pull_request paths of build.yml are the union of these globs; ":!" excludes a path from its build. +# The pull_request paths of build.yml are the union of the globs of BUILD_CHANGES and SCOPE_CHANGES; ":!" excludes a path from its build. BUILD_CHANGES = { "haskell": [ "src/**", @@ -68,7 +69,17 @@ BUILD_CHANGES = { ".github/**", ], "kotlin": ["apps/multiplatform/**", "Dockerfile*", ".github/**"], - "ios": ["apps/ios/**", "scripts/ci/ios-stub-libs.sh", ".github/**"], + "ios": ["apps/ios/**", "scripts/ci/ios-stub-libs.sh"], + "packages": [ + "packages/simplex-chat-nodejs/**", + "packages/simplex-chat-client/types/typescript/**", + "packages/simplex-chat-python/**", + "scripts/ci/package-tests.sh", + ], +} +SCOPE_CHANGES = { + "badge-web": ["apps/simplex-badge-service/web/**"], + "ci-scripts": ["scripts/ci/packages_release.py", "scripts/ci/test_packages_release.py"], } CABAL_FILE = "simplex-chat.cabal" CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M) @@ -496,7 +507,7 @@ def changed_since(base: str, sources: list[str]) -> bool: def build_pathspecs(build: str) -> list[str]: - return [path if path.startswith(":") else f":(glob){path}" for path in BUILD_CHANGES[build]] + return [path if path.startswith(":") else f":(glob){path}" for path in (BUILD_CHANGES | SCOPE_CHANGES)[build]] def changes(base: str) -> None: @@ -504,6 +515,7 @@ def changes(base: str) -> None: print(f"addon={'true' if changed_since(base, ADDON_SOURCES) else ''}") for build in BUILD_CHANGES: print(f"{build}={'true' if changed_since(base, build_pathspecs(build)) else ''}") + print(f"scopes={' '.join(scope for scope in SCOPE_CHANGES if changed_since(base, build_pathspecs(scope)))}") def print_build_plan(args: argparse.Namespace) -> None: @@ -959,6 +971,7 @@ def main(argv: list[str]) -> None: download_pinned_parser.add_argument("directory") changes_parser = commands.add_parser("changes") changes_parser.add_argument("base", help="the commit to compare with") + commands.add_parser("scopes") build_parser = commands.add_parser("build-plan") build_parser.add_argument("--base", required=True, help="the base commit of the pull request") prepare_parser = commands.add_parser("prepare") @@ -996,6 +1009,8 @@ def main(argv: list[str]) -> None: download_pinned(args.asset, args.directory) elif args.command == "changes": changes(args.base) + elif args.command == "scopes": + print(f"scopes={' '.join(SCOPE_CHANGES)}") elif args.command == "build-plan": print_build_plan(args) elif args.command == "prepare": diff --git a/scripts/ci/run-tests.sh b/scripts/ci/run-tests.sh new file mode 100755 index 0000000000..ef7c92112c --- /dev/null +++ b/scripts/ci/run-tests.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +if [ -n "${PR_BODY:-}" ]; then + TESTS=$(printf '%s\n' "$PR_BODY" | tr -d '\r' | awk '/^```ci-tests[ \t]*$/ {f = 1; next} /^```[ \t]*$/ {f = 0} f') +fi +args=() +while read -r p; do + if [ -n "$p" ]; then args+=(-m "$p"); fi +done <<< "${TESTS:-}" +if [ "${#args[@]}" -gt 0 ]; then + args+=(--fail-on=empty) +elif [ "${REDUCED:-}" = true ]; then + echo "No tests selected" + exit 0 +fi + +i=1 +attempts=${ATTEMPTS:-1} +while [ "$i" -le "$attempts" ]; do + if "$@" "${args[@]}"; then + break + elif [ "$i" -lt "$attempts" ]; then + echo "Attempt $i failed, retrying..." + sleep 1 + fi + i=$((i + 1)) +done +if [ "$i" -gt "$attempts" ]; then + echo "Attempt $attempts failed." + exit 1 +fi diff --git a/scripts/ci/scope-tests.sh b/scripts/ci/scope-tests.sh new file mode 100755 index 0000000000..fe4b2807a9 --- /dev/null +++ b/scripts/ci/scope-tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euo pipefail + +cd "$(dirname "$0")/../.." + +case "${1:-}" in + badge-web) + (cd apps/simplex-badge-service/web && npm ci && npm test) + ;; + ci-scripts) + python3 scripts/ci/test_packages_release.py + ;; + *) + echo "usage: $0 badge-web | ci-scripts" >&2 + exit 1 + ;; +esac diff --git a/scripts/ci/test_packages_release.py b/scripts/ci/test_packages_release.py index dfc92d20e5..fc1bde780e 100644 --- a/scripts/ci/test_packages_release.py +++ b/scripts/ci/test_packages_release.py @@ -27,6 +27,7 @@ from packages_release import ( TYPES_README, ADDON_PLATFORMS, BUILD_CHANGES, + SCOPE_CHANGES, LIBS_ASSETS, Package, Pins, @@ -245,7 +246,7 @@ class BuildPlanTest(unittest.TestCase): found = re.search(r'^ pull_request:\n paths:\n((?: - ".*"\n)+)', workflow, re.M) self.assertIsNotNone(found, "no pull_request paths in build.yml") paths = re.findall(r'^ - "(.*)"$', found[1], re.M) - globs = {path for build in BUILD_CHANGES.values() for path in build if not path.startswith(":")} + globs = {path for build in (BUILD_CHANGES | SCOPE_CHANGES).values() for path in build if not path.startswith(":")} self.assertEqual(sorted(paths), sorted(globs)) self.assertEqual(len(paths), len(set(paths)), "duplicate pull_request paths in build.yml") @@ -463,8 +464,8 @@ class ChangesTest(GitRepoTest): output = io.StringIO() with mock.patch.object(packages_release, "ROOT", self.repo), contextlib.redirect_stdout(output): changes(self.base) - values = dict(line.split("=", 1) for line in output.getvalue().splitlines()) - return {build for build in BUILD_CHANGES if values[build] == "true"} + self.values = dict(line.split("=", 1) for line in output.getvalue().splitlines()) + return {build for build in BUILD_CHANGES if self.values[build] == "true"} def test_ios_change_builds_only_ios(self): self.assertEqual(self.builds("apps/ios/Shared/View.swift"), {"ios"}) @@ -487,12 +488,23 @@ class ChangesTest(GitRepoTest): def test_build_image_change_builds_the_core_and_kotlin(self): self.assertEqual(self.builds("Dockerfile.build"), {"haskell", "kotlin"}) - def test_workflow_change_builds_everything(self): - self.assertEqual(self.builds(".github/workflows/build.yml"), {"haskell", "kotlin", "ios"}) + def test_workflow_change_builds_the_core_and_kotlin(self): + self.assertEqual(self.builds(".github/workflows/build.yml"), {"haskell", "kotlin"}) def test_unrelated_change_builds_nothing(self): self.assertEqual(self.builds("docs/guide.md", "packages/pkg/src.txt"), set()) + def test_package_change_builds_only_packages(self): + self.assertEqual(self.builds("packages/simplex-chat-python/src/simplex_chat/client.py"), {"packages"}) + + def test_badge_web_change_builds_the_core_and_tests_its_scope(self): + self.assertEqual(self.builds("apps/simplex-badge-service/web/src/catalog.ts"), {"haskell"}) + self.assertEqual(self.values["scopes"], "badge-web") + + def test_release_script_change_tests_the_ci_scripts(self): + self.assertEqual(self.builds("scripts/ci/packages_release.py"), {"haskell"}) + self.assertEqual(self.values["scopes"], "ci-scripts") + class PackageStateTest(GitRepoTest): def setUp(self): diff --git a/src/Simplex/Chat/Library/Subscriber.hs b/src/Simplex/Chat/Library/Subscriber.hs index aa4a0205ad..d36bdc194e 100644 --- a/src/Simplex/Chat/Library/Subscriber.hs +++ b/src/Simplex/Chat/Library/Subscriber.hs @@ -1688,22 +1688,26 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag User {userChatRelay} = user -- TODO [relays] owner, relays: TBC how to communicate member rejection rules from owner to relays memberJoinRequestViaRelay :: InvitationId -> VersionRangeChat -> Maybe SignedMsg -> Profile -> MemberId -> MemberKey -> Maybe MemberId -> CM () - memberJoinRequestViaRelay invId chatVRange signedMsg_ p joiningMemberId joiningMemberKey@(MemberKey joiningKey) viaRelay = do + memberJoinRequestViaRelay invId chatVRange signedMsg_ p@Profile {displayName} joiningMemberId joiningMemberKey@(MemberKey joiningKey) viaRelay = do (_ucl, gLinkInfo_) <- withStore $ \db -> getUserContactLinkById db userId uclId case gLinkInfo_ of Just GroupLinkInfo {groupId, memberRole = gLinkMemRole} -> do g@(GIK gInfo _) <- withStore $ \db -> getGroupInfoKeys db cxt user groupId - existing_ <- withStore' $ \db -> eitherToMaybe <$> runExceptT (getGroupMemberByMemberId db cxt user gInfo joiningMemberId) - let signed = memberSigned gInfo joiningMemberId joiningKey signedMsg_ - case existing_ of - Just rosterMem - -- a privileged memberId's key is owner-authoritative (the roster); the joiner must prove - -- possession of that exact key, otherwise this is an attempt to impersonate it - | isRosterRole (memberRole' rosterMem) -> - if verifyKey gInfo rosterMem signed - then acceptJoin g signed (Just rosterMem) (memberRole' rosterMem) - else messageError "memberJoinRequestViaRelay: rejected join claiming privileged memberId (key mismatch or invalid signature)" - _ -> acceptJoin g signed Nothing gLinkMemRole + if + | not (useRelays' gInfo) -> + messageWarning $ "memberJoinRequestViaRelay (group " <> groupName' gInfo <> "): ignored relay join request from " <> displayName <> " (group does not use relays)" + | otherwise -> do + existing_ <- withStore' $ \db -> eitherToMaybe <$> runExceptT (getGroupMemberByMemberId db cxt user gInfo joiningMemberId) + let signed = memberSigned gInfo joiningMemberId joiningKey signedMsg_ + case existing_ of + Just rosterMem + -- a privileged memberId's key is owner-authoritative (the roster); the joiner must prove + -- possession of that exact key, otherwise this is an attempt to impersonate it + | isRosterRole (memberRole' rosterMem) -> + if verifyKey gInfo rosterMem signed + then acceptJoin g signed (Just rosterMem) (memberRole' rosterMem) + else messageError "memberJoinRequestViaRelay: rejected join claiming privileged memberId (key mismatch or invalid signature)" + _ -> acceptJoin g signed Nothing gLinkMemRole Nothing -> messageError "memberJoinRequestViaRelay: no group link info for relay link" where diff --git a/tests/ChatTests/Groups.hs b/tests/ChatTests/Groups.hs index 01a504f13c..136e417ef2 100644 --- a/tests/ChatTests/Groups.hs +++ b/tests/ChatTests/Groups.hs @@ -39,7 +39,7 @@ import Simplex.Chat.Messages (CIMention (..), CIMentionMember (..), ChatItemId) import Simplex.Chat.Messages.Batch (encodeBinaryBatch, encodeFwdElement) import Simplex.Chat.Messages.CIContent (publicGroupNoE2EText) import Simplex.Chat.Options -import Simplex.Chat.Protocol (ChatMessage (ChatMessage), ChatMsgEvent (XGrpMemNew, XGrpRelayInv, XInfo, XMsgUpdate, XMsgNew, XMsgDel), FwdSender (FwdMember, FwdChannel), GrpMsgForward (GrpMsgForward), MsgContainer (..), MsgMention (..), MsgContent (..), VerifiedMsg (VMUnsigned), chatMsgToBody, mcSimple, msgContentText) +import Simplex.Chat.Protocol (ChatMessage (ChatMessage), ChatMsgEvent (XGrpMemNew, XGrpRelayInv, XInfo, XMember, XMsgUpdate, XMsgNew, XMsgDel), FwdSender (FwdMember, FwdChannel), GrpMsgForward (GrpMsgForward), MsgContainer (..), MsgMention (..), MsgContent (..), VerifiedMsg (VMUnsigned), chatMsgToBody, mcSimple, msgContentText) import Simplex.Chat.Types import Simplex.Chat.Types.MemberRelations (MemberRelation (..), getRelation, setRelation) import Simplex.Chat.Types.Shared (GroupMemberRole (..), GroupAcceptance (..)) @@ -135,6 +135,7 @@ chatGroupTests = do it "demotion does not remove group link" testGroupLinkDemotedAdmin it "host profile received" testGroupLinkHostProfileReceived it "existing contact merged" testGroupLinkExistingContactMerged + it "ignore relay join request in group without relays" testGroupLinkIgnoreRelayJoinRequest describe "group links - member screening" $ do it "reject member - blocked name" testGLinkRejectBlockedName it "accept member - only host approval" testGLinkApproveMember @@ -3629,6 +3630,32 @@ testGroupLinkExistingContactMerged = bob #> "#team hi there" alice <# "#team bob> hi there" +testGroupLinkIgnoreRelayJoinRequest :: HasCallStack => TestParams -> IO () +testGroupLinkIgnoreRelayJoinRequest ps = + withNewTestChatOpts ps (testOpts {coreOptions = testCoreOpts {logLevel = CLLWarning}}) "alice" aliceProfile $ \alice -> + withNewTestChat ps "bob" bobProfile $ \bob -> do + alice ##> "/g team" + alice <## "group #team is created" + alice <## "to add members use /a team or /create link #team" + alice ##> "/create link #team" + (gShortLink, _gLink) <- getGroupLinks alice "team" GRMember True + sendMemberJoinRequest bob gShortLink + alice <## "warning: memberJoinRequestViaRelay (group team): ignored relay join request from bob (group does not use relays)" + alice ##> "/ms team" + alice <## "alice (Alice): owner, you, created group" + +sendMemberJoinRequest :: TestCC -> String -> IO () +sendMemberJoinRequest cc groupLink = + withCCUser cc $ \user -> do + gLink :: ShortLinkContact <- either fail pure $ strDecode (B.pack groupLink) + (memberKey, _) <- atomically . C.generateKeyPair =<< C.newRandom + let joinGroup = do + (_, _, cReq) <- getShortLinkConnReq NRMInteractive user gLink + (connId, _) <- withAgent $ \a -> prepareConnectionToJoin a (aUserId user) True cReq PQSupportOff + dm <- encodeConnInfo $ XMember bobProfile (MemberId "member") (MemberKey memberKey) Nothing + withAgent $ \a -> joinConnection a NRMInteractive (aUserId user) connId True cReq dm PQSupportOff SMSubscribe + runExceptT joinGroup `runReaderT` chatController cc >>= either (fail . show) (\_ -> pure ()) + testGLinkRejectBlockedName :: HasCallStack => TestParams -> IO () testGLinkRejectBlockedName = testChatCfg2 cfg aliceProfile bobProfile $