From 48b1ef764bea7ebe8cd1dbf3144153f87efa633f Mon Sep 17 00:00:00 2001 From: sh <37271604+shumvgolove@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:19:24 +0000 Subject: [PATCH] ci: reproducible builds/refactor (#5808) * ci: reproducible builds/refactor * ci: fix mac desktop upload * ci: docker shell abort on error * scripts: add reproduce script * ci: add new reproduce workflow * scripts/reproduce-builds: change repo back to official --- .github/actions/prepare-build/action.yml | 52 ++ .github/actions/prepare-release/action.yml | 39 ++ .github/workflows/build.yml | 580 ++++++++++++--------- .github/workflows/reproduce-schedule.yml | 45 ++ Dockerfile.build | 84 +++ scripts/ci/linux_util_free_space.sh | 96 ++++ scripts/reproduce-builds.sh | 120 +++++ 7 files changed, 757 insertions(+), 259 deletions(-) create mode 100644 .github/actions/prepare-build/action.yml create mode 100644 .github/actions/prepare-release/action.yml create mode 100644 .github/workflows/reproduce-schedule.yml create mode 100644 Dockerfile.build create mode 100755 scripts/ci/linux_util_free_space.sh create mode 100644 scripts/reproduce-builds.sh diff --git a/.github/actions/prepare-build/action.yml b/.github/actions/prepare-build/action.yml new file mode 100644 index 0000000000..6682641419 --- /dev/null +++ b/.github/actions/prepare-build/action.yml @@ -0,0 +1,52 @@ +name: "Prebuilt steps for build" +description: "Reusable steps for multiple jobs" +inputs: + java_ver: + required: true + description: "Java version to install" + ghc_ver: + required: true + description: "GHC version to install" + github_ref: + required: true + description: "Git reference" + os: + required: true + description: "Target OS" + cache_path: + required: false + default: "~/.cabal/store" + description: "Cache path" + cabal_ver: + required: false + default: 3.10.1.0 + description: "GHC version to install" +runs: + using: "composite" + steps: + - name: Skip unreliable ghc 8.10.7 build on stable branch + shell: bash + if: inputs.ghc_ver == '8.10.7' && inputs.github_ref == 'refs/heads/stable' + run: exit 0 + + - name: Setup Haskell + uses: simplex-chat/setup-haskell-action@v2 + with: + ghc-version: ${{ inputs.ghc_ver }} + cabal-version: ${{ inputs.cabal_ver }} + + - name: Setup Java + if: startsWith(inputs.github_ref, 'refs/tags/v') + uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: ${{ inputs.java_ver }} + cache: 'gradle' + + - name: Restore cached build + uses: actions/cache@v4 + with: + path: | + ${{ inputs.cache_path }} + dist-newstyle + key: ${{ inputs.os }}-ghc${{ inputs.ghc_ver }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }} diff --git a/.github/actions/prepare-release/action.yml b/.github/actions/prepare-release/action.yml new file mode 100644 index 0000000000..e44e6ef0f2 --- /dev/null +++ b/.github/actions/prepare-release/action.yml @@ -0,0 +1,39 @@ +name: "Upload binary and update hash" +description: "Reusable steps for multiple jobs" +inputs: + bin_path: + required: true + description: "Path to binary to upload" + bin_name: + required: true + description: "Name of uploaded binary" + bin_hash: + required: true + description: "Message with SHA to include in release" + github_ref: + required: true + description: "Github reference" + github_token: + required: true + description: "Github token" +runs: + using: "composite" + steps: + - name: Linux upload AppImage to release + if: startsWith(inputs.github_ref, 'refs/tags/v') + uses: simplex-chat/upload-release-action@v2 + with: + repo_token: ${{ inputs.github_token }} + file: ${{ inputs.bin_path }} + asset_name: ${{ inputs.bin_name }} + tag: ${{ inputs.github_ref }} + + - name: Linux update AppImage hash + if: startsWith(inputs.github_ref, 'refs/tags/v') + uses: simplex-chat/action-gh-release@v2 + env: + GITHUB_TOKEN: ${{ inputs.github_token }} + with: + append_body: true + body: | + ${{ inputs.bin_hash }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39973dc017..de0b976bcc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,16 +22,57 @@ on: - "README.md" - "PRIVACY.md" +# This workflow uses custom actions (prepare-build and prepare-release) defined in: +# +# .github/actions/ +# ├── prepare-build +# │ └── action.yml +# └── prepare-release +# └── action.yml + +# Important! +# Do not use always(), it makes build unskippable. +# See: https://github.com/actions/runner/issues/1846#issuecomment-1246102753 + jobs: - prepare-release: - if: startsWith(github.ref, 'refs/tags/v') + +# ============================= +# Global variables +# ============================= + +# That is the only and less hacky way to setup global variables +# to use in strategy matrix (env:/YAML anchors doesn't work). +# See: https://github.com/orgs/community/discussions/56787#discussioncomment-6041789 +# https://github.com/actions/runner/issues/1182 +# https://stackoverflow.com/a/77549656 + + variables: + runs-on: ubuntu-latest + outputs: + GHC_VER: 9.6.3 + JAVA_VER: 17 + steps: + - name: Dummy job when we have just simple variables + if: false + run: echo + +# ============================= +# Create release +# ============================= + +# Create release, but only if it's triggered by tag push. +# On pull requests/commits push, this job will always complete. + + maybe-release: runs-on: ubuntu-latest steps: - name: Clone project + if: startsWith(github.ref, 'refs/tags/v') uses: actions/checkout@v3 - name: Build changelog id: build_changelog + if: startsWith(github.ref, 'refs/tags/v') uses: simplex-chat/release-changelog-builder-action@v4 with: configuration: .github/changelog_conf.json @@ -42,6 +83,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create release + if: startsWith(github.ref, 'refs/tags/v') uses: simplex-chat/action-gh-release@v1 with: body: ${{ steps.build_changelog.outputs.changelog }} @@ -52,183 +94,259 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - build: - name: build-${{ matrix.os }}-${{ matrix.ghc }} - if: always() - needs: prepare-release - runs-on: ${{ matrix.os }} +# ========================= +# Linux Build +# ========================= + + build-linux: + name: "ubuntu-${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" + needs: [maybe-release, variables] + runs-on: ubuntu-${{ matrix.os }} strategy: fail-fast: false matrix: include: - - os: ubuntu-20.04 + - os: 20.04 ghc: "8.10.7" - cache_path: ~/.cabal/store - - os: ubuntu-20.04 - ghc: "9.6.3" - cache_path: ~/.cabal/store - asset_name: simplex-chat-ubuntu-20_04-x86-64 + - os: 20.04 + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-ubuntu-20_04-x86-64 desktop_asset_name: simplex-desktop-ubuntu-20_04-x86_64.deb - - os: ubuntu-22.04 - ghc: "9.6.3" - cache_path: ~/.cabal/store - asset_name: simplex-chat-ubuntu-22_04-x86-64 + - os: 22.04 + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-ubuntu-22_04-x86-64 desktop_asset_name: simplex-desktop-ubuntu-22_04-x86_64.deb - - os: macos-latest - ghc: "9.6.3" - cache_path: ~/.cabal/store - asset_name: simplex-chat-macos-aarch64 - desktop_asset_name: simplex-desktop-macos-aarch64.dmg - - os: macos-13 - ghc: "9.6.3" - cache_path: ~/.cabal/store - asset_name: simplex-chat-macos-x86-64 - desktop_asset_name: simplex-desktop-macos-x86_64.dmg - - os: windows-latest - ghc: "9.6.3" - cache_path: C:/cabal - asset_name: simplex-chat-windows-x86-64 - desktop_asset_name: simplex-desktop-windows-x86_64.msi - steps: - - name: Skip unreliable ghc 8.10.7 build on stable branch - if: matrix.ghc == '8.10.7' && github.ref == 'refs/heads/stable' - run: exit 0 - - - name: Configure pagefile (Windows) - if: matrix.os == 'windows-latest' - uses: simplex-chat/configure-pagefile-action@v1.3 - with: - minimum-size: 16GB - maximum-size: 16GB - disk-root: "C:" - - - name: Clone project + - name: Checkout Code uses: actions/checkout@v3 - - name: Setup Haskell - uses: simplex-chat/setup-haskell-action@v2 - with: - ghc-version: ${{ matrix.ghc }} - cabal-version: "3.10.1.0" + # Otherwise we run out of disk space with Docker build + - name: Free disk space + shell: bash + run: ./scripts/ci/linux_util_free_space.sh - name: Restore cached build - id: restore_cache - uses: actions/cache/restore@v3 + uses: actions/cache@v4 with: path: | - ${{ matrix.cache_path }} + ~/.cabal/store dist-newstyle - key: ${{ matrix.os }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }} + key: ubuntu-${{ matrix.os }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }} - # / Unix + - name: Set up Docker Buildx + uses: simplex-chat/docker-setup-buildx-action@v3 - - name: Unix prepare cabal.project.local for Mac - if: matrix.os == 'macos-latest' + - name: Build and cache Docker image + uses: simplex-chat/docker-build-push-action@v6 + with: + context: . + load: true + file: Dockerfile.build + tags: build/${{ matrix.os }}:latest + build-args: | + TAG=${{ matrix.os }} + GHC=${{ matrix.ghc }} + + # Docker needs these flags for AppImage build: + # --device /dev/fuse + # --cap-add SYS_ADMIN + # --security-opt apparmor:unconfined + - name: Start container shell: bash run: | - echo "ignore-project: False" >> cabal.project.local - echo "package simplexmq" >> cabal.project.local - echo " extra-include-dirs: /opt/homebrew/opt/openssl@3.0/include" >> cabal.project.local - echo " extra-lib-dirs: /opt/homebrew/opt/openssl@3.0/lib" >> cabal.project.local - echo "" >> cabal.project.local - echo "package direct-sqlcipher" >> cabal.project.local - echo " extra-include-dirs: /opt/homebrew/opt/openssl@3.0/include" >> cabal.project.local - echo " extra-lib-dirs: /opt/homebrew/opt/openssl@3.0/lib" >> cabal.project.local - echo " flags: +openssl" >> cabal.project.local + docker run -t -d \ + --device /dev/fuse \ + --cap-add SYS_ADMIN \ + --security-opt apparmor:unconfined \ + --name builder \ + -v ~/.cabal:/root/.cabal \ + -v /home/runner/work/_temp:/home/runner/work/_temp \ + -v ${{ github.workspace }}:/project \ + build/${{ matrix.os }}:latest - - name: Unix prepare cabal.project.local for Mac - if: matrix.os == 'macos-13' - shell: bash - run: | - echo "ignore-project: False" >> cabal.project.local - echo "package simplexmq" >> cabal.project.local - echo " extra-include-dirs: /usr/local/opt/openssl@3.0/include" >> cabal.project.local - echo " extra-lib-dirs: /usr/local/opt/openssl@3.0/lib" >> cabal.project.local - echo "" >> cabal.project.local - echo "package direct-sqlcipher" >> cabal.project.local - echo " extra-include-dirs: /usr/local/opt/openssl@3.0/include" >> cabal.project.local - echo " extra-lib-dirs: /usr/local/opt/openssl@3.0/lib" >> cabal.project.local - echo " flags: +openssl" >> cabal.project.local - - - name: Install AppImage dependencies - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04' - run: sudo apt install -y desktop-file-utils - - - name: Install openssl for Mac - if: matrix.os == 'macos-latest' || matrix.os == 'macos-13' - run: brew install openssl@3.0 - - - name: Unix prepare cabal.project.local for Ubuntu - if: matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' + - name: Prepare cabal.project.local shell: bash run: | echo "ignore-project: False" >> cabal.project.local echo "package direct-sqlcipher" >> cabal.project.local echo " flags: +openssl" >> cabal.project.local - - name: Unix build CLI - id: unix_cli_build - if: matrix.os != 'windows-latest' + # chmod/git commands are used to workaround permission issues when cache is restored + - name: Build CLI + shell: docker exec -t builder sh -eu {0} + run: | + chmod -R 777 dist-newstyle ~/.cabal && git config --global --add safe.directory '*' + cabal clean + cabal update + cabal build -j --enable-tests + mkdir -p /out + for i in simplex-chat simplex-chat-test; do + bin=$(find /project/dist-newstyle -name "$i" -type f -executable) + chmod +x "$bin" + mv "$bin" /out/ + done + strip /out/simplex-chat + + - name: Copy tests from container shell: bash run: | - cabal build --enable-tests - path=$(cabal list-bin simplex-chat) - echo "bin_path=$path" >> $GITHUB_OUTPUT - echo "bin_hash=$(echo SHA2-512\(${{ matrix.asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + docker cp builder:/out/simplex-chat-test . - - name: Unix upload CLI binary to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os != 'windows-latest' - uses: simplex-chat/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.unix_cli_build.outputs.bin_path }} - asset_name: ${{ matrix.asset_name }} - tag: ${{ github.ref }} - - - name: Unix update CLI binary hash - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os != 'windows-latest' - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.unix_cli_build.outputs.bin_hash }} - - - name: Setup Java - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name - uses: actions/setup-java@v3 - with: - distribution: 'corretto' - java-version: '17' - cache: 'gradle' - - - name: Linux build desktop - id: linux_desktop_build - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04') + - name: Copy CLI from container and prepare it + id: linux_cli_prepare + if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name shell: bash + run: | + docker cp builder:/out/simplex-chat ./${{ matrix.cli_asset_name }} + path="${{ github.workspace }}/${{ matrix.cli_asset_name }}" + echo "bin_path=$path" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + + - name: Upload CLI + if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name + uses: ./.github/actions/prepare-release + with: + bin_path: ${{ steps.linux_cli_prepare.outputs.bin_path }} + bin_name: ${{ matrix.cli_asset_name }} + bin_hash: ${{ steps.linux_cli_prepare.outputs.bin_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Desktop + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + shell: docker exec -t builder sh -eu {0} run: | scripts/desktop/build-lib-linux.sh cd apps/multiplatform ./gradlew packageDeb - path=$(echo $PWD/release/main/deb/simplex_*_amd64.deb) + + - name: Prepare Desktop + id: linux_desktop_build + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + shell: bash + run: | + path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/deb/simplex_*_amd64.deb ) echo "package_path=$path" >> $GITHUB_OUTPUT echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - - name: Linux make AppImage - id: linux_appimage_build - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04' - shell: bash + - name: Upload Desktop + uses: ./.github/actions/prepare-release + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + with: + bin_path: ${{ steps.linux_desktop_build.outputs.package_path }} + bin_name: ${{ matrix.desktop_asset_name }} + bin_hash: ${{ steps.linux_desktop_build.outputs.package_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build AppImage + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + shell: docker exec -t builder sh -eu {0} run: | scripts/desktop/make-appimage-linux.sh - path=$(echo $PWD/apps/multiplatform/release/main/*imple*.AppImage) + + - name: Prepare AppImage + id: linux_appimage_build + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + shell: bash + run: | + path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/*imple*.AppImage) echo "appimage_path=$path" >> $GITHUB_OUTPUT echo "appimage_hash=$(echo SHA2-512\(simplex-desktop-x86_64.AppImage\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - - name: Mac build desktop + - name: Upload AppImage + if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + uses: ./.github/actions/prepare-release + with: + bin_path: ${{ steps.linux_appimage_build.outputs.appimage_path }} + bin_name: "simplex-desktop-x86_64.AppImage" + bin_hash: ${{ steps.linux_appimage_build.outputs.appimage_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Fix permissions for cache + shell: bash + run: | + sudo chmod -R 777 dist-newstyle ~/.cabal + sudo chown -R $(id -u):$(id -g) dist-newstyle ~/.cabal + + - name: Run tests + shell: bash + run: | + ./simplex-chat-test + +# ========================= +# MacOS Build +# ========================= + + build-macos: + name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" + needs: [maybe-release, variables] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-latest + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-macos-aarch64 + desktop_asset_name: simplex-desktop-macos-aarch64.dmg + openssl_dir: "/opt/homebrew/opt" + - os: macos-13 + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-macos-x86-64 + desktop_asset_name: simplex-desktop-macos-x86_64.dmg + openssl_dir: "/usr/local/opt" + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Prepare build + uses: ./.github/actions/prepare-build + with: + java_ver: ${{ needs.variables.outputs.JAVA_VER }} + ghc_ver: ${{ matrix.ghc }} + os: ${{ matrix.os }} + github_ref: ${{ github.ref }} + + - name: Install OpenSSL + run: brew install openssl@3.0 + + - name: Prepare cabal.project.local + shell: bash + run: | + echo "ignore-project: False" >> cabal.project.local + echo "package simplexmq" >> cabal.project.local + echo " extra-include-dirs: ${{ matrix.opnessl_dir }}/openssl@3.0/include" >> cabal.project.local + echo " extra-lib-dirs: ${{ matrix.openssl_dir}}/openssl@3.0/lib" >> cabal.project.local + echo "" >> cabal.project.local + echo "package direct-sqlcipher" >> cabal.project.local + echo " extra-include-dirs: ${{ matrix.openssl_dir }}/openssl@3.0/include" >> cabal.project.local + echo " extra-lib-dirs: ${{ matrix.openssl_dir }}/openssl@3.0/lib" >> cabal.project.local + echo " flags: +openssl" >> cabal.project.local + + - name: Build CLI + id: mac_cli_build + shell: bash + run: | + cabal build -j --enable-tests + path=$(cabal list-bin simplex-chat) + echo "bin_path=$path" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + + - name: Upload CLI + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/actions/prepare-release + with: + bin_path: ${{ steps.mac_cli_build.outputs.bin_path }} + bin_name: ${{ matrix.cli_asset_name }} + bin_hash: ${{ steps.mac_cli_build.outputs.bin_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Desktop id: mac_desktop_build - if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13') + if: startsWith(github.ref, 'refs/tags/v') shell: bash env: APPLE_SIMPLEX_SIGNING_KEYCHAIN: ${{ secrets.APPLE_SIMPLEX_SIGNING_KEYCHAIN }} @@ -240,85 +358,58 @@ jobs: echo "package_path=$path" >> $GITHUB_OUTPUT echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - - name: Linux upload desktop package to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04') - uses: simplex-chat/upload-release-action@v2 + - name: Upload Desktop + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/actions/prepare-release with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.linux_desktop_build.outputs.package_path }} - asset_name: ${{ matrix.desktop_asset_name }} - tag: ${{ github.ref }} + bin_path: ${{ steps.mac_desktop_build.outputs.package_path }} + bin_name: ${{ matrix.desktop_asset_name }} + bin_hash: ${{ steps.mac_desktop_build.outputs.package_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Linux update desktop package hash - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04') - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.linux_desktop_build.outputs.package_hash }} - - - name: Linux upload AppImage to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04' - uses: simplex-chat/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.linux_appimage_build.outputs.appimage_path }} - asset_name: simplex-desktop-x86_64.AppImage - tag: ${{ github.ref }} - - - name: Linux update AppImage hash - if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04' - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.linux_appimage_build.outputs.appimage_hash }} - - - name: Mac upload desktop package to release - if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13') - uses: simplex-chat/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.mac_desktop_build.outputs.package_path }} - asset_name: ${{ matrix.desktop_asset_name }} - tag: ${{ github.ref }} - - - name: Mac update desktop package hash - if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13') - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.mac_desktop_build.outputs.package_hash }} - - - name: Cache unix build - uses: actions/cache/save@v3 - if: matrix.os != 'windows-latest' - with: - path: | - ${{ matrix.cache_path }} - dist-newstyle - key: ${{ steps.restore_cache.outputs.cache-primary-key }} - - - name: Unix test - if: matrix.os != 'windows-latest' + - name: Run tests timeout-minutes: 40 shell: bash run: cabal test --test-show-details=direct - # Unix / +# ========================= +# Windows Build +# ========================= - # / Windows - # rm -rf dist-newstyle/src/direct-sq* is here because of the bug in cabal's dependency which prevents second build from finishing + build-windows: + name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" + needs: [maybe-release, variables] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-windows-x86-64 + desktop_asset_name: simplex-desktop-windows-x86_64.msi + steps: + - name: Checkout Code + uses: actions/checkout@v3 + - name: Prepare build + uses: ./.github/actions/prepare-build + with: + java_ver: ${{ needs.variables.outputs.JAVA_VER }} + ghc_ver: ${{ matrix.ghc }} + os: ${{ matrix.os }} + cache_path: "C:/cabal" + github_ref: ${{ github.ref }} + + - name: Configure pagefile (Windows) + uses: simplex-chat/configure-pagefile-action@v1.4 + with: + minimum-size: 16GB + maximum-size: 16GB + disk-root: "C:" + - name: 'Setup MSYS2' - if: matrix.os == 'windows-latest' uses: simplex-chat/setup-msys2@v2 with: msystem: ucrt64 @@ -331,10 +422,9 @@ jobs: toolchain:p cmake:p - - - name: Windows build - id: windows_build - if: matrix.os == 'windows-latest' + # rm -rf dist-newstyle/src/direct-sq* is here because of the bug in cabal's dependency which prevents second build from finishing + - name: Build CLI + id: windows_cli_build shell: msys2 {0} run: | export PATH=$PATH:/c/ghcup/bin:$(echo /c/tools/ghc-*/bin || echo) @@ -349,70 +439,42 @@ jobs: rm -rf dist-newstyle/src/direct-sq* sed -i "s/, unix /--, unix /" simplex-chat.cabal - cabal build --enable-tests + cabal build -j --enable-tests rm -rf dist-newstyle/src/direct-sq* path=$(cabal list-bin simplex-chat | tail -n 1) echo "bin_path=$path" >> $GITHUB_OUTPUT - echo "bin_hash=$(echo SHA2-512\(${{ matrix.asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - - name: Windows upload CLI binary to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest' - uses: simplex-chat/upload-release-action@v2 + - name: Upload CLI + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/actions/prepare-release with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.windows_build.outputs.bin_path }} - asset_name: ${{ matrix.asset_name }} - tag: ${{ github.ref }} + bin_path: ${{ steps.windows_cli_build.outputs.bin_path }} + bin_name: ${{ matrix.cli_asset_name }} + bin_hash: ${{ steps.windows_cli_build.outputs.bin_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Windows update CLI binary hash - if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest' - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.windows_build.outputs.bin_hash }} - - - name: Windows build desktop + - name: Build Desktop id: windows_desktop_build - if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest' + if: startsWith(github.ref, 'refs/tags/v') shell: msys2 {0} run: | export PATH=$PATH:/c/ghcup/bin:$(echo /c/tools/ghc-*/bin || echo) scripts/desktop/build-lib-windows.sh cd apps/multiplatform ./gradlew packageMsi + rm -rf dist-newstyle/src/direct-sq* path=$(echo $PWD/release/main/msi/*imple*.msi | sed 's#/\([a-z]\)#\1:#' | sed 's#/#\\#g') echo "package_path=$path" >> $GITHUB_OUTPUT echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - - name: Windows upload desktop package to release - if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest' - uses: simplex-chat/upload-release-action@v2 + - name: Upload Desktop + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/actions/prepare-release with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ steps.windows_desktop_build.outputs.package_path }} - asset_name: ${{ matrix.desktop_asset_name }} - tag: ${{ github.ref }} - - - name: Windows update desktop package hash - if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest' - uses: simplex-chat/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - append_body: true - body: | - ${{ steps.windows_desktop_build.outputs.package_hash }} - - - name: Cache windows build - uses: actions/cache/save@v3 - if: matrix.os == 'windows-latest' - with: - path: | - ${{ matrix.cache_path }} - dist-newstyle - key: ${{ steps.restore_cache.outputs.cache-primary-key }} - - # Windows / + bin_path: ${{ steps.windows_desktop_build.outputs.package_path }} + bin_name: ${{ matrix.desktop_asset_name }} + bin_hash: ${{ steps.windows_desktop_build.outputs.package_hash }} + github_ref: ${{ github.ref }} + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reproduce-schedule.yml b/.github/workflows/reproduce-schedule.yml new file mode 100644 index 0000000000..7de44addc7 --- /dev/null +++ b/.github/workflows/reproduce-schedule.yml @@ -0,0 +1,45 @@ +name: Reproduce latest release + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' # every day at 02:00 night + +jobs: + reproduce: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Get latest release + shell: bash + run: | + curl --proto '=https' \ + --tlsv1.2 \ + -sSf -L \ + 'https://api.github.com/repos/simplex-chat/simplex-chat/releases/latest' \ + 2>/dev/null | \ + grep -i "tag_name" | \ + awk -F \" '{print "TAG="$4}' >> $GITHUB_ENV + + - name: Execute reproduce script + run: | + ${GITHUB_WORKSPACE}/scripts/reproduce-builds.sh "$TAG" + + - name: Check if build has been reproduced + env: + url: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_URL }} + user: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_USER }} + pass: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_PASS }} + run: | + if [ -f "${GITHUB_WORKSPACE}/$TAG/_sha256sums" ]; then + exit 0 + else + curl --proto '=https' --tlsv1.2 -sSf \ + -u "${user}:${pass}" \ + -H 'Content-Type: application/json' \ + -d '{"title": "👾 GitHub: Runner", "description": "⛔️ '"$TAG"' did not reproduce."}' \ + "$url" + exit 1 + fi diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000000..cd0fd22aad --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,84 @@ +# syntax=docker/dockerfile:1.7.0-labs +ARG TAG=24.04 +FROM ubuntu:${TAG} AS build + +### Build stage + +ARG GHC=9.6.3 +ARG CABAL=3.10.1.0 +ARG JAVA=17 + +ENV TZ=Etc/UTC \ + DEBIAN_FRONTEND=noninteractive + +# Install curl, git and and simplexmq dependencies +RUN apt-get update && \ + apt-get install -y curl \ + libpq-dev \ + git \ + sqlite3 \ + libsqlite3-dev \ + build-essential \ + libgmp3-dev \ + zlib1g-dev \ + llvm \ + cmake \ + llvm-dev \ + libnuma-dev \ + libssl-dev \ + desktop-file-utils \ + openjdk-${JAVA}-jdk-headless \ + patchelf \ + ca-certificates \ + zip \ + wget \ + fuse3 \ + file \ + appstream \ + gpg \ + unzip &&\ + export JAVA_HOME=$(update-java-alternatives -l | head -n 1 | awk -F ' ' '{print $NF}') &&\ + ln -s /bin/fusermount /bin/fusermount3 || : + +# Specify bootstrap Haskell versions +ENV BOOTSTRAP_HASKELL_GHC_VERSION=${GHC} +ENV BOOTSTRAP_HASKELL_CABAL_VERSION=${CABAL} + +# Do not install Stack +ENV BOOTSTRAP_HASKELL_INSTALL_NO_STACK=true +ENV BOOTSTRAP_HASKELL_INSTALL_NO_STACK_HOOK=true + +# Install ghcup +RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 sh + +# Adjust PATH +ENV PATH="/root/.cabal/bin:/root/.ghcup/bin:$PATH" + +# Set both as default +RUN ghcup set ghc "${GHC}" && \ + ghcup set cabal "${CABAL}" + +#===================== +# Install Android SDK +#===================== +ARG SDK_VERSION=13114758 + +ENV SDK_VERSION=$SDK_VERSION \ + ANDROID_HOME=/root + +RUN curl -L -o tools.zip "https://dl.google.com/android/repository/commandlinetools-linux-${SDK_VERSION}_latest.zip" && \ + unzip tools.zip && rm tools.zip && \ + mv cmdline-tools tools && mkdir "$ANDROID_HOME/cmdline-tools" && mv tools "$ANDROID_HOME/cmdline-tools/" && \ + ln -s "$ANDROID_HOME/cmdline-tools/tools" "$ANDROID_HOME/cmdline-tools/latest" + +ENV PATH="$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/tools/bin" + +# https://askubuntu.com/questions/885658/android-sdk-repositories-cfg-could-not-be-loaded +RUN mkdir -p ~/.android ~/.gradle && \ + touch ~/.android/repositories.cfg && \ + echo 'org.gradle.console=plain' > ~/.gradle/gradle.properties &&\ + yes | sdkmanager --licenses >/dev/null + +ENV PATH=$PATH:$ANDROID_HOME/platform-tools:$ANDROID_HOME/build-tools + +WORKDIR /project diff --git a/scripts/ci/linux_util_free_space.sh b/scripts/ci/linux_util_free_space.sh new file mode 100755 index 0000000000..ef00eb886e --- /dev/null +++ b/scripts/ci/linux_util_free_space.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Taken from: https://github.com/apache/arrow/blob/main/ci/scripts/util_free_space.sh + +set -eux + +df -h +echo "::group::/usr/local/*" +du -hsc /usr/local/* +echo "::endgroup::" +# ~1GB +sudo rm -rf \ + /usr/local/aws-sam-cil \ + /usr/local/julia* || : +echo "::group::/usr/local/bin/*" +du -hsc /usr/local/bin/* +echo "::endgroup::" +# ~1GB (From 1.2GB to 214MB) +sudo rm -rf \ + /usr/local/bin/aliyun \ + /usr/local/bin/azcopy \ + /usr/local/bin/bicep \ + /usr/local/bin/cmake-gui \ + /usr/local/bin/cpack \ + /usr/local/bin/helm \ + /usr/local/bin/hub \ + /usr/local/bin/kubectl \ + /usr/local/bin/minikube \ + /usr/local/bin/node \ + /usr/local/bin/packer \ + /usr/local/bin/pulumi* \ + /usr/local/bin/sam \ + /usr/local/bin/stack \ + /usr/local/bin/terraform || : +# 142M +sudo rm -rf /usr/local/bin/oc || : \ +echo "::group::/usr/local/share/*" +du -hsc /usr/local/share/* +echo "::endgroup::" +# 506MB +sudo rm -rf /usr/local/share/chromium || : +# 1.3GB +sudo rm -rf /usr/local/share/powershell || : +echo "::group::/usr/local/lib/*" +du -hsc /usr/local/lib/* +echo "::endgroup::" +# 15GB +sudo rm -rf /usr/local/lib/android || : +# 341MB +sudo rm -rf /usr/local/lib/heroku || : +# 1.2GB +sudo rm -rf /usr/local/lib/node_modules || : +echo "::group::/opt/*" +du -hsc /opt/* +echo "::endgroup::" +# 679MB +sudo rm -rf /opt/az || : +echo "::group::/opt/microsoft/*" +du -hsc /opt/microsoft/* +echo "::endgroup::" +# 197MB +sudo rm -rf /opt/microsoft/powershell || : +echo "::group::/opt/hostedtoolcache/*" +du -hsc /opt/hostedtoolcache/* +echo "::endgroup::" +# 5.3GB +sudo rm -rf /opt/hostedtoolcache/CodeQL || : +# 1.4GB +sudo rm -rf /opt/hostedtoolcache/go || : +# 489MB +sudo rm -rf /opt/hostedtoolcache/PyPy || : +# 376MB +sudo rm -rf /opt/hostedtoolcache/node || : +# Remove Web browser packages +sudo apt purge -y \ + firefox \ + google-chrome-stable \ + microsoft-edge-stable +df -h diff --git a/scripts/reproduce-builds.sh b/scripts/reproduce-builds.sh new file mode 100644 index 0000000000..1334fec0ec --- /dev/null +++ b/scripts/reproduce-builds.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env sh +set -eu + +TAG="$1" + +tempdir="$(mktemp -d)" +init_dir="$PWD" + +repo_name="simplex-chat" +repo="https://github.com/simplex-chat/${repo_name}" + +cabal_local='ignore-project: False +package direct-sqlcipher + flags: +openssl' + +export DOCKER_BUILDKIT=1 + +cleanup() { + docker exec -t builder sh -c 'rm -rf ./dist-newstyle' 2>/dev/null || : + rm -rf -- "$tempdir" + docker rm --force builder 2>/dev/null || : + docker image rm local 2>/dev/null || : + cd "$init_dir" +} +trap 'cleanup' EXIT INT + +mkdir -p "$init_dir/$TAG/from-source" "$init_dir/$TAG/prebuilt" + +git -C "$tempdir" clone "$repo.git" &&\ + cd "$tempdir/${repo_name}" &&\ + git checkout "$TAG" + +for os in 20.04 22.04; do + os_url="$(printf '%s' "$os" | tr '.' '_')" + + # Build image + docker build \ + --no-cache \ + --build-arg TAG=${os} \ + --build-arg GHC=9.6.3 \ + -f "$tempdir/${repo_name}/Dockerfile.build" \ + -t local \ + . + + printf '%s' "$cabal_local" > "$tempdir/${repo_name}/cabal.project.local" + + # Run container in background + docker run -t -d \ + --name builder \ + -v "$tempdir/${repo_name}:/project" \ + local + + docker exec \ + -t \ + builder \ + sh -c 'cabal clean && cabal update && cabal build -j --enable-tests && mkdir -p /out && for i in simplex-chat; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable) && chmod +x "$bin" && mv "$bin" /out/; done && strip /out/simplex-chat' + + docker cp \ + builder:/out/simplex-chat \ + "$init_dir/$TAG/from-source/simplex-chat-ubuntu-${os_url}-x86-64" + + # Download prebuilt postgresql binary + curl -L \ + --output-dir "$init_dir/$TAG/prebuilt/" \ + -O \ + "$repo/releases/download/${TAG}/simplex-chat-ubuntu-${os_url}-x86-64" + + # Important! Remove dist-newstyle for the next interation + docker exec \ + -t \ + builder \ + sh -c 'rm -rf ./dist-newstyle' + + # Also restore git to previous state + git reset --hard && git clean -dfx + + # Stop containers, delete images + docker stop builder + docker rm --force builder + docker image rm local +done + +# Cleanup +rm -rf -- "$tempdir" +cd "$init_dir" + +# Final stage: compare hashes + +# Path to binaries +path_bin="$init_dir/$TAG" + +# Assume everything is okay for now +bad=0 + +# Check hashes for all binaries +for file in "$path_bin"/from-source/*; do + # Extract binary name + app="$(basename $file)" + + # Compute hash for compiled binary + compiled=$(sha256sum "$path_bin/from-source/$app" | awk '{print $1}') + # Compute hash for prebuilt binary + prebuilt=$(sha256sum "$path_bin/prebuilt/$app" | awk '{print $1}') + + # Compare + if [ "$compiled" != "$prebuilt" ]; then + # If hashes doesn't match, set bad... + bad=1 + + # ... and print affected binary + printf "%s - sha256sum hash doesn't match\n" "$app" + fi +done + +# If everything is still okay, compute checksums file +if [ "$bad" = 0 ]; then + sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' > "$path_bin/_sha256sums" + + printf 'Checksums computed - %s\n' "$path_bin/_sha256sums" +fi