ci: reproducible builds/refactor (#5808)

* ci: reproducible builds/refactor

* ci: fix mac desktop upload

* ci: docker shell abort on error

* scripts: add reproduce script

* ci: add new reproduce workflow

* scripts/reproduce-builds: change repo back to official

.github/actions/prepare-build/action.yml

@@ -0,0 +1,52 @@
+name: "Prebuilt steps for build"
+description: "Reusable steps for multiple jobs"
+inputs:
+   java_ver:
+     required: true
+     description: "Java version to install"
+   ghc_ver:
+     required: true
+     description: "GHC version to install"
+   github_ref:
+     required: true
+     description: "Git reference"
+   os:
+     required: true
+     description: "Target OS"
+   cache_path:
+     required: false
+     default: "~/.cabal/store"
+     description: "Cache path"
+   cabal_ver:
+     required: false
+     default: 3.10.1.0
+     description: "GHC version to install"
+runs:
+  using: "composite"
+  steps:
+    - name: Skip unreliable ghc 8.10.7 build on stable branch
+      shell: bash
+      if: inputs.ghc_ver == '8.10.7' && inputs.github_ref == 'refs/heads/stable'
+      run: exit 0
+
+    - name: Setup Haskell
+      uses: simplex-chat/setup-haskell-action@v2
+      with:
+        ghc-version: ${{ inputs.ghc_ver }}
+        cabal-version: ${{ inputs.cabal_ver }}
+
+    - name: Setup Java
+      if: startsWith(inputs.github_ref, 'refs/tags/v')
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'corretto'
+        java-version: ${{ inputs.java_ver }}
+        cache: 'gradle'
+
+    - name: Restore cached build
+      uses: actions/cache@v4
+      with:
+        path: |
+          ${{ inputs.cache_path }}
+          dist-newstyle
+        key: ${{ inputs.os }}-ghc${{ inputs.ghc_ver }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}

.github/actions/prepare-release/action.yml

@@ -0,0 +1,39 @@
+name: "Upload binary and update hash"
+description: "Reusable steps for multiple jobs"
+inputs:
+   bin_path:
+     required: true
+     description: "Path to binary to upload"
+   bin_name:
+     required: true
+     description: "Name of uploaded binary"
+   bin_hash:
+     required: true
+     description: "Message with SHA to include in release"
+   github_ref:
+     required: true
+     description: "Github reference"
+   github_token:
+     required: true
+     description: "Github token"
+runs:
+  using: "composite"
+  steps:
+    - name: Linux upload AppImage to release
+      if: startsWith(inputs.github_ref, 'refs/tags/v')
+      uses: simplex-chat/upload-release-action@v2
+      with:
+        repo_token: ${{ inputs.github_token }}
+        file: ${{ inputs.bin_path }}
+        asset_name: ${{ inputs.bin_name }}
+        tag: ${{ inputs.github_ref }}
+
+    - name: Linux update AppImage hash
+      if: startsWith(inputs.github_ref, 'refs/tags/v')
+      uses: simplex-chat/action-gh-release@v2
+      env:
+        GITHUB_TOKEN: ${{ inputs.github_token }}
+      with:
+        append_body: true
+        body: |
+          ${{ inputs.bin_hash }}

.github/workflows/build.yml

@@ -22,16 +22,57 @@ on:
       - "README.md"
       - "PRIVACY.md"
 
+# This workflow uses custom actions (prepare-build and prepare-release) defined in:
+# 
+# .github/actions/
+# ├── prepare-build
+# │   └── action.yml
+# └── prepare-release
+#     └── action.yml
+
+# Important!
+# Do not use always(), it makes build unskippable.
+# See: https://github.com/actions/runner/issues/1846#issuecomment-1246102753
+
 jobs:
-  prepare-release:
+
-    if: startsWith(github.ref, 'refs/tags/v')
+# =============================
+#       Global variables
+# =============================
+
+# That is the only and less hacky way to setup global variables
+# to use in strategy matrix (env:/YAML anchors doesn't work).
+# See: https://github.com/orgs/community/discussions/56787#discussioncomment-6041789
+#      https://github.com/actions/runner/issues/1182
+#      https://stackoverflow.com/a/77549656
+
+  variables:
+    runs-on: ubuntu-latest
+    outputs:
+      GHC_VER: 9.6.3
+      JAVA_VER: 17
+    steps:
+      - name: Dummy job when we have just simple variables
+        if: false
+        run: echo
+
+# =============================
+#       Create release
+# =============================
+
+# Create release, but only if it's triggered by tag push.
+# On pull requests/commits push, this job will always complete.
+
+  maybe-release:
     runs-on: ubuntu-latest
     steps:
       - name: Clone project
+        if: startsWith(github.ref, 'refs/tags/v')
         uses: actions/checkout@v3
 
       - name: Build changelog
         id: build_changelog
+        if: startsWith(github.ref, 'refs/tags/v')
         uses: simplex-chat/release-changelog-builder-action@v4
         with:
           configuration: .github/changelog_conf.json
@@ -42,6 +83,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create release
+        if: startsWith(github.ref, 'refs/tags/v')
         uses: simplex-chat/action-gh-release@v1
         with:
           body: ${{ steps.build_changelog.outputs.changelog }}
@@ -52,183 +94,259 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  build:
+# =========================
-    name: build-${{ matrix.os }}-${{ matrix.ghc }}
+#        Linux Build
-    if: always()
+# =========================
-    needs: prepare-release
+
-    runs-on: ${{ matrix.os }}
+  build-linux:
+    name: "ubuntu-${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}"
+    needs: [maybe-release, variables]
+    runs-on: ubuntu-${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: 20.04
             ghc: "8.10.7"
-            cache_path: ~/.cabal/store
+          - os: 20.04
-          - os: ubuntu-20.04
+            ghc: ${{ needs.variables.outputs.GHC_VER }}
-            ghc: "9.6.3"
+            cli_asset_name: simplex-chat-ubuntu-20_04-x86-64
-            cache_path: ~/.cabal/store
-            asset_name: simplex-chat-ubuntu-20_04-x86-64
             desktop_asset_name: simplex-desktop-ubuntu-20_04-x86_64.deb
-          - os: ubuntu-22.04
+          - os: 22.04
-            ghc: "9.6.3"
+            ghc: ${{ needs.variables.outputs.GHC_VER }}
-            cache_path: ~/.cabal/store
+            cli_asset_name: simplex-chat-ubuntu-22_04-x86-64
-            asset_name: simplex-chat-ubuntu-22_04-x86-64
             desktop_asset_name: simplex-desktop-ubuntu-22_04-x86_64.deb
-          - os: macos-latest
-            ghc: "9.6.3"
-            cache_path: ~/.cabal/store
-            asset_name: simplex-chat-macos-aarch64
-            desktop_asset_name: simplex-desktop-macos-aarch64.dmg
-          - os: macos-13
-            ghc: "9.6.3"
-            cache_path: ~/.cabal/store
-            asset_name: simplex-chat-macos-x86-64
-            desktop_asset_name: simplex-desktop-macos-x86_64.dmg
-          - os: windows-latest
-            ghc: "9.6.3"
-            cache_path: C:/cabal
-            asset_name: simplex-chat-windows-x86-64
-            desktop_asset_name: simplex-desktop-windows-x86_64.msi
-
     steps:
-      - name: Skip unreliable ghc 8.10.7 build on stable branch
+      - name: Checkout Code
-        if: matrix.ghc == '8.10.7' && github.ref == 'refs/heads/stable'
-        run: exit 0
-
-      - name: Configure pagefile (Windows)
-        if: matrix.os == 'windows-latest'
-        uses: simplex-chat/configure-pagefile-action@v1.3
-        with:
-          minimum-size: 16GB
-          maximum-size: 16GB
-          disk-root: "C:"
-
-      - name: Clone project
         uses: actions/checkout@v3
 
-      - name: Setup Haskell
+      # Otherwise we run out of disk space with Docker build
-        uses: simplex-chat/setup-haskell-action@v2
+      - name: Free disk space
-        with:
+        shell: bash
-          ghc-version: ${{ matrix.ghc }}
+        run: ./scripts/ci/linux_util_free_space.sh
-          cabal-version: "3.10.1.0"
 
       - name: Restore cached build
-        id: restore_cache
+        uses: actions/cache@v4
-        uses: actions/cache/restore@v3
         with:
           path: |
-            ${{ matrix.cache_path }}
+            ~/.cabal/store
             dist-newstyle
-          key: ${{ matrix.os }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}
+          key: ubuntu-${{ matrix.os }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}
 
-      # / Unix
+      - name: Set up Docker Buildx
+        uses: simplex-chat/docker-setup-buildx-action@v3
 
-      - name: Unix prepare cabal.project.local for Mac
+      - name: Build and cache Docker image
-        if: matrix.os == 'macos-latest'
+        uses: simplex-chat/docker-build-push-action@v6
+        with:
+          context: .
+          load: true
+          file: Dockerfile.build
+          tags: build/${{ matrix.os }}:latest
+          build-args: |
+            TAG=${{ matrix.os }}
+            GHC=${{ matrix.ghc }}
+
+      # Docker needs these flags for AppImage build:
+      # --device /dev/fuse
+      # --cap-add SYS_ADMIN
+      # --security-opt apparmor:unconfined
+      - name: Start container
         shell: bash
         run: |
-          echo "ignore-project: False"                                         >> cabal.project.local
+          docker run -t -d \
-          echo "package simplexmq"                                             >> cabal.project.local
+            --device /dev/fuse \
-          echo "    extra-include-dirs: /opt/homebrew/opt/openssl@3.0/include" >> cabal.project.local
+            --cap-add SYS_ADMIN \
-          echo "    extra-lib-dirs: /opt/homebrew/opt/openssl@3.0/lib"         >> cabal.project.local
+            --security-opt apparmor:unconfined \
-          echo ""                                                              >> cabal.project.local
+            --name builder \
-          echo "package direct-sqlcipher"                                      >> cabal.project.local
+            -v ~/.cabal:/root/.cabal \
-          echo "    extra-include-dirs: /opt/homebrew/opt/openssl@3.0/include" >> cabal.project.local
+            -v /home/runner/work/_temp:/home/runner/work/_temp \
-          echo "    extra-lib-dirs: /opt/homebrew/opt/openssl@3.0/lib"         >> cabal.project.local
+            -v ${{ github.workspace }}:/project \
-          echo "    flags: +openssl"                                           >> cabal.project.local
+            build/${{ matrix.os }}:latest
 
-      - name: Unix prepare cabal.project.local for Mac
+      - name: Prepare cabal.project.local
-        if: matrix.os == 'macos-13'
-        shell: bash
-        run: |
-          echo "ignore-project: False"                                         >> cabal.project.local
-          echo "package simplexmq"                                             >> cabal.project.local
-          echo "    extra-include-dirs: /usr/local/opt/openssl@3.0/include"    >> cabal.project.local
-          echo "    extra-lib-dirs: /usr/local/opt/openssl@3.0/lib"            >> cabal.project.local
-          echo ""                                                              >> cabal.project.local
-          echo "package direct-sqlcipher"                                      >> cabal.project.local
-          echo "    extra-include-dirs: /usr/local/opt/openssl@3.0/include"    >> cabal.project.local
-          echo "    extra-lib-dirs: /usr/local/opt/openssl@3.0/lib"            >> cabal.project.local
-          echo "    flags: +openssl"                                           >> cabal.project.local
-
-      - name: Install AppImage dependencies
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04'
-        run: sudo apt install -y desktop-file-utils
-
-      - name: Install openssl for Mac
-        if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
-        run: brew install openssl@3.0
-
-      - name: Unix prepare cabal.project.local for Ubuntu
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04'
         shell: bash
         run: |
           echo "ignore-project: False"    >> cabal.project.local
           echo "package direct-sqlcipher" >> cabal.project.local
           echo "    flags: +openssl"      >> cabal.project.local
 
-      - name: Unix build CLI
+      # chmod/git commands are used to workaround permission issues when cache is restored
-        id: unix_cli_build
+      - name: Build CLI
-        if: matrix.os != 'windows-latest'
+        shell: docker exec -t builder sh -eu {0}
+        run: |
+          chmod -R 777 dist-newstyle ~/.cabal && git config --global --add safe.directory '*'
+          cabal clean
+          cabal update
+          cabal build -j --enable-tests
+          mkdir -p /out
+          for i in simplex-chat simplex-chat-test; do
+            bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
+            chmod +x "$bin"
+            mv "$bin" /out/
+          done
+          strip /out/simplex-chat
+
+      - name: Copy tests from container
         shell: bash
         run: |
-          cabal build --enable-tests
+          docker cp builder:/out/simplex-chat-test .
-          path=$(cabal list-bin simplex-chat)
-          echo "bin_path=$path" >> $GITHUB_OUTPUT
-          echo "bin_hash=$(echo SHA2-512\(${{ matrix.asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Unix upload CLI binary to release
+      - name: Copy CLI from container and prepare it
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os != 'windows-latest'
+        id: linux_cli_prepare
-        uses: simplex-chat/upload-release-action@v2
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ steps.unix_cli_build.outputs.bin_path }}
-          asset_name: ${{ matrix.asset_name }}
-          tag: ${{ github.ref }}
-
-      - name: Unix update CLI binary hash
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os != 'windows-latest'
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.unix_cli_build.outputs.bin_hash }}
-
-      - name: Setup Java
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'corretto'
-          java-version: '17'
-          cache: 'gradle'
-
-      - name: Linux build desktop
-        id: linux_desktop_build
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04')
         shell: bash
+        run: |
+          docker cp builder:/out/simplex-chat ./${{ matrix.cli_asset_name }}
+          path="${{ github.workspace }}/${{ matrix.cli_asset_name }}"
+          echo "bin_path=$path" >> $GITHUB_OUTPUT
+          echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
+
+      - name: Upload CLI
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name
+        uses: ./.github/actions/prepare-release
+        with:
+          bin_path: ${{ steps.linux_cli_prepare.outputs.bin_path }}
+          bin_name: ${{ matrix.cli_asset_name }}
+          bin_hash: ${{ steps.linux_cli_prepare.outputs.bin_hash }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Desktop
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name
+        shell: docker exec -t builder sh -eu {0}
         run: |
           scripts/desktop/build-lib-linux.sh
           cd apps/multiplatform
           ./gradlew packageDeb
-          path=$(echo $PWD/release/main/deb/simplex_*_amd64.deb)
+
+      - name: Prepare Desktop
+        id: linux_desktop_build
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name
+        shell: bash
+        run: |
+          path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/deb/simplex_*_amd64.deb )
           echo "package_path=$path" >> $GITHUB_OUTPUT
           echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Linux make AppImage
+      - name: Upload Desktop
-        id: linux_appimage_build
+        uses: ./.github/actions/prepare-release
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04'
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name
-        shell: bash
+        with:
+          bin_path: ${{ steps.linux_desktop_build.outputs.package_path }}
+          bin_name: ${{ matrix.desktop_asset_name }}
+          bin_hash: ${{ steps.linux_desktop_build.outputs.package_hash }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build AppImage
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04'
+        shell: docker exec -t builder sh -eu {0}
         run: |
           scripts/desktop/make-appimage-linux.sh
-          path=$(echo $PWD/apps/multiplatform/release/main/*imple*.AppImage)
+
+      - name: Prepare AppImage
+        id: linux_appimage_build
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04'
+        shell: bash
+        run: |
+          path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/*imple*.AppImage)
           echo "appimage_path=$path" >> $GITHUB_OUTPUT
           echo "appimage_hash=$(echo SHA2-512\(simplex-desktop-x86_64.AppImage\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Mac build desktop
+      - name: Upload AppImage
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04'
+        uses: ./.github/actions/prepare-release
+        with:
+          bin_path: ${{ steps.linux_appimage_build.outputs.appimage_path }}
+          bin_name: "simplex-desktop-x86_64.AppImage"
+          bin_hash: ${{ steps.linux_appimage_build.outputs.appimage_hash }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Fix permissions for cache
+        shell: bash
+        run: |
+          sudo chmod -R 777 dist-newstyle ~/.cabal
+          sudo chown -R $(id -u):$(id -g) dist-newstyle ~/.cabal
+
+      - name: Run tests
+        shell: bash
+        run: |
+          ./simplex-chat-test
+
+# =========================
+#        MacOS Build
+# =========================
+
+  build-macos:
+    name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}"
+    needs: [maybe-release, variables]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-latest
+            ghc: ${{ needs.variables.outputs.GHC_VER }}
+            cli_asset_name: simplex-chat-macos-aarch64
+            desktop_asset_name: simplex-desktop-macos-aarch64.dmg
+            openssl_dir: "/opt/homebrew/opt"
+          - os: macos-13
+            ghc: ${{ needs.variables.outputs.GHC_VER }}
+            cli_asset_name: simplex-chat-macos-x86-64
+            desktop_asset_name: simplex-desktop-macos-x86_64.dmg
+            openssl_dir: "/usr/local/opt"
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Prepare build
+        uses: ./.github/actions/prepare-build
+        with:
+          java_ver: ${{ needs.variables.outputs.JAVA_VER }}
+          ghc_ver: ${{ matrix.ghc }}
+          os: ${{ matrix.os }}
+          github_ref: ${{ github.ref }}
+
+      - name: Install OpenSSL
+        run: brew install openssl@3.0
+
+      - name: Prepare cabal.project.local
+        shell: bash
+        run: |
+          echo "ignore-project: False"                                                    >> cabal.project.local
+          echo "package simplexmq"                                                        >> cabal.project.local
+          echo "    extra-include-dirs: ${{ matrix.opnessl_dir }}/openssl@3.0/include"    >> cabal.project.local
+          echo "    extra-lib-dirs: ${{ matrix.openssl_dir}}/openssl@3.0/lib"             >> cabal.project.local
+          echo ""                                                                         >> cabal.project.local
+          echo "package direct-sqlcipher"                                                 >> cabal.project.local
+          echo "    extra-include-dirs: ${{ matrix.openssl_dir }}/openssl@3.0/include"    >> cabal.project.local
+          echo "    extra-lib-dirs: ${{ matrix.openssl_dir }}/openssl@3.0/lib"            >> cabal.project.local
+          echo "    flags: +openssl"                                                      >> cabal.project.local
+
+      - name: Build CLI
+        id: mac_cli_build
+        shell: bash
+        run: |
+          cabal build -j --enable-tests
+          path=$(cabal list-bin simplex-chat)
+          echo "bin_path=$path" >> $GITHUB_OUTPUT
+          echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
+
+      - name: Upload CLI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: ./.github/actions/prepare-release
+        with:
+          bin_path: ${{ steps.mac_cli_build.outputs.bin_path }}
+          bin_name: ${{ matrix.cli_asset_name }}
+          bin_hash: ${{ steps.mac_cli_build.outputs.bin_hash }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Desktop
         id: mac_desktop_build
-        if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13')
+        if: startsWith(github.ref, 'refs/tags/v')
         shell: bash
         env:
           APPLE_SIMPLEX_SIGNING_KEYCHAIN: ${{ secrets.APPLE_SIMPLEX_SIGNING_KEYCHAIN }}
@@ -240,85 +358,58 @@ jobs:
           echo "package_path=$path" >> $GITHUB_OUTPUT
           echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Linux upload desktop package to release
+      - name: Upload Desktop
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04')
+        if: startsWith(github.ref, 'refs/tags/v')
-        uses: simplex-chat/upload-release-action@v2
+        uses: ./.github/actions/prepare-release
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          bin_path: ${{ steps.mac_desktop_build.outputs.package_path }}
-          file: ${{ steps.linux_desktop_build.outputs.package_path }}
+          bin_name: ${{ matrix.desktop_asset_name }}
-          asset_name: ${{ matrix.desktop_asset_name }}
+          bin_hash: ${{ steps.mac_desktop_build.outputs.package_hash }}
-          tag: ${{ github.ref }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Linux update desktop package hash
+      - name: Run tests
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && (matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04')
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.linux_desktop_build.outputs.package_hash }}
-
-      - name: Linux upload AppImage to release
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04'
-        uses: simplex-chat/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ steps.linux_appimage_build.outputs.appimage_path }}
-          asset_name: simplex-desktop-x86_64.AppImage
-          tag: ${{ github.ref }}
-
-      - name: Linux update AppImage hash
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.asset_name && matrix.os == 'ubuntu-20.04'
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.linux_appimage_build.outputs.appimage_hash }}
-
-      - name: Mac upload desktop package to release
-        if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13')
-        uses: simplex-chat/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ steps.mac_desktop_build.outputs.package_path }}
-          asset_name: ${{ matrix.desktop_asset_name }}
-          tag: ${{ github.ref }}
-
-      - name: Mac update desktop package hash
-        if: startsWith(github.ref, 'refs/tags/v') && (matrix.os == 'macos-latest' || matrix.os == 'macos-13')
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.mac_desktop_build.outputs.package_hash }}
-
-      - name: Cache unix build
-        uses: actions/cache/save@v3
-        if: matrix.os != 'windows-latest'
-        with:
-          path: |
-            ${{ matrix.cache_path }}
-            dist-newstyle
-          key: ${{ steps.restore_cache.outputs.cache-primary-key }}
-
-      - name: Unix test
-        if: matrix.os != 'windows-latest'
         timeout-minutes: 40
         shell: bash
         run: cabal test --test-show-details=direct
 
-      # Unix /
+# =========================
+#      Windows Build
+# =========================
 
-      # / Windows
+  build-windows:
-      # rm -rf dist-newstyle/src/direct-sq* is here because of the bug in cabal's dependency which prevents second build from finishing
+    name: "${{ matrix.os }} (CLI,Desktop), GHC: ${{ matrix.ghc }}"
+    needs: [maybe-release, variables]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            ghc: ${{ needs.variables.outputs.GHC_VER }}
+            cli_asset_name: simplex-chat-windows-x86-64
+            desktop_asset_name: simplex-desktop-windows-x86_64.msi
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
 
+      - name: Prepare build
+        uses: ./.github/actions/prepare-build
+        with:
+          java_ver: ${{ needs.variables.outputs.JAVA_VER }}
+          ghc_ver: ${{ matrix.ghc }}
+          os: ${{ matrix.os }}
+          cache_path: "C:/cabal"
+          github_ref: ${{ github.ref }}
+
+      - name: Configure pagefile (Windows)
+        uses: simplex-chat/configure-pagefile-action@v1.4
+        with:
+          minimum-size: 16GB
+          maximum-size: 16GB
+          disk-root: "C:"
+  
       - name: 'Setup MSYS2'
-        if: matrix.os == 'windows-latest'
         uses: simplex-chat/setup-msys2@v2
         with:
           msystem: ucrt64
@@ -331,10 +422,9 @@ jobs:
             toolchain:p
             cmake:p
 
-
+      # rm -rf dist-newstyle/src/direct-sq* is here because of the bug in cabal's dependency which prevents second build from finishing
-      - name: Windows build
+      - name: Build CLI
-        id: windows_build
+        id: windows_cli_build
-        if: matrix.os == 'windows-latest'
         shell: msys2 {0}
         run: |
           export PATH=$PATH:/c/ghcup/bin:$(echo /c/tools/ghc-*/bin || echo)
@@ -349,70 +439,42 @@ jobs:
 
           rm -rf dist-newstyle/src/direct-sq*
           sed -i "s/, unix /--, unix /" simplex-chat.cabal
-          cabal build --enable-tests
+          cabal build -j --enable-tests
           rm -rf dist-newstyle/src/direct-sq*
           path=$(cabal list-bin simplex-chat | tail -n 1)
           echo "bin_path=$path" >> $GITHUB_OUTPUT
-          echo "bin_hash=$(echo SHA2-512\(${{ matrix.asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
+          echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Windows upload CLI binary to release
+      - name: Upload CLI
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest'
+        if: startsWith(github.ref, 'refs/tags/v')
-        uses: simplex-chat/upload-release-action@v2
+        uses: ./.github/actions/prepare-release
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          bin_path: ${{ steps.windows_cli_build.outputs.bin_path }}
-          file: ${{ steps.windows_build.outputs.bin_path }}
+          bin_name: ${{ matrix.cli_asset_name }}
-          asset_name: ${{ matrix.asset_name }}
+          bin_hash: ${{ steps.windows_cli_build.outputs.bin_hash }}
-          tag: ${{ github.ref }}
+          github_ref: ${{ github.ref }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Windows update CLI binary hash
+      - name: Build Desktop
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest'
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.windows_build.outputs.bin_hash }}
-
-      - name: Windows build desktop
         id: windows_desktop_build
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest'
+        if: startsWith(github.ref, 'refs/tags/v')
         shell: msys2 {0}
         run: |
           export PATH=$PATH:/c/ghcup/bin:$(echo /c/tools/ghc-*/bin || echo)
           scripts/desktop/build-lib-windows.sh
           cd apps/multiplatform
           ./gradlew packageMsi
+          rm -rf dist-newstyle/src/direct-sq*
           path=$(echo $PWD/release/main/msi/*imple*.msi | sed 's#/\([a-z]\)#\1:#' | sed 's#/#\\#g')
           echo "package_path=$path" >> $GITHUB_OUTPUT
           echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT
 
-      - name: Windows upload desktop package to release
+      - name: Upload Desktop
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest'
+        if: startsWith(github.ref, 'refs/tags/v')
-        uses: simplex-chat/upload-release-action@v2
+        uses: ./.github/actions/prepare-release
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          bin_path: ${{ steps.windows_desktop_build.outputs.package_path }}
-          file: ${{ steps.windows_desktop_build.outputs.package_path }}
+          bin_name: ${{ matrix.desktop_asset_name }}
-          asset_name: ${{ matrix.desktop_asset_name }}
+          bin_hash: ${{ steps.windows_desktop_build.outputs.package_hash }}
-          tag: ${{ github.ref }}
+          github_ref: ${{ github.ref }}
-
+          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Windows update desktop package hash
-        if: startsWith(github.ref, 'refs/tags/v') && matrix.os == 'windows-latest'
-        uses: simplex-chat/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          append_body: true
-          body: |
-            ${{ steps.windows_desktop_build.outputs.package_hash }}
-
-      - name: Cache windows build
-        uses: actions/cache/save@v3
-        if: matrix.os == 'windows-latest'
-        with:
-          path: |
-            ${{ matrix.cache_path }}
-            dist-newstyle
-          key: ${{ steps.restore_cache.outputs.cache-primary-key }}
-
-      # Windows /

.github/workflows/reproduce-schedule.yml

@@ -0,0 +1,45 @@
+name: Reproduce latest release
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *' # every day at 02:00 night
+
+jobs:
+  reproduce:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get latest release
+        shell: bash
+        run: |
+          curl --proto '=https' \
+               --tlsv1.2 \
+               -sSf -L \
+               'https://api.github.com/repos/simplex-chat/simplex-chat/releases/latest' \
+               2>/dev/null | \
+          grep -i "tag_name" | \
+          awk -F \" '{print "TAG="$4}' >> $GITHUB_ENV
+
+      - name: Execute reproduce script
+        run: |
+          ${GITHUB_WORKSPACE}/scripts/reproduce-builds.sh "$TAG"
+
+      - name: Check if build has been reproduced
+        env:
+          url: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_URL }}
+          user: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_USER }}
+          pass: ${{ secrets.STATUS_SIMPLEX_WEBHOOK_PASS }}
+        run: |
+          if [ -f "${GITHUB_WORKSPACE}/$TAG/_sha256sums" ]; then
+            exit 0
+          else
+            curl --proto '=https' --tlsv1.2 -sSf \
+              -u "${user}:${pass}" \
+              -H 'Content-Type: application/json' \
+              -d '{"title": "👾 GitHub: Runner", "description": "⛔️ '"$TAG"' did not reproduce."}' \
+              "$url"
+            exit 1
+          fi

Dockerfile.build

@@ -0,0 +1,84 @@
+# syntax=docker/dockerfile:1.7.0-labs
+ARG TAG=24.04
+FROM ubuntu:${TAG} AS build
+
+### Build stage
+
+ARG GHC=9.6.3
+ARG CABAL=3.10.1.0
+ARG JAVA=17
+
+ENV TZ=Etc/UTC \
+    DEBIAN_FRONTEND=noninteractive
+
+# Install curl, git and and simplexmq dependencies
+RUN apt-get update && \
+    apt-get install -y curl \
+                       libpq-dev \
+                       git \
+                       sqlite3 \
+                       libsqlite3-dev \
+                       build-essential \
+                       libgmp3-dev \
+                       zlib1g-dev \
+                       llvm \
+                       cmake \
+                       llvm-dev \
+                       libnuma-dev \
+                       libssl-dev \
+                       desktop-file-utils \
+                       openjdk-${JAVA}-jdk-headless \
+                       patchelf \
+                       ca-certificates \
+                       zip \
+                       wget \
+                       fuse3 \
+                       file \
+                       appstream \
+                       gpg \
+                       unzip &&\
+    export JAVA_HOME=$(update-java-alternatives -l | head -n 1 | awk -F ' ' '{print $NF}') &&\
+    ln -s /bin/fusermount /bin/fusermount3 || :
+
+# Specify bootstrap Haskell versions
+ENV BOOTSTRAP_HASKELL_GHC_VERSION=${GHC}
+ENV BOOTSTRAP_HASKELL_CABAL_VERSION=${CABAL}
+
+# Do not install Stack
+ENV BOOTSTRAP_HASKELL_INSTALL_NO_STACK=true
+ENV BOOTSTRAP_HASKELL_INSTALL_NO_STACK_HOOK=true
+
+# Install ghcup
+RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 sh
+
+# Adjust PATH
+ENV PATH="/root/.cabal/bin:/root/.ghcup/bin:$PATH"
+
+# Set both as default
+RUN ghcup set ghc "${GHC}" && \
+    ghcup set cabal "${CABAL}"
+
+#=====================
+# Install Android SDK
+#=====================
+ARG SDK_VERSION=13114758
+
+ENV SDK_VERSION=$SDK_VERSION \
+    ANDROID_HOME=/root
+
+RUN curl -L -o tools.zip "https://dl.google.com/android/repository/commandlinetools-linux-${SDK_VERSION}_latest.zip" && \
+    unzip tools.zip && rm tools.zip && \
+    mv cmdline-tools tools && mkdir "$ANDROID_HOME/cmdline-tools" && mv tools "$ANDROID_HOME/cmdline-tools/" && \
+    ln -s "$ANDROID_HOME/cmdline-tools/tools" "$ANDROID_HOME/cmdline-tools/latest"
+
+ENV PATH="$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/tools/bin"
+
+# https://askubuntu.com/questions/885658/android-sdk-repositories-cfg-could-not-be-loaded
+RUN mkdir -p ~/.android ~/.gradle && \
+    touch ~/.android/repositories.cfg && \
+    echo 'org.gradle.console=plain' > ~/.gradle/gradle.properties &&\
+    yes | sdkmanager --licenses >/dev/null
+
+ENV PATH=$PATH:$ANDROID_HOME/platform-tools:$ANDROID_HOME/build-tools
+
+WORKDIR /project

scripts/ci/linux_util_free_space.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Taken from: https://github.com/apache/arrow/blob/main/ci/scripts/util_free_space.sh
+
+set -eux
+
+df -h
+echo "::group::/usr/local/*"
+du -hsc /usr/local/*
+echo "::endgroup::"
+# ~1GB
+sudo rm -rf \
+  /usr/local/aws-sam-cil \
+  /usr/local/julia* || :
+echo "::group::/usr/local/bin/*"
+du -hsc /usr/local/bin/*
+echo "::endgroup::"
+# ~1GB (From 1.2GB to 214MB)
+sudo rm -rf \
+  /usr/local/bin/aliyun \
+  /usr/local/bin/azcopy \
+  /usr/local/bin/bicep \
+  /usr/local/bin/cmake-gui \
+  /usr/local/bin/cpack \
+  /usr/local/bin/helm \
+  /usr/local/bin/hub \
+  /usr/local/bin/kubectl \
+  /usr/local/bin/minikube \
+  /usr/local/bin/node \
+  /usr/local/bin/packer \
+  /usr/local/bin/pulumi* \
+  /usr/local/bin/sam \
+  /usr/local/bin/stack \
+  /usr/local/bin/terraform || :
+# 142M
+sudo rm -rf /usr/local/bin/oc || : \
+echo "::group::/usr/local/share/*"
+du -hsc /usr/local/share/*
+echo "::endgroup::"
+# 506MB
+sudo rm -rf /usr/local/share/chromium || :
+# 1.3GB
+sudo rm -rf /usr/local/share/powershell || :
+echo "::group::/usr/local/lib/*"
+du -hsc /usr/local/lib/*
+echo "::endgroup::"
+# 15GB
+sudo rm -rf /usr/local/lib/android || :
+# 341MB
+sudo rm -rf /usr/local/lib/heroku || :
+# 1.2GB
+sudo rm -rf /usr/local/lib/node_modules || :
+echo "::group::/opt/*"
+du -hsc /opt/*
+echo "::endgroup::"
+# 679MB
+sudo rm -rf /opt/az || :
+echo "::group::/opt/microsoft/*"
+du -hsc /opt/microsoft/*
+echo "::endgroup::"
+# 197MB
+sudo rm -rf /opt/microsoft/powershell || :
+echo "::group::/opt/hostedtoolcache/*"
+du -hsc /opt/hostedtoolcache/*
+echo "::endgroup::"
+# 5.3GB
+sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+# 1.4GB
+sudo rm -rf /opt/hostedtoolcache/go || :
+# 489MB
+sudo rm -rf /opt/hostedtoolcache/PyPy || :
+# 376MB
+sudo rm -rf /opt/hostedtoolcache/node || :
+# Remove Web browser packages
+sudo apt purge -y \
+  firefox \
+  google-chrome-stable \
+  microsoft-edge-stable
+df -h

scripts/reproduce-builds.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env sh
+set -eu
+
+TAG="$1"
+
+tempdir="$(mktemp -d)"
+init_dir="$PWD"
+
+repo_name="simplex-chat"
+repo="https://github.com/simplex-chat/${repo_name}"
+
+cabal_local='ignore-project: False
+package direct-sqlcipher
+    flags: +openssl'
+
+export DOCKER_BUILDKIT=1
+
+cleanup() {
+	docker exec -t builder sh -c 'rm -rf ./dist-newstyle' 2>/dev/null || :
+	rm -rf -- "$tempdir"
+	docker rm --force builder 2>/dev/null || :
+	docker image rm local 2>/dev/null || :
+	cd "$init_dir"
+}
+trap 'cleanup' EXIT INT
+
+mkdir -p "$init_dir/$TAG/from-source" "$init_dir/$TAG/prebuilt"
+
+git -C "$tempdir" clone "$repo.git" &&\
+	cd "$tempdir/${repo_name}" &&\
+	git checkout "$TAG"
+
+for os in 20.04 22.04; do
+	os_url="$(printf '%s' "$os" | tr '.' '_')"
+
+	# Build image
+	docker build \
+		--no-cache \
+		--build-arg TAG=${os} \
+		--build-arg GHC=9.6.3 \
+		-f "$tempdir/${repo_name}/Dockerfile.build" \
+		-t local \
+		.
+
+	printf '%s' "$cabal_local" > "$tempdir/${repo_name}/cabal.project.local"
+
+	# Run container in background
+	docker run -t -d \
+		--name builder \
+		-v "$tempdir/${repo_name}:/project" \
+		local
+
+	docker exec \
+		-t \
+		builder \
+		sh -c 'cabal clean && cabal update && cabal build -j --enable-tests && mkdir -p /out && for i in simplex-chat; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable) && chmod +x "$bin" && mv "$bin" /out/; done && strip /out/simplex-chat'
+
+	docker cp \
+		builder:/out/simplex-chat \
+		"$init_dir/$TAG/from-source/simplex-chat-ubuntu-${os_url}-x86-64"
+
+	# Download prebuilt postgresql binary
+	curl -L \
+		--output-dir "$init_dir/$TAG/prebuilt/" \
+		-O \
+		"$repo/releases/download/${TAG}/simplex-chat-ubuntu-${os_url}-x86-64"
+	
+	# Important! Remove dist-newstyle for the next interation
+	docker exec \
+		-t \
+		builder \
+		sh -c 'rm -rf ./dist-newstyle'
+
+	# Also restore git to previous state 
+	git reset --hard && git clean -dfx
+
+	# Stop containers, delete images
+	docker stop builder
+	docker rm --force builder
+	docker image rm local
+done
+
+# Cleanup
+rm -rf -- "$tempdir"
+cd "$init_dir"
+
+# Final stage: compare hashes
+
+# Path to binaries
+path_bin="$init_dir/$TAG"
+
+# Assume everything is okay for now
+bad=0
+
+# Check hashes for all binaries
+for file in "$path_bin"/from-source/*; do
+	# Extract binary name
+	app="$(basename $file)"
+
+	# Compute hash for compiled binary
+	compiled=$(sha256sum "$path_bin/from-source/$app" | awk '{print $1}')
+	# Compute hash for prebuilt binary
+	prebuilt=$(sha256sum "$path_bin/prebuilt/$app" | awk '{print $1}')
+
+	# Compare
+	if [ "$compiled" != "$prebuilt" ]; then
+		# If hashes doesn't match, set bad...
+		bad=1
+
+		# ... and print affected binary
+		printf "%s - sha256sum hash doesn't match\n" "$app"
+	fi
+done
+
+# If everything is still okay, compute checksums file
+if [ "$bad" = 0 ]; then
+	sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' > "$path_bin/_sha256sums"
+
+	printf 'Checksums computed - %s\n' "$path_bin/_sha256sums"
+fi