From 5351fa68d06bf24f2910b4393b72da4f6983e6e1 Mon Sep 17 00:00:00 2001 From: sh <37271604+shumvgolove@users.noreply.github.com> Date: Wed, 23 Apr 2025 12:27:30 +0000 Subject: [PATCH] ci: switch to sha256 and skip 8.10.7 on release (#5837) * ci: skip 8.10.7 on release * ci: switch to sha256 * script/reproduce-builds: make it executable * scripts/reproduce-builds: rename to simplex-chat-reproduce-builds * ci: bump actions * ci: 20.04 is deprecated * scripts/reproduce-builds: remove Ubuntu 20.04 * docs: adjust reproduce script * ci: skip 8.10.7 in stable or release for Linux * ci: really skup 8.10.7 in stable or release * ci: remove useless linux checks * ci: remove timeout from mac tests * ci: fix action names * ci: setup swap for 8.10.7 * ci: bump swap to 30gb * ci: simplify * ci: 10 -> 3 retries * ci: retry only in stable or release --- .github/actions/prepare-build/action.yml | 5 - .github/actions/prepare-release/action.yml | 4 +- .github/actions/swap/action.yml | 44 ++++++++ .github/workflows/build.yml | 102 +++++++++++++----- docs/SERVER.md | 6 +- ...ds.sh => simplex-chat-reproduce-builds.sh} | 12 +-- 6 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 .github/actions/swap/action.yml rename scripts/{reproduce-builds.sh => simplex-chat-reproduce-builds.sh} (88%) mode change 100644 => 100755 diff --git a/.github/actions/prepare-build/action.yml b/.github/actions/prepare-build/action.yml index 6682641419..ce75b7a57c 100644 --- a/.github/actions/prepare-build/action.yml +++ b/.github/actions/prepare-build/action.yml @@ -24,11 +24,6 @@ inputs: runs: using: "composite" steps: - - name: Skip unreliable ghc 8.10.7 build on stable branch - shell: bash - if: inputs.ghc_ver == '8.10.7' && inputs.github_ref == 'refs/heads/stable' - run: exit 0 - - name: Setup Haskell uses: simplex-chat/setup-haskell-action@v2 with: diff --git a/.github/actions/prepare-release/action.yml b/.github/actions/prepare-release/action.yml index e44e6ef0f2..e0d32bd596 100644 --- a/.github/actions/prepare-release/action.yml +++ b/.github/actions/prepare-release/action.yml @@ -19,7 +19,7 @@ inputs: runs: using: "composite" steps: - - name: Linux upload AppImage to release + - name: Upload file with specific name if: startsWith(inputs.github_ref, 'refs/tags/v') uses: simplex-chat/upload-release-action@v2 with: @@ -28,7 +28,7 @@ runs: asset_name: ${{ inputs.bin_name }} tag: ${{ inputs.github_ref }} - - name: Linux update AppImage hash + - name: Add hash to release notes if: startsWith(inputs.github_ref, 'refs/tags/v') uses: simplex-chat/action-gh-release@v2 env: diff --git a/.github/actions/swap/action.yml b/.github/actions/swap/action.yml new file mode 100644 index 0000000000..87d670b147 --- /dev/null +++ b/.github/actions/swap/action.yml @@ -0,0 +1,44 @@ +name: 'Set Swap Space' +description: 'Add moar swap' +branding: + icon: 'crop' + color: 'orange' +inputs: + swap-size-gb: + description: 'Swap space to create, in Gigabytes.' + required: false + default: '10' +runs: + using: "composite" + steps: + - name: Swap space report before modification + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + swapon --show + echo + - name: Set Swap + shell: bash + run: | + export SWAP_FILE=$(swapon --show=NAME | tail -n 1) + echo "Swap file: $SWAP_FILE" + if [ -z "$SWAP_FILE" ]; then + SWAP_FILE=/opt/swapfile + else + sudo swapoff $SWAP_FILE + sudo rm $SWAP_FILE + fi + sudo fallocate -l ${{ inputs.swap-size-gb }}G $SWAP_FILE + sudo chmod 600 $SWAP_FILE + sudo mkswap $SWAP_FILE + sudo swapon $SWAP_FILE + - name: Swap space report after modification + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + swapon --show + echo diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de0b976bcc..ca1bc79510 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,7 +73,7 @@ jobs: - name: Build changelog id: build_changelog if: startsWith(github.ref, 'refs/tags/v') - uses: simplex-chat/release-changelog-builder-action@v4 + uses: simplex-chat/release-changelog-builder-action@v5 with: configuration: .github/changelog_conf.json failOnError: true @@ -84,7 +84,7 @@ jobs: - name: Create release if: startsWith(github.ref, 'refs/tags/v') - uses: simplex-chat/action-gh-release@v1 + uses: simplex-chat/action-gh-release@v2 with: body: ${{ steps.build_changelog.outputs.changelog }} prerelease: true @@ -106,26 +106,38 @@ jobs: fail-fast: false matrix: include: - - os: 20.04 + - os: 22.04 ghc: "8.10.7" - - os: 20.04 - ghc: ${{ needs.variables.outputs.GHC_VER }} - cli_asset_name: simplex-chat-ubuntu-20_04-x86-64 - desktop_asset_name: simplex-desktop-ubuntu-20_04-x86_64.deb + should_run: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} - os: 22.04 ghc: ${{ needs.variables.outputs.GHC_VER }} cli_asset_name: simplex-chat-ubuntu-22_04-x86-64 desktop_asset_name: simplex-desktop-ubuntu-22_04-x86_64.deb + should_run: true + - os: 24.04 + ghc: ${{ needs.variables.outputs.GHC_VER }} + cli_asset_name: simplex-chat-ubuntu-24_04-x86-64 + desktop_asset_name: simplex-desktop-ubuntu-24_04-x86_64.deb + should_run: true steps: - name: Checkout Code + if: matrix.should_run == true uses: actions/checkout@v3 + - name: Setup swap + if: matrix.ghc == '8.10.7' && matrix.should_run == true + uses: ./.github/actions/swap + with: + swap-size-gb: 30 + # Otherwise we run out of disk space with Docker build - name: Free disk space + if: matrix.should_run == true shell: bash run: ./scripts/ci/linux_util_free_space.sh - name: Restore cached build + if: matrix.should_run == true uses: actions/cache@v4 with: path: | @@ -134,9 +146,11 @@ jobs: key: ubuntu-${{ matrix.os }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }} - name: Set up Docker Buildx + if: matrix.should_run == true uses: simplex-chat/docker-setup-buildx-action@v3 - name: Build and cache Docker image + if: matrix.should_run == true uses: simplex-chat/docker-build-push-action@v6 with: context: . @@ -152,6 +166,7 @@ jobs: # --cap-add SYS_ADMIN # --security-opt apparmor:unconfined - name: Start container + if: matrix.should_run == true shell: bash run: | docker run -t -d \ @@ -165,6 +180,7 @@ jobs: build/${{ matrix.os }}:latest - name: Prepare cabal.project.local + if: matrix.should_run == true shell: bash run: | echo "ignore-project: False" >> cabal.project.local @@ -173,6 +189,7 @@ jobs: # chmod/git commands are used to workaround permission issues when cache is restored - name: Build CLI + if: matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | chmod -R 777 dist-newstyle ~/.cabal && git config --global --add safe.directory '*' @@ -188,22 +205,23 @@ jobs: strip /out/simplex-chat - name: Copy tests from container + if: matrix.should_run == true shell: bash run: | docker cp builder:/out/simplex-chat-test . - name: Copy CLI from container and prepare it id: linux_cli_prepare - if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true shell: bash run: | docker cp builder:/out/simplex-chat ./${{ matrix.cli_asset_name }} path="${{ github.workspace }}/${{ matrix.cli_asset_name }}" echo "bin_path=$path" >> $GITHUB_OUTPUT - echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-256\(${{ matrix.cli_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload CLI - if: startsWith(github.ref, 'refs/tags/v') && matrix.cli_asset_name + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true uses: ./.github/actions/prepare-release with: bin_path: ${{ steps.linux_cli_prepare.outputs.bin_path }} @@ -213,7 +231,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Build Desktop - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | scripts/desktop/build-lib-linux.sh @@ -222,16 +240,16 @@ jobs: - name: Prepare Desktop id: linux_desktop_build - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true shell: bash run: | path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/deb/simplex_*_amd64.deb ) echo "package_path=$path" >> $GITHUB_OUTPUT - echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "package_hash=$(echo SHA2-256\(${{ matrix.desktop_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload Desktop uses: ./.github/actions/prepare-release - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name + if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true with: bin_path: ${{ steps.linux_desktop_build.outputs.package_path }} bin_name: ${{ matrix.desktop_asset_name }} @@ -240,22 +258,22 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Build AppImage - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + if: startsWith(github.ref, 'refs/tags/v') && matrix.os == '22.04' && matrix.should_run == true shell: docker exec -t builder sh -eu {0} run: | scripts/desktop/make-appimage-linux.sh - name: Prepare AppImage id: linux_appimage_build - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + if: startsWith(github.ref, 'refs/tags/v') && matrix.os == '22.04' && matrix.should_run == true shell: bash run: | path=$(echo ${{ github.workspace }}/apps/multiplatform/release/main/*imple*.AppImage) echo "appimage_path=$path" >> $GITHUB_OUTPUT - echo "appimage_hash=$(echo SHA2-512\(simplex-desktop-x86_64.AppImage\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "appimage_hash=$(echo SHA2-256\(simplex-desktop-x86_64.AppImage\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload AppImage - if: startsWith(github.ref, 'refs/tags/v') && matrix.desktop_asset_name && matrix.os == '20.04' + if: startsWith(github.ref, 'refs/tags/v') && matrix.os == '22.04' && matrix.should_run == true uses: ./.github/actions/prepare-release with: bin_path: ${{ steps.linux_appimage_build.outputs.appimage_path }} @@ -265,15 +283,33 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Fix permissions for cache + if: matrix.should_run == true shell: bash run: | sudo chmod -R 777 dist-newstyle ~/.cabal sudo chown -R $(id -u):$(id -g) dist-newstyle ~/.cabal - name: Run tests + if: matrix.should_run == true + timeout-minutes: 120 shell: bash run: | - ./simplex-chat-test + i=1 + attempts=1 + ${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} && attempts=3 + while [ "$i" -le "$attempts" ]; do + if ./simplex-chat-test; then + break + else + echo "Attempt $i failed, retrying..." + i=$((i + 1)) + sleep 1 + fi + done + if [ "$i" -gt "$attempts" ]; then + echo "All "$attempts" attempts failed." + exit 1 + fi # ========================= # MacOS Build @@ -332,7 +368,7 @@ jobs: cabal build -j --enable-tests path=$(cabal list-bin simplex-chat) echo "bin_path=$path" >> $GITHUB_OUTPUT - echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-256\(${{ matrix.cli_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload CLI if: startsWith(github.ref, 'refs/tags/v') @@ -356,7 +392,7 @@ jobs: scripts/ci/build-desktop-mac.sh path=$(echo $PWD/apps/multiplatform/release/main/dmg/SimpleX-*.dmg) echo "package_path=$path" >> $GITHUB_OUTPUT - echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "package_hash=$(echo SHA2-256\(${{ matrix.desktop_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload Desktop if: startsWith(github.ref, 'refs/tags/v') @@ -369,9 +405,25 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests - timeout-minutes: 40 + timeout-minutes: 120 shell: bash - run: cabal test --test-show-details=direct + run: | + i=1 + attempts=1 + ${{ (github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} && attempts=3 + while [ "$i" -le "$attempts" ]; do + if cabal test --test-show-details=direct; then + break + else + echo "Attempt $i failed, retrying..." + i=$((i + 1)) + sleep 1 + fi + done + if [ "$i" -gt "$attempts" ]; then + echo "All "$attempts" attempts failed." + exit 1 + fi # ========================= # Windows Build @@ -443,7 +495,7 @@ jobs: rm -rf dist-newstyle/src/direct-sq* path=$(cabal list-bin simplex-chat | tail -n 1) echo "bin_path=$path" >> $GITHUB_OUTPUT - echo "bin_hash=$(echo SHA2-512\(${{ matrix.cli_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "bin_hash=$(echo SHA2-256\(${{ matrix.cli_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload CLI if: startsWith(github.ref, 'refs/tags/v') @@ -467,7 +519,7 @@ jobs: rm -rf dist-newstyle/src/direct-sq* path=$(echo $PWD/release/main/msi/*imple*.msi | sed 's#/\([a-z]\)#\1:#' | sed 's#/#\\#g') echo "package_path=$path" >> $GITHUB_OUTPUT - echo "package_hash=$(echo SHA2-512\(${{ matrix.desktop_asset_name }}\)= $(openssl sha512 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT + echo "package_hash=$(echo SHA2-256\(${{ matrix.desktop_asset_name }}\)= $(openssl sha256 $path | cut -d' ' -f 2))" >> $GITHUB_OUTPUT - name: Upload Desktop if: startsWith(github.ref, 'refs/tags/v') diff --git a/docs/SERVER.md b/docs/SERVER.md index 57f49b5588..f45403be8a 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -1603,19 +1603,19 @@ To reproduce the build you must have: 1. Download script: ```sh - curl -LO 'https://raw.githubusercontent.com/simplex-chat/simplexmq/refs/heads/master/scripts/reproduce-builds.sh' + curl -LO 'https://raw.githubusercontent.com/simplex-chat/simplexmq/refs/heads/master/scripts/simplexmq-reproduce-builds.sh' ``` 2. Make it executable: ```sh - chmod +x reproduce-builds.sh + chmod +x simplexmq-reproduce-builds.sh ``` 3. Execute the script with the required tag: ```sh - ./reproduce-builds.sh 'v6.3.1' + ./simplexmq-reproduce-builds.sh 'v6.3.1' ``` The script executes these steps (please review the script to confirm): diff --git a/scripts/reproduce-builds.sh b/scripts/simplex-chat-reproduce-builds.sh old mode 100644 new mode 100755 similarity index 88% rename from scripts/reproduce-builds.sh rename to scripts/simplex-chat-reproduce-builds.sh index 1334fec0ec..408d1d4a53 --- a/scripts/reproduce-builds.sh +++ b/scripts/simplex-chat-reproduce-builds.sh @@ -24,13 +24,13 @@ cleanup() { } trap 'cleanup' EXIT INT -mkdir -p "$init_dir/$TAG/from-source" "$init_dir/$TAG/prebuilt" +mkdir -p "$init_dir/$TAG-$repo_name/from-source" "$init_dir/$TAG-$repo_name/prebuilt" git -C "$tempdir" clone "$repo.git" &&\ cd "$tempdir/${repo_name}" &&\ git checkout "$TAG" -for os in 20.04 22.04; do +for os in 22.04 24.04; do os_url="$(printf '%s' "$os" | tr '.' '_')" # Build image @@ -57,11 +57,11 @@ for os in 20.04 22.04; do docker cp \ builder:/out/simplex-chat \ - "$init_dir/$TAG/from-source/simplex-chat-ubuntu-${os_url}-x86-64" + "$init_dir/$TAG-$repo_name/from-source/simplex-chat-ubuntu-${os_url}-x86-64" # Download prebuilt postgresql binary curl -L \ - --output-dir "$init_dir/$TAG/prebuilt/" \ + --output-dir "$init_dir/$TAG-$repo_name/prebuilt/" \ -O \ "$repo/releases/download/${TAG}/simplex-chat-ubuntu-${os_url}-x86-64" @@ -87,7 +87,7 @@ cd "$init_dir" # Final stage: compare hashes # Path to binaries -path_bin="$init_dir/$TAG" +path_bin="$init_dir/$TAG-$repo_name" # Assume everything is okay for now bad=0 @@ -114,7 +114,7 @@ done # If everything is still okay, compute checksums file if [ "$bad" = 0 ]; then - sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' > "$path_bin/_sha256sums" + sha256sum "$path_bin"/from-source/* | sed -e "s|$PWD/||g" -e 's|from-source/||g' -e "s|-$repo_name||g" > "$path_bin/_sha256sums" printf 'Checksums computed - %s\n' "$path_bin/_sha256sums" fi