CW-827 CI/CD update (#1948)

* CI update
- use existing build outputs in build_monero_all.sh
- update $HOME, fix gh actions
- add secrets earlier in the runtime (potentially speed up 'Build generated code' step)
- add windows dockerfile
- add linux/android dockerfile
- update android/linux ci script

* [skip slack] [run tests] Run tests on CI, fix tests

* [skip slack] [run tests] force enable kvm in android

* [skip slack] [run tests] remove inexisting flag

* [run tests] [skip slack] update tests

* add extra dependencies [skip ci]

* [skip slack] [run tests] Add secrets

* [skip slack] [run tests] Timeout test cases, continue on error

* [skip slack] [run tests] Xvfb fix, timeout fix

* [skip slack] [run tests] Start dbus to clean up the logs, use SIGKILL

* [skip slack] [run tests] Enable network manager

* [skip slack] [run tests] Screen record test, resize screen

* [skip slack] [run tests] Improve status report for tests

* [skip slack] [run tests] Increase framerate

* [skip slack] [run tests] Remove test that I am unable to fix locally easily from CI

* [skip slack] [run tests] Simplify ffmpeg command

* [skip slack] [run tests] Increase timeout, add comment

* [skip slack] Update dockerfile, migrate from mrcyjanek to cake-tech for the ghcr org

* Update lib/entities/default_settings_migration.dart

Co-authored-by: Omar Hatem <omarh.ismail1@gmail.com>

---------

Co-authored-by: Omar Hatem <omarh.ismail1@gmail.com>

scripts/android/.gitignore

@@ -0,0 +1 @@
+mwebd

scripts/android/build_monero_all.sh

@@ -8,50 +8,20 @@ cd "$(dirname "$0")"
 
 NPROC="-j$(nproc)"
 
-if [[ "x$(uname)" == "xDarwin" ]];
-then
-    USE_DOCKER="ON"
-    NPROC="-j1"
-fi
-
 ../prepare_moneroc.sh
 
-if [[ ! "x$RUNNER_OS" == "x" ]];
-then
-    REMOVE_CACHES=ON
-fi
-
-# NOTE: -j1 is intentional. Otherwise you will run into weird behaviour on macos
-if [[ ! "x$USE_DOCKER" == "x" ]];
-then
-    for COIN in monero wownero;
-    do
-        pushd ../monero_c
-            docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc g++ libtinfo5 gperf; ./build_single.sh ${COIN} x86_64-linux-android $NPROC"
-            # docker run --platform linux/amd64 -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc g++ libtinfo5 gperf; ./build_single.sh ${COIN} i686-linux-android $NPROC"
-            docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc g++ libtinfo5 gperf; ./build_single.sh ${COIN} armv7a-linux-androideabi $NPROC"
-            docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc g++ libtinfo5 gperf; ./build_single.sh ${COIN} aarch64-linux-android $NPROC"
-        popd
-    done
-else
-    for COIN in monero wownero;
-    do
-        pushd ../monero_c
-            env -i ./build_single.sh ${COIN} x86_64-linux-android $NPROC
-            [[ ! "x$REMOVE_CACHES" == "x" ]] && rm -rf ${COIN}/contrib/depends/x86_64-linux-android
-            # ./build_single.sh ${COIN} i686-linux-android $NPROC
-            # [[ ! "x$REMOVE_CACHES" == "x" ]] && rm -rf ${COIN}/contrib/depends/i686-linux-android
-            env -i ./build_single.sh ${COIN} armv7a-linux-androideabi $NPROC
-            [[ ! "x$REMOVE_CACHES" == "x" ]] && rm -rf ${COIN}/contrib/depends/armv7a-linux-androideabi
-            env -i ./build_single.sh ${COIN} aarch64-linux-android $NPROC
-            [[ ! "x$REMOVE_CACHES" == "x" ]] && rm -rf ${COIN}/contrib/depends/aarch64-linux-android
-
-        popd
-        unxz -f ../monero_c/release/${COIN}/x86_64-linux-android_libwallet2_api_c.so.xz
-
-        unxz -f ../monero_c/release/${COIN}/armv7a-linux-androideabi_libwallet2_api_c.so.xz
-
-        unxz -f ../monero_c/release/${COIN}/aarch64-linux-android_libwallet2_api_c.so.xz
-        [[ ! "x$REMOVE_CACHES" == "x" ]] && rm -rf ${COIN}/contrib/depends/{built,sources}
-    done
-fi
+for COIN in monero wownero;
+do
+    pushd ../monero_c
+        for target in {x86_64,aarch64}-linux-android armv7a-linux-androideabi
+        do
+            if [[ -f "release/${COIN}/${target}_libwallet2_api_c.so" ]];
+            then
+                echo "file exist, not building monero_c for ${COIN}/$target.";
+            else
+                env -i ./build_single.sh ${COIN} $target $NPROC
+                unxz -f ../monero_c/release/${COIN}/${target}_libwallet2_api_c.so.xz
+            fi
+        done
+    popd
+done

scripts/android/build_mwebd.sh

@@ -16,7 +16,4 @@ cd mwebd
 git reset --hard 555349415f76a42ec5c76152b64c4ab9aabc448f
 gomobile bind -target=android -androidapi 21 .
 mkdir -p ../../../cw_mweb/android/libs/
-mv ./mwebd.aar $_
-# cleanup:
-cd ..
-rm -rf mwebd
+cp ./mwebd.aar $_

scripts/linux/Dockerfile.linux

@@ -0,0 +1,148 @@
+# Usage:
+# docker build . -f Dockerfile.linux -t ghcr.io/cake-tech/cake_wallet:main-linux
+# docker push ghcr.io/cake-tech/cake_wallet:main-linux
+
+FROM --platform=linux/amd64 docker.io/debian:12
+
+LABEL org.opencontainers.image.source=https://github.com/cake-tech/cake_wallet
+
+ENV GOLANG_VERSION=1.23.4
+# comes from https://developer.android.com/studio/#command-tools
+ENV ANDROID_SDK_TOOLS_VERSION=11076708
+# https://developer.android.com/studio/releases/build-tools
+ENV ANDROID_PLATFORM_VERSION=34
+ENV ANDROID_BUILD_TOOLS_VERSION=34.0.0
+
+ENV FLUTTER_VERSION=3.24.0
+
+# If we ever need to migrate the home directory...
+RUN sed -i 's|^root:[^:]*:[^:]*:[^:]*:[^:]*:/root:|root:x:0:0:root:/root:|' /etc/passwd
+# mkdir -p /root && rm -rf /root && cp -a /root /root
+ENV HOME=/root
+# Heavily inspired by cirrusci images
+# https://github.com/cirruslabs/docker-images-android/blob/master/sdk/tools/Dockerfile
+# https://github.com/cirruslabs/docker-images-android/blob/master/sdk/34/Dockerfile
+# https://github.com/cirruslabs/docker-images-android/blob/master/sdk/34-ndk/Dockerfile
+# https://github.com/cirruslabs/docker-images-flutter/blob/master/sdk/Dockerfile
+
+ENV ANDROID_HOME=/opt/android-sdk-linux \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LANGUAGE=en_US:en
+
+ENV ANDROID_SDK_ROOT=$ANDROID_HOME \
+    PATH=${PATH}:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/platform-tools:${ANDROID_HOME}/emulator
+
+RUN set -o xtrace \
+    && cd /opt \
+    && apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y jq \
+    && apt-get install -y default-jdk \
+    && apt-get install -y sudo wget zip unzip git openssh-client curl bc software-properties-common build-essential ruby-full ruby-bundler libstdc++6 libpulse0 libglu1-mesa locales lcov libsqlite3-dev --no-install-recommends \
+    # for x86 emulators
+    && apt-get install -y libxtst6 libnss3-dev libnspr4 libxss1 libatk-bridge2.0-0 libgtk-3-0 libgdk-pixbuf2.0-0 \
+    && apt-get install -y -qq xxd \
+    && apt-get install -y lftp \
+    && apt-get install -qq -y sqlite3 libsqlite3-dev \
+    # linux desktop dependencies
+    && apt-get install -y clang cmake ninja-build pkg-config libgtk-3-dev \
+    # monero_c dependencies
+    && apt-get install -y ccache build-essential autoconf libtool gperf llvm \
+    # extra stuff for KVM
+    && apt-get install -y udev qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils \
+    # for linux tests
+    && apt-get install -y xvfb network-manager ffmpeg x11-utils \
+    && rm -rf /var/lib/apt/lists/* \
+    && sh -c 'echo "en_US.UTF-8 UTF-8" > /etc/locale.gen' \
+    && locale-gen \
+    && update-locale LANG=en_US.UTF-8
+
+# install nodejs for actions
+RUN apt-get update && \
+    apt-get install -y curl && \
+    curl -fsSL https://deb.nodesource.com/setup_23.x | bash - && \
+    apt-get install -y nodejs && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN wget https://go.dev/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz &&\
+    rm -rf /usr/local/go &&\
+    tar -C /usr/local -xzf go${GOLANG_VERSION}.linux-amd64.tar.gz
+
+ENV PATH=${PATH}:/usr/local/go/bin:${HOME}/go/bin
+ENV GOROOT=/usr/local/go
+ENV GOPATH=${HOME}/go
+RUN go install golang.org/x/mobile/cmd/gomobile@latest
+RUN gomobile init
+
+RUN wget -q https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip -O android-sdk-tools.zip \
+    && mkdir -p ${ANDROID_HOME}/cmdline-tools/ \
+    && unzip -q android-sdk-tools.zip -d ${ANDROID_HOME}/cmdline-tools/ \
+    && mv ${ANDROID_HOME}/cmdline-tools/cmdline-tools ${ANDROID_HOME}/cmdline-tools/latest \
+    && chown -R root:root $ANDROID_HOME \
+    && rm android-sdk-tools.zip \
+    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+    && yes | sdkmanager --licenses \
+    && wget -O /usr/bin/android-wait-for-emulator https://raw.githubusercontent.com/travis-ci/travis-cookbooks/master/community-cookbooks/android-sdk/files/default/android-wait-for-emulator \
+    && chmod +x /usr/bin/android-wait-for-emulator \
+    && sdkmanager platform-tools \
+    && mkdir -p ${HOME}/.android \
+    && touch ${HOME}/.android/repositories.cfg \
+    && git config --global user.email "czarek@cakewallet.com" \
+    && git config --global user.name "CakeWallet CI"
+
+# emulator is not available on linux/arm64 (https://issuetracker.google.com/issues/227219818)
+RUN if [ $(uname -m) == "x86_64" ]; then sdkmanager emulator ; fi
+
+# Extra dependencies to not download them for cake wallet build
+RUN yes | sdkmanager \
+    "platforms;android-$ANDROID_PLATFORM_VERSION" \
+    "build-tools;$ANDROID_BUILD_TOOLS_VERSION" \
+    "platforms;android-33" \
+    "build-tools;33.0.2" \
+    "build-tools;33.0.1" \
+    "build-tools;33.0.0" \
+    "build-tools;35.0.0"
+
+ENV ANDROID_NDK_VERSION=27.2.12479018
+
+# Extra ndk dependency for sp_scanner
+RUN yes | sdkmanager "ndk;$ANDROID_NDK_VERSION" \
+    "ndk;27.0.12077973"
+
+# https://github.com/ReactiveCircus/android-emulator-runner dependencies for tests
+RUN yes | sdkmanager "system-images;android-29;default;x86" \
+    "system-images;android-29;default;x86_64" \
+    "system-images;android-31;default;x86_64" \
+    "platforms;android-29"
+
+# fake the KVM status so android emulator doesn't complain (that much)
+RUN (addgroup kvm || true) && \
+    adduser root kvm && \
+    mkdir -p /etc/udev/rules.d/ && \
+    echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | tee /etc/udev/rules.d/99-kvm4all.rules
+
+ENV PATH=${HOME}/.cargo/bin:${PATH}
+
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
+    cargo install cargo-ndk && \
+    for target in aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android x86_64-unknown-linux-gnu; \
+    do \
+        rustup target add --toolchain stable $target; \
+    done
+
+
+ENV HOME=${HOME}
+ENV FLUTTER_HOME=${HOME}/sdks/flutter/${FLUTTER_VERSION}
+ENV FLUTTER_ROOT=$FLUTTER_HOME
+
+ENV PATH=${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin
+
+RUN git clone --depth 1 --branch ${FLUTTER_VERSION} https://github.com/flutter/flutter.git ${FLUTTER_HOME}
+
+RUN yes | flutter doctor --android-licenses \
+    && flutter doctor \
+    && chown -R root:root ${FLUTTER_HOME}
+
+RUN flutter precache

scripts/linux/build_monero_all.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-
-. ./config.sh
-
-
 set -x -e
 
 cd "$(dirname "$0")"
@@ -15,7 +11,15 @@ NPROC="-j$(nproc)"
 for COIN in monero wownero;
 do
     pushd ../monero_c
-        ./build_single.sh ${COIN} $(gcc -dumpmachine) $NPROC
+        for target in x86_64-linux-gnu
+        do
+            if [[ -f "release/${COIN}/${target}_libwallet2_api_c.so" ]];
+            then
+                echo "file exist, not building monero_c for ${COIN}/$target.";
+            else
+                ./build_single.sh ${COIN} $target $NPROC
+                unxz -f ../monero_c/release/${COIN}/${target}_libwallet2_api_c.so.xz
+            fi
+        done
     popd
-    unxz -f ../monero_c/release/${COIN}/$(gcc -dumpmachine)_libwallet2_api_c.so.xz
-done
+done

scripts/prepare_moneroc.sh

@@ -4,9 +4,9 @@ set -x -e
 
 cd "$(dirname "$0")"
 
-if [[ ! -d "monero_c" ]];
+if [[ ! -d "monero_c/.git" ]];
 then
-    git clone https://github.com/mrcyjanek/monero_c --branch master
+    git clone https://github.com/mrcyjanek/monero_c --branch master monero_c
     cd monero_c
     git checkout af5277f96073917185864d3596e82b67bee54e78
     git reset --hard

scripts/windows/.gitignore

@@ -0,0 +1 @@
+actions-runner

scripts/windows/Dockerfile.windows

@@ -0,0 +1,68 @@
+# Usage:
+# docker build . -f Dockerfile.windows -t ghcr.io/cake-tech/cake_wallet:main-windows
+# docker push ghcr.io/cake-tech/cake_wallet:main-windows
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2022
+
+ENV FLUTTER_VERSION=3.24.0
+ENV GIT_VERSION=2.47.1
+ENV VS_INSTALLED_DIR="C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools"
+ENV PATH="C:\Users\ContainerAdministrator\.cargo\bin;C:\ProgramData\chocolatey\bin;C:\flutter\flutter\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\OpenSSH\;C:\Users\ContainerAdministrator\AppData\Local\Microsoft\WindowsApps"
+ENV RUNNER_VERSION=2.321.0
+ENV RUNNER_URL=https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-win-x64-${RUNNER_VERSION}.zip
+ENV RUNNER_WORKDIR=_work
+
+RUN powershell -Command \
+    curl.exe -L https://aka.ms/vs/17/release/vc_redist.x64.exe -o vc_redist.x64.exe ; \
+    Start-Process -Wait -FilePath .\vc_redist.x64.exe -ArgumentList '/quiet', '/install' ; \
+    Remove-Item -Force vc_redist.x64.exe
+
+RUN powershell -Command \
+    $GIT_VERSION = [Environment]::GetEnvironmentVariable('GIT_VERSION'); \
+    curl.exe -L https://github.com/git-for-windows/git/releases/download/v$($GIT_VERSION).windows.1/Git-$($GIT_VERSION)-64-bit.exe -o git_installer.exe ; \
+    Start-Process -Wait -FilePath .\git_installer.exe -ArgumentList '/SILENT', '/NOICONS' ; \
+    Remove-Item -Force git_installer.exe
+
+RUN powershell -NoProfile -ExecutionPolicy Bypass -Command \
+    Set-ExecutionPolicy RemoteSigned -Scope Process; \
+    [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; \
+    Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicP -OutFile install.ps1; \
+    powershell -NoProfile -ExecutionPolicy Bypass -File install.ps1; \
+    Remove-Item -Force install.ps1
+
+RUN choco install -y visualstudio2022community
+RUN choco install -y visualstudio2022-workload-nativedesktop
+RUN choco install -y nodejs
+RUN choco install -y go
+RUN choco install -y 7zip
+
+RUN powershell -Command \
+    curl.exe -L https://win.rustup.rs -o rustup-init.exe; \
+    Start-Process -Wait -FilePath .\rustup-init.exe -ArgumentList "-y"; \
+    Remove-Item -Force .\rustup-init.exe
+
+RUN powershell -Command \
+    curl.exe -L https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -o C:\Windows\System32\nuget.exe
+
+RUN powershell -Command \
+    $FLUTTER_VERSION = [Environment]::GetEnvironmentVariable('FLUTTER_VERSION'); \
+    curl.exe -L https://storage.googleapis.com/flutter_infra_release/releases/stable/windows/flutter_windows_$($FLUTTER_VERSION)-stable.zip -o flutter.zip ; \
+    7z x flutter.zip -oC:\flutter -bsp1 -bse1 ; \
+    Remove-Item -Force flutter.zip
+
+RUN flutter precache
+
+WORKDIR C:\\actions-runner
+
+RUN powershell -Command \
+    curl.exe -L $env:RUNNER_URL -o 'actions-runner.zip'; \
+    7z x actions-runner.zip -oC:\actions-runner -bsp1 -bse1 ; \
+    Remove-Item -Path 'actions-runner.zip'
+
+COPY actions-runner/.credentials /actions-runner/.credentials
+COPY actions-runner/.credentials_rsaparams /actions-runner/.credentials_rsaparams
+COPY actions-runner/.runner /actions-runner/.runner
+
+COPY ci_entrypoint.ps1 /actions-runner/ci_entrypoint.ps1
+
+ENTRYPOINT ["powershell", "-File", "ci_entrypoint.ps1"]

scripts/windows/ci_entrypoint.ps1

@@ -0,0 +1,5 @@
+$runnerDir = "C:\actions-runner"
+$runCmd = "$runnerDir\run.cmd"
+
+Write-Host "Starting the runner..."
+& $runCmd

scripts/windows/ci_register.ps1

@@ -0,0 +1,30 @@
+# Variables for paths and config
+$runnerDir = "C:\actions-runner"
+$configCmd = "$runnerDir\config.cmd"
+$runCmd = "$runnerDir\run.cmd"
+
+# Check required environment variables
+if (-not $env:RUNNER_TOKEN) {
+    Write-Error "RUNNER_TOKEN is not set. Exiting."
+    exit 1
+}
+if (-not $env:RUNNER_REPO_URL) {
+    Write-Error "RUNNER_REPO_URL is not set. Exiting."
+    exit 1
+}
+$env:RUNNER_NAME = "windows-amd64-cake"
+$env:RUNNER_WORKDIR = "_work"
+
+# Register the runner
+Write-Host "Registering the runner..."
+Write-Host "--url $env:RUNNER_REPO_URL"
+Write-Host "--token $env:RUNNER_TOKEN"
+Write-Host "--name $env:RUNNER_NAME"
+Write-Host "--work $env:RUNNER_WORKDIR"
+
+& $configCmd --url $env:RUNNER_REPO_URL `
+             --token $env:RUNNER_TOKEN `
+             --name $env:RUNNER_NAME `
+             --work $env:RUNNER_WORKDIR `
+             --unattended `
+             --replace