From 95c34277cac1e7733b1f2cc8543f573e0f78533a Mon Sep 17 00:00:00 2001 From: Roland Kurmann Date: Sat, 1 Mar 2025 13:12:48 +0100 Subject: [PATCH] ci: add additional ci test platforms ubuntu-24.04-arm and macos-13 (#372) * ci: add support for ubuntu-24.04-arm and macos-13 in CI workflow * ci: update CI workflows to include Python 3.x and enhance macOS build configurations * ci: update Docker Buildx image to latest version * ci: push bullseye image also to ghcr * ci: run Docker natively on amd64 and arm64, avoid qemu emulation * ci: fix native docker releases * ci: comment out macOS build steps due to missing zbar shared library --- .github/workflows/ci.yml | 5 +-- .github/workflows/ci_docker.yml | 36 ++++++++++++++++---- .github/workflows/ci_release.yml | 56 ++++++++++++++++++++++---------- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6953604..1945820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,9 @@ jobs: strategy: matrix: - python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] - platform: [ubuntu-latest, macos-latest, windows-latest] + # 3.x is used to run code coverage + python-version: ["3.x", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + platform: [ubuntu-latest, macos-latest, windows-latest, ubuntu-24.04-arm, macos-13] # exclude: runs-on: ${{ matrix.platform }} diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 590f004..422ef05 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -27,7 +27,15 @@ jobs: build-and-push-docker-debian-image: name: Build Docker Bookworm image and push to repositories # run only when code is compiling and tests are passing - runs-on: ubuntu-latest + strategy: + matrix: + include: + - DOCKER_PLATFORM: linux/amd64 + platform: ubuntu-latest + - DOCKER_PLATFORM: linux/arm64 + platform: ubuntu-24.04-arm + + runs-on: ${{ matrix.platform }} # steps to perform in job steps: @@ -75,7 +83,7 @@ jobs: id: docker_build_qr_reader_latest uses: docker/build-push-action@v5 with: - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.DOCKER_PLATFORM }} # relative path to the place where source code with Dockerfile is located # TODO file:, move to docker/ context: . @@ -108,7 +116,15 @@ jobs: build-and-push-docker-alpine-image: name: Build Docker Alpine image and push to repositories # run only when code is compiling and tests are passing - runs-on: ubuntu-latest + strategy: + matrix: + include: + - DOCKER_PLATFORM: linux/amd64 + platform: ubuntu-latest + - DOCKER_PLATFORM: linux/arm64 + platform: ubuntu-24.04-arm + + runs-on: ${{ matrix.platform }} # steps to perform in job steps: @@ -152,7 +168,7 @@ jobs: uses: docker/build-push-action@v5 with: # relative path to the place where source code with Dockerfile is located - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.DOCKER_PLATFORM }} context: . file: docker/Dockerfile_only_txt # builder: ${{ steps.buildx.outputs.name }} @@ -184,7 +200,15 @@ jobs: build-and-push-docker-bullseye-image: name: Build Docker Bullseye image (for PyInstsaller) and push to repositories # run only when code is compiling and tests are passing - runs-on: ubuntu-latest + strategy: + matrix: + include: + - DOCKER_PLATFORM: linux/amd64 + platform: ubuntu-latest + - DOCKER_PLATFORM: linux/arm64 + platform: ubuntu-24.04-arm + + runs-on: ${{ matrix.platform }} # steps to perform in job steps: @@ -228,7 +252,7 @@ jobs: if: github.ref == 'refs/heads/master' uses: docker/build-push-action@v5 with: - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.DOCKER_PLATFORM }} # relative path to the place where source code with Dockerfile is located context: . file: docker/Dockerfile diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index edeb61c..2b63680 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -106,20 +106,23 @@ jobs: path: release_id.txt build-linux-executable-in-docker: - name: Build ${{ matrix.PLATFORM }} release in docker container + name: Build ${{ matrix.platform }} release in docker container # run only when code is compiling and tests are passing - runs-on: ubuntu-latest needs: create-release strategy: matrix: include: - - PLATFORM: linux/amd64 + - DOCKER_PLATFORM: linux/amd64 + platform: ubuntu-latest EXE: extract_otp_secrets_linux_x86_64 ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_x86_64 - - PLATFORM: linux/arm64 + - DOCKER_PLATFORM: linux/arm64 + platform: ubuntu-24.04-arm EXE: extract_otp_secrets_linux_arm64 ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_arm64 + runs-on: ${{ matrix.platform }} + # steps to perform in job steps: - name: Checkout code @@ -145,7 +148,7 @@ jobs: # TODO remove workaround when fixed with: driver-opts: | - image=moby/buildkit:v0.10.6 + image=moby/buildkit:latest - name: Login to DockerHub uses: docker/login-action@v3 @@ -172,11 +175,10 @@ jobs: # https://hub.docker.com/r/multiarch/qemu-user-static/ - name: Run Pyinstaller in container for ${{ matrix.EXE }} run: | - docker run --pull always --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --platform ${{ matrix.PLATFORM }} --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files docker.io/scit0/extract_otp_secrets:bullseye -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && PYTHONHASHSEED=31 && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name ${{ matrix.EXE }} --distpath /files/dist/ /files/src/extract_otp_secrets.py' + docker run --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files docker.io/scit0/extract_otp_secrets:bullseye -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && PYTHONHASHSEED=31 && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name ${{ matrix.EXE }} --distpath /files/dist/ /files/src/extract_otp_secrets.py' - name: Smoke tests linux/amd64 - if: matrix.PLATFORM == 'linux/amd64' + if: matrix.DOCKER_PLATFORM == 'linux/amd64' run: | dist/${{ matrix.EXE }} -V dist/${{ matrix.EXE }} -h @@ -189,9 +191,9 @@ jobs: dist/${{ matrix.EXE }} --qr CV2 example_export.png dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png - name: Smoke tests linux/arm64 - if: matrix.PLATFORM == 'linux/arm64' + if: matrix.DOCKER_PLATFORM == 'linux/arm64' run: | - docker run --platform ${{ matrix.PLATFORM }} --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files docker.io/scit0/extract_otp_secrets -c 'dist/${{ matrix.EXE }} -V && dist/${{ matrix.EXE }} -h && dist/${{ matrix.EXE }} example_export.png && dist/${{ matrix.EXE }} - < example_export.txt && dist/${{ matrix.EXE }} --qr ZBAR example_export.png && dist/${{ matrix.EXE }} --qr QREADER example_export.png && dist/${{ matrix.EXE }} --qr QREADER_DEEP example_export.png && dist/${{ matrix.EXE }} --qr CV2 example_export.png && dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png' + docker run --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files docker.io/scit0/extract_otp_secrets -c 'dist/${{ matrix.EXE }} -V && dist/${{ matrix.EXE }} -h && dist/${{ matrix.EXE }} example_export.png && dist/${{ matrix.EXE }} - < example_export.txt && dist/${{ matrix.EXE }} --qr ZBAR example_export.png && dist/${{ matrix.EXE }} --qr QREADER example_export.png && dist/${{ matrix.EXE }} --qr QREADER_DEEP example_export.png && dist/${{ matrix.EXE }} --qr CV2 example_export.png && dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png' - name: Load Release URL File from release job if: startsWith(github.ref, 'refs/tags/v') uses: actions/download-artifact@v4 @@ -247,19 +249,39 @@ jobs: UPLOAD: false CMD_BUILD: | pyinstaller -y --add-data $pythonLocation/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_ubuntu src/extract_otp_secrets.py - # TODO temp disable macos releases to due - # FileNotFoundError: Icon input file /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/PyInstaller/bootloader/images/icon-windowed.icns not found - # - os: macos-12 + - os: ubuntu-24.04-arm + TARGET: linux + EXE: extract_otp_secrets_ubuntu_arm64 + ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_arm64_ubuntu_latest + ASSET_MIME: application/x-executable + UPLOAD: false + CMD_BUILD: | + pyinstaller -y --add-data $pythonLocation/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_ubuntu_arm64 src/extract_otp_secrets.py + - os: macos-13 + TARGET: macos + # https://pyinstaller.org/en/stable/spec-files.html#spec-file-options-for-a-macos-bundle + EXE: extract_otp_secrets + ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64 + DMG: extract_otp_secrets.dmg + ASSET_NAME_DMG: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64.dmg + ASSET_MIME: application/octet-stream + UPLOAD: true + CMD_BUILD: | + VERSION_STR=$(setuptools-git-versioning) COPYRIGHT_YEARS='2020-2025' envsubst < installer/extract_otp_secrets_macos_template.spec > extract_otp_secrets_macos.spec + pyinstaller -y extract_otp_secrets_macos.spec + installer/build_dmg.sh + # Disable WARN: Cannot import pyzbar module. This problem is probably due to the missing zbar shared library. + # - os: macos-14 # TARGET: macos # # https://pyinstaller.org/en/stable/spec-files.html#spec-file-options-for-a-macos-bundle # EXE: extract_otp_secrets - # ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64 + # ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_arm64 # DMG: extract_otp_secrets.dmg - # ASSET_NAME_DMG: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64.dmg + # ASSET_NAME_DMG: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_arm64.dmg # ASSET_MIME: application/octet-stream # UPLOAD: true # CMD_BUILD: | - # VERSION_STR=$(setuptools-git-versioning) COPYRIGHT_YEARS='2020-2023' envsubst < installer/extract_otp_secrets_macos_template.spec > extract_otp_secrets_macos.spec + # VERSION_STR=$(setuptools-git-versioning) COPYRIGHT_YEARS='2020-2025' envsubst < installer/extract_otp_secrets_macos_template.spec > extract_otp_secrets_macos.spec # pyinstaller -y extract_otp_secrets_macos.spec # installer/build_dmg.sh steps: @@ -296,7 +318,7 @@ jobs: shell: bash run: | mkdir -p build/ - VERSION_STR=$(setuptools-git-versioning) VERSION_MAJOR=$(cut -d '.' -f 1 <<< "$(setuptools-git-versioning)") VERSION_MINOR=$(cut -d '.' -f 2 <<< "$(setuptools-git-versioning)") VERSION_PATCH=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n "s/^([0-9]+).*/\1/p") VERSION_BUILD=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n -e"s/^[0-9]+.+/99/p")$(($(git rev-list --count $(git tag | sort -V -r | sed '1!d')..HEAD))) COPYRIGHT_YEARS='2020-2023' envsubst < installer/win_file_version_info_template.txt > build/win_file_version_info.txt + VERSION_STR=$(setuptools-git-versioning) VERSION_MAJOR=$(cut -d '.' -f 1 <<< "$(setuptools-git-versioning)") VERSION_MINOR=$(cut -d '.' -f 2 <<< "$(setuptools-git-versioning)") VERSION_PATCH=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n "s/^([0-9]+).*/\1/p") VERSION_BUILD=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n -e"s/^[0-9]+.+/99/p")$(($(git rev-list --count $(git tag | sort -V -r | sed '1!d')..HEAD))) COPYRIGHT_YEARS='2020-2025' envsubst < installer/win_file_version_info_template.txt > build/win_file_version_info.txt - name: Build with pyinstaller for ${{ matrix.TARGET }} env: # Reproducible build: https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build