CI #43
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "*" | |
| pull_request: | |
| merge_group: | |
| workflow_dispatch: | |
| inputs: | |
| test_release: | |
| description: If true, publish to test.pypi.org | |
| required: true | |
| default: true | |
| type: boolean | |
| permissions: | |
| contents: read | |
| env: | |
| DOCKER_IMAGE: ghcr.io/onekey-sec/unblob | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_pre_commit: | |
| name: Check - pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: "false" | |
| - name: Setup Nix | |
| uses: cachix/install-nix-action@v30 | |
| with: | |
| install_url: https://releases.nixos.org/nix/nix-2.18.8/install | |
| - name: Check pre-commit hook | |
| uses: pre-commit/[email protected] | |
| check_pyright: | |
| name: Check - pyright | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| - name: Check - pyright | |
| run: uv run pyright . | |
| run_python_tests: | |
| name: Run tests (Python) | |
| needs: [check_pre_commit, check_pyright] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup git lfs | |
| uses: ./.github/actions/setup-git-lfs | |
| - name: Run pytest | |
| run: uv run pytest -vvv | |
| run_rust_tests: | |
| name: Run tests (Rust) | |
| needs: [check_pre_commit] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Run cargo test | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| run: cargo test | |
| build_linux_wheels: | |
| name: Build wheels (linux) | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') | |
| needs: [check_pre_commit] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - manylinux: manylinux2014 | |
| target: x86_64 | |
| - manylinux: manylinux2014 | |
| target: aarch64 | |
| - manylinux: musllinux_1_1 | |
| target: x86_64 | |
| # lief is not available for this platform (and no sdist is provided) | |
| # - manylinux: musllinux_1_1 | |
| # target: aarch64 | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: "false" | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.platform.target }} | |
| args: --release --out dist | |
| sccache: "true" | |
| manylinux: ${{ matrix.platform.manylinux }} | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.platform.manylinux }}-${{ matrix.platform.target }} | |
| path: dist | |
| - name: Check wheel (x86-manylinux) | |
| if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') | |
| shell: bash | |
| run: | | |
| set -e | |
| python3 -m venv wheel.venv | |
| source wheel.venv/bin/activate | |
| pip install dist/*.whl | |
| unblob --help | |
| - name: Check wheel (cross-manylinux) | |
| if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') | |
| uses: uraimo/run-on-arch-action@v2 | |
| with: | |
| arch: ${{ matrix.platform.target }} | |
| distro: ubuntu22.04 | |
| githubToken: ${{ github.token }} | |
| install: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3-venv libmagic1 | |
| run: | | |
| set -e | |
| python3 -m venv wheel.venv | |
| source wheel.venv/bin/activate | |
| pip install dist/*.whl | |
| unblob --version | |
| - name: Check wheel (x86-musllinux) | |
| if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: alpine:latest | |
| options: -v ${{ github.workspace }}:/io -w /io | |
| run: | | |
| set -e | |
| apk add py3-pip libmagic gcc lz4 musl-dev python3-dev | |
| python3 -m venv wheel.venv | |
| source wheel.venv/bin/activate | |
| pip install dist/*.whl | |
| unblob --version | |
| - name: Check wheel (cross-musllinux) | |
| if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') | |
| uses: uraimo/run-on-arch-action@v2 | |
| with: | |
| arch: ${{ matrix.platform.target }} | |
| distro: alpine_latest | |
| githubToken: ${{ github.token }} | |
| install: | | |
| apk add py3-pip libmagic gcc lz4 musl-dev python3-dev | |
| run: | | |
| set -e | |
| python3 -m venv wheel.venv | |
| source wheel.venv/bin/activate | |
| pip install dist/*.whl | |
| unblob --version | |
| build_macos_wheels: | |
| name: Build wheels (macos) | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') | |
| needs: [check_pre_commit] | |
| runs-on: ${{ matrix.platform.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - runner: macos-13 | |
| target: x86_64 | |
| - runner: macos-14 | |
| target: aarch64 | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: "false" | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.platform.target }} | |
| args: --release --out dist | |
| sccache: "true" | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-macos-${{ matrix.platform.target }} | |
| path: dist | |
| - name: Check wheel | |
| run: | | |
| set -e | |
| brew install libmagic | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install --find-links dist unblob | |
| unblob --version | |
| build_sdist: | |
| name: Build sdist | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') | |
| needs: [check_pre_commit] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: "false" | |
| - name: Build sdist | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-sdist | |
| path: dist | |
| build-image: | |
| name: Build Docker image | |
| if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') | |
| needs: [build_linux_wheels] | |
| permissions: | |
| # needed for sarif report upload | |
| security-events: write | |
| # needed for pushing to registry | |
| packages: write | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - docker: linux/amd64 | |
| wheel: wheels-manylinux2014-x86_64 | |
| - docker: linux/arm64 | |
| wheel: wheels-manylinux2014-aarch64 | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Download wheel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.platform.wheel }} | |
| path: dist | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform.docker }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=semver,pattern={{version}} | |
| type=sha | |
| - name: Set up QEMU | |
| if: matrix.platform.docker != 'linux/amd64' | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push by digest | |
| id: build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.platform.docker }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| tags: ${{ steps.meta.output.tags }} | |
| outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && github.event_name == 'push' }} | |
| - name: Docker container vulnerability scan | |
| id: scan | |
| uses: anchore/scan-action@v6 | |
| with: | |
| image: ${{ env.DOCKER_IMAGE }} | |
| fail-build: false | |
| severity-cutoff: critical | |
| only-fixed: true | |
| - name: Upload SARIF report | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: ${{ steps.scan.outputs.sarif }} | |
| - name: Check unblob - help | |
| run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help | |
| - name: Check unblob - show-external-dependencies | |
| run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies | |
| - name: Check unblob - run for a file with --verbose | |
| run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip | |
| - name: Export digest | |
| run: | | |
| mkdir -p /tmp/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "/tmp/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digest-${{ env.PLATFORM_PAIR }} | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge-and-push-image: | |
| if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-image | |
| permissions: | |
| # needed for pushing to registry | |
| packages: write | |
| steps: | |
| - name: Download digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp/digests | |
| pattern: digest-* | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=semver,pattern={{version}} | |
| type=sha | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create manifest list and push | |
| working-directory: /tmp/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} | |
| release: | |
| name: Release | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} | |
| needs: | |
| - run_python_tests | |
| - run_rust_tests | |
| - build_linux_wheels | |
| - build_macos_wheels | |
| - build_sdist | |
| permissions: | |
| # Use to sign the release artifacts | |
| id-token: write | |
| # Used to upload release artifacts | |
| contents: write | |
| # Used to generate artifact attestation | |
| attestations: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheels-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Generate artifact attestation | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-path: dist/* | |
| - name: Publish to PyPI | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_release) | |
| uses: PyO3/maturin-action@v1 | |
| env: | |
| MATURIN_PYPI_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && secrets.TEST_PYPI_API_TOKEN || secrets.POETRY_PYPI_TOKEN_PYPI }} | |
| MATURIN_REPOSITORY: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && 'testpypi' || 'pypi' }} | |
| with: | |
| command: upload | |
| args: --non-interactive --skip-existing dist/* |