CI.yml

name: CI

on:
  push:
    branches:
      - main
    tags:
      - "*"
  pull_request:
  merge_group:
  workflow_dispatch:
    inputs:
      test_release:
        description: If true, publish to test.pypi.org
        required: true
        default: true
        type: boolean

permissions:
  contents: read

env:
  DOCKER_IMAGE: ghcr.io/onekey-sec/unblob

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  check_pre_commit:
    name: Check - pre-commit
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies
        with:
          install-test-deps: "false"

      - name: Setup Nix
        uses: cachix/install-nix-action@v30
        with:
          install_url: https://releases.nixos.org/nix/nix-2.18.8/install

      - name: Check pre-commit hook
        uses: pre-commit/action@v3.0.1

  check_pyright:
    name: Check - pyright
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies

      - name: Check - pyright
        run: uv run pyright .

  run_python_tests:
    name: Run tests (Python)
    needs: [check_pre_commit, check_pyright]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies
        with:
          python-version: ${{ matrix.python-version }}

      - name: Setup git lfs
        uses: ./.github/actions/setup-git-lfs

      - name: Run pytest
        run: uv run pytest -vvv

  run_rust_tests:
    name: Run tests (Rust)
    needs: [check_pre_commit]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup sccache
        uses: mozilla-actions/sccache-action@v0.0.7

      - name: Run cargo test
        env:
          SCCACHE_GHA_ENABLED: "true"
          RUSTC_WRAPPER: "sccache"
        run: cargo test

  build_linux_wheels:
    name: Build wheels (linux)
    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies')
    needs: [check_pre_commit]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        platform:
          - manylinux: manylinux2014
            target: x86_64
          - manylinux: manylinux2014
            target: aarch64
          - manylinux: musllinux_1_1
            target: x86_64
          # lief is not available for this platform (and no sdist is provided)
          # - manylinux: musllinux_1_1
          #   target: aarch64
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies
        with:
          install-test-deps: "false"

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.platform.target }}
          args: --release --out dist
          sccache: "true"
          manylinux: ${{ matrix.platform.manylinux }}

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.platform.manylinux }}-${{ matrix.platform.target }}
          path: dist

      - name: Check wheel (x86-manylinux)
        if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux')
        shell: bash
        run: |
          set -e
          python3 -m venv wheel.venv
          source wheel.venv/bin/activate
          pip install dist/*.whl
          unblob --help

      - name: Check wheel (cross-manylinux)
        if: matrix.platform.target !=  'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux')
        uses: uraimo/run-on-arch-action@v2
        with:
          arch: ${{ matrix.platform.target }}
          distro: ubuntu22.04
          githubToken: ${{ github.token }}
          install: |
            apt-get update
            apt-get install -y --no-install-recommends python3-venv libmagic1
          run: |
            set -e
            python3 -m venv wheel.venv
            source wheel.venv/bin/activate
            pip install dist/*.whl
            unblob --version

      - name: Check wheel (x86-musllinux)
        if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux')
        uses: addnab/docker-run-action@v3
        with:
          image: alpine:latest
          options: -v ${{ github.workspace }}:/io -w /io
          run: |
            set -e
            apk add py3-pip libmagic gcc lz4 musl-dev python3-dev
            python3 -m venv wheel.venv
            source wheel.venv/bin/activate
            pip install dist/*.whl
            unblob --version

      - name: Check wheel (cross-musllinux)
        if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux')
        uses: uraimo/run-on-arch-action@v2
        with:
          arch: ${{ matrix.platform.target }}
          distro: alpine_latest
          githubToken: ${{ github.token }}
          install: |
            apk add py3-pip libmagic gcc lz4 musl-dev python3-dev
          run: |
            set -e
            python3 -m venv wheel.venv
            source wheel.venv/bin/activate
            pip install dist/*.whl
            unblob --version

  build_macos_wheels:
    name: Build wheels (macos)
    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies')
    needs: [check_pre_commit]
    runs-on: ${{ matrix.platform.runner }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - runner: macos-13
            target: x86_64
          - runner: macos-14
            target: aarch64
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies
        with:
          install-test-deps: "false"

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.platform.target }}
          args: --release --out dist
          sccache: "true"

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-macos-${{ matrix.platform.target }}
          path: dist

      - name: Check wheel
        run: |
          set -e
          brew install libmagic
          python3 -m venv .venv
          source .venv/bin/activate
          pip install --find-links dist unblob
          unblob --version

  build_sdist:
    name: Build sdist
    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies')
    needs: [check_pre_commit]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Setup 3rd party dependencies
        uses: ./.github/actions/setup-dependencies
        with:
          install-test-deps: "false"

      - name: Build sdist
        uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist

      - name: Upload sdist
        uses: actions/upload-artifact@v4
        with:
          name: wheels-sdist
          path: dist

  build-image:
    name: Build Docker image
    if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies')
    needs: [build_linux_wheels]
    permissions:
      security-events: write # sarif
      packages: write # registry
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        platform:
          - docker: linux/amd64
            wheel: wheels-manylinux2014-x86_64
          - docker: linux/arm64
            wheel: wheels-manylinux2014-aarch64
    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Download wheel
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.platform.wheel }}
          path: dist

      - name: Prepare
        run: |
          platform=${{ matrix.platform.docker }}
          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.DOCKER_IMAGE }}
          tags: |
            type=raw,value=latest,enable={{is_default_branch}}
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=sha

      - name: Set up QEMU
        if: matrix.platform.docker != 'linux/amd64'
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push by digest
        id: build
        uses: docker/build-push-action@v6
        with:
          context: .
          platforms: ${{ matrix.platform.docker }}
          labels: ${{ steps.meta.outputs.labels }}
          tags: ${{ steps.meta.output.tags }}
          outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{  github.repository_owner == 'onekey-sec' && github.event_name == 'push' }}

      - name: Docker container vulnerability scan
        id: scan
        uses: anchore/scan-action@v6
        with:
          image: ${{ env.DOCKER_IMAGE }}
          fail-build: false
          severity-cutoff: critical
          only-fixed: true

      - name: Upload SARIF report
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: ${{ steps.scan.outputs.sarif }}

      - name: Check unblob - help
        run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help

      - name: Check unblob - show-external-dependencies
        run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies

      - name: Check unblob - run for a file with --verbose
        run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip

      - name: Export digest
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"

      - name: Upload digest
        uses: actions/upload-artifact@v4
        with:
          name: digest-${{ env.PLATFORM_PAIR }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  merge-and-push-image:
    if: github.repository_owner == 'onekey-sec' && github.event_name == 'push'
    runs-on: ubuntu-latest
    needs:
      - build-image
    steps:
      - name: Download digests
        uses: actions/download-artifact@v4
        with:
          path: /tmp/digests
          pattern: digest-*
          merge-multiple: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.DOCKER_IMAGE }}
          tags: |
            type=raw,value=latest,enable={{is_default_branch}}
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=sha

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Create manifest list and push
        working-directory: /tmp/digests
        run: |
          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
            $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *)
      - name: Inspect image
        run: |
          docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }}

  release:
    name: Release
    runs-on: ubuntu-latest
    if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
    needs:
      - run_python_tests
      - run_rust_tests
      - build_linux_wheels
      - build_macos_wheels
      - build_sdist
    permissions:
      # Use to sign the release artifacts
      id-token: write
      # Used to upload release artifacts
      contents: write
      # Used to generate artifact attestation
      attestations: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: wheel-*
          path: dist
          merge-multiple: true

      - name: Generate artifact attestation
        uses: actions/attest-build-provenance@v1
        with:
          subject-path: dist/*

      - name: Publish to PyPI
        if: ${{ startsWith(github.ref, 'refs/tags/') }}
        uses: PyO3/maturin-action@v1
        env:
          MATURIN_PYPI_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && secrets.TEST_PYPI_API_TOKEN || secrets.POETRY_PYPI_TOKEN_PYPI }}
          MATURIN_REPOSITORY: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && 'testpypi' || 'pypi' }}
        with:
          command: upload
          args: --non-interactive --skip-existing dist/*