diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index 8bf31e5bc0..0f2e15b90a 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -3,14 +3,30 @@ inputs: description: "Python version to setup" required: false default: "3.9" + install-test-deps: + description: "Wether to install 3rd Party dependencies (for tests)" + required: false + default: "true" # unfortunately boolean variables are not supported name: "Setup dependencies" description: "Install all required dependencies for worflows to run." runs: using: "composite" steps: - - name: Install 3rd party dependencies - run: sudo unblob/install-deps.sh + - if: inputs.install-test-deps == 'true' + name: Install 3rd party dependencies + run: sudo ./install-deps.sh + shell: bash + + - name: Setup sccache # for speeding up Rust builds + uses: mozilla-actions/sccache-action@v0.0.7 + with: + disable_annotations: true # it is very spammy, but useful for diagnostics + + - name: Enable sccache + run: | + echo SCCACHE_GHA_ENABLED=true >> $GITHUB_ENV + echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV shell: bash - name: Install uv & Python @@ -18,8 +34,12 @@ runs: with: enable-cache: true cache-dependency-glob: "uv.lock" - python-version: ${{ matrix.python-version }} + python-version: ${{ inputs.python-version }} - name: Install the project run: uv sync shell: bash + + - name: Setup pip # some tools need it, and uv virtualenvs doesn't contain it + run: uv pip install pip + shell: bash diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 6e7a187837..90e3dcbded 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -4,7 +4,8 @@ queries: - uses: security-and-quality paths: - - 'unblob' + - 'python' + - 'rust' - 'tests' paths-ignore: diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000000..3f27dac5d6 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,455 @@ +name: CI + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + merge_group: + workflow_dispatch: + inputs: + test_release: + description: If true, publish to test.pypi.org + required: true + default: true + type: boolean + +permissions: + contents: read + +env: + DOCKER_IMAGE: ghcr.io/onekey-sec/unblob + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check_pre_commit: + name: Check - pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + + - name: Setup Nix + uses: cachix/install-nix-action@v30 + with: + install_url: https://releases.nixos.org/nix/nix-2.18.8/install + + - name: Check pre-commit hook + uses: pre-commit/action@v3.0.1 + + check_pyright: + name: Check - pyright + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + + - name: Check - pyright + run: uv run pyright . + + run_python_tests: + name: Run tests (Python) + needs: [check_pre_commit, check_pyright] + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + python-version: ${{ matrix.python-version }} + + - name: Setup git lfs + uses: ./.github/actions/setup-git-lfs + + - name: Run pytest + run: uv run pytest -vvv + + run_rust_tests: + name: Run tests (Rust) + needs: [check_pre_commit] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.7 + + - name: Run cargo test + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + run: cargo test + + build_linux_wheels: + name: Build wheels (linux) + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [check_pre_commit] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - manylinux: manylinux2014 + target: x86_64 + - manylinux: manylinux2014 + target: aarch64 + - manylinux: musllinux_1_1 + target: x86_64 + # lief is not available for this platform (and no sdist is provided) + # - manylinux: musllinux_1_1 + # target: aarch64 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: "true" + manylinux: ${{ matrix.platform.manylinux }} + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform.manylinux }}-${{ matrix.platform.target }} + path: dist + + - name: Check wheel (x86-manylinux) + if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') + shell: bash + run: | + set -e + python3 -m venv wheel.venv + source wheel.venv/bin/activate + pip install dist/*.whl + unblob --help + + - name: Check wheel (cross-manylinux) + if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') + uses: uraimo/run-on-arch-action@v2 + with: + arch: ${{ matrix.platform.target }} + distro: ubuntu22.04 + githubToken: ${{ github.token }} + install: | + apt-get update + apt-get install -y --no-install-recommends python3-venv libmagic1 + run: | + set -e + python3 -m venv wheel.venv + source wheel.venv/bin/activate + pip install dist/*.whl + unblob --version + + - name: Check wheel (x86-musllinux) + if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') + uses: addnab/docker-run-action@v3 + with: + image: alpine:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + set -e + apk add py3-pip libmagic gcc lz4 musl-dev python3-dev + python3 -m venv wheel.venv + source wheel.venv/bin/activate + pip install dist/*.whl + unblob --version + + - name: Check wheel (cross-musllinux) + if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') + uses: uraimo/run-on-arch-action@v2 + with: + arch: ${{ matrix.platform.target }} + distro: alpine_latest + githubToken: ${{ github.token }} + install: | + apk add py3-pip libmagic gcc lz4 musl-dev python3-dev + run: | + set -e + python3 -m venv wheel.venv + source wheel.venv/bin/activate + pip install dist/*.whl + unblob --version + + build_macos_wheels: + name: Build wheels (macos) + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [check_pre_commit] + runs-on: ${{ matrix.platform.runner }} + strategy: + fail-fast: false + matrix: + platform: + - runner: macos-13 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: "true" + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + + - name: Check wheel + run: | + set -e + brew install libmagic + python3 -m venv .venv + source .venv/bin/activate + pip install --find-links dist unblob + unblob --version + + build_sdist: + name: Build sdist + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [check_pre_commit] + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + build-image: + name: Build Docker image + if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [build_linux_wheels] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - docker: linux/amd64 + wheel: wheels-manylinux2014-x86_64 + - docker: linux/arm64 + wheel: wheels-manylinux2014-aarch64 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.platform.wheel }} + path: dist + + - name: Prepare + run: | + platform=${{ matrix.platform.docker }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=sha + + - name: Set up QEMU + if: matrix.platform.docker != 'linux/amd64' + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.platform.docker }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.output.tags }} + outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && github.event_name == 'push' }} + + - name: Docker container vulnerability scan + id: scan + uses: anchore/scan-action@v6 + with: + image: ${{ env.DOCKER_IMAGE }} + fail-build: false + severity-cutoff: critical + only-fixed: true + + - name: Upload SARIF report + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Check unblob - help + run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help + + - name: Check unblob - show-external-dependencies + run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies + + - name: Check unblob - run for a file with --verbose + run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-and-push-image: + if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' + runs-on: ubuntu-latest + needs: + - build-image + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=sha + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} + + release: + name: Release + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: + - run_python_tests + - run_rust_tests + - build_linux_wheels + - build_macos_wheels + - build_sdist + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheel-* + path: dist + merge-multiple: true + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist/* + + - name: Publish to PyPI + if: ${{ startsWith(github.ref, 'refs/tags/') }} + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && secrets.TEST_PYPI_API_TOKEN || secrets.POETRY_PYPI_TOKEN_PYPI }} + MATURIN_REPOSITORY: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && 'testpypi' || 'pypi' }} + with: + command: upload + args: --non-interactive --skip-existing dist/* diff --git a/.github/workflows/build-publish-image.yml b/.github/workflows/build-publish-image.yml deleted file mode 100644 index fde30e41b5..0000000000 --- a/.github/workflows/build-publish-image.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: Build Docker image - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - -env: - DOCKER_IMAGE: ghcr.io/onekey-sec/unblob - -jobs: - build-image: - if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup git lfs - uses: ./.github/actions/setup-git-lfs - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: uv build - run: uv build --wheel - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_IMAGE }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=semver,pattern={{version}} - type=sha - - - name: Set up QEMU - if: matrix.platform != 'linux/amd64' - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - tags: ${{ steps.meta.output.tags }} - outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && 'true' || 'false' }} - - - name: Docker container vulnerability scan - id: scan - uses: anchore/scan-action@v6 - with: - image: ${{ env.DOCKER_IMAGE }} - fail-build: false - severity-cutoff: critical - only-fixed: true - - - name: Upload SARIF report - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: ${{ steps.scan.outputs.sarif }} - - - name: Check unblob - help - run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help - - - name: Check unblob - show-external-dependencies - run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies - - - name: Check unblob - run for a file with --verbose - run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digest-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge-and-push-image: - if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' - runs-on: ubuntu-latest - needs: - - build-image - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digest-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_IMAGE }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=semver,pattern={{version}} - type=sha - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c4b5e52b13..0000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: PyPI Release - -on: - release: - types: [created] - -jobs: - release: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup dependencies - uses: ./.github/actions/setup-dependencies - - - run: uv build - - run: uv publish - env: - UV_PUBLISH_TOKEN: ${{secrets.POETRY_PYPI_TOKEN_PYPI}} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index fe06ad6919..0000000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Run checks and tests - -on: - push: - branches: - - main - pull_request: - merge_group: - - -jobs: - check_pre_commit: - name: Check - pre-commit - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: Setup Nix - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.18.8/install - - - name: Check pre-commit hook - uses: pre-commit/action@v3.0.1 - - check_pyright: - name: Check - pyright - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: Check - pyright - run: uv run pyright . - - run_tests: - name: Run tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - with: - python-version: ${{ matrix.python-version }} - - - name: Setup git lfs - uses: ./.github/actions/setup-git-lfs - - - name: Run pytest - run: uv run pytest -vvv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 365a71bcfa..a8ce50146a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,18 @@ repos: entry: uv run taplo lint args: - --default-schema-catalogs + - id: cargo-fmt # rustup component add rustfmt + name: Check rust (cargo fmt) + entry: cargo fmt --all -- + language: system + types: [rust] + pass_filenames: false + - id: cargo-clippy # rustup component add clippy + name: Check rust (cargo clippy) + entry: cargo clippy --tests --all-features -- -D warnings + language: system + pass_filenames: false + types: [rust] - repo: https://github.com/jendrikseipp/vulture rev: v2.14 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..4c4e0e4a39 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1050 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "bytemuck" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "enumflags2" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c041f5090df68b32bcd905365fd51769c8b9d553fe87fde0b683534f10c01bd2" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "landlock" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18738c5d4c7fae6727a96adb94722ef7ce82f3eafea0a11777e258a93816537e" +dependencies = [ + "enumflags2", + "libc", + "thiserror", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libm" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bda4c6077b0b08da2c48b172195795498381a7c8988c9e6212a6c55c5b9bd70" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "nalgebra" +version = "0.32.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +dependencies = [ + "approx", + "matrixmultiply", + "nalgebra-macros", + "num-complex", + "num-rational", + "num-traits", + "rand", + "rand_distr", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.2", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-log" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ac84e6eec1159bc2a575c9ae6723baa6ee9d45873e9bebad1e3ad7e8d28a443" +dependencies = [ + "arc-swap", + "log", + "pyo3", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "regex" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + +[[package]] +name = "ryu" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" + +[[package]] +name = "safe_arch" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3460605018fdc9612bce72735cba0d27efbcd9904780d44c7e3a9948f96148a" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "simba" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + +[[package]] +name = "statrs" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f697a07e4606a0a25c044de247e583a330dbb1731d11bc7350b81f48ad567255" +dependencies = [ + "approx", + "nalgebra", + "num-traits", + "rand", +] + +[[package]] +name = "syn" +version = "2.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "thiserror" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unblob-rust" +version = "0.0.0" +dependencies = [ + "approx", + "criterion", + "landlock", + "log", + "pyo3", + "pyo3-log", + "rand", + "statrs", + "thiserror", +] + +[[package]] +name = "unicode-ident" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wide" +version = "0.7.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b828f995bf1e9622031f8009f8481a85406ce1f4d4588ff746d872043e855690" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..5890389758 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] +members = [ + "rust", +] +resolver = "2" + +[workspace.package] +authors = ["ONEKEY "] +edition = "2021" +license = "MIT" +publish = false +rust-version = "1.82.0" +version = "0.0.0" diff --git a/Dockerfile b/Dockerfile index 88ae3da5de..f583d644b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN chown -R unblob /data WORKDIR /data/output -COPY unblob/install-deps.sh / +COPY install-deps.sh / RUN sh -xeu /install-deps.sh # You MUST do an uv build before to have the wheel to copy & install here (CI action will do this when building) diff --git a/devenv.nix b/devenv.nix index dd68703545..aecd0d2bf4 100644 --- a/devenv.nix +++ b/devenv.nix @@ -24,6 +24,7 @@ ]; }; }; + languages.rust.enable = true; env.UV_LINK_MODE = "copy"; diff --git a/docs/development.md b/docs/development.md index adaa135af0..08b5b11b4d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -41,9 +41,9 @@ where the exciting stuff is. - **Git LFS**: We have big integration test files, and we are using Git LFS to track them. [Install `git-lfs`](https://git-lfs.github.com/) from the website. -- **Rust** (_for unblob-native_): unblob has a [Rust extension](https://github.com/onekey-sec/unblob-native) for - performance intensive processing. Building it is entirely optional and requires [`rustup`](https://rustup.rs/) to be - installed on the host system. Follow the instructions on the [rustup website](https://rustup.rs/) to install it. +- **Rust** some functionality of unblob is implemented in Rust. Building it requires a Rust toolchain + (e.g. via [`rustup`](https://rustup.rs/)) to be installed on the host system. Follow the + instructions on the [rustup website](https://rustup.rs/) to install it. - **pyenv** (_Recommended_): When you are working with multiple versions of Python, pyenv makes it very easy to install and use different versions and make virtualenvs. diff --git a/docs/index.md b/docs/index.md index b92c0157b8..0e65b0ee1d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -74,10 +74,10 @@ unblob has been developed with the following objectives in mind: - **Speed** - we want unblob to be blazing fast, that's why we use multi-processing by default, make sure to write efficient code, use - memory-mapped files, and use [Hyperscan](https://github.com/intel/hyperscan) - as a high-performance matching library. Computation-intensive functions are - written in [Rust](https://github.com/onekey-sec/unblob-native) and - called from Python using specific bindings. + memory-mapped files, and use + [Hyperscan](https://github.com/intel/hyperscan) as a + high-performance matching library. Computation-intensive functions + are written in Rust and called from Python using specific bindings. ## How does it work? diff --git a/flake.lock b/flake.lock index 84f69f91b7..73c49b917b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,21 +1,5 @@ { "nodes": { - "advisory-db": { - "flake": false, - "locked": { - "lastModified": 1735928634, - "narHash": "sha256-Qg1vJOuEohAbdRmTTOLrbbGsyK9KRB54r3+aBuOMctM=", - "owner": "rustsec", - "repo": "advisory-db", - "rev": "63a2f39924f66ca89cf5761f299a8a244fe02543", - "type": "github" - }, - "original": { - "owner": "rustsec", - "repo": "advisory-db", - "type": "github" - } - }, "cachix": { "inputs": { "devenv": "devenv_2", @@ -82,21 +66,6 @@ "type": "github" } }, - "crane": { - "locked": { - "lastModified": 1736032295, - "narHash": "sha256-QNRlMxQTT3rdgsQb3QxljO14kE8xxdDXNJ/4jIm4u3Q=", - "owner": "ipetkov", - "repo": "crane", - "rev": "9fa361afe873c740d5ca10ff526463d5807eab88", - "type": "github" - }, - "original": { - "owner": "ipetkov", - "repo": "crane", - "type": "github" - } - }, "devenv": { "inputs": { "cachix": "cachix", @@ -309,24 +278,6 @@ "type": "github" } }, - "flake-utils_3": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "gitignore": { "inputs": { "nixpkgs": [ @@ -393,21 +344,6 @@ "type": "github" } }, - "nix-filter": { - "locked": { - "lastModified": 1731533336, - "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=", - "owner": "numtide", - "repo": "nix-filter", - "rev": "f7653272fd234696ae94229839a99b73c9ab7de0", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "nix-filter", - "type": "github" - } - }, "nix-github-actions": { "inputs": { "nixpkgs": [ @@ -716,8 +652,7 @@ "devenv": "devenv", "filter": "filter", "flake-compat": "flake-compat_3", - "nixpkgs": "nixpkgs_3", - "unblob-native": "unblob-native" + "nixpkgs": "nixpkgs_3" } }, "systems": { @@ -734,45 +669,6 @@ "repo": "default", "type": "github" } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "unblob-native": { - "inputs": { - "advisory-db": "advisory-db", - "crane": "crane", - "flake-utils": "flake-utils_3", - "nix-filter": "nix-filter", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1736087889, - "narHash": "sha256-BblrBZ8iB9UQjYYn7digOVmJsAqLcNNZTSNZE0tDSB8=", - "owner": "onekey-sec", - "repo": "unblob-native", - "rev": "cb61f36fe73ee6e1729a33818d0650e0b5ea902d", - "type": "github" - }, - "original": { - "owner": "onekey-sec", - "repo": "unblob-native", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index a1bc0fbea6..ab89e0d1d3 100644 --- a/flake.nix +++ b/flake.nix @@ -3,10 +3,6 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; inputs.filter.url = "github:numtide/nix-filter"; - inputs.unblob-native = { - url = "github:onekey-sec/unblob-native"; - inputs.nixpkgs.follows = "nixpkgs"; - }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; @@ -29,7 +25,6 @@ nixpkgs, devenv, filter, - unblob-native, ... }@inputs: let @@ -58,7 +53,6 @@ { overlays.default = nixpkgs.lib.composeManyExtensions [ filter.overlays.default - unblob-native.overlays.default (import ./overlay.nix) ]; packages = forAllSystems ( diff --git a/fuzzing/search_chunks_fuzzer.py b/fuzzing/search_chunks_fuzzer.py index 4e0d83a95c..5e2efe7997 100755 --- a/fuzzing/search_chunks_fuzzer.py +++ b/fuzzing/search_chunks_fuzzer.py @@ -23,7 +23,7 @@ def extract(inpath: Path, outpath: Path): # noqa: ARG001 with atheris.import_hook.instrument_imports( - include=["unblob"], exclude=["unblob_native"] + include=["unblob"], exclude=["unblob._rust"] ): from unblob.extractors.command import Command from unblob.file_utils import File diff --git a/unblob/install-deps.sh b/install-deps.sh similarity index 100% rename from unblob/install-deps.sh rename to install-deps.sh diff --git a/overlay.nix b/overlay.nix index 390e14d2e8..9c5051e03e 100644 --- a/overlay.nix +++ b/overlay.nix @@ -13,21 +13,31 @@ final: prev: pyproject_toml = (builtins.fromTOML (builtins.readFile ./pyproject.toml)); version = pyproject_toml.project.version; in - (prev.unblob.override { e2fsprogs = final.e2fsprogs-nofortify; }).overridePythonAttrs (super: { + (prev.unblob.override { e2fsprogs = final.e2fsprogs-nofortify; }).overridePythonAttrs (super: rec { inherit version; src = final.nix-filter { root = ./.; include = [ + "Cargo.lock" + "Cargo.toml" "pyproject.toml" - "unblob" + "python" + "rust" "tests" "README.md" ]; }; # remove this when packaging changes are upstreamed - build-system = with final.python3.pkgs; [ hatchling ]; + cargoDeps = final.rustPlatform.importCargoLock { + lockFile = ./Cargo.lock; + }; + + nativeBuildInputs = with final.rustPlatform; [ + cargoSetupHook + maturinBuildHook + ]; # override disabling of 'test_all_handlers[filesystem.extfs]' from upstream pytestFlagsArray = [ diff --git a/pyproject.toml b/pyproject.toml index 74ff7d7c2e..5619cb07b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "structlog>=24.1.0", "treelib>=1.7.0", "ubi-reader>=0.8.9", - "unblob-native>=0.1.5", ] description = "Extract files from any kind of container formats" license = { text = "MIT" } @@ -38,7 +37,7 @@ unblob = "unblob.cli:main" [dependency-groups] dev = [ - "atheris>=2.3,<3.0; python_version < '3.12'", + "atheris>=2.3,<3.0; sys_platform == 'linux' and python_version < '3.12'", "pre-commit>=3.5,<5.0", "pyright>=1.1.349", "pytest-cov>=3,<7", @@ -126,6 +125,9 @@ ignore-init-module-imports = true [tool.ruff.lint.flake8-comprehensions] allow-dict-calls-with-keyword-arguments = true # code like `dict(a=1, b=2)` is allowed +[tool.ruff.lint.isort] +known-first-party = ["unblob"] + [tool.ruff.lint.mccabe] max-complexity = 8 @@ -171,8 +173,8 @@ fail_under = 90 skip_covered = true [tool.vulture] -exclude = ["unblob/testing.py"] -paths = ["unblob/", "vulture_whitelist.py"] +exclude = ["python/unblob/testing.py"] +paths = ["python/", "vulture_whitelist.py"] [tool.pyright] exclude = [ @@ -181,6 +183,19 @@ exclude = [ ] typeCheckingMode = "standard" + +[tool.maturin] +locked = true +manifest-path = "rust/Cargo.toml" +module-name = "unblob._rust" +python-packages = ["unblob"] +python-source = "python" + +features = [ + "pyo3/abi3-py39", # https://docs.python.org/3/c-api/stable.html + "pyo3/extension-module", # This is an extension module +] + [build-system] -build-backend = "hatchling.build" -requires = ["hatchling"] +build-backend = "maturin" +requires = ["maturin>=1.0,<2.0"] diff --git a/unblob/__init__.py b/python/unblob/__init__.py similarity index 100% rename from unblob/__init__.py rename to python/unblob/__init__.py diff --git a/unblob/handlers/archive/__init__.py b/python/unblob/_rust/__init__.pyi similarity index 100% rename from unblob/handlers/archive/__init__.py rename to python/unblob/_rust/__init__.pyi diff --git a/python/unblob/_rust/math_tools.pyi b/python/unblob/_rust/math_tools.pyi new file mode 100644 index 0000000000..b7a3c7c8ee --- /dev/null +++ b/python/unblob/_rust/math_tools.pyi @@ -0,0 +1,2 @@ +def shannon_entropy(data: bytes) -> float: ... +def chi_square_probability(data: bytes) -> float: ... diff --git a/python/unblob/_rust/sandbox.pyi b/python/unblob/_rust/sandbox.pyi new file mode 100644 index 0000000000..43375a3e1b --- /dev/null +++ b/python/unblob/_rust/sandbox.pyi @@ -0,0 +1,23 @@ +import os + +import typing_extensions + +_Path: typing_extensions.TypeAlias = os.PathLike | str + +class AccessFS: + @staticmethod + def read(access_dir: _Path) -> AccessFS: ... + @staticmethod + def read_write(access_dir: _Path) -> AccessFS: ... + @staticmethod + def make_reg(access_dir: _Path) -> AccessFS: ... + @staticmethod + def make_dir(access_dir: _Path) -> AccessFS: ... + @staticmethod + def remove_dir(access_dir: _Path) -> AccessFS: ... + @staticmethod + def remove_file(access_dir: _Path) -> AccessFS: ... + +def restrict_access(*args: AccessFS) -> None: ... + +class SandboxError(Exception): ... diff --git a/unblob/cli.py b/python/unblob/cli.py similarity index 100% rename from unblob/cli.py rename to python/unblob/cli.py diff --git a/unblob/cli_options.py b/python/unblob/cli_options.py similarity index 100% rename from unblob/cli_options.py rename to python/unblob/cli_options.py diff --git a/unblob/dependencies.py b/python/unblob/dependencies.py similarity index 100% rename from unblob/dependencies.py rename to python/unblob/dependencies.py diff --git a/unblob/extractor.py b/python/unblob/extractor.py similarity index 100% rename from unblob/extractor.py rename to python/unblob/extractor.py diff --git a/unblob/extractors/README.md b/python/unblob/extractors/README.md similarity index 100% rename from unblob/extractors/README.md rename to python/unblob/extractors/README.md diff --git a/unblob/extractors/__init__.py b/python/unblob/extractors/__init__.py similarity index 100% rename from unblob/extractors/__init__.py rename to python/unblob/extractors/__init__.py diff --git a/unblob/extractors/command.py b/python/unblob/extractors/command.py similarity index 100% rename from unblob/extractors/command.py rename to python/unblob/extractors/command.py diff --git a/unblob/file_utils.py b/python/unblob/file_utils.py similarity index 100% rename from unblob/file_utils.py rename to python/unblob/file_utils.py diff --git a/unblob/finder.py b/python/unblob/finder.py similarity index 100% rename from unblob/finder.py rename to python/unblob/finder.py diff --git a/unblob/handlers/__init__.py b/python/unblob/handlers/__init__.py similarity index 100% rename from unblob/handlers/__init__.py rename to python/unblob/handlers/__init__.py diff --git a/unblob/handlers/compression/__init__.py b/python/unblob/handlers/archive/__init__.py similarity index 100% rename from unblob/handlers/compression/__init__.py rename to python/unblob/handlers/archive/__init__.py diff --git a/unblob/handlers/archive/_safe_tarfile.py b/python/unblob/handlers/archive/_safe_tarfile.py similarity index 100% rename from unblob/handlers/archive/_safe_tarfile.py rename to python/unblob/handlers/archive/_safe_tarfile.py diff --git a/unblob/handlers/archive/ar.py b/python/unblob/handlers/archive/ar.py similarity index 100% rename from unblob/handlers/archive/ar.py rename to python/unblob/handlers/archive/ar.py diff --git a/unblob/handlers/archive/arc.py b/python/unblob/handlers/archive/arc.py similarity index 100% rename from unblob/handlers/archive/arc.py rename to python/unblob/handlers/archive/arc.py diff --git a/unblob/handlers/archive/arj.py b/python/unblob/handlers/archive/arj.py similarity index 100% rename from unblob/handlers/archive/arj.py rename to python/unblob/handlers/archive/arj.py diff --git a/unblob/handlers/archive/autel/ecc.py b/python/unblob/handlers/archive/autel/ecc.py similarity index 100% rename from unblob/handlers/archive/autel/ecc.py rename to python/unblob/handlers/archive/autel/ecc.py diff --git a/unblob/handlers/archive/cab.py b/python/unblob/handlers/archive/cab.py similarity index 100% rename from unblob/handlers/archive/cab.py rename to python/unblob/handlers/archive/cab.py diff --git a/unblob/handlers/archive/cpio.py b/python/unblob/handlers/archive/cpio.py similarity index 100% rename from unblob/handlers/archive/cpio.py rename to python/unblob/handlers/archive/cpio.py diff --git a/unblob/handlers/archive/dlink/encrpted_img.py b/python/unblob/handlers/archive/dlink/encrpted_img.py similarity index 100% rename from unblob/handlers/archive/dlink/encrpted_img.py rename to python/unblob/handlers/archive/dlink/encrpted_img.py diff --git a/unblob/handlers/archive/dlink/shrs.py b/python/unblob/handlers/archive/dlink/shrs.py similarity index 100% rename from unblob/handlers/archive/dlink/shrs.py rename to python/unblob/handlers/archive/dlink/shrs.py diff --git a/unblob/handlers/archive/dmg.py b/python/unblob/handlers/archive/dmg.py similarity index 100% rename from unblob/handlers/archive/dmg.py rename to python/unblob/handlers/archive/dmg.py diff --git a/unblob/handlers/archive/engeniustech/engenius.py b/python/unblob/handlers/archive/engeniustech/engenius.py similarity index 100% rename from unblob/handlers/archive/engeniustech/engenius.py rename to python/unblob/handlers/archive/engeniustech/engenius.py diff --git a/unblob/handlers/archive/hp/bdl.py b/python/unblob/handlers/archive/hp/bdl.py similarity index 100% rename from unblob/handlers/archive/hp/bdl.py rename to python/unblob/handlers/archive/hp/bdl.py diff --git a/unblob/handlers/archive/hp/ipkg.py b/python/unblob/handlers/archive/hp/ipkg.py similarity index 100% rename from unblob/handlers/archive/hp/ipkg.py rename to python/unblob/handlers/archive/hp/ipkg.py diff --git a/unblob/handlers/archive/instar/bneg.py b/python/unblob/handlers/archive/instar/bneg.py similarity index 100% rename from unblob/handlers/archive/instar/bneg.py rename to python/unblob/handlers/archive/instar/bneg.py diff --git a/unblob/handlers/archive/instar/instar_hd.py b/python/unblob/handlers/archive/instar/instar_hd.py similarity index 100% rename from unblob/handlers/archive/instar/instar_hd.py rename to python/unblob/handlers/archive/instar/instar_hd.py diff --git a/unblob/handlers/archive/netgear/chk.py b/python/unblob/handlers/archive/netgear/chk.py similarity index 100% rename from unblob/handlers/archive/netgear/chk.py rename to python/unblob/handlers/archive/netgear/chk.py diff --git a/unblob/handlers/archive/netgear/trx.py b/python/unblob/handlers/archive/netgear/trx.py similarity index 100% rename from unblob/handlers/archive/netgear/trx.py rename to python/unblob/handlers/archive/netgear/trx.py diff --git a/unblob/handlers/archive/qnap/qnap_nas.py b/python/unblob/handlers/archive/qnap/qnap_nas.py similarity index 100% rename from unblob/handlers/archive/qnap/qnap_nas.py rename to python/unblob/handlers/archive/qnap/qnap_nas.py diff --git a/unblob/handlers/archive/rar.py b/python/unblob/handlers/archive/rar.py similarity index 100% rename from unblob/handlers/archive/rar.py rename to python/unblob/handlers/archive/rar.py diff --git a/unblob/handlers/archive/sevenzip.py b/python/unblob/handlers/archive/sevenzip.py similarity index 100% rename from unblob/handlers/archive/sevenzip.py rename to python/unblob/handlers/archive/sevenzip.py diff --git a/unblob/handlers/archive/stuffit.py b/python/unblob/handlers/archive/stuffit.py similarity index 100% rename from unblob/handlers/archive/stuffit.py rename to python/unblob/handlers/archive/stuffit.py diff --git a/unblob/handlers/archive/tar.py b/python/unblob/handlers/archive/tar.py similarity index 100% rename from unblob/handlers/archive/tar.py rename to python/unblob/handlers/archive/tar.py diff --git a/unblob/handlers/archive/xiaomi/hdr.py b/python/unblob/handlers/archive/xiaomi/hdr.py similarity index 100% rename from unblob/handlers/archive/xiaomi/hdr.py rename to python/unblob/handlers/archive/xiaomi/hdr.py diff --git a/unblob/handlers/archive/zip.py b/python/unblob/handlers/archive/zip.py similarity index 100% rename from unblob/handlers/archive/zip.py rename to python/unblob/handlers/archive/zip.py diff --git a/unblob/handlers/executable/__init__.py b/python/unblob/handlers/compression/__init__.py similarity index 100% rename from unblob/handlers/executable/__init__.py rename to python/unblob/handlers/compression/__init__.py diff --git a/unblob/handlers/compression/_gzip_reader.py b/python/unblob/handlers/compression/_gzip_reader.py similarity index 100% rename from unblob/handlers/compression/_gzip_reader.py rename to python/unblob/handlers/compression/_gzip_reader.py diff --git a/unblob/handlers/compression/bzip2.py b/python/unblob/handlers/compression/bzip2.py similarity index 100% rename from unblob/handlers/compression/bzip2.py rename to python/unblob/handlers/compression/bzip2.py diff --git a/unblob/handlers/compression/compress.py b/python/unblob/handlers/compression/compress.py similarity index 100% rename from unblob/handlers/compression/compress.py rename to python/unblob/handlers/compression/compress.py diff --git a/unblob/handlers/compression/gzip.py b/python/unblob/handlers/compression/gzip.py similarity index 100% rename from unblob/handlers/compression/gzip.py rename to python/unblob/handlers/compression/gzip.py diff --git a/unblob/handlers/compression/lz4.py b/python/unblob/handlers/compression/lz4.py similarity index 100% rename from unblob/handlers/compression/lz4.py rename to python/unblob/handlers/compression/lz4.py diff --git a/unblob/handlers/compression/lzh.py b/python/unblob/handlers/compression/lzh.py similarity index 100% rename from unblob/handlers/compression/lzh.py rename to python/unblob/handlers/compression/lzh.py diff --git a/unblob/handlers/compression/lzip.py b/python/unblob/handlers/compression/lzip.py similarity index 100% rename from unblob/handlers/compression/lzip.py rename to python/unblob/handlers/compression/lzip.py diff --git a/unblob/handlers/compression/lzma.py b/python/unblob/handlers/compression/lzma.py similarity index 100% rename from unblob/handlers/compression/lzma.py rename to python/unblob/handlers/compression/lzma.py diff --git a/unblob/handlers/compression/lzo.py b/python/unblob/handlers/compression/lzo.py similarity index 100% rename from unblob/handlers/compression/lzo.py rename to python/unblob/handlers/compression/lzo.py diff --git a/unblob/handlers/compression/xz.py b/python/unblob/handlers/compression/xz.py similarity index 100% rename from unblob/handlers/compression/xz.py rename to python/unblob/handlers/compression/xz.py diff --git a/unblob/handlers/compression/zlib.py b/python/unblob/handlers/compression/zlib.py similarity index 100% rename from unblob/handlers/compression/zlib.py rename to python/unblob/handlers/compression/zlib.py diff --git a/unblob/handlers/compression/zstd.py b/python/unblob/handlers/compression/zstd.py similarity index 100% rename from unblob/handlers/compression/zstd.py rename to python/unblob/handlers/compression/zstd.py diff --git a/unblob/handlers/filesystem/__init__.py b/python/unblob/handlers/executable/__init__.py similarity index 100% rename from unblob/handlers/filesystem/__init__.py rename to python/unblob/handlers/executable/__init__.py diff --git a/unblob/handlers/executable/elf.py b/python/unblob/handlers/executable/elf.py similarity index 100% rename from unblob/handlers/executable/elf.py rename to python/unblob/handlers/executable/elf.py diff --git a/unblob/handlers/filesystem/android/__init__.py b/python/unblob/handlers/filesystem/__init__.py similarity index 100% rename from unblob/handlers/filesystem/android/__init__.py rename to python/unblob/handlers/filesystem/__init__.py diff --git a/unblob/py.typed b/python/unblob/handlers/filesystem/android/__init__.py similarity index 100% rename from unblob/py.typed rename to python/unblob/handlers/filesystem/android/__init__.py diff --git a/unblob/handlers/filesystem/android/sparse.py b/python/unblob/handlers/filesystem/android/sparse.py similarity index 100% rename from unblob/handlers/filesystem/android/sparse.py rename to python/unblob/handlers/filesystem/android/sparse.py diff --git a/unblob/handlers/filesystem/cramfs.py b/python/unblob/handlers/filesystem/cramfs.py similarity index 100% rename from unblob/handlers/filesystem/cramfs.py rename to python/unblob/handlers/filesystem/cramfs.py diff --git a/unblob/handlers/filesystem/extfs.py b/python/unblob/handlers/filesystem/extfs.py similarity index 100% rename from unblob/handlers/filesystem/extfs.py rename to python/unblob/handlers/filesystem/extfs.py diff --git a/unblob/handlers/filesystem/fat.py b/python/unblob/handlers/filesystem/fat.py similarity index 100% rename from unblob/handlers/filesystem/fat.py rename to python/unblob/handlers/filesystem/fat.py diff --git a/unblob/handlers/filesystem/iso9660.py b/python/unblob/handlers/filesystem/iso9660.py similarity index 100% rename from unblob/handlers/filesystem/iso9660.py rename to python/unblob/handlers/filesystem/iso9660.py diff --git a/unblob/handlers/filesystem/jffs2.py b/python/unblob/handlers/filesystem/jffs2.py similarity index 100% rename from unblob/handlers/filesystem/jffs2.py rename to python/unblob/handlers/filesystem/jffs2.py diff --git a/unblob/handlers/filesystem/ntfs.py b/python/unblob/handlers/filesystem/ntfs.py similarity index 100% rename from unblob/handlers/filesystem/ntfs.py rename to python/unblob/handlers/filesystem/ntfs.py diff --git a/unblob/handlers/filesystem/romfs.py b/python/unblob/handlers/filesystem/romfs.py similarity index 100% rename from unblob/handlers/filesystem/romfs.py rename to python/unblob/handlers/filesystem/romfs.py diff --git a/unblob/handlers/filesystem/squashfs.py b/python/unblob/handlers/filesystem/squashfs.py similarity index 100% rename from unblob/handlers/filesystem/squashfs.py rename to python/unblob/handlers/filesystem/squashfs.py diff --git a/unblob/handlers/filesystem/ubi.py b/python/unblob/handlers/filesystem/ubi.py similarity index 100% rename from unblob/handlers/filesystem/ubi.py rename to python/unblob/handlers/filesystem/ubi.py diff --git a/unblob/handlers/filesystem/yaffs.py b/python/unblob/handlers/filesystem/yaffs.py similarity index 100% rename from unblob/handlers/filesystem/yaffs.py rename to python/unblob/handlers/filesystem/yaffs.py diff --git a/unblob/hookspecs.py b/python/unblob/hookspecs.py similarity index 100% rename from unblob/hookspecs.py rename to python/unblob/hookspecs.py diff --git a/unblob/identifiers.py b/python/unblob/identifiers.py similarity index 100% rename from unblob/identifiers.py rename to python/unblob/identifiers.py diff --git a/unblob/iter_utils.py b/python/unblob/iter_utils.py similarity index 100% rename from unblob/iter_utils.py rename to python/unblob/iter_utils.py diff --git a/unblob/logging.py b/python/unblob/logging.py similarity index 100% rename from unblob/logging.py rename to python/unblob/logging.py diff --git a/unblob/models.py b/python/unblob/models.py similarity index 100% rename from unblob/models.py rename to python/unblob/models.py diff --git a/unblob/parser.py b/python/unblob/parser.py similarity index 100% rename from unblob/parser.py rename to python/unblob/parser.py diff --git a/unblob/plugins.py b/python/unblob/plugins.py similarity index 100% rename from unblob/plugins.py rename to python/unblob/plugins.py diff --git a/unblob/pool.py b/python/unblob/pool.py similarity index 100% rename from unblob/pool.py rename to python/unblob/pool.py diff --git a/unblob/processing.py b/python/unblob/processing.py similarity index 99% rename from unblob/processing.py rename to python/unblob/processing.py index 7f271743b8..84a149edb4 100644 --- a/unblob/processing.py +++ b/python/unblob/processing.py @@ -9,8 +9,8 @@ import magic import plotext as plt from structlog import get_logger -from unblob_native import math_tools as mt +from unblob._rust import math_tools as mt from unblob.handlers import BUILTIN_DIR_HANDLERS, BUILTIN_HANDLERS, Handlers from .extractor import carve_unknown_chunk, carve_valid_chunk, fix_extracted_directory diff --git a/python/unblob/py.typed b/python/unblob/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/unblob/report.py b/python/unblob/report.py similarity index 100% rename from unblob/report.py rename to python/unblob/report.py diff --git a/unblob/sandbox.py b/python/unblob/sandbox.py similarity index 98% rename from unblob/sandbox.py rename to python/unblob/sandbox.py index d12b27f0ea..fb7df47df9 100644 --- a/unblob/sandbox.py +++ b/python/unblob/sandbox.py @@ -6,7 +6,8 @@ from typing import Callable, Optional, TypeVar from structlog import get_logger -from unblob_native.sandbox import ( + +from unblob._rust.sandbox import ( AccessFS, SandboxError, restrict_access, diff --git a/unblob/testing.py b/python/unblob/testing.py similarity index 98% rename from unblob/testing.py rename to python/unblob/testing.py index 196261d76a..e024e146dc 100644 --- a/unblob/testing.py +++ b/python/unblob/testing.py @@ -11,8 +11,8 @@ from lark.lark import Lark from lark.visitors import Discard, Transformer from pytest_cov.embed import cleanup_on_sigterm -from unblob_native.sandbox import AccessFS, SandboxError, restrict_access +from unblob._rust.sandbox import AccessFS, SandboxError, restrict_access from unblob.finder import build_hyperscan_database from unblob.logging import configure_logger from unblob.models import ProcessResult diff --git a/unblob/ui.py b/python/unblob/ui.py similarity index 100% rename from unblob/ui.py rename to python/unblob/ui.py diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000000..135e9cbadd --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,37 @@ +[package] +authors.workspace = true +edition.workspace = true +license.workspace = true +name = "unblob-rust" +rust-version.workspace = true +version.workspace = true + +[lib] +crate-type = [ + "cdylib", # for Python extension + "rlib", # for benchmarking +] +name = "unblob_rust" + +[dependencies] +log = "0.4.22" +pyo3 = "0.22.4" +pyo3-log = "0.11.0" +statrs = "0.17.1" +thiserror = "1.0.64" + +[target.'cfg(target_os = "linux")'.dependencies] +landlock = "0.4.1" + +[dev-dependencies] +approx = "0.5.0" +criterion = "0.5.1" +rand = "0.8.4" + +[[bench]] +harness = false +name = "benches_main" + +[lints.rust] +# Required for Rust >= 1.84 && pyo3 < 0.23 +unexpected_cfgs = { level = "allow", check-cfg = ['cfg(addr_of)'] } diff --git a/rust/benches/benches_main.rs b/rust/benches/benches_main.rs new file mode 100755 index 0000000000..d32eabe375 --- /dev/null +++ b/rust/benches/benches_main.rs @@ -0,0 +1,50 @@ +#![allow(clippy::identity_op)] +#![allow(non_snake_case, non_upper_case_globals)] + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use rand::prelude::*; + +const kB: usize = 1024; +const MB: usize = 1024 * kB; + +fn shannon_entropy(c: &mut Criterion) { + let mut sample = vec![0u8; 1 * MB]; + StdRng::seed_from_u64(5).fill(&mut sample[..]); + + let mut group = c.benchmark_group("Shannon entropy"); + + for sample_size in [256, 1 * kB, 64 * kB, 256 * kB, 1 * MB] { + group.throughput(Throughput::Bytes(sample_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(sample_size), + &sample_size, + |b, &size| { + b.iter(|| unblob_native::math_tools::shannon_entropy(&sample[0..size])); + }, + ); + } + group.finish(); +} + +fn chi_square_probability(c: &mut Criterion) { + let mut sample = vec![0u8; 1 * MB]; + StdRng::seed_from_u64(5).fill(&mut sample[..]); + + let mut group = c.benchmark_group("Chi square probability"); + + for sample_size in [256, 1 * kB, 64 * kB, 256 * kB, 1 * MB] { + group.throughput(Throughput::Bytes(sample_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(sample_size), + &sample_size, + |b, &size| { + b.iter(|| unblob_native::math_tools::chi_square_probability(&sample[0..size])); + }, + ); + } + group.finish(); +} + +criterion_group!(benches, shannon_entropy, chi_square_probability); + +criterion_main!(benches); diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000000..da974a537a --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,15 @@ +pub mod math_tools; +pub mod sandbox; + +use pyo3::prelude::*; + +/// Performance-critical functionality +#[pymodule] +fn _rust(m: &Bound<'_, PyModule>) -> PyResult<()> { + math_tools::init_module(m)?; + sandbox::init_module(m)?; + + pyo3_log::init(); + + Ok(()) +} diff --git a/rust/src/math_tools.rs b/rust/src/math_tools.rs new file mode 100644 index 0000000000..c3e5ae4c89 --- /dev/null +++ b/rust/src/math_tools.rs @@ -0,0 +1,146 @@ +use pyo3::prelude::*; +use statrs::distribution::{ChiSquared, ContinuousCDF}; + +pub fn shannon_entropy(data: &[u8]) -> f64 { + let mut entropy = 0.0; + let mut counts = [0; 256]; + + for &b in data { + counts[b as usize] += 1; + } + + for &count in &counts { + if count == 0 { + continue; + } + + let p = count as f64 / data.len() as f64; + entropy -= p * p.log2(); + } + + entropy +} +/// Calculates Shannon entropy of data +#[pyfunction(name = "shannon_entropy")] +pub fn py_shannon_entropy(py: Python, data: &[u8]) -> PyResult { + py.allow_threads(|| Ok(shannon_entropy(data))) +} + +pub fn chi_square_probability(data: &[u8]) -> f64 { + if data.is_empty() { + return 0.0; + } + + // Total number of possible byte values (0–255) + let num_bins = 256; + let expected_count = data.len() as f64 / num_bins as f64; + + // Frequency count for each byte value + let mut frequencies = [0u32; 256]; + for &byte in data { + frequencies[byte as usize] += 1; + } + + // Calculate chi-square statistic + let chi_square: f64 = frequencies + .iter() + .map(|&obs| { + let observed = obs as f64; + (observed - expected_count).powi(2) / expected_count + }) + .sum(); + + // Degrees of freedom: 255 (256 bins - 1) + let degrees_of_freedom = (num_bins - 1) as f64; + let chi_squared = ChiSquared::new(degrees_of_freedom).unwrap(); + + // Compute p-value (chi-square probability) + 1.0 - chi_squared.cdf(chi_square) +} +/// Calculates Chi Square of data +#[pyfunction(name = "chi_square_probability")] +pub fn py_chi_square_probability(py: Python, data: &[u8]) -> PyResult { + py.allow_threads(|| Ok(chi_square_probability(data))) +} + +pub fn init_module(root_module: &Bound<'_, PyModule>) -> PyResult<()> { + let module = PyModule::new_bound(root_module.py(), "math_tools")?; + module.add_function(wrap_pyfunction!(py_shannon_entropy, &module)?)?; + module.add_function(wrap_pyfunction!(py_chi_square_probability, &module)?)?; + + root_module.add_submodule(&module)?; + + root_module + .py() + .import_bound("sys")? + .getattr("modules")? + .set_item("unblob._rust.math", module)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + mod shannon { + use super::*; + + #[test] + fn test_shannon_entropy() { + let input = b"000111"; // 50% entropy distribution ~ 1 bit information + + assert_eq!(shannon_entropy(input), 1.0); + } + } + + mod chi_square { + use super::*; + use rand::prelude::*; + + #[test] + fn test_non_uniform_distribution() { + let uniform_distribution = [0u8; 4096]; + let chi_square_value = chi_square_probability(&uniform_distribution); + + assert_eq!( + chi_square_value, 0.0, + "Chi-square probability for fully non uniform distributions should be 0.0" + ); + } + + #[test] + fn test_uniform_distribution() { + let uniform_distribution: Vec = (0..=255).collect(); + let chi_square_value = chi_square_probability(&uniform_distribution); + + assert_eq!( + chi_square_value, 1.0, + "Chi-square probability for fully uniform distributions should be 1.0" + ); + } + + #[test] + fn test_random_distribution() { + let mut random_data = [0u8; 4096]; + StdRng::from_entropy().fill_bytes(&mut random_data); + let chi_square_value = chi_square_probability(&random_data); + + assert!( + chi_square_value > 0.0 && chi_square_value < 1.0, + "Chi-square probability for PRNG distribution should be within bounds" + ); + } + + #[test] + fn test_empty_data() { + let empty_data: Vec = Vec::new(); + let chi_square_value = chi_square_probability(&empty_data); + + assert_eq!( + chi_square_value, 0.0, + "Chi-square probability for empty data should be 0.0" + ); + } + } +} diff --git a/rust/src/sandbox/linux.rs b/rust/src/sandbox/linux.rs new file mode 100644 index 0000000000..50a66b2477 --- /dev/null +++ b/rust/src/sandbox/linux.rs @@ -0,0 +1,108 @@ +use landlock::{ + path_beneath_rules, Access, AccessFs, Ruleset, RulesetAttr, RulesetCreatedAttr, RulesetStatus, + ABI, +}; +use log; + +use std::path::Path; + +use crate::sandbox::{AccessFS, SandboxError}; + +impl AccessFS { + fn read(&self) -> Option<&Path> { + if let Self::Read(path) = self { + Some(path) + } else { + None + } + } + + fn read_write(&self) -> Option<&Path> { + if let Self::ReadWrite(path) = self { + Some(path) + } else { + None + } + } + + fn make_reg(&self) -> Option<&Path> { + if let Self::MakeReg(path) = self { + Some(path) + } else { + None + } + } + + fn make_dir(&self) -> Option<&Path> { + if let Self::MakeDir(path) = self { + Some(path) + } else { + None + } + } + + fn remove_dir(&self) -> Option<&Path> { + if let Self::RemoveDir(path) = self { + Some(path) + } else { + None + } + } + + fn remove_file(&self) -> Option<&Path> { + if let Self::RemoveFile(path) = self { + Some(path) + } else { + None + } + } +} + +pub fn restrict_access(access_rules: &[AccessFS]) -> Result<(), SandboxError> { + let abi = ABI::V2; + + let read_only: Vec<&Path> = access_rules.iter().filter_map(AccessFS::read).collect(); + + let read_write: Vec<&Path> = access_rules + .iter() + .filter_map(AccessFS::read_write) + .collect(); + + let create_file: Vec<&Path> = access_rules.iter().filter_map(AccessFS::make_reg).collect(); + + let create_directory: Vec<&Path> = access_rules.iter().filter_map(AccessFS::make_dir).collect(); + + let remove_directory: Vec<&Path> = access_rules + .iter() + .filter_map(AccessFS::remove_dir) + .collect(); + + let remove_file: Vec<&Path> = access_rules + .iter() + .filter_map(AccessFS::remove_file) + .collect(); + + let status = Ruleset::default() + .handle_access(AccessFs::from_all(abi))? + .create()? + .add_rules(path_beneath_rules(read_write, AccessFs::from_all(abi)))? + .add_rules(path_beneath_rules(create_file, AccessFs::MakeReg))? + .add_rules(path_beneath_rules(create_directory, AccessFs::MakeDir))? + .add_rules(path_beneath_rules(read_only, AccessFs::from_read(abi)))? + .add_rules(path_beneath_rules(remove_directory, AccessFs::RemoveDir))? + .add_rules(path_beneath_rules(remove_file, AccessFs::RemoveFile))? + .restrict_self()?; + + if status.ruleset == RulesetStatus::NotEnforced { + log::error!("Could not enforce restictions"); + return Err(SandboxError::NotEnforced); + } + + log::info!( + "Activated FS access restrictions; rules={:?}, status={:?}", + access_rules, + status.ruleset + ); + + Ok(()) +} diff --git a/rust/src/sandbox/mod.rs b/rust/src/sandbox/mod.rs new file mode 100644 index 0000000000..439d6752c6 --- /dev/null +++ b/rust/src/sandbox/mod.rs @@ -0,0 +1,126 @@ +#[cfg_attr(target_os = "linux", path = "linux.rs")] +#[cfg_attr(not(target_os = "linux"), path = "unsupported.rs")] +mod sandbox_impl; + +use pyo3::{create_exception, exceptions::PyException, prelude::*, types::PyTuple}; +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Clone, Debug)] +pub enum AccessFS { + Read(PathBuf), + ReadWrite(PathBuf), + MakeReg(PathBuf), + MakeDir(PathBuf), + RemoveDir(PathBuf), + RemoveFile(PathBuf), +} + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum SandboxError { + #[error("Sandboxing is not implemented on this system")] + NotImplemented, + #[error("Could not enforce sandbox restrictions")] + NotEnforced, + #[cfg(target_os = "linux")] + #[error(transparent)] + LandlockError(#[from] landlock::RulesetError), +} + +/// Enforces access restrictions +#[pyfunction(name = "restrict_access", signature=(*rules))] +fn py_restrict_access(rules: &Bound<'_, PyTuple>) -> PyResult<()> { + sandbox_impl::restrict_access( + &rules + .iter() + .map(|r| Ok(r.extract::()?.access)) + .collect::>>()?, + ) + .map_err(|err| PySandboxError::new_err((PySandboxErrorKind::from(&err), err.to_string()))) +} + +create_exception!(unblob_native.sandbox, PySandboxError, PyException); + +#[pyclass(eq, eq_int, name = "SandboxErrorKind")] +#[derive(PartialEq)] +enum PySandboxErrorKind { + NotImplemented, + NotEnforced, + Unknown, +} + +impl From<&SandboxError> for PySandboxErrorKind { + fn from(value: &SandboxError) -> Self { + #[allow(unreachable_patterns)] // There are conditional pattern variants that may not exist + match value { + SandboxError::NotImplemented => Self::NotImplemented, + SandboxError::NotEnforced => Self::NotEnforced, + _ => Self::Unknown, + } + } +} + +#[pyclass(name = "AccessFS", module = "unblob_native.sandbox")] +#[derive(Clone)] +struct PyAccessFS { + access: AccessFS, +} + +impl PyAccessFS { + fn new(access: AccessFS) -> Self { + Self { access } + } +} + +#[pymethods] +impl PyAccessFS { + #[staticmethod] + fn read(dir: PathBuf) -> Self { + Self::new(AccessFS::Read(dir)) + } + + #[staticmethod] + fn read_write(dir: PathBuf) -> Self { + Self::new(AccessFS::ReadWrite(dir)) + } + + #[staticmethod] + fn make_reg(dir: PathBuf) -> Self { + Self::new(AccessFS::MakeReg(dir)) + } + + #[staticmethod] + fn make_dir(dir: PathBuf) -> Self { + Self::new(AccessFS::MakeDir(dir)) + } + + #[staticmethod] + fn remove_dir(dir: PathBuf) -> Self { + Self::new(AccessFS::RemoveDir(dir)) + } + + #[staticmethod] + fn remove_file(dir: PathBuf) -> Self { + Self::new(AccessFS::RemoveFile(dir)) + } +} + +pub fn init_module(root_module: &Bound<'_, PyModule>) -> PyResult<()> { + let module = PyModule::new_bound(root_module.py(), "sandbox")?; + module.add_function(wrap_pyfunction!(py_restrict_access, &module)?)?; + module.add_class::()?; + module.add( + "SandboxError", + root_module.py().get_type_bound::(), + )?; + + root_module.add_submodule(&module)?; + root_module + .py() + .import_bound("sys")? + .getattr("modules")? + .set_item("unblob._rust.sandbox", module)?; + + Ok(()) +} diff --git a/rust/src/sandbox/unsupported.rs b/rust/src/sandbox/unsupported.rs new file mode 100644 index 0000000000..2e0d958d83 --- /dev/null +++ b/rust/src/sandbox/unsupported.rs @@ -0,0 +1,5 @@ +use crate::sandbox::{AccessFS, SandboxError}; + +pub fn restrict_access(_access_rules: &[AccessFS]) -> Result<(), SandboxError> { + Err(SandboxError::NotImplemented)? +} diff --git a/tests/rust/__init__.py b/tests/rust/__init__.py new file mode 100644 index 0000000000..9ca4c9c686 --- /dev/null +++ b/tests/rust/__init__.py @@ -0,0 +1,7 @@ +# placeholder, so test files won't share the global module with files in parent director: +# +# > import file mismatch: +# > imported module 'test_sandbox' has this __file__ attribute: +# > /build/source/tests/rust/test_sandbox.py +# > which is not the same as the test file we want to collect: +# > /build/source/tests/test_sandbox.py diff --git a/tests/rust/test_math.py b/tests/rust/test_math.py new file mode 100644 index 0000000000..2c9e586ce5 --- /dev/null +++ b/tests/rust/test_math.py @@ -0,0 +1,39 @@ +import pytest + +from unblob._rust import math_tools + +UNIFORM_DISTRIBUTION = bytes(x for x in range(256)) +NON_UNIFORM_DISTRIBUTION = bytes([0] * 256) + + +@pytest.mark.parametrize( + "data,entropy", + [ + pytest.param(b"", 0, id="empty"), + pytest.param(b"\x00", 0, id="0 bit"), + pytest.param(b"\x01\x01\x00\x00", 1.0, id="1 bit small"), + pytest.param(b"\x01\x01\x00\x00" * 1000, 1.0, id="1 bit large"), + pytest.param(b"\x00\x01\x02\x03", 2.0, id="2 bits"), + ], +) +def test_shannon_entropy(data: bytes, entropy: float): + assert math_tools.shannon_entropy(data) == pytest.approx(entropy) + + +@pytest.mark.parametrize( + "data,chi_square_value", + [ + pytest.param(b"", 0, id="empty"), + pytest.param(UNIFORM_DISTRIBUTION, 1.0, id="uniform distribution"), + pytest.param(NON_UNIFORM_DISTRIBUTION, 0.0, id="non uniform distribution"), + pytest.param( + UNIFORM_DISTRIBUTION + NON_UNIFORM_DISTRIBUTION, + 0.0, + id="partially uniform distribution", + ), + ], +) +def test_chi_square_entropy(data: bytes, chi_square_value: float): + assert math_tools.chi_square_probability(data) == pytest.approx( + chi_square_value, abs=1e-4 + ) diff --git a/tests/rust/test_sandbox.py b/tests/rust/test_sandbox.py new file mode 100644 index 0000000000..f97770f09c --- /dev/null +++ b/tests/rust/test_sandbox.py @@ -0,0 +1,131 @@ +import ctypes +import errno +import os +import platform +import threading +from pathlib import Path + +import pytest + +from unblob._rust.sandbox import AccessFS, SandboxError, restrict_access + +FILE_CONTENT = b"HELLO" + + +@pytest.mark.skipif(platform.system() == "Linux", reason="Linux is supported.") +def test_unsupported_platform(): + with pytest.raises(SandboxError): + restrict_access(AccessFS.read("/")) + + +@pytest.fixture(scope="session") +def sandbox_path(tmp_path_factory: pytest.TempPathFactory) -> Path: + sandbox_path = tmp_path_factory.mktemp("sandbox") + + file_path = sandbox_path / "file.txt" + dir_path = sandbox_path / "dir" + link_path = sandbox_path / "link" + + with file_path.open("wb") as f: + assert f.write(FILE_CONTENT) == len(FILE_CONTENT) + + dir_path.mkdir() + link_path.symlink_to(file_path) + + return sandbox_path + + +# In include/uapi/asm-generic/unistd.h: +# #define __NR_landlock_create_ruleset 444 +__NR_landlock_create_ruleset = 444 +# In include/uapi/linux/landlock.h: +# #define LANDLOCK_CREATE_RULESET_VERSION (1U << 0) +LANDLOCK_CREATE_RULESET_VERSION = 1 << 0 + + +def landlock_supported() -> int: + if platform.system() != "Linux": + return 0 + + # https://docs.kernel.org/userspace-api/landlock.html#creating-a-new-ruleset + libc = ctypes.CDLL(None, use_errno=True) + + max_abi_version = libc.syscall( + __NR_landlock_create_ruleset, + None, + ctypes.c_size_t(0), + ctypes.c_uint32(LANDLOCK_CREATE_RULESET_VERSION), + ) + if max_abi_version > 0: + return max_abi_version + + err = ctypes.get_errno() + if err in ( + errno.EOPNOTSUPP, # disabled at boot time + errno.ENOSYS, # not implememented + ): + return 0 + + raise RuntimeError("landlock_create_ruleset failed", err, os.strerror(err)) + + +@pytest.mark.skipif( + not landlock_supported(), reason="Landlock support is not available on this system" +) +def test_read_sandboxing(request: pytest.FixtureRequest, sandbox_path: Path): # noqa: C901 + exception = None + + def _run_catching_exceptions(fn): + def wrapper(): + __tracebackhide__ = True + nonlocal exception + try: + fn() + except BaseException as exc: + exception = exc + + return wrapper + + # Sandbox applies to the current thread and future threads spawned + # from it. + # + # Running the test on a new thread keeps the main-thread + # clean, so sandboxing won't interfere with other tests executed + # after this one. + + @_run_catching_exceptions + def _run_in_thread(): + restrict_access( + AccessFS.read("/"), + AccessFS.read(sandbox_path), + # allow pytest caching, coverage, etc... + AccessFS.read_write(request.config.rootpath), + ) + + with pytest.raises(PermissionError): + (sandbox_path / "some-dir").mkdir() + + with pytest.raises(PermissionError): + (sandbox_path / "some-file").touch() + + with pytest.raises(PermissionError): + (sandbox_path / "some-link").symlink_to("file.txt") + + for path in sandbox_path.rglob("**/*"): + if path.is_file() or path.is_symlink(): + with path.open("rb") as f: + assert f.read() == FILE_CONTENT + with pytest.raises(PermissionError): + assert path.open("r+") + with pytest.raises(PermissionError): + assert path.unlink() + elif path.is_dir(): + with pytest.raises(PermissionError): + path.rmdir() + + t = threading.Thread(target=_run_in_thread) + t.start() + t.join() + __tracebackhide__ = True + if exception: + raise exception diff --git a/uv.lock b/uv.lock index fd248e521a..af11566931 100644 --- a/uv.lock +++ b/uv.lock @@ -1513,12 +1513,11 @@ dependencies = [ { name = "structlog" }, { name = "treelib" }, { name = "ubi-reader" }, - { name = "unblob-native" }, ] [package.dev-dependencies] dev = [ - { name = "atheris", marker = "python_full_version < '3.12'" }, + { name = "atheris", marker = "python_full_version < '3.12' and sys_platform == 'linux'" }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -1556,12 +1555,11 @@ requires-dist = [ { name = "structlog", specifier = ">=24.1.0" }, { name = "treelib", specifier = ">=1.7.0" }, { name = "ubi-reader", specifier = ">=0.8.9" }, - { name = "unblob-native", specifier = ">=0.1.5" }, ] [package.metadata.requires-dev] dev = [ - { name = "atheris", marker = "python_full_version < '3.12'", specifier = ">=2.3,<3.0" }, + { name = "atheris", marker = "python_full_version < '3.12' and sys_platform == 'linux'", specifier = ">=2.3,<3.0" }, { name = "pre-commit", specifier = ">=3.5,<5.0" }, { name = "pyright", specifier = ">=1.1.349" }, { name = "pytest", specifier = ">=8.0.0" }, @@ -1578,20 +1576,6 @@ docs = [ { name = "pillow", specifier = ">=10.2.0,<12.0" }, ] -[[package]] -name = "unblob-native" -version = "0.1.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/93/b1/29b5b72365e0831cf29d06de619067d48f2468a2f36d27afadb2b7d6cf5f/unblob_native-0.1.5.tar.gz", hash = "sha256:1d092f2836e5ae6a4e8cf1ff88d6a9623b610aaeb33691f36c8efc169121c67e", size = 35186 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/57/eb1c87045ca95d71bd6eaacd551d25f403826e4ba17c662cd21ba4821a4c/unblob_native-0.1.5-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ce1b88c4115848cea243b3787b9d94d0db6c940a846b9fba70d81a3795a5d99c", size = 276023 }, - { url = "https://files.pythonhosted.org/packages/8a/3f/9cb99a7373ac1b46c4cf0826e6108a79a8e602ffaf6395454c26ff6e590c/unblob_native-0.1.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d8313ac34dc5a918855883133b31abe371775f7a591be3eb8a30c2b49b557413", size = 270894 }, - { url = "https://files.pythonhosted.org/packages/5e/12/638830aed0b5406dc0a6fe7c6dcbd94862cd848facf013892819ab0565a8/unblob_native-0.1.5-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:494ce29ef2739b2a809d26ca7f7d304842aac882aa49d1bd473c26906bac4a61", size = 316909 }, - { url = "https://files.pythonhosted.org/packages/46/9a/397011566a8c1224d5e0c3ba6866016a8a19697baf6bb9e0eee5ac6e9449/unblob_native-0.1.5-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed91269171aecb39f423054c1b68d28b034f1be1e5aeab9c5f7c4bb1a9c1630c", size = 319330 }, - { url = "https://files.pythonhosted.org/packages/d8/3c/a1cb249d4f011fa78052dde47566a9214691815a1ab022f77586bd62c331/unblob_native-0.1.5-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8a3f5d11a3e8c28e03d046d22095386352e6252bc9f941bac6a9145ea9c4e4de", size = 351780 }, - { url = "https://files.pythonhosted.org/packages/0e/55/33d8a8925441e36935063ab6d80a0c23984ba9a11169a0bd32ba9e1184d8/unblob_native-0.1.5-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:20f94274822ea7b26ece88fd886fcd6ec98508f492aed12d046e4f84ef3855cb", size = 351350 }, -] - [[package]] name = "urllib3" version = "2.3.0"