From bf23b21e41c169c25d088a97d06d43b5595e02b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Tue, 13 Apr 2021 00:01:44 +0000 Subject: Improve GitHub actions workflow (#41) * MacOS wheels should now support both x86_64 and ARM64. * We now build ARM wheels for Linux. * `abi3` is used to support multiple Python versions in a single wheel. --- .github/workflows/ci.yml | 334 +++++++++++++++++++++++++++++++++-------------- CHANGELOG.md | 2 + Cargo.toml | 7 +- README.md | 2 +- ci/set-target.sh | 40 ------ 5 files changed, 242 insertions(+), 143 deletions(-) delete mode 100755 ci/set-target.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 806059d..dd8d8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,12 +4,7 @@ on: release: types: [created] push: - branches: - - master pull_request: - schedule: - # Runs every Thursday at 20:23 GMT to avoid bit rot - - cron: "20 23 * * 4" jobs: lint: @@ -37,137 +32,282 @@ jobs: command: clippy args: --all-targets --all-features - build: - runs-on: ${{ matrix.os }} + macos: + runs-on: macos-latest needs: lint - strategy: - fail-fast: false - matrix: - rust-toolchain: [1.45, nightly] - python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] - architecture: [x86, x64] - exclude: - # Only build 32-bit for Windows - - os: macos-latest - architecture: x86 - - os: ubuntu-latest - architecture: x86 - # Only build on Rust 1.45 on Linux - - os: macos-latest - rust-toolchain: 1.45 - - os: windows-latest - rust-toolchain: 1.45 - steps: - - name: Checkout - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + python-version: 3.6 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-apple-darwin + profile: minimal + default: true + + - name: Install maturin + run: pip install maturin + + - name: Build wheels - x86_64 + run: | + maturin build -i python --target x86_64-apple-darwin --release --out dist + pip install adblock --no-index --find-links dist --force-reinstall + + - name: Build wheels - universal2 + env: + DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer + MACOSX_DEPLOYMENT_TARGET: "10.9" + PYO3_CROSS_LIB_DIR: /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib + run: | + # Build wheels + maturin build -i python --release --universal2 --out dist --no-sdist + pip install adblock --no-index --find-links dist --force-reinstall - - name: Install dependencies + - name: Run PyTest run: | - python -m pip install --upgrade pip - pip install poetry twine - poetry install + pip install pytest toml + pytest -vv --color=yes - - name: Set Rust target environment variable - run: bash ci/set-target.sh ${{ matrix.os }} ${{ matrix.architecture }} + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Install Rust + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + needs: lint + strategy: + matrix: + platform: + [ + { python-architecture: "x64", target: "x86_64-pc-windows-msvc" }, + { python-architecture: "x86", target: "i686-pc-windows-msvc" }, + ] + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + architecture: ${{ matrix.platform.python-architecture }} + + - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.rust-toolchain }} - target: ${{ env.ACTIONS_RUST_TARGET }} - override: true + toolchain: stable + target: ${{ matrix.platform.target }} + profile: minimal + default: true - - name: Build Python package - run: poetry run maturin build --interpreter python${{matrix.python_version}} --target ${{ env.ACTIONS_RUST_TARGET }} + - name: Install maturin + run: pip install maturin + + - name: Build wheels + run: | + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} + pip install adblock --no-index --find-links dist --force-reinstall + + - name: Run PyTest + run: | + pip install pytest toml + pytest -vv --color=yes - name: Check wheels with Twine - run: twine check target/wheels/* + run: pip install twine && twine check dist/* shell: bash - - name: Install Python package - run: poetry run pip install target/wheels/adblock*.whl + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist + + linux: + runs-on: ubuntu-latest + needs: lint + strategy: + matrix: + platform: + [ + { + toolchain: "1.45", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + { + toolchain: "nightly", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + { + toolchain: "stable", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + # FIXME: GitHub actions runs out of memory when this build is + # attempted. + # { + # toolchain: "stable", + # manylinux: "2010", + # target: "i686-unknown-linux-gnu", + # arch: "i686" + # }, + ] + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Build Wheels + run: | + echo 'curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${{ matrix.platform.toolchain }} + source ~/.cargo/env + export PATH=/opt/python/cp38-cp38/bin:$PATH + pip install maturin + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} + ' > build-wheel.sh + chmod +x build-wheel.sh + docker run --rm -v "$PWD":/io -w /io quay.io/pypa/manylinux${{ matrix.platform.manylinux }}_${{ matrix.platform.arch }} bash build-wheel.sh + + - name: Run PyTest + run: | + pip install adblock --no-index --find-links dist --force-reinstall + pip install pytest toml + pytest -vv --color=yes + + - name: Auditwheel Symbols + run: | + pip install auditwheel-symbols + auditwheel-symbols dist/*.whl + + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Run Python tests - run: poetry run pytest -vv --color=yes + - name: Upload wheels + if: matrix.platform.toolchain == 'stable' + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist - python-publish: - needs: build - runs-on: ${{ matrix.os }} + linux-cross: + runs-on: ubuntu-latest + needs: lint strategy: - fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] - architecture: [x86, x64] - exclude: - # Only build 32-bit for Windows - - os: macos-latest - architecture: x86 - - os: ubuntu-latest - architecture: x86 - + platform: + [ + { + manylinux: "2014", + target: "aarch64-unknown-linux-gnu", + arch: "aarch64", + }, + { + manylinux: "2014", + target: "armv7-unknown-linux-gnueabihf", + arch: "armv7", + }, + ] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + python-version: 3.6 - - name: Set Rust target environment variable - run: bash ci/set-target.sh ${{ matrix.os }} ${{ matrix.architecture }} - shell: bash + - name: Build Wheels + run: | + echo 'curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source ~/.cargo/env + rustup target add ${{ matrix.platform.target }} + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} + ' > build-wheel.sh + chmod +x build-wheel.sh + docker run --rm -v "$PWD":/io -w /io messense/manylinux2014-cross:${{ matrix.platform.arch }} bash build-wheel.sh - - name: Install latest nightly - uses: actions-rs/toolchain@v1 + - uses: uraimo/run-on-arch-action@v2.0.5 + name: Install built wheel with: - toolchain: nightly - target: ${{ env.ACTIONS_RUST_TARGET }} - override: true + arch: ${{ matrix.platform.arch }} + distro: ubuntu18.04 + # Mount the dist directory as /artifacts in the container + dockerRunArgs: | + --volume "${PWD}/dist:/artifacts" + install: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install -U pip + run: | + ls -lrth /artifacts + pip3 install adblock --no-index --find-links /artifacts --force-reinstall + cd ~ && python3 -c "import adblock" - - name: Install dependencies + - name: Auditwheel Symbols run: | - python -m pip install --upgrade pip - pip install poetry twine - poetry install - - name: Build Python package - run: poetry run maturin build --release --strip --interpreter python${{matrix.python_version}} --target ${{ env.ACTIONS_RUST_TARGET }} - - - name: List wheels - if: matrix.os == 'windows-latest' - run: dir target\wheels\ - - - name: List wheels - if: matrix.os != 'windows-latest' - run: find ./target/wheels/ - - - name: Install wheels - run: pip install target/wheels/adblock*.whl + pip install auditwheel-symbols + auditwheel-symbols dist/*.whl + + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') + - name: Upload wheels + uses: actions/upload-artifact@v2 with: - files: target/wheels/adblock*.whl - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: wheels + path: dist + + python-publish: + runs-on: ubuntu-latest + needs: [ macos, windows, linux, linux-cross ] + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Wheel filename sanity checks + run: | + num_abi3_whl=$(find | grep "\./adblock.*-abi3.*\.whl" | wc -l) + num_whl=$(find | grep "\./adblock.*\.whl" | wc -l) + test $num_abi3_whl -eq $num_whl + test $num_whl -ge 1 + find | grep "\./adblock-.*\.tar\.gz" - name: PyPi publish if: github.event_name == 'release' && github.event.action == 'created' env: TWINE_PASSWORD: ${{ secrets.PYPI }} - run: twine upload --non-interactive --skip-existing --username __token__ target/wheels/* - shell: bash + run: | + pip install --upgrade wheel pip setuptools twine + twine upload --non-interactive --skip-existing --username __token__ ./* + + - name: GitHub release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: ./* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} docs-publish: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d6716..3cd6a52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ## Unreleased --- +### Changes +* PyO3 is now configured to use [`abi3`](https://pyo3.rs/v0.13.2/building_and_distribution.html#py_limited_apiabi3). ## 0.4.3 - (2021-03-20) --- diff --git a/Cargo.toml b/Cargo.toml index 1a0fa77..caa62b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,11 +23,8 @@ debug = true [dependencies] adblock = { version = "=0.3.10", default-features = false, features = ["full-regex-handling", "embedded-domain-resolver"] } -pyo3 = "0.13" +pyo3 = { version = "0.13", features = ["abi3-py36", "extension-module"] } [lib] name = "adblock" -crate-type = ["rlib", "cdylib"] - -[features] -default = ["pyo3/extension-module"] +crate-type = ["rlib", "cdylib"] \ No newline at end of file diff --git a/README.md b/README.md index f5278e9..5889e57 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ maturin build --release |------------------|----------|------------|-----| | Python | `>=3.6` | `python3` | - | | Rust | `>=1.45` | `rust` | - | -| Maturin | `*` | `maturin` | https://github.com/PyO3/maturin | +| Maturin | `>=0.10` | `maturin` | https://github.com/PyO3/maturin | ### Developing diff --git a/ci/set-target.sh b/ci/set-target.sh deleted file mode 100755 index f0dfbf0..0000000 --- a/ci/set-target.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 OS ARCHITECTURE" >&2 - exit 1 -fi - -os=$1 -architecture=$2 - -output() { - echo "Setting ACTIONS_RUST_TARGET=$1" - echo "ACTIONS_RUST_TARGET=$1" >> "$GITHUB_ENV" - exit 0 -} - -if [ "$architecture" = "x64" ]; then - if [ "$os" = "ubuntu-latest" ]; then - output "x86_64-unknown-linux-gnu" - elif [ "$os" = "macos-latest" ]; then - output "x86_64-apple-darwin" - elif [ "$os" = "windows-latest" ]; then - output "x86_64-pc-windows-msvc" - else - echo "Unknown 64-bit OS: $os" - exit 1 - fi -elif [ "$architecture" = "x86" ]; then - if [ "$os" = "ubuntu-latest" ]; then - output "i686-unknown-linux-gnu" - elif [ "$os" = "windows-latest" ]; then - output "i686-pc-windows-msvc" - else - echo "Unknown 32-bit OS: $os" - exit 1 - fi -else - echo "Bad architecture: $architecture" - exit 1 -fi -- cgit v1.2.3