From bf23b21e41c169c25d088a97d06d43b5595e02b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20Dagur?= Date: Tue, 13 Apr 2021 00:01:44 +0000 Subject: Improve GitHub actions workflow (#41) * MacOS wheels should now support both x86_64 and ARM64. * We now build ARM wheels for Linux. * `abi3` is used to support multiple Python versions in a single wheel. --- .github/workflows/ci.yml | 334 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 237 insertions(+), 97 deletions(-) (limited to '.github') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 806059d..dd8d8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,12 +4,7 @@ on: release: types: [created] push: - branches: - - master pull_request: - schedule: - # Runs every Thursday at 20:23 GMT to avoid bit rot - - cron: "20 23 * * 4" jobs: lint: @@ -37,137 +32,282 @@ jobs: command: clippy args: --all-targets --all-features - build: - runs-on: ${{ matrix.os }} + macos: + runs-on: macos-latest needs: lint - strategy: - fail-fast: false - matrix: - rust-toolchain: [1.45, nightly] - python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] - architecture: [x86, x64] - exclude: - # Only build 32-bit for Windows - - os: macos-latest - architecture: x86 - - os: ubuntu-latest - architecture: x86 - # Only build on Rust 1.45 on Linux - - os: macos-latest - rust-toolchain: 1.45 - - os: windows-latest - rust-toolchain: 1.45 - steps: - - name: Checkout - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + python-version: 3.6 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-apple-darwin + profile: minimal + default: true + + - name: Install maturin + run: pip install maturin + + - name: Build wheels - x86_64 + run: | + maturin build -i python --target x86_64-apple-darwin --release --out dist + pip install adblock --no-index --find-links dist --force-reinstall + + - name: Build wheels - universal2 + env: + DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer + MACOSX_DEPLOYMENT_TARGET: "10.9" + PYO3_CROSS_LIB_DIR: /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib + run: | + # Build wheels + maturin build -i python --release --universal2 --out dist --no-sdist + pip install adblock --no-index --find-links dist --force-reinstall - - name: Install dependencies + - name: Run PyTest run: | - python -m pip install --upgrade pip - pip install poetry twine - poetry install + pip install pytest toml + pytest -vv --color=yes - - name: Set Rust target environment variable - run: bash ci/set-target.sh ${{ matrix.os }} ${{ matrix.architecture }} + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Install Rust + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + needs: lint + strategy: + matrix: + platform: + [ + { python-architecture: "x64", target: "x86_64-pc-windows-msvc" }, + { python-architecture: "x86", target: "i686-pc-windows-msvc" }, + ] + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + architecture: ${{ matrix.platform.python-architecture }} + + - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.rust-toolchain }} - target: ${{ env.ACTIONS_RUST_TARGET }} - override: true + toolchain: stable + target: ${{ matrix.platform.target }} + profile: minimal + default: true - - name: Build Python package - run: poetry run maturin build --interpreter python${{matrix.python_version}} --target ${{ env.ACTIONS_RUST_TARGET }} + - name: Install maturin + run: pip install maturin + + - name: Build wheels + run: | + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} + pip install adblock --no-index --find-links dist --force-reinstall + + - name: Run PyTest + run: | + pip install pytest toml + pytest -vv --color=yes - name: Check wheels with Twine - run: twine check target/wheels/* + run: pip install twine && twine check dist/* shell: bash - - name: Install Python package - run: poetry run pip install target/wheels/adblock*.whl + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist + + linux: + runs-on: ubuntu-latest + needs: lint + strategy: + matrix: + platform: + [ + { + toolchain: "1.45", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + { + toolchain: "nightly", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + { + toolchain: "stable", + manylinux: "2010", + target: "x86_64-unknown-linux-gnu", + arch: "x86_64", + }, + # FIXME: GitHub actions runs out of memory when this build is + # attempted. + # { + # toolchain: "stable", + # manylinux: "2010", + # target: "i686-unknown-linux-gnu", + # arch: "i686" + # }, + ] + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Build Wheels + run: | + echo 'curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${{ matrix.platform.toolchain }} + source ~/.cargo/env + export PATH=/opt/python/cp38-cp38/bin:$PATH + pip install maturin + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} + ' > build-wheel.sh + chmod +x build-wheel.sh + docker run --rm -v "$PWD":/io -w /io quay.io/pypa/manylinux${{ matrix.platform.manylinux }}_${{ matrix.platform.arch }} bash build-wheel.sh + + - name: Run PyTest + run: | + pip install adblock --no-index --find-links dist --force-reinstall + pip install pytest toml + pytest -vv --color=yes + + - name: Auditwheel Symbols + run: | + pip install auditwheel-symbols + auditwheel-symbols dist/*.whl + + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Run Python tests - run: poetry run pytest -vv --color=yes + - name: Upload wheels + if: matrix.platform.toolchain == 'stable' + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist - python-publish: - needs: build - runs-on: ${{ matrix.os }} + linux-cross: + runs-on: ubuntu-latest + needs: lint strategy: - fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] - architecture: [x86, x64] - exclude: - # Only build 32-bit for Windows - - os: macos-latest - architecture: x86 - - os: ubuntu-latest - architecture: x86 - + platform: + [ + { + manylinux: "2014", + target: "aarch64-unknown-linux-gnu", + arch: "aarch64", + }, + { + manylinux: "2014", + target: "armv7-unknown-linux-gnueabihf", + arch: "armv7", + }, + ] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + python-version: 3.6 - - name: Set Rust target environment variable - run: bash ci/set-target.sh ${{ matrix.os }} ${{ matrix.architecture }} - shell: bash + - name: Build Wheels + run: | + echo 'curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source ~/.cargo/env + rustup target add ${{ matrix.platform.target }} + maturin build -i python --release --out dist --no-sdist --target ${{ matrix.platform.target }} --manylinux ${{ matrix.platform.manylinux }} + ' > build-wheel.sh + chmod +x build-wheel.sh + docker run --rm -v "$PWD":/io -w /io messense/manylinux2014-cross:${{ matrix.platform.arch }} bash build-wheel.sh - - name: Install latest nightly - uses: actions-rs/toolchain@v1 + - uses: uraimo/run-on-arch-action@v2.0.5 + name: Install built wheel with: - toolchain: nightly - target: ${{ env.ACTIONS_RUST_TARGET }} - override: true + arch: ${{ matrix.platform.arch }} + distro: ubuntu18.04 + # Mount the dist directory as /artifacts in the container + dockerRunArgs: | + --volume "${PWD}/dist:/artifacts" + install: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install -U pip + run: | + ls -lrth /artifacts + pip3 install adblock --no-index --find-links /artifacts --force-reinstall + cd ~ && python3 -c "import adblock" - - name: Install dependencies + - name: Auditwheel Symbols run: | - python -m pip install --upgrade pip - pip install poetry twine - poetry install - - name: Build Python package - run: poetry run maturin build --release --strip --interpreter python${{matrix.python_version}} --target ${{ env.ACTIONS_RUST_TARGET }} - - - name: List wheels - if: matrix.os == 'windows-latest' - run: dir target\wheels\ - - - name: List wheels - if: matrix.os != 'windows-latest' - run: find ./target/wheels/ - - - name: Install wheels - run: pip install target/wheels/adblock*.whl + pip install auditwheel-symbols + auditwheel-symbols dist/*.whl + + - name: Check wheels with Twine + run: pip install twine && twine check dist/* shell: bash - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') + - name: Upload wheels + uses: actions/upload-artifact@v2 with: - files: target/wheels/adblock*.whl - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: wheels + path: dist + + python-publish: + runs-on: ubuntu-latest + needs: [ macos, windows, linux, linux-cross ] + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Wheel filename sanity checks + run: | + num_abi3_whl=$(find | grep "\./adblock.*-abi3.*\.whl" | wc -l) + num_whl=$(find | grep "\./adblock.*\.whl" | wc -l) + test $num_abi3_whl -eq $num_whl + test $num_whl -ge 1 + find | grep "\./adblock-.*\.tar\.gz" - name: PyPi publish if: github.event_name == 'release' && github.event.action == 'created' env: TWINE_PASSWORD: ${{ secrets.PYPI }} - run: twine upload --non-interactive --skip-existing --username __token__ target/wheels/* - shell: bash + run: | + pip install --upgrade wheel pip setuptools twine + twine upload --non-interactive --skip-existing --username __token__ ./* + + - name: GitHub release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: ./* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} docs-publish: runs-on: ubuntu-latest -- cgit v1.2.3