diff --git a/.github/workflows/linux-wheels.yaml b/.github/workflows/linux-wheels.yaml new file mode 100644 index 00000000..33305686 --- /dev/null +++ b/.github/workflows/linux-wheels.yaml @@ -0,0 +1,95 @@ +# Build Linux wheels using Docker (requires building LLVM from source) +# This is separate from the main wheels.yaml because it takes much longer +# +# Publishing is handled by release.yaml which waits for all build workflows. + +name: Build Linux Wheels + +on: + # Build on release tags (same as wheels.yaml) + push: + tags: + - 'v*' + # Manual trigger for testing (building LLVM takes ~30-60 minutes) + workflow_dispatch: + inputs: + arch: + description: 'Architecture to build' + required: true + default: 'both' + type: choice + options: + - x86_64 + - aarch64 + - both + +jobs: + build-x86_64: + name: Build Linux x86_64 wheels + runs-on: ubuntu-latest + # Build on tag pushes, or manual trigger with x86_64/both selected + if: github.event_name == 'push' || inputs.arch == 'x86_64' || inputs.arch == 'both' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + run: | + docker build -t atheris-builder-x86_64 -f deployment/Dockerfile deployment/ + + - name: Build wheels + run: | + docker run --rm \ + -v ${{ github.workspace }}:/atheris \ + atheris-builder-x86_64 + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: linux-x86_64-wheels + path: dist/*.whl + + build-aarch64: + name: Build Linux aarch64 wheels + runs-on: ubuntu-latest + # Build on tag pushes, or manual trigger with aarch64/both selected + if: github.event_name == 'push' || inputs.arch == 'aarch64' || inputs.arch == 'both' + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image (this takes a while - building LLVM on QEMU) + run: | + docker buildx build \ + --platform linux/arm64 \ + -t atheris-builder-aarch64 \ + -f deployment/Dockerfile.aarch64 \ + --load \ + deployment/ + timeout-minutes: 180 # Building LLVM on QEMU is slow + + - name: Build wheels + run: | + docker run --rm \ + --platform linux/arm64 \ + -v ${{ github.workspace }}:/atheris \ + atheris-builder-aarch64 + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: linux-aarch64-wheels + path: dist/*.whl + + # Linux wheels are published by the unified release.yaml workflow diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..9db0c997 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,118 @@ +# Unified release workflow that publishes all wheels to PyPI +# +# This workflow waits for both build workflows to complete before publishing, +# ensuring atomic releases with all artifacts (macOS + Linux + sdist). +# +# Triggered by tag pushes - waits for wheels.yaml and linux-wheels.yaml to finish. + +name: Release to PyPI + +on: + push: + tags: + - 'v*' + +jobs: + wait-for-builds: + name: Wait for build workflows + runs-on: ubuntu-latest + # aarch64 build can take up to 180 min; add buffer for all builds + timeout-minutes: 240 + permissions: + checks: read # Required by lewagon/wait-on-check-action + steps: + - name: Wait for macOS wheels workflow + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: 'Build wheels on macos-latest' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 30 + + - name: Wait for Linux x86_64 wheels workflow + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: 'Build Linux x86_64 wheels' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 60 + + - name: Wait for Linux aarch64 wheels workflow + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: 'Build Linux aarch64 wheels' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 60 + + - name: Wait for sdist build + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: 'Build source distribution' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 30 + + publish: + name: Publish to PyPI + needs: wait-for-builds + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/atheris + permissions: + id-token: write # Required for trusted publishing + actions: read # Required to download artifacts + + steps: + - name: Get workflow run IDs + id: get-runs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Get the most recent successful run of each workflow for this commit + # Note: --commit filters by headSha, works correctly for tag pushes + MACOS_RUN=$(gh run list --repo ${{ github.repository }} \ + --workflow "Build macOS Wheels" \ + --commit ${{ github.sha }} \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + echo "macos_run=$MACOS_RUN" >> $GITHUB_OUTPUT + + LINUX_RUN=$(gh run list --repo ${{ github.repository }} \ + --workflow "Build Linux Wheels" \ + --commit ${{ github.sha }} \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + echo "linux_run=$LINUX_RUN" >> $GITHUB_OUTPUT + + - name: Download macOS wheels + uses: actions/download-artifact@v4 + with: + run-id: ${{ steps.get-runs.outputs.macos_run }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: dist + merge-multiple: true + + - name: Download Linux wheels + uses: actions/download-artifact@v4 + with: + run-id: ${{ steps.get-runs.outputs.linux_run }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: dist + merge-multiple: true + + - name: List artifacts + run: ls -la dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # Skip files already uploaded (allows retrying failed releases) + skip-existing: true + # Uses trusted publishing - no API token needed + # Configure at: https://pypi.org/manage/project/atheris/settings/publishing/ diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml new file mode 100644 index 00000000..469ac754 --- /dev/null +++ b/.github/workflows/wheels.yaml @@ -0,0 +1,87 @@ +# Build macOS ARM64 wheels and source distribution +# Addresses: https://github.com/google/atheris/issues/52 +# +# NOTE: This workflow builds macOS ARM64 only. Linux wheels require building +# LLVM from source - see linux-wheels.yaml for Docker-based Linux builds. +# +# KNOWN LIMITATION: macOS wheels currently require Homebrew LLVM at runtime +# (brew install llvm) due to libc++ dependency. See TODO for future fix. +# +# Publishing is handled by release.yaml which waits for all build workflows. + +name: Build macOS Wheels + +on: + # Build on release tags (release.yaml handles publishing) + push: + tags: + - 'v*' + # Allow manual trigger for testing + workflow_dispatch: + # Build on PRs that modify build-related files (for testing) + pull_request: + paths: + - 'setup.py' + - 'pyproject.toml' + - '.github/workflows/wheels.yaml' + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # macOS ARM64 (native on macos-latest which is ARM) + - os: macos-latest + arch: arm64 + # Note: Linux wheels require building LLVM from source (see deployment/Dockerfile) + # Note: macOS x86_64 cross-compilation doesn't work well - skip for now + + steps: + - uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21 + env: + # Only build for the current architecture + CIBW_ARCHS: ${{ matrix.arch }} + + # macOS: Set up Homebrew LLVM environment + # Intel Macs use /usr/local, ARM Macs use /opt/homebrew + # Must link against Homebrew's libc++ to avoid symbol mismatch with system libc++ + CIBW_ENVIRONMENT_MACOS: > + CLANG_BIN="$(brew --prefix llvm)/bin/clang" + CC="$(brew --prefix llvm)/bin/clang" + CXX="$(brew --prefix llvm)/bin/clang++" + LDFLAGS="-L$(brew --prefix llvm)/lib/c++ -L$(brew --prefix llvm)/lib -Wl,-rpath,$(brew --prefix llvm)/lib/c++" + CPPFLAGS="-I$(brew --prefix llvm)/include" + + # macOS: Skip delocate - it fails on .a static library files + # TODO: The wheel currently requires Homebrew LLVM at runtime + # Future work: vendor libc++ or use static linking + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + run: pipx run build --sdist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + # Publishing is handled by release.yaml diff --git a/README.md b/README.md index 11235353..358eea91 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,14 @@ If you don't have `clang` installed or it's too old, you'll need to download and Apple Clang doesn't come with libFuzzer, so you'll need to install a new version of LLVM from head. Follow the instructions in Installing Against New LLVM below. +**Note for macOS ARM64 (Apple Silicon) wheels:** The prebuilt wheels on PyPI for macOS ARM64 require Homebrew LLVM to be installed at runtime due to libc++ dependencies: + +```bash +brew install llvm +``` + +This requirement exists because the wheels link against Homebrew's libc++ library. Future versions may bundle this dependency to make the wheels fully self-contained. + #### Installing Against New LLVM ```bash diff --git a/deployment/Dockerfile.aarch64 b/deployment/Dockerfile.aarch64 new file mode 100644 index 00000000..e34d9993 --- /dev/null +++ b/deployment/Dockerfile.aarch64 @@ -0,0 +1,53 @@ +# Copyright 2020 Google LLC +# Copyright 2026 RMC Infosec +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM quay.io/pypa/manylinux2014_aarch64 + +# Clang needs to be able to find python3 +RUN set -e -x -v; \ + ln -s /opt/python/cp38-cp38/bin/python /usr/bin/python3 + +RUN set -e -x -v; \ + yum install -y cmake; + +RUN set -e -x -v; \ + cd /root; \ + git clone https://github.com/llvm/llvm-project.git; + +RUN set -e -x -v; \ + cd /root/llvm-project; \ + git checkout 0982db188b661d6744b06244fda64d43dd80206e; \ + cmake -S llvm -B build -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"; + +RUN set -e -x -v; \ + cd /root/llvm-project/build; \ + make -j$(nproc) compiler-rt; + +RUN set -e -x -v; \ + python3 -m pip install auditwheel; + +RUN set -e -x -v; \ + /opt/python/cp311-cp311/bin/python3 -m pip install setuptools && \ + /opt/python/cp312-cp312/bin/python3 -m pip install setuptools && \ + /opt/python/cp313-cp313/bin/python3 -m pip install setuptools; + +WORKDIR /atheris + +CMD export LIBFUZZER_LIB="/root/llvm-project/build/lib/clang/$(ls /root/llvm-project/build/lib/clang/)/lib/aarch64-unknown-linux-gnu/libclang_rt.fuzzer_no_main.a"; \ + /opt/python/cp311-cp311/bin/python3 setup.py bdist_wheel -d /tmp/dist && \ + /opt/python/cp312-cp312/bin/python3 setup.py bdist_wheel -d /tmp/dist && \ + /opt/python/cp313-cp313/bin/python3 setup.py bdist_wheel -d /tmp/dist && \ + ( cd /tmp/dist && find /tmp/dist/* | xargs -I{} auditwheel repair --plat manylinux2014_aarch64 {} ) && \ + mkdir -p /atheris/dist && cp /tmp/dist/wheelhouse/* /atheris/dist/ diff --git a/pyproject.toml b/pyproject.toml index 5f4f3fb3..a09428fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,5 +14,34 @@ exclude = [ exclude = ["src/benchmark/"] ignore-init-module-imports = true # E501 Line too long -# F401 … imported but unused; consider adding to `__all__` or using a redundant alias +# F401 ... imported but unused; consider adding to `__all__` or using a redundant alias ignore = ["E501", "F401"] + + +[build-system] +requires = ["setuptools>=45", "wheel", "pybind11>=2.5.0"] +build-backend = "setuptools.build_meta" + + +[tool.cibuildwheel] +# Build for CPython only, skip PyPy and musllinux +skip = "pp* *-musllinux*" + +# Test that the built wheel can be imported (atheris doesn't have __version__) +test-command = "python -c \"import atheris; print('atheris loaded, FuzzedDataProvider:', hasattr(atheris, 'FuzzedDataProvider'))\"" + +# Build for Python 3.11, 3.12, 3.13 +build = "cp311-* cp312-* cp313-*" + +# Linux builds use Docker (see linux-wheels.yaml) because manylinux +# containers don't have libFuzzer pre-installed and building LLVM is slow. + +[tool.cibuildwheel.macos] +# Build for Apple Silicon only (x86_64 cross-compilation has issues) +archs = ["arm64"] + +# Install LLVM from Homebrew (Apple Clang doesn't include libFuzzer) +before-all = "brew install llvm" + +# Environment variables are set via CIBW_ENVIRONMENT_MACOS in the workflow +# This is needed because $(brew --prefix) differs between Intel and ARM Macs