diff --git a/.cross/Dockerfile.aarch64-unknown-linux-gnu b/.cross/Dockerfile.aarch64-unknown-linux-gnu new file mode 100644 index 0000000..b5b59c9 --- /dev/null +++ b/.cross/Dockerfile.aarch64-unknown-linux-gnu @@ -0,0 +1,50 @@ +FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + gcc-10-aarch64-linux-gnu \ + g++-10-aarch64-linux-gnu \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Set GCC 10 as default for Debian-style compiler names +RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ + update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ + update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ + update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 + +# Also handle Rust triplet names (aarch64-unknown-linux-gnu-g++) if present +RUN if [ -f /usr/bin/aarch64-unknown-linux-gnu-g++ ]; then \ + mv /usr/bin/aarch64-unknown-linux-gnu-g++ /usr/bin/aarch64-unknown-linux-gnu-g++.old; \ + fi && \ + if [ -f /usr/bin/aarch64-unknown-linux-gnu-gcc ]; then \ + mv /usr/bin/aarch64-unknown-linux-gnu-gcc /usr/bin/aarch64-unknown-linux-gnu-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-unknown-linux-gnu-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-unknown-linux-gnu-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "aarch64-unknown-linux-gnu-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "aarch64-unknown-linux-gnu-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 "$f"; \ + done || true + +# Verify GCC version (should show GCC 10) +RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.aarch64-unknown-linux-musl b/.cross/Dockerfile.aarch64-unknown-linux-musl new file mode 100644 index 0000000..4f05c19 --- /dev/null +++ b/.cross/Dockerfile.aarch64-unknown-linux-musl @@ -0,0 +1,66 @@ +FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + gcc-10-aarch64-linux-gnu \ + g++-10-aarch64-linux-gnu \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Set GCC 10 as default host compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 + +# Set GCC 10 as default aarch64 cross-compiler (Debian-style names) +RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ + update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ + update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ + update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 + +# Handle musl compiler names - both Debian-style and Rust triplet style +RUN if [ -f /usr/bin/aarch64-linux-musl-g++ ]; then \ + mv /usr/bin/aarch64-linux-musl-g++ /usr/bin/aarch64-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/aarch64-linux-musl-gcc ]; then \ + mv /usr/bin/aarch64-linux-musl-gcc /usr/bin/aarch64-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-linux-musl-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-linux-musl-gcc + +# Handle Rust triplet names +RUN if [ -f /usr/bin/aarch64-unknown-linux-musl-g++ ]; then \ + mv /usr/bin/aarch64-unknown-linux-musl-g++ /usr/bin/aarch64-unknown-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/aarch64-unknown-linux-musl-gcc ]; then \ + mv /usr/bin/aarch64-unknown-linux-musl-gcc /usr/bin/aarch64-unknown-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-unknown-linux-musl-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-unknown-linux-musl-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "aarch64-unknown-linux-musl-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "aarch64-unknown-linux-musl-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 "$f"; \ + done || true + +# Verify GCC version +RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.arm-unknown-linux-gnueabihf b/.cross/Dockerfile.arm-unknown-linux-gnueabihf new file mode 100644 index 0000000..88a5f54 --- /dev/null +++ b/.cross/Dockerfile.arm-unknown-linux-gnueabihf @@ -0,0 +1,53 @@ +FROM ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + gcc-10-arm-linux-gnueabihf \ + g++-10-arm-linux-gnueabihf \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Set GCC 10 as default for Debian-style compiler names +RUN update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 100 && \ + update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 100 && \ + update-alternatives --set arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 && \ + update-alternatives --set arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 + +# The cross-rs image for arm uses Rust triplet names (arm-unknown-linux-gnueabihf-g++) +# which come from crosstool-ng. We need to replace those with GCC 10 wrappers. +# First, backup the old compilers, then create symlinks to GCC 10 +RUN if [ -f /usr/bin/arm-unknown-linux-gnueabihf-g++ ]; then \ + mv /usr/bin/arm-unknown-linux-gnueabihf-g++ /usr/bin/arm-unknown-linux-gnueabihf-g++.old; \ + fi && \ + if [ -f /usr/bin/arm-unknown-linux-gnueabihf-gcc ]; then \ + mv /usr/bin/arm-unknown-linux-gnueabihf-gcc /usr/bin/arm-unknown-linux-gnueabihf-gcc.old; \ + fi && \ + ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-unknown-linux-gnueabihf-g++ && \ + ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-unknown-linux-gnueabihf-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "arm-unknown-linux-gnueabihf-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "arm-unknown-linux-gnueabihf-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 "$f"; \ + done || true + +# Verify GCC version (should show GCC 10) +RUN arm-linux-gnueabihf-g++ --version | head -1 +RUN which arm-unknown-linux-gnueabihf-g++ && arm-unknown-linux-gnueabihf-g++ --version | head -1 || echo "arm-unknown-linux-gnueabihf-g++ not in PATH" diff --git a/.cross/Dockerfile.x86_64-unknown-linux-musl b/.cross/Dockerfile.x86_64-unknown-linux-musl new file mode 100644 index 0000000..6663c72 --- /dev/null +++ b/.cross/Dockerfile.x86_64-unknown-linux-musl @@ -0,0 +1,58 @@ +FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Set GCC 10 as default host compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 + +# Handle musl compiler names - both Debian-style and Rust triplet style +RUN if [ -f /usr/bin/x86_64-linux-musl-g++ ]; then \ + mv /usr/bin/x86_64-linux-musl-g++ /usr/bin/x86_64-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/x86_64-linux-musl-gcc ]; then \ + mv /usr/bin/x86_64-linux-musl-gcc /usr/bin/x86_64-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/g++-10 /usr/bin/x86_64-linux-musl-g++ && \ + ln -sf /usr/bin/gcc-10 /usr/bin/x86_64-linux-musl-gcc + +# Handle Rust triplet names +RUN if [ -f /usr/bin/x86_64-unknown-linux-musl-g++ ]; then \ + mv /usr/bin/x86_64-unknown-linux-musl-g++ /usr/bin/x86_64-unknown-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/x86_64-unknown-linux-musl-gcc ]; then \ + mv /usr/bin/x86_64-unknown-linux-musl-gcc /usr/bin/x86_64-unknown-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/g++-10 /usr/bin/x86_64-unknown-linux-musl-g++ && \ + ln -sf /usr/bin/gcc-10 /usr/bin/x86_64-unknown-linux-musl-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "x86_64-unknown-linux-musl-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/g++-10 "$f"; \ + done || true && \ + find /x-tools -name "x86_64-unknown-linux-musl-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/gcc-10 "$f"; \ + done || true + +# Verify GCC version +RUN g++ --version | head -1 diff --git a/.github/workflows/build-cross-images.yml b/.github/workflows/build-cross-images.yml new file mode 100644 index 0000000..4dafa75 --- /dev/null +++ b/.github/workflows/build-cross-images.yml @@ -0,0 +1,73 @@ +name: Build Cross-Compilation Images + +on: + push: + paths: + - '.cross/Dockerfile.*' + - '.github/workflows/build-cross-images.yml' + branches: + - main + - 'fix/**' + workflow_dispatch: + +concurrency: + group: build-cross-images-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }}/cozodb-cross + +jobs: + build-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-unknown-linux-gnu + dockerfile: .cross/Dockerfile.aarch64-unknown-linux-gnu + - target: arm-unknown-linux-gnueabihf + dockerfile: .cross/Dockerfile.arm-unknown-linux-gnueabihf + - target: aarch64-unknown-linux-musl + dockerfile: .cross/Dockerfile.aarch64-unknown-linux-musl + - target: x86_64-unknown-linux-musl + dockerfile: .cross/Dockerfile.x86_64-unknown-linux-musl + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.target }} + tags: | + type=raw,value=latest + type=sha,prefix= + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1abd0ab..bb35484 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,64 +3,89 @@ name: Build precompiled NIFs push: branches: - main + - 'fix/**' tags: - - '*' + # Matches v0.3.2, v1.0.0, etc. + - 'v*' workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_release: name: 'NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }})' runs-on: '${{ matrix.job.os }}' + permissions: + contents: write strategy: fail-fast: false matrix: nif: - '2.17' job: - - target: arm-unknown-linux-gnueabihf - os: ubuntu-24.04 - use-cross: true - - target: aarch64-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true - - target: aarch64-unknown-linux-musl - os: ubuntu-24.04 - use-cross: true + # macOS targets (native builds) - target: aarch64-apple-darwin os: macos-latest - - target: riscv64gc-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true - target: x86_64-apple-darwin os: macos-latest + # Linux x86_64 native build - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 - - target: x86_64-unknown-linux-musl - os: ubuntu-24.04 - use-cross: true + # Linux ARM64 native build on ARM runner + - target: aarch64-unknown-linux-gnu + os: ubuntu-24.04-arm + # Windows x86_64 native build (no jemalloc - not available on Windows) + - target: x86_64-pc-windows-msvc + os: windows-latest + cargo-args: --no-default-features --features windows-default steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Extract project version and branch/tag + - name: Extract project version shell: bash run: | # Get the project version from cozodb.app.src - echo "PROJECT_VERSION=$(sed -n 's/.*{vsn, "\([^"]*\)".*/\1/p' - src.cozodb.app.src | head -n1)" >> $GITHUB_ENV - - # Determine if it's a tag or a branch - if [[ "$GITHUB_REF" == refs/tags/* ]]; then - BRANCH_OR_TAG="${GITHUB_REF#refs/tags/}" - else - BRANCH_OR_TAG="${GITHUB_REF#refs/heads/}" - fi - echo "BRANCH_OR_TAG=$BRANCH_OR_TAG" >> $GITHUB_ENV - - name: Install build dependencies - run: sudo apt-get update && sudo apt-get install -y build-essential clang libclang-dev pkg-config + echo "PROJECT_VERSION=$(sed -n 's/.*{vsn, "\([^"]*\)".*/\1/p' src/cozodb.app.src | head -n1)" >> $GITHUB_ENV + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y build-essential clang libclang-dev pkg-config liburing-dev + # RocksDB requires C++20 + echo "CXXFLAGS=-std=c++2a" >> $GITHUB_ENV + - name: Install build dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm pkg-config + - name: Install build dependencies (Windows) + if: runner.os == 'Windows' + run: | + choco install llvm -y + # Set LIBCLANG_PATH for bindgen + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV + # Force C++20 for MSVC (RocksDB requires it) + echo "CXXFLAGS=/std:c++20" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: toolchain: stable target: '${{ matrix.job.target }}' + - name: Install cross (for cross-compilation) + if: matrix.job.use-cross == true + run: | + cargo install cross --git https://github.com/cross-rs/cross + - name: Set cross environment + if: matrix.job.use-cross == true + run: | + # Point cross to our config file + echo "CROSS_CONFIG=${{ github.workspace }}/native/cozodb/Cross.toml" >> $GITHUB_ENV + # RocksDB requires C++20 + echo "CXXFLAGS=-std=c++2a" >> $GITHUB_ENV + # Debug: show Cross.toml content + echo "Cross.toml content:" + cat ${{ github.workspace }}/native/cozodb/Cross.toml - name: Build the project id: build-crate uses: philss/rustler-precompiled-action@v1.1.4 @@ -71,21 +96,17 @@ jobs: nif-version: '${{ matrix.nif }}' use-cross: '${{ matrix.job.use-cross }}' project-dir: native/cozodb - - name: Rename Artifact with Branch & Version - shell: bash - run: | - ORIGINAL_NAME="${{ steps.build-crate.outputs.file-name }}" - NEW_NAME="cozodb-${{ env.BRANCH_OR_TAG }}-${{ env.PROJECT_VERSION }}-${{ matrix.job.target }}.tar.gz" - mv "${{ steps.build-crate.outputs.file-path }}" "$NEW_NAME" - echo "RENAMED_ARTIFACT=$NEW_NAME" >> $GITHUB_ENV + cargo-args: '${{ matrix.job.cargo-args }}' - name: Artifact upload uses: actions/upload-artifact@v4 with: - name: '${{ env.RENAMED_ARTIFACT }}' - path: '${{ env.RENAMED_ARTIFACT }}' - - name: Publish archives and packages + name: '${{ steps.build-crate.outputs.file-name }}' + path: '${{ steps.build-crate.outputs.file-path }}' + retention-days: 5 + - name: Publish to GitHub Releases uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: - files: | - ${{ env.RENAMED_ARTIFACT }} - if: 'startsWith(github.ref, ''refs/tags/'')' + files: '${{ steps.build-crate.outputs.file-path }}' + make_latest: true + generate_release_notes: true diff --git a/native/cozodb/Cargo.toml b/native/cozodb/Cargo.toml index c4a70a1..cf2d70b 100644 --- a/native/cozodb/Cargo.toml +++ b/native/cozodb/Cargo.toml @@ -23,13 +23,9 @@ num_cpus = "1.17.0" # Only compile on non-MSVC platforms when jemalloc feature is enabled # Note: disable-initial-exec-tls is required for Docker/dlopen() compatibility # to avoid "cannot allocate memory in static TLS block" errors -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = { version = "0.6", features = ["stats", "disable_initial_exec_tls"], optional = true } -tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } -tikv-jemalloc-sys = { version = "0.6", features = ["stats", "disable_initial_exec_tls"], optional = true } - # ============================================================================== # Pin the following to match cozo's working version +# ============================================================================== graph = "=0.3.1" # uses graph_builder 0.4.0 graph_builder = "=0.4.0" # uses rayon 1.10.0 rayon = "=1.10.0" @@ -39,7 +35,15 @@ serde = { version = "1.0.199", features = ["derive"] } serde_derive = "1.0.199" lazy_static = "1.4.0" crossbeam = "0.8.4" -# ============================================================================== + +# Memory allocator - jemalloc (optional, controlled by feature flag) +# Only compile on non-MSVC platforms when jemalloc feature is enabled +# Note: disable-initial-exec-tls is required for Docker/dlopen() compatibility +# to avoid "cannot allocate memory in static TLS block" errors +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = { version = "0.6", features = ["stats", "disable_initial_exec_tls"], optional = true } +tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } +tikv-jemalloc-sys = { version = "0.6", features = ["stats", "disable_initial_exec_tls"], optional = true } [features] default = [ @@ -49,6 +53,13 @@ default = [ "jemalloc" # jemalloc with 0ms decay: best throughput + low memory retention ] +# Windows default - same as default but without jemalloc (not available on Windows/MSVC) +windows-default = [ + "cozo/storage-sqlite", + "cozo/storage-rocksdb", + "cozo/graph-algo", +] + # IMPORTANT: storage-rocksdb (cozorocks) and storage-new-rocksdb (rust-rocksdb) are # MUTUALLY EXCLUSIVE. Both link to RocksDB with different allocator configurations, # causing "pointer being freed was not allocated" crashes when used together. @@ -114,4 +125,3 @@ jemalloc-profiling = ["tikv-jemallocator/profiling", "tikv-jemalloc-ctl/profilin # RocksDB falls back to standard pread/pwrite which performs well. # Enable via: COZODB_IO_URING=true make build io-uring = ["cozo/io-uring"] - diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml new file mode 100644 index 0000000..a8848af --- /dev/null +++ b/native/cozodb/Cross.toml @@ -0,0 +1,40 @@ +# Cross compilation configuration for cozodb NIF +# See: https://github.com/cross-rs/cross + +[build.env] +passthrough = [ + "RUST_BACKTRACE", + "LIBCLANG_PATH", + "LLVM_CONFIG_PATH", + "BINDGEN_EXTRA_CLANG_ARGS", +] + +# ============================================================================== +# aarch64-unknown-linux-gnu - use clang for C++20 support +# ============================================================================== +[target.aarch64-unknown-linux-gnu] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" +pre-build = [ + "apt-get update && apt-get install -y libclang-dev clang llvm lld", +] + +[target.aarch64-unknown-linux-gnu.env] +passthrough = ["RUST_BACKTRACE"] +# Force clang as C++ compiler with C++20 and correct target +variables = [ + ["CXX_aarch64_unknown_linux_gnu", "clang++"], + ["CC_aarch64_unknown_linux_gnu", "clang"], + ["CXX_aarch64-unknown-linux-gnu", "clang++"], + ["CC_aarch64-unknown-linux-gnu", "clang"], + ["CXXFLAGS", "-std=c++20 --target=aarch64-linux-gnu -I/usr/aarch64-linux-gnu/include"], + ["CFLAGS", "--target=aarch64-linux-gnu"], +] + +# ============================================================================== +# riscv64 - already works with standard image +# ============================================================================== +[target.riscv64gc-unknown-linux-gnu] +image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" +pre-build = [ + "apt-get update && apt-get install -y libclang-dev clang llvm", +] diff --git a/src/cargo.hrl b/src/cargo.hrl index 65809df..d985d30 100644 --- a/src/cargo.hrl +++ b/src/cargo.hrl @@ -4,7 +4,32 @@ -endif. -ifndef(CARGO_HRL). -define(CARGO_HRL, 1). + +%% Try to load precompiled NIF first, fall back to source-compiled NIF. +%% This macro wraps cozodb_nif_loader for backward compatibility. -define(load_nif_from_crate(__CRATE, __INIT), + (fun() -> + __APP = ?CARGO_LOAD_APP, + __PATH = filename:join([code:priv_dir(__APP), "crates", __CRATE, __CRATE]), + case filelib:is_file(__PATH ++ ".so") orelse + filelib:is_file(__PATH ++ ".dylib") of + true -> + %% NIF already exists (either precompiled or source-built) + erlang:load_nif(__PATH, __INIT); + false -> + %% Try to download precompiled, then load + case cozodb_nif_loader:load_nif() of + ok -> ok; + {error, _} -> + %% Final fallback: try loading from source path anyway + erlang:load_nif(__PATH, __INIT) + end + end + end)() +). + +%% Legacy macro for direct source loading (skips precompiled download) +-define(load_nif_from_crate_source(__CRATE, __INIT), (fun() -> __APP = ?CARGO_LOAD_APP, __PATH = filename:join([code:priv_dir(__APP), "crates", __CRATE, __CRATE]), diff --git a/src/cozodb.app.src b/src/cozodb.app.src index 974ec4b..6248976 100644 --- a/src/cozodb.app.src +++ b/src/cozodb.app.src @@ -6,6 +6,9 @@ {applications, [ kernel, stdlib, + crypto, + inets, + ssl, telemetry ]}, {env, []}, diff --git a/src/cozodb_nif_loader.erl b/src/cozodb_nif_loader.erl new file mode 100644 index 0000000..cc1f400 --- /dev/null +++ b/src/cozodb_nif_loader.erl @@ -0,0 +1,339 @@ +-module(cozodb_nif_loader). + +-export([ + load_nif/0, + load_nif/1, + get_platform_info/0, + download_precompiled/1, + verify_checksum/2 +]). + +%% GitHub repository for downloading precompiled NIFs +%% Can be overridden with environment variable COZODB_GITHUB_REPO +-define(DEFAULT_GITHUB_REPO, "adiibanez/cozodb"). +-define(NIF_VERSION, "2.17"). +-define(APP_NAME, cozodb). +-define(CRATE_NAME, "cozodb"). + +-type platform_info() :: #{ + os => linux | darwin | windows, + arch => x86_64 | aarch64 | arm | riscv64, + abi => gnu | musl | gnueabihf | darwin | msvc, + target => binary() +}. + +%% @doc Load the NIF library, trying precompiled first, then falling back to source. +-spec load_nif() -> ok | {error, term()}. +load_nif() -> + load_nif([]). + +%% @doc Load the NIF library with options. +%% Options: +%% - force_download: boolean() - Force download even if local exists +%% - force_compile: boolean() - Skip precompiled, compile from source +%% - version: string() - Specific version to download +-spec load_nif(proplists:proplist()) -> ok | {error, term()}. +load_nif(Opts) -> + ForceCompile = proplists:get_value(force_compile, Opts, false), + case ForceCompile of + true -> + load_from_source(); + false -> + try_load_precompiled(Opts) + end. + +%% @doc Get platform information for the current system. +-spec get_platform_info() -> platform_info(). +get_platform_info() -> + OS = get_os(), + Arch = get_arch(), + ABI = get_abi(OS), + Target = build_target(OS, Arch, ABI), + #{ + os => OS, + arch => Arch, + abi => ABI, + target => Target + }. + +%% @doc Download precompiled NIF for the given version. +-spec download_precompiled(string()) -> {ok, file:filename()} | {error, term()}. +download_precompiled(Version) -> + #{target := Target} = get_platform_info(), + PrivDir = code:priv_dir(?APP_NAME), + CrateDir = filename:join([PrivDir, "crates", "cozodb"]), + ok = filelib:ensure_dir(filename:join(CrateDir, "dummy")), + + ArtifactName = build_artifact_name(Version, Target), + Url = build_download_url(Version, ArtifactName), + + TmpDir = filename:join([PrivDir, "tmp"]), + ok = filelib:ensure_dir(filename:join(TmpDir, "dummy")), + TmpFile = filename:join(TmpDir, ArtifactName), + + logger:info("Downloading precompiled NIF from: ~s", [Url]), + case download_file(Url, TmpFile) of + ok -> + extract_and_install(TmpFile, CrateDir, Version, Target); + {error, _} = DownloadErr -> + DownloadErr + end. + +%% @doc Verify a file's SHA256 checksum. +-spec verify_checksum(file:filename(), binary()) -> ok | {error, checksum_mismatch}. +verify_checksum(FilePath, ExpectedChecksum) -> + case file:read_file(FilePath) of + {ok, Content} -> + ActualChecksum = crypto:hash(sha256, Content), + ActualHex = bin_to_hex(ActualChecksum), + case string:lowercase(binary_to_list(ExpectedChecksum)) =:= + string:lowercase(binary_to_list(ActualHex)) of + true -> ok; + false -> {error, checksum_mismatch} + end; + {error, Reason} -> + {error, {file_read_error, Reason}} + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== + +try_load_precompiled(Opts) -> + PrivDir = code:priv_dir(?APP_NAME), + NifPath = filename:join([PrivDir, "crates", "cozodb", "cozodb"]), + + %% Check for NIF file with any extension (.so, .dylib, .dll) + NifExists = filelib:is_file(NifPath ++ ".so") orelse + filelib:is_file(NifPath ++ ".dylib") orelse + filelib:is_file(NifPath ++ ".dll"), + + case NifExists of + true -> + ForceDownload = proplists:get_value(force_download, Opts, false), + case ForceDownload of + true -> + download_and_load(Opts); + false -> + load_nif_file(NifPath) + end; + false -> + download_and_load(Opts) + end. + +download_and_load(Opts) -> + Version = proplists:get_value(version, Opts, get_app_version()), + case download_precompiled(Version) of + {ok, NifPath} -> + load_nif_file(NifPath); + {error, Reason} -> + logger:warning("Failed to download precompiled NIF: ~p, falling back to source", [Reason]), + load_from_source() + end. + +load_nif_file(Path) -> + case erlang:load_nif(Path, 0) of + ok -> ok; + {error, {reload, _}} -> ok; + {error, Reason} -> {error, {nif_load_error, Reason}} + end. + +load_from_source() -> + PrivDir = code:priv_dir(?APP_NAME), + NifPath = filename:join([PrivDir, "crates", "cozodb", "cozodb"]), + load_nif_file(NifPath). + +get_os() -> + case os:type() of + {unix, linux} -> linux; + {unix, darwin} -> darwin; + {win32, _} -> windows; + _ -> unknown + end. + +get_arch() -> + case erlang:system_info(system_architecture) of + Arch when is_list(Arch) -> + ArchLower = string:lowercase(Arch), + case {string:find(ArchLower, "x86_64"), string:find(ArchLower, "aarch64"), + string:find(ArchLower, "arm"), string:find(ArchLower, "riscv64")} of + {nomatch, nomatch, nomatch, nomatch} -> + case string:find(ArchLower, "x86") of + nomatch -> unknown; + _ -> x86_64 + end; + {_, nomatch, nomatch, nomatch} -> x86_64; + {nomatch, _, nomatch, nomatch} -> aarch64; + {nomatch, nomatch, _, nomatch} -> arm; + {nomatch, nomatch, nomatch, _} -> riscv64 + end; + _ -> + unknown + end. + +get_abi(linux) -> + case is_musl() of + true -> musl; + false -> + case get_arch() of + arm -> gnueabihf; + _ -> gnu + end + end; +get_abi(darwin) -> + darwin; +get_abi(windows) -> + msvc; +get_abi(_) -> + unknown. + +is_musl() -> + case os:cmd("ldd --version 2>&1") of + Output -> + string:find(string:lowercase(Output), "musl") =/= nomatch + end. + +build_target(darwin, Arch, _ABI) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-apple-darwin"]); +build_target(linux, Arch, musl) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-unknown-linux-musl"]); +build_target(linux, arm, gnueabihf) -> + <<"arm-unknown-linux-gnueabihf">>; +build_target(linux, Arch, gnu) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-unknown-linux-gnu"]); +build_target(windows, Arch, msvc) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-pc-windows-msvc"]); +build_target(_, _, _) -> + <<"unknown">>. + +%% Build artifact name in rustler_precompiled format: +%% lib{crate}-v{version}-nif-{nif_version}-{target}.{ext}.tar.gz +build_artifact_name(Version, Target) -> + Ext = get_lib_extension(Target), + lists:flatten(io_lib:format( + "lib~s-v~s-nif-~s-~s~s.tar.gz", + [?CRATE_NAME, Version, ?NIF_VERSION, Target, Ext] + )). + +%% Build the download URL for GitHub releases +build_download_url(Version, ArtifactName) -> + GithubRepo = get_github_repo(), + Tag = "v" ++ Version, + lists:flatten(io_lib:format( + "https://github.com/~s/releases/download/~s/~s", + [GithubRepo, Tag, ArtifactName] + )). + +%% Get the library file extension for the target +get_lib_extension(Target) -> + TargetStr = binary_to_list(Target), + case string:find(TargetStr, "windows") of + nomatch -> + %% Both Linux and macOS use .so for Erlang NIFs + ".so"; + _ -> + ".dll" + end. + +%% Get GitHub repo from environment or default +get_github_repo() -> + case os:getenv("COZODB_GITHUB_REPO") of + false -> ?DEFAULT_GITHUB_REPO; + Repo -> Repo + end. + +download_file(Url, DestPath) -> + ensure_http_client(), + case httpc:request(get, {Url, []}, [{timeout, 120000}], [{stream, DestPath}]) of + {ok, saved_to_file} -> + ok; + {ok, {{_, 200, _}, _, Body}} -> + file:write_file(DestPath, Body); + {ok, {{_, 404, _}, _, _}} -> + {error, not_found}; + {ok, {{_, Status, _}, _, _}} -> + {error, {http_error, Status}}; + {error, Reason} -> + {error, {download_failed, Reason}} + end. + +download_and_verify_checksum(ChecksumUrl, FilePath) -> + TmpChecksum = FilePath ++ ".sha256.tmp", + case download_file(ChecksumUrl, TmpChecksum) of + ok -> + case file:read_file(TmpChecksum) of + {ok, ChecksumContent} -> + file:delete(TmpChecksum), + [ExpectedChecksum | _] = binary:split(ChecksumContent, [<<" ">>, <<"\t">>, <<"\n">>]), + verify_checksum(FilePath, ExpectedChecksum); + {error, Reason} -> + file:delete(TmpChecksum), + {error, {checksum_read_error, Reason}} + end; + {error, not_found} -> + logger:warning("Checksum file not found, skipping verification"), + ok; + {error, Reason} -> + {error, {checksum_download_failed, Reason}} + end. + +%% Extract tar.gz and rename the NIF file to standard name +extract_and_install(TarFile, DestDir, Version, Target) -> + case erl_tar:extract(TarFile, [{cwd, DestDir}, compressed]) of + ok -> + file:delete(TarFile), + %% The extracted file has rustler_precompiled naming: + %% lib{crate}-v{version}-nif-{nif_version}-{target}.{ext} + Ext = get_lib_extension(Target), + ExtractedName = lists:flatten(io_lib:format( + "lib~s-v~s-nif-~s-~s~s", + [?CRATE_NAME, Version, ?NIF_VERSION, Target, Ext] + )), + ExtractedPath = filename:join(DestDir, ExtractedName), + %% Rename to standard name without lib prefix and version + FinalExt = case Ext of + ".dll" -> ".dll"; + _ -> ".so" + end, + FinalPath = filename:join(DestDir, "cozodb" ++ FinalExt), + case file:rename(ExtractedPath, FinalPath) of + ok -> + NifPath = filename:join(DestDir, "cozodb"), + {ok, NifPath}; + {error, RenameReason} -> + logger:warning("Failed to rename ~s to ~s: ~p", + [ExtractedPath, FinalPath, RenameReason]), + %% Try to use the extracted file directly + NifPath = filename:rootname(ExtractedPath), + {ok, NifPath} + end; + {error, Reason} -> + file:delete(TarFile), + {error, {extract_failed, Reason}} + end. + +ensure_http_client() -> + case application:ensure_all_started(inets) of + {ok, _} -> ok; + {error, {already_started, _}} -> ok + end, + case application:ensure_all_started(ssl) of + {ok, _} -> ok; + {error, {already_started, _}} -> ok + end. + +get_app_version() -> + case application:get_key(?APP_NAME, vsn) of + {ok, Vsn} -> Vsn; + undefined -> "0.0.0" + end. + +bin_to_hex(Bin) -> + << <<(hex_char(H)), (hex_char(L))>> || <> <= Bin >>. + +hex_char(N) when N < 10 -> $0 + N; +hex_char(N) -> $a + N - 10.