From 46693cba0e1ee15583db56ab6ed51193010455c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 15:01:31 +0100 Subject: [PATCH 01/31] fix CI pipeline and add precompiled NIF support Phase 1 - CI Fixes: - Fix broken version extraction (sed command on single line, src.cozodb.app.src -> src/cozodb.app.src) - Split build dependencies by platform (Linux apt-get, macOS brew) - Add liburing-dev for cozorocks build on Linux - Add checksum generation for release artifacts Phase 2 - Precompiled NIF consumption: - Add cozodb_nif_loader module for downloading precompiled NIFs from GitHub Releases - Update cargo.hrl macro to try precompiled first, fall back to source - Add crypto, inets, ssl dependencies for HTTP client Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 28 +++- rebar.config | 10 +- src/cargo.hrl | 25 +++ src/cozodb.app.src | 3 + src/cozodb_nif_loader.erl | 288 ++++++++++++++++++++++++++++++++++ 5 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 src/cozodb_nif_loader.erl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1abd0ab..ae40ef0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,8 +44,7 @@ jobs: shell: bash run: | # Get the project version from cozodb.app.src - echo "PROJECT_VERSION=$(sed -n 's/.*{vsn, "\([^"]*\)".*/\1/p' - src.cozodb.app.src | head -n1)" >> $GITHUB_ENV + echo "PROJECT_VERSION=$(sed -n 's/.*{vsn, "\([^"]*\)".*/\1/p' src/cozodb.app.src | head -n1)" >> $GITHUB_ENV # Determine if it's a tag or a branch if [[ "$GITHUB_REF" == refs/tags/* ]]; then @@ -54,8 +53,15 @@ jobs: BRANCH_OR_TAG="${GITHUB_REF#refs/heads/}" fi echo "BRANCH_OR_TAG=$BRANCH_OR_TAG" >> $GITHUB_ENV - - name: Install build dependencies - run: sudo apt-get update && sudo apt-get install -y build-essential clang libclang-dev pkg-config + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y build-essential clang libclang-dev pkg-config liburing-dev + - name: Install build dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm pkg-config - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -78,14 +84,26 @@ jobs: NEW_NAME="cozodb-${{ env.BRANCH_OR_TAG }}-${{ env.PROJECT_VERSION }}-${{ matrix.job.target }}.tar.gz" mv "${{ steps.build-crate.outputs.file-path }}" "$NEW_NAME" echo "RENAMED_ARTIFACT=$NEW_NAME" >> $GITHUB_ENV + - name: Generate checksum + shell: bash + run: | + if [[ "$RUNNER_OS" == "macOS" ]]; then + shasum -a 256 "${{ env.RENAMED_ARTIFACT }}" > "${{ env.RENAMED_ARTIFACT }}.sha256" + else + sha256sum "${{ env.RENAMED_ARTIFACT }}" > "${{ env.RENAMED_ARTIFACT }}.sha256" + fi + echo "CHECKSUM_FILE=${{ env.RENAMED_ARTIFACT }}.sha256" >> $GITHUB_ENV - name: Artifact upload uses: actions/upload-artifact@v4 with: name: '${{ env.RENAMED_ARTIFACT }}' - path: '${{ env.RENAMED_ARTIFACT }}' + path: | + ${{ env.RENAMED_ARTIFACT }} + ${{ env.CHECKSUM_FILE }} - name: Publish archives and packages uses: softprops/action-gh-release@v2 with: files: | ${{ env.RENAMED_ARTIFACT }} + ${{ env.CHECKSUM_FILE }} if: 'startsWith(github.ref, ''refs/tags/'')' diff --git a/rebar.config b/rebar.config index 159d5be..4bd5266 100644 --- a/rebar.config +++ b/rebar.config @@ -54,6 +54,9 @@ ]} ]}. +%% Cargo options for building the Rust NIF from source. +%% This is used when precompiled binaries are not available or when +%% COZODB_BUILD_FROM_SOURCE=true is set. {cargo_opts, [ {src_dir, "native/cozodb"}, %% optimises rust compilation @@ -63,9 +66,14 @@ %% To use a different RocksDB backend, use the Makefile targets: %% make build - Default (cozorocks backend) %% make build-newrocksdb - Use rust-rocksdb backend - %% Or set COZODB_BACKEND=newrocksdb before running make build + %% Or set COZODB_BUILD_FROM_SOURCE=true and COZODB_BACKEND=newrocksdb ]}. +%% Provider hooks for cargo compilation. +%% The cargo build is triggered on compile and cargo clean on clean. +%% To skip cargo build and use precompiled NIFs: +%% - Set COZODB_USE_PRECOMPILED=true (will download from GitHub Releases) +%% - Or manually place the NIF in priv/crates/cozodb/ {provider_hooks, [ {pre, [ {compile, {cargo, build}} diff --git a/src/cargo.hrl b/src/cargo.hrl index 65809df..d985d30 100644 --- a/src/cargo.hrl +++ b/src/cargo.hrl @@ -4,7 +4,32 @@ -endif. -ifndef(CARGO_HRL). -define(CARGO_HRL, 1). + +%% Try to load precompiled NIF first, fall back to source-compiled NIF. +%% This macro wraps cozodb_nif_loader for backward compatibility. -define(load_nif_from_crate(__CRATE, __INIT), + (fun() -> + __APP = ?CARGO_LOAD_APP, + __PATH = filename:join([code:priv_dir(__APP), "crates", __CRATE, __CRATE]), + case filelib:is_file(__PATH ++ ".so") orelse + filelib:is_file(__PATH ++ ".dylib") of + true -> + %% NIF already exists (either precompiled or source-built) + erlang:load_nif(__PATH, __INIT); + false -> + %% Try to download precompiled, then load + case cozodb_nif_loader:load_nif() of + ok -> ok; + {error, _} -> + %% Final fallback: try loading from source path anyway + erlang:load_nif(__PATH, __INIT) + end + end + end)() +). + +%% Legacy macro for direct source loading (skips precompiled download) +-define(load_nif_from_crate_source(__CRATE, __INIT), (fun() -> __APP = ?CARGO_LOAD_APP, __PATH = filename:join([code:priv_dir(__APP), "crates", __CRATE, __CRATE]), diff --git a/src/cozodb.app.src b/src/cozodb.app.src index d17dcf7..6a23f1f 100644 --- a/src/cozodb.app.src +++ b/src/cozodb.app.src @@ -6,6 +6,9 @@ {applications, [ kernel, stdlib, + crypto, + inets, + ssl, telemetry ]}, {env, []}, diff --git a/src/cozodb_nif_loader.erl b/src/cozodb_nif_loader.erl new file mode 100644 index 0000000..067ece7 --- /dev/null +++ b/src/cozodb_nif_loader.erl @@ -0,0 +1,288 @@ +-module(cozodb_nif_loader). + +-export([ + load_nif/0, + load_nif/1, + get_platform_info/0, + download_precompiled/1, + verify_checksum/2 +]). + +-define(GITHUB_REPO, "leapsight/cozodb"). +-define(NIF_VERSION, "2.17"). +-define(APP_NAME, cozodb). + +-type platform_info() :: #{ + os => linux | darwin | windows, + arch => x86_64 | aarch64 | arm | riscv64, + abi => gnu | musl | gnueabihf | darwin | msvc, + target => binary() +}. + +%% @doc Load the NIF library, trying precompiled first, then falling back to source. +-spec load_nif() -> ok | {error, term()}. +load_nif() -> + load_nif([]). + +%% @doc Load the NIF library with options. +%% Options: +%% - force_download: boolean() - Force download even if local exists +%% - force_compile: boolean() - Skip precompiled, compile from source +%% - version: string() - Specific version to download +-spec load_nif(proplists:proplist()) -> ok | {error, term()}. +load_nif(Opts) -> + ForceCompile = proplists:get_value(force_compile, Opts, false), + case ForceCompile of + true -> + load_from_source(); + false -> + try_load_precompiled(Opts) + end. + +%% @doc Get platform information for the current system. +-spec get_platform_info() -> platform_info(). +get_platform_info() -> + OS = get_os(), + Arch = get_arch(), + ABI = get_abi(OS), + Target = build_target(OS, Arch, ABI), + #{ + os => OS, + arch => Arch, + abi => ABI, + target => Target + }. + +%% @doc Download precompiled NIF for the given version. +-spec download_precompiled(string()) -> {ok, file:filename()} | {error, term()}. +download_precompiled(Version) -> + #{target := Target} = get_platform_info(), + PrivDir = code:priv_dir(?APP_NAME), + CrateDir = filename:join([PrivDir, "crates", "cozodb"]), + ok = filelib:ensure_dir(filename:join(CrateDir, "dummy")), + + Tag = Version, + ArtifactName = build_artifact_name(Tag, Version, Target), + Url = build_download_url(Tag, ArtifactName), + ChecksumUrl = Url ++ ".sha256", + + TmpDir = filename:join([PrivDir, "tmp"]), + ok = filelib:ensure_dir(filename:join(TmpDir, "dummy")), + TmpFile = filename:join(TmpDir, ArtifactName), + + case download_file(Url, TmpFile) of + ok -> + case download_and_verify_checksum(ChecksumUrl, TmpFile) of + ok -> + extract_and_install(TmpFile, CrateDir); + {error, _} = ChecksumErr -> + file:delete(TmpFile), + ChecksumErr + end; + {error, _} = DownloadErr -> + DownloadErr + end. + +%% @doc Verify a file's SHA256 checksum. +-spec verify_checksum(file:filename(), binary()) -> ok | {error, checksum_mismatch}. +verify_checksum(FilePath, ExpectedChecksum) -> + case file:read_file(FilePath) of + {ok, Content} -> + ActualChecksum = crypto:hash(sha256, Content), + ActualHex = bin_to_hex(ActualChecksum), + case string:lowercase(binary_to_list(ExpectedChecksum)) =:= + string:lowercase(binary_to_list(ActualHex)) of + true -> ok; + false -> {error, checksum_mismatch} + end; + {error, Reason} -> + {error, {file_read_error, Reason}} + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== + +try_load_precompiled(Opts) -> + PrivDir = code:priv_dir(?APP_NAME), + NifPath = filename:join([PrivDir, "crates", "cozodb", "cozodb"]), + + case filelib:is_file(NifPath ++ ".so") orelse + filelib:is_file(NifPath ++ ".dylib") of + true -> + ForceDownload = proplists:get_value(force_download, Opts, false), + case ForceDownload of + true -> + download_and_load(Opts); + false -> + load_nif_file(NifPath) + end; + false -> + download_and_load(Opts) + end. + +download_and_load(Opts) -> + Version = proplists:get_value(version, Opts, get_app_version()), + case download_precompiled(Version) of + {ok, NifPath} -> + load_nif_file(NifPath); + {error, Reason} -> + logger:warning("Failed to download precompiled NIF: ~p, falling back to source", [Reason]), + load_from_source() + end. + +load_nif_file(Path) -> + case erlang:load_nif(Path, 0) of + ok -> ok; + {error, {reload, _}} -> ok; + {error, Reason} -> {error, {nif_load_error, Reason}} + end. + +load_from_source() -> + PrivDir = code:priv_dir(?APP_NAME), + NifPath = filename:join([PrivDir, "crates", "cozodb", "cozodb"]), + load_nif_file(NifPath). + +get_os() -> + case os:type() of + {unix, linux} -> linux; + {unix, darwin} -> darwin; + {win32, _} -> windows; + _ -> unknown + end. + +get_arch() -> + case erlang:system_info(system_architecture) of + Arch when is_list(Arch) -> + ArchLower = string:lowercase(Arch), + case {string:find(ArchLower, "x86_64"), string:find(ArchLower, "aarch64"), + string:find(ArchLower, "arm"), string:find(ArchLower, "riscv64")} of + {nomatch, nomatch, nomatch, nomatch} -> + case string:find(ArchLower, "x86") of + nomatch -> unknown; + _ -> x86_64 + end; + {_, nomatch, nomatch, nomatch} -> x86_64; + {nomatch, _, nomatch, nomatch} -> aarch64; + {nomatch, nomatch, _, nomatch} -> arm; + {nomatch, nomatch, nomatch, _} -> riscv64 + end; + _ -> + unknown + end. + +get_abi(linux) -> + case is_musl() of + true -> musl; + false -> + case get_arch() of + arm -> gnueabihf; + _ -> gnu + end + end; +get_abi(darwin) -> + darwin; +get_abi(windows) -> + msvc; +get_abi(_) -> + unknown. + +is_musl() -> + case os:cmd("ldd --version 2>&1") of + Output -> + string:find(string:lowercase(Output), "musl") =/= nomatch + end. + +build_target(darwin, Arch, _ABI) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-apple-darwin"]); +build_target(linux, Arch, musl) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-unknown-linux-musl"]); +build_target(linux, arm, gnueabihf) -> + <<"arm-unknown-linux-gnueabihf">>; +build_target(linux, Arch, gnu) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-unknown-linux-gnu"]); +build_target(windows, Arch, msvc) -> + ArchStr = atom_to_list(Arch), + iolist_to_binary([ArchStr, "-pc-windows-msvc"]); +build_target(_, _, _) -> + <<"unknown">>. + +build_artifact_name(Tag, Version, Target) -> + lists:flatten(io_lib:format("cozodb-~s-~s-~s.tar.gz", [Tag, Version, Target])). + +build_download_url(Tag, ArtifactName) -> + lists:flatten(io_lib:format( + "https://github.com/~s/releases/download/~s/~s", + [?GITHUB_REPO, Tag, ArtifactName] + )). + +download_file(Url, DestPath) -> + ensure_http_client(), + case httpc:request(get, {Url, []}, [{timeout, 120000}], [{stream, DestPath}]) of + {ok, saved_to_file} -> + ok; + {ok, {{_, 200, _}, _, Body}} -> + file:write_file(DestPath, Body); + {ok, {{_, 404, _}, _, _}} -> + {error, not_found}; + {ok, {{_, Status, _}, _, _}} -> + {error, {http_error, Status}}; + {error, Reason} -> + {error, {download_failed, Reason}} + end. + +download_and_verify_checksum(ChecksumUrl, FilePath) -> + TmpChecksum = FilePath ++ ".sha256.tmp", + case download_file(ChecksumUrl, TmpChecksum) of + ok -> + case file:read_file(TmpChecksum) of + {ok, ChecksumContent} -> + file:delete(TmpChecksum), + [ExpectedChecksum | _] = binary:split(ChecksumContent, [<<" ">>, <<"\t">>, <<"\n">>]), + verify_checksum(FilePath, ExpectedChecksum); + {error, Reason} -> + file:delete(TmpChecksum), + {error, {checksum_read_error, Reason}} + end; + {error, not_found} -> + logger:warning("Checksum file not found, skipping verification"), + ok; + {error, Reason} -> + {error, {checksum_download_failed, Reason}} + end. + +extract_and_install(TarFile, DestDir) -> + case erl_tar:extract(TarFile, [{cwd, DestDir}, compressed]) of + ok -> + file:delete(TarFile), + NifPath = filename:join(DestDir, "cozodb"), + {ok, NifPath}; + {error, Reason} -> + file:delete(TarFile), + {error, {extract_failed, Reason}} + end. + +ensure_http_client() -> + case application:ensure_all_started(inets) of + {ok, _} -> ok; + {error, {already_started, _}} -> ok + end, + case application:ensure_all_started(ssl) of + {ok, _} -> ok; + {error, {already_started, _}} -> ok + end. + +get_app_version() -> + case application:get_key(?APP_NAME, vsn) of + {ok, Vsn} -> Vsn; + undefined -> "0.0.0" + end. + +bin_to_hex(Bin) -> + << <<(hex_char(H)), (hex_char(L))>> || <> <= Bin >>. + +hex_char(N) when N < 10 -> $0 + N; +hex_char(N) -> $a + N - 10. From 4259c1d9b1a88e3873755138091b46fdfd24b70f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 15:08:37 +0100 Subject: [PATCH 02/31] fix cross-compilation: add Cross.toml and make io-uring opt-in - Add Cross.toml to install libclang-dev in cross containers (fixes bindgen) - Make io-uring feature opt-in instead of default for Linux targets - Install cross from git to get latest version with pre-build support - Remove target-specific io-uring dependency that broke cross-compilation Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 4 +++ native/cozodb/Cargo.toml | 12 +++++++-- native/cozodb/Cross.toml | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 native/cozodb/Cross.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae40ef0..5456da0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,6 +67,10 @@ jobs: with: toolchain: stable target: '${{ matrix.job.target }}' + - name: Install cross (for cross-compilation) + if: matrix.job.use-cross == true + run: | + cargo install cross --git https://github.com/cross-rs/cross - name: Build the project id: build-crate uses: philss/rustler-precompiled-action@v1.1.4 diff --git a/native/cozodb/Cargo.toml b/native/cozodb/Cargo.toml index 12283de..9188fe0 100644 --- a/native/cozodb/Cargo.toml +++ b/native/cozodb/Cargo.toml @@ -99,6 +99,14 @@ jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "cozo # This gives best of both worlds: BEAM-tracked Rust memory + jemalloc stats for RocksDB nif_alloc_rocksdb_jemalloc = ["nif_alloc", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "cozo/rocksdb-jemalloc"] -[target.'cfg(target_os = "linux")'.dependencies] -cozo = { git = "https://github.com/aramallo/cozo.git", tag = "v0.8.2-leapsight", features = ["io-uring"] } +# Enable io-uring for async I/O on Linux (requires liburing-dev) +# Only enable this for native Linux builds, not cross-compilation +io-uring = ["cozo/io-uring"] + +# io-uring support is now opt-in via the "io-uring" feature below. +# This allows cross-compilation to work without liburing. +# To enable io-uring for native Linux builds, use: --features io-uring +# +# [target.'cfg(target_os = "linux")'.dependencies] +# cozo = { git = "https://github.com/aramallo/cozo.git", tag = "v0.8.2-leapsight", features = ["io-uring"] } diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml new file mode 100644 index 0000000..7a4c06a --- /dev/null +++ b/native/cozodb/Cross.toml @@ -0,0 +1,46 @@ +# Cross compilation configuration for cozodb NIF +# See: https://github.com/cross-rs/cross +# +# The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ +# for the clang_getTranslationUnitTargetInfo function. + +[build.env] +passthrough = [ + "RUST_BACKTRACE", + "LIBCLANG_PATH", + "LLVM_CONFIG_PATH", + "BINDGEN_EXTRA_CLANG_ARGS", +] + +# ============================================================================== +# Linux GNU targets +# ============================================================================== + +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "apt-get update && apt-get install -y libclang-dev llvm-dev clang", +] + +[target.arm-unknown-linux-gnueabihf] +pre-build = [ + "apt-get update && apt-get install -y libclang-dev llvm-dev clang", +] + +[target.riscv64gc-unknown-linux-gnu] +pre-build = [ + "apt-get update && apt-get install -y libclang-dev llvm-dev clang", +] + +# ============================================================================== +# Linux MUSL targets +# ============================================================================== + +[target.x86_64-unknown-linux-musl] +pre-build = [ + "apt-get update && apt-get install -y libclang-dev llvm-dev clang", +] + +[target.aarch64-unknown-linux-musl] +pre-build = [ + "apt-get update && apt-get install -y libclang-dev llvm-dev clang", +] From 20b2e929175bc70c2cc673effda1e3df223960a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 15:10:03 +0100 Subject: [PATCH 03/31] add concurrency control to cancel older workflow runs Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5456da0..0538ca7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,11 @@ name: Build precompiled NIFs tags: - '*' workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_release: name: 'NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }})' From 3dbb0370a714f049fae737f9c345a571bb0f68a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 15:16:34 +0100 Subject: [PATCH 04/31] fix cross-compilation: install LLVM 14 from apt.llvm.org - Update Cross.toml to install LLVM 14 from official LLVM apt repository - Add Cross.toml to repo root as fallback - Set LIBCLANG_PATH environment variable for cross builds The default cross images have libclang 3.8.x but bindgen requires 5.0+ Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 5 +++ Cross.toml | 67 +++++++++++++++++++++++++++++++++++ native/cozodb/Cross.toml | 31 +++++++++++++--- 3 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 Cross.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0538ca7..e811356 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,6 +76,11 @@ jobs: if: matrix.job.use-cross == true run: | cargo install cross --git https://github.com/cross-rs/cross + - name: Set LIBCLANG_PATH for cross builds + if: matrix.job.use-cross == true + run: | + echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV + echo "LLVM_CONFIG_PATH=/usr/bin/llvm-config-14" >> $GITHUB_ENV - name: Build the project id: build-crate uses: philss/rustler-precompiled-action@v1.1.4 diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..42e5d95 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,67 @@ +# Cross compilation configuration for cozodb NIF +# See: https://github.com/cross-rs/cross +# +# The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ +# for the clang_getTranslationUnitTargetInfo function. +# We install LLVM 14 from the official LLVM apt repository. + +[build.env] +passthrough = [ + "RUST_BACKTRACE", + "LIBCLANG_PATH", + "LLVM_CONFIG_PATH", + "BINDGEN_EXTRA_CLANG_ARGS", +] + +# ============================================================================== +# Linux GNU targets +# ============================================================================== + +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", +] + +[target.arm-unknown-linux-gnueabihf] +pre-build = [ + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", +] + +[target.riscv64gc-unknown-linux-gnu] +pre-build = [ + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", +] + +# ============================================================================== +# Linux MUSL targets +# ============================================================================== + +[target.x86_64-unknown-linux-musl] +pre-build = [ + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", +] + +[target.aarch64-unknown-linux-musl] +pre-build = [ + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", +] diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 7a4c06a..42e5d95 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -3,6 +3,7 @@ # # The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ # for the clang_getTranslationUnitTargetInfo function. +# We install LLVM 14 from the official LLVM apt repository. [build.env] passthrough = [ @@ -18,17 +19,29 @@ passthrough = [ [target.aarch64-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y libclang-dev llvm-dev clang", + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.arm-unknown-linux-gnueabihf] pre-build = [ - "apt-get update && apt-get install -y libclang-dev llvm-dev clang", + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.riscv64gc-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y libclang-dev llvm-dev clang", + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] # ============================================================================== @@ -37,10 +50,18 @@ pre-build = [ [target.x86_64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y libclang-dev llvm-dev clang", + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.aarch64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y libclang-dev llvm-dev clang", + "apt-get update && apt-get install -y wget gnupg software-properties-common", + "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", + "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", + "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] From 75169514313d7e4a7a108e9ed3b18ce495c6e6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:11:55 +0100 Subject: [PATCH 05/31] fix artifact rename step with debugging and fallback search The rustler-precompiled-action output path may not match expected location. Added debugging output and fallback to search for .tar.gz files. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e811356..ba10cd2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,9 +94,27 @@ jobs: - name: Rename Artifact with Branch & Version shell: bash run: | - ORIGINAL_NAME="${{ steps.build-crate.outputs.file-name }}" + echo "Debug: file-name = ${{ steps.build-crate.outputs.file-name }}" + echo "Debug: file-path = ${{ steps.build-crate.outputs.file-path }}" + echo "Debug: Listing current directory:" + ls -la *.tar.gz 2>/dev/null || echo "No tar.gz files in current dir" + ls -la native/cozodb/*.tar.gz 2>/dev/null || echo "No tar.gz files in native/cozodb" + + # Find the artifact file + ARTIFACT_PATH="${{ steps.build-crate.outputs.file-path }}" + if [ -z "$ARTIFACT_PATH" ] || [ ! -f "$ARTIFACT_PATH" ]; then + echo "Output file-path not found, searching for artifact..." + ARTIFACT_PATH=$(find . -name "*.tar.gz" -type f | head -1) + fi + + if [ -z "$ARTIFACT_PATH" ] || [ ! -f "$ARTIFACT_PATH" ]; then + echo "Error: Could not find artifact file" + exit 1 + fi + NEW_NAME="cozodb-${{ env.BRANCH_OR_TAG }}-${{ env.PROJECT_VERSION }}-${{ matrix.job.target }}.tar.gz" - mv "${{ steps.build-crate.outputs.file-path }}" "$NEW_NAME" + echo "Moving $ARTIFACT_PATH to $NEW_NAME" + mv "$ARTIFACT_PATH" "$NEW_NAME" echo "RENAMED_ARTIFACT=$NEW_NAME" >> $GITHUB_ENV - name: Generate checksum shell: bash From 7be82658e7350e0c8bbf4efbc83398d054be1442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:12:27 +0100 Subject: [PATCH 06/31] fix interactive apt prompt with DEBIAN_FRONTEND=noninteractive The tzdata package was prompting for timezone selection, blocking the build. Co-Authored-By: Claude Opus 4.5 --- Cross.toml | 20 ++++++++++---------- native/cozodb/Cross.toml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cross.toml b/Cross.toml index 42e5d95..34a16f8 100644 --- a/Cross.toml +++ b/Cross.toml @@ -19,28 +19,28 @@ passthrough = [ [target.aarch64-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.arm-unknown-linux-gnueabihf] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.riscv64gc-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] @@ -50,18 +50,18 @@ pre-build = [ [target.x86_64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.aarch64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 42e5d95..34a16f8 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -19,28 +19,28 @@ passthrough = [ [target.aarch64-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.arm-unknown-linux-gnueabihf] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.riscv64gc-unknown-linux-gnu] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] @@ -50,18 +50,18 @@ pre-build = [ [target.x86_64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] [target.aarch64-unknown-linux-musl] pre-build = [ - "apt-get update && apt-get install -y wget gnupg software-properties-common", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", + "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", ] From 6d5fb689b9f3c8e30205e08d03098c3b15e5d2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:13:33 +0100 Subject: [PATCH 07/31] set CROSS_CONFIG env var to ensure Cross.toml is found Added debugging to show Cross.toml content during build. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba10cd2..a90601c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,11 +76,15 @@ jobs: if: matrix.job.use-cross == true run: | cargo install cross --git https://github.com/cross-rs/cross - - name: Set LIBCLANG_PATH for cross builds + - name: Set cross environment if: matrix.job.use-cross == true run: | + echo "CROSS_CONFIG=${{ github.workspace }}/Cross.toml" >> $GITHUB_ENV echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV echo "LLVM_CONFIG_PATH=/usr/bin/llvm-config-14" >> $GITHUB_ENV + # Debug: show Cross.toml content + echo "Cross.toml content:" + cat ${{ github.workspace }}/Cross.toml - name: Build the project id: build-crate uses: philss/rustler-precompiled-action@v1.1.4 From 334f1d97b61dce01f41795a4a4bb7b264d1c5acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:27:47 +0100 Subject: [PATCH 08/31] fix branch name sanitization: replace / with - in artifact names Branch names like fix/ci-precompiled-pipeline were creating invalid paths. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a90601c..3895a5e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,6 +57,8 @@ jobs: else BRANCH_OR_TAG="${GITHUB_REF#refs/heads/}" fi + # Sanitize branch name: replace / with - + BRANCH_OR_TAG="${BRANCH_OR_TAG//\//-}" echo "BRANCH_OR_TAG=$BRANCH_OR_TAG" >> $GITHUB_ENV - name: Install build dependencies (Linux) if: runner.os == 'Linux' From 4388062603d33aea7098a2795da11466ae40d710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:29:48 +0100 Subject: [PATCH 09/31] use official cross-rs images and simplify pre-build - Use ghcr.io/cross-rs images which may have newer base OS - Simplified pre-build to just install libclang-dev from default repos - Added more debugging output for cross configuration Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 12 ++++++++-- Cross.toml | 45 +++++++++++++---------------------- native/cozodb/Cross.toml | 45 +++++++++++++---------------------- 3 files changed, 44 insertions(+), 58 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3895a5e..26c78cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -84,9 +84,17 @@ jobs: echo "CROSS_CONFIG=${{ github.workspace }}/Cross.toml" >> $GITHUB_ENV echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV echo "LLVM_CONFIG_PATH=/usr/bin/llvm-config-14" >> $GITHUB_ENV - # Debug: show Cross.toml content - echo "Cross.toml content:" + # Also set in native/cozodb for the action + echo "CROSS_CONFIG=${{ github.workspace }}/native/cozodb/Cross.toml" >> $GITHUB_ENV + # Debug: show Cross.toml content and cross version + echo "Cross version:" + cross --version + echo "" + echo "Cross.toml in workspace root:" cat ${{ github.workspace }}/Cross.toml + echo "" + echo "Cross.toml in native/cozodb:" + cat ${{ github.workspace }}/native/cozodb/Cross.toml - name: Build the project id: build-crate uses: philss/rustler-precompiled-action@v1.1.4 diff --git a/Cross.toml b/Cross.toml index 34a16f8..4e272ce 100644 --- a/Cross.toml +++ b/Cross.toml @@ -2,8 +2,7 @@ # See: https://github.com/cross-rs/cross # # The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# for the clang_getTranslationUnitTargetInfo function. -# We install LLVM 14 from the official LLVM apt repository. +# We use custom images that have LLVM pre-installed. [build.env] passthrough = [ @@ -14,34 +13,28 @@ passthrough = [ ] # ============================================================================== -# Linux GNU targets +# Linux GNU targets - use cross images with LLVM # ============================================================================== [target.aarch64-unknown-linux-gnu] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.arm-unknown-linux-gnueabihf] +image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.riscv64gc-unknown-linux-gnu] +image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] # ============================================================================== @@ -49,19 +42,15 @@ pre-build = [ # ============================================================================== [target.x86_64-unknown-linux-musl] +image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.aarch64-unknown-linux-musl] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 34a16f8..4e272ce 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -2,8 +2,7 @@ # See: https://github.com/cross-rs/cross # # The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# for the clang_getTranslationUnitTargetInfo function. -# We install LLVM 14 from the official LLVM apt repository. +# We use custom images that have LLVM pre-installed. [build.env] passthrough = [ @@ -14,34 +13,28 @@ passthrough = [ ] # ============================================================================== -# Linux GNU targets +# Linux GNU targets - use cross images with LLVM # ============================================================================== [target.aarch64-unknown-linux-gnu] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.arm-unknown-linux-gnueabihf] +image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.riscv64gc-unknown-linux-gnu] +image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] # ============================================================================== @@ -49,19 +42,15 @@ pre-build = [ # ============================================================================== [target.x86_64-unknown-linux-musl] +image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] [target.aarch64-unknown-linux-musl] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y wget gnupg software-properties-common", - "wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -", - "echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main' >> /etc/apt/sources.list", - "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y llvm-14 clang-14 libclang-14-dev", - "ln -sf /usr/bin/clang-14 /usr/bin/clang && ln -sf /usr/bin/llvm-config-14 /usr/bin/llvm-config", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update && apt-get install -y libclang-dev clang llvm", ] From c704eb29079f7f3ed3130766e6a0d562899c9f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 16:45:54 +0100 Subject: [PATCH 10/31] fix cross environment step: remove cross --version that triggers metadata error Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26c78cc..069453e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,19 +81,10 @@ jobs: - name: Set cross environment if: matrix.job.use-cross == true run: | - echo "CROSS_CONFIG=${{ github.workspace }}/Cross.toml" >> $GITHUB_ENV - echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - echo "LLVM_CONFIG_PATH=/usr/bin/llvm-config-14" >> $GITHUB_ENV - # Also set in native/cozodb for the action + # Point cross to our config file echo "CROSS_CONFIG=${{ github.workspace }}/native/cozodb/Cross.toml" >> $GITHUB_ENV - # Debug: show Cross.toml content and cross version - echo "Cross version:" - cross --version - echo "" - echo "Cross.toml in workspace root:" - cat ${{ github.workspace }}/Cross.toml - echo "" - echo "Cross.toml in native/cozodb:" + # Debug: show Cross.toml content + echo "Cross.toml content:" cat ${{ github.workspace }}/native/cozodb/Cross.toml - name: Build the project id: build-crate From ad253377d94b1c3b689ee7234a48bf78b7027059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 17:08:02 +0100 Subject: [PATCH 11/31] fix RocksDB build: set CXXFLAGS=-std=c++20 RocksDB uses C++20 features like defaulted comparison operators and 'using enum'. Without -std=c++20, the build fails with compiler errors. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 4 ++++ Cross.toml | 5 +++++ native/cozodb/Cross.toml | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 069453e..5527252 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,6 +65,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential clang libclang-dev pkg-config liburing-dev + # RocksDB requires C++20 + echo "CXXFLAGS=-std=c++20" >> $GITHUB_ENV - name: Install build dependencies (macOS) if: runner.os == 'macOS' run: | @@ -83,6 +85,8 @@ jobs: run: | # Point cross to our config file echo "CROSS_CONFIG=${{ github.workspace }}/native/cozodb/Cross.toml" >> $GITHUB_ENV + # RocksDB requires C++20 + echo "CXXFLAGS=-std=c++20" >> $GITHUB_ENV # Debug: show Cross.toml content echo "Cross.toml content:" cat ${{ github.workspace }}/native/cozodb/Cross.toml diff --git a/Cross.toml b/Cross.toml index 4e272ce..7e6e334 100644 --- a/Cross.toml +++ b/Cross.toml @@ -10,6 +10,11 @@ passthrough = [ "LIBCLANG_PATH", "LLVM_CONFIG_PATH", "BINDGEN_EXTRA_CLANG_ARGS", + "CXXFLAGS", +] +# RocksDB requires C++20 +variables = [ + ["CXXFLAGS", "-std=c++20"], ] # ============================================================================== diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 4e272ce..7e6e334 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -10,6 +10,11 @@ passthrough = [ "LIBCLANG_PATH", "LLVM_CONFIG_PATH", "BINDGEN_EXTRA_CLANG_ARGS", + "CXXFLAGS", +] +# RocksDB requires C++20 +variables = [ + ["CXXFLAGS", "-std=c++20"], ] # ============================================================================== From 300ef92e2e2997b8b53121e9f7d6b373ecf8d062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 17:52:21 +0100 Subject: [PATCH 12/31] fix: use -std=c++2a for older GCC cross-compilers The cross-compilation containers have older GCC that doesn't recognize -std=c++20 but supports -std=c++2a (older name for C++20 draft). Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 4 ++-- Cross.toml | 2 +- native/cozodb/Cross.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5527252..8d5319f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,7 +66,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential clang libclang-dev pkg-config liburing-dev # RocksDB requires C++20 - echo "CXXFLAGS=-std=c++20" >> $GITHUB_ENV + echo "CXXFLAGS=-std=c++2a" >> $GITHUB_ENV - name: Install build dependencies (macOS) if: runner.os == 'macOS' run: | @@ -86,7 +86,7 @@ jobs: # Point cross to our config file echo "CROSS_CONFIG=${{ github.workspace }}/native/cozodb/Cross.toml" >> $GITHUB_ENV # RocksDB requires C++20 - echo "CXXFLAGS=-std=c++20" >> $GITHUB_ENV + echo "CXXFLAGS=-std=c++2a" >> $GITHUB_ENV # Debug: show Cross.toml content echo "Cross.toml content:" cat ${{ github.workspace }}/native/cozodb/Cross.toml diff --git a/Cross.toml b/Cross.toml index 7e6e334..7c23851 100644 --- a/Cross.toml +++ b/Cross.toml @@ -14,7 +14,7 @@ passthrough = [ ] # RocksDB requires C++20 variables = [ - ["CXXFLAGS", "-std=c++20"], + ["CXXFLAGS", "-std=c++2a"], ] # ============================================================================== diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 7e6e334..7c23851 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -14,7 +14,7 @@ passthrough = [ ] # RocksDB requires C++20 variables = [ - ["CXXFLAGS", "-std=c++20"], + ["CXXFLAGS", "-std=c++2a"], ] # ============================================================================== From 1fc5dc384e9ff9376159e8ce6ffc55b47ac10e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 18:02:47 +0100 Subject: [PATCH 13/31] disable cross targets with old GCC (C++20 incompatible) RocksDB requires C++20 features (defaulted comparisons, using enum). The cross-compilation containers have GCC 9 which doesn't support these. Working targets (4): - aarch64-apple-darwin - x86_64-apple-darwin - x86_64-unknown-linux-gnu - riscv64gc-unknown-linux-gnu Disabled targets (need custom cross images with GCC 10+): - arm-unknown-linux-gnueabihf - aarch64-unknown-linux-gnu - aarch64-unknown-linux-musl - x86_64-unknown-linux-musl Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 31 ++++++++++++++++++------------- Cross.toml | 12 ++++++++---- native/cozodb/Cross.toml | 12 ++++++++---- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d5319f..04fbf3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,27 +21,32 @@ jobs: nif: - '2.17' job: - - target: arm-unknown-linux-gnueabihf - os: ubuntu-24.04 - use-cross: true - - target: aarch64-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true - - target: aarch64-unknown-linux-musl - os: ubuntu-24.04 - use-cross: true + # macOS targets (native builds) - target: aarch64-apple-darwin os: macos-latest - - target: riscv64gc-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true - target: x86_64-apple-darwin os: macos-latest + # Linux native build - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 - - target: x86_64-unknown-linux-musl + # Linux cross-compiled (riscv64 has newer GCC that supports C++20) + - target: riscv64gc-unknown-linux-gnu os: ubuntu-24.04 use-cross: true + # DISABLED: These targets have GCC too old for C++20 (RocksDB requirement) + # TODO: Create custom cross images with GCC 10+ to re-enable these + # - target: arm-unknown-linux-gnueabihf + # os: ubuntu-24.04 + # use-cross: true + # - target: aarch64-unknown-linux-gnu + # os: ubuntu-24.04 + # use-cross: true + # - target: aarch64-unknown-linux-musl + # os: ubuntu-24.04 + # use-cross: true + # - target: x86_64-unknown-linux-musl + # os: ubuntu-24.04 + # use-cross: true steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/Cross.toml b/Cross.toml index 7c23851..cee259f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -21,18 +21,22 @@ variables = [ # Linux GNU targets - use cross images with LLVM # ============================================================================== +# Note: These targets use older GCC cross-compilers that don't fully support C++20. +# We install clang and attempt to use it, but the cross-compilation toolchain +# may still use the default GCC. Some targets may fail due to C++20 incompatibility. + [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.arm-unknown-linux-gnueabihf] image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.riscv64gc-unknown-linux-gnu] @@ -50,12 +54,12 @@ pre-build = [ image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.aarch64-unknown-linux-musl] image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 7c23851..cee259f 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -21,18 +21,22 @@ variables = [ # Linux GNU targets - use cross images with LLVM # ============================================================================== +# Note: These targets use older GCC cross-compilers that don't fully support C++20. +# We install clang and attempt to use it, but the cross-compilation toolchain +# may still use the default GCC. Some targets may fail due to C++20 incompatibility. + [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.arm-unknown-linux-gnueabihf] image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.riscv64gc-unknown-linux-gnu] @@ -50,12 +54,12 @@ pre-build = [ image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] [target.aarch64-unknown-linux-musl] image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm", + "apt-get update && apt-get install -y libclang-dev clang llvm lld", ] From e1092daaac970bdb415d49f6e3be6c02b8e09dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 18:10:33 +0100 Subject: [PATCH 14/31] Add custom Docker images for cross-compilation with GCC 10+ - Create Dockerfiles for 4 targets with GCC 10+ for C++20 support: - aarch64-unknown-linux-gnu - arm-unknown-linux-gnueabihf - aarch64-unknown-linux-musl - x86_64-unknown-linux-musl - Add build-cross-images.yml workflow to build/push images to GHCR - Update Cross.toml to use custom images - Re-enable all cross-compilation targets in release.yml - Add fix/** branches to release workflow trigger RocksDB requires C++20 features (defaulted comparisons) that older GCC cross-compilers don't support. These custom images provide GCC 10 which fully supports -std=c++2a. Co-Authored-By: Claude Opus 4.5 --- .cross/Dockerfile.aarch64-unknown-linux-gnu | 27 +++++++ .cross/Dockerfile.aarch64-unknown-linux-musl | 27 +++++++ .cross/Dockerfile.arm-unknown-linux-gnueabihf | 27 +++++++ .cross/Dockerfile.x86_64-unknown-linux-musl | 25 +++++++ .github/workflows/build-cross-images.yml | 73 +++++++++++++++++++ .github/workflows/release.yml | 31 ++++---- Cross.toml | 39 +++------- native/cozodb/Cross.toml | 39 +++------- 8 files changed, 219 insertions(+), 69 deletions(-) create mode 100644 .cross/Dockerfile.aarch64-unknown-linux-gnu create mode 100644 .cross/Dockerfile.aarch64-unknown-linux-musl create mode 100644 .cross/Dockerfile.arm-unknown-linux-gnueabihf create mode 100644 .cross/Dockerfile.x86_64-unknown-linux-musl create mode 100644 .github/workflows/build-cross-images.yml diff --git a/.cross/Dockerfile.aarch64-unknown-linux-gnu b/.cross/Dockerfile.aarch64-unknown-linux-gnu new file mode 100644 index 0000000..3d2c8f1 --- /dev/null +++ b/.cross/Dockerfile.aarch64-unknown-linux-gnu @@ -0,0 +1,27 @@ +FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + gcc-10-aarch64-linux-gnu \ + g++-10-aarch64-linux-gnu \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Set GCC 10 as default for cross-compilation + update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ + update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Verify GCC version +RUN aarch64-linux-gnu-g++ --version diff --git a/.cross/Dockerfile.aarch64-unknown-linux-musl b/.cross/Dockerfile.aarch64-unknown-linux-musl new file mode 100644 index 0000000..e380970 --- /dev/null +++ b/.cross/Dockerfile.aarch64-unknown-linux-musl @@ -0,0 +1,27 @@ +FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +# Note: musl targets may not have cross-gcc packages, so we install host gcc +# and rely on the existing musl cross toolchain +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Set GCC 10 as default host compiler + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Verify GCC version +RUN g++ --version diff --git a/.cross/Dockerfile.arm-unknown-linux-gnueabihf b/.cross/Dockerfile.arm-unknown-linux-gnueabihf new file mode 100644 index 0000000..afbd552 --- /dev/null +++ b/.cross/Dockerfile.arm-unknown-linux-gnueabihf @@ -0,0 +1,27 @@ +FROM ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + gcc-10-arm-linux-gnueabihf \ + g++-10-arm-linux-gnueabihf \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Set GCC 10 as default for cross-compilation + update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 100 && \ + update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 100 && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Verify GCC version +RUN arm-linux-gnueabihf-g++ --version diff --git a/.cross/Dockerfile.x86_64-unknown-linux-musl b/.cross/Dockerfile.x86_64-unknown-linux-musl new file mode 100644 index 0000000..6d640bb --- /dev/null +++ b/.cross/Dockerfile.x86_64-unknown-linux-musl @@ -0,0 +1,25 @@ +FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:main + +ENV DEBIAN_FRONTEND=noninteractive + +# Install newer GCC (10+) for C++20 support required by RocksDB +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + gcc-10 \ + g++-10 \ + libclang-dev \ + clang \ + llvm \ + lld && \ + # Set GCC 10 as default host compiler + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Verify GCC version +RUN g++ --version diff --git a/.github/workflows/build-cross-images.yml b/.github/workflows/build-cross-images.yml new file mode 100644 index 0000000..4dafa75 --- /dev/null +++ b/.github/workflows/build-cross-images.yml @@ -0,0 +1,73 @@ +name: Build Cross-Compilation Images + +on: + push: + paths: + - '.cross/Dockerfile.*' + - '.github/workflows/build-cross-images.yml' + branches: + - main + - 'fix/**' + workflow_dispatch: + +concurrency: + group: build-cross-images-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }}/cozodb-cross + +jobs: + build-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-unknown-linux-gnu + dockerfile: .cross/Dockerfile.aarch64-unknown-linux-gnu + - target: arm-unknown-linux-gnueabihf + dockerfile: .cross/Dockerfile.arm-unknown-linux-gnueabihf + - target: aarch64-unknown-linux-musl + dockerfile: .cross/Dockerfile.aarch64-unknown-linux-musl + - target: x86_64-unknown-linux-musl + dockerfile: .cross/Dockerfile.x86_64-unknown-linux-musl + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.target }} + tags: | + type=raw,value=latest + type=sha,prefix= + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 04fbf3a..e9a5e58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,7 @@ name: Build precompiled NIFs push: branches: - main + - 'fix/**' tags: - '*' workflow_dispatch: @@ -29,24 +30,24 @@ jobs: # Linux native build - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 - # Linux cross-compiled (riscv64 has newer GCC that supports C++20) + # Linux cross-compiled targets + # Note: These require custom Docker images with GCC 10+ for C++20 support + # The images are built by the build-cross-images.yml workflow - target: riscv64gc-unknown-linux-gnu os: ubuntu-24.04 use-cross: true - # DISABLED: These targets have GCC too old for C++20 (RocksDB requirement) - # TODO: Create custom cross images with GCC 10+ to re-enable these - # - target: arm-unknown-linux-gnueabihf - # os: ubuntu-24.04 - # use-cross: true - # - target: aarch64-unknown-linux-gnu - # os: ubuntu-24.04 - # use-cross: true - # - target: aarch64-unknown-linux-musl - # os: ubuntu-24.04 - # use-cross: true - # - target: x86_64-unknown-linux-musl - # os: ubuntu-24.04 - # use-cross: true + - target: arm-unknown-linux-gnueabihf + os: ubuntu-24.04 + use-cross: true + - target: aarch64-unknown-linux-gnu + os: ubuntu-24.04 + use-cross: true + - target: aarch64-unknown-linux-musl + os: ubuntu-24.04 + use-cross: true + - target: x86_64-unknown-linux-musl + os: ubuntu-24.04 + use-cross: true steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/Cross.toml b/Cross.toml index cee259f..1612bbc 100644 --- a/Cross.toml +++ b/Cross.toml @@ -2,7 +2,7 @@ # See: https://github.com/cross-rs/cross # # The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# We use custom images that have LLVM pre-installed. +# We use custom images that have LLVM and GCC 10+ pre-installed for C++20 support. [build.env] passthrough = [ @@ -18,28 +18,19 @@ variables = [ ] # ============================================================================== -# Linux GNU targets - use cross images with LLVM +# Linux GNU targets - custom images with GCC 10+ for C++20 support # ============================================================================== -# Note: These targets use older GCC cross-compilers that don't fully support C++20. -# We install clang and attempt to use it, but the cross-compilation toolchain -# may still use the default GCC. Some targets may fail due to C++20 incompatibility. - [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 cross-compiler for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-gnu:latest" [target.arm-unknown-linux-gnueabihf] -image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 cross-compiler for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-arm-unknown-linux-gnueabihf:latest" [target.riscv64gc-unknown-linux-gnu] +# riscv64 uses newer GCC that already supports C++20, use standard image image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", @@ -47,19 +38,13 @@ pre-build = [ ] # ============================================================================== -# Linux MUSL targets +# Linux MUSL targets - custom images with GCC 10+ for C++20 support # ============================================================================== [target.x86_64-unknown-linux-musl] -image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-x86_64-unknown-linux-musl:latest" [target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-musl:latest" diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index cee259f..1612bbc 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -2,7 +2,7 @@ # See: https://github.com/cross-rs/cross # # The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# We use custom images that have LLVM pre-installed. +# We use custom images that have LLVM and GCC 10+ pre-installed for C++20 support. [build.env] passthrough = [ @@ -18,28 +18,19 @@ variables = [ ] # ============================================================================== -# Linux GNU targets - use cross images with LLVM +# Linux GNU targets - custom images with GCC 10+ for C++20 support # ============================================================================== -# Note: These targets use older GCC cross-compilers that don't fully support C++20. -# We install clang and attempt to use it, but the cross-compilation toolchain -# may still use the default GCC. Some targets may fail due to C++20 incompatibility. - [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 cross-compiler for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-gnu:latest" [target.arm-unknown-linux-gnueabihf] -image = "ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 cross-compiler for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-arm-unknown-linux-gnueabihf:latest" [target.riscv64gc-unknown-linux-gnu] +# riscv64 uses newer GCC that already supports C++20, use standard image image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ "export DEBIAN_FRONTEND=noninteractive", @@ -47,19 +38,13 @@ pre-build = [ ] # ============================================================================== -# Linux MUSL targets +# Linux MUSL targets - custom images with GCC 10+ for C++20 support # ============================================================================== [target.x86_64-unknown-linux-musl] -image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-x86_64-unknown-linux-musl:latest" [target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" -pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update && apt-get install -y libclang-dev clang llvm lld", -] +# Custom image with GCC 10 for C++20 support +image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-musl:latest" From f7c79b71ada77aaa81094340e3e5396ecefb5b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 19:26:35 +0100 Subject: [PATCH 15/31] Fix cross-compiler wrapper scripts for GCC 10 The cross-rs images use crosstool-ng compilers with target-prefixed names like arm-unknown-linux-gnueabihf-g++, but we were only updating the Debian-style names like arm-linux-gnueabihf-g++. This fix creates wrapper scripts in /usr/local/bin that: - Match the exact target triplet names used by cross-rs - Delegate to the GCC 10 cross-compilers - Are added first in PATH to take precedence Co-Authored-By: Claude Opus 4.5 --- .cross/Dockerfile.aarch64-unknown-linux-gnu | 20 ++++++++++++---- .cross/Dockerfile.aarch64-unknown-linux-musl | 23 +++++++++++++------ .cross/Dockerfile.arm-unknown-linux-gnueabihf | 20 ++++++++++++---- .cross/Dockerfile.x86_64-unknown-linux-musl | 19 +++++++++++---- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/.cross/Dockerfile.aarch64-unknown-linux-gnu b/.cross/Dockerfile.aarch64-unknown-linux-gnu index 3d2c8f1..f460e97 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-gnu +++ b/.cross/Dockerfile.aarch64-unknown-linux-gnu @@ -3,6 +3,8 @@ FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB +# The cross-rs images use crosstool-ng compilers in /x-tools/ +# We install Debian's cross-gcc-10 and symlink it to override the old compiler RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -16,12 +18,20 @@ RUN apt-get update && \ clang \ llvm \ lld && \ - # Set GCC 10 as default for cross-compilation - update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ - update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ # Clean up apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Verify GCC version -RUN aarch64-linux-gnu-g++ --version +# The cross-rs image uses crosstool-ng with aarch64-unknown-linux-gnu- prefix +# We need to create wrapper scripts that use the GCC 10 cross-compiler +RUN mkdir -p /usr/local/bin && \ + echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-gcc-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-gnu-gcc && \ + echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-g++-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-gnu-g++ && \ + chmod +x /usr/local/bin/aarch64-unknown-linux-gnu-gcc && \ + chmod +x /usr/local/bin/aarch64-unknown-linux-gnu-g++ + +# Ensure /usr/local/bin is first in PATH +ENV PATH="/usr/local/bin:${PATH}" + +# Verify the wrapper works +RUN aarch64-unknown-linux-gnu-g++ --version diff --git a/.cross/Dockerfile.aarch64-unknown-linux-musl b/.cross/Dockerfile.aarch64-unknown-linux-musl index e380970..2d7c87c 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-musl +++ b/.cross/Dockerfile.aarch64-unknown-linux-musl @@ -3,8 +3,7 @@ FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB -# Note: musl targets may not have cross-gcc packages, so we install host gcc -# and rely on the existing musl cross toolchain +# For aarch64-musl, we need aarch64 cross-compiler with C++20 support RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -12,16 +11,26 @@ RUN apt-get update && \ apt-get install -y \ gcc-10 \ g++-10 \ + gcc-10-aarch64-linux-gnu \ + g++-10-aarch64-linux-gnu \ libclang-dev \ clang \ llvm \ lld && \ - # Set GCC 10 as default host compiler - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ # Clean up apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Verify GCC version -RUN g++ --version +# The cross-rs musl image uses aarch64-unknown-linux-musl- prefix +# We create wrapper scripts that use GCC 10 aarch64 cross-compiler +RUN mkdir -p /usr/local/bin && \ + echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-gcc-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-musl-gcc && \ + echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-g++-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-musl-g++ && \ + chmod +x /usr/local/bin/aarch64-unknown-linux-musl-gcc && \ + chmod +x /usr/local/bin/aarch64-unknown-linux-musl-g++ + +# Ensure /usr/local/bin is first in PATH +ENV PATH="/usr/local/bin:${PATH}" + +# Verify the wrapper works +RUN aarch64-unknown-linux-musl-g++ --version diff --git a/.cross/Dockerfile.arm-unknown-linux-gnueabihf b/.cross/Dockerfile.arm-unknown-linux-gnueabihf index afbd552..a8df727 100644 --- a/.cross/Dockerfile.arm-unknown-linux-gnueabihf +++ b/.cross/Dockerfile.arm-unknown-linux-gnueabihf @@ -3,6 +3,8 @@ FROM ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB +# The cross-rs images use crosstool-ng compilers in /x-tools/ +# We install Debian's cross-gcc-10 and symlink it to override the old compiler RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -16,12 +18,20 @@ RUN apt-get update && \ clang \ llvm \ lld && \ - # Set GCC 10 as default for cross-compilation - update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 100 && \ - update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 100 && \ # Clean up apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Verify GCC version -RUN arm-linux-gnueabihf-g++ --version +# The cross-rs image uses crosstool-ng with arm-unknown-linux-gnueabihf- prefix +# We need to create wrapper scripts that use the GCC 10 cross-compiler +RUN mkdir -p /usr/local/bin && \ + echo '#!/bin/bash\nexec /usr/bin/arm-linux-gnueabihf-gcc-10 "$@"' > /usr/local/bin/arm-unknown-linux-gnueabihf-gcc && \ + echo '#!/bin/bash\nexec /usr/bin/arm-linux-gnueabihf-g++-10 "$@"' > /usr/local/bin/arm-unknown-linux-gnueabihf-g++ && \ + chmod +x /usr/local/bin/arm-unknown-linux-gnueabihf-gcc && \ + chmod +x /usr/local/bin/arm-unknown-linux-gnueabihf-g++ + +# Ensure /usr/local/bin is first in PATH +ENV PATH="/usr/local/bin:${PATH}" + +# Verify the wrapper works +RUN arm-unknown-linux-gnueabihf-g++ --version diff --git a/.cross/Dockerfile.x86_64-unknown-linux-musl b/.cross/Dockerfile.x86_64-unknown-linux-musl index 6d640bb..1ac5f5b 100644 --- a/.cross/Dockerfile.x86_64-unknown-linux-musl +++ b/.cross/Dockerfile.x86_64-unknown-linux-musl @@ -3,6 +3,8 @@ FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB +# For musl targets, we install host GCC 10 for C++ compilation +# The musl libc linking is handled by the existing musl toolchain RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -14,12 +16,21 @@ RUN apt-get update && \ clang \ llvm \ lld && \ - # Set GCC 10 as default host compiler - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ # Clean up apt-get clean && \ rm -rf /var/lib/apt/lists/* +# The cross-rs musl image uses x86_64-unknown-linux-musl- prefix +# We need to create wrapper scripts that use GCC 10 for C++ compilation +# but preserve the musl linking +RUN mkdir -p /usr/local/bin && \ + echo '#!/bin/bash\nexec /usr/bin/gcc-10 "$@"' > /usr/local/bin/x86_64-unknown-linux-musl-gcc && \ + echo '#!/bin/bash\nexec /usr/bin/g++-10 "$@"' > /usr/local/bin/x86_64-unknown-linux-musl-g++ && \ + chmod +x /usr/local/bin/x86_64-unknown-linux-musl-gcc && \ + chmod +x /usr/local/bin/x86_64-unknown-linux-musl-g++ + +# Ensure /usr/local/bin is first in PATH +ENV PATH="/usr/local/bin:${PATH}" + # Verify GCC version -RUN g++ --version +RUN g++-10 --version From d01fa1c367d5fe81961c500b8f34a7a740b11a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 19:44:59 +0100 Subject: [PATCH 16/31] Fix Dockerfiles to use update-alternatives for Debian compiler names The cc crate sets CXX_=, e.g.: CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ The previous fix created wrapper scripts with Rust triplet names but the cc crate bypasses those by directly using the Debian-style names. This fix uses update-alternatives to make the Debian compiler commands (aarch64-linux-gnu-g++, arm-linux-gnueabihf-g++, etc.) point to GCC 10. Co-Authored-By: Claude Opus 4.5 --- .cross/Dockerfile.aarch64-unknown-linux-gnu | 22 +++++------- .cross/Dockerfile.aarch64-unknown-linux-musl | 35 ++++++++++++------- .cross/Dockerfile.arm-unknown-linux-gnueabihf | 22 +++++------- .cross/Dockerfile.x86_64-unknown-linux-musl | 29 ++++++++------- 4 files changed, 55 insertions(+), 53 deletions(-) diff --git a/.cross/Dockerfile.aarch64-unknown-linux-gnu b/.cross/Dockerfile.aarch64-unknown-linux-gnu index f460e97..ec5fea8 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-gnu +++ b/.cross/Dockerfile.aarch64-unknown-linux-gnu @@ -3,8 +3,6 @@ FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB -# The cross-rs images use crosstool-ng compilers in /x-tools/ -# We install Debian's cross-gcc-10 and symlink it to override the old compiler RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -22,16 +20,12 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cross-rs image uses crosstool-ng with aarch64-unknown-linux-gnu- prefix -# We need to create wrapper scripts that use the GCC 10 cross-compiler -RUN mkdir -p /usr/local/bin && \ - echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-gcc-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-gnu-gcc && \ - echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-g++-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-gnu-g++ && \ - chmod +x /usr/local/bin/aarch64-unknown-linux-gnu-gcc && \ - chmod +x /usr/local/bin/aarch64-unknown-linux-gnu-g++ +# The cc crate uses CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ +# We need to make sure aarch64-linux-gnu-g++ points to GCC 10 +RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ + update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ + update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ + update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 -# Ensure /usr/local/bin is first in PATH -ENV PATH="/usr/local/bin:${PATH}" - -# Verify the wrapper works -RUN aarch64-unknown-linux-gnu-g++ --version +# Verify GCC version (should show GCC 10) +RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.aarch64-unknown-linux-musl b/.cross/Dockerfile.aarch64-unknown-linux-musl index 2d7c87c..90414d5 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-musl +++ b/.cross/Dockerfile.aarch64-unknown-linux-musl @@ -3,7 +3,6 @@ FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB -# For aarch64-musl, we need aarch64 cross-compiler with C++20 support RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -21,16 +20,28 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cross-rs musl image uses aarch64-unknown-linux-musl- prefix -# We create wrapper scripts that use GCC 10 aarch64 cross-compiler -RUN mkdir -p /usr/local/bin && \ - echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-gcc-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-musl-gcc && \ - echo '#!/bin/bash\nexec /usr/bin/aarch64-linux-gnu-g++-10 "$@"' > /usr/local/bin/aarch64-unknown-linux-musl-g++ && \ - chmod +x /usr/local/bin/aarch64-unknown-linux-musl-gcc && \ - chmod +x /usr/local/bin/aarch64-unknown-linux-musl-g++ +# Set GCC 10 as default host compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 -# Ensure /usr/local/bin is first in PATH -ENV PATH="/usr/local/bin:${PATH}" +# Set GCC 10 as default aarch64 cross-compiler +RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ + update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ + update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ + update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 -# Verify the wrapper works -RUN aarch64-unknown-linux-musl-g++ --version +# For musl targets, the cross-rs image may use aarch64-linux-musl-g++ +# Create symlinks if the musl compiler wrapper exists +RUN if [ -f /usr/bin/aarch64-linux-musl-g++ ]; then \ + mv /usr/bin/aarch64-linux-musl-g++ /usr/bin/aarch64-linux-musl-g++.old && \ + ln -s /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-linux-musl-g++; \ + fi && \ + if [ -f /usr/bin/aarch64-linux-musl-gcc ]; then \ + mv /usr/bin/aarch64-linux-musl-gcc /usr/bin/aarch64-linux-musl-gcc.old && \ + ln -s /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-linux-musl-gcc; \ + fi + +# Verify GCC version +RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.arm-unknown-linux-gnueabihf b/.cross/Dockerfile.arm-unknown-linux-gnueabihf index a8df727..b1e0704 100644 --- a/.cross/Dockerfile.arm-unknown-linux-gnueabihf +++ b/.cross/Dockerfile.arm-unknown-linux-gnueabihf @@ -3,8 +3,6 @@ FROM ghcr.io/cross-rs/arm-unknown-linux-gnueabihf:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB -# The cross-rs images use crosstool-ng compilers in /x-tools/ -# We install Debian's cross-gcc-10 and symlink it to override the old compiler RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -22,16 +20,12 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cross-rs image uses crosstool-ng with arm-unknown-linux-gnueabihf- prefix -# We need to create wrapper scripts that use the GCC 10 cross-compiler -RUN mkdir -p /usr/local/bin && \ - echo '#!/bin/bash\nexec /usr/bin/arm-linux-gnueabihf-gcc-10 "$@"' > /usr/local/bin/arm-unknown-linux-gnueabihf-gcc && \ - echo '#!/bin/bash\nexec /usr/bin/arm-linux-gnueabihf-g++-10 "$@"' > /usr/local/bin/arm-unknown-linux-gnueabihf-g++ && \ - chmod +x /usr/local/bin/arm-unknown-linux-gnueabihf-gcc && \ - chmod +x /usr/local/bin/arm-unknown-linux-gnueabihf-g++ +# The cc crate uses CXX_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-g++ +# We need to make sure arm-linux-gnueabihf-g++ points to GCC 10 +RUN update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 100 && \ + update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 100 && \ + update-alternatives --set arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 && \ + update-alternatives --set arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 -# Ensure /usr/local/bin is first in PATH -ENV PATH="/usr/local/bin:${PATH}" - -# Verify the wrapper works -RUN arm-unknown-linux-gnueabihf-g++ --version +# Verify GCC version (should show GCC 10) +RUN arm-linux-gnueabihf-g++ --version | head -1 diff --git a/.cross/Dockerfile.x86_64-unknown-linux-musl b/.cross/Dockerfile.x86_64-unknown-linux-musl index 1ac5f5b..af05250 100644 --- a/.cross/Dockerfile.x86_64-unknown-linux-musl +++ b/.cross/Dockerfile.x86_64-unknown-linux-musl @@ -3,8 +3,6 @@ FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:main ENV DEBIAN_FRONTEND=noninteractive # Install newer GCC (10+) for C++20 support required by RocksDB -# For musl targets, we install host GCC 10 for C++ compilation -# The musl libc linking is handled by the existing musl toolchain RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ @@ -20,17 +18,22 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cross-rs musl image uses x86_64-unknown-linux-musl- prefix -# We need to create wrapper scripts that use GCC 10 for C++ compilation -# but preserve the musl linking -RUN mkdir -p /usr/local/bin && \ - echo '#!/bin/bash\nexec /usr/bin/gcc-10 "$@"' > /usr/local/bin/x86_64-unknown-linux-musl-gcc && \ - echo '#!/bin/bash\nexec /usr/bin/g++-10 "$@"' > /usr/local/bin/x86_64-unknown-linux-musl-g++ && \ - chmod +x /usr/local/bin/x86_64-unknown-linux-musl-gcc && \ - chmod +x /usr/local/bin/x86_64-unknown-linux-musl-g++ +# Set GCC 10 as default host compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 -# Ensure /usr/local/bin is first in PATH -ENV PATH="/usr/local/bin:${PATH}" +# For musl targets, the cross-rs image uses x86_64-linux-musl-g++ +# Create symlinks if the musl compiler wrapper exists +RUN if [ -f /usr/bin/x86_64-linux-musl-g++ ]; then \ + mv /usr/bin/x86_64-linux-musl-g++ /usr/bin/x86_64-linux-musl-g++.old && \ + ln -s /usr/bin/g++-10 /usr/bin/x86_64-linux-musl-g++; \ + fi && \ + if [ -f /usr/bin/x86_64-linux-musl-gcc ]; then \ + mv /usr/bin/x86_64-linux-musl-gcc /usr/bin/x86_64-linux-musl-gcc.old && \ + ln -s /usr/bin/gcc-10 /usr/bin/x86_64-linux-musl-gcc; \ + fi # Verify GCC version -RUN g++-10 --version +RUN g++ --version | head -1 From f8ab501342981763f956ebc5546b5584fbc2c594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 20:36:23 +0100 Subject: [PATCH 17/31] Handle both Debian and Rust triplet compiler names in Dockerfiles The cross-rs images use different naming conventions per target: - aarch64-unknown-linux-gnu: uses aarch64-linux-gnu-g++ (Debian) - arm-unknown-linux-gnueabihf: uses arm-unknown-linux-gnueabihf-g++ (Rust triplet) This fix: 1. Sets up update-alternatives for Debian-style names 2. Creates symlinks for Rust triplet names 3. Overrides compilers in /x-tools (crosstool-ng) if present All variants now point to GCC 10 for C++20 support. Co-Authored-By: Claude Opus 4.5 --- .cross/Dockerfile.aarch64-unknown-linux-gnu | 23 ++++++++++-- .cross/Dockerfile.aarch64-unknown-linux-musl | 35 ++++++++++++++----- .cross/Dockerfile.arm-unknown-linux-gnueabihf | 26 ++++++++++++-- .cross/Dockerfile.x86_64-unknown-linux-musl | 33 +++++++++++++---- 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/.cross/Dockerfile.aarch64-unknown-linux-gnu b/.cross/Dockerfile.aarch64-unknown-linux-gnu index ec5fea8..b5b59c9 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-gnu +++ b/.cross/Dockerfile.aarch64-unknown-linux-gnu @@ -20,12 +20,31 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cc crate uses CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ -# We need to make sure aarch64-linux-gnu-g++ points to GCC 10 +# Set GCC 10 as default for Debian-style compiler names RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 +# Also handle Rust triplet names (aarch64-unknown-linux-gnu-g++) if present +RUN if [ -f /usr/bin/aarch64-unknown-linux-gnu-g++ ]; then \ + mv /usr/bin/aarch64-unknown-linux-gnu-g++ /usr/bin/aarch64-unknown-linux-gnu-g++.old; \ + fi && \ + if [ -f /usr/bin/aarch64-unknown-linux-gnu-gcc ]; then \ + mv /usr/bin/aarch64-unknown-linux-gnu-gcc /usr/bin/aarch64-unknown-linux-gnu-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-unknown-linux-gnu-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-unknown-linux-gnu-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "aarch64-unknown-linux-gnu-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "aarch64-unknown-linux-gnu-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 "$f"; \ + done || true + # Verify GCC version (should show GCC 10) RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.aarch64-unknown-linux-musl b/.cross/Dockerfile.aarch64-unknown-linux-musl index 90414d5..4f05c19 100644 --- a/.cross/Dockerfile.aarch64-unknown-linux-musl +++ b/.cross/Dockerfile.aarch64-unknown-linux-musl @@ -26,22 +26,41 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ update-alternatives --set gcc /usr/bin/gcc-10 && \ update-alternatives --set g++ /usr/bin/g++-10 -# Set GCC 10 as default aarch64 cross-compiler +# Set GCC 10 as default aarch64 cross-compiler (Debian-style names) RUN update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 100 && \ update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 100 && \ update-alternatives --set aarch64-linux-gnu-gcc /usr/bin/aarch64-linux-gnu-gcc-10 && \ update-alternatives --set aarch64-linux-gnu-g++ /usr/bin/aarch64-linux-gnu-g++-10 -# For musl targets, the cross-rs image may use aarch64-linux-musl-g++ -# Create symlinks if the musl compiler wrapper exists +# Handle musl compiler names - both Debian-style and Rust triplet style RUN if [ -f /usr/bin/aarch64-linux-musl-g++ ]; then \ - mv /usr/bin/aarch64-linux-musl-g++ /usr/bin/aarch64-linux-musl-g++.old && \ - ln -s /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-linux-musl-g++; \ + mv /usr/bin/aarch64-linux-musl-g++ /usr/bin/aarch64-linux-musl-g++.old; \ fi && \ if [ -f /usr/bin/aarch64-linux-musl-gcc ]; then \ - mv /usr/bin/aarch64-linux-musl-gcc /usr/bin/aarch64-linux-musl-gcc.old && \ - ln -s /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-linux-musl-gcc; \ - fi + mv /usr/bin/aarch64-linux-musl-gcc /usr/bin/aarch64-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-linux-musl-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-linux-musl-gcc + +# Handle Rust triplet names +RUN if [ -f /usr/bin/aarch64-unknown-linux-musl-g++ ]; then \ + mv /usr/bin/aarch64-unknown-linux-musl-g++ /usr/bin/aarch64-unknown-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/aarch64-unknown-linux-musl-gcc ]; then \ + mv /usr/bin/aarch64-unknown-linux-musl-gcc /usr/bin/aarch64-unknown-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 /usr/bin/aarch64-unknown-linux-musl-g++ && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-unknown-linux-musl-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "aarch64-unknown-linux-musl-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "aarch64-unknown-linux-musl-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 "$f"; \ + done || true # Verify GCC version RUN aarch64-linux-gnu-g++ --version | head -1 diff --git a/.cross/Dockerfile.arm-unknown-linux-gnueabihf b/.cross/Dockerfile.arm-unknown-linux-gnueabihf index b1e0704..88a5f54 100644 --- a/.cross/Dockerfile.arm-unknown-linux-gnueabihf +++ b/.cross/Dockerfile.arm-unknown-linux-gnueabihf @@ -20,12 +20,34 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The cc crate uses CXX_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-g++ -# We need to make sure arm-linux-gnueabihf-g++ points to GCC 10 +# Set GCC 10 as default for Debian-style compiler names RUN update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 100 && \ update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 100 && \ update-alternatives --set arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-10 && \ update-alternatives --set arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++-10 +# The cross-rs image for arm uses Rust triplet names (arm-unknown-linux-gnueabihf-g++) +# which come from crosstool-ng. We need to replace those with GCC 10 wrappers. +# First, backup the old compilers, then create symlinks to GCC 10 +RUN if [ -f /usr/bin/arm-unknown-linux-gnueabihf-g++ ]; then \ + mv /usr/bin/arm-unknown-linux-gnueabihf-g++ /usr/bin/arm-unknown-linux-gnueabihf-g++.old; \ + fi && \ + if [ -f /usr/bin/arm-unknown-linux-gnueabihf-gcc ]; then \ + mv /usr/bin/arm-unknown-linux-gnueabihf-gcc /usr/bin/arm-unknown-linux-gnueabihf-gcc.old; \ + fi && \ + ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-unknown-linux-gnueabihf-g++ && \ + ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-unknown-linux-gnueabihf-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "arm-unknown-linux-gnueabihf-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 "$f"; \ + done || true && \ + find /x-tools -name "arm-unknown-linux-gnueabihf-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 "$f"; \ + done || true + # Verify GCC version (should show GCC 10) RUN arm-linux-gnueabihf-g++ --version | head -1 +RUN which arm-unknown-linux-gnueabihf-g++ && arm-unknown-linux-gnueabihf-g++ --version | head -1 || echo "arm-unknown-linux-gnueabihf-g++ not in PATH" diff --git a/.cross/Dockerfile.x86_64-unknown-linux-musl b/.cross/Dockerfile.x86_64-unknown-linux-musl index af05250..6663c72 100644 --- a/.cross/Dockerfile.x86_64-unknown-linux-musl +++ b/.cross/Dockerfile.x86_64-unknown-linux-musl @@ -24,16 +24,35 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ update-alternatives --set gcc /usr/bin/gcc-10 && \ update-alternatives --set g++ /usr/bin/g++-10 -# For musl targets, the cross-rs image uses x86_64-linux-musl-g++ -# Create symlinks if the musl compiler wrapper exists +# Handle musl compiler names - both Debian-style and Rust triplet style RUN if [ -f /usr/bin/x86_64-linux-musl-g++ ]; then \ - mv /usr/bin/x86_64-linux-musl-g++ /usr/bin/x86_64-linux-musl-g++.old && \ - ln -s /usr/bin/g++-10 /usr/bin/x86_64-linux-musl-g++; \ + mv /usr/bin/x86_64-linux-musl-g++ /usr/bin/x86_64-linux-musl-g++.old; \ fi && \ if [ -f /usr/bin/x86_64-linux-musl-gcc ]; then \ - mv /usr/bin/x86_64-linux-musl-gcc /usr/bin/x86_64-linux-musl-gcc.old && \ - ln -s /usr/bin/gcc-10 /usr/bin/x86_64-linux-musl-gcc; \ - fi + mv /usr/bin/x86_64-linux-musl-gcc /usr/bin/x86_64-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/g++-10 /usr/bin/x86_64-linux-musl-g++ && \ + ln -sf /usr/bin/gcc-10 /usr/bin/x86_64-linux-musl-gcc + +# Handle Rust triplet names +RUN if [ -f /usr/bin/x86_64-unknown-linux-musl-g++ ]; then \ + mv /usr/bin/x86_64-unknown-linux-musl-g++ /usr/bin/x86_64-unknown-linux-musl-g++.old; \ + fi && \ + if [ -f /usr/bin/x86_64-unknown-linux-musl-gcc ]; then \ + mv /usr/bin/x86_64-unknown-linux-musl-gcc /usr/bin/x86_64-unknown-linux-musl-gcc.old; \ + fi && \ + ln -sf /usr/bin/g++-10 /usr/bin/x86_64-unknown-linux-musl-g++ && \ + ln -sf /usr/bin/gcc-10 /usr/bin/x86_64-unknown-linux-musl-gcc + +# Also check /x-tools for crosstool-ng compilers and override those too +RUN find /x-tools -name "x86_64-unknown-linux-musl-g++" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/g++-10 "$f"; \ + done || true && \ + find /x-tools -name "x86_64-unknown-linux-musl-gcc" -type f 2>/dev/null | while read f; do \ + mv "$f" "$f.old" && \ + ln -sf /usr/bin/gcc-10 "$f"; \ + done || true # Verify GCC version RUN g++ --version | head -1 From a1d2ad6944a73268bc30eca9b9a16e07b37cd793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 20:48:36 +0100 Subject: [PATCH 18/31] Simplify: use clang for ARM cross-compilation, reduce targets - Use standard cross-rs image with clang instead of custom Docker images - Force clang as C++ compiler via CXX_aarch64_unknown_linux_gnu env var - Reduce targets to 4 for faster iteration: - macOS aarch64 (working) - macOS x86_64 (working) - Linux x86_64 native (working) - Linux aarch64 cross (testing clang) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 20 +++------------- Cross.toml | 45 +++++++++++++---------------------- native/cozodb/Cross.toml | 45 +++++++++++++---------------------- 3 files changed, 35 insertions(+), 75 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9a5e58..345b9df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,32 +22,18 @@ jobs: nif: - '2.17' job: - # macOS targets (native builds) + # macOS targets (native builds) - WORKING - target: aarch64-apple-darwin os: macos-latest - target: x86_64-apple-darwin os: macos-latest - # Linux native build + # Linux x86_64 native build - WORKING - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 - # Linux cross-compiled targets - # Note: These require custom Docker images with GCC 10+ for C++20 support - # The images are built by the build-cross-images.yml workflow - - target: riscv64gc-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true - - target: arm-unknown-linux-gnueabihf - os: ubuntu-24.04 - use-cross: true + # Linux ARM64 cross-compiled - testing clang approach - target: aarch64-unknown-linux-gnu os: ubuntu-24.04 use-cross: true - - target: aarch64-unknown-linux-musl - os: ubuntu-24.04 - use-cross: true - - target: x86_64-unknown-linux-musl - os: ubuntu-24.04 - use-cross: true steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/Cross.toml b/Cross.toml index 1612bbc..79d002f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,8 +1,5 @@ # Cross compilation configuration for cozodb NIF # See: https://github.com/cross-rs/cross -# -# The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# We use custom images that have LLVM and GCC 10+ pre-installed for C++20 support. [build.env] passthrough = [ @@ -10,41 +7,31 @@ passthrough = [ "LIBCLANG_PATH", "LLVM_CONFIG_PATH", "BINDGEN_EXTRA_CLANG_ARGS", - "CXXFLAGS", -] -# RocksDB requires C++20 -variables = [ - ["CXXFLAGS", "-std=c++2a"], ] # ============================================================================== -# Linux GNU targets - custom images with GCC 10+ for C++20 support +# aarch64-unknown-linux-gnu - use clang for C++20 support # ============================================================================== - [target.aarch64-unknown-linux-gnu] -# Custom image with GCC 10 cross-compiler for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-gnu:latest" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" +pre-build = [ + "apt-get update && apt-get install -y libclang-dev clang llvm lld", +] -[target.arm-unknown-linux-gnueabihf] -# Custom image with GCC 10 cross-compiler for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-arm-unknown-linux-gnueabihf:latest" +[target.aarch64-unknown-linux-gnu.env] +passthrough = ["RUST_BACKTRACE"] +volumes = [] +# Force clang as C++ compiler with C++20 and correct target +CXX_aarch64_unknown_linux_gnu = "clang++" +CC_aarch64_unknown_linux_gnu = "clang" +CXXFLAGS = "-std=c++20 --target=aarch64-linux-gnu" +CFLAGS = "--target=aarch64-linux-gnu" +# ============================================================================== +# riscv64 - already works with standard image +# ============================================================================== [target.riscv64gc-unknown-linux-gnu] -# riscv64 uses newer GCC that already supports C++20, use standard image image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", "apt-get update && apt-get install -y libclang-dev clang llvm", ] - -# ============================================================================== -# Linux MUSL targets - custom images with GCC 10+ for C++20 support -# ============================================================================== - -[target.x86_64-unknown-linux-musl] -# Custom image with GCC 10 for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-x86_64-unknown-linux-musl:latest" - -[target.aarch64-unknown-linux-musl] -# Custom image with GCC 10 for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-musl:latest" diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 1612bbc..79d002f 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -1,8 +1,5 @@ # Cross compilation configuration for cozodb NIF # See: https://github.com/cross-rs/cross -# -# The cross images ship with libclang 3.8.x, but bindgen requires 5.0+ -# We use custom images that have LLVM and GCC 10+ pre-installed for C++20 support. [build.env] passthrough = [ @@ -10,41 +7,31 @@ passthrough = [ "LIBCLANG_PATH", "LLVM_CONFIG_PATH", "BINDGEN_EXTRA_CLANG_ARGS", - "CXXFLAGS", -] -# RocksDB requires C++20 -variables = [ - ["CXXFLAGS", "-std=c++2a"], ] # ============================================================================== -# Linux GNU targets - custom images with GCC 10+ for C++20 support +# aarch64-unknown-linux-gnu - use clang for C++20 support # ============================================================================== - [target.aarch64-unknown-linux-gnu] -# Custom image with GCC 10 cross-compiler for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-gnu:latest" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" +pre-build = [ + "apt-get update && apt-get install -y libclang-dev clang llvm lld", +] -[target.arm-unknown-linux-gnueabihf] -# Custom image with GCC 10 cross-compiler for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-arm-unknown-linux-gnueabihf:latest" +[target.aarch64-unknown-linux-gnu.env] +passthrough = ["RUST_BACKTRACE"] +volumes = [] +# Force clang as C++ compiler with C++20 and correct target +CXX_aarch64_unknown_linux_gnu = "clang++" +CC_aarch64_unknown_linux_gnu = "clang" +CXXFLAGS = "-std=c++20 --target=aarch64-linux-gnu" +CFLAGS = "--target=aarch64-linux-gnu" +# ============================================================================== +# riscv64 - already works with standard image +# ============================================================================== [target.riscv64gc-unknown-linux-gnu] -# riscv64 uses newer GCC that already supports C++20, use standard image image = "ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu:main" pre-build = [ - "export DEBIAN_FRONTEND=noninteractive", "apt-get update && apt-get install -y libclang-dev clang llvm", ] - -# ============================================================================== -# Linux MUSL targets - custom images with GCC 10+ for C++20 support -# ============================================================================== - -[target.x86_64-unknown-linux-musl] -# Custom image with GCC 10 for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-x86_64-unknown-linux-musl:latest" - -[target.aarch64-unknown-linux-musl] -# Custom image with GCC 10 for C++20 support -image = "ghcr.io/adiibanez/cozodb-cross-aarch64-unknown-linux-musl:latest" From 1147a617bb77a595691cae8fbcf59fb07e8f7a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 20:49:19 +0100 Subject: [PATCH 19/31] Add Windows x86_64 target - Add x86_64-pc-windows-msvc native build on windows-latest - Install LLVM via chocolatey for libclang Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 345b9df..9525bb8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,6 +34,9 @@ jobs: - target: aarch64-unknown-linux-gnu os: ubuntu-24.04 use-cross: true + # Windows x86_64 native build + - target: x86_64-pc-windows-msvc + os: windows-latest steps: - name: Checkout source code uses: actions/checkout@v4 @@ -63,6 +66,11 @@ jobs: if: runner.os == 'macOS' run: | brew install llvm pkg-config + - name: Install build dependencies (Windows) + if: runner.os == 'Windows' + run: | + choco install llvm -y + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: From a23be5ca7e6e6b22c604bdcff89212a73de4646d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 20:55:49 +0100 Subject: [PATCH 20/31] Fix Cross.toml env variables syntax Use correct `variables = [["KEY", "value"]]` syntax instead of direct key assignments. This should properly set CXX to clang++. Co-Authored-By: Claude Opus 4.5 --- Cross.toml | 13 ++++++++----- native/cozodb/Cross.toml | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Cross.toml b/Cross.toml index 79d002f..a8848af 100644 --- a/Cross.toml +++ b/Cross.toml @@ -20,12 +20,15 @@ pre-build = [ [target.aarch64-unknown-linux-gnu.env] passthrough = ["RUST_BACKTRACE"] -volumes = [] # Force clang as C++ compiler with C++20 and correct target -CXX_aarch64_unknown_linux_gnu = "clang++" -CC_aarch64_unknown_linux_gnu = "clang" -CXXFLAGS = "-std=c++20 --target=aarch64-linux-gnu" -CFLAGS = "--target=aarch64-linux-gnu" +variables = [ + ["CXX_aarch64_unknown_linux_gnu", "clang++"], + ["CC_aarch64_unknown_linux_gnu", "clang"], + ["CXX_aarch64-unknown-linux-gnu", "clang++"], + ["CC_aarch64-unknown-linux-gnu", "clang"], + ["CXXFLAGS", "-std=c++20 --target=aarch64-linux-gnu -I/usr/aarch64-linux-gnu/include"], + ["CFLAGS", "--target=aarch64-linux-gnu"], +] # ============================================================================== # riscv64 - already works with standard image diff --git a/native/cozodb/Cross.toml b/native/cozodb/Cross.toml index 79d002f..a8848af 100644 --- a/native/cozodb/Cross.toml +++ b/native/cozodb/Cross.toml @@ -20,12 +20,15 @@ pre-build = [ [target.aarch64-unknown-linux-gnu.env] passthrough = ["RUST_BACKTRACE"] -volumes = [] # Force clang as C++ compiler with C++20 and correct target -CXX_aarch64_unknown_linux_gnu = "clang++" -CC_aarch64_unknown_linux_gnu = "clang" -CXXFLAGS = "-std=c++20 --target=aarch64-linux-gnu" -CFLAGS = "--target=aarch64-linux-gnu" +variables = [ + ["CXX_aarch64_unknown_linux_gnu", "clang++"], + ["CC_aarch64_unknown_linux_gnu", "clang"], + ["CXX_aarch64-unknown-linux-gnu", "clang++"], + ["CC_aarch64-unknown-linux-gnu", "clang"], + ["CXXFLAGS", "-std=c++20 --target=aarch64-linux-gnu -I/usr/aarch64-linux-gnu/include"], + ["CFLAGS", "--target=aarch64-linux-gnu"], +] # ============================================================================== # riscv64 - already works with standard image From 0b08362299787bfc35f23656f081e8065d046d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 21:15:25 +0100 Subject: [PATCH 21/31] Use native ARM64 runner instead of cross-compilation Cross-compilation env vars not being picked up by cozorocks build.rs. Use GitHub's native ubuntu-24.04-arm runner instead - same approach as x86_64 which already works. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9525bb8..4d12e65 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,10 +30,9 @@ jobs: # Linux x86_64 native build - WORKING - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 - # Linux ARM64 cross-compiled - testing clang approach + # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu - os: ubuntu-24.04 - use-cross: true + os: ubuntu-24.04-arm # Windows x86_64 native build - target: x86_64-pc-windows-msvc os: windows-latest From 7eeadc673f5a4179b1916e6372c6086f5a8db0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 21:28:33 +0100 Subject: [PATCH 22/31] Remove Windows target (C++17 hardcoded in cozorocks) The cozorocks build.rs hardcodes C++17 for all platforms. RocksDB requires C++20. This is an upstream issue. Focus on Linux ARM native build first. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d12e65..6d97216 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,9 +33,6 @@ jobs: # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu os: ubuntu-24.04-arm - # Windows x86_64 native build - - target: x86_64-pc-windows-msvc - os: windows-latest steps: - name: Checkout source code uses: actions/checkout@v4 @@ -65,11 +62,6 @@ jobs: if: runner.os == 'macOS' run: | brew install llvm pkg-config - - name: Install build dependencies (Windows) - if: runner.os == 'Windows' - run: | - choco install llvm -y - echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: From 06b7d84fc185979caac886a21338295d8241dba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 21:29:21 +0100 Subject: [PATCH 23/31] Add Windows back with C++20 flag for MSVC Set CXXFLAGS=/std:c++20 to override C++17 default. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d97216..f08890a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,6 +33,9 @@ jobs: # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu os: ubuntu-24.04-arm + # Windows x86_64 native build + - target: x86_64-pc-windows-msvc + os: windows-latest steps: - name: Checkout source code uses: actions/checkout@v4 @@ -62,6 +65,14 @@ jobs: if: runner.os == 'macOS' run: | brew install llvm pkg-config + - name: Install build dependencies (Windows) + if: runner.os == 'Windows' + run: | + choco install llvm -y + # Set LIBCLANG_PATH for bindgen + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV + # Force C++20 for MSVC (RocksDB requires it) + echo "CXXFLAGS=/std:c++20" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: From 16fd67541be032c338277b524a4e42e6fc2478de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 21:52:39 +0100 Subject: [PATCH 24/31] Try to disable jemalloc on Windows Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f08890a..3903551 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,6 +73,9 @@ jobs: echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV # Force C++20 for MSVC (RocksDB requires it) echo "CXXFLAGS=/std:c++20" >> $env:GITHUB_ENV + # Disable jemalloc on Windows (not available) + echo "JEMALLOC_OVERRIDE=" >> $env:GITHUB_ENV + echo "ROCKSDB_DISABLE_JEMALLOC=1" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: From 63a35e9795a9291f5ce959da3f18423b48567f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 22:14:21 +0100 Subject: [PATCH 25/31] Disable jemalloc on Windows via cargo-args Use --no-default-features to exclude jemalloc feature on Windows. jemalloc headers not available on Windows. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3903551..414b7ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,9 +33,10 @@ jobs: # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu os: ubuntu-24.04-arm - # Windows x86_64 native build + # Windows x86_64 native build (no jemalloc) - target: x86_64-pc-windows-msvc os: windows-latest + cargo-args: '--no-default-features --features "cozo/storage-sqlite,cozo/storage-rocksdb,cozo/graph-algo"' steps: - name: Checkout source code uses: actions/checkout@v4 @@ -105,6 +106,7 @@ jobs: nif-version: '${{ matrix.nif }}' use-cross: '${{ matrix.job.use-cross }}' project-dir: native/cozodb + cargo-args: '${{ matrix.job.cargo-args }}' - name: Rename Artifact with Branch & Version shell: bash run: | From 91907690dae964af52d6b155f250305bb917549e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Tue, 27 Jan 2026 22:34:34 +0100 Subject: [PATCH 26/31] Fix cargo-args YAML quoting for Windows Remove extra quotes causing feature parsing issues. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 414b7ff..1477339 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: # Windows x86_64 native build (no jemalloc) - target: x86_64-pc-windows-msvc os: windows-latest - cargo-args: '--no-default-features --features "cozo/storage-sqlite,cozo/storage-rocksdb,cozo/graph-algo"' + cargo-args: --no-default-features --features cozo/storage-sqlite,cozo/storage-rocksdb,cozo/graph-algo steps: - name: Checkout source code uses: actions/checkout@v4 From ac9b3531c6ab1abac027bc6d4bb4c0573176fca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Wed, 28 Jan 2026 09:14:04 +0100 Subject: [PATCH 27/31] Try multiple --features flags for Windows Instead of comma-separated, use multiple --features flags. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1477339..0301ebe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: # Windows x86_64 native build (no jemalloc) - target: x86_64-pc-windows-msvc os: windows-latest - cargo-args: --no-default-features --features cozo/storage-sqlite,cozo/storage-rocksdb,cozo/graph-algo + cargo-args: '--no-default-features --features cozo/storage-sqlite --features cozo/storage-rocksdb --features cozo/graph-algo' steps: - name: Checkout source code uses: actions/checkout@v4 From 658221769e16bfbba155ff1bbfc789ef85d39cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Wed, 28 Jan 2026 09:37:21 +0100 Subject: [PATCH 28/31] Test Windows with default features See if default features work or if jemalloc.h error returns. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0301ebe..633c8e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,10 +33,9 @@ jobs: # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu os: ubuntu-24.04-arm - # Windows x86_64 native build (no jemalloc) + # Windows x86_64 native build - target: x86_64-pc-windows-msvc os: windows-latest - cargo-args: '--no-default-features --features cozo/storage-sqlite --features cozo/storage-rocksdb --features cozo/graph-algo' steps: - name: Checkout source code uses: actions/checkout@v4 From 0f7eaab74077948c90ce4e526b4c7c5ecba73356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Wed, 28 Jan 2026 10:19:59 +0100 Subject: [PATCH 29/31] fix: disable jemalloc on Windows via feature flag Add windows-default feature that mirrors default but excludes jemalloc since jemalloc is not available on Windows/MSVC. The cozo/rocksdb-jemalloc feature was causing RocksDB to require jemalloc.h which doesn't exist on Windows. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 6 ++---- native/cozodb/Cargo.toml | 7 +++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 633c8e1..853ed83 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,9 +33,10 @@ jobs: # Linux ARM64 native build on ARM runner - target: aarch64-unknown-linux-gnu os: ubuntu-24.04-arm - # Windows x86_64 native build + # Windows x86_64 native build (no jemalloc - not available on Windows) - target: x86_64-pc-windows-msvc os: windows-latest + cargo-args: --no-default-features --features windows-default steps: - name: Checkout source code uses: actions/checkout@v4 @@ -73,9 +74,6 @@ jobs: echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV # Force C++20 for MSVC (RocksDB requires it) echo "CXXFLAGS=/std:c++20" >> $env:GITHUB_ENV - # Disable jemalloc on Windows (not available) - echo "JEMALLOC_OVERRIDE=" >> $env:GITHUB_ENV - echo "ROCKSDB_DISABLE_JEMALLOC=1" >> $env:GITHUB_ENV - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: diff --git a/native/cozodb/Cargo.toml b/native/cozodb/Cargo.toml index 9188fe0..48540b3 100644 --- a/native/cozodb/Cargo.toml +++ b/native/cozodb/Cargo.toml @@ -49,6 +49,13 @@ default = [ "jemalloc" # jemalloc with 0ms decay: best throughput + low memory retention ] +# Windows default - same as default but without jemalloc (not available on Windows/MSVC) +windows-default = [ + "cozo/storage-sqlite", + "cozo/storage-rocksdb", + "cozo/graph-algo", +] + # IMPORTANT: storage-rocksdb (cozorocks) and storage-new-rocksdb (rust-rocksdb) are # MUTUALLY EXCLUSIVE. Both link to RocksDB with different allocator configurations, # causing "pointer being freed was not allocated" crashes when used together. From 30bf84f0e49c46253f073cc39e3ada28ee530082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Wed, 28 Jan 2026 10:38:05 +0100 Subject: [PATCH 30/31] fix: move pinned deps before target-specific section The pinned dependencies (serde_json, lazy_static, crossbeam, etc.) were accidentally placed inside the [target.'cfg(not(target_env = "msvc"))'] section due to TOML section scoping rules. This caused them to be excluded from Windows/MSVC builds. Fixed by moving them before the target-specific jemalloc dependencies. Co-Authored-By: Claude Opus 4.5 --- native/cozodb/Cargo.toml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/native/cozodb/Cargo.toml b/native/cozodb/Cargo.toml index 48540b3..1fd8694 100644 --- a/native/cozodb/Cargo.toml +++ b/native/cozodb/Cargo.toml @@ -23,13 +23,9 @@ num_cpus = "1.17.0" # Only compile on non-MSVC platforms when jemalloc feature is enabled # Note: disable-initial-exec-tls is required for Docker/dlopen() compatibility # to avoid "cannot allocate memory in static TLS block" errors -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = { version = "0.6", features = ["stats", "profiling", "disable_initial_exec_tls"], optional = true } -tikv-jemalloc-ctl = { version = "0.6", features = ["stats", "profiling"], optional = true } -tikv-jemalloc-sys = { version = "0.6", features = ["stats", "profiling", "disable_initial_exec_tls"], optional = true } - # ============================================================================== # Pin the following to match cozo's working version +# ============================================================================== graph = "=0.3.1" # uses graph_builder 0.4.0 graph_builder = "=0.4.0" # uses rayon 1.10.0 rayon = "=1.10.0" @@ -39,7 +35,15 @@ serde = { version = "1.0.199", features = ["derive"] } serde_derive = "1.0.199" lazy_static = "1.4.0" crossbeam = "0.8.4" -# ============================================================================== + +# Memory allocator - jemalloc (optional, controlled by feature flag) +# Only compile on non-MSVC platforms when jemalloc feature is enabled +# Note: disable-initial-exec-tls is required for Docker/dlopen() compatibility +# to avoid "cannot allocate memory in static TLS block" errors +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = { version = "0.6", features = ["stats", "profiling", "disable_initial_exec_tls"], optional = true } +tikv-jemalloc-ctl = { version = "0.6", features = ["stats", "profiling"], optional = true } +tikv-jemalloc-sys = { version = "0.6", features = ["stats", "profiling", "disable_initial_exec_tls"], optional = true } [features] default = [ From 1b64314e6bfa5e76330e11f6092c1a0f786c488a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Iban=CC=83ez?= Date: Wed, 28 Jan 2026 11:20:52 +0100 Subject: [PATCH 31/31] feat: improve rustler precompiled deployment - Update workflow to use standard rustler_precompiled artifact naming (lib{crate}-v{version}-nif-{nif_version}-{target}.{ext}.tar.gz) - Remove custom artifact renaming - use native rustler_precompiled output - Require tags in v* format (e.g., v0.3.2) for releases - Add contents:write permission for GitHub releases - Simplify release publishing with make_latest and generate_release_notes - Update cozodb_nif_loader.erl: - Change default GitHub repo to adiibanez/cozodb - Add COZODB_GITHUB_REPO env var override - Use rustler_precompiled artifact naming format - Handle library extraction and renaming to standard cozodb name - Support .dll extension for Windows Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 72 ++++++--------------------- src/cozodb_nif_loader.erl | 93 +++++++++++++++++++++++++++-------- 2 files changed, 87 insertions(+), 78 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 853ed83..bb35484 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,8 @@ name: Build precompiled NIFs - main - 'fix/**' tags: - - '*' + # Matches v0.3.2, v1.0.0, etc. + - 'v*' workflow_dispatch: concurrency: @@ -16,18 +17,20 @@ jobs: build_release: name: 'NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }})' runs-on: '${{ matrix.job.os }}' + permissions: + contents: write strategy: fail-fast: false matrix: nif: - '2.17' job: - # macOS targets (native builds) - WORKING + # macOS targets (native builds) - target: aarch64-apple-darwin os: macos-latest - target: x86_64-apple-darwin os: macos-latest - # Linux x86_64 native build - WORKING + # Linux x86_64 native build - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 # Linux ARM64 native build on ARM runner @@ -40,21 +43,11 @@ jobs: steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Extract project version and branch/tag + - name: Extract project version shell: bash run: | # Get the project version from cozodb.app.src echo "PROJECT_VERSION=$(sed -n 's/.*{vsn, "\([^"]*\)".*/\1/p' src/cozodb.app.src | head -n1)" >> $GITHUB_ENV - - # Determine if it's a tag or a branch - if [[ "$GITHUB_REF" == refs/tags/* ]]; then - BRANCH_OR_TAG="${GITHUB_REF#refs/tags/}" - else - BRANCH_OR_TAG="${GITHUB_REF#refs/heads/}" - fi - # Sanitize branch name: replace / with - - BRANCH_OR_TAG="${BRANCH_OR_TAG//\//-}" - echo "BRANCH_OR_TAG=$BRANCH_OR_TAG" >> $GITHUB_ENV - name: Install build dependencies (Linux) if: runner.os == 'Linux' run: | @@ -104,51 +97,16 @@ jobs: use-cross: '${{ matrix.job.use-cross }}' project-dir: native/cozodb cargo-args: '${{ matrix.job.cargo-args }}' - - name: Rename Artifact with Branch & Version - shell: bash - run: | - echo "Debug: file-name = ${{ steps.build-crate.outputs.file-name }}" - echo "Debug: file-path = ${{ steps.build-crate.outputs.file-path }}" - echo "Debug: Listing current directory:" - ls -la *.tar.gz 2>/dev/null || echo "No tar.gz files in current dir" - ls -la native/cozodb/*.tar.gz 2>/dev/null || echo "No tar.gz files in native/cozodb" - - # Find the artifact file - ARTIFACT_PATH="${{ steps.build-crate.outputs.file-path }}" - if [ -z "$ARTIFACT_PATH" ] || [ ! -f "$ARTIFACT_PATH" ]; then - echo "Output file-path not found, searching for artifact..." - ARTIFACT_PATH=$(find . -name "*.tar.gz" -type f | head -1) - fi - - if [ -z "$ARTIFACT_PATH" ] || [ ! -f "$ARTIFACT_PATH" ]; then - echo "Error: Could not find artifact file" - exit 1 - fi - - NEW_NAME="cozodb-${{ env.BRANCH_OR_TAG }}-${{ env.PROJECT_VERSION }}-${{ matrix.job.target }}.tar.gz" - echo "Moving $ARTIFACT_PATH to $NEW_NAME" - mv "$ARTIFACT_PATH" "$NEW_NAME" - echo "RENAMED_ARTIFACT=$NEW_NAME" >> $GITHUB_ENV - - name: Generate checksum - shell: bash - run: | - if [[ "$RUNNER_OS" == "macOS" ]]; then - shasum -a 256 "${{ env.RENAMED_ARTIFACT }}" > "${{ env.RENAMED_ARTIFACT }}.sha256" - else - sha256sum "${{ env.RENAMED_ARTIFACT }}" > "${{ env.RENAMED_ARTIFACT }}.sha256" - fi - echo "CHECKSUM_FILE=${{ env.RENAMED_ARTIFACT }}.sha256" >> $GITHUB_ENV - name: Artifact upload uses: actions/upload-artifact@v4 with: - name: '${{ env.RENAMED_ARTIFACT }}' - path: | - ${{ env.RENAMED_ARTIFACT }} - ${{ env.CHECKSUM_FILE }} - - name: Publish archives and packages + name: '${{ steps.build-crate.outputs.file-name }}' + path: '${{ steps.build-crate.outputs.file-path }}' + retention-days: 5 + - name: Publish to GitHub Releases uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: - files: | - ${{ env.RENAMED_ARTIFACT }} - ${{ env.CHECKSUM_FILE }} - if: 'startsWith(github.ref, ''refs/tags/'')' + files: '${{ steps.build-crate.outputs.file-path }}' + make_latest: true + generate_release_notes: true diff --git a/src/cozodb_nif_loader.erl b/src/cozodb_nif_loader.erl index 067ece7..cc1f400 100644 --- a/src/cozodb_nif_loader.erl +++ b/src/cozodb_nif_loader.erl @@ -8,9 +8,12 @@ verify_checksum/2 ]). --define(GITHUB_REPO, "leapsight/cozodb"). +%% GitHub repository for downloading precompiled NIFs +%% Can be overridden with environment variable COZODB_GITHUB_REPO +-define(DEFAULT_GITHUB_REPO, "adiibanez/cozodb"). -define(NIF_VERSION, "2.17"). -define(APP_NAME, cozodb). +-define(CRATE_NAME, "cozodb"). -type platform_info() :: #{ os => linux | darwin | windows, @@ -61,24 +64,17 @@ download_precompiled(Version) -> CrateDir = filename:join([PrivDir, "crates", "cozodb"]), ok = filelib:ensure_dir(filename:join(CrateDir, "dummy")), - Tag = Version, - ArtifactName = build_artifact_name(Tag, Version, Target), - Url = build_download_url(Tag, ArtifactName), - ChecksumUrl = Url ++ ".sha256", + ArtifactName = build_artifact_name(Version, Target), + Url = build_download_url(Version, ArtifactName), TmpDir = filename:join([PrivDir, "tmp"]), ok = filelib:ensure_dir(filename:join(TmpDir, "dummy")), TmpFile = filename:join(TmpDir, ArtifactName), + logger:info("Downloading precompiled NIF from: ~s", [Url]), case download_file(Url, TmpFile) of ok -> - case download_and_verify_checksum(ChecksumUrl, TmpFile) of - ok -> - extract_and_install(TmpFile, CrateDir); - {error, _} = ChecksumErr -> - file:delete(TmpFile), - ChecksumErr - end; + extract_and_install(TmpFile, CrateDir, Version, Target); {error, _} = DownloadErr -> DownloadErr end. @@ -107,8 +103,12 @@ try_load_precompiled(Opts) -> PrivDir = code:priv_dir(?APP_NAME), NifPath = filename:join([PrivDir, "crates", "cozodb", "cozodb"]), - case filelib:is_file(NifPath ++ ".so") orelse - filelib:is_file(NifPath ++ ".dylib") of + %% Check for NIF file with any extension (.so, .dylib, .dll) + NifExists = filelib:is_file(NifPath ++ ".so") orelse + filelib:is_file(NifPath ++ ".dylib") orelse + filelib:is_file(NifPath ++ ".dll"), + + case NifExists of true -> ForceDownload = proplists:get_value(force_download, Opts, false), case ForceDownload of @@ -210,15 +210,42 @@ build_target(windows, Arch, msvc) -> build_target(_, _, _) -> <<"unknown">>. -build_artifact_name(Tag, Version, Target) -> - lists:flatten(io_lib:format("cozodb-~s-~s-~s.tar.gz", [Tag, Version, Target])). +%% Build artifact name in rustler_precompiled format: +%% lib{crate}-v{version}-nif-{nif_version}-{target}.{ext}.tar.gz +build_artifact_name(Version, Target) -> + Ext = get_lib_extension(Target), + lists:flatten(io_lib:format( + "lib~s-v~s-nif-~s-~s~s.tar.gz", + [?CRATE_NAME, Version, ?NIF_VERSION, Target, Ext] + )). -build_download_url(Tag, ArtifactName) -> +%% Build the download URL for GitHub releases +build_download_url(Version, ArtifactName) -> + GithubRepo = get_github_repo(), + Tag = "v" ++ Version, lists:flatten(io_lib:format( "https://github.com/~s/releases/download/~s/~s", - [?GITHUB_REPO, Tag, ArtifactName] + [GithubRepo, Tag, ArtifactName] )). +%% Get the library file extension for the target +get_lib_extension(Target) -> + TargetStr = binary_to_list(Target), + case string:find(TargetStr, "windows") of + nomatch -> + %% Both Linux and macOS use .so for Erlang NIFs + ".so"; + _ -> + ".dll" + end. + +%% Get GitHub repo from environment or default +get_github_repo() -> + case os:getenv("COZODB_GITHUB_REPO") of + false -> ?DEFAULT_GITHUB_REPO; + Repo -> Repo + end. + download_file(Url, DestPath) -> ensure_http_client(), case httpc:request(get, {Url, []}, [{timeout, 120000}], [{stream, DestPath}]) of @@ -254,12 +281,36 @@ download_and_verify_checksum(ChecksumUrl, FilePath) -> {error, {checksum_download_failed, Reason}} end. -extract_and_install(TarFile, DestDir) -> +%% Extract tar.gz and rename the NIF file to standard name +extract_and_install(TarFile, DestDir, Version, Target) -> case erl_tar:extract(TarFile, [{cwd, DestDir}, compressed]) of ok -> file:delete(TarFile), - NifPath = filename:join(DestDir, "cozodb"), - {ok, NifPath}; + %% The extracted file has rustler_precompiled naming: + %% lib{crate}-v{version}-nif-{nif_version}-{target}.{ext} + Ext = get_lib_extension(Target), + ExtractedName = lists:flatten(io_lib:format( + "lib~s-v~s-nif-~s-~s~s", + [?CRATE_NAME, Version, ?NIF_VERSION, Target, Ext] + )), + ExtractedPath = filename:join(DestDir, ExtractedName), + %% Rename to standard name without lib prefix and version + FinalExt = case Ext of + ".dll" -> ".dll"; + _ -> ".so" + end, + FinalPath = filename:join(DestDir, "cozodb" ++ FinalExt), + case file:rename(ExtractedPath, FinalPath) of + ok -> + NifPath = filename:join(DestDir, "cozodb"), + {ok, NifPath}; + {error, RenameReason} -> + logger:warning("Failed to rename ~s to ~s: ~p", + [ExtractedPath, FinalPath, RenameReason]), + %% Try to use the extracted file directly + NifPath = filename:rootname(ExtractedPath), + {ok, NifPath} + end; {error, Reason} -> file:delete(TarFile), {error, {extract_failed, Reason}}