From 8c820ffcd085c408cf31328668a32fa544b4b854 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Wed, 17 Dec 2025 18:28:53 +0000 Subject: [PATCH 1/4] Add plumbing for arm64 binaries --- dev/tasks/macros.jinja | 6 ++++-- dev/tasks/r/github.packages.yml | 38 ++++++++++++++++++++++----------- r/tools/nixlibs.R | 12 +++++++++-- r/tools/test-nixlibs.R | 4 ++-- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 6c17eecabb2..c1a29f86033 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -247,11 +247,13 @@ env: path: repo/libarrow {% endif %} {% if get_nix %} - - name: Get Linux binary + {% for arch in ["x86_64", "arm64"] %} + - name: Get Linux {{ arch }} binary uses: actions/download-artifact@v4 with: - name: r-libarrow-linux-x86_64 + name: r-libarrow-linux-{{ arch }} path: repo/libarrow + {% endfor %} {% endif %} {% if get_mac %} {% for arch in ["x86_64", "arm64"] %} diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index f57901371bd..e36ba9ccd56 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -111,14 +111,22 @@ jobs: {{ '${{ env.PKG_FILE }}' }}.sha512 linux-cpp: - name: C++ Binary Linux - runs-on: ubuntu-latest + name: C++ Binary Linux {{ '${{ matrix.arch }}' }} + runs-on: {{ '${{ matrix.runs-on }}' }} needs: source strategy: fail-fast: false + matrix: + include: + - arch: x86_64 + runs-on: ubuntu-latest + ubuntu: "22.04" + - arch: arm64 + runs-on: ubuntu-24.04-arm + ubuntu: "24.04" env: - PKG_ID: r-libarrow-linux-x86_64 - PKG_FILE: r-libarrow-linux-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} + PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} @@ -126,7 +134,7 @@ jobs: - name: Build libarrow shell: bash env: - UBUNTU: "22.04" + UBUNTU: {{ '${{ matrix.ubuntu }}' }} {{ macros.github_set_sccache_envvars()|indent(8) }} run: | source arrow/ci/scripts/util_enable_core_dumps.sh @@ -290,8 +298,8 @@ jobs: path: arrow_* test-linux-binary: needs: [source, linux-cpp] - name: Test binary {{ '${{ matrix.config.image }}' }} - runs-on: ubuntu-latest + name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} + runs-on: {{ '${{ matrix.config.runner }}' }} container: {{ '${{ matrix.config.image }}' }} strategy: fail-fast: false @@ -302,13 +310,17 @@ jobs: # an OS that is not in the allowlist, so we have to opt-in to use the # binary. Other env vars used in r_docker_configure.sh can be added # here and wired up in the later steps. - - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE"} + # x86_64 tests + - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) - # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE"} - - {image: "rhub/ubuntu-release"} # currently ubuntu-22.04 - - {image: "rstudio/r-base:4.1-jammy"} - - {image: "rstudio/r-base:4.2-jammy"} - - {image: "rstudio/r-base:4.3-noble"} + # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} + - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-22.04 + - {image: "rstudio/r-base:4.1-jammy", runner: "ubuntu-latest"} + - {image: "rstudio/r-base:4.2-jammy", runner: "ubuntu-latest"} + - {image: "rstudio/r-base:4.3-noble", runner: "ubuntu-latest"} + # ARM64 tests + - {image: "rstudio/r-base:4.2-jammy", runner: "ubuntu-24.04-arm"} + - {image: "rstudio/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} steps: # Get the arrow checkout just for the docker config scripts # Don't need submodules for this (hence false arg to macro): they fail on diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 9d0a2604682..f93b7a05e8a 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -255,13 +255,21 @@ check_allowlist <- function( any(grepl(paste(allowlist, collapse = "|"), os)) } +normalise_arch <- function(arch) { + if (arch %in% c("aarch64", "arm64")) { + return("arm64") + } + arch +} + select_binary <- function( os = tolower(Sys.info()[["sysname"]]), arch = tolower(Sys.info()[["machine"]]), test_program = test_for_curl_and_openssl ) { - if (identical(os, "darwin") || (identical(os, "linux") && identical(arch, "x86_64"))) { - # We only host x86 linux binaries and x86 & arm64 macos today + arch <- normalise_arch(arch) + + if (identical(os, "darwin") || identical(os, "linux")) { binary <- tryCatch( # Somehow the test program system2 call errors on the sanitizer builds # so globally handle the possibility that this could fail diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index b1d6214fd87..f7711c97ce4 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -52,8 +52,8 @@ test_that("identify_binary() based on LIBARROW_BINARY", { test_that("select_binary() based on system", { expect_output( - expect_null(select_binary("linux", arch = "aarch64")), # Not built today - "Building on linux aarch64" + expect_null(select_binary("freebsd", arch = "x86_64")), + "Building on freebsd x86_64" ) }) From 8a502a9ae443c2bb98b2b062f6f9148e49e49731 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 18 Dec 2025 12:40:23 +0000 Subject: [PATCH 2/4] add in architecture --- dev/tasks/r/github.packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index e36ba9ccd56..93950231aa3 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -134,6 +134,7 @@ jobs: - name: Build libarrow shell: bash env: + ARCH: {{ '${{ matrix.arch == "x86_64" && "amd64" || "arm64v8" }}' }} UBUNTU: {{ '${{ matrix.ubuntu }}' }} {{ macros.github_set_sccache_envvars()|indent(8) }} run: | From 7fa239cc90cdb66a2c53e0521bf908ace32c7f7f Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 18 Dec 2025 14:12:48 +0000 Subject: [PATCH 3/4] Fix quotes --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 93950231aa3..d6aece2b8cf 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -134,7 +134,7 @@ jobs: - name: Build libarrow shell: bash env: - ARCH: {{ '${{ matrix.arch == "x86_64" && "amd64" || "arm64v8" }}' }} + ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64v8' }}" }} UBUNTU: {{ '${{ matrix.ubuntu }}' }} {{ macros.github_set_sccache_envvars()|indent(8) }} run: | From cfd61693d65106ecdf7536db835b9b3c1bd005a4 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 19 Dec 2025 11:15:12 +0000 Subject: [PATCH 4/4] Update rstudio/r-base to posit/r-base --- compose.yaml | 2 +- dev/tasks/r/github.linux.versions.yml | 4 ++-- dev/tasks/r/github.packages.yml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose.yaml b/compose.yaml index b89d3082c79..18cd75385ed 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1761,7 +1761,7 @@ services: cache_from: - ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck args: - base: rstudio/r-base:4.2-focal + base: posit/r-base:4.2-focal r_dev: ${ARROW_R_DEV} tz: ${TZ} shm_size: *shm-size diff --git a/dev/tasks/r/github.linux.versions.yml b/dev/tasks/r/github.linux.versions.yml index b7b55ca8252..e5ed151a937 100644 --- a/dev/tasks/r/github.linux.versions.yml +++ b/dev/tasks/r/github.linux.versions.yml @@ -21,12 +21,12 @@ jobs: r-versions: - name: "rstudio/r-base:{{ MATRIX }}-jammy" + name: "posit/r-base:{{ MATRIX }}-jammy" runs-on: ubuntu-latest strategy: fail-fast: false matrix: - # See https://hub.docker.com/r/rstudio/r-base + # See https://hub.docker.com/r/posit/r-base r_version: # We test devel, release, and oldrel in regular CI. # This is for older versions diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index d6aece2b8cf..ddfb44a58a6 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -316,12 +316,12 @@ jobs: # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-22.04 - - {image: "rstudio/r-base:4.1-jammy", runner: "ubuntu-latest"} - - {image: "rstudio/r-base:4.2-jammy", runner: "ubuntu-latest"} - - {image: "rstudio/r-base:4.3-noble", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.1-jammy", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.2-jammy", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} # ARM64 tests - - {image: "rstudio/r-base:4.2-jammy", runner: "ubuntu-24.04-arm"} - - {image: "rstudio/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.2-jammy", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} steps: # Get the arrow checkout just for the docker config scripts # Don't need submodules for this (hence false arg to macro): they fail on @@ -362,7 +362,7 @@ jobs: if: false needs: test-linux-binary runs-on: ubuntu-latest - container: "rstudio/r-base:4.2-centos7" + container: "posit/r-base:4.2-centos7" steps: - uses: actions/download-artifact@v4 with: