From 9167a6e967fed76ab92c103f4286f94950dcb3b8 Mon Sep 17 00:00:00 2001 From: Hiroshi Shinaoka Date: Fri, 6 Feb 2026 10:50:49 +0900 Subject: [PATCH] Add cross-language MPI interop tests and update README - Add experimental/POC status, upstream merge goal, and Julia/Python motivation to README - Add MPMD-based interop tests: Rust + mpi4py + MPI.jl sharing MPI_COMM_WORLD under a single mpiexec - Add CI workflow for interop tests (.github/workflows/interop.yml) - Tests gracefully skip when Python/Julia deps are not installed Co-Authored-By: Claude Opus 4.6 --- .github/workflows/interop.yml | 55 +++++++++++++++++ README.md | 20 ++++++ examples/interop_test.rs | 35 +++++++++++ tests/interop/run_interop.sh | 113 ++++++++++++++++++++++++++++++++++ tests/interop/test_mpi4py.py | 35 +++++++++++ tests/interop/test_mpi_jl.jl | 37 +++++++++++ 6 files changed, 295 insertions(+) create mode 100644 .github/workflows/interop.yml create mode 100644 examples/interop_test.rs create mode 100755 tests/interop/run_interop.sh create mode 100644 tests/interop/test_mpi4py.py create mode 100644 tests/interop/test_mpi_jl.jl diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml new file mode 100644 index 00000000..5e0916aa --- /dev/null +++ b/.github/workflows/interop.yml @@ -0,0 +1,55 @@ +name: Interoperability Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + interop: + name: MPI interop (${{ matrix.mpi }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - mpi: mpich + mpi-pkg: libmpich-dev + - mpi: openmpi + mpi-pkg: libopenmpi-dev + + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y ${{ matrix.mpi-pkg }} llvm-dev libclang-dev pkgconf + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install mpi4py + run: pip install mpi4py + + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: "1" + + - name: Install MPI.jl + run: julia -e 'using Pkg; Pkg.add("MPI")' + + - name: Run interop tests (mpi-sys-backend) + run: bash tests/interop/run_interop.sh --backend mpi-sys-backend + + # TODO: Enable mpi-rt-sys-backend tests once MPIwrapper is available in CI + # - name: Run interop tests (mpi-rt-sys-backend) + # run: bash tests/interop/run_interop.sh --backend mpi-rt-sys-backend + # env: + # MPI_RT_LIB: /path/to/libmpiwrapper.so diff --git a/README.md b/README.md index 76d8d39f..f5964a88 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,15 @@ **Rust [MPI] bindings with runtime library loading.** Fork of [rsmpi] v0.8.1. +> **Status: Experimental / Proof of Concept.** +> This project aims to eventually merge its runtime-loading backend upstream into rsmpi. + The key addition over upstream rsmpi is the `mpi-rt-sys-backend`: an [MPIABI]-based backend that loads MPI at runtime via `dlopen`, requiring **no C compiler, system MPI headers, or libclang at build time**. +### Motivation + +The primary goal is to enable **calling Rust MPI code from Julia and Python** without build-time MPI dependencies. By loading MPI at runtime, Rust libraries can share the same MPI communicator with [MPI.jl] and [mpi4py], enabling seamless multi-language HPC workflows. + [actions-shield]: https://github.com/tensor4all/rsmpi-rt/workflows/Test/badge.svg [actions]: https://github.com/tensor4all/rsmpi-rt/actions [doc-shield]: https://img.shields.io/badge/docs-GitHub%20Pages-blue @@ -17,6 +24,8 @@ The key addition over upstream rsmpi is the `mpi-rt-sys-backend`: an [MPIABI]-ba [MPI]: http://www.mpi-forum.org [rsmpi]: https://github.com/rsmpi/rsmpi [MPIABI]: https://github.com/eschnett/MPItrampoline +[MPI.jl]: https://github.com/JuliaParallel/MPI.jl +[mpi4py]: https://mpi4py.readthedocs.io/ ## Quick Start @@ -104,6 +113,17 @@ See the [upstream rsmpi README](https://github.com/rsmpi/rsmpi#requirements) for | `derive` | `#[derive(Equivalence)]` for sending structs over MPI | | `complex` | Support for `num-complex` types | +## Interoperability Tests + +Integration tests verify that Rust MPI code can run alongside [mpi4py] and [MPI.jl] under the same `mpiexec`, sharing `MPI_COMM_WORLD` via MPMD launch: + +```bash +# Run the cross-language interop test (requires Python/mpi4py and Julia/MPI.jl) +bash tests/interop/run_interop.sh +``` + +See [tests/interop/](tests/interop/) for details. + ## Documentation - [API docs (GitHub Pages)][doc] diff --git a/examples/interop_test.rs b/examples/interop_test.rs new file mode 100644 index 00000000..9858f35b --- /dev/null +++ b/examples/interop_test.rs @@ -0,0 +1,35 @@ +//! Rust MPI interop test program for MPMD launch with Python/Julia. +//! +//! This program participates in a cross-language MPI test: +//! - All ranks call MPI_Barrier to verify shared MPI_COMM_WORLD +//! - Rank 0 (this program) broadcasts a value to all ranks +//! - All ranks verify they received the broadcast +//! +//! Run via: tests/interop/run_interop.sh + +use mpi::traits::*; + +fn main() { + let universe = mpi::initialize().unwrap(); + let world = universe.world(); + let rank = world.rank(); + let size = world.size(); + + // Phase 1: All ranks report in and synchronize + println!("[Rust rank {}] MPI_COMM_WORLD size = {}", rank, size); + world.barrier(); + + // Phase 2: Rank 0 broadcasts a magic value (42) to all ranks + let root = world.process_at_rank(0); + let mut value: i32 = if rank == 0 { 42 } else { 0 }; + root.broadcast_into(&mut value); + + assert_eq!(value, 42, "Broadcast value mismatch on rank {}", rank); + println!("[Rust rank {}] broadcast received: {}", rank, value); + + // Phase 3: Final barrier + world.barrier(); + if rank == 0 { + println!("[Rust rank 0] interop test PASSED"); + } +} diff --git a/tests/interop/run_interop.sh b/tests/interop/run_interop.sh new file mode 100755 index 00000000..518e292a --- /dev/null +++ b/tests/interop/run_interop.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Cross-language MPI interoperability test. +# +# Launches Rust, Python, and Julia MPI programs under a single mpiexec +# using MPMD (Multiple Program Multiple Data) mode, sharing MPI_COMM_WORLD. +# +# Prerequisites: +# - MPI implementation (MPICH or OpenMPI) +# - Python 3 with mpi4py: pip install mpi4py +# - Julia with MPI.jl: julia -e 'using Pkg; Pkg.add("MPI")' +# - For mpi-rt-sys-backend: MPI_RT_LIB environment variable set +# +# Usage: +# bash tests/interop/run_interop.sh [--backend mpi-sys-backend|mpi-rt-sys-backend] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +BACKEND="mpi-sys-backend" +while [[ $# -gt 0 ]]; do + case $1 in + --backend) BACKEND="$2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +echo "=== Cross-language MPI interop test (backend: $BACKEND) ===" + +# Build Rust test binary +echo "Building Rust test binary..." +FEATURES="$BACKEND" +if [ "$BACKEND" = "mpi-rt-sys-backend" ]; then + DEFAULT_FEATURES="--no-default-features" +else + DEFAULT_FEATURES="" +fi +cargo build --manifest-path "$PROJECT_DIR/Cargo.toml" \ + $DEFAULT_FEATURES --features "$FEATURES" \ + --example interop_test 2>&1 + +RUST_BIN="$PROJECT_DIR/target/debug/examples/interop_test" + +# Check prerequisites +check_cmd() { + if ! command -v "$1" &>/dev/null; then + echo "SKIP: $1 not found" + return 1 + fi + return 0 +} + +HAS_PYTHON=false +HAS_JULIA=false + +if check_cmd python3; then + if python3 -c "import mpi4py" 2>/dev/null; then + HAS_PYTHON=true + else + echo "SKIP: mpi4py not installed (pip install mpi4py)" + fi +fi + +if check_cmd julia; then + if julia -e 'using MPI' 2>/dev/null; then + HAS_JULIA=true + else + echo "SKIP: MPI.jl not installed (julia -e 'using Pkg; Pkg.add(\"MPI\")')" + fi +fi + +# --- Test 1: Rust only (baseline) --- +echo "" +echo "--- Test 1: Rust-only MPI (2 ranks) ---" +mpiexec -n 2 "$RUST_BIN" +echo "PASSED" + +# --- Test 2: Rust + Python (MPMD) --- +if [ "$HAS_PYTHON" = true ]; then + echo "" + echo "--- Test 2: Rust + Python MPMD (2 ranks) ---" + mpiexec -n 1 "$RUST_BIN" : -n 1 python3 "$SCRIPT_DIR/test_mpi4py.py" + echo "PASSED" +else + echo "" + echo "--- Test 2: Rust + Python MPMD --- SKIPPED" +fi + +# --- Test 3: Rust + Julia (MPMD) --- +if [ "$HAS_JULIA" = true ]; then + echo "" + echo "--- Test 3: Rust + Julia MPMD (2 ranks) ---" + mpiexec -n 1 "$RUST_BIN" : -n 1 julia "$SCRIPT_DIR/test_mpi_jl.jl" + echo "PASSED" +else + echo "" + echo "--- Test 3: Rust + Julia MPMD --- SKIPPED" +fi + +# --- Test 4: All three languages (MPMD) --- +if [ "$HAS_PYTHON" = true ] && [ "$HAS_JULIA" = true ]; then + echo "" + echo "--- Test 4: Rust + Python + Julia MPMD (3 ranks) ---" + mpiexec -n 1 "$RUST_BIN" : -n 1 python3 "$SCRIPT_DIR/test_mpi4py.py" : -n 1 julia "$SCRIPT_DIR/test_mpi_jl.jl" + echo "PASSED" +else + echo "" + echo "--- Test 4: Rust + Python + Julia MPMD --- SKIPPED" +fi + +echo "" +echo "=== All interop tests completed ===" diff --git a/tests/interop/test_mpi4py.py b/tests/interop/test_mpi4py.py new file mode 100644 index 00000000..d1feaea8 --- /dev/null +++ b/tests/interop/test_mpi4py.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""mpi4py interop test for MPMD launch with Rust and Julia. + +This script participates in a cross-language MPI test: +- All ranks call MPI_Barrier to verify shared MPI_COMM_WORLD +- Rank 0 broadcasts a value (42) to all ranks +- All ranks verify they received the broadcast + +Run via: tests/interop/run_interop.sh +""" + +from mpi4py import MPI + + +def main(): + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + + # Phase 1: All ranks report in and synchronize + print(f"[Python rank {rank}] MPI_COMM_WORLD size = {size}") + comm.Barrier() + + # Phase 2: Rank 0 broadcasts a magic value (42) to all ranks + value = comm.bcast(42 if rank == 0 else None, root=0) + + assert value == 42, f"Broadcast value mismatch on rank {rank}: got {value}" + print(f"[Python rank {rank}] broadcast received: {value}") + + # Phase 3: Final barrier + comm.Barrier() + + +if __name__ == "__main__": + main() diff --git a/tests/interop/test_mpi_jl.jl b/tests/interop/test_mpi_jl.jl new file mode 100644 index 00000000..2746683a --- /dev/null +++ b/tests/interop/test_mpi_jl.jl @@ -0,0 +1,37 @@ +#!/usr/bin/env julia +# +# MPI.jl interop test for MPMD launch with Rust and Python. +# +# This script participates in a cross-language MPI test: +# - All ranks call MPI_Barrier to verify shared MPI_COMM_WORLD +# - Rank 0 broadcasts a value (42) to all ranks +# - All ranks verify they received the broadcast +# +# Run via: tests/interop/run_interop.sh + +using MPI + +function main() + MPI.Init() + + comm = MPI.COMM_WORLD + rank = MPI.Comm_rank(comm) + size = MPI.Comm_size(comm) + + # Phase 1: All ranks report in and synchronize + println("[Julia rank $rank] MPI_COMM_WORLD size = $size") + MPI.Barrier(comm) + + # Phase 2: Rank 0 broadcasts a magic value (42) to all ranks + value = MPI.bcast(rank == 0 ? Int32(42) : Int32(0), comm; root=0) + + @assert value == 42 "Broadcast value mismatch on rank $rank: got $value" + println("[Julia rank $rank] broadcast received: $value") + + # Phase 3: Final barrier + MPI.Barrier(comm) + + MPI.Finalize() +end + +main()