Multirail Multiplexing - Scratch PR

.github/workflows/ci.yml

@@ -144,14 +144,19 @@ jobs:
           - config_name: Manpages
             sos_config: --enable-manpages --enable-pmi-simple
             libfabric_version: v1.13.x
+          - config_name: Without OFI inject
+            sos_config: --disable-ofi-inject --enable-pmi-simple
+            libfabric_version: v1.13.x
 
     name: OFI ${{ matrix.libfabric_version }} (${{ matrix.config_name }})
 
     steps:
       - name: Checking OS version
         run: |
           echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
       - name: Install dependencies
         run: |
           sudo apt-get install -y gfortran rpm mpich libmpich-dev libhwloc-dev
@@ -167,7 +172,7 @@ jobs:
           key: libfabric-${{ matrix.libfabric_version }}-${{ env.OS_NAME }}
       - name: Checkout libfabric
         if: steps.cache-libfabric.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: ofiwg/libfabric
           path: repos/libfabric
@@ -192,7 +197,7 @@ jobs:
           key: xpmem-${{ matrix.xpmem_version }}-${{ env.OS_NAME }}
       - name: Checkout XPMEM
         if: ${{ matrix.xpmem_version }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: hjelmn/xpmem
           path: repos/xpmem
@@ -223,7 +228,7 @@ jobs:
           make check TESTS= -j
           ${{ matrix.env_setup }}
           SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2" check
-          cat test/unit/hello.log
+          cat modules/tests-sos/test/unit/hello.log
       - name: Test RPM (${{ matrix.rpm_build }})
         if: ${{ matrix.rpm_build }}
         run: |
@@ -453,7 +458,7 @@ jobs:
 #          export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH
 #          ${{ matrix.sos_pm_pre }}
 #          SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${{ matrix.sos_pm }} -np 2" check
-#          cat test/unit/hello.log
+#          cat modules/tests-sos/test/unit/hello.log
 #          ${{ matrix.sos_pm_post }}
 #      - name: Test SOS (hello)
 #        if: ${{ matrix.sos_transport == 'portals4' }}
@@ -462,7 +467,7 @@ jobs:
 #          make check TESTS= -j
 #          export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH
 #          ${{ matrix.sos_pm_pre }}
-#          SHMEM_INFO=1 ${{ matrix.sos_pm }} -np 1 test/unit/hello
+#          SHMEM_INFO=1 ${{ matrix.sos_pm }} -np 1 modules/tests-sos/test/unit/hello
 #          ${{ matrix.sos_pm_post }}
 #
   UCX:
@@ -474,16 +479,17 @@ jobs:
           - config_name: ucx-1.9.0
             ucx_version: v1.9.0
             xpmem_version: master
-        sos_config: [--enable-pmi-simple --disable-fortran --with-hwloc=no,
-                     --enable-pmi-mpi CC=mpicc --disable-fortran,
+        sos_config: [--enable-pmi-simple --disable-fortran,
                      --with-cma --enable-error-checking --enable-profiling
                      --enable-pmi-simple --disable-fortran --with-hwloc=no,
                      --with-xpmem --enable-error-checking --enable-pmi-simple --with-hwloc=no]
     steps:
       - name: Checking OS version
         run: |
           echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
       - name: Install dependencies
         run: |
           sudo apt-get install -y gfortran mpich libmpich-dev
@@ -492,7 +498,7 @@ jobs:
 
       # XPMEM
       - name: Checkout XPMEM
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: hjelmn/xpmem
           path: repos/xpmem
@@ -517,7 +523,7 @@ jobs:
           key: ucx-${{ matrix.ucx_version}}-${{ env.OS_NAME }}
       - name: Checkout UCX
         if: steps.cache-ucx.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: openucx/ucx
           path: repos/ucx
@@ -545,7 +551,7 @@ jobs:
           cd build
           make check TESTS= -j
           SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2" check
-          cat test/unit/hello.log
+          cat modules/tests-sos/test/unit/hello.log
 
   Portals4:
     runs-on: ubuntu-20.04
@@ -573,7 +579,9 @@ jobs:
       - name: Checking OS version
         run: |
           echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
       - name: Install dependencies
         run: |
           sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev
@@ -582,7 +590,7 @@ jobs:
 
       # XPMEM
       - name: Checkout XPMEM
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: hjelmn/xpmem
           path: repos/xpmem
@@ -607,7 +615,7 @@ jobs:
           key: portals4-${{ matrix.portals4_version}}-${{ env.OS_NAME }}
       - name: Checkout Portals4
         if: steps.cache-portals4.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: regrant/portals4
           path: repos/portals4
@@ -634,7 +642,7 @@ jobs:
         run: |
           cd build
           make check TESTS= -j
-          ${SOS_PM} -np 1 test/unit/hello
+          ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello
 
   XPMEM_Only:
     runs-on: ubuntu-20.04
@@ -650,7 +658,9 @@ jobs:
       - name: Checking OS version
         run: |
           echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
       - name: Install dependencies
         run: |
           sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev
@@ -659,7 +669,7 @@ jobs:
 
       # XPMEM
       - name: Checkout XPMEM
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: hjelmn/xpmem
           path: repos/xpmem
@@ -688,4 +698,4 @@ jobs:
           cd build
           make check TESTS= -j
           SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2" check
-          ${SOS_PM} -np 1 test/unit/hello
+          ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "modules/tests-sos"]
+	path = modules/tests-sos
+	url = ../../openshmem-org/tests-sos.git
+	branch = main

Makefile.am

@@ -21,7 +21,7 @@ endif
 
 dist_doc_DATA = README NEWS LICENSE
 
-SUBDIRS = bindings mpp pmi-simple src test
+SUBDIRS = bindings mpp pmi-simple src modules
 
 if ENABLE_MANPAGES
 SUBDIRS += man

autogen.sh

@@ -2,6 +2,33 @@
 
 test -d ./config || mkdir ./config
 
+tests_sos_dir=modules/tests-sos
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+current_dir="$(pwd)"
+
+# Check if the script is being run from within the root SOS source directory
+if [ "$script_dir" != "$current_dir" ]; then
+    echo "Error: Please run ./autogen.sh from the top-level directory of SOS."
+    exit 1
+fi
+
+if [ ! -d "${tests_sos_dir}" ]; then
+    echo "Error: the tests-sos submodule directory ${tests_sos_dir} does not exist"
+    exit 1
+elif [ ! -f "${tests_sos_dir}/test/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/apps/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/include/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/performance/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/shmemx/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/spec-example/Makefile.am" ] ||
+     [ ! -f "${tests_sos_dir}/test/unit/Makefile.am" ]; then
+    echo "Error: test submodule (${tests_sos_dir}) contents are missing."
+    echo "Please run the following command to download the SOS tests:"
+    echo "   git submodule update --init"
+    exit 1
+fi
+
 FILES=./man/*.1
 echo -n "dist_man1_MANS =" > ./man/Makefile.am
 for f in $FILES

configure.ac

@@ -967,15 +967,17 @@ AC_CONFIG_FILES([Makefile
   mpp/shmem-def.h
   src/Makefile
   src/sandia-openshmem.pc
-  test/Makefile
-  test/unit/Makefile
-  test/shmemx/Makefile
-  test/spec-example/Makefile
-  test/include/Makefile
-  test/performance/Makefile
-  test/performance/shmem_perf_suite/Makefile
-  test/performance/tests/Makefile
-  test/apps/Makefile])
+  modules/Makefile
+  modules/tests-sos/Makefile
+  modules/tests-sos/test/Makefile
+  modules/tests-sos/test/unit/Makefile
+  modules/tests-sos/test/shmemx/Makefile
+  modules/tests-sos/test/spec-example/Makefile
+  modules/tests-sos/test/include/Makefile
+  modules/tests-sos/test/performance/Makefile
+  modules/tests-sos/test/performance/shmem_perf_suite/Makefile
+  modules/tests-sos/test/performance/tests/Makefile
+  modules/tests-sos/test/apps/Makefile])
 
 AC_OUTPUT
 

test/Makefile.am → modules/Makefile.am

@@ -4,11 +4,11 @@
 # DE-AC04-94AL85000 with Sandia Corporation, the U.S.  Government
 # retains certain rights in this software.
 #
-# Copyright (c) 2017 Intel Corporation. All rights reserved.
+# Copyright (c) 2024 Intel Corporation. All rights reserved.
 # This software is available to you under the BSD license.
 #
 # This file is part of the Sandia OpenSHMEM software package. For license
 # information, see the LICENSE file in the top level directory of the
 # distribution.
 
-SUBDIRS = unit shmemx performance apps spec-example include
+SUBDIRS = tests-sos

sandia-openshmem.spec.in

@@ -49,17 +49,17 @@ rm -f %{buildroot}%{_libdir}/*.la
 # FIXME: It would be better to configure with --disable-libtool-wrapper so we
 #        can pull from the standard dirs instead of .libs
 install -d %{testdir}/unit
-install test/unit/.libs/* %{testdir}/unit
+install modules/tests-sos/test/unit/.libs/* %{testdir}/unit
 install -d %{testdir}/shmemx
-install test/shmemx/.libs/* %{testdir}/shmemx
+install modules/tests-sos/test/shmemx/.libs/* %{testdir}/shmemx
 install -d %{testdir}/spec-example
-install test/spec-example/.libs/* %{testdir}/spec-example
+install modules/tests-sos/test/spec-example/.libs/* %{testdir}/spec-example
 install -d %{testdir}/apps
-install test/apps/.libs/* %{testdir}/apps
+install modules/tests-sos/test/apps/.libs/* %{testdir}/apps
 install -d %{testdir}/performance/shmem_perf_suite
-install test/performance/shmem_perf_suite/.libs/* %{testdir}/performance/shmem_perf_suite
+install modules/tests-sos/test/performance/shmem_perf_suite/.libs/* %{testdir}/performance/shmem_perf_suite
 install -d %{testdir}/performance/tests
-install test/performance/tests/.libs/* %{testdir}/performance/tests
+install modules/tests-sos/test/performance/tests/.libs/* %{testdir}/performance/tests
 
 %clean
 rm -rf %{buildroot}

scripts/docker/Dockerfile

@@ -97,7 +97,7 @@ RUN make install
 
 # Build SOS
 WORKDIR $SOS_INSTALL
-RUN git clone https://github.com/Sandia-OpenSHMEM/SOS.git
+RUN git clone --recurse-submodules https://github.com/Sandia-OpenSHMEM/SOS.git
 WORKDIR SOS
 RUN ./autogen.sh
 # To build SOS w/ basic Libfabric

scripts/simple-build-ofi.sh

@@ -82,7 +82,7 @@ echo ========================================================================
 echo = BUILDING SANDIA OPENSHMEM
 echo ========================================================================
 
-git clone -b $SOS_VERSION --depth $CLONE_DEPTH https://github.com/Sandia-OpenSHMEM/SOS.git sos
+git clone -b $SOS_VERSION --depth $CLONE_DEPTH --recurse-submodules https://github.com/Sandia-OpenSHMEM/SOS.git sos
 cd sos
 ./autogen.sh
 ./configure --prefix=$INSTALL_DIR --with-ofi=$INSTALL_DIR $SOS_CONFIG_OPTS

src/runtime-mpi.c

@@ -28,7 +28,7 @@
 
 /* Note: Increase MAX_KV_COUNT if more key/values are needed.  MAX_KV_COUNT is
  * 2 * the number of key/value pairs. */
-#define MAX_KV_COUNT 20
+#define MAX_KV_COUNT 40
 #define MAX_KV_LENGTH 512
 
 static int rank = -1;

src/shmem_comm.h

@@ -32,30 +32,45 @@
 
 static inline
 void
-shmem_internal_put_scalar(shmem_ctx_t ctx, void *target, const void *source, size_t len, int pe)
+shmem_internal_put_nb(shmem_ctx_t ctx, void *target, const void *source, size_t len, int pe,
+                      long *completion)
 {
-    shmem_internal_assert(len > 0);
+    if (len == 0)
+        return;
 
     if (shmem_shr_transport_use_write(ctx, target, source, len, pe)) {
-        shmem_shr_transport_put_scalar(ctx, target, source, len, pe);
+        shmem_shr_transport_put(ctx, target, source, len, pe);
     } else {
-        shmem_transport_put_scalar((shmem_transport_ctx_t *)ctx, target, source, len, pe);
+        shmem_transport_put_nb((shmem_transport_ctx_t *)ctx, target, source, len, pe, completion);
     }
 }
 
 
 static inline
 void
-shmem_internal_put_nb(shmem_ctx_t ctx, void *target, const void *source, size_t len, int pe,
-                      long *completion)
+shmem_internal_put_wait(shmem_ctx_t ctx, long *completion)
 {
-    if (len == 0)
-        return;
+    shmem_transport_put_wait((shmem_transport_ctx_t *)ctx, completion);
+    /* on-node is always blocking, so this is a no-op for them */
+}
+
+
+static inline
+void
+shmem_internal_put_scalar(shmem_ctx_t ctx, void *target, const void *source, size_t len, int pe)
+{
+    shmem_internal_assert(len > 0);
 
     if (shmem_shr_transport_use_write(ctx, target, source, len, pe)) {
-        shmem_shr_transport_put(ctx, target, source, len, pe);
+        shmem_shr_transport_put_scalar(ctx, target, source, len, pe);
     } else {
-        shmem_transport_put_nb((shmem_transport_ctx_t *)ctx, target, source, len, pe, completion);
+#ifndef DISABLE_OFI_INJECT
+        shmem_transport_put_scalar((shmem_transport_ctx_t *)ctx, target, source, len, pe);
+#else
+        long completion = 0;
+        shmem_transport_put_nb((shmem_transport_ctx_t *)ctx, target, source, len, pe, &completion);
+	shmem_internal_put_wait(ctx, &completion);
+#endif
     }
 }
 
@@ -106,15 +121,6 @@ shmem_internal_put_ct_nb(shmemx_ct_t ct, void *target, const void *source, size_
 }
 
 
-static inline
-void
-shmem_internal_put_wait(shmem_ctx_t ctx, long *completion)
-{
-    shmem_transport_put_wait((shmem_transport_ctx_t *)ctx, completion);
-    /* on-node is always blocking, so this is a no-op for them */
-}
-
-
 static inline
 void
 shmem_internal_get(shmem_ctx_t ctx, void *target, const void *source, size_t len, int pe)