Quicksilver SYCL version

SYCL/CMakeLists.txt

@@ -0,0 +1,175 @@
+# ModificationsCopyright(C) 2023 Intel Corporation
+
+# Redistributionand use in source and binary forms, with or without modification,
+
+# arepermitted provided that the following conditions are met:
+
+# 1. Redistributions of source code must retain the above copyright notice,
+
+# thislist of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+
+# thislist of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its contributors
+
+# maybe used to endorse or promote products derived from this software
+
+# withoutspecific prior written permission.
+
+# THISSOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+
+# ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+
+# THEIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+
+# AREDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+
+# BELIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+
+# ORCONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+
+# OFSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+
+# ORBUSINESS INTERRUPTION)
+# HOWEVERCAUSED AND ON ANY THEORY OF LIABILITY,
+
+# WHETHERIN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING NEGLIGENCE
+
+# OROTHERWISE)
+# ARISINGIN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+
+# EVEN
+
+# IFADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# SPDX-License-Identifier: BSD-3-Clause
+
+cmake_minimum_required(VERSION 2.80)
+project(qs LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+option(ENABLE_KERNEL_PROFILING "Build using kernel profiling" OFF)
+option(GPU_AOT "Build AOT for Intel GPU" OFF)
+option(USE_NVIDIA_BACKEND "Build for NVIDIA backend" OFF)
+option(USE_AMDHIP_BACKEND "Build for AMD HIP backend" OFF)
+option(USE_SM "Specifies which streaming multiprocessor architecture to use" OFF)
+
+include_directories(
+    ${CMAKE_SOURCE_DIR}/src/
+)
+
+set(SRC_LIST
+    ${CMAKE_SOURCE_DIR}/src/CoralBenchmark.cc
+    ${CMAKE_SOURCE_DIR}/src/CycleTracking.cc
+    ${CMAKE_SOURCE_DIR}/src/DecompositionObject.cc
+    ${CMAKE_SOURCE_DIR}/src/DirectionCosine.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/EnergySpectrum.cc
+    ${CMAKE_SOURCE_DIR}/src/GlobalFccGrid.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/GridAssignmentObject.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/InputBlock.cc
+    ${CMAKE_SOURCE_DIR}/src/MC_Base_Particle.cc
+    ${CMAKE_SOURCE_DIR}/src/MC_Domain.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/MC_Fast_Timer.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/MC_Particle_Buffer.cc
+    ${CMAKE_SOURCE_DIR}/src/MeshPartition.cc
+    ${CMAKE_SOURCE_DIR}/src/MonteCarlo.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/MpiCommObject.cc
+    ${CMAKE_SOURCE_DIR}/src/Parameters.cc
+    ${CMAKE_SOURCE_DIR}/src/ParticleVault.cc
+    ${CMAKE_SOURCE_DIR}/src/ParticleVaultContainer.cc
+    ${CMAKE_SOURCE_DIR}/src/PopulationControl.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/SharedMemoryCommObject.cc
+    ${CMAKE_SOURCE_DIR}/src/Tallies.cc
+    ${CMAKE_SOURCE_DIR}/src/cmdLineParser.cc
+    ${CMAKE_SOURCE_DIR}/src/cudaFunctions.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/initMC.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/main.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/parseUtils.cc
+    ${CMAKE_SOURCE_DIR}/src/utils.cc
+    ${CMAKE_SOURCE_DIR}/src/utilsMpi.cc.dp.cpp
+    ${CMAKE_SOURCE_DIR}/src/Random.cc
+)
+
+# Declare the executable target built from your sources
+# add_executable(gpu_example test.cc)
+
+# enable kernel profiling on demand
+if(GEN9)
+    message("-- NOTE: Building for GEN9 archetecture")
+    add_definitions(-DGEN9)
+endif()
+
+if(ENABLE_KERNEL_PROFILING)
+    message("-- NOTE: Enabling Kernel time measurement")
+    add_definitions(-DENABLE_KERNEL_PROFILING)
+endif()
+
+if(DEBUG)
+    message("-- NOTE: Enabling debug mode")
+    add_definitions(-DDEBUG)
+endif()
+
+set(DEF_INTEL_WL_CXX_FLAGS  " -DRUN_ON_GPU=1 -DHAVE_SYCL=1 ")
+set(DEF_NVIDIA_WL_CXX_FLAGS " -DHAVE_SYCL=1 ")
+set(DEF_AMD_WL_CXX_FLAGS    " -DUNIFIED_HOST=1 -DHAVE_SYCL=1 ")
+
+set(DEF_INTEL_GENERAL_CXX_FLAGS  " -O3 -fsycl -ffast-math ")
+set(DEF_NVIDIA_GENERAL_CXX_FLAGS " -O3 -fsycl -ffast-math ")
+set(DEF_AMD_GENERAL_CXX_FLAGS    " -O3 -fsycl -ffast-math ")
+
+
+# -DCMAKE_CXX_FLAGS=" -blah -blah " overrides the default flags (BOTH general and WL specific)
+# -DOVERRIDE_GENERAL_CXX_FLAGS=" -blah -blah " overrides the general flags only (and not the workload specific flags)
+# passing in both CMAKE_CXX_FLAGS and OVERRIDE_GENERAL_CXX_FLAGS is not allowed, in order to prevent ambiguity
+
+#set(USE_DEFAULT_FLAGS OFF)
+if(NOT "${CMAKE_CXX_FLAGS}" STREQUAL "" AND NOT "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
+    message(FATAL_ERROR "Both  CMAKE_CXX_FLAGS and OVERRIDE_GENERAL_CXX_FLAGS cannot be passed in together")
+elseif("${CMAKE_CXX_FLAGS}" STREQUAL "" AND "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
+    message(STATUS "Using DEFAULT compilation flags")
+    set(INTEL_GPU_CXX_FLAGS        "${DEF_INTEL_GENERAL_CXX_FLAGS}   ${DEF_INTEL_WL_CXX_FLAGS}")
+    set(NVIDIA_GPU_CXX_FLAGS       "${DEF_NVIDIA_GENERAL_CXX_FLAGS}  ${DEF_NVIDIA_WL_CXX_FLAGS}")
+    set(AMD_GPU_CXX_FLAGS          "${DEF_AMD_GENERAL_CXX_FLAGS}     ${DEF_AMD_WL_CXX_FLAGS}")
+elseif(NOT "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
+    message(STATUS "OVERRIDING GENERAL compilation flags")
+    set(INTEL_GPU_CXX_FLAGS        "${OVERRIDE_GENERAL_CXX_FLAGS}    ${DEF_INTEL_WL_CXX_FLAGS}")
+    set(NVIDIA_GPU_CXX_FLAGS       "${OVERRIDE_GENERAL_CXX_FLAGS}    ${DEF_NVIDIA_WL_CXX_FLAGS}")
+    set(AMD_GPU_CXX_FLAGS          "${OVERRIDE_GENERAL_CXX_FLAGS}    ${DEF_AMD_WL_CXX_FLAGS}")
+elseif(NOT "${CMAKE_CXX_FLAGS}" STREQUAL "")
+    message(STATUS "OVERRIDING GENERAL and WORKLOAD SPECIFIC compilation flags")
+    set(INTEL_GPU_CXX_FLAGS        "${CMAKE_CXX_FLAGS}")
+    set(NVIDIA_GPU_CXX_FLAGS       "${CMAKE_CXX_FLAGS}")
+    set(AMD_GPU_CXX_FLAGS          "${CMAKE_CXX_FLAGS}")
+endif()
+
+if(GPU_AOT)
+    message(STATUS "Enabling INTEL backend")
+    set(CMAKE_CXX_FLAGS "${INTEL_GPU_CXX_FLAGS}")
+    if((${GPU_AOT} STREQUAL "pvc") OR(${GPU_AOT} STREQUAL "PVC"))
+        message(STATUS "Enabling Intel GPU AOT compilation for ${GPU_AOT}")
+        string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=spir64_gen -Xs \"-device 0x0bd5 -revision_id 0x2f\" -Xs \"-options -ze-opt-large-register-file\" ")
+    else()
+        message(STATUS "Using custom AOT compilation flag ${GPU_AOT}")
+        string(APPEND CMAKE_CXX_FLAGS " ${GPU_AOT} ") # User should be aware of advanced AOT compilation flags
+    endif()
+elseif(USE_NVIDIA_BACKEND)
+    message(STATUS "Enabling NVIDIA backend")
+    set(CMAKE_CXX_FLAGS "${NVIDIA_GPU_CXX_FLAGS}")
+    string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=nvptx64-nvidia-cuda -Xsycl-target-backend --cuda-gpu-arch=sm_${USE_SM} ")
+elseif(USE_AMDHIP_BACKEND)
+    message(STATUS "Enabling AMD HIP backend for ${USE_AMDHIP_BACKEND} AMD architecture")
+    set(CMAKE_CXX_FLAGS "${AMD_GPU_CXX_FLAGS}")
+    string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=${USE_AMDHIP_BACKEND}  ")
+else()
+    # JIT case
+    message(STATUS "Enabling INTEL backend")
+    set(CMAKE_CXX_FLAGS "${INTEL_GPU_CXX_FLAGS}")
+endif()
+
+message(STATUS "CXX Compilation flags set to: ${CMAKE_CXX_FLAGS}")
+
+add_executable(qs ${SRC_LIST})
+target_link_libraries(qs sycl stdc++ stdc++fs)

SYCL/LICENSE.md

@@ -0,0 +1,28 @@
+Modifications Copyright (C) 2023 Intel Corporation
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+SPDX-License-Identifier: BSD-3-Clause

README.md → SYCL/README.md

@@ -19,35 +19,47 @@ vectorization potential.
 For more information, visit the
 [LLNL co-design pages.](https://codesign.llnl.gov/quicksilver.php)
 
+**To build sycl version**
 
-Building Quicksilver
---------------------
+source /path/to/oneAPI package
 
-Instructions to build Quicksilver can be found in the
-Makefile. Quicksilver is a relatively easy to build code with no
-external dependencies (except MPI and OpenMP).  You should be able to
-build Quicksilver on nearly any system by customizing the values of
-only four variables in the Makefile:
+mkdir build && cd build
 
-* CXX The name of the C++ compiler (with path if necessary)
-  Quicksilver uses C++11 features, so a C++11 compliant compiler
-  should be used.
+CXX=icpx cmake ../ -DGPU_AOT=PVC
 
-* CXXFLAGS Command line switches to pass to the C++ compiler when
-  compiling objects *and* when linking the executable.
+make -sj
 
-* CPPFLAGS Command line switches to pass to the compiler *only* when
-  compiling objects
+**To build sycl version on nvidia backend**
 
-* LDFLAGS Command line switches to pass to the compiler *only*
-  when linking the executable
+source /path/to/clang/
 
-Sample definitions for a number of common systems are provided.
+mkdir build && cd build
 
-Quicksilver recognizes a number of pre-processor macros that enable or
-disable various code features such as MPI, OpenMP, etc.  These are
-described in the Makefile.
+//For A100 Machine
 
+CC=clang CXX=clang++ cmake ../ -DUSE_NVIDIA_BACKEND=YES -DUSE_SM=80
+
+//For H100 Machine
+
+CC=clang CXX=clang++ cmake ../ -DUSE_NVIDIA_BACKEND=YES -DUSE_SM=90
+
+make -sj
+
+**To build sycl version on amd backend**
+
+source /path/to/clang/
+
+mkdir build && cd build
+
+//For MI-100 Machine
+
+CC=clang CXX=clang++ cmake ../ -DUSE_AMDHIP_BACKEND=gfx908
+
+//For MI-250 Machine
+
+CC=clang CXX=clang++ cmake ../ -DUSE_AMDHIP_BACKEND=gfx90a
+
+make -sj
 
 Running Quicksilver
 -------------------
@@ -63,18 +75,27 @@ Quicksilver also has the property that the output of every run is a
 valid input file.  Hence you can repeat any run for which you have the
 output file by using that output as an input file.
 
+For benchmarking run the example "Examples/CORAL2_Benchmark/Problem1/Coral2_P1_1.inp"
 
-License and Distribution Information
-------------------------------------
+**To run sycl version**
+
+export QS_DEVICE=GPU
+
+./qs -i ../Examples/AllScattering/scatteringOnly.inp
 
-Quicksilver is available [on github](https://github.com/LLNL/Quicksilver)
+**To run sycl version on nvidia backend**
 
+export QS_DEVICE=GPU
 
-Quicksilver is open source software with a BSD license.  See
-[LICENSE.md](https://github.com/LLNL/Quicksilver/blob/master/LICENSE.md)
+./qs -i ../Examples/AllScattering/scatteringOnly.inp
 
-This work was performed under the auspices of the U.S. Department of
-Energy by Lawrence Livermore National Laboratory under Contract
-DE-AC52-07NA27344.
+**To run sycl version on amd backend**
+
+export QS_DEVICE=GPU
+
+ONEAPI_DEVICE_SELECTOR=hip:* ./qs -i ../Examples/AllScattering/scatteringOnly.inp
+
+License and Distribution Information
+------------------------------------
 
-LLNL-CODE-684037
+Quicksilver is available [on github](https://github.com/LLNL/Quicksilver)