tenzing/load-env.sh at master · sandialabs/tenzing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#! /bin/bash

# Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the
# terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this
# software.

export TENZING_ACK_NOTICE=1

host=`hostname`

if [[ "$host" =~ .*ascicgpu.* ]]; then
    echo "$host" matched ascicgpu

    echo "export CUDAARCHS=70"
    export CUDAARCHS="70" # for cmake 3.20+

    echo "module purge"
    module purge

    mkdir -p /tmp/$USER
    export TMPDIR=/tmp/$USER

    echo "module load sierra-devel/nvidia"
    module load sierra-devel/nvidia

    echo "module load cde/v2/cmake/3.19.2"
    module load cde/v2/cmake/3.19.2

    which cmake
    which gcc
    which nvcc
    which mpirun
elif [[ "$host" =~ .*vortex.* ]]; then
# CUDA 10.1 & cmake 3.18.0 together cause some problem with recognizing the `-pthread` flag.

    echo "$host" matched vortex

    echo "export CUDAARCHS=70"
    export CUDAARCHS="70" # for cmake 3.20+
    export NUMA_ROOT_DIR=$HOME/software/numactl2.0.14-gcc7.3.1

    echo module --force purge
    module --force purge

    echo module load cmake/3.18.0
    module load cmake/3.18.0
    echo module load cuda/10.2.89
    module load cuda/10.2.89
    echo module load gcc/7.3.1
    module load gcc/7.3.1
    echo module load spectrum-mpi/rolling-release
    module load spectrum-mpi/rolling-release

    which cmake
    which gcc
    which nvcc
    which mpirun
elif [[ "$host" =~ .*weaver.* ]]; then
# CUDA 10.1 & cmake 3.18.0 together cause some problem with recognizing the `-pthread` flag.

    echo "$host" matched weaver

    echo "export CUDAARCHS=70"
    export CUDAARCHS="70" # for cmake 3.20+

    echo module --force purge
    module --force purge

    echo module load cmake/3.19.3
    module load cmake/3.19.3
    echo module load cuda/10.2.2
    module load cuda/10.2.2
    echo module load gcc/7.2.0
    module load gcc/7.2.0
    echo module load openmpi/4.0.5
    module load openmpi/4.0.5

    which cmake
    which gcc
    which nvcc
    which mpirun
elif [[ "$NERSC_HOST" =~ .*perlmutter.* ]]; then
# CUDA 10.1 & cmake 3.18.0 together cause some problem with recognizing the `-pthread` flag.

    echo "$NERSC_HOST" matched perlmutter

    echo "export CUDAARCHS=80"
    export CUDAARCHS="80" # for cmake 3.20+
    echo "export MPICH_GPU_SUPPORT_ENABLED=1"
    export MPICH_GPU_SUPPORT_ENABLED=1
    echo "export CRAY_ACCEL_TARGET=nvidia80"
    export CRAY_ACCEL_TARGET=nvidia80

    echo module load PrgEnv-gnu
    module load PrgEnv-gnu
    echo module load cmake/3.22.0
    module load cmake/3.22.0
    echo module load cudatoolkit
    module load cudatoolkit
    echo module load cpe-cuda
    module load cpe-cuda

    which cmake
    which gcc
    which nvcc
    which mpirun
fi