Perlmutter: SW Install Updates (#5648)

- [x] profile: avoid repetition, use `SW_DIR` variable as in `install_...` scripts - [x] move from CFS to PSCRATCH (more stable, faster, where the binary lives); uses an undocumented, purge-exempt location for container images/software - [x] build our own boost (SW stack consistency) - [x] get our own CCache (prior one is gone) - [x] RT tested
ECP-WarpX · Feb 18, 2025 · 0659286 · 0659286
1 parent 072341c
commit 0659286
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 38 deletions.
diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst
@@ -76,7 +76,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended)
       .. code-block:: bash
 
          bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh
-         source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate
+         source ${PSCRATCH}/storage/sw/warpx/perlmutter/gpu/venvs/warpx-gpu/bin/activate
 
       .. dropdown:: Script Details
          :color: light
@@ -126,7 +126,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended)
       .. code-block:: bash
 
          bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh
-         source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate
+         source ${PSCRATCH}/storage/sw/warpx/perlmutter/cpu/venvs/warpx-cpu/bin/activate
 
       .. dropdown:: Script Details
          :color: light

diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh
@@ -31,7 +31,7 @@ fi
 
 # Remove old dependencies #####################################################
 #
-SW_DIR="${CFS}/${proj}/${USER}/sw/perlmutter/cpu"
+SW_DIR="${PSCRATCH}/storage/sw/warpx/perlmutter/cpu"
 rm -rf ${SW_DIR}
 mkdir -p ${SW_DIR}
 
@@ -44,9 +44,29 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true
 # General extra dependencies ##################################################
 #
 
+# build parallelism
+PARALLEL=16
+
 # tmpfs build directory: avoids issues often seen with $HOME and is faster
 build_dir=$(mktemp -d)
 
+# CCache
+curl -Lo ccache.tar.xz https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz
+tar -xf ccache.tar.xz
+mv ccache-4.10.2-linux-x86_64 ${SW_DIR}/ccache-4.10.2
+rm -rf ccache.tar.xz
+
+# Boost (QED tables)
+rm -rf $HOME/src/boost-temp
+mkdir -p $HOME/src/boost-temp
+curl -Lo $HOME/src/boost-temp/boost.tar.gz https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz
+tar -xzf $HOME/src/boost-temp/boost.tar.gz -C $HOME/src/boost-temp
+cd $HOME/src/boost-temp/boost_1_82_0
+./bootstrap.sh --with-libraries=math --prefix=${SW_DIR}/boost-1.82.0
+./b2 cxxflags="-std=c++17" install -j ${PARALLEL}
+cd -
+rm -rf $HOME/src/boost-temp
+
 # c-blosc (I/O compression)
 if [ -d $HOME/src/c-blosc ]
 then
@@ -59,7 +79,7 @@ else
 fi
 rm -rf $HOME/src/c-blosc-pm-cpu-build
 cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
-cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel 16
+cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/c-blosc-pm-cpu-build
 
 # ADIOS2
@@ -74,7 +94,7 @@ else
 fi
 rm -rf $HOME/src/adios2-pm-cpu-build
 cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
-cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j 16
+cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j ${PARALLEL}
 rm -rf ${build_dir}/adios2-pm-cpu-build
 
 # BLAS++ (for PSATD+RZ)
@@ -89,7 +109,7 @@ else
 fi
 rm -rf $HOME/src/blaspp-pm-cpu-build
 CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31
-cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16
+cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/blaspp-pm-cpu-build
 
 # LAPACK++ (for PSATD+RZ)
@@ -104,7 +124,7 @@ else
 fi
 rm -rf $HOME/src/lapackpp-pm-cpu-build
 CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31
-cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16
+cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/lapackpp-pm-cpu-build
 
 # Python ######################################################################

diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh
@@ -31,7 +31,7 @@ fi
 
 # Remove old dependencies #####################################################
 #
-SW_DIR="${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu"
+SW_DIR="${PSCRATCH}/storage/sw/warpx/perlmutter/gpu"
 rm -rf ${SW_DIR}
 mkdir -p ${SW_DIR}
 
@@ -44,9 +44,29 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true
 # General extra dependencies ##################################################
 #
 
+# build parallelism
+PARALLEL=16
+
 # tmpfs build directory: avoids issues often seen with $HOME and is faster
 build_dir=$(mktemp -d)
 
+# CCache
+curl -Lo ccache.tar.xz https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz
+tar -xf ccache.tar.xz
+mv ccache-4.10.2-linux-x86_64 ${SW_DIR}/ccache-4.10.2
+rm -rf ccache.tar.xz
+
+# Boost (QED tables)
+rm -rf $HOME/src/boost-temp
+mkdir -p $HOME/src/boost-temp
+curl -Lo $HOME/src/boost-temp/boost.tar.gz https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz
+tar -xzf $HOME/src/boost-temp/boost.tar.gz -C $HOME/src/boost-temp
+cd $HOME/src/boost-temp/boost_1_82_0
+./bootstrap.sh --with-libraries=math --prefix=${SW_DIR}/boost-1.82.0
+./b2 cxxflags="-std=c++17" install -j ${PARALLEL}
+cd -
+rm -rf $HOME/src/boost-temp
+
 # c-blosc (I/O compression)
 if [ -d $HOME/src/c-blosc ]
 then
@@ -59,7 +79,7 @@ else
 fi
 rm -rf $HOME/src/c-blosc-pm-gpu-build
 cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
-cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel 16
+cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/c-blosc-pm-gpu-build
 
 # ADIOS2
@@ -74,7 +94,7 @@ else
 fi
 rm -rf $HOME/src/adios2-pm-gpu-build
 cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
-cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j 16
+cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j ${PARALLEL}
 rm -rf ${build_dir}/adios2-pm-gpu-build
 
 # BLAS++ (for PSATD+RZ)
@@ -89,7 +109,7 @@ else
 fi
 rm -rf $HOME/src/blaspp-pm-gpu-build
 CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31
-cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16
+cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/blaspp-pm-gpu-build
 
 # LAPACK++ (for PSATD+RZ)
@@ -104,7 +124,7 @@ else
 fi
 rm -rf $HOME/src/lapackpp-pm-gpu-build
 CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31
-cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16
+cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel ${PARALLEL}
 rm -rf ${build_dir}/lapackpp-pm-gpu-build
 
 # Python ######################################################################

diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example
@@ -10,32 +10,36 @@ module load cpu
 module load cmake/3.30.2
 module load cray-fftw/3.3.10.6
 
+# missing modules installed here
+export SW_DIR=${PSCRATCH}/storage/sw/warpx/perlmutter/cpu
+
 # optional: for QED support with detailed tables
-export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-12.3.0/boost-1.83.0-nxqk3hnci5g3wqv75wvsmuke3w74mzxi
+export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:${CMAKE_PREFIX_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:${LD_LIBRARY_PATH}
 
 # optional: for openPMD and PSATD+RZ support
 module load cray-hdf5-parallel/1.12.2.9
-export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH}
 
-export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:${LD_LIBRARY_PATH}
 
-export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH}
+export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH}
 
 # optional: CCache
-export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH
+export PATH=${SW_DIR}/ccache-4.10.2:$PATH
 
 # optional: for Python bindings or libEnsemble
 module load cray-python/3.11.5
 
-if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu" ]
+if [ -d "${SW_DIR}/venvs/warpx-cpu" ]
 then
-  source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate
+  source ${SW_DIR}/venvs/warpx-cpu/bin/activate
 fi
 
 # an alias to request an interactive batch node for one hour

diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example
@@ -14,32 +14,36 @@ module load craype-accel-nvidia80
 module load cudatoolkit
 module load cmake/3.30.2
 
+# missing modules installed here
+export SW_DIR=${PSCRATCH}/storage/sw/warpx/perlmutter/gpu
+
 # optional: for QED support with detailed tables
-export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-12.3.0/boost-1.83.0-nxqk3hnci5g3wqv75wvsmuke3w74mzxi
+export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:${CMAKE_PREFIX_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:${LD_LIBRARY_PATH}
 
 # optional: for openPMD and PSATD+RZ support
 module load cray-hdf5-parallel/1.12.2.9
-export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH}
+export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH}
 
-export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:${LD_LIBRARY_PATH}
 
-export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH}
+export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH}
 
 # optional: CCache
-export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH
+export PATH=${SW_DIR}/ccache-4.10.2:$PATH
 
 # optional: for Python bindings or libEnsemble
 module load cray-python/3.11.5
 
-if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu" ]
+if [ -d "${SW_DIR}/venvs/warpx-gpu" ]
 then
-  source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate
+  source ${SW_DIR}/venvs/warpx-gpu/bin/activate
 fi
 
 # an alias to request an interactive batch node for one hour