Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Doc: Major Cori Update #2978

Merged
merged 3 commits into from
Mar 18, 2022
Merged

Doc: Major Cori Update #2978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
834017c
Doc: Major Cori Update
ax3l Mar 17, 2022
9480968
Cori: Work-Around ADIOS 2.7.1 ZeroMQ
ax3l Mar 18, 2022
2996b46
Fix haswell/knl mismatch
ax3l Mar 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions Docs/source/install/hpc/cori.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,25 @@ And install ADIOS2, BLAS++ and LAPACK++:
# c-blosc (I/O compression)
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
rm -rf src/c-blosc-knl-build
cmake -S src/c-blosc -B src/c-blosc-knl-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-knl-install
cmake -S src/c-blosc -B src/c-blosc-knl-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/c-blosc-1.12.1-install
cmake --build src/c-blosc-knl-build --target install --parallel 16

# ADIOS2
git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
rm -rf src/adios2-knl-build
cmake -S src/adios2 -B src/adios2-knl-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-knl-install
cmake -S src/adios2 -B src/adios2-knl-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/adios2-2.7.1-install
cmake --build src/adios2-knl-build --target install --parallel 16
ax3l marked this conversation as resolved.
Show resolved Hide resolved

# BLAS++ (for PSATD+RZ)
git clone https://bitbucket.org/icl/blaspp.git src/blaspp
rm -rf src/blaspp-knl-build
cmake -S src/blaspp -B src/blaspp-knl-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/blaspp-master-knl-install
cmake -S src/blaspp -B src/blaspp-knl-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/blaspp-master-install
ax3l marked this conversation as resolved.
Show resolved Hide resolved
cmake --build src/blaspp-knl-build --target install --parallel 16

# LAPACK++ (for PSATD+RZ)
git clone https://bitbucket.org/icl/lapackpp.git src/lapackpp
rm -rf src/lapackpp-knl-build
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-knl-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lapackpp-master-knl-install
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-knl-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/lapackpp-master-install
ax3l marked this conversation as resolved.
Show resolved Hide resolved
cmake --build src/lapackpp-knl-build --target install --parallel 16

For PICMI and Python workflows, also install a virtual environment:
Expand All @@ -74,8 +74,8 @@ For PICMI and Python workflows, also install a virtual environment:
python3 -m pip install --user --upgrade pip
python3 -m pip install --user virtualenv

python3 -m venv $HOME/sw/venvs/knl_warpx
source $HOME/sw/venvs/knl_warpx/bin/activate
python3 -m venv $HOME/sw/knl/venvs/knl_warpx
source $HOME/sw/knl/venvs/knl_warpx/bin/activate

python3 -m pip install --upgrade pip
python3 -m pip install --upgrade wheel
Expand Down Expand Up @@ -108,13 +108,13 @@ And install ADIOS2, BLAS++ and LAPACK++:
# c-blosc (I/O compression)
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
rm -rf src/c-blosc-haswell-build
cmake -S src/c-blosc -B src/c-blosc-haswell-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-haswell-install
cmake -S src/c-blosc -B src/c-blosc-haswell-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/c-blosc-1.12.1-install
cmake --build src/c-blosc-haswell-build --target install --parallel 16

# ADIOS2
git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
rm -rf src/adios2-haswell-build
cmake -S src/adios2 -B src/adios2-haswell-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-haswell-install
cmake -S src/adios2 -B src/adios2-haswell-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/adios2-2.7.1-install
cmake --build src/adios2-haswell-build --target install --parallel 16

# BLAS++ (for PSATD+RZ)
Expand All @@ -126,7 +126,7 @@ And install ADIOS2, BLAS++ and LAPACK++:
# LAPACK++ (for PSATD+RZ)
git clone https://bitbucket.org/icl/lapackpp.git src/lapackpp
rm -rf src/lapackpp-haswell-build
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-haswell-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lapackpp-master-haswell-install
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-haswell-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/lapackpp-master-install
cmake --build src/lapackpp-haswell-build --target install --parallel 16

For PICMI and Python workflows, also install a virtual environment:
Expand All @@ -137,8 +137,8 @@ For PICMI and Python workflows, also install a virtual environment:
python3 -m pip install --user --upgrade pip
python3 -m pip install --user virtualenv

python3 -m venv $HOME/sw/venvs/haswell_warpx
source $HOME/sw/venvs/haswell_warpx/bin/activate
python3 -m venv $HOME/sw/haswell/venvs/haswell_warpx
source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate

python3 -m pip install --upgrade pip
python3 -m pip install --upgrade wheel
Expand Down Expand Up @@ -173,12 +173,12 @@ And install ADIOS2:
# c-blosc (I/O compression)
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
rm -rf src/c-blosc-gpu-build
cmake -S src/c-blosc -B src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-gpu-install
cmake -S src/c-blosc -B src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/c-blosc-1.12.1-install
cmake --build src/c-blosc-gpu-build --target install --parallel 16

git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
rm -rf src/adios2-gpu-build
cmake -S src/adios2 -B src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-gpu-install
cmake -S src/adios2 -B src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/adios2-2.7.1-install
cmake --build src/adios2-gpu-build --target install --parallel 16

For PICMI and Python workflows, also install a virtual environment:
Expand All @@ -189,8 +189,8 @@ For PICMI and Python workflows, also install a virtual environment:
python3 -m pip install --user --upgrade pip
python3 -m pip install --user virtualenv

python3 -m venv $HOME/sw/venvs/gpu_warpx
source $HOME/sw/venvs/gpu_warpx/bin/activate
python3 -m venv $HOME/sw/cori_gpu/venvs/gpu_warpx
source $HOME/sw/cori_gpu/venvs/gpu_warpx/bin/activate

python3 -m pip install --upgrade pip
python3 -m pip install --upgrade wheel
Expand Down
10 changes: 5 additions & 5 deletions Tools/machines/cori-nersc/gpu_warpx.profile.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ module purge
module load modules
module load cgpu
module load esslurm
module load gcc/8.3.0 cuda/11.4.0 cmake/3.21.3
module load gcc/8.3.0 cuda/11.4.0 cmake/3.22.1
module load openmpi

export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-gpu-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-gpu-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/adios2-2.7.1-install:$CMAKE_PREFIX_PATH

if [ -d "$HOME/sw/venvs/gpu_warpx" ]
if [ -d "$HOME/sw/cori_gpu/venvs/cori_gpu_warpx" ]
then
source $HOME/sw/venvs/gpu_warpx/bin/activate
source $HOME/sw/cori_gpu/venvs/cori_gpu_warpx/bin/activate
fi

# compiler environment hints
Expand Down
18 changes: 9 additions & 9 deletions Tools/machines/cori-nersc/haswell_warpx.profile.example
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
module swap PrgEnv-intel PrgEnv-gnu
module load cmake/3.21.3
module load cmake/3.22.1
module switch cray-libsci cray-libsci/20.09.1
module load cray-hdf5-parallel/1.10.5.2
module load cray-fftw/3.3.8.4
module load cray-python/3.7.3.2
module load cray-fftw/3.3.8.10
module load cray-python/3.9.7.1

export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-haswell-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-haswell-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master-haswell-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master-haswell-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-master-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-master-install:$CMAKE_PREFIX_PATH

if [ -d "$HOME/sw/venvs/haswell_warpx" ]
if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ]
then
source $HOME/sw/venvs/haswell_warpx/bin/activate
source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate
fi
18 changes: 9 additions & 9 deletions Tools/machines/cori-nersc/knl_warpx.profile.example
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
module swap craype-haswell craype-mic-knl
module swap PrgEnv-intel PrgEnv-gnu
module load cmake/3.21.3
module load cmake/3.22.1
module switch cray-libsci cray-libsci/20.09.1
module load cray-hdf5-parallel/1.10.5.2
module load cray-fftw/3.3.8.4
module load cray-python/3.7.3.2
module load cray-fftw/3.3.8.10
module load cray-python/3.9.7.1

export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-knl-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-knl-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master-knl-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master-knl-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-master-install:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-master-install:$CMAKE_PREFIX_PATH

if [ -d "$HOME/sw/venvs/knl_warpx" ]
if [ -d "$HOME/sw/knl/venvs/knl_warpx" ]
then
source $HOME/sw/venvs/knl_warpx/bin/activate
source $HOME/sw/knl/venvs/knl_warpx/bin/activate
fi

export CXXFLAGS="-march=knl"
Expand Down