📋 Instructions to install complete SmartSim suite on JASMIN

install-smartsim-smartredis.sh

# For JASMIN, using a Conda environment is recommended for package installations
# due to lack of sudo privileges for users.

# NOTE: Ensure both cudatoolkit/11.8 and cudnn/8.9.7.29 are loaded in your environment 
# If not refer to `z3.install-cuda-cudnn.sh` below.

# Create a new Conda virtual environment
conda create --name venv python=3.11
conda activate venv

# Ensure your gcc, g++ and gfortran versions are 11.4.0
conda install -c conda-forge gcc=11.4.0 gxx=11.4.0 gfortran=11.4.0

# Update cmake, make and git to latest updates
conda install -c conda-forge cmake make git

# Install SmartSim and SmartRedis Python packages using pip
pip install smartsim smartredis

# Build SmartSim for GPU, skipping TensorFlow and ONNX backends
smart clean && smart build --device cuda-11 --skip-tensorflow --skip-onnx

# Ensure you are in your home directory
cd ${HOME}

# Clone the SmartRedis repository from GitHub
git clone https://github.com/CrayLabs/SmartRedis.git smartredis

# Install SmartRedis with Fortran bindings using CMake
cd smartredis
mkdir build
cd build
cmake .. -DCMAKE_INSTALL_PREFIX=${HOME}/smartredis -DBUILD_FORTRAN=ON
make install -j$(nproc)

# Add the SmartSim and SmartRedis libraries to your PATH by editing ~/.bashrc
vi ~/.bashrc

# Add the following line to the end of the file:
export LD_LIBRARY_PATH=${HOME}/smartredis/lib64:$LD_LIBRARY_PATH

# [OPTIONAL]: Create a handy alias for compiling Fortran with SmartRedis
alias gfortran-smartredis="gfortran -I${HOME}/smartredis/include -L${HOME}/smartredis/lib64 -lsmartredis -lsmartredis-fortran -cpp"

# Apply the changes made to the ~/.bashrc file
source ~/.bashrc

z1.install-redis.sh

# CAUTION: DO NOT use conda install redis -c conda-forge
# The version conda has is outdated and does not support RedisAI

# Ensure you are in your home directory
cd ${HOME}

# Download the Redis tar file
wget https://download.redis.io/redis-stable.tar.gz

# Extract the tar file
tar -xzvf redis-stable.tar.gz
cd redis-stable

# Build the Redis binaries using all available CPU cores
make -j$(nproc)  # This dynamically sets the number of cores for your system

# Modify the Makefile to set the custom installation directory
# Open the Makefile for editing
vi src/Makefile

# Inside the Makefile, find and change the following lines:
# PREFIX?=/home/users/USERNAME/redis-stable
# INSTALL_BIN=$(PREFIX)/bin
# Replace USERNAME with your actual username

# Once the changes are made, run the following to install Redis
make install

# Add the Redis binaries to your PATH by editing ~/.bashrc
vi ~/.bashrc

# Add the following line to the end of the file:
export PATH=$PATH:${HOME}/redis-stable/bin

# Apply the changes made to the ~/.bashrc file
source ~/.bashrc

# Set up the Redis environment variable for connecting to the Redis server
export SSDB="localhost:6379"

z2.install-redisai.sh

# CAUTION: RedisAI (https://github.com/RedisAI/RedisAI) is not maintained anymore.
# Please email ([email protected]) for help with installation.

# NOTE: Please ensure the following are installed in your ${HOME} directory:
# a. CUDA and CUDNN (refer to `z3.install-cuda-cudnn.sh` below)
# b. Redis (refer to `z1.install-reids.sh` above)

# Ensure you are in your home directory
cd ${HOME}

# Git clone the RedisAI repository with submodules
git clone --recursive https://github.com/RedisAI/RedisAI redisai

# Install dependencies
cd redisai
bash $(pwd)/get_deps.sh gpu  # Corrected typo to get_deps.sh

# Note: Ignore the Makefile installation in README. 
# Follow the CMake installation steps below.

# Create a build directory
mkdir build
cd build

# Build and install RedisAI using CMake, ensuring proper paths for dependencies
cmake -DCMAKE_INSTALL_PREFIX=${HOME}/redisai \
      -DDEPS_PATH=$(pwd)/../deps/linux-x64-gpu \
      -DDEVICE=gpu \
      -DBUILD_TF=OFF \
      -DBUILD_TFLITE=OFF \
      -DBUILD_ORT=OFF \
      -DCUDNN_LIBRARY_PATH=$CUDNN_LIBRARY \
      $(pwd)/..

# Compile the code using all available CPU cores
make -j$(nproc)

# Start Redis server and load redisai
redis-server --loadmodule ~/redisai/redisai.so &

z3.install-cuda-cudnn.sh

# CAUTION: Installing CUDA and CUDNN using Conda is not recommended.
# Follow the source build installation process below.

# Ensure you are in your home directory
cd ${HOME}

# Download CUDA 11.8 and CUDNN 8.9.7 directly from NVIDIA
wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
wget "https://developer.download.nvidia.com/compute/cudnn/secure/8.9.7/local_installers/12.x/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz?mlSgafHZ4lAwSLJj-yU745bcFPawcn2SwkYIjNH7tbfv4Ll7ms1dhq2IlVT03refUfWqPgj61P_5K_o5kh0zoyLwE5uBokBF2A7tFUxDMDKUrhDgsmSlNzL8nxbGfglb2pOez01un7HSg--Uo4gNuswoTp8vuzU5_-EXg0vcpAtxu18Z_CPPZoLksL2oN0j12Fk9dP39coz6Udh0sCBvR1I=&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLyJ9"

# Install CUDA 11.8
# When running the installer, ensure you deselect everything except the driver
# and set the installation path for both libraries and binaries to /home/users/USERNAME/cuda-11.8
sh cuda_11.8.0_520.61.05_linux.run

# Install CUDNN 8.9.7
tar -xzvf cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz?mlSgafHZ4lAwSLJj-yU745bcFPawcn2SwkYIjNH7tbfv4Ll7ms1dhq2IlVT03refUfWqPgj61P_5K_o5kh0zoyLwE5uBokBF2A7tFUxDMDKUrhDgsmSlNzL8nxbGfglb2pOez01un7HSg--Uo4gNuswoTp8vuzU5_-EXg0vcpAtxu18Z_CPPZoLksL2oN0j12Fk9dP39coz6Udh0sCBvR1I=&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLyJ9
mv cudnn-linux-x86_64-8.9.7.29_cuda12-archive cudnn-8.9.7.29_cuda11 # Rename the folder

# Copy libcuda.so from your system's /usr/lib64 to the CUDA library directory
cp /usr/lib64/libcuda.so ${HOME}/cuda-11.8/lib64

# Create a modulefile directory for CUDA 11.8 and CUDNN 8.9.7.29
mkdir cuda_118_modulefiles

# Create a modulefile for CUDA 11.8
vi cuda_118_modulefiles/cudatoolkit/cuda-118

# Paste this
# ------
#%Module -*- tcl -*- ##
## modulefile
proc ModulesHelp { } {

puts stderr "\tAdds CUDA 11.8.0 to your environment,"

}

module-whatis "CUDA 11.8.0 development libraries"

conflict cudatoolkit

set          cuda_home             /home/users/USERNAME/cuda-11.8
setenv       CUDA_HOME             $cuda_home
setenv       CUDA_TOOLKIT_ROOT_DIR $cuda_home
set          cuda_lib              $cuda_home/lib64
setenv       CUDA_LIBRARY          $cuda_lib
prepend-path LD_LIBRARY_PATH       $cuda_lib
prepend-path CRAY_LD_LIBRARY_PATH  $cuda_lib
setenv       CUDA_CUDART_LIBRARY   $cuda_lib/libcudart.so

prepend-path PATH                  $cuda_home/bin
setenv       CUDA_NVCC_EXECUTABLE  $cuda_home/bin/nvcc
setenv       CUDA_INCLUDE_DIRS     $cuda_home/include
# ------

# Create a modulefile for CUDNN 8.9.7.2
vi cuda_118_modulefiles/cudnn/8.9.7.2

# Paste this
# ------
#%Module -*- tcl -*- ##
## modulefile
proc ModulesHelp { } {

puts stderr "\tAdds CUDNN 8.9.7.29 to your environment,"

}

module-whatis "CUDNN 8.9.7.29 development libraries"


set          CUDNN_ROOT         /home/users/USERNAME/cudnn-8.9.7.29_cuda11
set          cudnn_lib          $CUDNN_ROOT/lib
setenv       CUDNN_INSTALL_PATH $CUDNN_ROOT
setenv       CUDNN_LIBRARY      $cudnn_lib/libcudnn.so
setenv       CUDNN_INCLUDE_DIR  $CUDNN_ROOT/include

prepend-path LD_LIBRARY_PATH    $cudnn_lib
# ------

# Add these lines by editing ~/.bashrc to ensure they are loaded everytime
vi ~/.bashrc

# Add these lines at the end of the file:
export LD_LIBRARY_PATH=${HOME}/cuda-11.8/lib64
module use ${HOME}/cuda_118_modulefiles/
module load cudatoolkit/11.8 cudnn/8.9.7.29

# Apply the changes made to the ~/.bashrc file
source ~/.bashrc

# Verify whether the installed CUDA version is the correct one
# This should output: /home/users/USERNAME/cuda-11.8
echo $CUDA_HOME

# [OPTIONAL] Clean up
rm -i cuda_11.8.0_520.61.05_linux.run cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz?mlSgafHZ4lAwSLJj-yU745bcFPawcn2SwkYIjNH7tbfv4Ll7ms1dhq2IlVT03refUfWqPgj61P_5K_o5kh0zoyLwE5uBokBF2A7tFUxDMDKUrhDgsmSlNzL8nxbGfglb2pOez01un7HSg--Uo4gNuswoTp8vuzU5_-EXg0vcpAtxu18Z_CPPZoLksL2oN0j12Fk9dP39coz6Udh0sCBvR1I=&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLyJ9