#!/bin/bash
#SBATCH --job-name=CP2K_GPU
#SBATCH --partition=gpu
#SBATCH --nodes=1
#SBATCH --ntasks=2
#SBATCH --cpus-per-task=24
#SBATCH --gres=gpu:1
#SBATCH --mem=140G
#SBATCH --time=1-00:00:00
#SBATCH --output=cp2k.%j.out
#SBATCH --error=cp2k.%j.err

INPUT=GOSystem87_SA.inp
OUTPUT=GOSystem87_SA.out

set -e
ulimit -s unlimited

echo "========================================="
echo "CP2K GPU RUN"
echo "HOST: $(hostname)"
echo "DATE: $(date)"
echo "========================================="

# --------------------------------------------------
# Load CP2K environment
# --------------------------------------------------

source $HOME/load_cp2k_gpu_stack.sh

# --------------------------------------------------
# OpenMP
# --------------------------------------------------

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}

# --------------------------------------------------
# DBCSR diagnostics
# --------------------------------------------------
# Set to 2 for first validation run
# Change back to 0 after GPU usage is confirmed
# --------------------------------------------------

export DBCSR_ACC_VERBOSE=2

# --------------------------------------------------
# ELPA GPU
# --------------------------------------------------

export ELPA_DEFAULT_nvidia_gpu=1

# --------------------------------------------------
# UCX
# --------------------------------------------------

export OMPI_MCA_pml=ucx
export OMPI_MCA_osc=ucx

export UCX_TLS=rc,self,sm,cuda_copy,cuda_ipc

# --------------------------------------------------
# CP2K executable
# --------------------------------------------------

CP2K_BIN=$HOME/cp2k_build_zone/cp2k-2026.1/build/bin/cp2k.psmp

# --------------------------------------------------
# Diagnostics
# --------------------------------------------------

echo
echo "========== GPU =========="
nvidia-smi

echo
echo "========== CUDA =========="
echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"

echo
echo "========== CPU =========="
lscpu | grep "Model name"

echo
echo "========== MPI =========="
which mpirun
mpirun --version

echo
echo "========== CP2K =========="
ls -lh ${CP2K_BIN}

echo
echo "========== LIBRARIES =========="
if ldd ${CP2K_BIN} | grep "not found"; then
    echo "ERROR: Missing libraries detected."
    exit 1
fi

echo
echo "========== STARTING CP2K =========="
date

# --------------------------------------------------
# Run CP2K
# --------------------------------------------------

mpirun \
  --bind-to core \
  --map-by ppr:1:socket:PE=${OMP_NUM_THREADS} \
  -x OMP_NUM_THREADS \
  ${CP2K_BIN} \
  -i ${INPUT} \
  -o ${OUTPUT}

echo
echo "========== FINISHED =========="
date

