#!/bin/bash
#SBATCH --time=1-00:00:00
#SBATCH --mem-per-cpu=3072
#SBATCH --cpus-per-task=1
#SBATCH --partition=long
#SBATCH --array=0
#SBATCH --job-name="tof_mc"
#SBATCH --comment="CBM TOF"
#SBATCH --workdir=/lustre/nyx/cbm/users/csimon/simulation/mc
#SBATCH --open-mode=truncate

if [ ! -z "$SLURM_SUBMIT_DIR" ]; then
  wdir=`pwd`
else
  echo 'script needs to be invoked by sbatch'
  exit 1
fi

source /lustre/nyx/cbm/users/csimon/cbmroot/development/csimon/bld/v1805_may18_new/config.sh

if [ -z "$VMCWORKDIR" ]; then
  echo 'no ROOT environment available'
  exit 1
fi

if [ -z "$cSetup" ]; then
  if [ -z "$1" ]; then
    echo 'no setup specified'
    exit 1
  else
    cSetup=$1
  fi
fi

if [ -z "$bNoTransportPhysics" ]; then
  if [ -z "$2" ]; then
    echo 'no run type specified'
    exit 1
  else
    bNoTransportPhysics=$2
  fi
fi

if [ -z "$iNEventsPerMCRun" ]; then
  if [ -z "$3" ]; then
    echo 'no number of events per MC run specified'
    exit 1
  else
    iNEventsPerMCRun=$3
  fi
fi

if [ -z "$iNFilesPerMCRun" ]; then
  if [ -z "$4" ]; then
    echo 'no number of files per MC run specified'
    exit 1
  else
    iNFilesPerMCRun=$4
  fi
fi

if [ -z "$mcFileName" ]; then
  if [ -z "$5" ]; then
    echo 'no MC file name specified'
    exit 1
  else
    mcFileName=$5
  fi
fi

if [ -z "$cGeometry" ]; then
  if [ -z "$6" ]; then
    echo 'no geometry tag specified'
    exit 1
  else
    cGeometry=$6
  fi
fi

if [ -z "$dTargetInteraction" ]; then
  if [ -z "$7" ]; then
    echo 'no target interaction specified'
    exit 1
  else
    dTargetInteraction=$7
  fi
fi


InputFileName=${mcFileName}.`printf "%05d" $((1+(SLURM_ARRAY_TASK_ID-1)*iNFilesPerMCRun))`.root

outdir=${wdir}/`printf "%03d" ${SLURM_ARRAY_TASK_ID}`
mkdir -p ${outdir}/data

if [ ! -d "${outdir}" ]; then
  echo 'output directory does not exist and cannot be created'
  exit 1
else

  cd ${outdir}

  cp -v ${VMCWORKDIR}/macro/tof/digitizer/.rootrc ${outdir}/
  cp -v ${VMCWORKDIR}/macro/tof/digitizer/rootlogon.C ${outdir}/

  # document the steering scripts/macros and the geometry file used in this MC simulation run
  if [ "${SLURM_ARRAY_TASK_ID}" == "1" ]; then
    cp -v ${VMCWORKDIR}/macro/tof/digitizer/production_mc.sh ${wdir}/
    cp -v ${VMCWORKDIR}/macro/tof/digitizer/run_mc.sh ${wdir}/
    cp -v ${VMCWORKDIR}/macro/tof/digitizer/run_mc_hic.C ${wdir}/
    cp -v ${VMCWORKDIR}/geometry/tof/tof_${cGeometry}.geo.root ${wdir}/
    cp -v ${VMCWORKDIR}/geometry/setup/setup_${cSetup}.C ${wdir}/
    cp -v ${VMCWORKDIR}/macro/tof/setup/counters_${cSetup}.C ${wdir}/
  fi

  # Occasionally, an incorrect memory layout of the 32-bit Geant3 on a 64-bit machine
  # may occur and terminate the 'root' process when instantiating 'TGeant3TGeo' right
  # at the beginning.
  #
  # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  # locf_() Warning: changing base from 7f9a00000000 to 7f9900000000!!!
  # This may result in program crash or incorrect results
  # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  #
  # To not lose the MC data to be produced by the affected job array task
  # its sbatch log file is grepped for a dedicated string that marks a successful
  # completion of the 'root' macro. In case this string is not found the 'root' process
  # is started again until the string finally appears in the log.
  #
  # see also:
  # https://sft.its.cern.ch/jira/browse/VMC-1
  # https://forum.gsi.de/index.php?t=msg&th=3886&
  #
  while : ; do
    root -b -q ''${VMCWORKDIR}'/macro/tof/digitizer/run_mc_hic.C('${iNEventsPerMCRun}',"'${cSetup}'",'${SLURM_ARRAY_TASK_ID}',\
                                                                 "'${InputFileName}'",'${iNFilesPerMCRun}','${bNoTransportPhysics}',\
                                                                 '${dTargetInteraction}')'
    sleep 30
    if [ "`grep 'MC Real time' ${wdir}/slurm-${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}.out | wc -l`" == "0" ]; then
      rm -rfv ${outdir}/data/*
    else
      break
    fi
  done

  echo ""
  echo "Array job ran on ${SLURM_JOB_CPUS_PER_NODE} CPUs of node ${SLURM_JOB_NODELIST} in partition ${SLURM_JOB_PARTITION}."
  echo ""

  mv -v ${wdir}/slurm-${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}.out ${outdir}/

fi