#!/bin/bash

# Submission script for GridEngine (GE). Each job will 
# be executed via the jobScript.sh
# This jobScript supports up to 7 parameters. Edit 
# the user specific part of the script according to 
# your program.
#
# Input to the script is a filelist with 1 file per line.
# For each file a job is started. With the parameter 
# nFilesPerJob a comma separated filelist will be 
# generated and handed to the job script. This feature
# is usefull when running many small jobs. Each
# job has its own logfile. All needed directories for the
# logfiles will be created if non existing.
#
# IMPORTANT: the hera/prometheus cluster jobs will only
# see the /hera file system. All needed scripts, programs
# and parameters have to be located on /hera or the job
# will crash. This script syncs your working dir to the submission
# dir on /hera . Make sure your scripts use the submission dir!
# Software should be taken from /cvmfs/hades.gsi.de/install/
#
# job log files will be named like inputfiles. If nFilesPerJob > 1
# the log files will contain the partnumber.
#
######################################################################
#   CONFIGURATION

user=$(whoami)
currentDir=$(pwd)

batch_mode=1

nfiles=900      # number of files
nevents=900     # number of events per file

ana_step=analysis     #sim, reco, analysis
energy=25gev
pluto_reac=18    # choose a reaction simulated in pluto from the table in script pluto_table.sh
mvd=mvd3_051020         # nomvd, mvd2, mvd3, mvd3_051020, mvd4
field_scale=1.0  # scaling factor of magnetic field
field_cutoff="no" # "no", "yes"
delta="no"
interaction_rate=
release=
recosettings=     # empty, _mvd_l1, _mvd_lit, _nomvd_litl1, _delta

submmissionbase=/hera/cbm/users/${user}/sub/dielectron/auau/centr/${energy}
submissiondir=${submmissionbase}/dielectron
 nFilesPerJob=1                                   # number of files to be analyzed by 1 job (default==1)
    jobscript=${submissiondir}/jobScript.sh       # exec script (full path, call without dot, set it executable!)
        macro=global_${ana_step}.C 
    outputdir=/hera/cbm/users/${user}/analysis/dielectron/auau/centr/${energy} # outputdir for files AND logFiles
pathoutputlog=${outputdir}/out/${field_scale}_field_${mvd}_pluto${pluto_reac} # protocol from batch farm for each file
     filename=${macro}_pluto${pluto_reac}_${field_scale}_field_${mvd}${recosettings}   # filename of log file if nFilesPerJob > 1 (partnumber will be appended)
if [ ${field_cutoff} = "yes" ]; then
     filename=${filename}_cut_${field_cutoff}
     pathoutputlog=${outputdir}/out/${field_scale}_field_${mvd}_pluto${pluto_reac}_cut_${field_cutoff} # protocol from batch farm for each file
fi
if [ ${delta} = "yes" ]; then
     filename=${filename}_delta_${interaction_rate}khz
     pathoutputlog=${outputdir}/out/${field_scale}_field_${mvd}_pluto${pluto_reac}_delta_${interaction_rate}khz # protocol from batch farm for each file
fi
par1=${submissiondir}/LitEnv.sh                   # optional par1 : environment script
par2=""                                           # optional par2 : filenumber from filelist later set
par3=${outputdir}                                 # optional par3 : outputdir (MAINDIRROOT)
par4=${nevents}                                   # optional par4 : number of events
par5=${submissiondir}/$macro                      # optional par5 : root macro
par6=${energy}                                    # optional par6
par7=${pluto_reac}                                        # optional par7 : Pluto decay channel
par8=${mvd}
par9=${field_scale}
par10=${delta}
par11=${pathoutputlog}
par12=${filename}
par13=${interaction_rate}
par14=${recosettings}
resources="-l h_rt=1:0:0,h_vmem=2G"                           # runtime < 10h, mem < 2GB

filelistname=all_files.list
filelist=${currentDir}/${filelistname}  # file list in local dir! not in submissiondir!!!
if [ ${batch_mode} -eq 1 ]
then
  par2=${submissiondir}/${filelistname}
else
  par2=${filelist}
fi


createList=yes   # (yes/no) use this to create files list with generic names (for simulation, testing)
                 # use "no" if you have a filelist available


######################################################################


#---------------------------------------------------------------------
# create a file list for submission (simulation, testing etc.)
# for real data you will have a filelist with real filenames
if [ "$createList" == "yes" ]
then
   if [ -f $filelist ]
   then
     echo "===> REMOVING EXISTING FILELIST : $filelist"
     rm -f $filelist 
   fi

   echo "===> CREATE FILELIST : $filelist"
   for ((ct=1;ct<=$nfiles;ct++))
   do
      file=$(printf "%04i" $ct)
      echo ${file} >> $filelist
   done
fi
#---------------------------------------------------------------------



nFiles=$( cat $filelist | wc -l)

#---------------------------------------------------------------------
# create needed dirs
if [ ! -d $submmissionbase ]
then
    echo "===> CREATE SUBMISSIONBASEDIR : $submmissionbase"
    mkdir -p $submmissionbase
else
    echo "===> USE SUBMISSIONBASEDIR : $submmissionbase"
fi

if [ ! -d $pathoutputlog ]
then
    echo "===> CREATE LOGDIR : $pathoutputlog"
    mkdir -p $pathoutputlog
else
    echo "===> USE LOGDIR : $pathoutputlog"
fi
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# sync the local modified stuff 
# to the submission dir
echo "===> SYNC CURENTDIR TO SUBMISSIONDIR : rsync  -vHaz $currentDir ${submmissionbase}"
 rsync  -vHaz --include="*/" --include="*.sh" --include="*.C" --include="*.list" --exclude="*" --prune-empty-dirs $currentDir ${submmissionbase}/

syncStat=$?

if [ ! $syncStat -eq 0 ]
then
     echo "===> ERROR : SYNCHRONIZATION ENCOUNTERED PROBLEMS"
fi


echo "-------------------------------------------------"
#---------------------------------------------------------------------


command="-j y -wd ${submissiondir} ${resources} -o ${pathoutputlog}/ -t 1:${nfiles} \
${jobscript}  ${par1}   ${par2}    ${par3}     ${par4} ${par5} ${par6} ${par7} ${par8} ${par9} ${par10} ${par11} ${par12} ${par13} ${par14}"
#jobscript.sh LitEnv.sh filelist   MAINDIRROOT nev     macro   energy
     
echo qsub ${command}

if [ ${batch_mode} -eq 1 ]
then
  qsub ${command}
else
  ${jobscript}  ${par1}   ${par2}    ${par3}  ${par4} ${par5} ${par6} ${par7} ${par8} ${par9} ${par10} ${par11} ${par12} ${par13} ${par14}
fi
#---------------------------------------------------------------------