import glob,sys,subprocess,time,os,ROOT,commands,random
pandapath=os.environ.get('PANDAPATH')
print pandapath
sys.path.append(pandapath+'/macro/tpc/FOPI/python/argparse-1.2.1')
import argparse

def checkRECO(recfolder):
    recofile=recfolder+'/Cosmics_FopiMC_'+str(args.field)+args.gas+str(args.gain)+'_1_recl.reco.root '
    if not os.path.isfile(recofile):
        print 'checking: file doesnt exist'
        return False
    rfile=ROOT.TFile(recofile)
    tree=rfile.Get('cbmsim')
    if tree==None:
        print 'checking: tree not found'
        return False
    trentr=tree.GetEntriesFast()
    if trentr<args.events:
        print 'cheking: ',trentr,args.events
        return False
    return True

def genMC(folder,phi=-400):
    command=''
    command+='root -l macro/tpc/FOPI/runMCFOPI'
    if args.sphere:
        command+='Sphere'
    elif args.cyl:
        command+='Cyl'
    elif args.const:
        command+='_const'
    elif args.cosmics:
        command+='Cosmics'
    elif args.gibuu:
        command+='_GiBUU_mb'
    elif args.target:
        command+='_target_mb'
    else:
        print 'no MC generator given!'
        exit()

    command+='.C\('
    if (args.sphere or args.gibuu or args.cosmics) and args.phi[0]!=-400:
        command+='{0:2.0f}'.format(phi)+','
    else:
        command+=str(args.field)+','
    command+='\\\"'+args.gas+'\\\",'
    command+=str(args.gain) + ','
    command+='\\\"'+folder+'\\\",'
    command+=str(args.events)
    command+=',1,0,\\\"\\\"' #run,seed,addon
    if args.const or args.target:
        command+=','+str(args.mode)

    if args.const or args.cyl or args.target:
        if args.phi!=-400:
            command+=','+str(phi)
        else:
            command+=',0'

    if args.const or args.target:
        command+=',0' #theta

    if args.const or args.cyl:
        if args.nobfield:
            command+=',0'
        else:
            command+=',1'
    command+='\) -q -b\n'

    return command

def genDIGI(mcfolder,phi):
    if args.sphere or args.gibuu or args.cosmics:
        if args.phi[0]!=-400:
            field=phi
        else:
            field=86
    else:
        field=args.field
    
    folder=mcfolder
    
    command=''
    command+='root -l macro/tpc/FOPI/runDigiFOPICosmics.C\('
    command+='\\\"'+folder+'/Cosmics_FopiMC_{0:2.0f}'.format(field)+args.gas+str(args.gain)+'_1.mc.root\\\"' #filename
    command+=','+'{0:2.0f}'.format(field) #drift field
    command+=',\\\"'+args.gas+'\\\"' #gas mixture
    command+=','+str(args.gain) #gem gain

    if args.distort!="None": #distortion map
        command+=',\\\"'+args.distort+'\\\"'
    else:
        command+=',\\\"\\\"' 

    command+=','+str(args.events) #nEvents
    command+=',0' #evStart
    command+=',-1'#outnum
    
    if args.simpleResponse: #simple response
        command+=',kTRUE'


    command+='\) -q -b\n'
    return command

def genRecFolder(l1,l2,phi):
    recfolder=''
    fname=''

    if args.fC[0]!=-1:
        fname+='fC'+str(l1)
    if args.fG[0]!=-1:
        fname+='fG'+str(l2)
    if args.errors.count(0)<len(args.errors):
        fname+=errnames[l1]

    if (mcfolder[str(phi)]!=args.recoFolder and args.recoFolder!="None"):
        recfolder=args.recoFolder+'/'+fname
    else:
        recfolder=mcfolder[str(phi)]+'/'+fname

    if args.cutCov:
        recfolder+='_cutCov'
    if args.wpc:
        recfolder+='_wpc'

    if not os.path.exists(recfolder):
        os.system('mkdir '+recfolder)

    return recfolder

def genRECO(mcfolder,recfolder,l1=None,l2=None,start=0,stop=0,phi=-400):

    if not os.path.exists(recfolder):
        os.system('mkdir '+recfolder)
    if args.sphere or args.gibuu or args.cosmics:
        if args.phi[0]!=-400 and phi!=-400:
            field=phi
        else:
            field=86
    else:
        field=args.field               
        
    nobfield=args.nobfield
    hombfield=args.hombfield
    if nobfield and hombfield:
        print 'can not have homogenous and no bfield!! DECIDE!'
        exit()

    proxZ=args.proxZ
    if args.cyl and (phi<=4 or phi>=176):
        nobfield=True
        #hombfield=True
        proxZ=15

    
    

    command=''
    command+='root -l  FOPI/runRecoFOPI_MC_mb_recl_clcorr.C\('

    if args.digiFolder!="":
        command+='\\\"'+mcfolder+'/'+args.digiFolder+'\\\"'
    else:
        command+='\\\"'+mcfolder+'/Cosmics_FopiMC_{0:2.0f}'.format(field)+args.gas+str(args.gain)+'_1.raw.root\\\"'

    command+=','+'{0:2.0f}'.format(field)
    command+=',\\\"'+args.gas+'\\\",'
    command+=str(args.gain) + ','
    command+='1,'
    command+=str(stop)+','
    if args.corrs.count(0)<len(args.corrs):
        if l2==1 or l2==3:
            command+='true,true,'
        else:
            command+='false,false,'
    else:
        command+='true,true,'
    command+=str(args.reclfac)+','+str(args.recloff)+','+str(args.reclmin)+',' #recl_fac,recl_off,recl_min
    command+=str(args.proxc)+','+str(proxZ)+','+str(args.helixc)+',' #proxc,proxZs,helixc
    command+=str(start)+','#nEvStart,

    if args.split!=-1:
        command+=str(int(stop/args.split))+',' #outnum
    else:
        command+='0,' #outnum

    command+='\\\"NULL\\\",' #outfolderDrs
    command+='\\\"'+recfolder+'\\\"' #outfolder

    if args.fC[0]!=-1:
        command+=','+str(l1) #fC
    else:
        command+=',22' #fC

    if args.fG[0]!=-1:
        command+=','+str(l2) #fG
    elif args.errors.count(0)<len(args.errors):
        command+=','+str(errmode[l1]) #fG
    else:
        command+=',-1' #fG

    if nobfield:
        command+=',0'  #bfield
    elif hombfield:
        command+=',2' # bfield
    else:
        command+=',1' #bfield

    if args.corrs.count(0)<len(args.corrs):
        if l2==2 or l2==3:
            command+=',true'  #clcorr
        else:
            command+=',false' #clcorr
    else:
        command+=',false'

    if args.wpc:
        command+=',true'
    else:
        command+=',false'

    if args.cutCov:
        command+=',true'
    else:
        command+=',false'

    command+=','+str(args.scaleMode)
    nameadd=args.nameadd+getNameadd(errmode[l1])

    
    command+=',\\\"'+nameadd+'\\\"'
    if args.digiClcorr:
        command+=',true'
    command+='\) -q -b\n'
    return command

def genRECOdata(l1=None,l2=None):
    recfolder=genRecFolder(l1,l2)
    if not os.path.exists(recfolder):
        os.system('mkdir '+recfolder)

    command=''
    command+='python FOPI/BatchTools/submitCosmic.py '
    if args.runs=='None':
        print 'please specify the runs'
        exit()
    command+='--runs '+args.runs+' '
    command+='--path '+recfolder+' '
    command+='--fC '+str(l1)

    return command

def genANA(recfolder,l1=None,l2=None,phi=-400):
    command=''
    nameadd=args.nameadd+getNameadd(errmode[l1])
    
    anatodo=1
    if args.move and args.split!=-1:
        anatodo=0
        
        recofiles=[]
        for line in open('{0}/parts{1}{2}/files.txt'.format(recofolder,nameadd,corrnames[l2])):
            anatodo+=1
            recofiles.append(line.replace('\n',''))
                         
    for i in range(anatodo):
        command+='python FOPI/mberger/analyseFOPIReco_cosmics2.py '
        
        if args.move:
            anafile=recofiles[i]
        else:
            anafile=''
            anafile+=recfolder+'/Cosmics_FopiMC_'
            if args.phi[0]!=-400 and (args.sphere or args.gibuu or args.cosmics):
                anafile+='{0:2.0f}'.format(phi)
            else:
                anafile+=str(args.field)
            anafile+=args.gas+str(args.gain)+'_1'
            anafile+=nameadd
            if args.corrs.count(0)<len(args.corrs):
                anafile+=corrnames[l2]
            anafile+='.reco.root '

                    
        command+=anafile
        command+=' --outfname {0}'.format(anafile.replace('.reco.root','.hists.root'))
        
        '''
        command+='--outfname '+recfolder+'/Cosmics_FopiMC_'
        if args.phi[0]!=-400 and (args.sphere or args.gibuu or args.cosmics):
            command+='{0:2.0f}'.format(phi)
        else:
            command+=str(args.field)
   
        command+=args.gas+str(args.gain)+'_1'
        command+=nameadd
        if args.corrs.count(0)<len(args.corrs):
            command+=corrnames[l2]
        command+='.hists.root'
        '''
                    
        if args.events!=0 and args.split==-1:
            command+=' --nEvts '+str(args.events)
        if args.split!=-1:
            command+=' --nEvts '+str(args.split)

        if args.rangefile!='':
            command+=' --rangefile '+args.rangefile
        if args.move:
            command+=' --nopostprocess'

        gen='undefined'
        if args.const:
            gen='const'
        if args.cyl:
            gen='cyl'
        if args.sphere:
            gen='sphere'
        if args.cosmics:
            gen='gosmics'
        if args.gibuu:
            gen='gibuu'

        #errs="{0} {1} {2}".format(args.errors[0],args.errors[1],args.errors[2])
        #corr="{0} {1} {2} {3}".format(args.corrs[0],args.corrs[1],args.corrs[2],args.corrs[3])
        errs=errmode[l1]
        corr=corrnames[l2]
        command+=' --info "generator={0},angle={1},error={2},correction={3},field={4},gas={5},gain={6},name={7}"'.format(gen,phi,errs,corr,args.field,args.gas,args.gain,args.nameadd)
        command+='\n'

    return command

def genMerge(recofolder,l1=None,l2=None,phi=-400):
    command=''
    command+='python FOPI/BatchTools/mergerecofiles.py '
    command+=recofolder+'/parts/'
    outname='Cosmics_FopiMC_'
    
  
    if args.phi[0]!=-400 and phi!=-400 and (args.sphere or args.gibuu or args.cosmics):
        outname+='{0:2.0f}'.format(phi)
    else:
        outname+=str(args.field)

    outname+=args.gas+str(args.gain)+'_1'
    nameadd=args.nameadd+getNameadd(errmode[l1])

    outname+=nameadd

    if args.corrs.count(0)<len(args.corrs):
        outname+=corrnames[l2]
        
    outname+='.reco.root'
    
    
    if args.move:
        command+=' --move {0}/parts{1}{2}'.format(recofolder,nameadd,corrnames[l2])
    else:
        command+=' --out {0}'.format(outname)   
        
    command+=' --delIfEntry '+str(args.events)
    command+=' --pattern "_1'+nameadd+corrnames[l2]+'_[0-9]"'
    command+=' \n'
    return command

def getMemUsage(p):
    mem=0
    shared=0
    private=0
    pid=p.pid

    if os.path.isfile('/proc/'+str(pid)+'/smaps'):
        memfile=open('/proc/'+str(pid)+'/smaps','r')
    else:
        return mem

    for line in memfile:
        words=line.split()
        if words[0].find('Size:')!=-1:
            mem+=int(words[1])
        if words[0].find('Shared')!=-1:
            shared+=int(words[1])
        if words[0].find('Private')!=-1:
            private+=int(words[1])

#    print mem/1024, private/1024, shared/1024, (mem-shared)/1024
#    print psutil.Process(pid).get_ext_memory_info()
    return mem

def getNameadd(mode):
    nameadd=''
    if args.scaleMode==56 and mode==1:
        nameadd+='_scaled'
    if args.scaleMode==57 and mode==1:
        nameadd+='_numScaled'
    if args.scaleMode==65 and mode==1:
        nameadd+='_stepOnly'
    if args.scaleMode==66 and mode==1:
        nameadd+'_scaled_stepOnly'
    if args.digiClcorr:
        nameadd+='_digiClCorr'
    return nameadd

parser=argparse.ArgumentParser(description='start FOPI-ROOT stuff')
parser.add_argument('--mc',help='generate mc file',action='store_true')
parser.add_argument('--digi',help='generate digi file',action='store_true')
parser.add_argument('--reco',help='generate a reco file',action='store_true')
parser.add_argument('--ana',help='generate a hist file',action='store_true')
parser.add_argument('--merge',help='only merger reco files',action='store_true')

parser.add_argument('--data',help='do reco for data (default=mc)',action='store_true')
parser.add_argument('--runs',help='wich runs to reco in case of data',type=str,default='None')

parser.add_argument('--field',help='drift field strength to use',type=int,default=86)
parser.add_argument('--gas',help='gas to use (ar/ne)',type=str,default='ar')
parser.add_argument('--gain',help='Gain to use',type=int,default=85)
parser.add_argument('--nobfield',help='disable b-field',action='store_true')
parser.add_argument('--hombfield',help='use a homogenous b field',action='store_true')
parser.add_argument('--reclfac',help='the reclustering sqrt factor',type=float,default=0.0)
parser.add_argument('--recloff',help='the reclustering offset',type=float,default=1.)
parser.add_argument('--reclmin',help='the minimal reclustering clustersize',type=int,default=5)
parser.add_argument('--proxZ',help='Riemann Trackfinder Z stretch',type=float,default=1.45)
parser.add_argument('--proxc',help='Riemann Trackfinder proximity cut',type=float,default=2.6)
parser.add_argument('--helixc',help='Riemann Trackfinder helix proximity cut',type=float,default=1.7)
parser.add_argument('--wpc',help='use weighted plane construction in reco',action='store_true')
parser.add_argument('--cutCov',help='use cutting of cov instead of projection',action='store_true')
parser.add_argument('--scaleMode',help='0=phys, 1=pos, 2=diff, 3=mean pol, 4=mean phys, 20=diff*phys',type=int,default=99)
parser.add_argument('--nameadd',help='add to the reco file name',type=str,default="")

#the different generators
parser.add_argument('--const',help='Do constant MC generator',action='store_true')
parser.add_argument('--cyl',help='Do Cylinder MC generator',action='store_true')
parser.add_argument('--sphere',help='Do Sperical MC generator',action='store_true')
parser.add_argument('--cosmics',help='Do Cosmics MC generator',action='store_true')
parser.add_argument('--gibuu',help='Do GiBUU MC generator',action='store_true')
parser.add_argument('--target',help='Do target MC generator',action='store_true')

parser.add_argument('--move',help='move the reco file parts, dont merge',action='store_true')
parser.add_argument('--events',help='number of events to process',type=int,default=0)
parser.add_argument('--split',help='RECO ONLY: split files into junks of n events',type=int,default=-1)
parser.add_argument('--splitnum',help='which split number to run',type=int,default=-1)
parser.add_argument('--dsplit',help='RECO ONLY: splits even more',action='store_true')
parser.add_argument('--mode',help='which mode to run.\n for const: 0=along y; 1=along x; 2=any angle ',type=int, default=2)
parser.add_argument('--distort',help='use given distortion map',type=str,default="None")

parser.add_argument('--mcFolder',help='folder to store or find the MC file in',type=str,default="None")
parser.add_argument('--digiFolder',help='folder to store or find the DIGI file in',type=str,default="")
parser.add_argument('--recoFolder',help='folder to store or find the RECO file in',type=str,default="None")

parser.add_argument('--fC',help='Loop over values of fC (start stop step)',type=float,default=[-1,-1,-1],nargs=3)
parser.add_argument('--fG',help='Loop over values of fG (start stop step)',type=float,default=[-1,-1,-1],nargs=3)
parser.add_argument('--errors',help='Loop over the 3 different error calculation modes:\n Standard, No, New',type=int,default=[0,0,0],nargs=3)
parser.add_argument('--corrs',help='Loop over corrections:\n standard, recl, clcorr, clcorr+recl',type=int,default=[0,0,0,0],nargs=4)
parser.add_argument('--phi',help='loop over phi angles (const); theta angles (cyl); drift fields (sphere)',type=float,default=[-400,-400,-400],nargs=3)
parser.add_argument('--simpleResponse',help='use the simple pad response in digi',action='store_true')
parser.add_argument('--digiClcorr',help='correct digis for field effects',action='store_true')

parser.add_argument('--multiMC',help='one MC file for every run',action='store_true')
parser.add_argument('--nProcess',help='number of processes to run parallel',default=1,type=int)
parser.add_argument('--nRun',help='number of this run_stuff version. necessary if mor than 1 run_stuff.py will be started.',type=int,default=1)
parser.add_argument('--keepRunFile',help='keep the runfile. dont delete it',action='store_true')
parser.add_argument('--checkev',help='check if the number of events in the file is less than the given value, if yes redo the file',action='store_true')
parser.add_argument('--log',help='write  output to log file run_stuff.log',action='store_true')
parser.add_argument('--screen',help='use screen instead of terminal',action='store_true')
parser.add_argument('--batch',help='use batch farm',action='store_true')
parser.add_argument('--jobfile',help='the file to write the dependencies into',type=str,default='./dep.txt')
parser.add_argument('--depfile',help='the file to read the dependencies from',type=str,default='NONE')
parser.add_argument('--depar',help='use dependencys from array',action='store_true')
parser.add_argument('--rangefile',help='the file with individual ranges for histograms. (only needed when --ana).',
                    type=str,default='')
parser.add_argument('--hlp',help='print help',action='store_true')
args=parser.parse_args()

if args.hlp:
    parser.print_help()
    exit(0)

if args.mcFolder!="None" and not os.path.exists(args.mcFolder):
    print 'making folder:',args.mcFolder
    os.system('mkdir '+args.mcFolder)
if args.recoFolder!=args.mcFolder and args.recoFolder!="None" and not os.path.exists(args.recoFolder):
    print 'making folder:',args.recofolder
    os.system('mkdir '+args.recoFolder)
if args.batch:
    args.nProces=1000000000
    if args.mc or args.digi:
        args.multiMC=True


dependency=""
depar=[]
print 'The depfile used:',args.depfile
if args.depfile!='NONE':
    depfile=open(args.depfile,'read')
    for line in depfile:
#        dependency+=line.split('.')[0]+':'
#        print line.split('.')[0]
        dependency+=line.replace('\n','')+':'
        depar.append(line.replace('\n',''))


dependency=dependency[0:len(dependency)-1]
#print dependency


print 'followind dependencies are used:',dependency

if os.path.isfile(args.jobfile):
    os.system('mv -v ' +args.jobfile+' '+args.jobfile.replace('.txt','_old.txt'))

jobfile=open(args.jobfile,'write')


loop1var=[1,1,1]
loop2var=[1,1,1]
errnames=['standard','no_error','new_error']
errmode=[-1,-2,1]
corrnames=['','_recl','_clcorr','_recl_clcorr']

philoop=[1,1,1]

if args.fC[0]!=-1:
    loop1var=args.fC
if args.fG[0]!=-1:
    loop2var=args.fG

if args.errors.count(0)<len(args.errors):
    loop1var=[0,2,1]
if args.corrs.count(0)<len(args.corrs):
    loop2var=[0,3,1]
if args.phi[0]!=-400:
    philoop=args.phi

l1=loop1var[0]
l2=loop2var[0]
l1num=0
l2num=0

phi=philoop[0]
phinum=0

running={}
procinfo={}
to_del=[]
doMC={}
doDigi={}
#lock=False
lock={}
mcfolder={}
depcounter=-1

while phi<=philoop[1]:

    if lock.get(str(phi),None)==None:
        lock[str(phi)]=False
    if doMC.get(str(phi),None)==None:
        doMC[str(phi)]=True
    if doDigi.get(str(phi),None)==None:
        doDigi[str(phi)]=True

    if args.const:
        if mcfolder.get(str(phi),None)==None:
            if args.phi[0]!=-400:
                mcfolder[str(phi)]=args.mcFolder+'/phi_'+str(phi)
            else:
                mcfolder[str(phi)]=args.mcFolder
    elif args.cyl:
        # be aware, if cylgen: phi is used for theta!
        if mcfolder.get(str(phi),None)==None:
            if args.phi[0]!=-400:
                mcfolder[str(phi)]=args.mcFolder+'/theta_'+str(phi)
            else:
                mcfolder[str(phi)]=args.mcFolder
    elif args.sphere or args.gibuu or args.cosmics or args.target:
        if mcfolder.get(str(phi),None)==None:
            if args.phi[0]!=-400:
                mcfolder[str(phi)]=args.mcFolder+'/drift_'+str(phi)
            else:
                mcfolder[str(phi)]=args.mcFolder+'/drift_'+str(args.field)+'.0'
    else:
        if mcfolder.get(str(phi),None)==None:
            if args.phi[0]!=-400:
                mcfolder[str(phi)]=args.mcFolder+'/num_'+str(phi)
            else:
                mcfolder[str(phi)]=args.mcFolder

    if not os.path.exists(mcfolder[str(phi)]):
        os.system('mkdir '+mcfolder[str(phi)])
        
    print 'phi=',phi

    while l2 <= loop2var[1]:
        while l1 <= loop1var[1]:
            title='nothing'
            #if less processes are running the defined start a new one.
            if len(running)<args.nProcess:


                #if error looping is enabled, skip the ones which are set to 0
                if args.errors.count(0)<len(args.errors) and args.errors[l1]==0:
                    if not lock[str(phi)]:
                        l1+=loop1var[2]
                        l1num+=1
                    continue

                if args.corrs.count(0)<len(args.corrs) and args.corrs[l2]==0:
                    if not lock[str(phi)]:
                        l1+=loop1var[2]
                        l1num+=1
                    continue

                recofolder=genRecFolder(l1,l2,phi)
                randID=random.randint(0,10000)
                loopfname="loopfile"+str(args.nRun)+'_'+str(l1num)+'_'+str(l2num)+'_'+str(phi)+str(randID)
                if args.mc and doMC[str(phi)]:
                    loopfname+='_MC'
                if args.digi and doDigi[str(phi)]:
                    loopfname+='_Digi'
                loopfile=open(loopfname,'w')
                loopfile.write('#!/bin/bash\n')
                if args.screen or args.batch:
                    loopfile.write('pwd\n')
                    loopfile.write('export PANDAPATH='+pandapath+'\n')
                    loopfile.write('cd $PANDAPATH\n')
                    loopfile.write('pwd\n')
                    loopfile.write('echo '+loopfname+'\n')
                    loopfile.write('. ./setenv\n')


                #generate the line in file to generate MC. only done once
                if args.mc and doMC[str(phi)]:
                    loopfile.write(genMC(mcfolder[str(phi)],phi))
                    lock[str(phi)]=True
                    if args.multiMC:
                        lock[str(phi)]=False
                    doMC[str(phi)]=False


                #generate the line in file to do digi. only done once
                if args.digi and doDigi[str(phi)]:
                    loopfile.write(genDIGI(mcfolder[str(phi)],phi))
                    lock[str(phi)]=True
                    if args.multiMC:
                        lock[str(phi)]=False
                    doDigi[str(phi)]=False


                #generate the line in file to do reco. here the loop applies.
                if args.reco and not lock[str(phi)]:     
                    splitcounter=0
                    addon=''
                    if args.split!=-1:
                        start=0
                        stop=0
                        addon='/parts'
                    else:
                        start=0
                        stop=args.events+1

                    while stop!=args.events:
                        stop+=args.split
                        if stop>args.events:
                            stop=args.events

                        if args.data:
                            loopfile.write(genRECOdata(l1,l2))
                        else:
                            if args.checkev and not checkRECO(recofolder) and not args.merge:
                                print 'Checking enabled but events dont match -> Starting'
                                loopfile.write(genRECO(mcfolder[str(phi)],recofolder+addon,l1,l2,start,stop,phi))
                            elif not args.checkev and not args.merge:
                                loopfile.write(genRECO(mcfolder[str(phi)],recofolder+addon,l1,l2,start,stop,phi))
                            elif not args.merge:
                                print 'Checking enabled and file is ok -> skipping ('+str(l1)+','+str(l2)+')'
                        start=stop
                    if args.split!=-1:
                        print 'writing merge'
                        loopfile.write(genMerge(recofolder,l1,l2,phi))

                #generate the line in file to do ana. 
                if args.ana and not lock[str(phi)]:
                    loopfile.write(genANA(recofolder,l1,l2,phi))

                #loopfile.write('rm '+loopfname)
                loopfile.close()

    #            loopfile=open(loopfname,'w')
    #            loopfile.close()
                title=''
                if args.const:
                    title+='const'+str(args.mode)+'_'
                if args.cyl:
                    title+='cyl_'
                if args.sphere:
                    title+='sphere_'
                if args.gibuu:
                    title+='gibuu_'
                if args.cosmics:
                    title+='cosmics_'
                if args.target:
                    title+='target_'
                    
                if lock[str(phi)] or args.multiMC:
                    if args.mc:
                        title+='MC_'
                    if args.digi:
                        title+='DIGI_'

                if not lock[str(phi)] and not args.multiMC:
                    if args.reco:
                        title+='RECO_'
                    if args.ana:
                        title+='ANA_'
                    if args.errors.count(0)<len(args.errors):
                        title+=errnames[l1]+'_'
                    if args.corrs.count(0)<len(args.corrs):
                        title+=corrnames[l2]+'_'
                if args.phi[0]!=-1:
                    title+='P'+str(phi)+'_'
                    
                title+='R'+str(args.nRun)

                os.system('chmod a+x '+loopfname)
                if args.screen:
                    runCommand=['screen','-D','-m','-S',title,'./'+loopfname]
                    if args.log:
                        os.environ['SCREEN_LOG_NAME']=title+"_"+loopfname
                        runCommand.insert(1,'-L')
                elif args.batch:
                    title=title.replace('_','')
                    title=title.replace('.','')
                    title+='ID'+str(randID)
                    if args.split!=-1 and not (args.mc or args.digi):
                        splitcounter=-1
                        preambel=''
                        jobs_depend=''
                        for line in open(loopfname,'r'):
                            if line.find('root -l')==-1 and line.find('python')==-1:
                                preambel+=line
                                continue
                            splitcounter+=1
                            newtitle=title+'S'+str(splitcounter)
                            tmpfile=open(loopfname+'_S'+str(splitcounter),'w')
                            tmpfile.write(preambel)
                            tmpfile.write(line)
                            tmpfile.close()
                            os.system('chmod a+x '+loopfname+'_S'+str(splitcounter))
                            if line.find('root -l')!=-1 and (args.splitnum==-1 or args.splitnum==splitcounter):

                                qsub='sbatch -J '+newtitle
                                qsub+=' -o ~/sgeout/'+newtitle+' -e ~/sgeout/'+newtitle
                                if dependency!="":
                                    qsub+='-d afterok:'+dependency+' '
                                qsub+=' --time=30 --mem-per-cpu=1000 -p farmq '
                                qsub+=loopfname+'_S'+str(splitcounter)
                                
                                status, jobid = commands.getstatusoutput(qsub)
                                jobid=jobid[jobid.find('job')+4:len(jobid)]
                                jobs_depend+=jobid+':'
                                print qsub
                                print 'job:',jobid
                            if line.find('python')!=-1:
                                if len(jobs_depend)>0:
                                    if  jobs_depend[-1]==':':
                                        jobs_depend=jobs_depend[0:len(jobs_depend)-1]
                                if dependency!="":
                                    jobs_depend+=':'+dependency
                                    if jobs_depend[0]==':':
                                        jobs_depend=jobs_depend[1:len(jobs_depend)]
            

                                qsub='sbatch -J '+newtitle
                                qsub+=' --time=30 --mem-per-cpu=1000 -p farmq '
                                qsub+=' -o ~/sgeout/'+newtitle
                                qsub+=' -e ~/sgeout/'+newtitle
                                if jobs_depend!='':
                                    qsub+=' -d afterok:'+jobs_depend
                                qsub+=' '+loopfname+'_S'+str(splitcounter)

                                print qsub
                                status, jobid = commands.getstatusoutput(qsub)
                                print status,jobid
                                jobid=jobid[jobid.find('job')+4:len(jobid)]
                                print 'job:',jobid
                                jobfile.write(jobid+'\n')
                                if line.find('mergerecofiles')!=-1:
                                    jobs_depend=jobid
                                    newtitle='S2_'+newtitle
                    else:
                        runCommand=['sbatch','-J '+title,'-o ~/sgeout/'+title,'-e ~/sgeout/'+title,'--time=120 -mem-per-cpu=4000 -p farmq']
                        depcounter+=1
                        if args.depar:
                            dependency=depar[depcounter]
                        if dependency!="":
                            runCommand.append('-d afterok:'+dependency)
                        runCommand.append(' '+loopfname)
                        
                else:
                    runCommand=['gnome-terminal','--title='+title,'--disable-factory','-e','./'+loopfname]
                    #logging not working. to debug later
                    if args.log:
                        runCommand.append('> run_stuff.log')

                print 'starting: l1=',l1,'('+str(l1num)+') l2=',l2,'('+str(l2num)+')'+ 'phi=',phi,'('+str(phi)+')'
                print loopfname
                if args.batch and args.split==-1:
                    newcomm=''
                    for c in runCommand:
                        newcomm+=c+' '
                    status, jobid = commands.getstatusoutput(newcomm)
                    jobid=jobid[jobid.find('job')+4:len(jobid)]
                    #print newcomm
                    print 'job:',jobid
                    jobfile.write(jobid+'\n')

                    #os.system(newcomm)
                elif not args.batch:
                    running[loopfname]=subprocess.Popen(runCommand)

                    procinfo[loopfname]={}
                    procinfo[loopfname]['start_time']=time.time()
                    procinfo[loopfname]['mem_usage']=[]
                    if os.path.exists(recofolder):
                        procinfo[loopfname]['recFolder']=recofolder
                    elif os.path.exists(mcfolder[str(phi)]):
                        procinfo[loopfname]['recFolder']=mcfolder[str(phi)]
                    else:
                        procinfo[loopfname]['recFolder']='./'
                    procinfo[loopfname]['phi']=phi
                print 'checking lock:',lock[str(phi)]

                if not lock[str(phi)]:
                    print 'no lock'
                    l1+=loop1var[2]
                    l1num+=1
            if args.batch:
                time.sleep(.01)
            else:
                time.sleep(2)


            #check how many processes are already finished and write them down to del list.
            for p in running:
                running[p].poll()
                if running[p].returncode==0:
                    to_del.append(p)
                    procinfo[p]['stop_time']=time.time()
                    procinfo[p]['runtime']=procinfo[p]['stop_time']-procinfo[p]['start_time']
                    print 'job',p,'has finished'
#                    lock[str(procinfo[p]['phi'])]=False
                else:
                    if not args.screen:
                        procinfo[p]['mem_usage'].append(getMemUsage(running[p]))
                    print 'job',p,'is still running'

                if lock[str(phi)] and (p.find('MC')!=-1 or p.find('Digi')!=-1 ):
                    print 'lock is enabled, waiting now for:',p
                    running[p].wait()
                    procinfo[p]['stop_time']=time.time()
                    procinfo[p]['runtime']=procinfo[p]['stop_time']-procinfo[p]['start_time']
                    to_del.append(p)
                    print 'lock is enabled, job',p,'finished'
                    lock[str(phi)]=False
                    print 'lock for',phi,'disabled'

            #delete finished jobs
            for p in to_del:
                if running.get(p,None)!=None:
                    running.pop(p)
                if len(procinfo[p]['mem_usage'])!=0:
                    usedMem=max(procinfo[p]['mem_usage'])/1024
                    timestep=procinfo[p]['runtime']/len(procinfo[p]['mem_usage'])
                else:
                    usedMem=0
                    timestep=0
                if not args.keepRunFile:
                    os.system('rm '+p)

                usagefile=ROOT.TFile(procinfo[p]['recFolder']+'/procinfo.root','recreate')
                memgraph=ROOT.TGraph()

                for i in range(len(procinfo[p]['mem_usage'])):
                    memgraph.SetPoint(i,i*timestep,procinfo[p]['mem_usage'][i]/1024)
                memgraph.Write()
                usagefile.Close()
            del to_del[:]
            print 'there are',len(running),'running jobs'

        l2+=loop2var[2]
        l2num+=1
        l1=loop1var[0]
        l1num=0
        
    l2num=0
    l2=loop2var[0]
    phi+=philoop[2]
    phinum+=1

while len(running)>0:
    for p in running:
        running[p].poll()
        if running[p].returncode==0:
            to_del.append(p)
            procinfo[p]['stop_time']=time.time()
            procinfo[p]['runtime']=procinfo[p]['stop_time']-procinfo[p]['start_time']
        else:
            if not args.screen:
                procinfo[p]['mem_usage'].append(getMemUsage(running[p]))

    #delete finished jobs
    for p in to_del:
        if running.get(p,None)!=None:
            running.pop(p)
        print 'job:',p,'finished'
        if len(procinfo[p]['mem_usage'])>0:
            print 'Memory used:',sum(procinfo[p]['mem_usage'])/len(procinfo[p]['mem_usage'])/1024,'MB'
        print 'Runtime:',procinfo[p]['runtime'],'seconds'
        if os.path.isfile(p)and not args.keepRunFile:
            os.system('rm '+p)
    time.sleep(2)

jobfile.close()
#if not args.batch:
#    u=raw_input('wait to finish')