import glob, sys, os, string, io, ROOT, math, commands pandapath=os.environ.get('PANDAPATH') sys.path.append(pandapath+'/macro/tpc/FOPI/python/argparse-1.2.1') sys.path.append(pandapath+'/macro/tpc/FOPI/python/xlrd-0.9.2') sys.path.append(pandapath+'/macro/tpc/FOPI/python/xlwt-0.7.5') sys.path.append(pandapath+'/macro/tpc/FOPI') from ROOT import std from parFile import create_parfile import argparse import xlrd def search(num): count=-1 for i in range(len(info)): if num in info[i]: count = i return count parser=argparse.ArgumentParser(description='runs the runReco_mb.C root script for a choosable range of files') parser.add_argument('path',help='path to data files',type=str) parser.add_argument('--outpath',help='outpath',type=str,default="outfiles_e12/Data/Cosmics/parts/") parser.add_argument('--info',help='file with the information of the datas',default="macro/tpc/FOPI/vwalbre/FieldCorr/simfiles_vw.xls",type=str) parser.add_argument('--hlp',help='show this help',action='store_true') parser.add_argument('--low',help='Lowest run number',type=int,default=3701) parser.add_argument('--high',help='Highest run number',type=int,default=99999) parser.add_argument('--runs',help='a range of run numbers to analyze, either comma separated single runs or a range -',type=str,default="") parser.add_argument('--wallcl',help='wallclock',type=int,default=0) #parser.add_argument('--clfile',help='Cluster Correction File (line-1) in tpc.files.par', type=int,default=58) parser.add_argument('--junksize',help='split the job into junks of size junksize',type=float,default=2000) parser.add_argument('--noreclust',help='no reclust',action='store_true') parser.add_argument('--noclcorr',help='no cluster correction',action='store_true') parser.add_argument('--nogain',help='disable gain correction',action='store_const',const=0,default=1) parser.add_argument('--fC',help='error parameter fC (scales only std error linear up)',type=float,default=14) parser.add_argument('--fG', help='error parameter 2 (-1 std error; -2 unscaled shape cov; <-1 scaled shape cov; >0 covOnPlane)', type=int,default=-1) parser.add_argument('--scaleMode', help='which scaling mode to use? 55 cov on plane; 56 scaled cov on plane', type=int,default=55) parser.add_argument('--fopi',help='turn on fopi detector tracks in reco',action='store_true') parser.add_argument('--fopipath',help='the path to the fopi data',type=str,default='') parser.add_argument('--onlyone',help='only one part in comb. with part',action='store_true') parser.add_argument('--part',help='only one part',type=int,default=-1) parser.add_argument('--inter',help='use interactive session instead of batch farm',action='store_true') parser.add_argument('--file',help='create file with ids and jobname',action='store_true') parser.add_argument('--errorfile',help='calculates only the files in the errorfile from mergerecofiles.py',type=str,default="") parser.add_argument('--recreateParfile',help='recreate the parfile',action='store_true') parser.add_argument('--clusterErrorFile',help='line number of the clusterErrorFile in tpc.files.par',type=int,default=-1) parser.add_argument('--nameadd',help='string to add to recofile',type=str,default='') parser.add_argument('--mergerlist',help='create a list with all files for the merger',type=str,default="mergerlist.txt") args=parser.parse_args() if args.hlp: parser.print_help() exit(0) ROOT.gROOT.ProcessLine(".x rootlogon.C") ROOT.gROOT.ProcessLine('gStyle->SetPalette(1)') ROOT.gROOT.ProcessLine('gROOT->SetStyle("Plain")') nEvts=args.junksize gaincorr=args.nogain joblist = [] submitScript='macro/tpc/FOPI/BatchTools/submitCosmicData.sh' #load datafiles #{ FileList = os.listdir(args.path) FileList.sort() #} #load the information of the DataFiles #{ infofile = xlrd.open_workbook(args.info) sheetinfo = infofile.sheet_by_name(u'DATA') erfilelist=[] info = [] for row_number in xrange(sheetinfo.nrows): if row_number == 0 or row_number == 1: continue info.append(sheetinfo.row_values(row_number)) for row in range(len(info)): if info[row][0] != '': info[row][0]=int(info[row][0]) if info[row][4] != '' : info[row][4]=round(info[row][4],1) #} if args.file: batchfile=open("batchfile.txt", "w") if args.errorfile: erfiles=open(args.errorfile, "r") for i in erfiles: print i erfilelist.append(i.replace(".reco.root\n","")) print "error files:",erfilelist runs=args.runs runList=[] colindex = runs.find(",") if colindex > 0: runs = runs.split(",") for i in range(len(runs)) : dashIndex = runs[i].find("-") if dashIndex > 0 : runs[i] = runs[i].split("-") for j in range(int(runs[i][0]), int(runs[i][1])+1) : runList.append(j) if dashIndex < 0 and len(runs) > 1 : runList.append(int(runs[i])) else : dashIndex = runs.find("-") if dashIndex > 0 : runs = runs.split("-") for j in range(int(runs[0]), int(runs[1])+1): runList.append(j) if dashIndex < 0 and len(runs) > 1 : runList = [int(runs)] if runs=="": for ir in range(args.low,args.high): runList.append(ir) print 'the runList:',runList #prepare file for batch #{ count=0 mergerfile=open(args.outpath.replace("/parts","")+"/"+args.mergerlist,'w') for files in FileList: if files[-5:] != ".root": print "No ROOT File given" continue fpath=args.path+files #search information beg=files.find("_") runNumber=int(files[beg+1:beg+5]) if not (runNumber in runList): continue count = search(runNumber) if count==-1: print "No information found to:", args.path+files continue #reject not cosmic files and defect files #{ #if info[count][1] != "cosmics": # print fpath, ": ", "No Cosmic:", info[count][1] # continue if str(info[count][2]) != '': print fpath, ": ", str(info[count][2]) continue #} vd = float(info[count] [3]/1000) field = int(info[count] [4]) efie = float(info[count] [5]) gas = str(info[count] [7]) gain = int(info[count] [6]*100) bfield = float(info[count] [8]) gain_par = float(info[count] [9]) devMapFile = int(info[count][10]) clusterErrorFile = int(info[count][11]) print "runNumber:", runNumber #Check datafile and parfile #{ RFile=ROOT.TFile.Open(fpath) tree=RFile.Get("tpcEvent") if tree==None: print "no tree found in file ",fpath RFile.Close() continue nofEvts=tree.GetEntriesFast() RFile.Close() if os.path.isfile("tpc/FOPI/par/tpc.run"+str(runNumber)+".par") and not args.recreateParfile: print 'parfile ok' else: if gas == 'NeCO2': gasf = 34 elif gas == 'ArCO2': gasf = 35 else: print "No TpcGasFile for:", gas continue if vd == 0: print "vdrift is not known... taking it from gasfile" wallc = args.wallcl print "Creating parfile..." create_parfile(runNumber, gasf, efie, gain_par, gain, devMapFile, vd, wallc,clusterErrorFile) print "Checking if parfile creating was successful..." if os.path.isfile("tpc/FOPI/par/tpc.run"+str(runNumber)+".par"): print 'parfile ok' else: print "Not possible to creat a parfile" #} #Submit #{ start=0 print "total number of events in file:", nofEvts nparts = int(float(nofEvts)/float(nEvts)) print "there will be", nparts,"files" jobid='' for i in range(nparts+1): stop=start+nEvts if stop > nofEvts: stop=nofEvts if args.part!=-1: if i != args.part: start=stop continue if args.fopi: outFileName="run_" else: outFileName= "runC_" outFileName+=str(runNumber)+"_"+str(field)+"_"+str(gas)+"_"+str(gain)+"_" num=runNumber varlist="" os.environ['datapath']=args.path varlist+='"datapath"' os.environ['num']=str(num) varlist+=',"num"' os.environ['start']=str(start) varlist+=',"start"' os.environ['stop']=str(stop) varlist+=',"stop"' os.environ['outpath']=str(args.outpath) varlist+=',"outpath"' os.environ['gaincorr']=str(gaincorr) varlist+=',"gaincorr"' os.environ['errorfC']=str(args.fC) varlist+=',"errorfC"' os.environ['errorfG']=str(args.fG) varlist+=',"errorfG"' os.environ['errorScale']=str(args.scaleMode) varlist+=',"errorScale"' os.environ['fopi']='true' if args.fopi else 'false' varlist+='fopi' os.environ['fopipath']=args.fopipath varlist+='fopipath' if args.fopi: name="run" else: name="runC" name += str(num)+"_part"+str(i) if bfield==0.0: os.environ['bfield']="0" else: os.environ['bfield']="1" varlist+=',"bfield"' if args.noreclust: os.environ['reclust']="false" else: os.environ['reclust']="true" outFileName+="recl_" name+="_recl" varlist+=',"reclust"' if args.noclcorr: os.environ['clcorr']="false" else: os.environ['clcorr']="true" outFileName+="clcorr_" name+="_clcorr" varlist+=',"clcorr"' if args.fG==-1: outFileName+="stdErr_" name+="_stdErr" elif args.fG==-2: outFileName+="shapeErr_" name+="_shapeErr" if args.scaleMode!=99: outFileName+="scale{0}_".format(args.scaleMode) name+="_scale{0}".format(args.scaleMode) elif args.fG>0: outFileName+="planeErr_" name+="_planeErr" if args.scaleMode==56: outFileName+="scaled_" name+="_scaled" if args.scaleMode==61: outFileName+="param_" name+="_param" outFileName+=args.nameadd name+="_"+args.nameadd os.environ['outnum']=str(i) outFileName+="part_{0:03d}".format(i) print "OutFile:", outFileName os.environ['outFileName']=str(outFileName) varlist+=',"outfileName"' #print "Name of Batch Job:", name os.environ['name']=str(name) varlist+=',"name"' if args.inter: os.system('. '+submitScript) start=stop continue if args.errorfile!="": if not outFileName in erfilelist: start=stop mergerfile.write(outFileName+".reco.root\n") continue qsub='sbatch -J '+name qsub+=' -o ~/sgeout/'+name qsub+=' -e ~/sgeout/'+name #qsub+=' -W depend=afterok:'+jobid qsub+=' --time=120 --mem-per-cpu=2000 '+submitScript print 'submitting:',name status, jobid = commands.getstatusoutput(qsub) if args.file: batchfile.write("jobid: "+ str(jobid) + " Name: " + str(name)+ "\n") mergerfile.write(outFileName+".reco.root\n") os.system('sleep 0.001') start=stop print "*****" #} #} mergerfile.close() if args.file: batchfile.close()