import glob,sys,subprocess,time,os,ROOT
pandapath=os.environ.get('PANDAPATH')
home=os.environ.get('HOME')
sys.path.append(pandapath+'/macro/tpc/FOPI/python/argparse-1.2.1')
import argparse, commands

parser=argparse.ArgumentParser(description='check if batchfiles were fiished correctly')
parser.add_argument('--path',help='path to batch output folder',type=str,default=home+"/sgeout/")
parser.add_argument('--dev',help='delete files which have n events',type=int,default=[-1],nargs='+')
parser.add_argument('--dme',help='delete files which are mergers',action='store_true')
parser.add_argument('--complete',help='delete completed files',action='store_true')
parser.add_argument('--delolder',help='delete files older than (JJJJ-MM-DD-HH-mm)',type=str,default='')
parser.add_argument('--resubmit',help='resubmit jobs which failed',action='store_true')
parser.add_argument('--resubmitAll',help='resubmit jobs which are sill in path',action='store_true')
parser.add_argument('--resubExclude',help='exclude files with this pattern in filename from resubmitting',type=str,default='aPatternHopefullyNotInTheFileName')
parser.add_argument('--delResub',help='delete output files which were resubmitted',action='store_true')
args=parser.parse_args()

#print args.path
#os.system('ls '+args.path)

runtimes=open('batch_runtimes.txt','a')
ndefect=0
for dirpath,dirnames,filenames in os.walk(args.path):
    #filenames.sort()
    totev=-1
    for filename in filenames:
#        infile=open(dirpath+filename,'r')
        good=False
        delete=False

        if args.delolder!='':
            ctime=time.localtime(os.path.getctime(dirpath+filename))
            oldtime=time.strptime(args.delolder,'%Y-%m-%d-%H-%M')
            if ctime<oldtime:
                #print "file created: {0} will be deleted".format(time.ctime(os.path.getctime(args.path+'/'+filename)))
                print 'filename {0} is too old'.format(filename)
                os.system('rm -v {0}'.format(dirpath+filename))
                continue
                
        rtime=0
        rtimeType=''
        nevents=0
        force_stop=False
        for line in reversed(list(open(dirpath+filename,'r'))):

            if line.find("CANCELLED")!=-1:
                print  dirpath+filename,'was cancelled!'
                good=False
                force_stop=True

            if line.find("Real time")!=-1:
                rtime=float(line.split()[2])
                if(filename.find('RECO'))!=-1:
                    rtimeType='RECO'
                #print rtime,rtimeType

            if line.find('total runtime')!=-1:
                rtime=float(line.split()[2])
                if(filename.find('ANA'))!=-1:
                    rtimeType='ANA'
                print rtime,rtimeType

            if line.find('IEVENT=')!=-1:
                words=line.split()
                print dirpath+filename,'has',words[3],'events'
                nevents=int(words[3])
                if args.dev[0]!=-1 and int(words[3]) in args.dev:
                    delete=True
                good=True

            if line.find('************** EVENT:')!=-1:
                words=line.split()
                print dirpath+filename,'has',words[2],'events'
                
                try:
                    int(words[2])
                except ValueError:
                    good=False

                nevents=int(words[2])
                if args.dev[0]!=-1 and int(words[2]) in args.dev:
                    delete=True
                good=True

            if line.find('Total number of events:')!=-1:
                totev=int(line.split(':')[-1])
                print dirpath+filename,'was a merger with',totev,'events'
                nevents=totev
                if args.dme and totev in args.dev:
                    delete=True
                good=True
                

            if line.find('merging all above files')!=-1:
                pass
            
            if line.find('Events:')!=-1:
                words=line.split()
                print dirpath+filename,'has', words[1],'events'
                nevents=int(words[1])
                good=True
                if args.dev[0]!=-1 and int(words[1]) in args.dev:
                    delete=True

#            if line.find('all done')!=-1:
#                print dirpath+filename,'is complete'
#                if rtimeType!='ANA':
#                    delete=True
#                good=True

            if line.find('99%')!=-1:
                words=line.split()
                print dirpath+filename,'is 100% complete'
                if (words[1] =='99%' or words[0]=='99%') and args.complete:
                    if rtimeType!='ANA':
                        delete=True
                good=True

            if line.find('Good event:')!=-1:
                words=line.split()
                print dirpath+filename ,'has',words[4],'events'
                nevents=int(words[4])
                good=True
                if args.dev[0]!=-1 and int(words[4]) in args.dev:
                    delete=True

            if line.find('no files to merge in')!=-1:
                print dirpath+filename,'was a merger with nothing to merge'
                good=True

            if line.find('moved')!=-1 and line.find('files, found')!=-1:
                print  dirpath+filename,'was a merger who moved files'
                good=True
                if args.complete:
                    delete=True
                
            if line.find('all events processed')!=-1:
                print dirpath+filename,'processed all events (is 100% complete)'
                good=True
                if args.complete:
                    delete=True
            
            if line.find('writing times')!=-1:
                print dirpath+filename,'was a bat paramscan file'
                good=True
                if args.complete:
                    delete=True
                    
            if line.find('All done!')!=-1:
                print dirpath+filename,'processed all bins'
                good=True
                if args.complete:
                    delete=True

            if line.find('% *************')!=-1:
                print line.split()[1],'done of file',filename
                good=True
                
            if line.find('RealTime=')!=-1:
                print dirpath+filename,'completed reco'
                good=True
                if args.complete:
                    delete=True

            if line.find('% done')!=-1:
                words=line.split()
                print "{0} done".format(words[0])
                good=True

            if line.find('all merged')!=-1:
                print "{0}{1} merged everything".format(dirpath,filename)
                good=True
                delete=args.complete

            if delete:
                os.system('rm -v '+dirpath+filename)
                if(rtimeType!=''):
                    print '{0};{1};{2}\n'.format(rtimeType,rtime,nevents)
                    runtimes.write('{0};{1};{2}\n'.format(rtimeType,rtime,nevents))
                
            if good or force_stop:
                break
 
        if not good:
            print '{0}{1} seems to be defect'.format(dirpath,filename)
            ndefect+=1

        if (args.resubmit and not good) or args.resubmitAll:
            if(filename.find(args.resubExclude))==-1:
                for line in (open(dirpath+filename,'r')):
                    if line.find('loopfile')!=-1:
                        loopfname=line.replace("\n","")
                        break
                print loopfname
                
                qsub='sbatch -J '+filename
                qsub+=' --time=120 --mem-per-cpu=2000 -p farmq '
                qsub+=' -o ~/sgeout/'+filename
                qsub+=' -e ~/sgeout/'+filename
                qsub+=' '+loopfname

                #print qsub
                status, jobid = commands.getstatusoutput(qsub)
            if args.delResub:
                os.system('rm -v '+dirpath+filename)

print 'there were',ndefect,'defect files and',len(filenames),'files in total' 
runtimes.close()