import os,time,math #os.system('. rootlogin 534-07') import ROOT def makeNewUser(): newUser={} newUser['njobs']=0 newUser['nRjobs']=0 newUser['nQjobs']=0 newUser['totTime']=0 newUser['nHjobs']=0 newUser['Rids']={} newUser['AvgTime']=0 newUser['share']=100 newUser['Avg8Time']=0 newUser['Avg2Time']=0 return newUser colors=[1,2,3,4,94,6,7,8,9,31,41,51,61,28,81,46,87,95,97,30,100] readonly=True writeonly=False #readonly=False #writeonly=True if not writeonly: gcanv=ROOT.TCanvas('canvas','Batch usage',2000,2000) #pad for graph gpad_1=ROOT.TPad('gpad_1','a pad',0,0,0.68,1) gpad_1.Divide(1,3) gpad_1.cd(1).SetLeftMargin(0.03) gpad_1.cd(2).SetLeftMargin(0.03) gpad_1.cd(3).SetLeftMargin(0) gpad_1.cd(1).SetRightMargin(0.05) gpad_1.cd(2).SetRightMargin(0.05) gpad_1.cd(3).SetRightMargin(0) gpad_1.cd(3).Divide(2,1) gpad_1.cd(3).cd(1).SetLeftMargin(0.05) gpad_1.cd(3).cd(1).SetRightMargin(0.045) gpad_1.cd(3).cd(2).SetLeftMargin(0.045) gpad_1.cd(3).cd(2).SetRightMargin(0.08) gpad_3=ROOT.TPad('gpad_3','a pad',0.68,0,1,1) gpad_3.Divide(1,3) gpad_3.cd(1).SetLeftMargin(0.1) gpad_3.cd(1).SetRightMargin(0.02) gpad_3.cd(2).SetLeftMargin(0.1) gpad_3.cd(2).SetRightMargin(0.02) gpad_3.cd(3).Divide(2,1) gcanv.cd() gpad_1.Draw() gpad_3.Draw() first=True reset=True changed=False users_g={} minis={} maxis={} timerange= 8*60*60 #in seconds #usepath='/scratch/e12i/knucl/' usepath='./outfiles_e12/' #timerange=2000*60*60 if os.path.isfile(usepath+'batchuseage.txt'): for line in open(usepath+'batchuseage.txt','r'): words=line.split(';') user=words[0] if user=='walbrech': user=='walbrecht' if user=='schweiss': user='schweisshelm' timestamp=float(words[-1]) if (users_g.get(user,None))==None: users_g[user]={} users_g[user]['counter']=0 users_g[user]['njobs']=ROOT.TGraph() users_g[user]['njobs'].SetTitle("Total Jobs") users_g[user]['nRjobs']=ROOT.TGraph() users_g[user]['nRjobs'].SetTitle("Running Jobs") users_g[user]['nQjobs']=ROOT.TGraph() users_g[user]['nQjobs'].SetTitle('Queued Jobs') users_g[user]['nHjobs']=ROOT.TGraph() users_g[user]['nHjobs'].SetTitle("Halted Jobs") users_g[user]['totTime']=0 users_g[user]['AvgTime']=ROOT.TGraph() users_g[user]['AvgTime'].SetTitle("Average Computing Time") users_g[user]['share']=ROOT.TGraph() users_g[user]['share'].SetTitle('Fair Share Rank') users_g[user]['njobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nRjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nHjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nQjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['AvgTime'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['share'].SetPoint(users_g[user]['counter'],timestamp,0) if timestamp > time.time()-timerange: last_x,last_y=ROOT.Double(-1),ROOT.Double(-1) if users_g[user]['counter']>=0: users_g[user]['njobs'].GetPoint(users_g[user]['counter'],last_x,last_y) if last_y == 0: users_g[user]['counter']+=1 users_g[user]['njobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nRjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nHjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nQjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['AvgTime'].SetPoint(users_g[user]['counter'],timestamp,0) share=100 if len(words)>8: share=float(words[7]) if share<0: share=0 users_g[user]['share'].SetPoint(users_g[user]['counter'],timestamp,share) users_g[user]['counter']+=1 users_g[user]['njobs'].SetPoint(users_g[user]['counter'],timestamp,int(words[1])) users_g[user]['nRjobs'].SetPoint(users_g[user]['counter'],timestamp,int(words[2])) users_g[user]['nHjobs'].SetPoint(users_g[user]['counter'],timestamp,int(words[3])) users_g[user]['nQjobs'].SetPoint(users_g[user]['counter'],timestamp,int(words[4])) users_g[user]['AvgTime'].SetPoint(users_g[user]['counter'],timestamp,float(words[6])) share=100 if len(words)>8: share=float(words[7]) if share<0: share=0 users_g[user]['share'].SetPoint(users_g[user]['counter'],timestamp,share) maxis['njobs']=max(maxis.get('njobs',0),int(words[1])) maxis['nRjobs']=max(maxis.get('nRjobs',0),int(words[2])) maxis['nHjobs']=max(maxis.get('nHjobs',0),int(words[3])) maxis['nQjobs']=max(maxis.get('nQjobs',0),int(words[4])) maxis['AvgTime']=max(maxis.get('AvgTime',0),float(words[6])) if len(words)>8: maxis['share']=max(maxis.get('share',0),share) while(1): if not readonly: os.system('qstat -n -1 > batchtmp.txt') os.system('cp batchtmp.txt '+usepath) os.system('diagnose -f > batchtmp_fair.txt') os.system('cp batchtmp_fair.txt '+usepath) if first: users={} else: for u in users: for val in users[u]: users[u][val]=0 linecount=-1 inblock=False users_f={} for line in open(usepath+'batchtmp_fair.txt'): linecount+=1 if line[0]=='\n' or line[0]=='-': continue if line.find('GROUP')!=-1: inblock=True continue if line.find('CLASS')!=-1: inblock=False if inblock: words=line.split() users_f[words[0]]=float(words[1]) else: continue linecount=-1 if reset: changed=False else: reset=True for line in open(usepath+'batchtmp.txt','r'): linecount+=1 if linecount<5: continue words=line.split() if len(words)<10: continue uname=words[1] if uname=='walbrech': uname='walbrecht' if uname=='schweiss': uname='schweisshelm' if (users.get(uname,None))==None: users[uname]=makeNewUser() changed=True if words[9]=='R': users[uname]['nRjobs']+=1 users[uname]['njobs']+=1 #users[uname]['Rids'][words[0].split('.')[0]]=words[11].split('/')[0] elif words[9]=='Q': users[uname]['njobs']+=1 users[uname]['nQjobs']+=1 elif words[9]=='H': users[uname]['nHjobs']+=1 users[uname]['njobs']+=1 if words[10]!='--': times=words[10].split(':') minutes=int(times[0])*60+int(times[1]) users[uname]['totTime']+=minutes if not readonly: os.system('rm batchtmp.txt') for fu in users_f: if users.get(fu,None) != None: users[fu]['share']=100-users_f[fu] else: users[fu]=makeNewUser() users[fu]['share']=100-users_f[fu] if (first or changed): if changed and not first: del hRunning del hQueued del hHalted del hTotJobs del hTotTime del hTot8Time del hTot2Time compUsers=len([x for x in users if users[x]['njobs']>0 ]) hRunning=ROOT.TH1I('hRunning','Running',compUsers,0,compUsers) hRunning.SetFillStyle(3004) hRunning.SetFillColor(2) hRunning.SetLineColor(2) hRunning.SetStats(0) hRunning.SetMarkerSize(1.5) hRunning.SetMarkerColor(2) hQueued=ROOT.TH1I('hQueued','Queued',compUsers,0,compUsers) hQueued.SetFillStyle(3005) hQueued.SetLineColor(3) hQueued.SetFillColor(3) hQueued.SetStats(0) hQueued.SetMarkerSize(1.5) hQueued.SetMarkerColor(3) hHalted=ROOT.TH1I('hHalted','Halted',compUsers,0,compUsers) hHalted.SetFillStyle(3006) hHalted.SetFillColor(4) hHalted.SetLineColor(4) hHalted.SetStats(0) hHalted.SetMarkerSize(1.5) hHalted.SetMarkerColor(4) hTotJobs=ROOT.TH1I('hTotJobs','Jobs',compUsers,0,compUsers) hTotJobs.SetFillStyle(0) hTotJobs.SetLineColor(1) hTotJobs.SetStats(0) hTotJobs.SetMarkerSize(1.5) hTotJobs.SetMarkerColor(1) hTotTime=ROOT.TH1I('hTotTime','Average computing time per job',compUsers,0,compUsers) hTotTime.SetLineColor(1) hTotTime.SetStats(0) hTotTime.SetMarkerSize(2) hTotTime.SetMarkerColor(1) hTot8Time=ROOT.TH1I('hTot8Time','8h Average computing time per job',compUsers,0,compUsers) hTot8Time.SetLineColor(6) hTot8Time.SetStats(0) hTot8Time.SetMarkerSize(2) hTot8Time.SetMarkerColor(1) hTot2Time=ROOT.TH1I('hTot2Time','2h Average computing time per job',compUsers,0,compUsers) hTot2Time.SetLineColor(6) hTot2Time.SetStats(0) hTot2Time.SetMarkerSize(2) hTot2Time.SetMarkerColor(1) legend=ROOT.TLegend(0,0.1,0.9,0.9,"Legend") legend.SetFillColor(0) legend.AddEntry(hTotJobs,"Total Job Number","f") legend.AddEntry(hRunning,"Running Jobs","f") legend.AddEntry(hQueued,"Queued Jobs","f") legend.AddEntry(hHalted,"Halted Jobs","f") first=False else: hRunning.Reset() hQueued.Reset() hHalted.Reset() hTotJobs.Reset() hTotTime.Reset() hTot8Time.Reset() usercount=-1 todel=[] if not readonly: outfile=open(usepath+'batchuseage.txt','a') for g in maxis: maxis[g]=0 for user in users: if users[user]['njobs']!=0: usercount+=1 if(users[user]['nRjobs']!=0): users[user]['AvgTime']=float(users[user]['totTime']/users[user]['nRjobs']) else: users[user]['AvgTime']=0 hRunning.Fill(usercount,users[user]['nRjobs']) hQueued.Fill(usercount,users[user]['nQjobs']) hHalted.Fill(usercount,users[user]['nHjobs']) hTotJobs.Fill(usercount,users[user]['njobs']) hTotJobs.GetXaxis().SetBinLabel(usercount+1,user) hTotTime.Fill(usercount,users[user]['AvgTime']) hTotTime.GetXaxis().SetBinLabel(usercount+1,user) if (users_g.get(user,None))==None: users_g[user]={} users_g[user]['counter']=-1 users_g[user]['njobs']=ROOT.TGraph() users_g[user]['njobs'].SetTitle("Total Jobs") users_g[user]['nRjobs']=ROOT.TGraph() users_g[user]['nRjobs'].SetTitle("Runnung Jobs") users_g[user]['nQjobs']=ROOT.TGraph() users_g[user]['nQjobs'].SetTitle('Queued Jobs') users_g[user]['nHjobs']=ROOT.TGraph() users_g[user]['nHjobs'].SetTitle("Halted Jobs") users_g[user]['totTime']=0 users_g[user]['AvgTime']=ROOT.TGraph() users_g[user]['AvgTime'].SetTitle("Average Computing Time") users_g[user]['share']=ROOT.TGraph() users_g[user]['share'].SetTitle('Fair Share Rank') timestamp=time.time() last_x,last_y=ROOT.Double(-1),ROOT.Double(-1) if users_g[user]['counter']>=0: users_g[user]['njobs'].GetPoint(users_g[user]['counter'],last_x,last_y) if last_y == 0: users_g[user]['counter']+=1 users_g[user]['njobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nRjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nHjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['nQjobs'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['AvgTime'].SetPoint(users_g[user]['counter'],timestamp,0) users_g[user]['share'].SetPoint(users_g[user]['counter'],timestamp,users[user]['share']) users_g[user]['counter']+=1 users_g[user]['njobs'].SetPoint(users_g[user]['counter'],timestamp,users[user]['njobs']) users_g[user]['nRjobs'].SetPoint(users_g[user]['counter'],timestamp,users[user]['nRjobs']) users_g[user]['nHjobs'].SetPoint(users_g[user]['counter'],timestamp,users[user]['nHjobs']) users_g[user]['nQjobs'].SetPoint(users_g[user]['counter'],timestamp,users[user]['nQjobs']) users_g[user]['AvgTime'].SetPoint(users_g[user]['counter'],timestamp,users[user]['AvgTime']) users_g[user]['share'].SetPoint(users_g[user]['counter'],timestamp,users[user]['share']) users_g[user]['totTime']+=users[user]['totTime'] x,y=ROOT.Double(0),ROOT.Double(0) npoints=users_g[user]['njobs'].GetN() users_g[user]['njobs'].GetPoint(npoints-1,x,y) counter=0 while float(x)>time.time()-timerange and counter5: #print 'bigger' continue maxis[p]=max(maxis.get(p,0),float(y)) if not readonly: outfile.write(user+';' +str(users[user]['njobs'])+';' +str(users[user]['nRjobs'])+';' +str(users[user]['nHjobs'])+';' +str(users[user]['nQjobs'])+';' +str(users[user]['totTime'])+';' +str(users[user]['AvgTime'])+';' +str(users[user]['share'])+';' +str(timestamp) +'\n') if users[user]['njobs']==0: todel.append(user) if not readonly: outfile.close() for u in todel: changed=True reset=False users.pop(u) print 'Last update:',time.asctime( time.localtime(time.time()) ) if not writeonly: hTotJobs.SetMinimum(0) hTotTime.SetMinimum(0) gpad_3.cd(1) hTotJobs.Draw() hTotJobs.Draw('same TEXT45') hRunning.Draw('same') hRunning.Draw('same TEXT45') hQueued.Draw('same') hQueued.Draw('same TEXT45') hHalted.Draw('same') hHalted.Draw('same TEXT45') gpad_3.cd(2) hTotTime.Draw() hTotTime.Draw('same TEXT0') gpad_3.cd(3).cd(2) legend.Draw() gpad_3.Update() gcanv.cd(1) opt="AL" col=-1 runleg=ROOT.TLegend(0,0,1,1) runleg.SetFillColor(0) runleg.SetBorderSize(0) runleg.SetTextSize(0.1) runleg_e=[] for user in users_g: npoints=users_g[user]['njobs'].GetN() users_g[user]['njobs'].GetPoint(npoints-1,x,y) counter=0 to_include=False while float(x)>time.time()-timerange and counter