Skip to content
Snippets Groups Projects
Commit e1940e7c authored by Antonio Falabella's avatar Antonio Falabella
Browse files

Accounting script with site feature

parent 8990e7b9
No related branches found
No related tags found
No related merge requests found
Pipeline #
#!/usr/bin/env python
import sys,os,time
import psycopg2
import calendar
from pytz import timezone
from pprint import pprint
import pickle
def now():
return time.ctime(time.time())
def mlog(f,m,dbg=True):
if dbg:
f.write("%s: "%now()+m+'\n')
f.flush()
logf = open('/tmp/accounting_update_test.log','a')
localzone = timezone('Europe/Rome')
class dbconn():
def __init__(self,database='acct',host='dbfarm-1.cr.cnaf.infn.it',user='acct',password='acct'):
self.conn = psycopg2.connect(database=database,host=host,user=user,password=password)
self.curs = self.conn.cursor()
qc = dbconn()
'''
CREATE OR REPLACE FUNCTION proc_strip(text)
RETURNS text
LANGUAGE sql
AS $function$
SELECT ltrim($1,'248*')
$function$;
'''
sq = """
SELECT queue,sum(numprocessors), exechosts, sum(wallt), sum(cput),siteName FROM (
SELECT
queue,numprocessors,exechosts,runtime*numprocessors as wallt, utime+stime as cput,
(fromhost like 'ce%%')::int as isgrid, proc_strip(exechosts) as nodetype,
CASE queue
WHEN 'T3_BO' THEN 'INFN-T3-BO'
WHEN 'T3_BO_FAST' THEN 'INFN-T3-BO'
WHEN 'lhcb_tier2' THEN 'INFN-CNAF-LHCB'
WHEN 'cert_t2' THEN 'INFN-CNAF-LHCB'
ELSE
CASE fromhost
WHEN 'ce01-lhcb-t2' THEN 'INFN-CNAF-LHCB'
WHEN 'ce02-lhcb-t2' THEN 'INFN-CNAF-LHCB'
WHEN 'cebo-t3-01' THEN 'INFN-BOLOGNA-T3'
WHEN 'cebo-t3-02' THEN 'INFN-BOLOGNA-T3'
ELSE
CASE substr(proc_strip(exechosts),1,5)
WHEN 'vnode' THEN 'INFN-ARUBA'
WHEN 'ba-3-' THEN 'BARI-RECAS'
WHEN 'wn-20' THEN 'INFN-T1'
ELSE 'UNKNOWN'
END
END
END
AS siteName
FROM {0}
WHERE
eventtimeepoch BETWEEN to_unixtime('{1}') AND
to_unixtime('{1}')+86400 AND exechosts != '' ) as A
GROUP BY queue,exechosts,siteName
"""
if len(sys.argv) != 2:
print "\n Only one argument allowed!"
exit(1)
#DT = 86400
#Tnow = int(time.time())
#T1 = time.strftime("%Y-%m-%d",time.gmtime(Tnow))
#T0 = time.strftime("%Y-%m-%d",time.gmtime(Tnow - DT))
else :
T0 = sys.argv[1]
DT = 86400
Tnow = int(time.time())
T1 = time.strftime("%Y-%m-%d",time.gmtime(Tnow))
if (T0==T1):
print "Accounting available only for last day!\n"
exit(1)
pg_tables = ["gridjob","job"]
T = {"gridjob":[],"job":[]}
L = {"gridjob":[],"job":[]}
# Save to file for debug
debug = True
if (not (os.path.exists("backup.pkl") or not(debug))):
for p in pg_tables:
query = sq.format(p,T0,T0)
mlog(logf, "executing:"+ qc.curs.mogrify(sq,(T0)*4))
qc.curs.execute(query)
T[p] = qc.curs.fetchall()
f = open('/usr/share/lsf/var/cache/machine_job/node_hs06_cores.txt','r')
spec = [tuple(x.rstrip().split(';')) for x in f]
f.close()
d = dict([(x[0],map(int,x[1:])) for x in spec])
get_hn = lambda s : s.split('*')[-1]
for t0 in T[p]:
t = list(t0)
hn = t[2] = get_hn(t0[2])
L[p].append(tuple(t+d.get(hn,[11,1,1])))
pickle.dump(L, open("backup.pkl","wb"))
print "File created"
qc.conn.close()
else:
L = pickle.load(open("backup.pkl","rb"))
print "Loaded list"
print len(L)
import pprint
#pprint.pprint(L)
D = {"gridjob":
{"INFN-T1":{},"BARI-RECAS":{},"INFN-ARUBA":{}},
"job":
{"INFN-T1":{},"BARI-RECAS":{},"INFN-ARUBA":{}}
}
#print L
for submit in L.keys():
for q,nj,wn,wt,cpt,site,hs06,ncore,ns in L[submit]:
if site in ["INFN-T1","INFN-CNAF-LHCB","INFN-BOLOGNA-T3","INFN-T3-BO"]:
site = "INFN-T1"
if D[submit][site].has_key(q):
D[submit][site][q][0] += nj
D[submit][site][q][1] += hs06
D[submit][site][q][2] += wt
D[submit][site][q][3] += cpt
D[submit][site][q][4] += wt * hs06/ (ns + 0.0)
D[submit][site][q][5] += cpt * hs06/ (ns + 0.0)
else:
D[submit][site][q] = []
D[submit][site][q].append(nj)
D[submit][site][q].append(hs06)
D[submit][site][q].append(wt)
D[submit][site][q].append(cpt)
D[submit][site][q].append( wt * hs06/ (ns + 0.0))
D[submit][site][q].append(cpt * hs06/ (ns + 0.0))
mlog(logf,"D = %s"%repr(D))
#Inserisco i Grid (D[True])
localzone = timezone('Europe/Rome')
yesterday_ts = int(time.mktime(time.strptime(T0,"%Y-%m-%d")) - 86400 + localzone._utcoffset.seconds)
from CarbonClient import CarbonClient
carbon_server = '131.154.96.41'
carbon_port = 2003
# Connect carbon
try:
carbon = CarbonClient(carbon_server, carbon_port)
except:
mlog(logf,"Couldn't connect to %(server)s on port %(port)d, is carbon-agent.py running?" % { 'server': carbon_server, 'port': carbon_port })
#print "Couldn't connect to %(server)s on port %(port)d, is carbon-agent.py running?" % { 'server': carbon_server, 'port': carbon_port }
#sys.exit(2)
carbon = None
for submit in D.keys():
for site in D[submit].keys():
for queue,(njobs,HStot,wct,cpt,wcths_sec,cpths_sec) in D[submit][site].items():
HSwct, HScpt = wcths_sec/DT, cpths_sec/DT
if submit=="gridjob":
submit_type = 'grid'
else:
submit_type = 'local'
if carbon:
carbon.insert('farming.acc.%s.%s.%s.njobs' % (site, queue, submit_type), njobs, yesterday_ts)
carbon.insert('farming.acc.%s.%s.%s.ndone' % (site, queue, submit_type), njobs, yesterday_ts)
carbon.insert('farming.acc.%s.%s.%s.cpt.sec' % (site, queue, submit_type), cpt, yesterday_ts)
carbon.insert('farming.acc.%s.%s.%s.cpt.hs06' % (site, queue, submit_type), HScpt, yesterday_ts)
carbon.insert('farming.acc.%s.%s.%s.wct.sec' % (site, queue, submit_type), wct, yesterday_ts)
carbon.insert('farming.acc.%s.%s.%s.wct.hs06' % (site, queue, submit_type), HSwct, yesterday_ts)
pprint.pprint(D)
sys.exit(0)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment