Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
A
acc_mon_scripts
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Antonio Falabella
acc_mon_scripts
Commits
e1940e7c
There was a problem fetching the pipeline metadata.
Commit
e1940e7c
authored
8 years ago
by
Antonio Falabella
Browse files
Options
Downloads
Patches
Plain Diff
Accounting script with site feature
parent
8990e7b9
No related branches found
No related tags found
No related merge requests found
Pipeline
#
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
accounting_update_pgdb_openstack_new_query.py
+196
-0
196 additions, 0 deletions
accounting_update_pgdb_openstack_new_query.py
with
196 additions
and
0 deletions
accounting_update_pgdb_openstack_new_query.py
0 → 100644
+
196
−
0
View file @
e1940e7c
#!/usr/bin/env python
import
sys
,
os
,
time
import
psycopg2
import
calendar
from
pytz
import
timezone
from
pprint
import
pprint
import
pickle
def
now
():
return
time
.
ctime
(
time
.
time
())
def
mlog
(
f
,
m
,
dbg
=
True
):
if
dbg
:
f
.
write
(
"
%s:
"
%
now
()
+
m
+
'
\n
'
)
f
.
flush
()
logf
=
open
(
'
/tmp/accounting_update_test.log
'
,
'
a
'
)
localzone
=
timezone
(
'
Europe/Rome
'
)
class
dbconn
():
def
__init__
(
self
,
database
=
'
acct
'
,
host
=
'
dbfarm-1.cr.cnaf.infn.it
'
,
user
=
'
acct
'
,
password
=
'
acct
'
):
self
.
conn
=
psycopg2
.
connect
(
database
=
database
,
host
=
host
,
user
=
user
,
password
=
password
)
self
.
curs
=
self
.
conn
.
cursor
()
qc
=
dbconn
()
'''
CREATE OR REPLACE FUNCTION proc_strip(text)
RETURNS text
LANGUAGE sql
AS $function$
SELECT ltrim($1,
'
248*
'
)
$function$;
'''
sq
=
"""
SELECT queue,sum(numprocessors), exechosts, sum(wallt), sum(cput),siteName FROM (
SELECT
queue,numprocessors,exechosts,runtime*numprocessors as wallt, utime+stime as cput,
(fromhost like
'
ce%%
'
)::int as isgrid, proc_strip(exechosts) as nodetype,
CASE queue
WHEN
'
T3_BO
'
THEN
'
INFN-T3-BO
'
WHEN
'
T3_BO_FAST
'
THEN
'
INFN-T3-BO
'
WHEN
'
lhcb_tier2
'
THEN
'
INFN-CNAF-LHCB
'
WHEN
'
cert_t2
'
THEN
'
INFN-CNAF-LHCB
'
ELSE
CASE fromhost
WHEN
'
ce01-lhcb-t2
'
THEN
'
INFN-CNAF-LHCB
'
WHEN
'
ce02-lhcb-t2
'
THEN
'
INFN-CNAF-LHCB
'
WHEN
'
cebo-t3-01
'
THEN
'
INFN-BOLOGNA-T3
'
WHEN
'
cebo-t3-02
'
THEN
'
INFN-BOLOGNA-T3
'
ELSE
CASE substr(proc_strip(exechosts),1,5)
WHEN
'
vnode
'
THEN
'
INFN-ARUBA
'
WHEN
'
ba-3-
'
THEN
'
BARI-RECAS
'
WHEN
'
wn-20
'
THEN
'
INFN-T1
'
ELSE
'
UNKNOWN
'
END
END
END
AS siteName
FROM {0}
WHERE
eventtimeepoch BETWEEN to_unixtime(
'
{1}
'
) AND
to_unixtime(
'
{1}
'
)+86400 AND exechosts !=
''
) as A
GROUP BY queue,exechosts,siteName
"""
if
len
(
sys
.
argv
)
!=
2
:
print
"
\n
Only one argument allowed!
"
exit
(
1
)
#DT = 86400
#Tnow = int(time.time())
#T1 = time.strftime("%Y-%m-%d",time.gmtime(Tnow))
#T0 = time.strftime("%Y-%m-%d",time.gmtime(Tnow - DT))
else
:
T0
=
sys
.
argv
[
1
]
DT
=
86400
Tnow
=
int
(
time
.
time
())
T1
=
time
.
strftime
(
"
%Y-%m-%d
"
,
time
.
gmtime
(
Tnow
))
if
(
T0
==
T1
):
print
"
Accounting available only for last day!
\n
"
exit
(
1
)
pg_tables
=
[
"
gridjob
"
,
"
job
"
]
T
=
{
"
gridjob
"
:[],
"
job
"
:[]}
L
=
{
"
gridjob
"
:[],
"
job
"
:[]}
# Save to file for debug
debug
=
True
if
(
not
(
os
.
path
.
exists
(
"
backup.pkl
"
)
or
not
(
debug
))):
for
p
in
pg_tables
:
query
=
sq
.
format
(
p
,
T0
,
T0
)
mlog
(
logf
,
"
executing:
"
+
qc
.
curs
.
mogrify
(
sq
,(
T0
)
*
4
))
qc
.
curs
.
execute
(
query
)
T
[
p
]
=
qc
.
curs
.
fetchall
()
f
=
open
(
'
/usr/share/lsf/var/cache/machine_job/node_hs06_cores.txt
'
,
'
r
'
)
spec
=
[
tuple
(
x
.
rstrip
().
split
(
'
;
'
))
for
x
in
f
]
f
.
close
()
d
=
dict
([(
x
[
0
],
map
(
int
,
x
[
1
:]))
for
x
in
spec
])
get_hn
=
lambda
s
:
s
.
split
(
'
*
'
)[
-
1
]
for
t0
in
T
[
p
]:
t
=
list
(
t0
)
hn
=
t
[
2
]
=
get_hn
(
t0
[
2
])
L
[
p
].
append
(
tuple
(
t
+
d
.
get
(
hn
,[
11
,
1
,
1
])))
pickle
.
dump
(
L
,
open
(
"
backup.pkl
"
,
"
wb
"
))
print
"
File created
"
qc
.
conn
.
close
()
else
:
L
=
pickle
.
load
(
open
(
"
backup.pkl
"
,
"
rb
"
))
print
"
Loaded list
"
print
len
(
L
)
import
pprint
#pprint.pprint(L)
D
=
{
"
gridjob
"
:
{
"
INFN-T1
"
:{},
"
BARI-RECAS
"
:{},
"
INFN-ARUBA
"
:{}},
"
job
"
:
{
"
INFN-T1
"
:{},
"
BARI-RECAS
"
:{},
"
INFN-ARUBA
"
:{}}
}
#print L
for
submit
in
L
.
keys
():
for
q
,
nj
,
wn
,
wt
,
cpt
,
site
,
hs06
,
ncore
,
ns
in
L
[
submit
]:
if
site
in
[
"
INFN-T1
"
,
"
INFN-CNAF-LHCB
"
,
"
INFN-BOLOGNA-T3
"
,
"
INFN-T3-BO
"
]:
site
=
"
INFN-T1
"
if
D
[
submit
][
site
].
has_key
(
q
):
D
[
submit
][
site
][
q
][
0
]
+=
nj
D
[
submit
][
site
][
q
][
1
]
+=
hs06
D
[
submit
][
site
][
q
][
2
]
+=
wt
D
[
submit
][
site
][
q
][
3
]
+=
cpt
D
[
submit
][
site
][
q
][
4
]
+=
wt
*
hs06
/
(
ns
+
0.0
)
D
[
submit
][
site
][
q
][
5
]
+=
cpt
*
hs06
/
(
ns
+
0.0
)
else
:
D
[
submit
][
site
][
q
]
=
[]
D
[
submit
][
site
][
q
].
append
(
nj
)
D
[
submit
][
site
][
q
].
append
(
hs06
)
D
[
submit
][
site
][
q
].
append
(
wt
)
D
[
submit
][
site
][
q
].
append
(
cpt
)
D
[
submit
][
site
][
q
].
append
(
wt
*
hs06
/
(
ns
+
0.0
))
D
[
submit
][
site
][
q
].
append
(
cpt
*
hs06
/
(
ns
+
0.0
))
mlog
(
logf
,
"
D = %s
"
%
repr
(
D
))
#Inserisco i Grid (D[True])
localzone
=
timezone
(
'
Europe/Rome
'
)
yesterday_ts
=
int
(
time
.
mktime
(
time
.
strptime
(
T0
,
"
%Y-%m-%d
"
))
-
86400
+
localzone
.
_utcoffset
.
seconds
)
from
CarbonClient
import
CarbonClient
carbon_server
=
'
131.154.96.41
'
carbon_port
=
2003
# Connect carbon
try
:
carbon
=
CarbonClient
(
carbon_server
,
carbon_port
)
except
:
mlog
(
logf
,
"
Couldn
'
t connect to %(server)s on port %(port)d, is carbon-agent.py running?
"
%
{
'
server
'
:
carbon_server
,
'
port
'
:
carbon_port
})
#print "Couldn't connect to %(server)s on port %(port)d, is carbon-agent.py running?" % { 'server': carbon_server, 'port': carbon_port }
#sys.exit(2)
carbon
=
None
for
submit
in
D
.
keys
():
for
site
in
D
[
submit
].
keys
():
for
queue
,(
njobs
,
HStot
,
wct
,
cpt
,
wcths_sec
,
cpths_sec
)
in
D
[
submit
][
site
].
items
():
HSwct
,
HScpt
=
wcths_sec
/
DT
,
cpths_sec
/
DT
if
submit
==
"
gridjob
"
:
submit_type
=
'
grid
'
else
:
submit_type
=
'
local
'
if
carbon
:
carbon
.
insert
(
'
farming.acc.%s.%s.%s.njobs
'
%
(
site
,
queue
,
submit_type
),
njobs
,
yesterday_ts
)
carbon
.
insert
(
'
farming.acc.%s.%s.%s.ndone
'
%
(
site
,
queue
,
submit_type
),
njobs
,
yesterday_ts
)
carbon
.
insert
(
'
farming.acc.%s.%s.%s.cpt.sec
'
%
(
site
,
queue
,
submit_type
),
cpt
,
yesterday_ts
)
carbon
.
insert
(
'
farming.acc.%s.%s.%s.cpt.hs06
'
%
(
site
,
queue
,
submit_type
),
HScpt
,
yesterday_ts
)
carbon
.
insert
(
'
farming.acc.%s.%s.%s.wct.sec
'
%
(
site
,
queue
,
submit_type
),
wct
,
yesterday_ts
)
carbon
.
insert
(
'
farming.acc.%s.%s.%s.wct.hs06
'
%
(
site
,
queue
,
submit_type
),
HSwct
,
yesterday_ts
)
pprint
.
pprint
(
D
)
sys
.
exit
(
0
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment