Skip to content
Snippets Groups Projects
Commit 4c63ddba authored by Andrea Michelotti's avatar Andrea Michelotti
Browse files

Merge branch 'offset_upgrade' into main

parents f85665d4 47e021d0
No related branches found
No related tags found
No related merge requests found
Pipeline #105911 failed
......@@ -5,31 +5,63 @@
`pip install -r requirements.txt`
### help
usage: memcached2Influx.py [-h] [-u USERNAME] [-p PASSWORD] [-s SERVER] [-d DATABASE]
[-f FILE] [-n NAME] [-k KEY] [-po PORT] [-r RATE]
usage: memcached2Influx.py [-h] [-u USERNAME] [-p PASSWORD] -is INFLUXSERVER
-id INFLUXDATABASE [-ip INFLUXPORT] -ms
MEMCACHEDSERVER [-mp MEMCACHEDPORT] [-k "KEY,KEY2,KEY3"]
[-kn KEYNAME] [-kr KEYRATE] [-f FILE]
[-fr FILERATE]
options:
`-h` or `--help` show this help message and exit
`-u` or `--username` --username USERNAME
the username needed to log in the db
`-p` or `--password` --password PASSWORD
the password needed to log in the db
`-s` or `--server` --server SERVER
the ip address of the server
`-d` or `--database` --database DATABASE
the name of the database you want to log in
`-f` or `--file` --file FILE the configuration file path
`-n` or `--name` --name NAME if you're not using a configuration file, use this parameter to
chose the name of the measurement
`-k` or `--key` KEY the key needed to find the data in the memcached DB
`-po` or `--port` PORT the port associated with the server address
`-r` or `--rate` RATE specify the seconds between 2 different push in the influx db,
use this option only if you're not using a configuration file
`-h` or `--help` show this help message and exit
`-u` or `--username` the username needed to log in the db
`-p` or `--password` the password needed to log in the db
`-is` or `--influxServer` the address of the influx server
`-id` or `--influxDatabase` the name of the database you want to log in
`-ip` or `--influxPort` the port associated with the server address
`-ms` or `--memcachedServer` the address of the memcached server
`-mp` or `--memcachedPort` the port of the memcached server
`-k` or `--key` the key needed to find the data in the memcached DB
`-kn` or `--keyName` if you're not using a configuration file, use this parameter to chose the name of the measurement
`-kr` or `--keyRate` when the program is not working with a configuration file, specify the seconds between 2 different push in the influx db
`-f` or `--file` the configuration file path
`-fr` or `--fileRate` when using a configuration file, specify the sleep time of the loop that check when and wich one of the key must be pushed
### test
some test of funtionality
```
python memcached2Influx.py -s vldantemon003.lnf.infn.it -po 8086 -f '/home/riccardo/Random bs go/Git/memcached2influx/memcached2influx/cofnigurationFile.txt'
python memcached2Influx.py -is vldantemon003.lnf.infn.it -id dcsMemDb -ip 8086 -ms 192.168.198.20 -mp 11211 -f '/home/riccardo/Random bs go/Git/memcached2influx/memcached2influx/configurationFile.txt'
or
ython memcached2Influx.py -s vldantemon003.lnf.infn.it -po 8086 -k DAFNESTATELAB_JDAT -r 10
python memcached2Influx.py -is vldantemon003.lnf.infn.it -ip 8086 -id dcsMemDb
-k DAFNESTATELAB_JDAT -kn test -kr 10 -ms 192.168.198.20 -mp 11211
```
### usage
this application has been made to transfer data from a memcached server to an influx server, but there are 2 different way to do so.
`using a configuration file.`
In the project folder you can find a "configurationFile.txt" this file contain all the necessary data to retrive and optimize the process of transiction between the 2 server, and in order to use such file you should use a command line like this:
```
python memcached2Influx.py -is vldantemon003.lnf.infn.it -id dcsMemDb -ip 8086 -ms 192.168.198.20 -mp 11211 -f '/home/riccardo/Random bs go/Git/memcached2influx/memcached2influx/configurationFile.txt'
```
`using key and rate`
But if you don't have all this information all you don't want to use a configuration file, you can just use a key and a rate instead.
With just a key and a publishing rate the program will know just what data you want to pubish and at wich rate, in order to do so you should use a command line like this
```
python memcached2Influx.py -is vldantemon003.lnf.infn.it -ip 8086 -id dcsMemDb
-k DAFNESTATELAB_JDAT -kn test -kr 10 -ms 192.168.198.20 -mp 11211
```
{
"dataset":
[
{
"name":"DEL0",
"type":"double",
"description":"readout delay",
"offset":4,
"len":8,
"factor":0,
"lbe":true,
"keybind":"KCKA*I_E_BUF"
},
\ No newline at end of file
{
"dataset":[
{
"name":"flame_state",
"type":"json",
"description":"Flame state",
"offset":0,
"len":512,
"keybind":"FLAME_STATE"
},{
"name":"flame_state",
"type":"json",
"description":"Flame state STR",
"offset":0,
"len":512,
"keybind":"FLAME_STATE_STR"
}]
}
......@@ -9,7 +9,6 @@ import json
import time
#Kiwi
def jsonKey2Influx(key, clientMemcached, name):
data = clientMemcached.get(key)
data = json.loads(data)
......@@ -26,17 +25,41 @@ def jsonKey2Influx(key, clientMemcached, name):
def jsonData2Influx(fileData,clientMemcached):
payload = []
jsonData = clientMemcached.get(fileData['keybind'])
fileData['memcached'] = json.loads(jsonData)
memcached = json.loads(jsonData)
fileData["rate"] = 5
fileData["currentTime"] = 0.0
payload.append({
"measurement": fileData["name"],
"tags": {
"key": fileData["keybind"]
},
"fields": fileData["memcached"]
"fields": memcached,
"parameter":fileData
})
return payload
def byteData2Influx(fileData,clientMemcached):
payload = []
byteArray = clientMemcached.get(fileData['keybind'])
memcached =byteArray[int(fileData['offset'])]
fileData["currentTime"] = 0.0
try:
fileData["rate"] = int(fileData["rate"])
except:
fileData["rate"] = 5
payload.append({
"measurement": fileData["name"],
"tags": {
"key": fileData["keybind"]
},
"fields": {
fileData["name"] : memcached
},
"parameter" : fileData
})
return payload
def findTheKey(configFile):
data = {}
......@@ -47,14 +70,13 @@ def findTheKey(configFile):
#knowing that the keybind is always the last value, the return is set right after it
if(match):
#match = re.search(pattern,line)
parameter = re.sub(r'(,|"|( ←))(?![^\[\]]*\])',"",match.group(2))
parameter = re.sub(r'(,|"|( ←)|\n)(?![^\[\]]*\])',"",match.group(2))
data[match.group(1)] = parameter
elif(line.find('keybind') != -1):
line = re.sub(r',|( ←)',"",line)
pattern = r'"(.+)":"(.+)"'
match = re.search(pattern,line)
data[match.group(1)] = match.group(2)
data['name'] = data['name'][0:-1]
return data
......@@ -64,66 +86,113 @@ parser = argparse.ArgumentParser()
parser.add_argument("-u", "--username", required = False ,default="", help = "the username needed to log in the db")
parser.add_argument("-p", "--password", required = False ,default="", help = "the password needed to log in the db")
parser.add_argument("-s", "--server", required = False , default="", help = "the ip address of the server")
parser.add_argument("-d", "--database", required = False, default = "" , help = "the name of the database you want to log in")
parser.add_argument("-f", "--file",required = False, help = "the configuration file path")
parser.add_argument("-n", "--name",required = False, help = "if you're not using a configuration file, use this parameter to chose the name of the measurement")
parser.add_argument("-is", "--influxServer",required = True, help = "the address of the influx server")
parser.add_argument("-id", "--influxDatabase", required = True, help = "the name of the database you want to log in")
parser.add_argument("-ip", "--influxPort",required = False, default = 8086, help = "the port associated with the server address")
parser.add_argument("-ms", "--memcachedServer",required = True, help = "the address of the memcached server")
parser.add_argument("-mp", "--memcachedPort",required = False, default = 11211, help = "the port of the memcached server")
parser.add_argument("-k", "--key",required = False, help = "the key needed to find the data in the memcached DB")
parser.add_argument("-po", "--port", help = "the port associated with the server address")
parser.add_argument("-r", "--rate", required = False, default = 5, help = "specify the seconds between 2 different push in the influx db, use this option only if you're not using a configuration file")
parser.add_argument("-kn", "--keyName",required = False, help = "if you're not using a configuration file, use this parameter to chose the name of the measurement")
parser.add_argument("-kr", "--keyRate", required = False, default = 5, help = "when the program is not working with a configuration file, specify the seconds between 2 different push in the influx db")
parser.add_argument("-f", "--file",required = False, help = "the configuration file path")
parser.add_argument("-fr", "--fileRate", required = False, default = 0.1, help = "when using a configuration file, specify the sleep time of the loop that check when and wich one of the key must be pushed")
args = parser.parse_args()
#log in the influx DB
#dcsMemDb
#python memcached2Influx.py -s vldantemon003.lnf.infn.it -po 8086 -f '/home/riccardo/Random bs go/Test/cofnigurationFile.txt'
try:
clientInflux = InfluxDBClient(host=args.server, port=args.port, username=args.username, password=args.password)
clientInflux.switch_database("dcsMemDb")
clientInflux = InfluxDBClient(host=args.influxServer, port=args.influxPort, username=args.username, password=args.password)
clientInflux.switch_database(args.influxDatabase)
print("succesfully logged in\n")
except:
sys.exit("Error: Invalid parameters, please try again")
print("Error: impossible to connect to influx with the following parameter")
print("username = "+args.username)
print("password = "+args.password)
print("database = "+args.influxServer)
print("port = "+args.influxPort)
sys.exit
#log in the memcached DB
#memcached_server = "192.168.198.20"
#mc_port = 11211
try:
clientMemcached = base.Client( ("192.168.198.20" , 11211) , connect_timeout=20.0)
clientMemcached = base.Client( (args.memcachedServer , args.memcachedPort) , connect_timeout=20.0)
except:
sys.exit('Cannot reach the memcached server, try again later')
if args.key:
refreshRate = 5
if args.rate:
refreshRate = int(args.rate)
vector_key=split_list = args.key.split(",") # Splitting at comma delimiter
try:
args.keyRate = int(args.keyRate)
except ValueError as e :
print("the inserted rate value is not valid\nThe default value of 5 second will be set")
args.keyRate = 5
name = ""
if(args.keyName):
name = args.keyName
else:
name = args.key
while True:
payload = []
if args.name:
payload = jsonKey2Influx(args.key, clientMemcached, args.name)
else:
payload = jsonKey2Influx(args.key, clientMemcached, args.key)
print('Publishing data')
print(payload)
clientInflux.write_points(payload)
time.sleep(refreshRate)
for s in vector_key:
payload = jsonKey2Influx(s, clientMemcached, s)
print('Publishing data')
print(payload)
clientInflux.write_points(payload)
time.sleep(args.keyRate)
elif args.file:
try:
args.fileRate = float(args.fileRate)
except ValueError as e :
print("the inserted rate value is not valid\nThe default value of 0.1 second will be set")
args.fileRate = 0.1
try:
open(args.file,'r')
except FileNotFoundError as e:
sys.exit("Error: " + args.iptable + " is not valid or does not point to a DBFile.")
if os.path.isfile(args.file):
with open(args.file,'r') as configFile:
slowest = 0
payloadList = []
while (True):
fileData = findTheKey(configFile)
payload = []
if fileData == None:
print('list done')
break
if fileData["type"] == "json":
payload = jsonData2Influx(fileData,clientMemcached)
print('Publishing into influx:')
print(payload)
print('\n')
clientInflux.write_points(payload)
\ No newline at end of file
elif fileData["type"] == "json":
payloadList.append(jsonData2Influx(fileData,clientMemcached))
elif fileData["type"] == "double":
payloadList.append(byteData2Influx(fileData,clientMemcached))
while (True):
time.sleep(args.fileRate)
for payload in payloadList:
if (payload[0]["parameter"]["currentTime"] >= payload[0]["parameter"]["rate"]):
payload[0]["parameter"]["currentTime"] = 0
name = payload[0]["parameter"]["name"]
data = ({
"measurement": name,
"tags": {
"key": payload[0]["parameter"]["keybind"]
},
"fields": {
name : payload[0]["fields"][name]
},
})
print("\nPublishing to influx:")
print(data)
print()
wrapData = [data]
clientInflux.write_points(wrapData)
else:
payload[0]["parameter"]["currentTime"] = payload[0]["parameter"]["currentTime"] +args.fileRate
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment