148 lines
5.3 KiB
Python
Executable File
148 lines
5.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import sys
|
|
import os
|
|
|
|
import re
|
|
|
|
import json
|
|
import psycopg2
|
|
|
|
import logging
|
|
|
|
from sqlalchemy import create_engine, text
|
|
from utils.time import timestamp_fmt as ts
|
|
from utils.config import set_config as setting
|
|
|
|
def extract_value(patterns, source, default='Not Defined'):
|
|
ip = {}
|
|
for pattern in patterns:
|
|
pattern = f'r"{pattern}:\s*(\d{1,3}(?:\.\d{1,3}){3})"'
|
|
matches = re.search(pattern, source, re.IGNORECASE)
|
|
if matches:
|
|
ip.update({pattern: matches.group(1)})
|
|
else:
|
|
ip.update({pattern: default})
|
|
return ip
|
|
|
|
def write_db(engine, records):
|
|
with engine.connect() as conn:
|
|
conn.execute(text("""
|
|
INSERT INTO dataraw (unit_name, unit_type, tool_name, tool_type, ip_centralina, ip_gateway, event_timestamp, battery_level, temperature, nodes_jsonb)
|
|
VALUES
|
|
""" + ",".join([
|
|
f"(:{i}_unit_name, :{i}_unit_type, :{i}_tool_name, :{i}_tool_type, :{i}_ip_centralina, :{i}_ip_gateway, :{i}_event_timestamp, :{i}_battery_level, :{i}_temperature, :{i}_nodes_jsonb)"
|
|
for i in range(len(records))
|
|
]) + """
|
|
ON CONFLICT ON CONSTRAINT dataraw_unique
|
|
DO UPDATE SET
|
|
unit_type = EXCLUDED.unit_type,
|
|
tool_type = EXCLUDED.tool_type,
|
|
ip_centralina = EXCLUDED.ip_centralina,
|
|
ip_gateway = EXCLUDED.ip_gateway,
|
|
battery_level = EXCLUDED.battery_level,
|
|
temperature = EXCLUDED.temperature,
|
|
nodes_jsonb = EXCLUDED.nodes_jsonb;
|
|
"""), {f"{i}_{key}": value for i, record in enumerate(records) for key, value in record.items()})
|
|
|
|
conn.commit()
|
|
|
|
def elab_csv(engine, cfg):
|
|
with engine.connect() as conn:
|
|
cur = conn.cursor()
|
|
cur.execute(f'select unit_name, unit_type, tool_name, tool_type, tool_data from {cfg.dbrectable } r ')
|
|
|
|
unit_name, unit_type, tool_name, tool_type, tool_data = cur.fetchone()
|
|
data_list = str(tool_data).strip("('{\"").strip("\"}\',)").split('","')
|
|
|
|
# Estrarre le informazioni degli ip dalla header
|
|
infos = extract_value(cfg.csv_infos, data_list[:9])
|
|
|
|
records = []
|
|
# Elabora le righe dei dati a partire dalla riga 8 in poi
|
|
for line in data_list:
|
|
if ";|;" in line:
|
|
# Rimuovi spazi bianchi o caratteri di nuova riga
|
|
input_data = line.strip().replace('\\n', '')
|
|
|
|
# Suddividi la stringa in sezioni usando ";|;" come separatore
|
|
parts = input_data.split(';|;')
|
|
|
|
# Verifica che ci siano almeno tre parti (timestamp, misure e nodi)
|
|
if len(parts) < 3:
|
|
print(f"Riga non valida: {input_data}")
|
|
continue
|
|
|
|
# Estrai la data/ora e le prime misurazioni
|
|
timestamp = parts[0]
|
|
measurements = parts[1]
|
|
|
|
# Estrai i valori di ciascun nodo e formatta i dati come JSON
|
|
nodes = parts[2:]
|
|
node_list = []
|
|
for i, node_data in enumerate(nodes, start=1):
|
|
node_dict = {"num": i}
|
|
# Dividi ogni nodo in valori separati da ";"
|
|
node_values = node_data.split(';')
|
|
for j, value in enumerate(node_values, start=0):
|
|
# Imposta i valori a -9999 se trovi "Dis."
|
|
node_dict['val' + str(j)] = -9999 if value == "Dis." else float(value)
|
|
node_list.append(node_dict)
|
|
|
|
# Prepara i dati per l'inserimento/aggiornamento
|
|
record = {
|
|
"unit_name": unit_name.upper(),
|
|
"unit_type": unit_type.upper(),
|
|
"tool_name": tool_name.upper(),
|
|
"tool_type": tool_type.upper(),
|
|
"ip_centralina": infos['IP'],
|
|
"ip_subnet": infos['Subnet'],
|
|
"ip_gateway": infos['Gateway'],
|
|
"Web_port": infos['Web port'],
|
|
"Ftp_port": infos['Ftp port'],
|
|
"event_timestamp": timestamp,
|
|
"battery_level": float(measurements.split(';')[0]),
|
|
"temperature": float(measurements.split(';')[1]),
|
|
"nodes_jsonb": json.dumps(node_list) # Converti la lista di dizionari in una stringa JSON
|
|
}
|
|
|
|
records.append(record)
|
|
|
|
# Se abbiamo raggiunto 500 record, esegui l'inserimento in batch
|
|
if len(records) >= 500:
|
|
print("raggiunti 500 record scrivo sul db")
|
|
write_db(engine, records)
|
|
records = []
|
|
|
|
write_db(engine, records)
|
|
|
|
|
|
def main():
|
|
# Load the configuration settings
|
|
cfg = setting.config()
|
|
|
|
try:
|
|
# Configura la connessione al database PostgreSQL
|
|
engine = create_engine(f'postgresql://{cfg.dbuser}:{cfg.dbpass}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbschema}')
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(message)s",
|
|
filename=cfg.logfilename,
|
|
level=logging.INFO,
|
|
)
|
|
elab_csv(engine, cfg)
|
|
|
|
except KeyboardInterrupt:
|
|
logging.info(
|
|
"Info: {}.".format("Shutdown requested...exiting")
|
|
)
|
|
|
|
except Exception:
|
|
print(
|
|
"{} - PID {:>5} >> Error: {}.".format(
|
|
ts.timestamp("log"), os.getpid(), sys.exc_info()[1]
|
|
)
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
main() |