alex/cvs_ase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prova1

Browse Source
This commit is contained in:
Alessandro Battilani
2024-11-10 15:58:19 +01:00
parent fbac5aa277
commit afd028b794
1 changed files with 23 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
transform_file.py
View File

@@ -1,7 +1,7 @@
import pandas as pd
import json import json
import psycopg2 import psycopg2
from sqlalchemy import create_engine from sqlalchemy import create_engine, text
from sqlalchemy.dialects.postgresql import insert
# Configura la connessione al database PostgreSQL # Configura la connessione al database PostgreSQL
engine = create_engine('postgresql://asepg:batt1l0@10.211.114.101:5432/asedb') engine = create_engine('postgresql://asepg:batt1l0@10.211.114.101:5432/asedb')
@@ -41,6 +41,7 @@ for line in lines[7:]:
nodes = parts[2:] nodes = parts[2:]
node_list = [] node_list = []
records = []
for i, node_data in enumerate(nodes, start=1): for i, node_data in enumerate(nodes, start=1):
# Dividi ogni nodo in valori separati da ";" # Dividi ogni nodo in valori separati da ";"
node_values = node_data.split(';') node_values = node_data.split(';')
@@ -58,20 +59,34 @@ for line in lines[7:]:
} }
node_list.append(node_dict) node_list.append(node_dict)
# Crea il DataFrame con una colonna JSONB per i nodi # Prepara i dati per l'inserimento/aggiornamento
df = pd.DataFrame([{ record = {
"event_timestamp": pd.to_datetime(timestamp),
"tipo_centralina": tipo_centralina, "tipo_centralina": tipo_centralina,
"unit": unit, "unit": unit,
"ip_centralina": ip_centralina, "ip_centralina": ip_centralina,
"path": path, "path": path,
"ip_gateway": ip_gateway, "ip_gateway": ip_gateway,
"event_timestamp": timestamp,
"battery_level": float(measurements.split(';')[0]), "battery_level": float(measurements.split(';')[0]),
"temperature": float(measurements.split(';')[1]), "temperature": float(measurements.split(';')[1]),
"nodes_jsonb": json.dumps(node_list) # Converti la lista di dizionari in una stringa JSON "nodes_jsonb": json.dumps(node_list) # Converti la lista di dizionari in una stringa JSON
}]) }
records.append(record)
# Carica il DataFrame in una tabella PostgreSQL # Esegui l'upsert con un comando SQL
df.to_sql('dataraw', engine, if_exists='append', index=False) with engine.connect() as conn:
with conn.execution_options(isolation_level='AUTOCOMMIT'):
result = conn.execute(text("""
INSERT INTO dataraw (tipo_centralina, unit, ip_centralina, path, ip_gateway, event_timestamp, battery_level, temperature, nodes_jsonb)
VALUES (:tipo_centralina, :unit, :ip_centralina, :path, :ip_gateway, :event_timestamp, :battery_level, :temperature, :nodes_jsonb)
ON CONFLICT ON CONSTRAINT dataraw_unique
DO UPDATE SET
path = EXCLUDED.path,
ip_gateway = EXCLUDED.ip_gateway,
battery_level = EXCLUDED.battery_level,
temperature = EXCLUDED.temperature,
nodes_jsonb = EXCLUDED.nodes_jsonb;
"""), records)
print(result)
print("Tutte le righe del file sono state caricate con successo nella tabella PostgreSQL!") print("Tutte le righe del file sono state caricate con successo nella tabella PostgreSQL!")