GD RSSI + normalizza orario
This commit is contained in:
@@ -6,7 +6,8 @@ from utils.database import FLAG_TO_TIMESTAMP, BATCH_SIZE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def load_data(cfg: object, matrice_valori: list, pool: object) -> bool:
|
||||
|
||||
async def load_data(cfg: object, matrice_valori: list, pool: object, type: str) -> bool:
|
||||
"""Carica una lista di record di dati grezzi nel database.
|
||||
|
||||
Esegue un'operazione di inserimento massivo (executemany) per caricare i dati.
|
||||
@@ -17,6 +18,7 @@ async def load_data(cfg: object, matrice_valori: list, pool: object) -> bool:
|
||||
cfg (object): L'oggetto di configurazione contenente i nomi delle tabelle e i parametri di re-tentativo.
|
||||
matrice_valori (list): Una lista di tuple, dove ogni tupla rappresenta una riga da inserire.
|
||||
pool (object): Il pool di connessioni al database.
|
||||
type (str): tipo di caricamento dati. Per GD fa l'update del tool DT corrispondente
|
||||
|
||||
Returns:
|
||||
bool: True se il caricamento ha avuto successo, False altrimenti.
|
||||
@@ -24,44 +26,60 @@ async def load_data(cfg: object, matrice_valori: list, pool: object) -> bool:
|
||||
if not matrice_valori:
|
||||
logger.info("Nulla da caricare.")
|
||||
return True
|
||||
sql_insert_RAWDATA = f"""
|
||||
INSERT INTO {cfg.dbrawdata} (
|
||||
`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,
|
||||
`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,
|
||||
`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`,
|
||||
`BatLevelModule`,`TemperatureModule`, `RssiModule`
|
||||
)
|
||||
VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s
|
||||
) as new_data
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`BatLevel` = IF({cfg.dbrawdata}.`BatLevel` != new_data.`BatLevel`, new_data.`BatLevel`, {cfg.dbrawdata}.`BatLevel`),
|
||||
`Temperature` = IF({cfg.dbrawdata}.`Temperature` != new_data.Temperature, new_data.Temperature, {cfg.dbrawdata}.`Temperature`),
|
||||
`Val0` = IF({cfg.dbrawdata}.`Val0` != new_data.Val0 AND new_data.`Val0` IS NOT NULL, new_data.Val0, {cfg.dbrawdata}.`Val0`),
|
||||
`Val1` = IF({cfg.dbrawdata}.`Val1` != new_data.Val1 AND new_data.`Val1` IS NOT NULL, new_data.Val1, {cfg.dbrawdata}.`Val1`),
|
||||
`Val2` = IF({cfg.dbrawdata}.`Val2` != new_data.Val2 AND new_data.`Val2` IS NOT NULL, new_data.Val2, {cfg.dbrawdata}.`Val2`),
|
||||
`Val3` = IF({cfg.dbrawdata}.`Val3` != new_data.Val3 AND new_data.`Val3` IS NOT NULL, new_data.Val3, {cfg.dbrawdata}.`Val3`),
|
||||
`Val4` = IF({cfg.dbrawdata}.`Val4` != new_data.Val4 AND new_data.`Val4` IS NOT NULL, new_data.Val4, {cfg.dbrawdata}.`Val4`),
|
||||
`Val5` = IF({cfg.dbrawdata}.`Val5` != new_data.Val5 AND new_data.`Val5` IS NOT NULL, new_data.Val5, {cfg.dbrawdata}.`Val5`),
|
||||
`Val6` = IF({cfg.dbrawdata}.`Val6` != new_data.Val6 AND new_data.`Val6` IS NOT NULL, new_data.Val6, {cfg.dbrawdata}.`Val6`),
|
||||
`Val7` = IF({cfg.dbrawdata}.`Val7` != new_data.Val7 AND new_data.`Val7` IS NOT NULL, new_data.Val7, {cfg.dbrawdata}.`Val7`),
|
||||
`Val8` = IF({cfg.dbrawdata}.`Val8` != new_data.Val8 AND new_data.`Val8` IS NOT NULL, new_data.Val8, {cfg.dbrawdata}.`Val8`),
|
||||
`Val9` = IF({cfg.dbrawdata}.`Val9` != new_data.Val9 AND new_data.`Val9` IS NOT NULL, new_data.Val9, {cfg.dbrawdata}.`Val9`),
|
||||
`ValA` = IF({cfg.dbrawdata}.`ValA` != new_data.ValA AND new_data.`ValA` IS NOT NULL, new_data.ValA, {cfg.dbrawdata}.`ValA`),
|
||||
`ValB` = IF({cfg.dbrawdata}.`ValB` != new_data.ValB AND new_data.`ValB` IS NOT NULL, new_data.ValB, {cfg.dbrawdata}.`ValB`),
|
||||
`ValC` = IF({cfg.dbrawdata}.`ValC` != new_data.ValC AND new_data.`ValC` IS NOT NULL, new_data.ValC, {cfg.dbrawdata}.`ValC`),
|
||||
`ValD` = IF({cfg.dbrawdata}.`ValD` != new_data.ValD AND new_data.`ValD` IS NOT NULL, new_data.ValD, {cfg.dbrawdata}.`ValD`),
|
||||
`ValE` = IF({cfg.dbrawdata}.`ValE` != new_data.ValE AND new_data.`ValE` IS NOT NULL, new_data.ValE, {cfg.dbrawdata}.`ValE`),
|
||||
`ValF` = IF({cfg.dbrawdata}.`ValF` != new_data.ValF AND new_data.`ValF` IS NOT NULL, new_data.ValF, {cfg.dbrawdata}.`ValF`),
|
||||
`BatLevelModule` = IF({cfg.dbrawdata}.`BatLevelModule` != new_data.BatLevelModule, new_data.BatLevelModule, {cfg.dbrawdata}.`BatLevelModule`),
|
||||
`TemperatureModule` = IF({cfg.dbrawdata}.`TemperatureModule` != new_data.TemperatureModule, new_data.TemperatureModule, {cfg.dbrawdata}.`TemperatureModule`),
|
||||
`RssiModule` = IF({cfg.dbrawdata}.`RssiModule` != new_data.RssiModule, new_data.RssiModule, {cfg.dbrawdata}.`RssiModule`),
|
||||
`Created_at` = NOW()
|
||||
"""
|
||||
#logger.info(f"Query insert: {sql_insert_RAWDATA}.")
|
||||
|
||||
if type == "gd" and matrice_valori[0][0] == "RSSI":
|
||||
matrice_valori.pop(0)
|
||||
sql_load_RAWDATA = f"""
|
||||
UPDATE {cfg.dbrawdata} t1
|
||||
JOIN (
|
||||
SELECT id
|
||||
FROM {cfg.dbrawdata}
|
||||
WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s
|
||||
AND TIMESTAMP(`EventDate`, `EventTime`) BETWEEN %s AND %s
|
||||
ORDER BY ABS(TIMESTAMPDIFF(SECOND, TIMESTAMP(`EventDate`, `EventTime`), %s))
|
||||
LIMIT 1
|
||||
) t2 ON t1.id = t2.id
|
||||
SET t1.BatLevelModule = %s, t1.TemperatureModule = %s, t1.RssiModule = %s
|
||||
"""
|
||||
else:
|
||||
sql_load_RAWDATA = f"""
|
||||
INSERT INTO {cfg.dbrawdata} (
|
||||
`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,
|
||||
`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,
|
||||
`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`,
|
||||
`BatLevelModule`,`TemperatureModule`, `RssiModule`
|
||||
)
|
||||
VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s
|
||||
) as new_data
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`BatLevel` = IF({cfg.dbrawdata}.`BatLevel` != new_data.`BatLevel`, new_data.`BatLevel`, {cfg.dbrawdata}.`BatLevel`),
|
||||
`Temperature` = IF({cfg.dbrawdata}.`Temperature` != new_data.Temperature, new_data.Temperature, {cfg.dbrawdata}.`Temperature`),
|
||||
`Val0` = IF({cfg.dbrawdata}.`Val0` != new_data.Val0 AND new_data.`Val0` IS NOT NULL, new_data.Val0, {cfg.dbrawdata}.`Val0`),
|
||||
`Val1` = IF({cfg.dbrawdata}.`Val1` != new_data.Val1 AND new_data.`Val1` IS NOT NULL, new_data.Val1, {cfg.dbrawdata}.`Val1`),
|
||||
`Val2` = IF({cfg.dbrawdata}.`Val2` != new_data.Val2 AND new_data.`Val2` IS NOT NULL, new_data.Val2, {cfg.dbrawdata}.`Val2`),
|
||||
`Val3` = IF({cfg.dbrawdata}.`Val3` != new_data.Val3 AND new_data.`Val3` IS NOT NULL, new_data.Val3, {cfg.dbrawdata}.`Val3`),
|
||||
`Val4` = IF({cfg.dbrawdata}.`Val4` != new_data.Val4 AND new_data.`Val4` IS NOT NULL, new_data.Val4, {cfg.dbrawdata}.`Val4`),
|
||||
`Val5` = IF({cfg.dbrawdata}.`Val5` != new_data.Val5 AND new_data.`Val5` IS NOT NULL, new_data.Val5, {cfg.dbrawdata}.`Val5`),
|
||||
`Val6` = IF({cfg.dbrawdata}.`Val6` != new_data.Val6 AND new_data.`Val6` IS NOT NULL, new_data.Val6, {cfg.dbrawdata}.`Val6`),
|
||||
`Val7` = IF({cfg.dbrawdata}.`Val7` != new_data.Val7 AND new_data.`Val7` IS NOT NULL, new_data.Val7, {cfg.dbrawdata}.`Val7`),
|
||||
`Val8` = IF({cfg.dbrawdata}.`Val8` != new_data.Val8 AND new_data.`Val8` IS NOT NULL, new_data.Val8, {cfg.dbrawdata}.`Val8`),
|
||||
`Val9` = IF({cfg.dbrawdata}.`Val9` != new_data.Val9 AND new_data.`Val9` IS NOT NULL, new_data.Val9, {cfg.dbrawdata}.`Val9`),
|
||||
`ValA` = IF({cfg.dbrawdata}.`ValA` != new_data.ValA AND new_data.`ValA` IS NOT NULL, new_data.ValA, {cfg.dbrawdata}.`ValA`),
|
||||
`ValB` = IF({cfg.dbrawdata}.`ValB` != new_data.ValB AND new_data.`ValB` IS NOT NULL, new_data.ValB, {cfg.dbrawdata}.`ValB`),
|
||||
`ValC` = IF({cfg.dbrawdata}.`ValC` != new_data.ValC AND new_data.`ValC` IS NOT NULL, new_data.ValC, {cfg.dbrawdata}.`ValC`),
|
||||
`ValD` = IF({cfg.dbrawdata}.`ValD` != new_data.ValD AND new_data.`ValD` IS NOT NULL, new_data.ValD, {cfg.dbrawdata}.`ValD`),
|
||||
`ValE` = IF({cfg.dbrawdata}.`ValE` != new_data.ValE AND new_data.`ValE` IS NOT NULL, new_data.ValE, {cfg.dbrawdata}.`ValE`),
|
||||
`ValF` = IF({cfg.dbrawdata}.`ValF` != new_data.ValF AND new_data.`ValF` IS NOT NULL, new_data.ValF, {cfg.dbrawdata}.`ValF`),
|
||||
`BatLevelModule` = IF({cfg.dbrawdata}.`BatLevelModule` != new_data.BatLevelModule, new_data.BatLevelModule, {cfg.dbrawdata}.`BatLevelModule`),
|
||||
`TemperatureModule` = IF({cfg.dbrawdata}.`TemperatureModule` != new_data.TemperatureModule, new_data.TemperatureModule, {cfg.dbrawdata}.`TemperatureModule`),
|
||||
`RssiModule` = IF({cfg.dbrawdata}.`RssiModule` != new_data.RssiModule, new_data.RssiModule, {cfg.dbrawdata}.`RssiModule`),
|
||||
`Created_at` = NOW()
|
||||
"""
|
||||
#logger.info(f"Query insert: {sql_load_RAWDATA}.")
|
||||
#logger.info(f"Matrice valori da inserire: {matrice_valori}.")
|
||||
rc = False
|
||||
async with pool.acquire() as conn:
|
||||
@@ -73,7 +91,7 @@ async def load_data(cfg: object, matrice_valori: list, pool: object) -> bool:
|
||||
for i in range(0, len(matrice_valori), BATCH_SIZE):
|
||||
batch = matrice_valori[i:i + BATCH_SIZE]
|
||||
|
||||
await cur.executemany(sql_insert_RAWDATA, batch)
|
||||
await cur.executemany(sql_load_RAWDATA, batch)
|
||||
await conn.commit()
|
||||
|
||||
logger.info(f"Completed batch {i//BATCH_SIZE + 1}/{(len(matrice_valori)-1)//BATCH_SIZE + 1}")
|
||||
|
||||
Reference in New Issue
Block a user