alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix async

Browse Source
This commit is contained in:
alex
2025-05-27 23:50:25 +02:00
parent 670972bd45
commit c40378b654
17 changed files with 181 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
utils/csv/data_preparation.py
View File

@@ -1,5 +1,4 @@
#!.venv/bin/python
from utils.database.connection import connetti_db
from utils.database.nodes_query import get_nodes_type
import utils.timestamp.date_check as date_check
import logging
@@ -9,31 +8,15 @@ from itertools import islice
logger = logging.getLogger(__name__)
def get_data(cfg: object, id: int) -> tuple:
"""
Retrieves data for a specific tool from the database.
async def get_data(cfg: object, id: int, pool) -> tuple:
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f'select unit_name, tool_name, tool_data from {cfg.dbrectable} where id = {id}')
unit_name, tool_name, tool_data = await cur.fetchone()
This function connects to the database using the provided configuration,
executes a query to retrieve the unit name, tool name ID, and tool data
associated with the given ID from the raw data table, and returns the results.
return unit_name, tool_name, tool_data
Args:
cfg: A configuration object containing database connection parameters
and table names (cfg.dbname, cfg.dbrectable).
id: The ID of the tool record to retrieve.
Returns:
A tuple containing the unit name, tool name ID, and tool data.
"""
with connetti_db(cfg) as conn:
cur = conn.cursor()
cur.execute(f'select unit_name, tool_name, tool_data from {cfg.dbname}.{cfg.dbrectable} where id = {id}')
unit_name, tool_name, tool_data = cur.fetchone()
cur.close()
conn.close()
return unit_name, tool_name, tool_data
def make_pipe_sep_matrix(cfg: object, id: int) -> list:
async def make_pipe_sep_matrix(cfg: object, id: int, pool) -> list:
"""
Processes raw tool data and transforms it into a matrix format for database insertion.
@@ -55,7 +38,7 @@ def make_pipe_sep_matrix(cfg: object, id: int) -> list:
EventTime, BatLevel, Temperature, followed by up to 16 additional
measurement values (Val0 to ValF), padded with None if necessary.
"""
UnitName, ToolNameID, ToolData = get_data(cfg, id)
UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
righe = ToolData.splitlines()
matrice_valori = []
for riga in [riga for riga in righe if ';|;' in riga]:
@@ -68,7 +51,7 @@ def make_pipe_sep_matrix(cfg: object, id: int) -> list:
return matrice_valori
def make_ain_din_matrix(cfg: object, id: int) -> list:
async def make_ain_din_matrix(cfg: object, id: int, pool) -> list:
"""
Processes raw location (LOC) tool data and transforms it into a matrix format for database insertion.
@@ -90,7 +73,7 @@ def make_ain_din_matrix(cfg: object, id: int) -> list:
A list of lists (matrix) representing the processed LOC data. Each inner
list contains data fields similar to `make_matrix`, adjusted for LOC data.
"""
UnitName, ToolNameID, ToolData = get_data(cfg, id)
UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
righe = ToolData.splitlines()
matrice_valori = []
@@ -113,8 +96,8 @@ def make_ain_din_matrix(cfg: object, id: int) -> list:
return matrice_valori
def make_channels_matrix(cfg: object, id: int) -> list:
UnitName, ToolNameID, ToolData = get_data(cfg, id)
async def make_channels_matrix(cfg: object, id: int, pool) -> list:
UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
righe = ToolData.splitlines()
matrice_valori = []

8
utils/csv/loaders.py
View File

@@ -6,7 +6,7 @@ import logging
logger = logging.getLogger(__name__)
async def main_loader(cfg: object, id: int, action: str) -> None:
async def main_loader(cfg: object, id: int, pool, action: str) -> None:
type_matrix_mapping = {
"pipe_separator": make_pipe_sep_matrix,
"analogic_digital": make_ain_din_matrix,
@@ -15,11 +15,11 @@ async def main_loader(cfg: object, id: int, action: str) -> None:
if action in type_matrix_mapping:
function_to_call = type_matrix_mapping[action]
# Create a matrix of values from the data
matrice_valori = function_to_call(cfg, id)
matrice_valori = await function_to_call(cfg, id, pool)
logger.info("matrice valori creata")
# Load the data into the database
if load_data(cfg, matrice_valori):
update_status(cfg, id, DATA_LOADED)
if await load_data(cfg, matrice_valori, pool):
await update_status(cfg, id, DATA_LOADED, pool)
else:
logger.warning(f"Action '{action}' non riconosciuta.")

104
utils/database/loader_action.py
View File

@@ -1,5 +1,4 @@
#!.venv/bin/python
from utils.database.connection import connetti_db
import logging
logger = logging.getLogger(__name__)
@@ -7,12 +6,12 @@ logger = logging.getLogger(__name__)
timestamp_cols = ['inserted_at', 'loaded_at', 'elaborated_at']
def load_data(cfg: object, matrice_valori: list) -> bool :
async def load_data(cfg: object, matrice_valori: list, pool) -> bool :
if not matrice_valori:
logger.info("Nulla da caricare.")
return True
sql_insert_RAWDATA = f'''
INSERT IGNORE INTO {cfg.dbname}.{cfg.dbrawdata} (
INSERT INTO {cfg.dbrawdata} (
`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,
`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,
`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`,
@@ -23,44 +22,67 @@ def load_data(cfg: object, matrice_valori: list) -> bool :
%s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s
)
) as new_data
ON DUPLICATE KEY UPDATE
`BatLevel` = IF({cfg.dbrawdata}.`BatLevel` != new_data.`BatLevel`, new_data.`BatLevel`, {cfg.dbrawdata}.`BatLevel`),
`Temperature` = IF({cfg.dbrawdata}.`Temperature` != new_data.Temperature, new_data.Temperature, {cfg.dbrawdata}.`Temperature`),
`Val0` = IF({cfg.dbrawdata}.`Val0` != new_data.Val0, new_data.Val0, {cfg.dbrawdata}.`Val0`),
`Val1` = IF({cfg.dbrawdata}.`Val1` != new_data.Val1, new_data.Val1, {cfg.dbrawdata}.`Val1`),
`Val2` = IF({cfg.dbrawdata}.`Val2` != new_data.Val2, new_data.Val2, {cfg.dbrawdata}.`Val2`),
`Val3` = IF({cfg.dbrawdata}.`Val3` != new_data.Val3, new_data.Val3, {cfg.dbrawdata}.`Val3`),
`Val4` = IF({cfg.dbrawdata}.`Val4` != new_data.Val4, new_data.Val4, {cfg.dbrawdata}.`Val4`),
`Val5` = IF({cfg.dbrawdata}.`Val5` != new_data.Val5, new_data.Val5, {cfg.dbrawdata}.`Val5`),
`Val6` = IF({cfg.dbrawdata}.`Val6` != new_data.Val6, new_data.Val6, {cfg.dbrawdata}.`Val6`),
`Val7` = IF({cfg.dbrawdata}.`Val7` != new_data.Val7, new_data.Val7, {cfg.dbrawdata}.`Val7`),
`Val8` = IF({cfg.dbrawdata}.`Val8` != new_data.Val8, new_data.Val8, {cfg.dbrawdata}.`Val8`),
`Val9` = IF({cfg.dbrawdata}.`Val9` != new_data.Val9, new_data.Val9, {cfg.dbrawdata}.`Val9`),
`ValA` = IF({cfg.dbrawdata}.`ValA` != new_data.ValA, new_data.ValA, {cfg.dbrawdata}.`ValA`),
`ValB` = IF({cfg.dbrawdata}.`ValB` != new_data.ValB, new_data.ValB, {cfg.dbrawdata}.`ValB`),
`ValC` = IF({cfg.dbrawdata}.`ValC` != new_data.ValC, new_data.ValC, {cfg.dbrawdata}.`ValC`),
`ValD` = IF({cfg.dbrawdata}.`ValD` != new_data.ValD, new_data.ValD, {cfg.dbrawdata}.`ValD`),
`ValE` = IF({cfg.dbrawdata}.`ValE` != new_data.ValE, new_data.ValE, {cfg.dbrawdata}.`ValE`),
`ValF` = IF({cfg.dbrawdata}.`ValF` != new_data.ValF, new_data.ValF, {cfg.dbrawdata}.`ValF`),
`BatLevelModule` = IF({cfg.dbrawdata}.`BatLevelModule` != new_data.BatLevelModule, new_data.BatLevelModule, {cfg.dbrawdata}.`BatLevelModule`),
`TemperatureModule` = IF({cfg.dbrawdata}.`TemperatureModule` != new_data.TemperatureModule, new_data.TemperatureModule, {cfg.dbrawdata}.`TemperatureModule`),
`RssiModule` = IF({cfg.dbrawdata}.`RssiModule` != new_data.RssiModule, new_data.RssiModule, {cfg.dbrawdata}.`RssiModule`),
`Created_at` = NOW()
'''
with connetti_db(cfg) as conn:
cur = conn.cursor()
try:
cur.executemany(sql_insert_RAWDATA, matrice_valori)
conn.commit()
logging.info("Data loaded.")
rc = True
except Exception as e:
conn.rollback()
logging.error(f"Error: {e}.")
rc = False
finally:
conn.close()
return rc
def update_status(cfg: object, id: int, status: int) -> None:
with connetti_db(cfg) as conn:
cur = conn.cursor()
try:
cur.execute(f'update {cfg.dbname}.{cfg.dbrectable} set locked = 0, status = {status}, {timestamp_cols[status]} = now() where id = {id}')
conn.commit()
logging.info("Status updated.")
except Exception as e:
conn.rollback()
logging.error(f'Error: {e}')
async with pool.acquire() as conn:
async with conn.cursor() as cur:
try:
await cur.executemany(sql_insert_RAWDATA, matrice_valori)
await conn.commit()
logging.info("Data loaded.")
rc = True
except Exception as e:
await conn.rollback()
logging.error(f"Error: {e}.")
rc = False
finally:
return rc
def get_matlab_cmd(cfg: object, unit: str, tool: str) -> tuple:
with connetti_db(cfg) as conn:
cur = conn.cursor()
try:
cur.execute(f'''select m.matcall, t.ftp_send , t.unit_id, s.`desc` as statustools, t.api_send, u.inoltro_api, u.inoltro_api_url, u.inoltro_api_bearer_token, IFNULL(u.duedate, "") as duedate
from matfuncs as m
inner join tools as t on t.matfunc = m.id
inner join units as u on u.id = t.unit_id
inner join statustools as s on t.statustool_id = s.id
where t.name = "{tool}" and u.name = "{unit}"''')
return cur.fetchone()
except Exception as e:
logging.error(f'Error: {e}')
async def update_status(cfg: object, id: int, status: int, pool) -> None:
async with pool.acquire() as conn:
async with conn.cursor() as cur:
try:
await cur.execute(f'update {cfg.dbrectable} set locked = 0, status = {status}, {timestamp_cols[status]} = now() where id = {id}')
await conn.commit()
logging.info("Status updated.")
except Exception as e:
await conn.rollback()
logging.error(f'Error: {e}')
async def get_matlab_cmd(cfg: object, unit: str, tool: str, pool) -> tuple:
async with pool.acquire() as conn:
async with conn.cursor() as cur:
try:
await cur.execute(f'''select m.matcall, t.ftp_send , t.unit_id, s.`desc` as statustools, t.api_send, u.inoltro_api, u.inoltro_api_url, u.inoltro_api_bearer_token, IFNULL(u.duedate, "") as duedate
from matfuncs as m
inner join tools as t on t.matfunc = m.id
inner join units as u on u.id = t.unit_id
inner join statustools as s on t.statustool_id = s.id
where t.name = "{tool}" and u.name = "{unit}"''')
return cur.fetchone()
except Exception as e:
logging.error(f'Error: {e}')

4
utils/parsers/by_type/g201_g201.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as channels_main_loader
async def main_loader(cfg: object, id: int) -> None:
await channels_main_loader(cfg, id, "channels")
async def main_loader(cfg: object, id: int, pool) -> None:
await channels_main_loader(cfg, id, pool,"channels")

4
utils/parsers/by_type/g301_g301.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g801_iptm.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g801_loc.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as analog_dig_main_loader
async def main_loader(cfg: object, id: int) -> None:
await analog_dig_main_loader(cfg, id, "analogic_digital")
async def main_loader(cfg: object, id: int, pool) -> None:
await analog_dig_main_loader(cfg, id, pool, "analogic_digital")

4
utils/parsers/by_type/g801_mums.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g801_mux.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as channels_main_loader
async def main_loader(cfg: object, id: int) -> None:
await channels_main_loader(cfg, id, "channels")
async def main_loader(cfg: object, id: int, pool) -> None:
await channels_main_loader(cfg, id, pool, "channels")

4
utils/parsers/by_type/g802_dsas.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g802_loc.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as analog_dig_main_loader
async def main_loader(cfg: object, id: int) -> None:
await analog_dig_main_loader(cfg, id, "analogic_digital")
async def main_loader(cfg: object, id: int, pool) -> None:
await analog_dig_main_loader(cfg, id, pool, "analogic_digital")

4
utils/parsers/by_type/g802_modb.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g802_mums.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

4
utils/parsers/by_type/g802_mux.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as channels_main_loader
async def main_loader(cfg: object, id: int) -> None:
await channels_main_loader(cfg, id, "channels")
async def main_loader(cfg: object, id: int, pool) -> None:
await channels_main_loader(cfg, id, pool, "channels")

2
utils/parsers/by_type/gs1_gs1.py
View File

@@ -1,4 +1,4 @@
from .tlp_tlp import main_loader as tlp_tlp_main_loader
async def main_loader(cfg: object, id: int) -> None:
async def main_loader(cfg: object, id: int, pool) -> None:
await tlp_tlp_main_loader(cfg, id)

4
utils/parsers/by_type/hortus_hortus.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
await pipe_sep_main_loader(cfg, id, "pipe_separator")
async def main_loader(cfg: object, id: int, pool) -> None:
await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")