alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

loc ok

Browse Source
This commit is contained in:
alex
2025-05-25 23:23:00 +02:00
parent 43acf4f415
commit 95c8ced201
23 changed files with 55 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
elab_orchestrator.py
View File

@@ -11,7 +11,8 @@ import subprocess
# Import custom modules for configuration and database connection
from utils.config import loader_ftp_csv as setting
from utils.database.connection import connetti_db
from utils.database.loader_action import DATA_LOADED, get_matlab_cmd
from utils.database.loader_action import get_matlab_cmd
from utils.database import DATA_LOADED
# Initialize the logger for this module
logger = logging.getLogger(__name__)

7
load_orchestrator.py
View File

@@ -10,7 +10,7 @@ import os
# Import custom modules for configuration and database connection
from utils.config import loader_load_data as setting
from utils.database.connection import connetti_db
from utils.database.loader_action import CSV_RECEIVED
from utils.database import CSV_RECEIVED
# Initialize the logger for this module
logger = logging.getLogger()
@@ -49,7 +49,7 @@ async def worker(worker_id: int, queue: asyncio.Queue, cfg: object) -> None:
await asyncio.sleep(CSV_PROCESSING_DELAY)
else:
logger.debug(f"Worker {worker_id} - Elaborazione completata correttamente")
await asyncio.sleep(CSV_PROCESSING_DELAY*worker_id)
await asyncio.sleep(CSV_PROCESSING_DELAY)
# Segnala che il lavoro è completato
queue.task_done()
@@ -107,9 +107,8 @@ async def load_csv(cfg: object) -> tuple:
logger.debug(f"Caricamento dinamico del modulo: {module_name}")
modulo = importlib.import_module(module_name)
logger.debug(f"Funzione 'main_loader' caricata dal modulo {module_name}")
return True, True
except (ImportError, AttributeError) as e:
logger.error(f"Errore nel caricamento del modulo {module_name}: {e}", exc_info=debug_mode)
logger.warning(f"Modulo {module_name} non trovato: {e}", exc_info=debug_mode)
if not modulo:
logger.error(f"Nessun modulo trovato {module_names}")

23
utils/csv/data_preparation.py
View File

@@ -91,19 +91,29 @@ def make_ain_din_matrix(cfg: object, id: int) -> list:
list contains data fields similar to `make_matrix`, adjusted for LOC data.
"""
UnitName, ToolNameID, ToolData = get_data(cfg, id)
node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
righe = ToolData.splitlines()
matrice_valori = []
pattern = r'(?:\d{4}/\d{2}/\d{2}|\d{2}/\d{2}/\d{4}) \d{2}:\d{2}:\d{2}(;[^;]+)+'
pattern = r'^(?:\d{4}\/\d{2}\/\d{2}|\d{2}\/\d{2}\/\d{4}) \d{2}:\d{2}:\d{2}(?:;\d+\.\d+){2}(?:;\d+){4}$'
if node_ains or node_dins:
for riga in [riga for riga in righe if re.match(pattern, riga)]:
timestamp, battery_voltage, unit_temperature, analog_input1, analog_input2, digital_input1, digital_input2 = riga.split(';')
event_date, event_time = timestamp.split(' ')
valori = [analog_input1, analog_input2, digital_input1, digital_input2]
matrice_valori.append([UnitName, ToolNameID, 1, date_check.conforma_data(event_date), event_time, battery_voltage, unit_temperature] + valori + ([None] * (19 - len(valori))))
timestamp, batlevel, temperature, analog_input1, analog_input2, digital_input1, digital_input2 = riga.split(';')
EventDate, EventTime = timestamp.split(' ')
if any(node_dins):
for node_num, digital_act in enumerate([digital_input1, digital_input2], start=1):
matrice_valori.append([UnitName, ToolNameID, node_num, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + [digital_act] + ([None] * (19 - 1)))
else:
logger.info(f"Nessun Ingresso digitale per {UnitName} {ToolNameID}")
if any(node_ains):
for node_num, analog_act in enumerate([analog_input1, analog_input2], start=1):
matrice_valori.append([UnitName, ToolNameID, node_num, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + [analog_act] + ([None] * (19 - 1)))
else:
logger.info(f"Nessun Ingresso analogico per {UnitName} {ToolNameID}")
return matrice_valori
def make_channels_matrix(cfg: object, id: int) -> list:
UnitName, ToolNameID, ToolData = get_data(cfg, id)
node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
righe = ToolData.splitlines()
matrice_valori = []
for riga in [riga for riga in righe if ';|;' in riga]:
@@ -111,7 +121,6 @@ def make_channels_matrix(cfg: object, id: int) -> list:
EventDate, EventTime = timestamp.split(' ')
valori_splitted = [valore for valore in rilevazioni.split(';') if valore != '|']
valori_iter = iter(valori_splitted)
node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
valori_nodi = [list(islice(valori_iter, channels)) for channels in node_channels]

5
utils/csv/loaders.py
View File

@@ -1,4 +1,5 @@
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_pipe_sep_matrix, make_ain_din_matrix, make_channels_matrix
import logging
@@ -15,6 +16,8 @@ async def main_loader(cfg: object, id: int, action: str) -> None:
function_to_call = type_matrix_mapping[action]
# Create a matrix of values from the data
matrice_valori = function_to_call(cfg, id)
logger.info("matrice valori creata")
# Load the data into the database
if load_data(cfg, matrice_valori):
update_status(cfg, id, DATA_LOADED)

3
utils/database/__init__.py
View File

@@ -0,0 +1,3 @@
CSV_RECEIVED = 0
DATA_LOADED = 1
DATA_ELABORATED = 2

7
utils/database/loader_action.py
View File

@@ -4,14 +4,13 @@ import logging
logger = logging.getLogger(__name__)
CSV_RECEIVED = 0
DATA_LOADED = 1
DATA_ELABORATED = 2
timestamp_cols = ['inserted_at', 'loaded_at', 'elaborated_at']
def load_data(cfg: object, matrice_valori: list) -> bool :
if not matrice_valori:
logger.info("Nulla da caricare.")
return True
sql_insert_RAWDATA = f'''
INSERT IGNORE INTO {cfg.dbname}.{cfg.dbrawdata} (
`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,

2
utils/parsers/by_type/d2w_d2w.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

3
utils/parsers/by_type/g201_g201.py
View File

@@ -1,6 +1,7 @@
#!.venv/bin/python
# Import necessary modules
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_matrix
import logging

2
utils/parsers/by_type/g301_g301.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

2
utils/parsers/by_type/g801_iptm.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

2
utils/parsers/by_type/g801_loc.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as analog_dig_main_loader
async def main_loader(cfg: object, id: int) -> None:
return analog_dig_main_loader(cfg, id, "analogic_digital")
await analog_dig_main_loader(cfg, id, "analogic_digital")

2
utils/parsers/by_type/g801_mums.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

3
utils/parsers/by_type/g801_musa.py
View File

@@ -1,6 +1,7 @@
#!.venv/bin/python
# Import necessary modules
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_matrix
import logging

3
utils/parsers/by_type/g801_mux.py
View File

@@ -1,6 +1,7 @@
#!.venv/bin/python
# Import necessary modules
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_matrix
import logging

2
utils/parsers/by_type/g802_dsas.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

3
utils/parsers/by_type/g802_gd.py
View File

@@ -1,6 +1,7 @@
#!.venv/bin/python
# Import necessary modules
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_matrix
import logging

2
utils/parsers/by_type/g802_loc.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as analog_dig_main_loader
async def main_loader(cfg: object, id: int) -> None:
return analog_dig_main_loader(cfg, id, "analogic_digital")
await analog_dig_main_loader(cfg, id, "analogic_digital")

2
utils/parsers/by_type/g802_modb.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

2
utils/parsers/by_type/g802_mums.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

2
utils/parsers/by_type/g802_mux.py
View File

@@ -1,4 +1,4 @@
from .g801_mux import main_loader as g801_mux_main_loader
async def main_loader(cfg: object, id: int) -> None:
return g801_mux_main_loader(cfg, id)
await g801_mux_main_loader(cfg, id)

4
utils/parsers/by_type/gs1_gs1.py
View File

@@ -1,4 +1,4 @@
from ..tlp_tlp import main_loader as tlp_tlp_main_loader
from .tlp_tlp import main_loader as tlp_tlp_main_loader
async def main_loader(cfg: object, id: int) -> None:
return tlp_tlp_main_loader(cfg, id)
await tlp_tlp_main_loader(cfg, id)

2
utils/parsers/by_type/hortus_hortus.py
View File

@@ -1,4 +1,4 @@
from utils.csv.loaders import main_loader as pipe_sep_main_loader
async def main_loader(cfg: object, id: int) -> None:
return pipe_sep_main_loader(cfg, id, "pipe_separator")
await pipe_sep_main_loader(cfg, id, "pipe_separator")

3
utils/parsers/tlp_tlp.py → utils/parsers/by_type/tlp_tlp.py
View File

@@ -1,6 +1,7 @@
#!.venv/bin/python
# Import necessary modules
from utils.database.loader_action import load_data, update_status, DATA_LOADED
from utils.database.loader_action import load_data, update_status
from utils.database import DATA_LOADED
from utils.csv.data_preparation import make_matrix
import logging