loc ok

elab_orchestrator.py

@@ -11,7 +11,8 @@ import subprocess
 # Import custom modules for configuration and database connection
 from utils.config import loader_ftp_csv as setting
 from utils.database.connection import connetti_db
-from utils.database.loader_action import DATA_LOADED, get_matlab_cmd
+from utils.database.loader_action import get_matlab_cmd
+from utils.database import DATA_LOADED
 
 # Initialize the logger for this module
 logger = logging.getLogger(__name__)

load_orchestrator.py

@@ -10,7 +10,7 @@ import os
 # Import custom modules for configuration and database connection
 from utils.config import loader_load_data as setting
 from utils.database.connection import connetti_db
-from utils.database.loader_action import CSV_RECEIVED
+from utils.database import CSV_RECEIVED
 
 # Initialize the logger for this module
 logger = logging.getLogger()
@@ -49,7 +49,7 @@ async def worker(worker_id: int, queue: asyncio.Queue, cfg: object) -> None:
                 await asyncio.sleep(CSV_PROCESSING_DELAY)
             else:
                 logger.debug(f"Worker {worker_id} - Elaborazione completata correttamente")
-                await asyncio.sleep(CSV_PROCESSING_DELAY*worker_id)
+                await asyncio.sleep(CSV_PROCESSING_DELAY)
 
             # Segnala che il lavoro è completato
             queue.task_done()
@@ -107,9 +107,8 @@ async def load_csv(cfg: object) -> tuple:
                         logger.debug(f"Caricamento dinamico del modulo: {module_name}")
                         modulo = importlib.import_module(module_name)
                         logger.debug(f"Funzione 'main_loader' caricata dal modulo {module_name}")
-                        return True, True
                     except (ImportError, AttributeError) as e:
-                        logger.error(f"Errore nel caricamento del modulo {module_name}: {e}", exc_info=debug_mode)
+                        logger.warning(f"Modulo {module_name} non trovato: {e}", exc_info=debug_mode)
 
                 if not modulo:
                     logger.error(f"Nessun modulo trovato {module_names}")

utils/csv/data_preparation.py

@@ -91,19 +91,29 @@ def make_ain_din_matrix(cfg: object, id: int) -> list:
         list contains data fields similar to `make_matrix`, adjusted for LOC data.
     """
     UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
     righe = ToolData.splitlines()
     matrice_valori = []
-    pattern = r'(?:\d{4}/\d{2}/\d{2}|\d{2}/\d{2}/\d{4}) \d{2}:\d{2}:\d{2}(;[^;]+)+'
+    pattern = r'^(?:\d{4}\/\d{2}\/\d{2}|\d{2}\/\d{2}\/\d{4}) \d{2}:\d{2}:\d{2}(?:;\d+\.\d+){2}(?:;\d+){4}$'
-    for riga in [riga for riga in righe if re.match(pattern, riga)]:
+    if node_ains or node_dins:
-        timestamp, battery_voltage, unit_temperature, analog_input1, analog_input2, digital_input1, digital_input2  = riga.split(';')
+        for riga in [riga for riga in righe if re.match(pattern, riga)]:
-        event_date, event_time = timestamp.split(' ')
+            timestamp, batlevel, temperature, analog_input1, analog_input2, digital_input1, digital_input2  = riga.split(';')
-        valori = [analog_input1, analog_input2, digital_input1, digital_input2]
+            EventDate, EventTime = timestamp.split(' ')
-        matrice_valori.append([UnitName, ToolNameID, 1, date_check.conforma_data(event_date), event_time, battery_voltage, unit_temperature] + valori + ([None] * (19 - len(valori))))
+            if any(node_dins):
-
+                for node_num, digital_act in enumerate([digital_input1, digital_input2], start=1):
+                    matrice_valori.append([UnitName, ToolNameID, node_num, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + [digital_act] + ([None] * (19 - 1)))
+            else:
+                logger.info(f"Nessun Ingresso digitale per {UnitName} {ToolNameID}")
+                if any(node_ains):
+                    for node_num, analog_act in enumerate([analog_input1, analog_input2], start=1):
+                        matrice_valori.append([UnitName, ToolNameID, node_num, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + [analog_act] + ([None] * (19 - 1)))
+                else:
+                    logger.info(f"Nessun Ingresso analogico per {UnitName} {ToolNameID}")
     return matrice_valori
 
 def make_channels_matrix(cfg: object, id: int) -> list:
     UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
     righe = ToolData.splitlines()
     matrice_valori = []
     for riga in [riga for riga in righe if ';|;' in riga]:
@@ -111,7 +121,6 @@ def make_channels_matrix(cfg: object, id: int) -> list:
         EventDate, EventTime = timestamp.split(' ')
         valori_splitted = [valore for valore in rilevazioni.split(';') if valore != '|']
         valori_iter = iter(valori_splitted)
-        node_channels, node_types, node_ains, node_dins = get_nodes_type(cfg, ToolNameID, UnitName)
 
         valori_nodi = [list(islice(valori_iter, channels)) for channels in node_channels]
 

utils/csv/loaders.py

@@ -1,4 +1,5 @@
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_pipe_sep_matrix, make_ain_din_matrix, make_channels_matrix
 
 import logging
@@ -15,6 +16,8 @@ async def main_loader(cfg: object, id: int, action: str) -> None:
         function_to_call = type_matrix_mapping[action]
         # Create a matrix of values from the data
         matrice_valori = function_to_call(cfg, id)
+
+        logger.info("matrice valori creata")
         # Load the data into the database
         if load_data(cfg, matrice_valori):
             update_status(cfg, id, DATA_LOADED)

utils/database/__init__.py

@@ -0,0 +1,3 @@
+CSV_RECEIVED = 0
+DATA_LOADED = 1
+DATA_ELABORATED = 2

utils/database/loader_action.py

@@ -4,14 +4,13 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-CSV_RECEIVED = 0
-DATA_LOADED = 1
-DATA_ELABORATED = 2
-
 timestamp_cols = ['inserted_at', 'loaded_at', 'elaborated_at']
 
 
 def load_data(cfg: object, matrice_valori: list) -> bool :
+    if not matrice_valori:
+        logger.info("Nulla da caricare.")
+        return True
     sql_insert_RAWDATA = f'''
     INSERT IGNORE INTO {cfg.dbname}.{cfg.dbrawdata} (
         `UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,

utils/parsers/by_type/d2w_d2w.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g201_g201.py

@@ -1,6 +1,7 @@
 #!.venv/bin/python
 # Import necessary modules
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_matrix
 import logging
 

utils/parsers/by_type/g301_g301.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g801_iptm.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g801_loc.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as analog_dig_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return analog_dig_main_loader(cfg, id, "analogic_digital")
+    await analog_dig_main_loader(cfg, id, "analogic_digital")

utils/parsers/by_type/g801_mums.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g801_musa.py

@@ -1,6 +1,7 @@
 #!.venv/bin/python
 # Import necessary modules
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_matrix
 import logging
 

utils/parsers/by_type/g801_mux.py

@@ -1,6 +1,7 @@
 #!.venv/bin/python
 # Import necessary modules
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_matrix
 import logging
 

utils/parsers/by_type/g802_dsas.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g802_gd.py

@@ -1,6 +1,7 @@
 #!.venv/bin/python
 # Import necessary modules
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_matrix
 import logging
 

utils/parsers/by_type/g802_loc.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as analog_dig_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return analog_dig_main_loader(cfg, id, "analogic_digital")
+    await analog_dig_main_loader(cfg, id, "analogic_digital")

utils/parsers/by_type/g802_modb.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g802_mums.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/g802_mux.py

@@ -1,4 +1,4 @@
 from .g801_mux import main_loader as g801_mux_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return g801_mux_main_loader(cfg, id)
+    await g801_mux_main_loader(cfg, id)

utils/parsers/by_type/gs1_gs1.py

@@ -1,4 +1,4 @@
-from ..tlp_tlp import main_loader as tlp_tlp_main_loader
+from .tlp_tlp import main_loader as tlp_tlp_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return tlp_tlp_main_loader(cfg, id)
+    await tlp_tlp_main_loader(cfg, id)

utils/parsers/by_type/hortus_hortus.py

@@ -1,4 +1,4 @@
 from utils.csv.loaders import main_loader as pipe_sep_main_loader
 
 async def main_loader(cfg: object, id: int) -> None:
-    return pipe_sep_main_loader(cfg, id, "pipe_separator")
+    await pipe_sep_main_loader(cfg, id, "pipe_separator")

utils/parsers/by_type/tlp_tlp.py

@@ -1,6 +1,7 @@
 #!.venv/bin/python
 # Import necessary modules
-from utils.database.loader_action import load_data, update_status, DATA_LOADED
+from utils.database.loader_action import load_data, update_status
+from utils.database import DATA_LOADED
 from utils.csv.data_preparation import make_matrix
 import logging