evol 3

ftpcsvreceiver.ini

@@ -19,6 +19,7 @@
 
 [csvelab]
   logFilename = csvElab.log
+  max_threads = 5
 
 [db]
   hostname = 10.211.114.173
@@ -29,14 +30,14 @@
   dbSchema = public
   userTableName = virtusers
   recTableName = received
-  rawTableName = rawdatacor
+  rawTableName = RAWDATACOR
 
 [unit]
   Types = G801|G201|G301|G802|D2W|GFLOW|CR1000X|TLP|GS1
   Names = ID[0-9]{4}|IX[0-9]{4}
 
 [tool]
-  Types = MUX|MUMS|MODB|IPTM|MUSA|LOC|GD|D2W|CR1000X|G301|NESA
+  Types = MUX|MUMS|MODB|IPTM|MUSA|LOC|GD|D2W|CR1000X|G301|NESA|GS1|G201|TLP|DSAS
   Names = LOC[0-9]{4}|DT[0-9]{4}|GD[0-9]{4}|[0-9]{18}|measurement
 
 [csv]

orchestrator.py

@@ -1,79 +1,71 @@
-#!/usr/bin/env python3
+#!.venv/bin/python
 
 import mysql.connector
-import os
-import sys
 import logging
 import importlib
+import time
+import threading
 
-from utils.time import timestamp_fmt as ts
 from utils.config import loader as setting
-#from unit_tool_mod import g801_mums, g801_mux
+from utils.database.connection import connetti_db
 
-def conn_db(cfg):
+logger = logging.getLogger(__name__)
-    """Establishes a connection to the MySQL database.
 
-    Args:
+def elab_csv(cfg, threads):
-        cfg: The configuration object containing database connection details.
-
-    Returns:
-        A MySQL database connection object.
-    """
     try:
-        conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
+        with connetti_db(cfg) as conn:
-        conn.autocommit = True
-        return conn
-    except mysql.connector.Error as e:
-        print(f"Error: {e}")
-        logging.error(f'{e}')
-    return None
-
-def elab_csv(cfg):
-    try:
-        with conn_db(cfg) as conn:
             cur = conn.cursor()
             cur.execute(f'select id, unit_name, unit_type, tool_name, tool_type, tool_data from {cfg.dbname}.{cfg.dbrectable} where locked = 0 and status = 0 limit 1')
             id, unit_name, unit_type, tool_name, tool_type, tool_data = cur.fetchone()
-            cur.execute(f'update {cfg.dbname}.{cfg.dbrectable} set locked = 1 where id = {id}')
+            if id:
+                cur.execute(f'update {cfg.dbname}.{cfg.dbrectable} set locked = 1 where id = {id}')
+
+                module_name = f'utils.parsers.{unit_type.lower()}_{tool_type.lower()}'
+                modulo = importlib.import_module(module_name)
+
+                funzione = getattr(modulo, "main_loader")
+
+                # Chiamare la funzione
+                thread = threading.Thread(target = funzione, args=(cfg, id))
+                threads.append(thread)
+                thread.start()
+                return True
+            else:
+                time.sleep(20)
+                return False
+
+
     except mysql.connector.Error as e:
         print(f"Error: {e}")
-        logging.error(f'{e}')
+        logger.error(f'{e}')
-
-    module_name = f'unit_tool_mod.{unit_type.lower()}_{tool_type.lower()}'
-    modulo = importlib.import_module(module_name)
-
-    funzione = getattr(modulo, "chi_sono")
-
-    # Chiamare la funzione
-    risultato = funzione(unit_name, tool_name)
-    print(f'risultato: {risultato}')
-
 
 def main():
     # Load the configuration settings
-    cfg = setting.config()
+    cfg = setting.Config()
 
     try:
         # Configura la connessione al database PostgreSQL
         # Configure logging
         logging.basicConfig(
-            format="%(asctime)s %(message)s",
+            format="%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s ",
-            filename=cfg.elablog,
+            filename=cfg.logfilename,
             level=logging.INFO,
         )
-        elab_csv(cfg)
+        threads = []
+        while True:
+            while len(threads) > cfg.max_threads:
+                for thread in threads:
+                    if not thread.is_alive():
+                        threads.remove(thread)
+            if elab_csv(cfg, threads):
+                logger.info(f"Threads in execution: {len(threads)}")
+                pass
 
     except KeyboardInterrupt:
-        logging.info(
+        logger.info("Info: Shutdown requested...exiting")
-            "Info: {}.".format("Shutdown requested...exiting")
-        )
 
-    except Exception:
+    except Exception as e:
-        print(
+        logger.info(f"Error: {e}.")
-            "{} - PID {:>5} >> Error: {}.".format(
-                ts.timestamp("log"), os.getpid(), sys.exc_info()[1]
-            )
-        )
 
 if __name__ == "__main__":
     main()

utils/config/loader.py

@@ -25,6 +25,7 @@ class Config:
 
         # LOADER setting
         self.elablog = c.get("csvelab", "logFilename")
+        self.max_threads = c.getint("csvelab", "max_threads")
 
         # DB setting
         self.dbhost = c.get("db", "hostname")
@@ -35,7 +36,7 @@ class Config:
         self.dbschema = c.get("db", "dbSchema")
         self.dbusertable = c.get("db", "userTableName")
         self.dbrectable = c.get("db", "recTableName")
-        self.dbdataraw = c.get("db", "rawTableName")
+        self.dbrawdata = c.get("db", "rawTableName")
 
         # unit setting
         self.units_name = [part for part in c.get("unit", "Names").split('|')]

utils/database/loader.py

@@ -0,0 +1,32 @@
+#!.venv/bin/python
+from utils.database.connection import connetti_db
+import logging
+
+logger = logging.getLogger(__name__)
+
+def load_data(cfg, matrice_valori):
+    sql_insert_RAWDATA = f'''
+    INSERT IGNORE INTO {cfg.dbname}.{cfg.dbrawdata} (
+        `UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,
+        `Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,
+        `Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`,
+        `BatLevelModule`,`TemperatureModule`, `RssiModule`
+    )
+    VALUES (
+        %s, %s, %s, %s, %s, %s, %s,
+        %s, %s, %s, %s, %s, %s, %s, %s,
+        %s, %s, %s, %s, %s, %s, %s, %s,
+        %s, %s, %s
+    )
+    '''
+    with connetti_db(cfg) as conn:
+        cur = conn.cursor()
+        try:
+            cur.executemany(sql_insert_RAWDATA, matrice_valori)
+            conn.commit()
+        except Exception as e:
+            conn.rollback()
+            print(f'Error: {e}')
+        finally:
+            cur.close()
+            conn.close()

utils/parsers/cr1000x_cr1000x.py

@@ -0,0 +1,3 @@
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/d2w_d2w.py

@@ -0,0 +1,3 @@
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/data_preparation.py

@@ -0,0 +1,28 @@
+#!.venv/bin/python
+from utils.database.connection import connetti_db
+import utils.timestamp.date_check as date_check
+import logging
+
+logger = logging.getLogger(__name__)
+
+def get_data(cfg, id):
+    with connetti_db(cfg) as conn:
+        cur = conn.cursor()
+        cur.execute(f'select unit_name, tool_name, tool_data from {cfg.dbname}.{cfg.dbrectable} where id = {id}')
+        unit_name, tool_name, tool_data = cur.fetchone()
+        cur.close()
+        conn.close()
+        return unit_name, tool_name, tool_data
+
+def make_matrix(cfg, id):
+    UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    righe = ToolData.splitlines()
+    matrice_valori = []
+    for riga in [riga for riga in righe if ';|;' in riga]:
+        timestamp, batlevel, temperature, rilevazioni = riga.split(';',3)
+        EventDate, EventTime = timestamp.split(' ')
+        valori_nodi = rilevazioni.lstrip('|;').rstrip(';').split(';|;') # Toglie '|;' iniziali, toglie eventuali ';' finali, dividi per ';|;'
+        for num_nodo, valori_nodo in enumerate(valori_nodi, start=1):
+            valori = valori_nodo.split(';')
+            matrice_valori.append([UnitName, ToolNameID, num_nodo, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + valori + ([None] * (19 - len(valori))))
+    return matrice_valori

utils/parsers/g201_g201.py

@@ -0,0 +1,3 @@
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g301_g301.py

@@ -0,0 +1,3 @@
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g801_iptm.py

@@ -0,0 +1,3 @@
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g801_loc.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g801_mums.py

@@ -1,3 +1,10 @@
-def chi_sono(unit, tool):
+#!.venv/bin/python
-    print(f'g801_mums: {unit} - {tool}')
+from utils.database.loader import load_data
-    return f'g801_mums: {unit} - {tool}'
+from utils.parsers.data_preparation import make_matrix
+import logging
+
+logger = logging.getLogger(__name__)
+
+def main_loader(cfg, id):
+    matrice_valori = make_matrix(cfg, id)
+    load_data(cfg, matrice_valori)

utils/parsers/g801_musa.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g801_mux.py

@@ -1,3 +1,5 @@
+import time
 def chi_sono(unit, tool):
-    print(f'g801_mux: {unit} - {tool}')
+    print(f'{__name__}: {unit} - {tool}')
-    return f'g801_mux: {unit} - {tool}'
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_dsas.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_gd.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_loc.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_modb.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_mums.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/g802_mux.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/gs1_gs1.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/tlp_loc.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'

utils/parsers/tlp_tlp.py

@@ -0,0 +1,5 @@
+import time
+def chi_sono(unit, tool):
+    print(f'{__name__}: {unit} - {tool}')
+    time.sleep(20)
+    return f'{__name__}: {unit} - {tool}'