diff --git a/env/elab.ini b/env/elab.ini index 1fc273e..b75b7f9 100644 --- a/env/elab.ini +++ b/env/elab.ini @@ -1,5 +1,5 @@ [logging] - logFilename = ./elab_data.log + logFilename = ../logs/elab_data.log [threads] max_num = 10 diff --git a/env/ftp.ini b/env/ftp.ini index 36794d2..a913f17 100644 --- a/env/ftp.ini +++ b/env/ftp.ini @@ -18,7 +18,7 @@ path = /home/alex/aseftp/csvfs/ [logging] - logFilename = ./ftp_csv_rec.log + logFilename = ../logs/ftp_csv_rec.log [unit] Types = G801|G201|G301|G802|D2W|GFLOW|CR1000X|TLP|GS1|HORTUS|RIFKL|HEALTH-|READINGS-|INTEGRITY MONITOR|MESSPUNKTEPINI_|HIRPINIA|CO_[0-9]{4}_[0-9]|ISI CSV LOG diff --git a/env/load.ini b/env/load.ini index 2f11ff0..263c574 100644 --- a/env/load.ini +++ b/env/load.ini @@ -1,5 +1,5 @@ [logging] - logFilename = ./load_raw_data.log + logFilename = ../logs/load_raw_data.log [threads] max_num = 5 \ No newline at end of file diff --git a/env/send.ini b/env/send.ini index 1396c25..2bbc6ca 100644 --- a/env/send.ini +++ b/env/send.ini @@ -1,5 +1,5 @@ [logging] - logFilename = ./send_data.log + logFilename = ../logs/send_data.log [threads] max_num = 5 diff --git a/lista_gd.txt b/lista_gd.txt deleted file mode 100644 index c41ad8e..0000000 --- a/lista_gd.txt +++ /dev/null @@ -1 +0,0 @@ -select count(id) from RAWDATACOR where date(created_at) = '2025-07-27' and concat(UnitName,ToolNameID) in ('ID0005_DT0000','ID0005_DT0036','ID0005_DT0037','ID0005_DT0038','ID0005_DT0039','ID0005_DT0040','ID0005_DT0041','ID0005_DT0042','ID0005_DT0043','ID0006_DT0044','ID0006_DT0045','ID0006_DT0046','ID0006_DT0047','ID0006_DT0048','ID0006_DT0049','ID0007_DT0050','ID0007_DT0051','ID0007_DT0052','ID0007_DT0053','ID0010_DT0054','ID0010_DT0055','ID0012_DT0001','ID0012_DT0102','ID0012_DT0103','ID0012_DT0104','ID0013_DT0001','ID0034_DT0002','ID0039_DT0001','ID0039_DT0002','ID0039_DT0003','ID0039_DT0004','ID0039_DT0005','ID0039_DT0006','ID0039_DT0007','ID0039_DT0008','ID0190_DT0001','ID0190_DT0002','ID0190_DT0003','ID0190_DT0004','ID0190_DT0005','ID0198_DT0229','ID0198_DT0230','ID0199_DT0010','ID0215_DT0234','ID0215_DT0235','ID0215_DT0236','ID0215_DT0237','ID0226_DT0000','ID0226_DT0239','ID0226_DT0240','ID0226_DT0241','ID0226_DT0242','ID0226_DT0243','ID0226_DT0244','ID0226_DT0245','ID0226_DT0246','ID0226_DT0247','ID0226_DT0248','ID0226_DT0249','ID0226_DT0250','ID0226_DT0251') diff --git a/elab_orchestrator.py b/src/elab_orchestrator.py similarity index 51% rename from elab_orchestrator.py rename to src/elab_orchestrator.py index 9585546..90c2108 100755 --- a/elab_orchestrator.py +++ b/src/elab_orchestrator.py @@ -41,38 +41,45 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None: while True: try: logger.info("Inizio elaborazione") - record = await get_next_csv_atomic(pool, cfg.dbrectable, WorkflowFlags.DATA_LOADED, WorkflowFlags.DATA_ELABORATED) - if record: id, unit_type, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record] - tool_elab_info = await get_matlab_command(cfg, tool_name, unit_name, pool) - if tool_elab_info: - if tool_elab_info['statustools'].lower() in cfg.elab_status: - logger.info(f"Elaborazione id {id} per {unit_name} {tool_name} ") + if tool_type.lower() != "gd": # i tool GD non devono essere elaborati + tool_elab_info = await get_matlab_command(cfg, tool_name.upper(), unit_name.upper(), pool) + if tool_elab_info: + if tool_elab_info['statustools'].lower() in cfg.elab_status: + logger.info(f"Elaborazione id {id} per {unit_name} {tool_name} ") - matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh {cfg.matlab_runtime} {unit_name} {tool_name}" - proc = await asyncio.create_subprocess_shell( - matlab_cmd, - cwd=cfg.matlab_func_path, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) + matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh {cfg.matlab_runtime} {unit_name.upper()} {tool_name.upper()}" + proc = await asyncio.create_subprocess_shell( + matlab_cmd, + cwd=cfg.matlab_func_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) - stdout, stderr = await proc.communicate() + stdout, stderr = await proc.communicate() - if proc.returncode != 0: - logger.error("Errore durante l'elaborazione") - logger.error(stderr.decode().strip()) - with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f: - f.write(stderr.decode().strip()) + if proc.returncode != 0: + logger.error("Errore durante l'elaborazione") + logger.error(stderr.decode().strip()) + with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f: + f.write(stderr.decode().strip()) + else: + logger.info(stdout.decode().strip()) + await update_status(cfg, id, WorkflowFlags.DATA_ELABORATED, pool) + await unlock(cfg, id, pool) + await asyncio.sleep(ELAB_PROCESSING_DELAY) else: - logger.info(stdout.decode().strip()) + logger.info(f"id {id} - {unit_name} - {tool_name} {tool_elab_info['statustools']}: MatLab calc by-passed.") await update_status(cfg, id, WorkflowFlags.DATA_ELABORATED, pool) - await unlock(cfg, id, pool) - await asyncio.sleep(ELAB_PROCESSING_DELAY) - else: - logger.info(f"id {id} - {unit_name} - {tool_name} {tool_elab_info['statustools']}: MatLab calc by-passed.") + await update_status(cfg, id, WorkflowFlags.DUMMY_ELABORATED, pool) + await unlock(cfg, id, pool) + else: + await update_status(cfg, id, WorkflowFlags.DATA_ELABORATED, pool) + await update_status(cfg, id, WorkflowFlags.DUMMY_ELABORATED, pool) + await unlock(cfg, id, pool) + else: logger.info("Nessun record disponibile") await asyncio.sleep(NO_RECORD_SLEEP) diff --git a/ftp_csv_receiver.py b/src/ftp_csv_receiver.py similarity index 100% rename from ftp_csv_receiver.py rename to src/ftp_csv_receiver.py diff --git a/load_ftp_users.py b/src/load_ftp_users.py similarity index 100% rename from load_ftp_users.py rename to src/load_ftp_users.py diff --git a/load_orchestrator.py b/src/load_orchestrator.py similarity index 100% rename from load_orchestrator.py rename to src/load_orchestrator.py diff --git a/send_orchestrator.py b/src/send_orchestrator.py similarity index 100% rename from send_orchestrator.py rename to src/send_orchestrator.py diff --git a/utils/__init__.py b/src/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to src/utils/__init__.py diff --git a/utils/config/__init__.py b/src/utils/config/__init__.py similarity index 100% rename from utils/config/__init__.py rename to src/utils/config/__init__.py diff --git a/utils/config/loader_ftp_csv.py b/src/utils/config/loader_ftp_csv.py similarity index 97% rename from utils/config/loader_ftp_csv.py rename to src/utils/config/loader_ftp_csv.py index 6a9b8a3..63612ea 100644 --- a/utils/config/loader_ftp_csv.py +++ b/src/utils/config/loader_ftp_csv.py @@ -7,7 +7,7 @@ class Config: def __init__(self): c = ConfigParser() - c.read(["env/ftp.ini", "env/db.ini"]) + c.read(["../env/ftp.ini", "../env/db.ini"]) # FTP setting self.service_port = c.getint("ftpserver", "service_port") diff --git a/utils/config/loader_load_data.py b/src/utils/config/loader_load_data.py similarity index 94% rename from utils/config/loader_load_data.py rename to src/utils/config/loader_load_data.py index 8d21e78..c03fc70 100644 --- a/utils/config/loader_load_data.py +++ b/src/utils/config/loader_load_data.py @@ -7,7 +7,7 @@ class Config: def __init__(self): c = ConfigParser() - c.read(["env/load.ini", "env/db.ini"]) + c.read(["../env/load.ini", "../env/db.ini"]) # LOG setting self.logfilename = c.get("logging", "logFilename") diff --git a/utils/config/loader_matlab_elab.py b/src/utils/config/loader_matlab_elab.py similarity index 96% rename from utils/config/loader_matlab_elab.py rename to src/utils/config/loader_matlab_elab.py index c6fe9d2..eb3cc4b 100644 --- a/utils/config/loader_matlab_elab.py +++ b/src/utils/config/loader_matlab_elab.py @@ -7,7 +7,7 @@ class Config: def __init__(self): c = ConfigParser() - c.read(["env/elab.ini", "env/db.ini"]) + c.read(["../env/elab.ini", "../env/db.ini"]) # LOG setting self.logfilename = c.get("logging", "logFilename") diff --git a/utils/config/loader_send_data.py b/src/utils/config/loader_send_data.py similarity index 94% rename from utils/config/loader_send_data.py rename to src/utils/config/loader_send_data.py index 04db549..df04f08 100644 --- a/utils/config/loader_send_data.py +++ b/src/utils/config/loader_send_data.py @@ -7,7 +7,7 @@ class Config: def __init__(self): c = ConfigParser() - c.read(["env/send.ini", "env/db.ini"]) + c.read(["../env/send.ini", "../env/db.ini"]) # LOG setting self.logfilename = c.get("logging", "logFilename") diff --git a/utils/config/users_loader.py b/src/utils/config/users_loader.py similarity index 92% rename from utils/config/users_loader.py rename to src/utils/config/users_loader.py index 55ee303..e48eb3f 100644 --- a/utils/config/users_loader.py +++ b/src/utils/config/users_loader.py @@ -7,7 +7,7 @@ class Config: def __init__(self): c = ConfigParser() - c.read(["env/db.ini"]) + c.read(["../env/db.ini"]) # DB setting self.dbhost = c.get("db", "hostname") diff --git a/utils/csv/__init__.py b/src/utils/csv/__init__.py similarity index 100% rename from utils/csv/__init__.py rename to src/utils/csv/__init__.py diff --git a/utils/csv/data_preparation.py b/src/utils/csv/data_preparation.py similarity index 100% rename from utils/csv/data_preparation.py rename to src/utils/csv/data_preparation.py diff --git a/utils/csv/loaders.py b/src/utils/csv/loaders.py similarity index 100% rename from utils/csv/loaders.py rename to src/utils/csv/loaders.py diff --git a/utils/csv/parser.py b/src/utils/csv/parser.py similarity index 100% rename from utils/csv/parser.py rename to src/utils/csv/parser.py diff --git a/utils/database/__init__.py b/src/utils/database/__init__.py similarity index 52% rename from utils/database/__init__.py rename to src/utils/database/__init__.py index 9c43f9a..4ee18c3 100644 --- a/utils/database/__init__.py +++ b/src/utils/database/__init__.py @@ -1,9 +1,10 @@ class WorkflowFlags: - CSV_RECEIVED = 0 # 0000 - DATA_LOADED = 1 # 0001 - DATA_ELABORATED = 2 # 0010 - SENT_RAW_DATA = 4 # 0100 - SENT_ELAB_DATA = 8 # 1000 + CSV_RECEIVED = 0 # 0000 + DATA_LOADED = 1 # 0001 + DATA_ELABORATED = 2 # 0010 + SENT_RAW_DATA = 4 # 0100 + SENT_ELAB_DATA = 8 # 1000 + DUMMY_ELABORATED = 16 # 10000 # Mappatura flag -> colonna timestamp @@ -12,7 +13,8 @@ FLAG_TO_TIMESTAMP = { WorkflowFlags.DATA_LOADED: "loaded_at", WorkflowFlags.DATA_ELABORATED: "elaborated_at", WorkflowFlags.SENT_RAW_DATA: "sent_raw_at", - WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at" + WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at", + WorkflowFlags.DUMMY_ELABORATED: "elaborated_at" } # Dimensione degli split della matrice per il caricamento diff --git a/utils/database/connection.py b/src/utils/database/connection.py similarity index 100% rename from utils/database/connection.py rename to src/utils/database/connection.py diff --git a/utils/database/elab_query.py b/src/utils/database/elab_query.py similarity index 97% rename from utils/database/elab_query.py rename to src/utils/database/elab_query.py index 5aeb918..b7cab53 100644 --- a/utils/database/elab_query.py +++ b/src/utils/database/elab_query.py @@ -14,7 +14,7 @@ async def get_data_as_csv(cfg: dict, id_recv: int, unit: str, tool: str, matlab_ Args: cfg (dict): Configuration dictionary (not directly used in the query but passed for consistency). - id (int): The ID of the record being processed (used for logging). + id_recv (int): The ID of the record being processed (used for logging). pool (object): The database connection pool. unit (str): The name of the unit to filter the data. tool (str): The ID of the tool to filter the data. diff --git a/utils/database/loader_action.py b/src/utils/database/loader_action.py similarity index 100% rename from utils/database/loader_action.py rename to src/utils/database/loader_action.py diff --git a/utils/database/matlab_query.py b/src/utils/database/matlab_query.py similarity index 100% rename from utils/database/matlab_query.py rename to src/utils/database/matlab_query.py diff --git a/utils/database/nodes_query.py b/src/utils/database/nodes_query.py similarity index 100% rename from utils/database/nodes_query.py rename to src/utils/database/nodes_query.py diff --git a/utils/ftp/__init__.py b/src/utils/ftp/__init__.py similarity index 100% rename from utils/ftp/__init__.py rename to src/utils/ftp/__init__.py diff --git a/utils/ftp/file_management.py b/src/utils/ftp/file_management.py similarity index 100% rename from utils/ftp/file_management.py rename to src/utils/ftp/file_management.py diff --git a/utils/ftp/send_data.py b/src/utils/ftp/send_data.py similarity index 99% rename from utils/ftp/send_data.py rename to src/utils/ftp/send_data.py index fdff40b..4b6bacb 100644 --- a/utils/ftp/send_data.py +++ b/src/utils/ftp/send_data.py @@ -110,7 +110,7 @@ async def send_elab_csv_to_customer(cfg: dict, id: int, unit: str, tool: str, cs finally: csv_buffer.close() -def parse_ftp_parms(ftp_parms): +def parse_ftp_parms(ftp_parms: str) -> dict: """ Parses a string of FTP parameters into a dictionary. diff --git a/utils/ftp/user_admin.py b/src/utils/ftp/user_admin.py similarity index 100% rename from utils/ftp/user_admin.py rename to src/utils/ftp/user_admin.py diff --git a/utils/orchestrator_utils.py b/src/utils/orchestrator_utils.py similarity index 100% rename from utils/orchestrator_utils.py rename to src/utils/orchestrator_utils.py diff --git a/utils/parsers/__init__.py b/src/utils/parsers/__init__.py similarity index 100% rename from utils/parsers/__init__.py rename to src/utils/parsers/__init__.py diff --git a/utils/parsers/by_name/__init__.py b/src/utils/parsers/by_name/__init__.py similarity index 100% rename from utils/parsers/by_name/__init__.py rename to src/utils/parsers/by_name/__init__.py diff --git a/utils/parsers/by_type/__init__.py b/src/utils/parsers/by_type/__init__.py similarity index 100% rename from utils/parsers/by_type/__init__.py rename to src/utils/parsers/by_type/__init__.py diff --git a/utils/parsers/by_type/cr1000x_cr1000x.py b/src/utils/parsers/by_type/cr1000x_cr1000x.py similarity index 100% rename from utils/parsers/by_type/cr1000x_cr1000x.py rename to src/utils/parsers/by_type/cr1000x_cr1000x.py diff --git a/utils/parsers/by_type/d2w_d2w.py b/src/utils/parsers/by_type/d2w_d2w.py similarity index 100% rename from utils/parsers/by_type/d2w_d2w.py rename to src/utils/parsers/by_type/d2w_d2w.py diff --git a/utils/parsers/by_type/g201_g201.py b/src/utils/parsers/by_type/g201_g201.py similarity index 100% rename from utils/parsers/by_type/g201_g201.py rename to src/utils/parsers/by_type/g201_g201.py diff --git a/utils/parsers/by_type/g301_g301.py b/src/utils/parsers/by_type/g301_g301.py similarity index 100% rename from utils/parsers/by_type/g301_g301.py rename to src/utils/parsers/by_type/g301_g301.py diff --git a/utils/parsers/by_type/g801_iptm.py b/src/utils/parsers/by_type/g801_iptm.py similarity index 100% rename from utils/parsers/by_type/g801_iptm.py rename to src/utils/parsers/by_type/g801_iptm.py diff --git a/utils/parsers/by_type/g801_loc.py b/src/utils/parsers/by_type/g801_loc.py similarity index 100% rename from utils/parsers/by_type/g801_loc.py rename to src/utils/parsers/by_type/g801_loc.py diff --git a/utils/parsers/by_type/g801_mums.py b/src/utils/parsers/by_type/g801_mums.py similarity index 100% rename from utils/parsers/by_type/g801_mums.py rename to src/utils/parsers/by_type/g801_mums.py diff --git a/utils/parsers/by_type/g801_musa.py b/src/utils/parsers/by_type/g801_musa.py similarity index 100% rename from utils/parsers/by_type/g801_musa.py rename to src/utils/parsers/by_type/g801_musa.py diff --git a/utils/parsers/by_type/g801_mux.py b/src/utils/parsers/by_type/g801_mux.py similarity index 100% rename from utils/parsers/by_type/g801_mux.py rename to src/utils/parsers/by_type/g801_mux.py diff --git a/utils/parsers/by_type/g802_dsas.py b/src/utils/parsers/by_type/g802_dsas.py similarity index 100% rename from utils/parsers/by_type/g802_dsas.py rename to src/utils/parsers/by_type/g802_dsas.py diff --git a/utils/parsers/by_type/g802_gd.py b/src/utils/parsers/by_type/g802_gd.py similarity index 100% rename from utils/parsers/by_type/g802_gd.py rename to src/utils/parsers/by_type/g802_gd.py diff --git a/utils/parsers/by_type/g802_loc.py b/src/utils/parsers/by_type/g802_loc.py similarity index 100% rename from utils/parsers/by_type/g802_loc.py rename to src/utils/parsers/by_type/g802_loc.py diff --git a/utils/parsers/by_type/g802_modb.py b/src/utils/parsers/by_type/g802_modb.py similarity index 100% rename from utils/parsers/by_type/g802_modb.py rename to src/utils/parsers/by_type/g802_modb.py diff --git a/utils/parsers/by_type/g802_mums.py b/src/utils/parsers/by_type/g802_mums.py similarity index 100% rename from utils/parsers/by_type/g802_mums.py rename to src/utils/parsers/by_type/g802_mums.py diff --git a/utils/parsers/by_type/g802_mux.py b/src/utils/parsers/by_type/g802_mux.py similarity index 100% rename from utils/parsers/by_type/g802_mux.py rename to src/utils/parsers/by_type/g802_mux.py diff --git a/utils/parsers/by_type/gs1_gs1.py b/src/utils/parsers/by_type/gs1_gs1.py similarity index 100% rename from utils/parsers/by_type/gs1_gs1.py rename to src/utils/parsers/by_type/gs1_gs1.py diff --git a/utils/parsers/by_type/hortus_hortus.py b/src/utils/parsers/by_type/hortus_hortus.py similarity index 100% rename from utils/parsers/by_type/hortus_hortus.py rename to src/utils/parsers/by_type/hortus_hortus.py diff --git a/utils/parsers/by_type/isi_csv_log_vulink.py b/src/utils/parsers/by_type/isi_csv_log_vulink.py similarity index 100% rename from utils/parsers/by_type/isi_csv_log_vulink.py rename to src/utils/parsers/by_type/isi_csv_log_vulink.py diff --git a/utils/parsers/by_type/tlp_loc.py b/src/utils/parsers/by_type/tlp_loc.py similarity index 100% rename from utils/parsers/by_type/tlp_loc.py rename to src/utils/parsers/by_type/tlp_loc.py diff --git a/utils/parsers/by_type/tlp_tlp.py b/src/utils/parsers/by_type/tlp_tlp.py similarity index 100% rename from utils/parsers/by_type/tlp_tlp.py rename to src/utils/parsers/by_type/tlp_tlp.py diff --git a/utils/timestamp/__init__.py b/src/utils/timestamp/__init__.py similarity index 100% rename from utils/timestamp/__init__.py rename to src/utils/timestamp/__init__.py diff --git a/utils/timestamp/date_check.py b/src/utils/timestamp/date_check.py similarity index 100% rename from utils/timestamp/date_check.py rename to src/utils/timestamp/date_check.py