From d1582b8f9e55b9653a220191e6b54a22713ca464 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 22 Aug 2025 21:15:10 +0200 Subject: [PATCH] add multi file logs filter errors --- src/elab_orchestrator.py | 19 +++++++----- src/utils/connect/send_email.py | 14 +++++++++ src/utils/general.py | 55 ++++++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/elab_orchestrator.py b/src/elab_orchestrator.py index 5e150af..9d72f08 100755 --- a/src/elab_orchestrator.py +++ b/src/elab_orchestrator.py @@ -12,6 +12,7 @@ from utils.csv.loaders import get_next_csv_atomic from utils.orchestrator_utils import run_orchestrator, worker_context from utils.database.loader_action import update_status, unlock from utils.connect.send_email import send_error_email +from utils.general import read_error_lines_from_logs # Initialize the logger for this module logger = logging.getLogger() @@ -64,20 +65,24 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None: if proc.returncode != 0: logger.error("Errore durante l'elaborazione") logger.error(stderr.decode().strip()) - with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f: - f.write(stderr.decode().strip()) if proc.returncode == 124: error_type = f"Matlab elab excessive duration: killed after {cfg.matlab_timeout} seconds." else: error_type = f"Matlab elab failed: {proc.returncode}." + """ - da verificare i log degli errori come sono creati... dal perl: - "cd $MatlabErrorPath; cat _${unit}_${tool}*_\*_output_error.txt > ${unit}${tool}_output_error.txt"; + + da verificare i log dove prenderli + + with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f: + f.write(stderr.decode().strip()) + # errors = [line for line in stderr.decode().strip() if line.startswith("Error")] + # warnings = [line for line in stderr.decode().strip() if not line.startswith("Error")] """ - errors = [line for line in stderr.decode().strip() if line.startswith("Error")] - warnings = [line for line in stderr.decode().strip() if line.startswith("Warning")] - await send_error_email(unit_name.upper(), tool_name.upper(), tool_elab_info['matcall'], error_type, errors, warnings) # da verificare + + errors, warnings = await read_error_lines_from_logs(cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt") + await send_error_email(unit_name.upper(), tool_name.upper(), tool_elab_info['matcall'], error_type, errors, warnings) else: diff --git a/src/utils/connect/send_email.py b/src/utils/connect/send_email.py index 6635715..e147600 100644 --- a/src/utils/connect/send_email.py +++ b/src/utils/connect/send_email.py @@ -7,6 +7,20 @@ cfg = setting.Config() logger = logging.getLogger(__name__) async def send_error_email(unit_name: str, tool_name: str, matlab_cmd: str, matlab_error: str, errors: list, warnings: list) -> None: + """ + Sends an error email containing details about a MATLAB processing failure. + + The email includes information about the unit, tool, MATLAB command, error message, + and lists of specific errors and warnings encountered. + + Args: + unit_name (str): The name of the unit involved in the processing. + tool_name (str): The name of the tool involved in the processing. + matlab_cmd (str): The MATLAB command that was executed. + matlab_error (str): The main MATLAB error message. + errors (list): A list of detailed error messages from MATLAB. + warnings (list): A list of detailed warning messages from MATLAB. + """ # Creazione dell'oggetto messaggio msg = EmailMessage() diff --git a/src/utils/general.py b/src/utils/general.py index 4de451b..0b7a833 100644 --- a/src/utils/general.py +++ b/src/utils/general.py @@ -1,7 +1,60 @@ +import glob +import os + +import logging + +logger = logging.getLogger() + def alterna_valori(val1: str, val2: str) -> any: """ Restituisce alternativamente il primo ed il secondo valore """ while True: yield val1 - yield val2 \ No newline at end of file + yield val2 + + +async def read_error_lines_from_logs(base_path: str, pattern: str) -> tuple[list[str], list[str]]: + """ + Reads error and warning lines from log files matching a given pattern within a base path. + + This asynchronous function searches for log files, reads their content, and categorizes + lines starting with 'Error' as errors and all other non-empty lines as warnings. + + Args: + base_path (str): The base directory where log files are located. + pattern (str): The glob-style pattern to match log filenames (e.g., "*.txt", "prefix_*_output_error.txt"). + + Returns: + tuple[list[str], list[str]]: A tuple containing two lists: + - The first list contains all extracted error messages. + - The second list contains all extracted warning messages.""" + # Costruisce il path completo con il pattern + search_pattern = os.path.join(base_path, pattern) + + # Trova tutti i file che corrispondono al pattern + matching_files = glob.glob(search_pattern) + + if not matching_files: + logger.warning(f"Nessun file trovato per il pattern: {search_pattern}") + return [], [] + + errors = [] + warnings = [] + + for file_path in matching_files: + try: + with open(file_path, 'r', encoding='utf-8') as file: + lines = file.readlines() + for line in lines: + stripped_line = line.strip() + if stripped_line: # Ignora righe vuote + if stripped_line.startswith('Error'): + errors.append(stripped_line) + else: + warnings.append(stripped_line) + + except Exception as e: + logger.error(f"Errore durante la lettura del file {file_path}: {e}") + + return errors, warnings