add multi file logs filter errors

src/elab_orchestrator.py

@@ -12,6 +12,7 @@ from utils.csv.loaders import get_next_csv_atomic
 from utils.orchestrator_utils import run_orchestrator, worker_context
 from utils.database.loader_action import update_status, unlock
 from utils.connect.send_email import send_error_email
+from utils.general import read_error_lines_from_logs
 
 # Initialize the logger for this module
 logger = logging.getLogger()
@@ -64,20 +65,24 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None:
                             if proc.returncode != 0:
                                 logger.error("Errore durante l'elaborazione")
                                 logger.error(stderr.decode().strip())
-                                with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f:
-                                    f.write(stderr.decode().strip())
 
                                 if proc.returncode == 124:
                                     error_type = f"Matlab elab excessive duration: killed after {cfg.matlab_timeout} seconds."
                                 else:
                                     error_type = f"Matlab elab failed: {proc.returncode}."
+
                                 """
-                                    da verificare i log degli errori come sono creati... dal perl:
+
-                                    "cd $MatlabErrorPath; cat _${unit}_${tool}*_\*_output_error.txt > ${unit}${tool}_output_error.txt";
+                                da verificare i log dove prenderli
+
+                                with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f:
+                                    f.write(stderr.decode().strip())
+                                # errors = [line for line in stderr.decode().strip() if line.startswith("Error")]
+                                # warnings = [line for line in stderr.decode().strip() if not line.startswith("Error")]
                                 """
-                                errors = [line for line in stderr.decode().strip() if line.startswith("Error")]
+
-                                warnings = [line for line in stderr.decode().strip() if line.startswith("Warning")]
+                                errors, warnings = await read_error_lines_from_logs(cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt")
-                                await send_error_email(unit_name.upper(), tool_name.upper(), tool_elab_info['matcall'], error_type, errors, warnings) # da verificare
+                                await send_error_email(unit_name.upper(), tool_name.upper(), tool_elab_info['matcall'], error_type, errors, warnings)
 
 
                             else:

src/utils/connect/send_email.py

@@ -7,6 +7,20 @@ cfg = setting.Config()
 logger = logging.getLogger(__name__)
 
 async def send_error_email(unit_name: str, tool_name: str, matlab_cmd: str, matlab_error: str, errors: list, warnings: list) -> None:
+    """
+    Sends an error email containing details about a MATLAB processing failure.
+
+    The email includes information about the unit, tool, MATLAB command, error message,
+    and lists of specific errors and warnings encountered.
+
+    Args:
+        unit_name (str): The name of the unit involved in the processing.
+        tool_name (str): The name of the tool involved in the processing.
+        matlab_cmd (str): The MATLAB command that was executed.
+        matlab_error (str): The main MATLAB error message.
+        errors (list): A list of detailed error messages from MATLAB.
+        warnings (list): A list of detailed warning messages from MATLAB.
+    """
 
     # Creazione dell'oggetto messaggio
     msg = EmailMessage()

src/utils/general.py

@@ -1,3 +1,10 @@
+import glob
+import os
+
+import logging
+
+logger = logging.getLogger()
+
 def alterna_valori(val1: str, val2: str) -> any:
     """
     Restituisce alternativamente il primo ed il secondo valore
@@ -5,3 +12,49 @@ def alterna_valori(val1: str, val2: str) -> any:
     while True:
         yield val1
         yield val2
+
+
+async def read_error_lines_from_logs(base_path: str, pattern: str) -> tuple[list[str], list[str]]:
+    """
+    Reads error and warning lines from log files matching a given pattern within a base path.
+
+    This asynchronous function searches for log files, reads their content, and categorizes
+    lines starting with 'Error' as errors and all other non-empty lines as warnings.
+
+    Args:
+        base_path (str): The base directory where log files are located.
+        pattern (str): The glob-style pattern to match log filenames (e.g., "*.txt", "prefix_*_output_error.txt").
+
+    Returns:
+        tuple[list[str], list[str]]: A tuple containing two lists:
+                                     - The first list contains all extracted error messages.
+                                     - The second list contains all extracted warning messages."""
+    # Costruisce il path completo con il pattern
+    search_pattern = os.path.join(base_path, pattern)
+
+    # Trova tutti i file che corrispondono al pattern
+    matching_files = glob.glob(search_pattern)
+
+    if not matching_files:
+        logger.warning(f"Nessun file trovato per il pattern: {search_pattern}")
+        return [], []
+
+    errors = []
+    warnings = []
+
+    for file_path in matching_files:
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                lines = file.readlines()
+                for line in lines:
+                    stripped_line = line.strip()
+                    if stripped_line:  # Ignora righe vuote
+                        if stripped_line.startswith('Error'):
+                            errors.append(stripped_line)
+                        else:
+                            warnings.append(stripped_line)
+
+        except Exception as e:
+            logger.error(f"Errore durante la lettura del file {file_path}: {e}")
+
+    return errors, warnings