alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lint con ruff

Browse Source
This commit is contained in:
alex
2025-09-22 22:30:54 +02:00
parent 35527c89cd
commit fb2b2724ed
54 changed files with 585 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/elab_orchestrator.py
View File

@@ -4,18 +4,18 @@ Orchestratore dei worker che lanciano le elaborazioni
"""
# Import necessary libraries
import logging
import asyncio
import logging
# Import custom modules for configuration and database connection
from utils.config import loader_matlab_elab as setting
from utils.database import WorkflowFlags
from utils.database.action_query import get_tool_info, check_flag_elab
from utils.csv.loaders import get_next_csv_atomic
from utils.orchestrator_utils import run_orchestrator, worker_context
from utils.database.loader_action import update_status, unlock
from utils.connect.send_email import send_error_email
from utils.csv.loaders import get_next_csv_atomic
from utils.database import WorkflowFlags
from utils.database.action_query import check_flag_elab, get_tool_info
from utils.database.loader_action import unlock, update_status
from utils.general import read_error_lines_from_logs
from utils.orchestrator_utils import run_orchestrator, worker_context
# Initialize the logger for this module
logger = logging.getLogger()
@@ -51,18 +51,16 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None:
record = await get_next_csv_atomic(pool, cfg.dbrectable, WorkflowFlags.DATA_LOADED, WorkflowFlags.DATA_ELABORATED)
if record:
rec_id, _, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
if tool_type.lower() != "gd": # i tool GD non devono essere elaborati ???
if tool_type.lower() != "gd": # i tool GD non devono essere elaborati ???
tool_elab_info = await get_tool_info(WorkflowFlags.DATA_ELABORATED, unit_name.upper(), tool_name.upper(), pool)
if tool_elab_info:
if tool_elab_info['statustools'].lower() in cfg.elab_status:
if tool_elab_info["statustools"].lower() in cfg.elab_status:
logger.info("Elaborazione ID %s per %s %s", rec_id, unit_name, tool_name)
await update_status(cfg, rec_id, WorkflowFlags.START_ELAB, pool)
matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh {cfg.matlab_runtime} {unit_name.upper()} {tool_name.upper()}"
matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh \
{cfg.matlab_runtime} {unit_name.upper()} {tool_name.upper()}"
proc = await asyncio.create_subprocess_shell(
matlab_cmd,
cwd=cfg.matlab_func_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
matlab_cmd, cwd=cfg.matlab_func_path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
@@ -82,9 +80,12 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None:
# errors = [line for line in stderr.decode().strip() if line.startswith("Error")]
# warnings = [line for line in stderr.decode().strip() if not line.startswith("Error")]
errors, warnings = await read_error_lines_from_logs(cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt")
await send_error_email(unit_name.upper(), tool_name.upper(), tool_elab_info['matcall'], error_type, errors, warnings)
errors, warnings = await read_error_lines_from_logs(
cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt"
)
await send_error_email(
unit_name.upper(), tool_name.upper(), tool_elab_info["matcall"], error_type, errors, warnings
)
else:
logger.info(stdout.decode().strip())
@@ -92,7 +93,9 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None:
await unlock(cfg, rec_id, pool)
await asyncio.sleep(ELAB_PROCESSING_DELAY)
else:
logger.info("ID %s %s - %s %s: MatLab calc by-passed.", rec_id, unit_name, tool_name, tool_elab_info['statustools'])
logger.info(
"ID %s %s - %s %s: MatLab calc by-passed.", rec_id, unit_name, tool_name, tool_elab_info["statustools"]
)
await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
await update_status(cfg, rec_id, WorkflowFlags.DUMMY_ELABORATED, pool)
await unlock(cfg, rec_id, pool)
@@ -108,7 +111,7 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None:
logger.info("Flag fermo elaborazione attivato")
await asyncio.sleep(NO_RECORD_SLEEP)
except Exception as e: # pylint: disable=broad-except
except Exception as e: # pylint: disable=broad-except
logger.error("Errore durante l'esecuzione: %s", e, exc_info=debug_mode)
await asyncio.sleep(1)
@@ -117,5 +120,6 @@ async def main():
"""Funzione principale che avvia l'elab_orchestrator."""
await run_orchestrator(setting.Config, worker)
if __name__ == "__main__":
asyncio.run(main())
asyncio.run(main())