ASE/load_orchestrator.py

#!.venv/bin/python

# Import necessary libraries
import logging
import importlib
import asyncio
import os
import aiomysql

# Import custom modules for configuration and database connection
from utils.config import loader_load_data as setting
from utils.database import CSV_RECEIVED

# Initialize the logger for this module
logger = logging.getLogger()

# Delay tra un processamento CSV e il successivo (in secondi)
CSV_PROCESSING_DELAY = 0.1
# Tempo di attesa se non ci sono record da elaborare
NO_RECORD_SLEEP = 20

async def get_next_csv_atomic(pool, table_name):
    """Preleva atomicamente il prossimo CSV da elaborare"""
    async with pool.acquire() as conn:
        async with conn.cursor() as cur:
            # Usa SELECT FOR UPDATE per lock atomico
            await cur.execute(f"""
                SELECT id, unit_type, tool_type, unit_name, tool_name
                FROM {table_name}
                WHERE locked = 0 AND status = %s
                ORDER BY id
                LIMIT 1
                FOR UPDATE SKIP LOCKED
            """, (CSV_RECEIVED,))

            result = await cur.fetchone()
            if result:
                await cur.execute(f"""
                    UPDATE {table_name}
                    SET locked = 1
                    WHERE id = %s
                """, (result[0],))
                await conn.commit()

            return result

async def worker(worker_id: int, cfg: object, pool) -> None:
    debug_mode = (logging.getLogger().getEffectiveLevel() == logging.DEBUG)
    logger.info(f"Worker {worker_id} - Avviato")

    while True:
        try:
            logger.info(f"Worker {worker_id} - Inizio elaborazione")

            record = await get_next_csv_atomic(pool, cfg.dbrectable)

            if record:
                success = await load_csv(record, cfg, worker_id, pool)
                if not success:
                    logger.error(f"Worker {worker_id} - Errore durante l'elaborazione")
                await asyncio.sleep(CSV_PROCESSING_DELAY)
            else:
                await asyncio.sleep(NO_RECORD_SLEEP)

        except Exception as e:
            logger.error(f"Worker {worker_id} - Errore durante l'esecuzione: {e}", exc_info=debug_mode)
            await asyncio.sleep(1)

async def load_csv(record: tuple, cfg: object, worker_id: int, pool) -> bool:

    debug_mode = (logging.getLogger().getEffectiveLevel() == logging.DEBUG)
    logger.debug(f"Worker {worker_id} - Inizio ricerca nuovo CSV da elaborare")

    id, unit_type, tool_type, unit_name, tool_name = record
    logger.info(f"Worker {worker_id} - Trovato CSV da elaborare: ID={id}, Tipo={unit_type}_{tool_type}, Nome={unit_name}_{tool_name}")

    # Costruisce il nome del modulo da caricare dinamicamente
    module_names = [f'utils.parsers.by_name.{unit_name.lower()}_{tool_name.lower()}',
                    f'utils.parsers.by_name.{unit_name.lower()}_{tool_type.lower()}',
                    f'utils.parsers.by_name.{unit_name.lower()}_all',
                    f'utils.parsers.by_type.{unit_type.lower()}_{tool_type.lower()}']
    modulo = None
    for module_name in module_names:
        try:
            logger.debug(f"Worker {worker_id} - Caricamento dinamico del modulo: {module_name}")
            modulo = importlib.import_module(module_name)
            logger.debug(f"Worker {worker_id} - Funzione 'main_loader' caricata dal modulo {module_name}")
            break
        except (ImportError, AttributeError) as e:
            logger.info(f"Worker {worker_id} - Modulo {module_name} non presente o non valido. {e}", exc_info=debug_mode)

    if not modulo:
        logger.error(f"Worker {worker_id} - Nessun modulo trovato {module_names}")
        return False

    # Ottiene la funzione 'main_loader' dal modulo

    funzione = getattr(modulo, "main_loader")

    # Esegui la funzione

    logger.info(f"Worker {worker_id} - Elaborazione con modulo {modulo} per ID={id}")
    await funzione(cfg, id, pool)
    logger.info(f"Worker {worker_id} - Elaborazione completata per ID={id}")
    return True



async def main():
    """Main function: avvia i worker e gestisce il ciclo principale."""
    logger.info("Avvio del sistema...")

    cfg = setting.Config()
    logger.info("Configurazione caricata correttamente")

    try:
        # Configura il logging globale
        log_level = os.getenv("LOG_LEVEL", "INFO").upper()
        debug_mode = (logging.getLogger().getEffectiveLevel() == logging.DEBUG)

        logging.basicConfig(
            format="%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s ",
            filename=cfg.logfilename,
            level=log_level,
        )
        logger.info("Logging configurato correttamente")


        # Numero massimo di worker concorrenti
        logger.info(f"Avvio di {cfg.max_threads} worker concorrenti")

        pool = await aiomysql.create_pool(
            host=cfg.dbhost,
            user=cfg.dbuser,
            password=cfg.dbpass,
            db=cfg.dbname,
            minsize=1,
            maxsize=cfg.max_threads*4
        )

        # Avvia i worker
        workers = [
            asyncio.create_task(worker(i, cfg, pool))
            for i in range(cfg.max_threads)
        ]

        logger.info("Sistema avviato correttamente. In attesa di nuovi task...")

        try:
            await asyncio.gather(*workers, return_exceptions=debug_mode)
        finally:
            pool.close()
            await pool.wait_closed()

    except KeyboardInterrupt:
        logger.info("Info: Shutdown richiesto... chiusura in corso")

    except Exception as e:
        logger.error(f"Errore principale: {e}", exc_info=debug_mode)


if __name__ == "__main__":
    asyncio.run(main())