96 lines
4.0 KiB
Python
Executable File
96 lines
4.0 KiB
Python
Executable File
#!.venv/bin/python
|
|
|
|
# Import necessary libraries
|
|
import mysql.connector
|
|
import logging
|
|
import importlib
|
|
import time
|
|
import threading
|
|
|
|
# Import custom modules for configuration and database connection
|
|
from utils.config import loader as setting
|
|
from utils.database.connection import connetti_db
|
|
from utils.database.loader_action import CSV_RECEIVED
|
|
|
|
# Initialize the logger for this module
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Function to elaborate CSV data
|
|
def load_csv(cfg: object, threads: list) -> bool:
|
|
try:
|
|
# Establish a database connection
|
|
with connetti_db(cfg) as conn:
|
|
cur = conn.cursor()
|
|
# Select a single record from the raw data table that is not currently locked and has a status of 0
|
|
cur.execute(f'select id, unit_name, unit_type, tool_name, tool_type, tool_data from {cfg.dbname}.{cfg.dbrectable} where locked = 0 and status = {CSV_RECEIVED} limit 1')
|
|
id, unit_name, unit_type, tool_name, tool_type, tool_data = cur.fetchone()
|
|
if id:
|
|
# If a record is found, lock it by updating the 'locked' field to 1
|
|
cur.execute(f'update {cfg.dbname}.{cfg.dbrectable} set locked = 1 where id = {id}')
|
|
|
|
# Construct the module name based on unit and tool types for dynamic import
|
|
module_name = f'utils.parsers.{unit_type.lower()}_{tool_type.lower()}'
|
|
# Dynamically import the module
|
|
modulo = importlib.import_module(module_name)
|
|
|
|
# Get the 'main_loader' function from the imported module
|
|
funzione = getattr(modulo, "main_loader")
|
|
|
|
# Create a new thread to execute the 'main_loader' function with the configuration and record ID
|
|
thread = threading.Thread(target = funzione, args=(cfg, id))
|
|
# Add the thread to the list of active threads
|
|
threads.append(thread)
|
|
# Start the thread's execution
|
|
thread.start()
|
|
# Return True to indicate that a record was processed
|
|
return True
|
|
else:
|
|
# If no record is found, wait for 20 seconds before attempting to fetch again
|
|
time.sleep(20)
|
|
# Return False to indicate that no record was processed
|
|
return False
|
|
|
|
|
|
except mysql.connector.Error as e:
|
|
# Handle database connection errors
|
|
print(f"Error: {e}")
|
|
logger.error(f'{e}')
|
|
|
|
def main():
|
|
# Load the configuration settings
|
|
cfg = setting.Config()
|
|
|
|
try:
|
|
# Configure logging to write log messages to a file with a specific format
|
|
logging.basicConfig(
|
|
format="%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s ",
|
|
filename=cfg.logfilename,
|
|
level=logging.INFO,
|
|
)
|
|
# Initialize an empty list to keep track of active threads
|
|
threads = []
|
|
# Enter an infinite loop to continuously process records
|
|
while True:
|
|
# Check if the number of active threads exceeds the maximum allowed
|
|
while len(threads) > cfg.max_threads:
|
|
# Iterate through the list of threads
|
|
for thread in threads:
|
|
# If a thread is no longer alive (has finished execution)
|
|
if not thread.is_alive():
|
|
# Remove it from the list of active threads
|
|
threads.remove(thread)
|
|
# Attempt to process a CSV record
|
|
if load_csv(cfg, threads):
|
|
# If a record was successfully processed, log the number of threads currently running
|
|
logger.info(f"Threads in execution: {len(threads)}")
|
|
pass
|
|
|
|
except KeyboardInterrupt:
|
|
# Handle a keyboard interrupt (e.g., Ctrl+C) to gracefully shut down the program
|
|
logger.info("Info: Shutdown requested...exiting")
|
|
|
|
except Exception as e:
|
|
logger.info(f"Error: {e}.")
|
|
|
|
if __name__ == "__main__":
|
|
main() |