From 55383e51b85bc7ce67d39942b17bf1028d93efae Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 19 Aug 2025 22:08:57 +0200 Subject: [PATCH] docs db __init__ --- src/utils/csv/loaders.py | 1 - src/utils/database/__init__.py | 20 +++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/utils/csv/loaders.py b/src/utils/csv/loaders.py index 0c00c06..5966999 100644 --- a/src/utils/csv/loaders.py +++ b/src/utils/csv/loaders.py @@ -98,7 +98,6 @@ async def get_next_csv_atomic(pool: object, table_name: str, status: int, next_s raise e async def main_old_script_loader(cfg: object, id: int, pool: object, script_name: str) -> None: - #async def main_loader(cfg: object, id: int, pool: object) -> None: """ This function retrieves CSV data, writes it to a temporary file, executes an external Python script to process it, diff --git a/src/utils/database/__init__.py b/src/utils/database/__init__.py index b696a1b..6c2c04c 100644 --- a/src/utils/database/__init__.py +++ b/src/utils/database/__init__.py @@ -1,22 +1,36 @@ class WorkflowFlags: + """ + Defines integer flags representing different stages in a data processing workflow. + Each flag is a power of 2, allowing them to be combined using bitwise operations + to represent multiple states simultaneously. + """ CSV_RECEIVED = 0 # 0000 DATA_LOADED = 1 # 0001 START_ELAB = 2 # 0010 DATA_ELABORATED = 4 # 0100 SENT_RAW_DATA = 8 # 1000 SENT_ELAB_DATA = 16 # 10000 - DUMMY_ELABORATED = 32 # 100000 + DUMMY_ELABORATED = 32 # 100000 (Used for testing or specific dummy elaborations) # Mappatura flag -> colonna timestamp FLAG_TO_TIMESTAMP = { + WorkflowFlags.CSV_RECEIVED: "inserted_at", WorkflowFlags.DATA_LOADED: "loaded_at", WorkflowFlags.START_ELAB: "start_elab_at", WorkflowFlags.DATA_ELABORATED: "elaborated_at", WorkflowFlags.SENT_RAW_DATA: "sent_raw_at", WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at", - WorkflowFlags.DUMMY_ELABORATED: "elaborated_at" + WorkflowFlags.DUMMY_ELABORATED: "elaborated_at" # Shares the same timestamp column as DATA_ELABORATED } +""" +A dictionary mapping each WorkflowFlag to the corresponding database column +name that stores the timestamp when that workflow stage was reached. +""" # Dimensione degli split della matrice per il caricamento -BATCH_SIZE = 1000 \ No newline at end of file +BATCH_SIZE = 1000 +""" +The number of records to process in a single batch when loading data into the database. +This helps manage memory usage and improve performance for large datasets. +""" \ No newline at end of file