docs db __init__
This commit is contained in:
@@ -98,7 +98,6 @@ async def get_next_csv_atomic(pool: object, table_name: str, status: int, next_s
|
||||
raise e
|
||||
|
||||
async def main_old_script_loader(cfg: object, id: int, pool: object, script_name: str) -> None:
|
||||
#async def main_loader(cfg: object, id: int, pool: object) -> None:
|
||||
"""
|
||||
This function retrieves CSV data, writes it to a temporary file,
|
||||
executes an external Python script to process it,
|
||||
|
||||
@@ -1,22 +1,36 @@
|
||||
class WorkflowFlags:
|
||||
"""
|
||||
Defines integer flags representing different stages in a data processing workflow.
|
||||
Each flag is a power of 2, allowing them to be combined using bitwise operations
|
||||
to represent multiple states simultaneously.
|
||||
"""
|
||||
CSV_RECEIVED = 0 # 0000
|
||||
DATA_LOADED = 1 # 0001
|
||||
START_ELAB = 2 # 0010
|
||||
DATA_ELABORATED = 4 # 0100
|
||||
SENT_RAW_DATA = 8 # 1000
|
||||
SENT_ELAB_DATA = 16 # 10000
|
||||
DUMMY_ELABORATED = 32 # 100000
|
||||
DUMMY_ELABORATED = 32 # 100000 (Used for testing or specific dummy elaborations)
|
||||
|
||||
# Mappatura flag -> colonna timestamp
|
||||
FLAG_TO_TIMESTAMP = {
|
||||
|
||||
WorkflowFlags.CSV_RECEIVED: "inserted_at",
|
||||
WorkflowFlags.DATA_LOADED: "loaded_at",
|
||||
WorkflowFlags.START_ELAB: "start_elab_at",
|
||||
WorkflowFlags.DATA_ELABORATED: "elaborated_at",
|
||||
WorkflowFlags.SENT_RAW_DATA: "sent_raw_at",
|
||||
WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at",
|
||||
WorkflowFlags.DUMMY_ELABORATED: "elaborated_at"
|
||||
WorkflowFlags.DUMMY_ELABORATED: "elaborated_at" # Shares the same timestamp column as DATA_ELABORATED
|
||||
}
|
||||
"""
|
||||
A dictionary mapping each WorkflowFlag to the corresponding database column
|
||||
name that stores the timestamp when that workflow stage was reached.
|
||||
"""
|
||||
|
||||
# Dimensione degli split della matrice per il caricamento
|
||||
BATCH_SIZE = 1000
|
||||
"""
|
||||
The number of records to process in a single batch when loading data into the database.
|
||||
This helps manage memory usage and improve performance for large datasets.
|
||||
"""
|
||||
Reference in New Issue
Block a user