docs db __init__
This commit is contained in:
@@ -98,7 +98,6 @@ async def get_next_csv_atomic(pool: object, table_name: str, status: int, next_s
|
|||||||
raise e
|
raise e
|
||||||
|
|
||||||
async def main_old_script_loader(cfg: object, id: int, pool: object, script_name: str) -> None:
|
async def main_old_script_loader(cfg: object, id: int, pool: object, script_name: str) -> None:
|
||||||
#async def main_loader(cfg: object, id: int, pool: object) -> None:
|
|
||||||
"""
|
"""
|
||||||
This function retrieves CSV data, writes it to a temporary file,
|
This function retrieves CSV data, writes it to a temporary file,
|
||||||
executes an external Python script to process it,
|
executes an external Python script to process it,
|
||||||
|
|||||||
@@ -1,22 +1,36 @@
|
|||||||
class WorkflowFlags:
|
class WorkflowFlags:
|
||||||
|
"""
|
||||||
|
Defines integer flags representing different stages in a data processing workflow.
|
||||||
|
Each flag is a power of 2, allowing them to be combined using bitwise operations
|
||||||
|
to represent multiple states simultaneously.
|
||||||
|
"""
|
||||||
CSV_RECEIVED = 0 # 0000
|
CSV_RECEIVED = 0 # 0000
|
||||||
DATA_LOADED = 1 # 0001
|
DATA_LOADED = 1 # 0001
|
||||||
START_ELAB = 2 # 0010
|
START_ELAB = 2 # 0010
|
||||||
DATA_ELABORATED = 4 # 0100
|
DATA_ELABORATED = 4 # 0100
|
||||||
SENT_RAW_DATA = 8 # 1000
|
SENT_RAW_DATA = 8 # 1000
|
||||||
SENT_ELAB_DATA = 16 # 10000
|
SENT_ELAB_DATA = 16 # 10000
|
||||||
DUMMY_ELABORATED = 32 # 100000
|
DUMMY_ELABORATED = 32 # 100000 (Used for testing or specific dummy elaborations)
|
||||||
|
|
||||||
# Mappatura flag -> colonna timestamp
|
# Mappatura flag -> colonna timestamp
|
||||||
FLAG_TO_TIMESTAMP = {
|
FLAG_TO_TIMESTAMP = {
|
||||||
|
|
||||||
WorkflowFlags.CSV_RECEIVED: "inserted_at",
|
WorkflowFlags.CSV_RECEIVED: "inserted_at",
|
||||||
WorkflowFlags.DATA_LOADED: "loaded_at",
|
WorkflowFlags.DATA_LOADED: "loaded_at",
|
||||||
WorkflowFlags.START_ELAB: "start_elab_at",
|
WorkflowFlags.START_ELAB: "start_elab_at",
|
||||||
WorkflowFlags.DATA_ELABORATED: "elaborated_at",
|
WorkflowFlags.DATA_ELABORATED: "elaborated_at",
|
||||||
WorkflowFlags.SENT_RAW_DATA: "sent_raw_at",
|
WorkflowFlags.SENT_RAW_DATA: "sent_raw_at",
|
||||||
WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at",
|
WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at",
|
||||||
WorkflowFlags.DUMMY_ELABORATED: "elaborated_at"
|
WorkflowFlags.DUMMY_ELABORATED: "elaborated_at" # Shares the same timestamp column as DATA_ELABORATED
|
||||||
}
|
}
|
||||||
|
"""
|
||||||
|
A dictionary mapping each WorkflowFlag to the corresponding database column
|
||||||
|
name that stores the timestamp when that workflow stage was reached.
|
||||||
|
"""
|
||||||
|
|
||||||
# Dimensione degli split della matrice per il caricamento
|
# Dimensione degli split della matrice per il caricamento
|
||||||
BATCH_SIZE = 1000
|
BATCH_SIZE = 1000
|
||||||
|
"""
|
||||||
|
The number of records to process in a single batch when loading data into the database.
|
||||||
|
This helps manage memory usage and improve performance for large datasets.
|
||||||
|
"""
|
||||||
Reference in New Issue
Block a user