reorg ini e log

utils/csv/data_preparation.py

@@ -0,0 +1,120 @@
+#!.venv/bin/python
+from utils.database.connection import connetti_db
+import utils.timestamp.date_check as date_check
+import logging
+import re
+
+from itertools import islice
+
+logger = logging.getLogger(__name__)
+
+def get_data(cfg: object, id: int) -> tuple:
+    """
+    Retrieves data for a specific tool from the database.
+
+    This function connects to the database using the provided configuration,
+    executes a query to retrieve the unit name, tool name ID, and tool data
+    associated with the given ID from the raw data table, and returns the results.
+
+    Args:
+        cfg: A configuration object containing database connection parameters
+             and table names (cfg.dbname, cfg.dbrectable).
+        id: The ID of the tool record to retrieve.
+
+    Returns:
+        A tuple containing the unit name, tool name ID, and tool data.
+    """
+    with connetti_db(cfg) as conn:
+        cur = conn.cursor()
+        cur.execute(f'select unit_name, tool_name, tool_data from {cfg.dbname}.{cfg.dbrectable} where id = {id}')
+        unit_name, tool_name, tool_data = cur.fetchone()
+        cur.close()
+        conn.close()
+        return unit_name, tool_name, tool_data
+
+def make_matrix(cfg: object, id: int) -> list:
+    """
+    Processes raw tool data and transforms it into a matrix format for database insertion.
+
+    This function retrieves raw tool data using `get_data`, splits it into individual
+    readings (rows), and further parses each reading into individual values.  It
+    handles data where multiple nodes might be reporting values within a single
+    reading.  The resulting matrix is a list of lists, where each inner list
+    represents a row of data ready for insertion into the database.  Missing
+    values are padded with `None` to ensure consistent row length.
+
+    Args:
+        cfg: A configuration object containing database connection parameters
+             and table names.
+        id: The ID of the tool record to process.
+
+    Returns:
+        A list of lists (matrix) representing the processed data.  Each inner list
+        contains the following elements: UnitName, ToolNameID, NodeNum, EventDate,
+        EventTime, BatLevel, Temperature, followed by up to 16 additional
+        measurement values (Val0 to ValF), padded with None if necessary.
+    """
+    UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    righe = ToolData.splitlines()
+    matrice_valori = []
+    for riga in [riga for riga in righe if ';|;' in riga]:
+        timestamp, batlevel, temperature, rilevazioni = riga.split(';',3)
+        EventDate, EventTime = timestamp.split(' ')
+        valori_nodi = rilevazioni.lstrip('|;').rstrip(';').split(';|;') # Toglie '|;' iniziali, toglie eventuali ';' finali, dividi per ';|;'
+        for num_nodo, valori_nodo in enumerate(valori_nodi, start=1):
+            valori = valori_nodo.split(';')
+            matrice_valori.append([UnitName, ToolNameID, num_nodo, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + valori + ([None] * (19 - len(valori))))
+
+    return matrice_valori
+
+def make_loc_matrix(cfg: object, id: int) -> list:
+    """
+    Processes raw location (LOC) tool data and transforms it into a matrix format for database insertion.
+
+    This function retrieves raw LOC tool data using `get_data`, splits it into
+    individual readings (rows), and parses each reading into individual values
+    specific to the LOC data format (timestamp, battery level, temperature, and
+    four additional values: ain1, ain2, din1, din2).  The resulting matrix is a list
+    of lists, where each inner list represents a row of data ready for insertion
+    into the database.  Missing values are padded with `None` to ensure consistent
+    row length.  It uses a regular expression to filter lines that match the
+    expected LOC data format.
+
+    Args:
+        cfg: A configuration object containing database connection parameters
+             and table names.
+        id: The ID of the tool record to process.
+
+    Returns:
+        A list of lists (matrix) representing the processed LOC data.  Each inner
+        list contains data fields similar to `make_matrix`, adjusted for LOC data.
+    """
+    UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    righe = ToolData.splitlines()
+    matrice_valori = []
+    pattern = r'(?:\d{4}/\d{2}/\d{2}|\d{2}/\d{2}/\d{4}) \d{2}:\d{2}:\d{2}(;[^;]+)+'
+    for riga in [riga for riga in righe if re.match(pattern, riga)]:
+        timestamp, battery_voltage, unit_temperature, analog_input1, analog_input2, digital_input1, digital_input2  = riga.split(';')
+        event_date, event_time = timestamp.split(' ')
+        valori = [analog_input1, analog_input2, digital_input1, digital_input2]
+        matrice_valori.append([UnitName, ToolNameID, 1, date_check.conforma_data(event_date), event_time, battery_voltage, unit_temperature] + valori + ([None] * (19 - len(valori))))
+
+    return matrice_valori
+
+def make_matrix_with_channels(cfg: object, id: int, node_channels: list) -> list:
+    UnitName, ToolNameID, ToolData = get_data(cfg, id)
+    righe = ToolData.splitlines()
+    matrice_valori = []
+    for riga in [riga for riga in righe if ';|;' in riga]:
+        timestamp, batlevel, temperature, rilevazioni = riga.split(';',3)
+        EventDate, EventTime = timestamp.split(' ')
+        valori_splitted = [valore for valore in rilevazioni.split(';') if valore != '|']
+        valori_iter = iter(valori_splitted)
+        valori_nodi = [list(islice(valori_iter, channels)) for channels in node_channels]
+
+        for num_nodo, valori in enumerate(valori_nodi, start=1):
+            matrice_valori.append([UnitName, ToolNameID, num_nodo, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + valori + ([None] * (19 - len(valori))))
+
+    return matrice_valori
+
+