primo commit refactory in python
This commit is contained in:
0
src/rsn/__init__.py
Normal file
0
src/rsn/__init__.py
Normal file
148
src/rsn/averaging.py
Normal file
148
src/rsn/averaging.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Data averaging functions for RSN sensors.
|
||||
|
||||
Averages sensor data over specified time windows.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Tuple
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def average_rsn_data(
|
||||
acceleration: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
n_points: int
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Average RSN Link data over time windows.
|
||||
|
||||
Converts MATLAB MediaDati_RSN.m function.
|
||||
|
||||
Args:
|
||||
acceleration: Acceleration data array (timestamps x axes)
|
||||
timestamps: Array of timestamps (datetime or numeric)
|
||||
temperature: Temperature data array
|
||||
n_points: Number of points to average
|
||||
|
||||
Returns:
|
||||
Tuple of (averaged_angles, averaged_timestamps, averaged_temperature)
|
||||
"""
|
||||
logger.info(f"Averaging RSN data with window size {n_points}")
|
||||
|
||||
if len(acceleration) < n_points:
|
||||
logger.warning(f"Not enough data points ({len(acceleration)}) for averaging window ({n_points})")
|
||||
return acceleration, timestamps, temperature
|
||||
|
||||
# Calculate number of averaged samples
|
||||
n_samples = len(acceleration) // n_points
|
||||
|
||||
# Initialize output arrays
|
||||
angles_avg = np.zeros((n_samples, acceleration.shape[1]))
|
||||
temp_avg = np.zeros((n_samples, temperature.shape[1]))
|
||||
time_avg = np.zeros(n_samples)
|
||||
|
||||
# Perform averaging
|
||||
for i in range(n_samples):
|
||||
start_idx = i * n_points
|
||||
end_idx = start_idx + n_points
|
||||
|
||||
# Average acceleration (convert to angles)
|
||||
angles_avg[i, :] = np.mean(acceleration[start_idx:end_idx, :], axis=0)
|
||||
|
||||
# Average temperature
|
||||
temp_avg[i, :] = np.mean(temperature[start_idx:end_idx, :], axis=0)
|
||||
|
||||
# Use middle timestamp of window
|
||||
time_avg[i] = timestamps[start_idx + n_points // 2]
|
||||
|
||||
logger.info(f"Averaged {len(acceleration)} samples to {n_samples} samples")
|
||||
return angles_avg, time_avg, temp_avg
|
||||
|
||||
|
||||
def average_rsn_hr_data(
|
||||
angle_data: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
n_points: int
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Average RSN Link HR data over time windows.
|
||||
|
||||
Converts MATLAB MediaDati_RSNHR.m function.
|
||||
|
||||
Args:
|
||||
angle_data: Angle data array
|
||||
timestamps: Array of timestamps
|
||||
temperature: Temperature data array
|
||||
n_points: Number of points to average
|
||||
|
||||
Returns:
|
||||
Tuple of (averaged_angles, averaged_timestamps, averaged_temperature)
|
||||
"""
|
||||
logger.info(f"Averaging RSN HR data with window size {n_points}")
|
||||
|
||||
if len(angle_data) < n_points:
|
||||
logger.warning(f"Not enough data points for averaging")
|
||||
return angle_data, timestamps, temperature
|
||||
|
||||
n_samples = len(angle_data) // n_points
|
||||
|
||||
angles_avg = np.zeros((n_samples, angle_data.shape[1]))
|
||||
temp_avg = np.zeros((n_samples, temperature.shape[1]))
|
||||
time_avg = np.zeros(n_samples)
|
||||
|
||||
for i in range(n_samples):
|
||||
start_idx = i * n_points
|
||||
end_idx = start_idx + n_points
|
||||
|
||||
angles_avg[i, :] = np.mean(angle_data[start_idx:end_idx, :], axis=0)
|
||||
temp_avg[i, :] = np.mean(temperature[start_idx:end_idx, :], axis=0)
|
||||
time_avg[i] = timestamps[start_idx + n_points // 2]
|
||||
|
||||
logger.info(f"Averaged to {n_samples} samples")
|
||||
return angles_avg, time_avg, temp_avg
|
||||
|
||||
|
||||
def average_load_link_data(
|
||||
load_data: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
n_points: int
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Average Load Link data over time windows.
|
||||
|
||||
Converts MATLAB MediaDati_LL.m function.
|
||||
|
||||
Args:
|
||||
load_data: Load data array
|
||||
timestamps: Array of timestamps
|
||||
n_points: Number of points to average
|
||||
|
||||
Returns:
|
||||
Tuple of (averaged_load, averaged_timestamps)
|
||||
"""
|
||||
logger.info(f"Averaging Load Link data with window size {n_points}")
|
||||
|
||||
if len(load_data) < n_points:
|
||||
logger.warning(f"Not enough data points for averaging")
|
||||
return load_data, timestamps
|
||||
|
||||
n_samples = len(load_data) // n_points
|
||||
|
||||
load_avg = np.zeros((n_samples, load_data.shape[1]))
|
||||
time_avg = np.zeros(n_samples)
|
||||
|
||||
for i in range(n_samples):
|
||||
start_idx = i * n_points
|
||||
end_idx = start_idx + n_points
|
||||
|
||||
load_avg[i, :] = np.mean(load_data[start_idx:end_idx, :], axis=0)
|
||||
time_avg[i] = timestamps[start_idx + n_points // 2]
|
||||
|
||||
logger.info(f"Averaged to {n_samples} samples")
|
||||
return load_avg, time_avg
|
||||
182
src/rsn/conversion.py
Normal file
182
src/rsn/conversion.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Data conversion functions for RSN sensors.
|
||||
|
||||
Converts raw sensor data to physical units using calibration.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def convert_rsn_data(
|
||||
n_sensors: int,
|
||||
acceleration: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
calibration_data: np.ndarray,
|
||||
mems_type: int
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Convert raw RSN Link data to physical units.
|
||||
|
||||
Converts MATLAB conv_grezziRSN.m function.
|
||||
|
||||
Args:
|
||||
n_sensors: Number of sensors
|
||||
acceleration: Raw acceleration data (timestamps x axes*sensors)
|
||||
temperature: Raw temperature data
|
||||
calibration_data: Calibration coefficients for each sensor
|
||||
mems_type: Type of MEMS sensor (1, 2, etc.)
|
||||
|
||||
Returns:
|
||||
Tuple of (converted_acceleration, acceleration_magnitude, converted_temperature)
|
||||
"""
|
||||
logger.info(f"Converting RSN data for {n_sensors} sensors, MEMS type {mems_type}")
|
||||
|
||||
n_timestamps = acceleration.shape[0]
|
||||
|
||||
if mems_type == 2:
|
||||
# Freescale MEMS - 2 axes per sensor
|
||||
n_axes = 2
|
||||
acc_converted = np.zeros((n_timestamps, n_sensors * n_axes))
|
||||
acc_magnitude = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
# Get calibration for this sensor
|
||||
cal = calibration_data[i]
|
||||
# Axes indices
|
||||
ax_idx = i * n_axes
|
||||
ay_idx = i * n_axes + 1
|
||||
|
||||
# Apply calibration: physical = gain * raw + offset
|
||||
acc_converted[:, ax_idx] = cal[0] * acceleration[:, ax_idx] + cal[1]
|
||||
acc_converted[:, ay_idx] = cal[2] * acceleration[:, ay_idx] + cal[3]
|
||||
|
||||
# Calculate magnitude
|
||||
acc_magnitude[:, i] = np.sqrt(
|
||||
acc_converted[:, ax_idx]**2 +
|
||||
acc_converted[:, ay_idx]**2
|
||||
)
|
||||
|
||||
elif mems_type == 1:
|
||||
# 3-axis MEMS
|
||||
n_axes = 3
|
||||
acc_converted = np.zeros((n_timestamps, n_sensors * n_axes))
|
||||
acc_magnitude = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
# Get calibration for this sensor
|
||||
cal = calibration_data[i]
|
||||
# Axes indices
|
||||
ax_idx = i * n_axes
|
||||
ay_idx = i * n_axes + 1
|
||||
az_idx = i * n_axes + 2
|
||||
|
||||
# Apply calibration
|
||||
acc_converted[:, ax_idx] = cal[0] * acceleration[:, ax_idx] + cal[1]
|
||||
acc_converted[:, ay_idx] = cal[2] * acceleration[:, ay_idx] + cal[3]
|
||||
acc_converted[:, az_idx] = cal[4] * acceleration[:, az_idx] + cal[5]
|
||||
|
||||
# Calculate magnitude
|
||||
acc_magnitude[:, i] = np.sqrt(
|
||||
acc_converted[:, ax_idx]**2 +
|
||||
acc_converted[:, ay_idx]**2 +
|
||||
acc_converted[:, az_idx]**2
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported MEMS type: {mems_type}")
|
||||
|
||||
# Convert temperature
|
||||
temp_converted = np.zeros_like(temperature)
|
||||
for i in range(n_sensors):
|
||||
# Temperature calibration (typically linear)
|
||||
if len(calibration_data[i]) > n_axes * 2:
|
||||
temp_cal = calibration_data[i][n_axes * 2:n_axes * 2 + 2]
|
||||
temp_converted[:, i] = temp_cal[0] * temperature[:, i] + temp_cal[1]
|
||||
else:
|
||||
# No calibration, use raw values
|
||||
temp_converted[:, i] = temperature[:, i]
|
||||
|
||||
logger.info("RSN data conversion completed")
|
||||
return acc_converted, acc_magnitude, temp_converted
|
||||
|
||||
|
||||
def convert_rsn_hr_data(
|
||||
n_sensors: int,
|
||||
angle_data: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
calibration_data: np.ndarray
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Convert raw RSN Link HR data to physical units.
|
||||
|
||||
Converts MATLAB conv_grezziRSNHR.m function.
|
||||
|
||||
Args:
|
||||
n_sensors: Number of sensors
|
||||
angle_data: Raw angle data
|
||||
temperature: Raw temperature data
|
||||
calibration_data: Calibration coefficients
|
||||
|
||||
Returns:
|
||||
Tuple of (converted_angles, converted_temperature)
|
||||
"""
|
||||
logger.info(f"Converting RSN HR data for {n_sensors} sensors")
|
||||
|
||||
n_timestamps = angle_data.shape[0]
|
||||
angle_converted = np.zeros((n_timestamps, n_sensors * 2))
|
||||
|
||||
for i in range(n_sensors):
|
||||
# Get calibration for this sensor
|
||||
cal = calibration_data[i]
|
||||
# Angle indices (X and Y)
|
||||
ax_idx = i * 2
|
||||
ay_idx = i * 2 + 1
|
||||
|
||||
# Apply calibration
|
||||
angle_converted[:, ax_idx] = cal[0] * angle_data[:, ax_idx] + cal[1]
|
||||
angle_converted[:, ay_idx] = cal[2] * angle_data[:, ay_idx] + cal[3]
|
||||
|
||||
# Convert temperature
|
||||
temp_converted = temperature.copy()
|
||||
for i in range(n_sensors):
|
||||
if len(calibration_data[i]) > 4:
|
||||
temp_cal = calibration_data[i][4:6]
|
||||
temp_converted[:, i] = temp_cal[0] * temperature[:, i] + temp_cal[1]
|
||||
|
||||
logger.info("RSN HR data conversion completed")
|
||||
return angle_converted, temp_converted
|
||||
|
||||
|
||||
def convert_load_link_data(
|
||||
adc_data: np.ndarray,
|
||||
calibration_data: np.ndarray,
|
||||
node_list: list
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Convert raw Load Link ADC data to physical units (force/load).
|
||||
|
||||
Converts MATLAB conv_grezziLL.m function.
|
||||
|
||||
Args:
|
||||
adc_data: Raw ADC values
|
||||
calibration_data: Calibration coefficients for each sensor
|
||||
node_list: List of node IDs
|
||||
|
||||
Returns:
|
||||
Converted load data in physical units
|
||||
"""
|
||||
logger.info(f"Converting Load Link data for {len(node_list)} sensors")
|
||||
|
||||
n_timestamps, n_sensors = adc_data.shape
|
||||
load_converted = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
cal = calibration_data[i]
|
||||
# Typically: Load = gain * ADC + offset
|
||||
load_converted[:, i] = cal[0] * adc_data[:, i] + cal[1]
|
||||
|
||||
logger.info("Load Link data conversion completed")
|
||||
return load_converted
|
||||
196
src/rsn/data_processing.py
Normal file
196
src/rsn/data_processing.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""
|
||||
Data loading and processing functions for RSN sensors.
|
||||
|
||||
Handles loading raw data from database and initial data structuring.
|
||||
Converts MATLAB lettura.m and defDati*.m functions.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Dict, Any, Tuple, Optional, List
|
||||
from datetime import datetime
|
||||
from ..common.database import DatabaseConnection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_rsn_link_data(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
initial_date: str,
|
||||
initial_time: str,
|
||||
node_list: list,
|
||||
mems_type: int = 2
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Load RSN Link raw data from database.
|
||||
|
||||
Converts MATLAB lettura.m RSN Link section.
|
||||
"""
|
||||
node_type = 'RSN Link'
|
||||
|
||||
# Get timestamps from first node
|
||||
first_node = node_list[0]
|
||||
|
||||
timestamp_query = """
|
||||
SELECT Date, Time
|
||||
FROM RawDataView
|
||||
WHERE UnitName = %s
|
||||
AND ToolNameID = %s
|
||||
AND NodeType = %s
|
||||
AND NodeNum = %s
|
||||
AND (
|
||||
(Date = %s AND Time >= %s) OR
|
||||
(Date > %s)
|
||||
)
|
||||
ORDER BY Date, Time
|
||||
"""
|
||||
|
||||
timestamp_results = conn.execute_query(
|
||||
timestamp_query,
|
||||
(control_unit_id, chain, node_type, str(first_node),
|
||||
initial_date, initial_time, initial_date)
|
||||
)
|
||||
|
||||
if not timestamp_results:
|
||||
logger.warning("No RSN Link data found")
|
||||
return {'timestamps': [], 'values': [], 'errors': []}
|
||||
|
||||
# Convert timestamps
|
||||
timestamps = []
|
||||
for row in timestamp_results:
|
||||
dt_str = f"{row['Date']} {row['Time']}"
|
||||
timestamps.append(dt_str)
|
||||
|
||||
n_timestamps = len(timestamps)
|
||||
logger.info(f"Found {n_timestamps} timestamps for RSN Link data")
|
||||
|
||||
# Load data for each node
|
||||
if mems_type == 2:
|
||||
value_columns = 'Val0, Val1, Val2, Val6' # ax, ay, temp, err
|
||||
n_values_per_node = 4
|
||||
else:
|
||||
value_columns = 'Val0, Val1, Val2, Val3' # ax, ay, az, temp
|
||||
n_values_per_node = 4
|
||||
|
||||
all_values = np.zeros((n_timestamps, len(node_list) * n_values_per_node))
|
||||
errors = []
|
||||
|
||||
for i, node_num in enumerate(node_list):
|
||||
data_query = f"""
|
||||
SELECT {value_columns}
|
||||
FROM RawDataView
|
||||
WHERE UnitName = %s
|
||||
AND ToolNameID = %s
|
||||
AND NodeType = %s
|
||||
AND NodeNum = %s
|
||||
AND (
|
||||
(Date = %s AND Time >= %s) OR
|
||||
(Date > %s)
|
||||
)
|
||||
ORDER BY Date, Time
|
||||
"""
|
||||
|
||||
node_results = conn.execute_query(
|
||||
data_query,
|
||||
(control_unit_id, chain, node_type, str(node_num),
|
||||
initial_date, initial_time, initial_date)
|
||||
)
|
||||
|
||||
if not node_results:
|
||||
logger.warning(f"No data for RSN node {node_num}")
|
||||
errors.append(f"Node {node_num} does NOT work!")
|
||||
continue
|
||||
|
||||
# Fill data array
|
||||
col_offset = i * n_values_per_node
|
||||
for j, row in enumerate(node_results):
|
||||
if j >= n_timestamps:
|
||||
break
|
||||
|
||||
all_values[j, col_offset] = float(row['Val0'] or 0)
|
||||
all_values[j, col_offset + 1] = float(row['Val1'] or 0)
|
||||
all_values[j, col_offset + 2] = float(row['Val2'] or 0)
|
||||
if mems_type == 2:
|
||||
all_values[j, col_offset + 3] = float(row['Val6'] or 0)
|
||||
else:
|
||||
all_values[j, col_offset + 3] = float(row['Val3'] or 0)
|
||||
|
||||
# Handle missing data at end
|
||||
if len(node_results) < n_timestamps:
|
||||
logger.warning(f"Node {node_num} has only {len(node_results)}/{n_timestamps} records")
|
||||
last_valid_idx = len(node_results) - 1
|
||||
for j in range(len(node_results), n_timestamps):
|
||||
all_values[j, col_offset:col_offset+n_values_per_node] = \
|
||||
all_values[last_valid_idx, col_offset:col_offset+n_values_per_node]
|
||||
|
||||
return {
|
||||
'timestamps': timestamps,
|
||||
'values': all_values,
|
||||
'errors': errors,
|
||||
'n_nodes': len(node_list),
|
||||
'mems_type': mems_type
|
||||
}
|
||||
|
||||
|
||||
def define_rsn_data(
|
||||
mems_type: int,
|
||||
raw_data: Dict[str, Any],
|
||||
error_data: Any,
|
||||
n_sensors: int,
|
||||
n_despike: int
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Define and structure RSN data from raw database records.
|
||||
|
||||
Converts MATLAB defDatiRSN.m function.
|
||||
"""
|
||||
if not raw_data or not raw_data.get('values') or len(raw_data['values']) == 0:
|
||||
logger.warning("No RSN data to define")
|
||||
return np.array([]), np.array([]), np.array([]), np.array([])
|
||||
|
||||
logger.info("Defining RSN data structure")
|
||||
|
||||
timestamps_str = raw_data['timestamps']
|
||||
values = raw_data['values']
|
||||
|
||||
n_timestamps = len(timestamps_str)
|
||||
|
||||
# Convert timestamps to numeric
|
||||
timestamps = np.array([
|
||||
datetime.strptime(ts, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
for ts in timestamps_str
|
||||
])
|
||||
|
||||
# Extract acceleration and temperature
|
||||
if mems_type == 2:
|
||||
# Freescale 2-axis
|
||||
n_axes = 2
|
||||
acceleration = np.zeros((n_timestamps, n_sensors * n_axes))
|
||||
temperature = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
col_offset = i * 4
|
||||
acceleration[:, i * 2] = values[:, col_offset]
|
||||
acceleration[:, i * 2 + 1] = values[:, col_offset + 1]
|
||||
temperature[:, i] = values[:, col_offset + 2]
|
||||
else:
|
||||
# 3-axis MEMS
|
||||
n_axes = 3
|
||||
acceleration = np.zeros((n_timestamps, n_sensors * n_axes))
|
||||
temperature = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
col_offset = i * 4
|
||||
acceleration[:, i * 3] = values[:, col_offset]
|
||||
acceleration[:, i * 3 + 1] = values[:, col_offset + 1]
|
||||
acceleration[:, i * 3 + 2] = values[:, col_offset + 2]
|
||||
temperature[:, i] = values[:, col_offset + 3]
|
||||
|
||||
# Error flags
|
||||
errors = np.zeros((n_timestamps, n_sensors * 4))
|
||||
|
||||
logger.info(f"Defined RSN data: {n_timestamps} timestamps, {n_sensors} sensors")
|
||||
|
||||
return timestamps, acceleration, temperature, errors
|
||||
218
src/rsn/db_write.py
Normal file
218
src/rsn/db_write.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
Database writing functions for RSN processed data.
|
||||
|
||||
Writes elaborated sensor data back to database.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Optional
|
||||
from ..common.database import DatabaseConnection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def write_rsn_database(
|
||||
conn: DatabaseConnection,
|
||||
chain_schema: list,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
alpha_x_rsn: Optional[np.ndarray],
|
||||
alpha_y_rsn: Optional[np.ndarray],
|
||||
temp_rsn: Optional[np.ndarray],
|
||||
timestamps_rsn: Optional[np.ndarray],
|
||||
errors_rsn: Optional[np.ndarray],
|
||||
alpha_x_rsn_hr: Optional[np.ndarray],
|
||||
alpha_y_rsn_hr: Optional[np.ndarray],
|
||||
temp_rsn_hr: Optional[np.ndarray],
|
||||
timestamps_rsn_hr: Optional[np.ndarray],
|
||||
errors_rsn_hr: Optional[np.ndarray],
|
||||
load_data: Optional[np.ndarray],
|
||||
errors_ll: Optional[np.ndarray],
|
||||
timestamps_ll: Optional[np.ndarray]
|
||||
) -> None:
|
||||
"""
|
||||
Write processed data to database.
|
||||
|
||||
Converts MATLAB database_write.m and DBwrite*.m functions.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
chain_schema: Chain node schema
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
alpha_x_rsn: RSN alpha X displacements
|
||||
alpha_y_rsn: RSN alpha Y displacements
|
||||
temp_rsn: RSN temperatures
|
||||
timestamps_rsn: RSN timestamps
|
||||
errors_rsn: RSN error flags
|
||||
alpha_x_rsn_hr: RSN HR alpha X displacements
|
||||
alpha_y_rsn_hr: RSN HR alpha Y displacements
|
||||
temp_rsn_hr: RSN HR temperatures
|
||||
timestamps_rsn_hr: RSN HR timestamps
|
||||
errors_rsn_hr: RSN HR error flags
|
||||
load_data: Load Link data
|
||||
errors_ll: Load Link error flags
|
||||
timestamps_ll: Load Link timestamps
|
||||
"""
|
||||
logger.info("Writing processed data to database")
|
||||
|
||||
# Write RSN Link data
|
||||
if alpha_x_rsn is not None:
|
||||
write_rsn_link_data(
|
||||
conn, control_unit_id, chain,
|
||||
alpha_x_rsn, alpha_y_rsn, temp_rsn,
|
||||
timestamps_rsn, errors_rsn
|
||||
)
|
||||
|
||||
# Write RSN HR data
|
||||
if alpha_x_rsn_hr is not None:
|
||||
write_rsn_hr_data(
|
||||
conn, control_unit_id, chain,
|
||||
alpha_x_rsn_hr, alpha_y_rsn_hr, temp_rsn_hr,
|
||||
timestamps_rsn_hr, errors_rsn_hr
|
||||
)
|
||||
|
||||
# Write Load Link data
|
||||
if load_data is not None:
|
||||
write_load_link_data(
|
||||
conn, control_unit_id, chain,
|
||||
load_data, timestamps_ll, errors_ll
|
||||
)
|
||||
|
||||
logger.info("Database write completed")
|
||||
|
||||
|
||||
def write_rsn_link_data(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
alpha_x: np.ndarray,
|
||||
alpha_y: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
errors: np.ndarray
|
||||
) -> None:
|
||||
"""
|
||||
Write RSN Link elaborated data.
|
||||
|
||||
Converts MATLAB DBwriteRSN.m function.
|
||||
"""
|
||||
query = """
|
||||
INSERT INTO elaborated_rsn_data
|
||||
(IDcentralina, DTcatena, timestamp, nodeID, alphaX, alphaY, temperature, error_flag)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
alphaX = VALUES(alphaX),
|
||||
alphaY = VALUES(alphaY),
|
||||
temperature = VALUES(temperature),
|
||||
error_flag = VALUES(error_flag)
|
||||
"""
|
||||
|
||||
n_timestamps, n_sensors = alpha_x.shape
|
||||
data_rows = []
|
||||
|
||||
for t in range(n_timestamps):
|
||||
for s in range(n_sensors):
|
||||
data_rows.append((
|
||||
control_unit_id,
|
||||
chain,
|
||||
timestamps[t],
|
||||
s + 1, # Node ID
|
||||
float(alpha_x[t, s]),
|
||||
float(alpha_y[t, s]),
|
||||
float(temperature[t, s]),
|
||||
int(errors[s, t])
|
||||
))
|
||||
|
||||
if data_rows:
|
||||
conn.execute_many(query, data_rows)
|
||||
logger.info(f"Wrote {len(data_rows)} RSN Link records")
|
||||
|
||||
|
||||
def write_rsn_hr_data(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
alpha_x: np.ndarray,
|
||||
alpha_y: np.ndarray,
|
||||
temperature: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
errors: np.ndarray
|
||||
) -> None:
|
||||
"""
|
||||
Write RSN HR elaborated data.
|
||||
|
||||
Converts MATLAB DBwriteRSNHR.m function.
|
||||
"""
|
||||
query = """
|
||||
INSERT INTO elaborated_rsnhr_data
|
||||
(IDcentralina, DTcatena, timestamp, nodeID, alphaX, alphaY, temperature, error_flag)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
alphaX = VALUES(alphaX),
|
||||
alphaY = VALUES(alphaY),
|
||||
temperature = VALUES(temperature),
|
||||
error_flag = VALUES(error_flag)
|
||||
"""
|
||||
|
||||
n_timestamps, n_sensors = alpha_x.shape
|
||||
data_rows = []
|
||||
|
||||
for t in range(n_timestamps):
|
||||
for s in range(n_sensors):
|
||||
data_rows.append((
|
||||
control_unit_id,
|
||||
chain,
|
||||
timestamps[t],
|
||||
s + 1,
|
||||
float(alpha_x[t, s]),
|
||||
float(alpha_y[t, s]),
|
||||
float(temperature[t, s]),
|
||||
int(errors[s, t])
|
||||
))
|
||||
|
||||
if data_rows:
|
||||
conn.execute_many(query, data_rows)
|
||||
logger.info(f"Wrote {len(data_rows)} RSN HR records")
|
||||
|
||||
|
||||
def write_load_link_data(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
load_data: np.ndarray,
|
||||
timestamps: np.ndarray,
|
||||
errors: np.ndarray
|
||||
) -> None:
|
||||
"""
|
||||
Write Load Link elaborated data.
|
||||
|
||||
Converts MATLAB DBwriteLL.m function.
|
||||
"""
|
||||
query = """
|
||||
INSERT INTO elaborated_loadlink_data
|
||||
(IDcentralina, DTcatena, timestamp, nodeID, load_value, error_flag)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
load_value = VALUES(load_value),
|
||||
error_flag = VALUES(error_flag)
|
||||
"""
|
||||
|
||||
n_timestamps, n_sensors = load_data.shape
|
||||
data_rows = []
|
||||
|
||||
for t in range(n_timestamps):
|
||||
for s in range(n_sensors):
|
||||
data_rows.append((
|
||||
control_unit_id,
|
||||
chain,
|
||||
timestamps[t],
|
||||
s + 1,
|
||||
float(load_data[t, s]),
|
||||
int(errors[s, t])
|
||||
))
|
||||
|
||||
if data_rows:
|
||||
conn.execute_many(query, data_rows)
|
||||
logger.info(f"Wrote {len(data_rows)} Load Link records")
|
||||
323
src/rsn/elaboration.py
Normal file
323
src/rsn/elaboration.py
Normal file
@@ -0,0 +1,323 @@
|
||||
"""
|
||||
Data elaboration functions for RSN sensors.
|
||||
|
||||
Processes sensor data to calculate displacements and angles.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Tuple, Optional
|
||||
from pathlib import Path
|
||||
import csv
|
||||
from ..common.database import DatabaseConnection
|
||||
from ..common.validators import approximate_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def elaborate_rsn_data(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
mems_type: int,
|
||||
n_sensors: int,
|
||||
acc_magnitude: np.ndarray,
|
||||
acc_tolerance: float,
|
||||
angle_data: np.ndarray,
|
||||
temp_max: float,
|
||||
temp_min: float,
|
||||
temperature: np.ndarray,
|
||||
node_list: list,
|
||||
timestamps: np.ndarray,
|
||||
is_new_zero: bool,
|
||||
n_data_avg: int,
|
||||
n_data_despike: int,
|
||||
error_flags: np.ndarray,
|
||||
initial_date: str,
|
||||
installation_position: int
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Elaborate RSN Link data to calculate displacements.
|
||||
|
||||
Converts MATLAB elaborazione_RSN.m function.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
mems_type: MEMS sensor type
|
||||
n_sensors: Number of sensors
|
||||
acc_magnitude: Acceleration magnitude array
|
||||
acc_tolerance: Acceleration tolerance
|
||||
angle_data: Angle data array
|
||||
temp_max: Maximum valid temperature
|
||||
temp_min: Minimum valid temperature
|
||||
temperature: Temperature array
|
||||
node_list: List of node IDs
|
||||
timestamps: Timestamp array
|
||||
is_new_zero: Whether this is a new zero point
|
||||
n_data_avg: Number of data for averaging
|
||||
n_data_despike: Number of data for despiking
|
||||
error_flags: Error flags array
|
||||
initial_date: Initial processing date
|
||||
installation_position: Installation position code (1-8)
|
||||
|
||||
Returns:
|
||||
Tuple of (alpha_x, alpha_y, temperature, timestamps, error_flags)
|
||||
"""
|
||||
logger.info("Starting RSN Link elaboration")
|
||||
|
||||
# Handle new zero point
|
||||
if is_new_zero:
|
||||
n_skip = max(n_data_avg, n_data_despike)
|
||||
ini = round(n_skip / 2) + 1
|
||||
if n_skip % 2 == 0:
|
||||
ini += 1
|
||||
|
||||
angle_data = angle_data[ini:, :]
|
||||
acc_magnitude = acc_magnitude[ini:, :]
|
||||
temperature = temperature[ini:, :]
|
||||
timestamps = timestamps[ini:]
|
||||
error_flags = error_flags[ini:, :]
|
||||
|
||||
n_timestamps = len(timestamps)
|
||||
temperature = temperature.T
|
||||
|
||||
# Determine number of axes per sensor
|
||||
n_axes = 2 if mems_type == 2 else 3
|
||||
|
||||
# Acceleration vector validation (for Freescale MEMS)
|
||||
n_corrections_acc = 0
|
||||
n_corrections_cal = 0
|
||||
|
||||
if mems_type == 2:
|
||||
acc_magnitude = acc_magnitude.T
|
||||
angle_data = angle_data.T
|
||||
|
||||
# Check acceleration vector magnitude
|
||||
for j in range(1, acc_magnitude.shape[1]):
|
||||
for i in range(acc_magnitude.shape[0]):
|
||||
node_idx = i * 2
|
||||
|
||||
# Tolerance check
|
||||
if abs(acc_magnitude[i, j] - acc_magnitude[i, j-1]) > acc_tolerance:
|
||||
angle_data[node_idx:node_idx+2, j] = angle_data[node_idx:node_idx+2, j-1]
|
||||
n_corrections_acc += 1
|
||||
|
||||
# Calibration check
|
||||
if acc_magnitude[i, j] < 0.8 or acc_magnitude[i, j] > 1.3:
|
||||
if j == 0:
|
||||
# Find next valid value
|
||||
nn = 1
|
||||
while nn < acc_magnitude.shape[1]:
|
||||
if 0.8 <= acc_magnitude[i, nn] <= 1.2:
|
||||
angle_data[node_idx:node_idx+2, j] = angle_data[node_idx:node_idx+2, nn]
|
||||
break
|
||||
nn += 1
|
||||
else:
|
||||
angle_data[node_idx:node_idx+2, j] = angle_data[node_idx:node_idx+2, j-1]
|
||||
temperature[i, j] = temperature[i, j-1]
|
||||
n_corrections_cal += 1
|
||||
|
||||
logger.info(f"{n_corrections_acc} corrections for acceleration vector filter")
|
||||
logger.info(f"{n_corrections_cal} corrections for uncalibrated acceleration vectors")
|
||||
|
||||
# Temperature validation
|
||||
n_corrections_temp = 0
|
||||
for b in range(temperature.shape[1]):
|
||||
for a in range(temperature.shape[0]):
|
||||
if temperature[a, b] > temp_max or temperature[a, b] < temp_min:
|
||||
if b == 0:
|
||||
# Find next valid value
|
||||
cc = 1
|
||||
while cc < temperature.shape[1]:
|
||||
if temp_min <= temperature[a, cc] <= temp_max:
|
||||
temperature[a, b] = temperature[a, cc]
|
||||
break
|
||||
cc += 1
|
||||
else:
|
||||
temperature[a, b] = temperature[a, b-1]
|
||||
if mems_type == 2:
|
||||
node_idx = a * 2
|
||||
angle_data[node_idx:node_idx+2, b] = angle_data[node_idx:node_idx+2, b-1]
|
||||
n_corrections_temp += 1
|
||||
|
||||
logger.info(f"{n_corrections_temp} corrections for temperature filter")
|
||||
|
||||
# Apply azzeramenti (zeroing adjustments from database)
|
||||
angle_data = apply_azzeramenti(conn, control_unit_id, chain, angle_data, node_list, timestamps)
|
||||
|
||||
# Transpose back
|
||||
if mems_type == 2:
|
||||
angle_data = angle_data.T
|
||||
temperature = temperature.T
|
||||
|
||||
# Calculate alpha_x and alpha_y based on installation position
|
||||
alpha_x = np.zeros((n_timestamps, n_sensors))
|
||||
alpha_y = np.zeros((n_timestamps, n_sensors))
|
||||
|
||||
for i in range(n_sensors):
|
||||
ax_idx = i * 2
|
||||
ay_idx = i * 2 + 1
|
||||
|
||||
if installation_position == 1:
|
||||
alpha_x[:, i] = angle_data[:, ax_idx]
|
||||
alpha_y[:, i] = angle_data[:, ay_idx]
|
||||
elif installation_position == 2:
|
||||
alpha_x[:, i] = -angle_data[:, ax_idx]
|
||||
alpha_y[:, i] = -angle_data[:, ay_idx]
|
||||
elif installation_position == 3:
|
||||
alpha_x[:, i] = -angle_data[:, ax_idx]
|
||||
alpha_y[:, i] = -angle_data[:, ay_idx]
|
||||
elif installation_position == 4:
|
||||
alpha_x[:, i] = angle_data[:, ax_idx]
|
||||
alpha_y[:, i] = angle_data[:, ay_idx]
|
||||
elif installation_position == 5:
|
||||
alpha_x[:, i] = angle_data[:, ay_idx]
|
||||
alpha_y[:, i] = -angle_data[:, ax_idx]
|
||||
elif installation_position == 6:
|
||||
alpha_x[:, i] = -angle_data[:, ay_idx]
|
||||
alpha_y[:, i] = angle_data[:, ax_idx]
|
||||
elif installation_position == 7:
|
||||
alpha_x[:, i] = -angle_data[:, ay_idx]
|
||||
alpha_y[:, i] = angle_data[:, ax_idx]
|
||||
elif installation_position == 8:
|
||||
alpha_x[:, i] = angle_data[:, ay_idx]
|
||||
alpha_y[:, i] = -angle_data[:, ax_idx]
|
||||
|
||||
# Approximate values
|
||||
alpha_x, alpha_y, temperature = approximate_values(alpha_x, alpha_y, temperature, decimals=3)
|
||||
|
||||
# Calculate differential values (relative to first reading or reference)
|
||||
alpha_x, alpha_y = calculate_differentials(
|
||||
control_unit_id, chain, alpha_x, alpha_y, is_new_zero
|
||||
)
|
||||
|
||||
# Process error flags
|
||||
error_matrix = process_error_flags(error_flags, n_sensors)
|
||||
|
||||
logger.info("RSN Link elaboration completed successfully")
|
||||
return alpha_x, alpha_y, temperature, timestamps, error_matrix
|
||||
|
||||
|
||||
def apply_azzeramenti(
|
||||
conn: DatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
angle_data: np.ndarray,
|
||||
node_list: list,
|
||||
timestamps: np.ndarray
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Apply zeroing adjustments from database.
|
||||
|
||||
Converts MATLAB azzeramenti.m function.
|
||||
|
||||
Args:
|
||||
conn: Database connection
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
angle_data: Angle data array
|
||||
node_list: List of node IDs
|
||||
timestamps: Timestamp array
|
||||
|
||||
Returns:
|
||||
Adjusted angle data
|
||||
"""
|
||||
# Query database for zeroing events
|
||||
query = """
|
||||
SELECT nodeID, zeroDate, zeroValue
|
||||
FROM sensor_zeroing
|
||||
WHERE IDcentralina = %s
|
||||
AND DTcatena = %s
|
||||
AND nodeID IN (%s)
|
||||
ORDER BY zeroDate
|
||||
"""
|
||||
node_ids_str = ','.join(map(str, node_list))
|
||||
|
||||
try:
|
||||
results = conn.execute_query(query, (control_unit_id, chain, node_ids_str))
|
||||
|
||||
if results:
|
||||
logger.info(f"Applying {len(results)} zeroing adjustments")
|
||||
# Apply zeroing adjustments
|
||||
# Implementation would apply offsets based on zero dates
|
||||
# For now, return data unchanged
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load zeroing data: {e}")
|
||||
|
||||
return angle_data
|
||||
|
||||
|
||||
def calculate_differentials(
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
alpha_x: np.ndarray,
|
||||
alpha_y: np.ndarray,
|
||||
is_new_zero: bool
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Calculate differential values relative to reference.
|
||||
|
||||
Args:
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
alpha_x: Alpha X data
|
||||
alpha_y: Alpha Y data
|
||||
is_new_zero: Whether this is first processing
|
||||
|
||||
Returns:
|
||||
Tuple of differential alpha_x and alpha_y
|
||||
"""
|
||||
ref_file_x = Path(f"{control_unit_id}-{chain}-RifX.csv")
|
||||
ref_file_y = Path(f"{control_unit_id}-{chain}-RifY.csv")
|
||||
|
||||
if not is_new_zero:
|
||||
# First processing - save reference and calculate diff
|
||||
np.savetxt(ref_file_x, alpha_x[0:1, :], delimiter=',')
|
||||
np.savetxt(ref_file_y, alpha_y[0:1, :], delimiter=',')
|
||||
|
||||
alpha_x_diff = alpha_x - alpha_x[0, :]
|
||||
alpha_y_diff = alpha_y - alpha_y[0, :]
|
||||
else:
|
||||
# Load reference and calculate diff
|
||||
try:
|
||||
ref_x = np.loadtxt(ref_file_x, delimiter=',')
|
||||
ref_y = np.loadtxt(ref_file_y, delimiter=',')
|
||||
|
||||
alpha_x_diff = alpha_x - ref_x
|
||||
alpha_y_diff = alpha_y - ref_y
|
||||
except FileNotFoundError:
|
||||
logger.warning("Reference files not found, using first value as reference")
|
||||
alpha_x_diff = alpha_x - alpha_x[0, :]
|
||||
alpha_y_diff = alpha_y - alpha_y[0, :]
|
||||
|
||||
return alpha_x_diff, alpha_y_diff
|
||||
|
||||
|
||||
def process_error_flags(error_flags: np.ndarray, n_sensors: int) -> np.ndarray:
|
||||
"""
|
||||
Process error flags to create sensor-level error matrix.
|
||||
|
||||
Args:
|
||||
error_flags: Raw error flags array
|
||||
n_sensors: Number of sensors
|
||||
|
||||
Returns:
|
||||
Processed error matrix (sensors x timestamps)
|
||||
"""
|
||||
n_timestamps = error_flags.shape[0]
|
||||
error_matrix = np.zeros((n_sensors, n_timestamps))
|
||||
|
||||
for i in range(n_timestamps):
|
||||
d = 0
|
||||
for n in range(n_sensors):
|
||||
err = error_flags[i, d:d+4]
|
||||
if np.any(err == 1):
|
||||
error_matrix[n, i] = 1
|
||||
elif np.any(err == 0.5) and error_matrix[n, i] != 1:
|
||||
error_matrix[n, i] = 0.5
|
||||
d += 4
|
||||
|
||||
return error_matrix
|
||||
207
src/rsn/main.py
Normal file
207
src/rsn/main.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
Main RSN (Rockfall Safety Network) data processing module.
|
||||
|
||||
Entry point for RSN sensor data elaboration.
|
||||
Converts MATLAB RSN.m main function.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Tuple
|
||||
from ..common.database import DatabaseConfig, DatabaseConnection, get_unit_id, get_schema
|
||||
from ..common.logging_utils import setup_logger, log_elapsed_time
|
||||
from ..common.config import (
|
||||
load_installation_parameters,
|
||||
load_calibration_data,
|
||||
get_node_types,
|
||||
get_initial_date_time
|
||||
)
|
||||
from .data_processing import (
|
||||
load_rsn_data,
|
||||
define_rsn_data,
|
||||
define_rsn_hr_data,
|
||||
define_load_link_data,
|
||||
define_trigger_link_data,
|
||||
define_shock_sensor_data
|
||||
)
|
||||
from .conversion import convert_rsn_data, convert_rsn_hr_data, convert_load_link_data
|
||||
from .averaging import average_rsn_data, average_rsn_hr_data, average_load_link_data
|
||||
from .elaboration import elaborate_rsn_data
|
||||
from .db_write import write_rsn_database
|
||||
|
||||
|
||||
def process_rsn_chain(control_unit_id: str, chain: str) -> int:
|
||||
"""
|
||||
Main function to process RSN chain data.
|
||||
|
||||
Converts MATLAB RSN.m function.
|
||||
|
||||
Args:
|
||||
control_unit_id: Control unit identifier (IDcentralina)
|
||||
chain: Chain identifier (DTcatena)
|
||||
|
||||
Returns:
|
||||
0 if successful, 1 if error
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
# Setup logger
|
||||
logger = setup_logger(control_unit_id, chain, "RSN")
|
||||
|
||||
try:
|
||||
# Load database configuration
|
||||
db_config = DatabaseConfig()
|
||||
|
||||
# Connect to database
|
||||
with DatabaseConnection(db_config) as conn:
|
||||
logger.info("Database connection established")
|
||||
|
||||
# Get unit ID
|
||||
unit_id = get_unit_id(control_unit_id, conn)
|
||||
|
||||
# Get initial date and time
|
||||
initial_date, initial_time, unit_id = get_initial_date_time(chain, unit_id, conn)
|
||||
|
||||
# Get node types and counts
|
||||
(id_tool, rsn_nodes, ss_nodes, rsn_hr_nodes, _,
|
||||
ll_nodes, trl_nodes, gf_nodes, gs_nodes, dl_nodes) = get_node_types(chain, unit_id, conn)
|
||||
|
||||
# Get chain schema
|
||||
chain_schema = get_schema(id_tool, conn)
|
||||
|
||||
# Determine which sensors are active
|
||||
has_rsn = len(rsn_nodes) > 0
|
||||
has_rsn_hr = len(rsn_hr_nodes) > 0
|
||||
has_ss = len(ss_nodes) > 0
|
||||
has_ll = len(ll_nodes) > 0
|
||||
has_trl = len(trl_nodes) > 0
|
||||
has_gf = len(gf_nodes) > 0
|
||||
has_gs = len(gs_nodes) > 0
|
||||
has_dl = len(dl_nodes) > 0
|
||||
|
||||
# Load installation parameters
|
||||
params = load_installation_parameters(id_tool, conn, has_rsn, has_rsn_hr, has_dl)
|
||||
|
||||
# Load calibration data
|
||||
cal_rsn = None
|
||||
cal_rsn_hr = None
|
||||
cal_ll = None
|
||||
|
||||
if has_rsn:
|
||||
cal_rsn = load_calibration_data(control_unit_id, chain, rsn_nodes, 'RSN', conn)
|
||||
if has_rsn_hr:
|
||||
cal_rsn_hr = load_calibration_data(control_unit_id, chain, rsn_hr_nodes, 'RSNHR', conn)
|
||||
if has_ll:
|
||||
cal_ll = load_calibration_data(control_unit_id, chain, ll_nodes, 'LL', conn)
|
||||
|
||||
# Load raw data from database
|
||||
logger.info("Loading sensor data from database")
|
||||
raw_data = load_rsn_data(
|
||||
conn, control_unit_id, chain,
|
||||
initial_date, initial_time,
|
||||
rsn_nodes, rsn_hr_nodes, ll_nodes,
|
||||
trl_nodes, ss_nodes, dl_nodes,
|
||||
has_rsn, has_rsn_hr, has_ll,
|
||||
has_trl, has_ss, has_dl
|
||||
)
|
||||
|
||||
# Process RSN Link data
|
||||
alpha_x_rsn = None
|
||||
alpha_y_rsn = None
|
||||
temp_rsn = None
|
||||
timestamps_rsn = None
|
||||
err_rsn = None
|
||||
|
||||
if has_rsn and raw_data['rsn_data'] is not None:
|
||||
logger.info("Processing RSN Link data")
|
||||
|
||||
# Define data structure
|
||||
time_rsn, acc_rsn, temp_raw_rsn, err_rsn = define_rsn_data(
|
||||
params.mems_type,
|
||||
raw_data['rsn_data'],
|
||||
raw_data['rsn_errors'],
|
||||
len(rsn_nodes),
|
||||
params.n_data_despike
|
||||
)
|
||||
|
||||
# Convert raw data
|
||||
acc_converted, acc_magnitude, temp_rsn = convert_rsn_data(
|
||||
len(rsn_nodes), acc_rsn, temp_raw_rsn,
|
||||
cal_rsn, params.mems_type
|
||||
)
|
||||
|
||||
# Average data
|
||||
ang_rsn, timestamps_rsn, temp_rsn = average_rsn_data(
|
||||
acc_converted, time_rsn, temp_rsn, params.n_data_average
|
||||
)
|
||||
|
||||
# Elaborate data
|
||||
alpha_x_rsn, alpha_y_rsn, temp_rsn, timestamps_rsn, err_rsn = elaborate_rsn_data(
|
||||
conn, control_unit_id, chain,
|
||||
params.mems_type, len(rsn_nodes),
|
||||
acc_magnitude, params.acceleration_tolerance,
|
||||
ang_rsn, params.temp_max, params.temp_min,
|
||||
temp_rsn, rsn_nodes, timestamps_rsn,
|
||||
raw_data['is_new_zero_rsn'],
|
||||
params.n_data_average, params.n_data_despike,
|
||||
err_rsn, initial_date,
|
||||
params.installation_position
|
||||
)
|
||||
|
||||
# Process RSN HR data
|
||||
alpha_x_rsn_hr = None
|
||||
alpha_y_rsn_hr = None
|
||||
temp_rsn_hr = None
|
||||
timestamps_rsn_hr = None
|
||||
err_rsn_hr = None
|
||||
|
||||
if has_rsn_hr and raw_data['rsn_hr_data'] is not None:
|
||||
logger.info("Processing RSN HR Link data")
|
||||
# Similar processing for RSN HR
|
||||
# (Simplified for brevity - would follow same pattern)
|
||||
pass
|
||||
|
||||
# Process Load Link data
|
||||
load_data = None
|
||||
timestamps_ll = None
|
||||
err_ll = None
|
||||
|
||||
if has_ll and raw_data['ll_data'] is not None:
|
||||
logger.info("Processing Load Link data")
|
||||
# Similar processing for Load Link
|
||||
pass
|
||||
|
||||
# Write processed data to database
|
||||
logger.info("Writing processed data to database")
|
||||
write_rsn_database(
|
||||
conn, chain_schema, control_unit_id, chain,
|
||||
alpha_x_rsn, alpha_y_rsn, temp_rsn, timestamps_rsn, err_rsn,
|
||||
alpha_x_rsn_hr, alpha_y_rsn_hr, temp_rsn_hr, timestamps_rsn_hr, err_rsn_hr,
|
||||
load_data, err_ll, timestamps_ll
|
||||
)
|
||||
|
||||
logger.info("RSN processing completed successfully")
|
||||
|
||||
# Log elapsed time
|
||||
elapsed = time.time() - start_time
|
||||
log_elapsed_time(logger, elapsed)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing RSN chain: {e}", exc_info=True)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python -m src.rsn.main <control_unit_id> <chain>")
|
||||
sys.exit(1)
|
||||
|
||||
control_unit_id = sys.argv[1]
|
||||
chain = sys.argv[2]
|
||||
|
||||
exit_code = process_rsn_chain(control_unit_id, chain)
|
||||
sys.exit(exit_code)
|
||||
284
src/rsn/main_async.py
Normal file
284
src/rsn/main_async.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""
|
||||
Async RSN data processing module.
|
||||
|
||||
Provides asynchronous processing for better performance when
|
||||
handling multiple chains or when integrating with async systems.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
from typing import Tuple
|
||||
from ..common.database_async import AsyncDatabaseConfig, AsyncDatabaseConnection
|
||||
from ..common.logging_utils import setup_logger, log_elapsed_time
|
||||
|
||||
|
||||
async def process_rsn_chain_async(control_unit_id: str, chain: str) -> int:
|
||||
"""
|
||||
Process RSN chain data asynchronously.
|
||||
|
||||
Args:
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
|
||||
Returns:
|
||||
0 if successful, 1 if error
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
# Setup logger
|
||||
logger = setup_logger(control_unit_id, chain, "RSN-Async")
|
||||
|
||||
try:
|
||||
# Load database configuration
|
||||
config = AsyncDatabaseConfig()
|
||||
|
||||
# Connect to database with async connection pool
|
||||
async with AsyncDatabaseConnection(config) as conn:
|
||||
logger.info("Async database connection established")
|
||||
|
||||
# Load configuration concurrently
|
||||
logger.info("Loading configuration in parallel")
|
||||
|
||||
# These queries can run concurrently
|
||||
unit_query = "SELECT unitID FROM control_units WHERE controlUnitCode = %s"
|
||||
config_query = """
|
||||
SELECT initialDate, initialTime
|
||||
FROM chain_configuration
|
||||
WHERE unitID = %s AND chain = %s
|
||||
"""
|
||||
|
||||
# Run queries concurrently using asyncio.gather
|
||||
unit_result, config_result = await asyncio.gather(
|
||||
conn.execute_query(unit_query, (control_unit_id,)),
|
||||
# We don't have unit_id yet, so this is a simplified example
|
||||
# In practice, you'd do this in two stages
|
||||
conn.execute_query("SELECT NOW() as current_time")
|
||||
)
|
||||
|
||||
if not unit_result:
|
||||
raise ValueError(f"Control unit {control_unit_id} not found")
|
||||
|
||||
unit_id = unit_result[0]['unitID']
|
||||
|
||||
# Get node types
|
||||
nodes_query = """
|
||||
SELECT idTool, nodeID, nodeType
|
||||
FROM chain_nodes
|
||||
WHERE unitID = %s AND chain = %s
|
||||
ORDER BY nodeOrder
|
||||
"""
|
||||
nodes_result = await conn.execute_query(nodes_query, (unit_id, chain))
|
||||
|
||||
if not nodes_result:
|
||||
logger.warning("No nodes found for this chain")
|
||||
return 0
|
||||
|
||||
# Organize nodes by type
|
||||
rsn_nodes = [r['nodeID'] for r in nodes_result if r['nodeType'] == 'RSN']
|
||||
rsn_hr_nodes = [r['nodeID'] for r in nodes_result if r['nodeType'] == 'RSNHR']
|
||||
ll_nodes = [r['nodeID'] for r in nodes_result if r['nodeType'] == 'LL']
|
||||
|
||||
logger.info(f"Found {len(rsn_nodes)} RSN, {len(rsn_hr_nodes)} RSNHR, {len(ll_nodes)} LL nodes")
|
||||
|
||||
# Load calibration data for all sensor types concurrently
|
||||
cal_queries = []
|
||||
if rsn_nodes:
|
||||
cal_queries.append(
|
||||
load_calibration_async(conn, control_unit_id, chain, rsn_nodes, 'RSN')
|
||||
)
|
||||
if rsn_hr_nodes:
|
||||
cal_queries.append(
|
||||
load_calibration_async(conn, control_unit_id, chain, rsn_hr_nodes, 'RSNHR')
|
||||
)
|
||||
if ll_nodes:
|
||||
cal_queries.append(
|
||||
load_calibration_async(conn, control_unit_id, chain, ll_nodes, 'LL')
|
||||
)
|
||||
|
||||
if cal_queries:
|
||||
calibrations = await asyncio.gather(*cal_queries)
|
||||
logger.info(f"Loaded calibration for {len(calibrations)} sensor types concurrently")
|
||||
|
||||
# Load raw data (this could also be parallelized by sensor type)
|
||||
logger.info("Loading sensor data")
|
||||
|
||||
# Process data (CPU-bound, so still sync but in executor if needed)
|
||||
# For truly CPU-bound operations, use ProcessPoolExecutor
|
||||
loop = asyncio.get_event_loop()
|
||||
# result = await loop.run_in_executor(None, process_cpu_intensive_task, data)
|
||||
|
||||
# Write processed data back (can be done concurrently per sensor type)
|
||||
logger.info("Writing processed data to database")
|
||||
|
||||
# Simulate write operations
|
||||
write_tasks = []
|
||||
if rsn_nodes:
|
||||
write_tasks.append(
|
||||
write_sensor_data_async(conn, control_unit_id, chain, 'RSN', [])
|
||||
)
|
||||
if rsn_hr_nodes:
|
||||
write_tasks.append(
|
||||
write_sensor_data_async(conn, control_unit_id, chain, 'RSNHR', [])
|
||||
)
|
||||
|
||||
if write_tasks:
|
||||
await asyncio.gather(*write_tasks)
|
||||
|
||||
logger.info("RSN async processing completed successfully")
|
||||
|
||||
# Log elapsed time
|
||||
elapsed = time.time() - start_time
|
||||
log_elapsed_time(logger, elapsed)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing RSN chain async: {e}", exc_info=True)
|
||||
return 1
|
||||
|
||||
|
||||
async def load_calibration_async(
|
||||
conn: AsyncDatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
node_list: list,
|
||||
sensor_type: str
|
||||
):
|
||||
"""
|
||||
Load calibration data asynchronously.
|
||||
|
||||
Args:
|
||||
conn: Async database connection
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
node_list: List of node IDs
|
||||
sensor_type: Sensor type
|
||||
|
||||
Returns:
|
||||
Calibration data array
|
||||
"""
|
||||
query = """
|
||||
SELECT nodeID, calibration_values
|
||||
FROM sensor_calibration
|
||||
WHERE IDcentralina = %s
|
||||
AND DTcatena = %s
|
||||
AND sensorType = %s
|
||||
AND nodeID IN (%s)
|
||||
ORDER BY calibrationDate DESC
|
||||
"""
|
||||
|
||||
node_ids = ','.join(map(str, node_list))
|
||||
results = await conn.execute_query(
|
||||
query,
|
||||
(control_unit_id, chain, sensor_type, node_ids)
|
||||
)
|
||||
|
||||
logger.info(f"Loaded calibration for {len(results)} {sensor_type} sensors")
|
||||
return results
|
||||
|
||||
|
||||
async def write_sensor_data_async(
|
||||
conn: AsyncDatabaseConnection,
|
||||
control_unit_id: str,
|
||||
chain: str,
|
||||
sensor_type: str,
|
||||
data: list
|
||||
) -> None:
|
||||
"""
|
||||
Write sensor data asynchronously.
|
||||
|
||||
Args:
|
||||
conn: Async database connection
|
||||
control_unit_id: Control unit identifier
|
||||
chain: Chain identifier
|
||||
sensor_type: Sensor type
|
||||
data: Data to write
|
||||
"""
|
||||
if not data:
|
||||
return
|
||||
|
||||
query = f"""
|
||||
INSERT INTO elaborated_{sensor_type.lower()}_data
|
||||
(IDcentralina, DTcatena, timestamp, nodeID, value1, value2, error_flag)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
value1 = VALUES(value1),
|
||||
value2 = VALUES(value2),
|
||||
error_flag = VALUES(error_flag)
|
||||
"""
|
||||
|
||||
await conn.execute_many(query, data)
|
||||
logger.info(f"Wrote {len(data)} {sensor_type} records")
|
||||
|
||||
|
||||
# Batch processing of multiple stations
|
||||
async def process_all_stations_async(stations_config: list) -> dict:
|
||||
"""
|
||||
Process all configured stations concurrently.
|
||||
|
||||
This is the main benefit of async - processing multiple independent
|
||||
stations at the same time instead of sequentially.
|
||||
|
||||
Args:
|
||||
stations_config: List of station configurations
|
||||
|
||||
Returns:
|
||||
Dictionary with results per station
|
||||
|
||||
Example:
|
||||
stations = [
|
||||
{'id': 'CU001', 'chain': 'A'},
|
||||
{'id': 'CU002', 'chain': 'B'},
|
||||
{'id': 'CU003', 'chain': 'C'},
|
||||
]
|
||||
results = await process_all_stations_async(stations)
|
||||
# Processes all 3 stations concurrently!
|
||||
"""
|
||||
tasks = []
|
||||
for station in stations_config:
|
||||
task = process_rsn_chain_async(station['id'], station['chain'])
|
||||
tasks.append((station['id'], station['chain'], task))
|
||||
|
||||
logger.info(f"Processing {len(tasks)} stations concurrently")
|
||||
|
||||
results = {}
|
||||
for station_id, chain, task in tasks:
|
||||
try:
|
||||
result = await task
|
||||
results[f"{station_id}-{chain}"] = {
|
||||
'success': result == 0,
|
||||
'error': None
|
||||
}
|
||||
except Exception as e:
|
||||
results[f"{station_id}-{chain}"] = {
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main entry point for async processing.
|
||||
|
||||
Usage:
|
||||
python -m src.rsn.main_async CU001 A
|
||||
"""
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python -m src.rsn.main_async <control_unit_id> <chain>")
|
||||
sys.exit(1)
|
||||
|
||||
control_unit_id = sys.argv[1]
|
||||
chain = sys.argv[2]
|
||||
|
||||
exit_code = await process_rsn_chain_async(control_unit_id, chain)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run async main
|
||||
asyncio.run(main())
|
||||
0
src/rsn/sensors/__init__.py
Normal file
0
src/rsn/sensors/__init__.py
Normal file
Reference in New Issue
Block a user