alex/matlab-python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add comprehensive validation system and migrate to .env configuration

Browse Source 
This commit includes:

1. Database Configuration Migration:
   - Migrated from DB.txt (Java JDBC) to .env (python-dotenv)
   - Added .env.example template with clear variable names
   - Updated database.py to use environment variables
   - Added python-dotenv>=1.0.0 to dependencies
   - Updated .gitignore to exclude sensitive files

2. Validation System (1,294 lines):
   - comparator.py: Statistical comparison with RMSE, correlation, tolerances
   - db_extractor.py: Database queries for all sensor types
   - validator.py: High-level validation orchestration
   - cli.py: Command-line interface for validation
   - README.md: Comprehensive validation documentation

3. Validation Features:
   - Compare Python vs MATLAB outputs from database
   - Support for all sensor types (RSN, Tilt, ATD)
   - Statistical metrics: max abs/rel diff, RMSE, correlation
   - Configurable tolerances (abs, rel, max)
   - Detailed validation reports
   - CLI and programmatic APIs

4. Examples and Documentation:
   - validate_example.sh: Bash script example
   - validate_example.py: Python programmatic example
   - Updated main README with validation section
   - Added validation workflow and troubleshooting guide

Benefits:
- ✅ No Java driver needed (native Python connectors)
- ✅ Secure .env configuration (excluded from git)
- ✅ Comprehensive validation against MATLAB
- ✅ Statistical confidence in migration accuracy
- ✅ Automated validation reports

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
alex
2025-10-13 15:34:13 +02:00
parent 876ef073fc
commit 23c53cf747
25 changed files with 7476 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

410
src/tilt/main.py
View File

@@ -2,21 +2,385 @@
Main Tilt sensor data processing module.
Entry point for tiltmeter sensor data elaboration.
Similar structure to RSN module but for tilt/inclinometer sensors.
Processes TLHR, BL, PL, KLHR and other tilt sensor types.
"""
import time
import logging
from typing import Tuple
from ..common.database import DatabaseConfig, DatabaseConnection, get_unit_id, get_schema
from ..common.database import DatabaseConfig, DatabaseConnection, get_unit_id
from ..common.logging_utils import setup_logger, log_elapsed_time
from ..common.config import load_installation_parameters, load_calibration_data
from .data_processing import (
load_tilt_link_hr_data, define_tilt_link_hr_data,
load_biaxial_link_data, define_biaxial_link_data,
load_pendulum_link_data, define_pendulum_link_data,
load_k_link_hr_data, define_k_link_hr_data
)
from .conversion import (
convert_tilt_link_hr_data, convert_biaxial_link_data,
convert_pendulum_link_data, convert_k_link_hr_data
)
from .averaging import (
average_tilt_link_hr_data, average_biaxial_link_data,
average_pendulum_link_data, average_k_link_hr_data
)
from .elaboration import (
elaborate_tilt_link_hr_data, elaborate_biaxial_link_data,
elaborate_pendulum_link_data, elaborate_k_link_hr_data
)
from .db_write import (
write_tilt_link_hr_data, write_biaxial_link_data,
write_pendulum_link_data, write_k_link_hr_data
)
def process_tlhr_sensors(conn, control_unit_id: str, chain: str, node_list: list,
params: dict, logger: logging.Logger) -> bool:
"""
Process TLHR (Tilt Link High Resolution) sensors.
Args:
conn: Database connection
control_unit_id: Control unit identifier
chain: Chain identifier
node_list: List of TLHR node IDs
params: Installation parameters
logger: Logger instance
Returns:
True if successful, False otherwise
"""
try:
n_sensors = len(node_list)
logger.info(f"Processing {n_sensors} TLHR sensors")
# Load calibration data
calibration_data = load_calibration_data(control_unit_id, chain, 'TLHR', conn)
# Get parameters
initial_date = params.get('initial_date')
initial_time = params.get('initial_time')
n_points = params.get('n_points_avg', 100)
n_despike = params.get('n_despike', 5)
temp_max = params.get('temp_max', 80.0)
temp_min = params.get('temp_min', -30.0)
# Load raw data from database
logger.info("Loading TLHR raw data from database")
raw_data = load_tilt_link_hr_data(conn, control_unit_id, chain,
initial_date, initial_time, node_list)
if raw_data is None or len(raw_data) == 0:
logger.warning("No TLHR data found")
return True
# Define data structure (handle NaN, despike, scale wrapping)
logger.info("Structuring TLHR data")
angle_data, timestamps, temperature, err_flag = define_tilt_link_hr_data(
raw_data, n_sensors, n_despike, temp_max, temp_min
)
if angle_data is None:
logger.warning("TLHR data definition failed")
return True
# Convert raw to physical units
logger.info("Converting TLHR data")
angle_converted, temperature_converted, err_flag = convert_tilt_link_hr_data(
angle_data, temperature, calibration_data, n_sensors
)
# Average with Gaussian smoothing
logger.info(f"Averaging TLHR data with {n_points} points")
angle_avg, temperature_avg, err_flag = average_tilt_link_hr_data(
angle_converted, timestamps, temperature_converted, n_points
)
# Elaborate (calculate displacements, differentials)
logger.info("Elaborating TLHR data")
x_global, y_global, z_global, x_local, y_local, z_local, \
x_diff, y_diff, z_diff, err_flag = elaborate_tilt_link_hr_data(
conn, control_unit_id, chain, n_sensors, angle_avg,
temp_max, temp_min, temperature_avg, err_flag, params
)
# Write to database
logger.info("Writing TLHR data to database")
write_tilt_link_hr_data(
conn, control_unit_id, chain, x_global, y_global, z_global,
x_local, y_local, z_local, x_diff, y_diff, z_diff,
timestamps, node_list, err_flag
)
logger.info(f"TLHR processing completed: {len(timestamps)} records")
return True
except Exception as e:
logger.error(f"Error processing TLHR sensors: {e}", exc_info=True)
return False
def process_bl_sensors(conn, control_unit_id: str, chain: str, node_list: list,
params: dict, logger: logging.Logger) -> bool:
"""
Process BL (Biaxial Link) sensors.
Args:
conn: Database connection
control_unit_id: Control unit identifier
chain: Chain identifier
node_list: List of BL node IDs
params: Installation parameters
logger: Logger instance
Returns:
True if successful, False otherwise
"""
try:
n_sensors = len(node_list)
logger.info(f"Processing {n_sensors} BL sensors")
# Load calibration data
calibration_data = load_calibration_data(control_unit_id, chain, 'BL', conn)
# Get parameters
initial_date = params.get('initial_date')
initial_time = params.get('initial_time')
n_points = params.get('n_points_avg', 100)
n_despike = params.get('n_despike', 5)
temp_max = params.get('temp_max', 80.0)
temp_min = params.get('temp_min', -30.0)
# Load raw data
logger.info("Loading BL raw data from database")
raw_data = load_biaxial_link_data(conn, control_unit_id, chain,
initial_date, initial_time, node_list)
if raw_data is None or len(raw_data) == 0:
logger.warning("No BL data found")
return True
# Define data structure
logger.info("Structuring BL data")
angle_data, timestamps, temperature, err_flag = define_biaxial_link_data(
raw_data, n_sensors, n_despike, temp_max, temp_min
)
if angle_data is None:
logger.warning("BL data definition failed")
return True
# Convert
logger.info("Converting BL data")
angle_converted, temperature_converted, err_flag = convert_biaxial_link_data(
angle_data, temperature, calibration_data, n_sensors
)
# Average
logger.info(f"Averaging BL data with {n_points} points")
angle_avg, temperature_avg, err_flag = average_biaxial_link_data(
angle_converted, timestamps, temperature_converted, n_points
)
# Elaborate
logger.info("Elaborating BL data")
x_global, y_global, z_global, x_diff, y_diff, z_diff, err_flag = \
elaborate_biaxial_link_data(
conn, control_unit_id, chain, n_sensors, angle_avg,
temp_max, temp_min, temperature_avg, err_flag, params
)
# Write to database
logger.info("Writing BL data to database")
write_biaxial_link_data(
conn, control_unit_id, chain, x_global, y_global, z_global,
x_diff, y_diff, z_diff, timestamps, node_list, err_flag
)
logger.info(f"BL processing completed: {len(timestamps)} records")
return True
except Exception as e:
logger.error(f"Error processing BL sensors: {e}", exc_info=True)
return False
def process_pl_sensors(conn, control_unit_id: str, chain: str, node_list: list,
params: dict, logger: logging.Logger) -> bool:
"""
Process PL (Pendulum Link) sensors.
Args:
conn: Database connection
control_unit_id: Control unit identifier
chain: Chain identifier
node_list: List of PL node IDs
params: Installation parameters
logger: Logger instance
Returns:
True if successful, False otherwise
"""
try:
n_sensors = len(node_list)
logger.info(f"Processing {n_sensors} PL sensors")
# Load calibration data
calibration_data = load_calibration_data(control_unit_id, chain, 'PL', conn)
# Get parameters
initial_date = params.get('initial_date')
initial_time = params.get('initial_time')
n_points = params.get('n_points_avg', 100)
n_despike = params.get('n_despike', 5)
temp_max = params.get('temp_max', 80.0)
temp_min = params.get('temp_min', -30.0)
# Load raw data
logger.info("Loading PL raw data from database")
raw_data = load_pendulum_link_data(conn, control_unit_id, chain,
initial_date, initial_time, node_list)
if raw_data is None or len(raw_data) == 0:
logger.warning("No PL data found")
return True
# Define data structure
logger.info("Structuring PL data")
angle_data, timestamps, temperature, err_flag = define_pendulum_link_data(
raw_data, n_sensors, n_despike, temp_max, temp_min
)
if angle_data is None:
logger.warning("PL data definition failed")
return True
# Convert
logger.info("Converting PL data")
angle_converted, temperature_converted, err_flag = convert_pendulum_link_data(
angle_data, temperature, calibration_data, n_sensors
)
# Average
logger.info(f"Averaging PL data with {n_points} points")
angle_avg, temperature_avg, err_flag = average_pendulum_link_data(
angle_converted, timestamps, temperature_converted, n_points
)
# Elaborate
logger.info("Elaborating PL data")
x_global, y_global, z_global, x_diff, y_diff, z_diff, err_flag = \
elaborate_pendulum_link_data(
conn, control_unit_id, chain, n_sensors, angle_avg,
temp_max, temp_min, temperature_avg, err_flag, params
)
# Write to database
logger.info("Writing PL data to database")
write_pendulum_link_data(
conn, control_unit_id, chain, x_global, y_global, z_global,
x_diff, y_diff, z_diff, timestamps, node_list, err_flag
)
logger.info(f"PL processing completed: {len(timestamps)} records")
return True
except Exception as e:
logger.error(f"Error processing PL sensors: {e}", exc_info=True)
return False
def process_klhr_sensors(conn, control_unit_id: str, chain: str, node_list: list,
params: dict, logger: logging.Logger) -> bool:
"""
Process KLHR (K Link High Resolution) sensors.
Args:
conn: Database connection
control_unit_id: Control unit identifier
chain: Chain identifier
node_list: List of KLHR node IDs
params: Installation parameters
logger: Logger instance
Returns:
True if successful, False otherwise
"""
try:
n_sensors = len(node_list)
logger.info(f"Processing {n_sensors} KLHR sensors")
# Load calibration data
calibration_data = load_calibration_data(control_unit_id, chain, 'KLHR', conn)
# Get parameters
initial_date = params.get('initial_date')
initial_time = params.get('initial_time')
n_points = params.get('n_points_avg', 100)
n_despike = params.get('n_despike', 5)
temp_max = params.get('temp_max', 80.0)
temp_min = params.get('temp_min', -30.0)
# Load raw data
logger.info("Loading KLHR raw data from database")
raw_data = load_k_link_hr_data(conn, control_unit_id, chain,
initial_date, initial_time, node_list)
if raw_data is None or len(raw_data) == 0:
logger.warning("No KLHR data found")
return True
# Define data structure
logger.info("Structuring KLHR data")
angle_data, timestamps, temperature, err_flag = define_k_link_hr_data(
raw_data, n_sensors, n_despike, temp_max, temp_min
)
if angle_data is None:
logger.warning("KLHR data definition failed")
return True
# Convert
logger.info("Converting KLHR data")
angle_converted, temperature_converted, err_flag = convert_k_link_hr_data(
angle_data, temperature, calibration_data, n_sensors
)
# Average
logger.info(f"Averaging KLHR data with {n_points} points")
angle_avg, temperature_avg, err_flag = average_k_link_hr_data(
angle_converted, timestamps, temperature_converted, n_points
)
# Elaborate
logger.info("Elaborating KLHR data")
x_global, y_global, z_global, x_diff, y_diff, z_diff, err_flag = \
elaborate_k_link_hr_data(
conn, control_unit_id, chain, n_sensors, angle_avg,
temp_max, temp_min, temperature_avg, err_flag, params
)
# Write to database
logger.info("Writing KLHR data to database")
write_k_link_hr_data(
conn, control_unit_id, chain, x_global, y_global, z_global,
x_diff, y_diff, z_diff, timestamps, node_list, err_flag
)
logger.info(f"KLHR processing completed: {len(timestamps)} records")
return True
except Exception as e:
logger.error(f"Error processing KLHR sensors: {e}", exc_info=True)
return False
def process_tilt_chain(control_unit_id: str, chain: str) -> int:
"""
Main function to process Tilt chain data.
Supports sensor types: TLHR, BL, PL, KLHR
Args:
control_unit_id: Control unit identifier
chain: Chain identifier
@@ -43,12 +407,13 @@ def process_tilt_chain(control_unit_id: str, chain: str) -> int:
# Load node configuration
logger.info("Loading tilt sensor configuration")
# Query for tilt sensor types (TL, TLH, TLHR, BL, PL, etc.)
# Query for tilt sensor types
query = """
SELECT idTool, nodeID, nodeType, sensorModel
FROM chain_nodes
WHERE unitID = %s AND chain = %s
AND nodeType IN ('TL', 'TLH', 'TLHR', 'TLHRH', 'BL', 'PL', 'RL', 'ThL', 'IPL', 'IPLHR', 'KL', 'KLHR', 'PT100')
AND nodeType IN ('TLHR', 'BL', 'PL', 'KLHR', 'TL', 'TLH', 'TLHRH',
'RL', 'ThL', 'IPL', 'IPLHR', 'KL', 'PT100')
ORDER BY nodeOrder
"""
results = conn.execute_query(query, (unit_id, chain))
@@ -73,34 +438,43 @@ def process_tilt_chain(control_unit_id: str, chain: str) -> int:
params = load_installation_parameters(id_tool, conn)
# Process each sensor type
# TL - Tilt Link (basic biaxial inclinometer)
if 'TL' in tilt_sensors:
logger.info(f"Processing {len(tilt_sensors['TL'])} TL sensors")
# Load, convert, average, elaborate, write
# Implementation would follow RSN pattern
success = True
# TLHR - Tilt Link High Resolution
# TLHR - Tilt Link High Resolution (most common)
if 'TLHR' in tilt_sensors:
logger.info(f"Processing {len(tilt_sensors['TLHR'])} TLHR sensors")
# Similar processing
if not process_tlhr_sensors(conn, control_unit_id, chain,
tilt_sensors['TLHR'], params, logger):
success = False
# BL - Biaxial Link
if 'BL' in tilt_sensors:
logger.info(f"Processing {len(tilt_sensors['BL'])} BL sensors")
if not process_bl_sensors(conn, control_unit_id, chain,
tilt_sensors['BL'], params, logger):
success = False
# PL - Pendulum Link
if 'PL' in tilt_sensors:
logger.info(f"Processing {len(tilt_sensors['PL'])} PL sensors")
if not process_pl_sensors(conn, control_unit_id, chain,
tilt_sensors['PL'], params, logger):
success = False
# Additional sensor types...
# KLHR - K Link High Resolution
if 'KLHR' in tilt_sensors:
if not process_klhr_sensors(conn, control_unit_id, chain,
tilt_sensors['KLHR'], params, logger):
success = False
logger.info("Tilt processing completed successfully")
# Log completion status
if success:
logger.info("Tilt processing completed successfully")
else:
logger.warning("Tilt processing completed with errors")
# Log elapsed time
elapsed = time.time() - start_time
log_elapsed_time(logger, elapsed)
return 0
return 0 if success else 1
except Exception as e:
logger.error(f"Error processing Tilt chain: {e}", exc_info=True)