fix: Add timeout settings and retry logic to MySQL connector
Configuration improvements: - Set read_timeout=300 (5 minutes) to handle long queries - Set write_timeout=300 (5 minutes) for writes - Set max_allowed_packet=64MB to handle larger data transfers Retry logic: - Added retry mechanism with max 3 retries on fetch failure - Auto-reconnect on connection loss before retry - Better error messages showing retry attempts This fixes the 'connection is lost' error that occurs during long-running migrations by: 1. Giving MySQL queries more time to complete 2. Allowing larger packet sizes for bulk data 3. Automatically recovering from connection drops Fixes: 'Connection is lost' error during full migration
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
"""Data transformation from MySQL to PostgreSQL format."""
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime
|
||||
from datetime import datetime, time, timedelta
|
||||
from config import (
|
||||
RAWDATACOR_COLUMNS,
|
||||
ELABDATADISP_FIELD_MAPPING,
|
||||
@@ -14,6 +14,36 @@ logger = get_logger(__name__)
|
||||
class DataTransformer:
|
||||
"""Transform MySQL data to PostgreSQL format."""
|
||||
|
||||
@staticmethod
|
||||
def _convert_time(event_time: Any) -> time:
|
||||
"""Convert event_time to datetime.time object.
|
||||
|
||||
Handles multiple input types:
|
||||
- str: Parse from "HH:MM:SS" format
|
||||
- timedelta: Convert from MySQL TIME type (stored as timedelta)
|
||||
- time: Return as-is
|
||||
|
||||
Args:
|
||||
event_time: Time value from MySQL (str, timedelta, or time)
|
||||
|
||||
Returns:
|
||||
datetime.time object
|
||||
"""
|
||||
if isinstance(event_time, str):
|
||||
return datetime.strptime(event_time, "%H:%M:%S").time()
|
||||
elif isinstance(event_time, timedelta):
|
||||
# MySQL returns TIME as timedelta
|
||||
# Extract seconds from timedelta and convert to time
|
||||
total_seconds = int(event_time.total_seconds())
|
||||
hours = total_seconds // 3600
|
||||
minutes = (total_seconds % 3600) // 60
|
||||
seconds = total_seconds % 60
|
||||
return time(hour=hours, minute=minutes, second=seconds)
|
||||
elif isinstance(event_time, time):
|
||||
return event_time
|
||||
else:
|
||||
raise ValueError(f"Unsupported event_time type: {type(event_time)}")
|
||||
|
||||
@staticmethod
|
||||
def transform_rawdatacor_row(mysql_row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Transform a RAWDATACOR row from MySQL to PostgreSQL format.
|
||||
@@ -41,14 +71,35 @@ class DataTransformer:
|
||||
"unit": unit if unit else None,
|
||||
}
|
||||
|
||||
# Combine event_date and event_time into event_timestamp
|
||||
event_date = mysql_row.get("EventDate")
|
||||
event_time = mysql_row.get("EventTime")
|
||||
if event_date is not None and event_time is not None:
|
||||
event_time_obj = DataTransformer._convert_time(event_time)
|
||||
event_timestamp = datetime.combine(event_date, event_time_obj)
|
||||
elif event_date is None or event_time is None:
|
||||
# Log a warning for records with missing date/time
|
||||
missing = []
|
||||
if event_date is None:
|
||||
missing.append("EventDate")
|
||||
if event_time is None:
|
||||
missing.append("EventTime")
|
||||
logger.warning(
|
||||
f"Row {mysql_row.get('id')} has NULL {', '.join(missing)}. "
|
||||
f"Using default timestamp: 1970-01-01 00:00:00"
|
||||
)
|
||||
# Use default timestamp for records with missing date/time
|
||||
event_timestamp = datetime(1970, 1, 1, 0, 0, 0)
|
||||
else:
|
||||
event_timestamp = None
|
||||
|
||||
# Create PostgreSQL row
|
||||
pg_row = {
|
||||
"id": mysql_row["id"],
|
||||
"unit_name": mysql_row.get("UnitName"),
|
||||
"tool_name_id": mysql_row["ToolNameID"],
|
||||
"node_num": mysql_row["NodeNum"],
|
||||
"event_date": mysql_row["EventDate"],
|
||||
"event_time": mysql_row["EventTime"],
|
||||
"event_timestamp": event_timestamp,
|
||||
"bat_level": mysql_row["BatLevel"],
|
||||
"temperature": mysql_row["Temperature"],
|
||||
"measurements": measurements,
|
||||
@@ -90,14 +141,35 @@ class DataTransformer:
|
||||
k: v for k, v in measurements.items() if v
|
||||
}
|
||||
|
||||
# Combine event_date and event_time into event_timestamp
|
||||
event_date = mysql_row.get("EventDate")
|
||||
event_time = mysql_row.get("EventTime")
|
||||
if event_date is not None and event_time is not None:
|
||||
event_time_obj = DataTransformer._convert_time(event_time)
|
||||
event_timestamp = datetime.combine(event_date, event_time_obj)
|
||||
elif event_date is None or event_time is None:
|
||||
# Log a warning for records with missing date/time
|
||||
missing = []
|
||||
if event_date is None:
|
||||
missing.append("EventDate")
|
||||
if event_time is None:
|
||||
missing.append("EventTime")
|
||||
logger.warning(
|
||||
f"Row {mysql_row.get('idElabData')} has NULL {', '.join(missing)}. "
|
||||
f"Using default timestamp: 1970-01-01 00:00:00"
|
||||
)
|
||||
# Use default timestamp for records with missing date/time
|
||||
event_timestamp = datetime(1970, 1, 1, 0, 0, 0)
|
||||
else:
|
||||
event_timestamp = None
|
||||
|
||||
# Create PostgreSQL row
|
||||
pg_row = {
|
||||
"id_elab_data": mysql_row["idElabData"],
|
||||
"unit_name": mysql_row.get("UnitName"),
|
||||
"tool_name_id": mysql_row["ToolNameID"],
|
||||
"node_num": mysql_row["NodeNum"],
|
||||
"event_date": mysql_row["EventDate"],
|
||||
"event_time": mysql_row["EventTime"],
|
||||
"event_timestamp": event_timestamp,
|
||||
"state": mysql_row.get("State"),
|
||||
"calc_err": mysql_row.get("calcerr", 0),
|
||||
"measurements": measurements,
|
||||
@@ -150,8 +222,7 @@ class DataTransformer:
|
||||
"unit_name",
|
||||
"tool_name_id",
|
||||
"node_num",
|
||||
"event_date",
|
||||
"event_time",
|
||||
"event_timestamp",
|
||||
"bat_level",
|
||||
"temperature",
|
||||
"measurements",
|
||||
@@ -166,8 +237,7 @@ class DataTransformer:
|
||||
"unit_name",
|
||||
"tool_name_id",
|
||||
"node_num",
|
||||
"event_date",
|
||||
"event_time",
|
||||
"event_timestamp",
|
||||
"state",
|
||||
"calc_err",
|
||||
"measurements",
|
||||
|
||||
Reference in New Issue
Block a user