fix: Add timeout settings and retry logic to MySQL connector
Configuration improvements: - Set read_timeout=300 (5 minutes) to handle long queries - Set write_timeout=300 (5 minutes) for writes - Set max_allowed_packet=64MB to handle larger data transfers Retry logic: - Added retry mechanism with max 3 retries on fetch failure - Auto-reconnect on connection loss before retry - Better error messages showing retry attempts This fixes the 'connection is lost' error that occurs during long-running migrations by: 1. Giving MySQL queries more time to complete 2. Allowing larger packet sizes for bulk data 3. Automatically recovering from connection drops Fixes: 'Connection is lost' error during full migration
This commit is contained in:
@@ -26,6 +26,9 @@ class MySQLConnector:
|
||||
database=self.settings.mysql.database,
|
||||
charset="utf8mb4",
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
read_timeout=300, # 5 minutes read timeout
|
||||
write_timeout=300, # 5 minutes write timeout
|
||||
max_allowed_packet=67108864, # 64MB max packet
|
||||
)
|
||||
logger.info(
|
||||
f"Connected to MySQL: {self.settings.mysql.host}:"
|
||||
@@ -86,22 +89,38 @@ class MySQLConnector:
|
||||
batch_size = self.settings.migration.batch_size
|
||||
|
||||
offset = 0
|
||||
max_retries = 3
|
||||
|
||||
while True:
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
query = f"SELECT * FROM `{table}` LIMIT %s OFFSET %s"
|
||||
cursor.execute(query, (batch_size, offset))
|
||||
rows = cursor.fetchall()
|
||||
retries = 0
|
||||
while retries < max_retries:
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
query = f"SELECT * FROM `{table}` LIMIT %s OFFSET %s"
|
||||
cursor.execute(query, (batch_size, offset))
|
||||
rows = cursor.fetchall()
|
||||
|
||||
if not rows:
|
||||
break
|
||||
if not rows:
|
||||
return
|
||||
|
||||
yield rows
|
||||
offset += len(rows)
|
||||
yield rows
|
||||
offset += len(rows)
|
||||
break # Success, exit retry loop
|
||||
|
||||
except pymysql.Error as e:
|
||||
logger.error(f"Failed to fetch rows from {table}: {e}")
|
||||
raise
|
||||
except pymysql.Error as e:
|
||||
retries += 1
|
||||
if retries >= max_retries:
|
||||
logger.error(f"Failed to fetch rows from {table} after {max_retries} retries: {e}")
|
||||
raise
|
||||
else:
|
||||
logger.warning(f"Fetch failed (retry {retries}/{max_retries}): {e}")
|
||||
# Reconnect and retry
|
||||
try:
|
||||
self.disconnect()
|
||||
self.connect()
|
||||
except Exception as reconnect_error:
|
||||
logger.error(f"Failed to reconnect: {reconnect_error}")
|
||||
raise
|
||||
|
||||
def fetch_rows_since(
|
||||
self,
|
||||
@@ -147,6 +166,59 @@ class MySQLConnector:
|
||||
logger.error(f"Failed to fetch rows from {table}: {e}")
|
||||
raise
|
||||
|
||||
def fetch_rows_from_id(
|
||||
self,
|
||||
table: str,
|
||||
primary_key: str,
|
||||
start_id: Optional[int] = None,
|
||||
batch_size: Optional[int] = None
|
||||
) -> Generator[List[Dict[str, Any]], None, None]:
|
||||
"""Fetch rows after a specific ID for resumable migrations.
|
||||
|
||||
Args:
|
||||
table: Table name
|
||||
primary_key: Primary key column name
|
||||
start_id: Start ID (fetch rows with ID > start_id), None to fetch from start
|
||||
batch_size: Number of rows per batch (uses config default if None)
|
||||
|
||||
Yields:
|
||||
Batches of row dictionaries
|
||||
"""
|
||||
if batch_size is None:
|
||||
batch_size = self.settings.migration.batch_size
|
||||
|
||||
offset = 0
|
||||
while True:
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
if start_id is not None:
|
||||
query = (
|
||||
f"SELECT * FROM `{table}` "
|
||||
f"WHERE `{primary_key}` > %s "
|
||||
f"ORDER BY `{primary_key}` ASC "
|
||||
f"LIMIT %s OFFSET %s"
|
||||
)
|
||||
cursor.execute(query, (start_id, batch_size, offset))
|
||||
else:
|
||||
query = (
|
||||
f"SELECT * FROM `{table}` "
|
||||
f"ORDER BY `{primary_key}` ASC "
|
||||
f"LIMIT %s OFFSET %s"
|
||||
)
|
||||
cursor.execute(query, (batch_size, offset))
|
||||
|
||||
rows = cursor.fetchall()
|
||||
|
||||
if not rows:
|
||||
break
|
||||
|
||||
yield rows
|
||||
offset += len(rows)
|
||||
|
||||
except pymysql.Error as e:
|
||||
logger.error(f"Failed to fetch rows from {table}: {e}")
|
||||
raise
|
||||
|
||||
def get_table_structure(self, table: str) -> Dict[str, Any]:
|
||||
"""Get table structure (column info).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user