fix: Add timeout settings and retry logic to MySQL connector

Configuration improvements:
- Set read_timeout=300 (5 minutes) to handle long queries
- Set write_timeout=300 (5 minutes) for writes
- Set max_allowed_packet=64MB to handle larger data transfers

Retry logic:
- Added retry mechanism with max 3 retries on fetch failure
- Auto-reconnect on connection loss before retry
- Better error messages showing retry attempts

This fixes the 'connection is lost' error that occurs during
long-running migrations by:
1. Giving MySQL queries more time to complete
2. Allowing larger packet sizes for bulk data
3. Automatically recovering from connection drops

Fixes: 'Connection is lost' error during full migration
This commit is contained in:
2025-12-21 09:53:34 +01:00
parent 821cda850e
commit b09cfcf9df
8 changed files with 761 additions and 119 deletions

View File

@@ -26,6 +26,9 @@ class MySQLConnector:
database=self.settings.mysql.database,
charset="utf8mb4",
cursorclass=pymysql.cursors.DictCursor,
read_timeout=300, # 5 minutes read timeout
write_timeout=300, # 5 minutes write timeout
max_allowed_packet=67108864, # 64MB max packet
)
logger.info(
f"Connected to MySQL: {self.settings.mysql.host}:"
@@ -86,22 +89,38 @@ class MySQLConnector:
batch_size = self.settings.migration.batch_size
offset = 0
max_retries = 3
while True:
try:
with self.connection.cursor() as cursor:
query = f"SELECT * FROM `{table}` LIMIT %s OFFSET %s"
cursor.execute(query, (batch_size, offset))
rows = cursor.fetchall()
retries = 0
while retries < max_retries:
try:
with self.connection.cursor() as cursor:
query = f"SELECT * FROM `{table}` LIMIT %s OFFSET %s"
cursor.execute(query, (batch_size, offset))
rows = cursor.fetchall()
if not rows:
break
if not rows:
return
yield rows
offset += len(rows)
yield rows
offset += len(rows)
break # Success, exit retry loop
except pymysql.Error as e:
logger.error(f"Failed to fetch rows from {table}: {e}")
raise
except pymysql.Error as e:
retries += 1
if retries >= max_retries:
logger.error(f"Failed to fetch rows from {table} after {max_retries} retries: {e}")
raise
else:
logger.warning(f"Fetch failed (retry {retries}/{max_retries}): {e}")
# Reconnect and retry
try:
self.disconnect()
self.connect()
except Exception as reconnect_error:
logger.error(f"Failed to reconnect: {reconnect_error}")
raise
def fetch_rows_since(
self,
@@ -147,6 +166,59 @@ class MySQLConnector:
logger.error(f"Failed to fetch rows from {table}: {e}")
raise
def fetch_rows_from_id(
self,
table: str,
primary_key: str,
start_id: Optional[int] = None,
batch_size: Optional[int] = None
) -> Generator[List[Dict[str, Any]], None, None]:
"""Fetch rows after a specific ID for resumable migrations.
Args:
table: Table name
primary_key: Primary key column name
start_id: Start ID (fetch rows with ID > start_id), None to fetch from start
batch_size: Number of rows per batch (uses config default if None)
Yields:
Batches of row dictionaries
"""
if batch_size is None:
batch_size = self.settings.migration.batch_size
offset = 0
while True:
try:
with self.connection.cursor() as cursor:
if start_id is not None:
query = (
f"SELECT * FROM `{table}` "
f"WHERE `{primary_key}` > %s "
f"ORDER BY `{primary_key}` ASC "
f"LIMIT %s OFFSET %s"
)
cursor.execute(query, (start_id, batch_size, offset))
else:
query = (
f"SELECT * FROM `{table}` "
f"ORDER BY `{primary_key}` ASC "
f"LIMIT %s OFFSET %s"
)
cursor.execute(query, (batch_size, offset))
rows = cursor.fetchall()
if not rows:
break
yield rows
offset += len(rows)
except pymysql.Error as e:
logger.error(f"Failed to fetch rows from {table}: {e}")
raise
def get_table_structure(self, table: str) -> Dict[str, Any]:
"""Get table structure (column info).