diff --git a/src/connectors/mysql_connector.py b/src/connectors/mysql_connector.py index a9da2cf..6a071d6 100644 --- a/src/connectors/mysql_connector.py +++ b/src/connectors/mysql_connector.py @@ -337,7 +337,8 @@ class MySQLConnector: table: str, partition: str, limit: Optional[int] = None, - offset: int = 0 + offset: int = 0, + start_id: Optional[int] = None ) -> Generator[List[Dict[str, Any]], None, None]: """Fetch consolidation groups from a partition. @@ -352,6 +353,7 @@ class MySQLConnector: partition: Partition name limit: Batch size for consolidation (uses config default if None) offset: Starting offset for pagination (unused, kept for compatibility) + start_id: Resume from this ID (fetch id > start_id). If None, starts from beginning Yields: Lists of rows grouped by consolidation key (complete groups only) @@ -365,7 +367,7 @@ class MySQLConnector: # Determine ID column name id_column = "idElabData" if table == "ELABDATADISP" else "id" max_retries = 3 - last_id = None + last_id = start_id buffered_group = [] # Buffer incomplete group at batch boundary last_buffered_key = None diff --git a/src/migrator/full_migration.py b/src/migrator/full_migration.py index c5d5c02..80d332f 100644 --- a/src/migrator/full_migration.py +++ b/src/migrator/full_migration.py @@ -120,6 +120,16 @@ class FullMigrator: logger.info(f"[{partition_idx}/{len(partitions)}] Processing partition {partition}...") partition_group_count = 0 + # Determine resume point within this partition + # If resuming and this is the last completed partition, start from last_id + start_id = None + if last_completed_partition == partition and previous_migrated_count > 0: + # For resume within same partition, we need to query the last ID inserted + # This is a simplified approach: just continue from ID tracking + start_id = self._get_last_migrated_id(pg_conn, pg_table) + if start_id: + logger.info(f"Resuming partition {partition} from ID > {start_id}") + # Accumulate rows for batch insertion to reduce database round-trips insert_buffer = [] # Use smaller batch size for more frequent updates: batch_size * 5 = 50k rows @@ -130,7 +140,8 @@ class FullMigrator: # Each group is a list of rows with the same (unit, tool, date, time) for group_rows in mysql_conn.fetch_consolidation_groups_from_partition( mysql_table, - partition + partition, + start_id=start_id ): if not group_rows: break