diff --git a/src/connectors/mysql_connector.py b/src/connectors/mysql_connector.py index f94341a..3c08559 100644 --- a/src/connectors/mysql_connector.py +++ b/src/connectors/mysql_connector.py @@ -257,11 +257,15 @@ class MySQLConnector: with self.connection.cursor() as cursor: # Use keyset pagination: fetch by id > last_id # This is much more efficient than OFFSET for large tables + # Order by id first for pagination, then by consolidation key to keep + # related nodes together in the same batch + order_clause = f"`{id_column}` ASC, `UnitName` ASC, `ToolNameID` ASC, `EventDate` ASC, `EventTime` ASC, `NodeNum` ASC" + if last_id is None: - query = f"SELECT * FROM `{table}` ORDER BY `{id_column}` ASC LIMIT %s" + query = f"SELECT * FROM `{table}` ORDER BY {order_clause} LIMIT %s" cursor.execute(query, (batch_size,)) else: - query = f"SELECT * FROM `{table}` WHERE `{id_column}` > %s ORDER BY `{id_column}` ASC LIMIT %s" + query = f"SELECT * FROM `{table}` WHERE `{id_column}` > %s ORDER BY {order_clause} LIMIT %s" cursor.execute(query, (last_id, batch_size)) rows = cursor.fetchall()