diff --git a/src/migrator/full_migration.py b/src/migrator/full_migration.py index 51394cf..65c5155 100644 --- a/src/migrator/full_migration.py +++ b/src/migrator/full_migration.py @@ -113,7 +113,8 @@ class FullMigrator: # Accumulate rows for batch insertion to reduce database round-trips insert_buffer = [] - insert_buffer_size = self.settings.migration.batch_size * 10 # Larger INSERT batches + # Use smaller batch size for more frequent updates: batch_size * 5 = 50k rows + insert_buffer_size = self.settings.migration.batch_size * 5 fetched_in_buffer = 0 # Track MySQL rows fetched (before consolidation) # Fetch consolidation groups from partition @@ -144,12 +145,14 @@ class FullMigrator: migrated += inserted batch_count += 1 progress.update(fetched_in_buffer) - - # Update state every 10 batches to track progress - if batch_count % 10 == 0: - self._update_migration_state( - pg_conn, migrated, None, migration_start_time - ) + # Update migration state after every batch flush + self._update_migration_state( + pg_conn, migrated, None, migration_start_time + ) + logger.debug( + f"Partition {partition}: flushed {inserted} rows, " + f"total migrated: {migrated}" + ) insert_buffer = [] fetched_in_buffer = 0 @@ -164,6 +167,10 @@ class FullMigrator: self._update_migration_state( pg_conn, migrated, None, migration_start_time ) + logger.debug( + f"Partition {partition} final flush: {inserted} rows, " + f"total migrated: {migrated}" + ) logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated")