diff --git a/.env.example b/.env.example index 4acb4e6..72e7f80 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ POSTGRES_DATABASE=migrated_db BATCH_SIZE=10000 LOG_LEVEL=INFO DRY_RUN=false +CONSOLIDATION_GROUP_LIMIT=50000 # Performance Testing BENCHMARK_OUTPUT_DIR=benchmark_results diff --git a/src/migrator/full_migration.py b/src/migrator/full_migration.py index e437f93..03dd014 100644 --- a/src/migrator/full_migration.py +++ b/src/migrator/full_migration.py @@ -105,8 +105,9 @@ class FullMigrator: partitions = mysql_conn.get_table_partitions(mysql_table) logger.info(f"Found {len(partitions)} partitions for {mysql_table}") - for partition in partitions: - logger.info(f"Processing partition {partition}...") + for partition_idx, partition in enumerate(partitions, 1): + logger.info(f"[{partition_idx}/{len(partitions)}] Processing partition {partition}...") + partition_group_count = 0 # Fetch consolidation groups from partition # Each group is a list of rows with the same (unit, tool, date, time) @@ -129,6 +130,7 @@ class FullMigrator: if inserted > 0: migrated += inserted batch_count += 1 + partition_group_count += 1 progress.update(len(group_rows)) # Update state every 10 consolidations to track progress @@ -137,6 +139,8 @@ class FullMigrator: pg_conn, migrated, None, migration_start_time ) + logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated") + # Get final actual count from PostgreSQL final_count = pg_conn.get_row_count(pg_table) logger.info(f"Final count from PostgreSQL: {final_count}")