From b6886293f60b4c619645467949d848754e7c7c8d Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 25 Dec 2025 22:10:43 +0100 Subject: [PATCH] Add detailed partition progress logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Log shows: - Current partition index and total ([X/Y]) - Partition name being processed - Number of groups consolidated per partition after completion This helps track migration progress when processing 18 partitions, making it easier to identify slow partitions or issues. 🤖 Generated with Claude Code Co-Authored-By: Claude Haiku 4.5 --- .env.example | 1 + src/migrator/full_migration.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 4acb4e6..72e7f80 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ POSTGRES_DATABASE=migrated_db BATCH_SIZE=10000 LOG_LEVEL=INFO DRY_RUN=false +CONSOLIDATION_GROUP_LIMIT=50000 # Performance Testing BENCHMARK_OUTPUT_DIR=benchmark_results diff --git a/src/migrator/full_migration.py b/src/migrator/full_migration.py index e437f93..03dd014 100644 --- a/src/migrator/full_migration.py +++ b/src/migrator/full_migration.py @@ -105,8 +105,9 @@ class FullMigrator: partitions = mysql_conn.get_table_partitions(mysql_table) logger.info(f"Found {len(partitions)} partitions for {mysql_table}") - for partition in partitions: - logger.info(f"Processing partition {partition}...") + for partition_idx, partition in enumerate(partitions, 1): + logger.info(f"[{partition_idx}/{len(partitions)}] Processing partition {partition}...") + partition_group_count = 0 # Fetch consolidation groups from partition # Each group is a list of rows with the same (unit, tool, date, time) @@ -129,6 +130,7 @@ class FullMigrator: if inserted > 0: migrated += inserted batch_count += 1 + partition_group_count += 1 progress.update(len(group_rows)) # Update state every 10 consolidations to track progress @@ -137,6 +139,8 @@ class FullMigrator: pg_conn, migrated, None, migration_start_time ) + logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated") + # Get final actual count from PostgreSQL final_count = pg_conn.get_row_count(pg_table) logger.info(f"Final count from PostgreSQL: {final_count}")