Add detailed partition progress logging

Log shows:
- Current partition index and total ([X/Y])
- Partition name being processed
- Number of groups consolidated per partition after completion

This helps track migration progress when processing 18 partitions,
making it easier to identify slow partitions or issues.

🤖 Generated with Claude Code

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-25 22:10:43 +01:00
parent 255fb1c520
commit b6886293f6
2 changed files with 7 additions and 2 deletions

View File

@@ -16,6 +16,7 @@ POSTGRES_DATABASE=migrated_db
BATCH_SIZE=10000
LOG_LEVEL=INFO
DRY_RUN=false
CONSOLIDATION_GROUP_LIMIT=50000
# Performance Testing
BENCHMARK_OUTPUT_DIR=benchmark_results

View File

@@ -105,8 +105,9 @@ class FullMigrator:
partitions = mysql_conn.get_table_partitions(mysql_table)
logger.info(f"Found {len(partitions)} partitions for {mysql_table}")
for partition in partitions:
logger.info(f"Processing partition {partition}...")
for partition_idx, partition in enumerate(partitions, 1):
logger.info(f"[{partition_idx}/{len(partitions)}] Processing partition {partition}...")
partition_group_count = 0
# Fetch consolidation groups from partition
# Each group is a list of rows with the same (unit, tool, date, time)
@@ -129,6 +130,7 @@ class FullMigrator:
if inserted > 0:
migrated += inserted
batch_count += 1
partition_group_count += 1
progress.update(len(group_rows))
# Update state every 10 consolidations to track progress
@@ -137,6 +139,8 @@ class FullMigrator:
pg_conn, migrated, None, migration_start_time
)
logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated")
# Get final actual count from PostgreSQL
final_count = pg_conn.get_row_count(pg_table)
logger.info(f"Final count from PostgreSQL: {final_count}")