Add detailed partition progress logging
Log shows: - Current partition index and total ([X/Y]) - Partition name being processed - Number of groups consolidated per partition after completion This helps track migration progress when processing 18 partitions, making it easier to identify slow partitions or issues. 🤖 Generated with Claude Code Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ POSTGRES_DATABASE=migrated_db
|
||||
BATCH_SIZE=10000
|
||||
LOG_LEVEL=INFO
|
||||
DRY_RUN=false
|
||||
CONSOLIDATION_GROUP_LIMIT=50000
|
||||
|
||||
# Performance Testing
|
||||
BENCHMARK_OUTPUT_DIR=benchmark_results
|
||||
|
||||
@@ -105,8 +105,9 @@ class FullMigrator:
|
||||
partitions = mysql_conn.get_table_partitions(mysql_table)
|
||||
logger.info(f"Found {len(partitions)} partitions for {mysql_table}")
|
||||
|
||||
for partition in partitions:
|
||||
logger.info(f"Processing partition {partition}...")
|
||||
for partition_idx, partition in enumerate(partitions, 1):
|
||||
logger.info(f"[{partition_idx}/{len(partitions)}] Processing partition {partition}...")
|
||||
partition_group_count = 0
|
||||
|
||||
# Fetch consolidation groups from partition
|
||||
# Each group is a list of rows with the same (unit, tool, date, time)
|
||||
@@ -129,6 +130,7 @@ class FullMigrator:
|
||||
if inserted > 0:
|
||||
migrated += inserted
|
||||
batch_count += 1
|
||||
partition_group_count += 1
|
||||
progress.update(len(group_rows))
|
||||
|
||||
# Update state every 10 consolidations to track progress
|
||||
@@ -137,6 +139,8 @@ class FullMigrator:
|
||||
pg_conn, migrated, None, migration_start_time
|
||||
)
|
||||
|
||||
logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated")
|
||||
|
||||
# Get final actual count from PostgreSQL
|
||||
final_count = pg_conn.get_row_count(pg_table)
|
||||
logger.info(f"Final count from PostgreSQL: {final_count}")
|
||||
|
||||
Reference in New Issue
Block a user