From 0cb4a0f71ec00de83d4b93939d075856a9b61ff2 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 23 Dec 2025 15:40:50 +0100 Subject: [PATCH] fix: Update progress tracking to use MySQL row count instead of PostgreSQL count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The progress bar was appearing frozen because: - Total was set to MySQL rows to process (111M) - Progress was updated by PostgreSQL rows inserted (11M after consolidation) - This created a 10:1 mismatch, making progress appear to crawl Solution: - Track progress based on MySQL rows processed (matches total) - Use batch_size (MySQL rows) instead of inserted count (PostgreSQL rows) - Change batch_max_id calculation to use original batch instead of transformed This ensures the progress bar advances at a visible rate while still maintaining accurate row count tracking from the database. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Haiku 4.5 --- src/migrator/full_migration.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/migrator/full_migration.py b/src/migrator/full_migration.py index f79acd5..84f1ca3 100644 --- a/src/migrator/full_migration.py +++ b/src/migrator/full_migration.py @@ -104,6 +104,9 @@ class FullMigrator: if not batch: break + # Track MySQL rows processed for progress (before consolidation) + batch_size = len(batch) + # Transform batch with consolidation enabled transformed = DataTransformer.transform_batch( mysql_table, @@ -120,14 +123,14 @@ class FullMigrator: ) if inserted > 0: - # For consolidated batches, count transformed rows, not source rows - migrated += inserted - progress.update(inserted) + # Update progress based on MySQL rows processed + # (not PostgreSQL rows inserted, since consolidation reduces count) + progress.update(batch_size) # Update state after each batch for resume capability # Use MAX id of the batch (represents last MySQL id processed) batch_max_id = max( - int(row.get("id", 0)) for row in transformed + int(row.get("id", 0)) for row in batch ) # Get actual row count from PostgreSQL for accuracy actual_count = pg_conn.get_row_count(pg_table) @@ -135,12 +138,13 @@ class FullMigrator: pg_conn, actual_count, batch_max_id, migration_start_time ) + # Get final actual count from PostgreSQL + final_count = pg_conn.get_row_count(pg_table) logger.info( - f"✓ Migration complete: {migrated} rows migrated " - f"to {pg_table}" + f"✓ Migration complete: {final_count} total rows in {pg_table}" ) - return migrated + return final_count except Exception as e: logger.error(f"Migration failed: {e}")