fix: Update progress tracking to use MySQL row count instead of PostgreSQL count
The progress bar was appearing frozen because: - Total was set to MySQL rows to process (111M) - Progress was updated by PostgreSQL rows inserted (11M after consolidation) - This created a 10:1 mismatch, making progress appear to crawl Solution: - Track progress based on MySQL rows processed (matches total) - Use batch_size (MySQL rows) instead of inserted count (PostgreSQL rows) - Change batch_max_id calculation to use original batch instead of transformed This ensures the progress bar advances at a visible rate while still maintaining accurate row count tracking from the database. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -104,6 +104,9 @@ class FullMigrator:
|
|||||||
if not batch:
|
if not batch:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
# Track MySQL rows processed for progress (before consolidation)
|
||||||
|
batch_size = len(batch)
|
||||||
|
|
||||||
# Transform batch with consolidation enabled
|
# Transform batch with consolidation enabled
|
||||||
transformed = DataTransformer.transform_batch(
|
transformed = DataTransformer.transform_batch(
|
||||||
mysql_table,
|
mysql_table,
|
||||||
@@ -120,14 +123,14 @@ class FullMigrator:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if inserted > 0:
|
if inserted > 0:
|
||||||
# For consolidated batches, count transformed rows, not source rows
|
# Update progress based on MySQL rows processed
|
||||||
migrated += inserted
|
# (not PostgreSQL rows inserted, since consolidation reduces count)
|
||||||
progress.update(inserted)
|
progress.update(batch_size)
|
||||||
|
|
||||||
# Update state after each batch for resume capability
|
# Update state after each batch for resume capability
|
||||||
# Use MAX id of the batch (represents last MySQL id processed)
|
# Use MAX id of the batch (represents last MySQL id processed)
|
||||||
batch_max_id = max(
|
batch_max_id = max(
|
||||||
int(row.get("id", 0)) for row in transformed
|
int(row.get("id", 0)) for row in batch
|
||||||
)
|
)
|
||||||
# Get actual row count from PostgreSQL for accuracy
|
# Get actual row count from PostgreSQL for accuracy
|
||||||
actual_count = pg_conn.get_row_count(pg_table)
|
actual_count = pg_conn.get_row_count(pg_table)
|
||||||
@@ -135,12 +138,13 @@ class FullMigrator:
|
|||||||
pg_conn, actual_count, batch_max_id, migration_start_time
|
pg_conn, actual_count, batch_max_id, migration_start_time
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Get final actual count from PostgreSQL
|
||||||
|
final_count = pg_conn.get_row_count(pg_table)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"✓ Migration complete: {migrated} rows migrated "
|
f"✓ Migration complete: {final_count} total rows in {pg_table}"
|
||||||
f"to {pg_table}"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return migrated
|
return final_count
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Migration failed: {e}")
|
logger.error(f"Migration failed: {e}")
|
||||||
|
|||||||
Reference in New Issue
Block a user