alex/mysql2postgres
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Only update last_completed_partition when partition is fully processed

Browse Source 
Previously, last_completed_partition was updated during batch flushes while
the partition was still being processed. This caused resume to skip partitions
that were only partially completed.

Now, last_completed_partition is only updated AFTER all consolidation groups
in a partition have been processed and the final buffer flush is complete.

🤖 Generated with Claude Code

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
alex
2025-12-26 00:49:14 +01:00
parent 32f90fdd47
commit 6ca97f0ba4
4 changed files with 195 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/migrator/full_migration.py
View File

@@ -166,9 +166,9 @@ class FullMigrator:
batch_count += 1
progress.update(fetched_in_buffer)
# Update migration state after every batch flush
# Do NOT set last_completed_partition yet - partition is still being processed
self._update_migration_state(
pg_conn, migrated, None, migration_start_time,
last_partition=partition
pg_conn, migrated, None, migration_start_time
)
logger.debug(
f"Partition {partition}: flushed {inserted} rows, "
@@ -185,16 +185,21 @@ class FullMigrator:
migrated += inserted
batch_count += 1
progress.update(fetched_in_buffer)
# Still don't set last_completed_partition - partition is still being finalized
self._update_migration_state(
pg_conn, migrated, None, migration_start_time,
last_partition=partition
pg_conn, migrated, None, migration_start_time
)
logger.debug(
f"Partition {partition} final flush: {inserted} rows, "
f"total migrated: {migrated}"
)
# NOW partition is complete - update with completed partition
logger.info(f"Partition {partition} complete: {partition_group_count} groups consolidated")
self._update_migration_state(
pg_conn, migrated, None, migration_start_time,
last_partition=partition
)
# Get final actual count from PostgreSQL
final_count = pg_conn.get_row_count(pg_table)