fix incremental

config.py

@@ -52,6 +52,7 @@ class MigrationSettings(BaseSettings):
     consolidation_group_limit: int = 10000
     log_level: str = "INFO"
     dry_run: bool = False
+    progress_log_interval: int = 50000
 
 
 class BenchmarkSettings(BaseSettings):
@@ -152,20 +153,32 @@ ELABDATADISP_FIELD_MAPPING = {
 # PostgreSQL Partition years (from both tables)
 PARTITION_YEARS = list(range(2014, 2032))  # 2014-2031
 
+# Consolidation key definition (same for both tables)
+# Multiple MySQL rows with same key but different NodeNum → 1 PostgreSQL row
+# MySQL source fields
+CONSOLIDATION_KEY_FIELDS = ["UnitName", "ToolNameID", "EventDate", "EventTime"]
+# Keys for tracking in migration_state.last_key (NOT actual PostgreSQL target columns)
+# Note: In PostgreSQL target, EventDate+EventTime become event_timestamp
+CONSOLIDATION_KEY_PG_FIELDS = ["unit_name", "tool_name_id", "event_date", "event_time"]
+
 # Table configurations - support both uppercase and lowercase keys
 _rawdatacor_config = {
     "mysql_table": "RAWDATACOR",
     "postgres_table": "rawdatacor",
-    "primary_key": "id",
-    "postgres_pk": "id",  # Primary key column name in PostgreSQL
-    "partition_key": "event_timestamp",
+    "mysql_pk": "id",  # MySQL primary key
+    "postgres_pk": "id",  # PostgreSQL auto-increment primary key
+    "mysql_max_id_field": "id",  # Field to track max ID from MySQL
+    "consolidation_key": CONSOLIDATION_KEY_FIELDS,
+    "consolidation_key_pg": CONSOLIDATION_KEY_PG_FIELDS,
 }
 _elabdatadisp_config = {
     "mysql_table": "ELABDATADISP",
     "postgres_table": "elabdatadisp",
-    "primary_key": "idElabData",
-    "postgres_pk": "id_elab_data",  # Primary key column name in PostgreSQL
-    "partition_key": "event_timestamp",
+    "mysql_pk": "idElabData",  # MySQL primary key
+    "postgres_pk": "id",  # PostgreSQL auto-increment primary key
+    "mysql_max_id_field": "idElabData",  # Field to track max ID from MySQL
+    "consolidation_key": CONSOLIDATION_KEY_FIELDS,
+    "consolidation_key_pg": CONSOLIDATION_KEY_PG_FIELDS,
 }
 
 TABLE_CONFIGS = {