alex/mysql2postgres
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Use actual PostgreSQL row count for total_rows_migrated tracking

Browse Source 
Replace session-level counting with direct table COUNT queries to ensure
total_rows_migrated always reflects actual reality in PostgreSQL. This fixes
the discrepancy where the counter was only tracking rows from the current session
and didn't account for earlier insertions or duplicates from failed resume attempts.

Key improvements:
- Use get_row_count() after each batch to get authoritative total
- Preserve previous count on resume and accumulate across sessions
- Remove dependency on error-prone session-level counters
- Ensures migration_state.total_rows_migrated matches actual table row count

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
alex
2025-12-23 15:33:27 +01:00
parent b09cfcf9df
commit 0f217379ea
8 changed files with 646 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/transformers/data_transformer.py
View File

@@ -1,5 +1,5 @@
"""Data transformation from MySQL to PostgreSQL format."""
from typing import Dict, Any, List
from typing import Dict, Any, List, Tuple
from datetime import datetime, time, timedelta
from config import (
RAWDATACOR_COLUMNS,
@@ -45,17 +45,16 @@ class DataTransformer:
raise ValueError(f"Unsupported event_time type: {type(event_time)}")
@staticmethod
def transform_rawdatacor_row(mysql_row: Dict[str, Any]) -> Dict[str, Any]:
"""Transform a RAWDATACOR row from MySQL to PostgreSQL format.
def _build_measurement_for_node(mysql_row: Dict[str, Any]) -> Dict[str, Any]:
"""Build measurement object for a single node.
Args:
mysql_row: Row dictionary from MySQL
Returns:
Transformed row dictionary for PostgreSQL
Measurement dictionary for this node (without node key wrapper)
"""
# Create measurements JSONB
measurements = {}
measurement = {}
# Map Val0-ValF with their units
for i, val_col in enumerate(RAWDATACOR_COLUMNS["val_columns"]):
@@ -66,10 +65,31 @@ class DataTransformer:
# Only add to JSONB if value is not None
if value is not None:
measurements[str(i)] = {
"value": str(value),
"unit": unit if unit else None,
}
measurement[str(i)] = {"value": str(value)}
# Only add unit if it's not None (saves ~20% space)
if unit:
measurement[str(i)]["unit"] = unit
return measurement
@staticmethod
def transform_rawdatacor_row(mysql_row: Dict[str, Any], measurements: Dict[str, Any] = None) -> Dict[str, Any]:
"""Transform a RAWDATACOR row from MySQL to PostgreSQL format.
Args:
mysql_row: Row dictionary from MySQL
measurements: Pre-built measurements JSONB (for consolidated nodes).
If None, builds measurements from mysql_row.
Returns:
Transformed row dictionary for PostgreSQL
"""
# If measurements not provided, build from single row
if measurements is None:
node_num = mysql_row.get("NodeNum")
node_measurements = DataTransformer._build_measurement_for_node(mysql_row)
# Wrap with node number as key for consolidation compatibility
measurements = {str(node_num): node_measurements} if node_num is not None else {}
# Combine event_date and event_time into event_timestamp
event_date = mysql_row.get("EventDate")
@@ -94,11 +114,11 @@ class DataTransformer:
event_timestamp = None
# Create PostgreSQL row
# Note: node_num is now stored in measurements JSONB, not as separate column
pg_row = {
"id": mysql_row["id"],
"unit_name": mysql_row.get("UnitName"),
"tool_name_id": mysql_row["ToolNameID"],
"node_num": mysql_row["NodeNum"],
"event_timestamp": event_timestamp,
"bat_level": mysql_row["BatLevel"],
"temperature": mysql_row["Temperature"],
@@ -179,25 +199,103 @@ class DataTransformer:
return pg_row
@staticmethod
def consolidate_rawdatacor_batch(
rows: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
"""Consolidate RAWDATACOR rows by (unit_name, tool_name_id, event_timestamp).
Groups multiple nodes with the same key into a single row with measurements
keyed by node number. Uses MAX(id) as the consolidated row ID for proper resume.
Args:
rows: List of row dictionaries from MySQL, ordered by
(UnitName, ToolNameID, EventDate, EventTime, NodeNum)
Returns:
List of consolidated row dictionaries ready for transformation
"""
if not rows:
return []
# Group rows by consolidation key
groups = {}
group_order = [] # Track order of first appearance
for row in rows:
# Build consolidation key
unit_name = row.get("UnitName")
tool_name_id = row["ToolNameID"]
event_date = row.get("EventDate")
event_time = row.get("EventTime")
# Create a hashable key
key = (unit_name, tool_name_id, event_date, event_time)
if key not in groups:
groups[key] = []
group_order.append(key)
groups[key].append(row)
# Transform each group into a consolidated row
consolidated_rows = []
for key in group_order:
group_rows = groups[key]
# Build consolidated measurements with nodes as keys
consolidated_measurements = {}
for row in group_rows:
node_num = row.get("NodeNum")
node_measurements = DataTransformer._build_measurement_for_node(row)
# Store measurements with node number as key
consolidated_measurements[str(node_num)] = node_measurements
# Use the row with minimum id as template for other fields
min_id_row = min(group_rows, key=lambda r: r["id"])
# Use the row with maximum id for the consolidated row ID (for proper resume)
max_id_row = max(group_rows, key=lambda r: r["id"])
# Create consolidated row with pre-built measurements
consolidated_row = DataTransformer.transform_rawdatacor_row(
min_id_row,
measurements=consolidated_measurements
)
# Update id to MAX(id) of the group (represents last MySQL row processed)
consolidated_row["id"] = max_id_row["id"]
consolidated_rows.append(consolidated_row)
return consolidated_rows
@staticmethod
def transform_batch(
table: str,
rows: List[Dict[str, Any]]
rows: List[Dict[str, Any]],
consolidate: bool = False
) -> List[Dict[str, Any]]:
"""Transform a batch of rows from MySQL to PostgreSQL format.
Args:
table: Table name ('RAWDATACOR' or 'ELABDATADISP')
rows: List of row dictionaries from MySQL
consolidate: If True and table is RAWDATACOR, consolidate nodes
Returns:
List of transformed row dictionaries for PostgreSQL
"""
if table == "RAWDATACOR":
return [
DataTransformer.transform_rawdatacor_row(row)
for row in rows
]
if consolidate:
# Consolidate rows by key first, then they're already transformed
return DataTransformer.consolidate_rawdatacor_batch(rows)
else:
return [
DataTransformer.transform_rawdatacor_row(row)
for row in rows
]
elif table == "ELABDATADISP":
return [
DataTransformer.transform_elabdatadisp_row(row)
@@ -221,7 +319,6 @@ class DataTransformer:
"id",
"unit_name",
"tool_name_id",
"node_num",
"event_timestamp",
"bat_level",
"temperature",

6
src/transformers/schema_transformer.py
View File

@@ -16,11 +16,11 @@ def create_rawdatacor_schema() -> str:
CREATE SEQUENCE IF NOT EXISTS rawdatacor_id_seq;
-- Create RAWDATACOR table with partitioning
-- Note: node_num is stored in measurements JSONB, not as a separate column
CREATE TABLE IF NOT EXISTS rawdatacor (
id BIGINT NOT NULL DEFAULT nextval('rawdatacor_id_seq'),
unit_name VARCHAR(32),
tool_name_id VARCHAR(32) NOT NULL,
node_num INTEGER NOT NULL,
event_timestamp TIMESTAMP NOT NULL,
bat_level NUMERIC(4,2) NOT NULL,
temperature NUMERIC(5,2) NOT NULL,
@@ -55,8 +55,8 @@ CREATE TABLE IF NOT EXISTS rawdatacor_default
# Add indexes
sql += """
-- Create indexes
CREATE INDEX IF NOT EXISTS idx_unit_tool_node_datetime_raw
ON rawdatacor(unit_name, tool_name_id, node_num, event_timestamp);
CREATE INDEX IF NOT EXISTS idx_unit_tool_datetime_raw
ON rawdatacor(unit_name, tool_name_id, event_timestamp);
CREATE INDEX IF NOT EXISTS idx_unit_tool_raw
ON rawdatacor(unit_name, tool_name_id);