"""Configuration management using Pydantic settings.""" from pydantic_settings import BaseSettings from pydantic import ConfigDict from typing import Optional import os class MySQLConfig(BaseSettings): """MySQL source database configuration.""" model_config = ConfigDict( env_prefix="MYSQL_", case_sensitive=False, extra="ignore", env_file=".env", env_file_encoding="utf-8" ) host: str port: int user: str password: str database: str class PostgreSQLConfig(BaseSettings): """PostgreSQL target database configuration.""" model_config = ConfigDict( env_prefix="POSTGRES_", case_sensitive=False, extra="ignore", env_file=".env", env_file_encoding="utf-8" ) host: str port: int user: str password: str database: str class MigrationSettings(BaseSettings): """Migration settings.""" model_config = ConfigDict( case_sensitive=False, extra="ignore", env_file=".env", env_file_encoding="utf-8" ) batch_size: int = 10000 consolidation_group_limit: int = 10000 log_level: str = "INFO" dry_run: bool = False progress_log_interval: int = 50000 class BenchmarkSettings(BaseSettings): """Benchmark settings.""" model_config = ConfigDict( env_prefix="BENCHMARK_", case_sensitive=False, extra="ignore", env_file=".env", env_file_encoding="utf-8" ) output_dir: str = "benchmark_results" iterations: int = 5 class Settings(BaseSettings): """All application settings.""" model_config = ConfigDict( env_file=".env", env_file_encoding="utf-8", case_sensitive=False, extra="ignore" ) mysql: MySQLConfig postgres: PostgreSQLConfig migration: MigrationSettings benchmark: BenchmarkSettings @classmethod def from_env(cls): """Load settings from environment variables.""" return cls( mysql=MySQLConfig(), postgres=PostgreSQLConfig(), migration=MigrationSettings(), benchmark=BenchmarkSettings(), ) # Lazy load settings _settings: Optional[Settings] = None def get_settings() -> Settings: """Get application settings, loading from .env if necessary.""" global _settings if _settings is None: _settings = Settings.from_env() return _settings # Schema transformation definitions RAWDATACOR_COLUMNS = { "val_columns": ["Val0", "Val1", "Val2", "Val3", "Val4", "Val5", "Val6", "Val7", "Val8", "Val9", "ValA", "ValB", "ValC", "ValD", "ValE", "ValF"], "unit_columns": ["Val0_unitmisure", "Val1_unitmisure", "Val2_unitmisure", "Val3_unitmisure", "Val4_unitmisure", "Val5_unitmisure", "Val6_unitmisure", "Val7_unitmisure", "Val8_unitmisure", "Val9_unitmisure", "ValA_unitmisure", "ValB_unitmisure", "ValC_unitmisure", "ValD_unitmisure", "ValE_unitmisure", "ValF_unitmisure"], } ELABDATADISP_MEASUREMENT_FIELDS = { "shifts": ["XShift", "YShift", "ZShift", "HShift", "HShiftDir", "HShift_local"], "coordinates": ["X", "Y", "Z", "Xstar", "Zstar"], "kinematics": ["speed", "speed_local", "acceleration", "acceleration_local"], "sensors": ["T_node", "load_value", "water_level", "pressure"], "calculated": ["AlfaX", "AlfaY", "Area"], } ELABDATADISP_FIELD_MAPPING = { # shifts mapping (source -> (category, key)) "XShift": ("shifts", "x"), "YShift": ("shifts", "y"), "ZShift": ("shifts", "z"), "HShift": ("shifts", "h"), "HShiftDir": ("shifts", "h_dir"), "HShift_local": ("shifts", "h_local"), # coordinates mapping "X": ("coordinates", "x"), "Y": ("coordinates", "y"), "Z": ("coordinates", "z"), "Xstar": ("coordinates", "x_star"), "Zstar": ("coordinates", "z_star"), # kinematics mapping "speed": ("kinematics", "speed"), "speed_local": ("kinematics", "speed_local"), "acceleration": ("kinematics", "acceleration"), "acceleration_local": ("kinematics", "acceleration_local"), # sensors mapping "T_node": ("sensors", "t_node"), "load_value": ("sensors", "load_value"), "water_level": ("sensors", "water_level"), "pressure": ("sensors", "pressure"), # calculated mapping "AlfaX": ("calculated", "alfa_x"), "AlfaY": ("calculated", "alfa_y"), "Area": ("calculated", "area"), } # PostgreSQL Partition years (from both tables) PARTITION_YEARS = list(range(2014, 2032)) # 2014-2031 # Consolidation key definition (same for both tables) # Multiple MySQL rows with same key but different NodeNum → 1 PostgreSQL row # MySQL source fields CONSOLIDATION_KEY_FIELDS = ["UnitName", "ToolNameID", "EventDate", "EventTime"] # Keys for tracking in migration_state.last_key (NOT actual PostgreSQL target columns) # Note: In PostgreSQL target, EventDate+EventTime become event_timestamp CONSOLIDATION_KEY_PG_FIELDS = ["unit_name", "tool_name_id", "event_date", "event_time"] # Table configurations - support both uppercase and lowercase keys _rawdatacor_config = { "mysql_table": "RAWDATACOR", "postgres_table": "rawdatacor", "mysql_pk": "id", # MySQL primary key "postgres_pk": "id", # PostgreSQL auto-increment primary key "mysql_max_id_field": "id", # Field to track max ID from MySQL "consolidation_key": CONSOLIDATION_KEY_FIELDS, "consolidation_key_pg": CONSOLIDATION_KEY_PG_FIELDS, } _elabdatadisp_config = { "mysql_table": "ELABDATADISP", "postgres_table": "elabdatadisp", "mysql_pk": "idElabData", # MySQL primary key "postgres_pk": "id", # PostgreSQL auto-increment primary key "mysql_max_id_field": "idElabData", # Field to track max ID from MySQL "consolidation_key": CONSOLIDATION_KEY_FIELDS, "consolidation_key_pg": CONSOLIDATION_KEY_PG_FIELDS, } TABLE_CONFIGS = { "rawdatacor": _rawdatacor_config, "RAWDATACOR": _rawdatacor_config, "elabdatadisp": _elabdatadisp_config, "ELABDATADISP": _elabdatadisp_config, }