alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: alex:dc6ccc1744d37a4ebd35257078cf6fc187c5343a
Branches Tags
alex:main-stable alex:main
...
compare: alex:cc7a136cf3ffe9048a531aa6d6cc31690ad5cc69
Branches Tags
alex:main-stable alex:main

2 Commits
dc6ccc1744 ... cc7a136cf3

Author SHA1 Message Date
alex
cc7a136cf3 refactory 2025-04-28 22:29:35 +02:00
alex
40ef173694 evol 2 2025-04-27 17:21:36 +02:00
30 changed files with 632 additions and 464 deletions
Expand all files Collapse all files

2
CsvLoader.py
View File

@@ -11,7 +11,7 @@ import shutil
from utils.time import timestamp_fmt as ts
from utils.time import date_refmt as df
from utils.config import set_config as setting
from utils.config import loader as setting
class sqlraw:

316
FtpCsvReceiver.py
View File

@@ -1,316 +0,0 @@
#!.venv/bin/python
"""This module implements an FTP server with custom commands for managing virtual users and handling CSV file uploads."""
import sys
import os
# import ssl
import re
import logging
import mysql.connector
from hashlib import sha256
from pathlib import Path
from utils.time import timestamp_fmt as ts
from utils.config import set_config as setting
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
def conn_db(cfg):
"""Establishes a connection to the MySQL database.
Args:
cfg: The configuration object containing database connection details.
Returns:
A MySQL database connection object.
"""
return mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
def extract_value(patterns, primary_source, secondary_source, default='Not Defined'):
"""Extracts the first match for a list of patterns from the primary source.
Falls back to the secondary source if no match is found.
"""
for source in (primary_source, secondary_source):
for pattern in patterns:
matches = re.findall(pattern, source, re.IGNORECASE)
if matches:
return matches[0] # Return the first match immediately
return default # Return default if no matches are found
class DummySha256Authorizer(DummyAuthorizer):
"""Custom authorizer that uses SHA256 for password hashing and manages users from a database."""
def __init__(self, cfg):
"""Initializes the authorizer, adds the admin user, and loads users from the database.
Args:
cfg: The configuration object.
"""
super().__init__()
self.add_user(
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
# Define the database connection
conn = conn_db(cfg)
# Create a cursor
cur = conn.cursor()
cur.execute(f'SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL')
for ftpuser, hash, virtpath, perm in cur.fetchall():
self.add_user(ftpuser, hash, virtpath, perm)
"""
Create the user's directory if it does not exist.
"""
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
def validate_authentication(self, username, password, handler):
# Validate the user's password against the stored hash
hash = sha256(password.encode("UTF-8")).hexdigest()
try:
if self.user_table[username]["pwd"] != hash:
raise KeyError
except KeyError:
raise AuthenticationFailed
class ASEHandler(FTPHandler):
"""Custom FTP handler that extends FTPHandler with custom commands and file handling."""
def __init__(self, conn, server, ioloop=None):
"""Initializes the handler, adds custom commands, and sets up command permissions.
Args:
conn: The connection object.
server: The FTP server object.
ioloop: The I/O loop object.
"""
super().__init__(conn, server, ioloop)
self.proto_cmds = FTPHandler.proto_cmds.copy()
# Add custom FTP commands for managing virtual users - command in lowercase
self.proto_cmds.update(
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
)
self.proto_cmds.update(
{'SITE DISU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> DISU USERNAME (disable virtual user).')}
)
self.proto_cmds.update(
{'SITE ENAU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ENAU USERNAME (enable virtual user).')}
)
self.proto_cmds.update(
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
help='Syntax: SITE <SP> LSTU (list virtual users).')}
)
def on_file_received(self, file):
"""Handles the event when a file is successfully received.
Args:
file: The path to the received file.
"""
if not os.stat(file).st_size:
os.remove(file)
logging.info(f'File {file} was empty: removed.')
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if (fileExtension.upper() in (cfg.fileext)):
with open(file, 'r') as csvfile:
lines = csvfile.readlines()
unit_name = extract_value(cfg.units_name, filename, str(lines[0:9]))
unit_type = extract_value(cfg.units_type, filename, str(lines[0:9]))
tool_name = extract_value(cfg.tools_name, filename, str(lines[0:9]))
tool_type = extract_value(cfg.tools_type, filename, str(lines[0:9]))
conn = conn_db(cfg)
# Create a cursor
cur = conn.cursor()
try:
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (filename, unit_name, unit_type, tool_name, tool_type, tool_data) VALUES (%s, %s, %s, %s, %s, %s)", (filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines)))
conn.commit()
conn.close()
except Exception as e:
logging.error(f'File {file} not loaded. Held in user path.')
logging.error(f'{e}')
else:
os.remove(file)
logging.info(f'File {file} loaded: removed.')
def on_incomplete_file_received(self, file):
"""Removes partially uploaded files.
Args:
file: The path to the incomplete file.
"""
os.remove(file)
def ftp_SITE_ADDU(self, line):
"""Adds a virtual user, creates their directory, and saves their details to the database.
"""
cfg = self.cfg
try:
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = sha256(password.encode("UTF-8")).hexdigest() # Hash the password
except IndexError:
self.respond('501 SITE ADDU failed. Command needs 2 arguments')
else:
try:
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.respond(f'551 Error in create virtual user path: {e}')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm=cfg.defperm)
# Save the user to the database
# Define the database connection
conn = conn_db(cfg)
# Create a cursor
cur = conn.cursor()
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbusertable} (ftpuser, hash, virtpath, perm) VALUES ('{user}', '{hash}', '{cfg.virtpath + user}', '{cfg.defperm}')")
conn.commit()
conn.close()
logging.info(f"User {user} created.")
self.respond('200 SITE ADDU successful.')
except Exception as e:
self.respond(f'501 SITE ADDU failed: {e}.')
print(e)
def ftp_SITE_DISU(self, line):
"""Removes a virtual user from the authorizer and marks them as deleted in the database."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Remove the user from the authorizer
self.authorizer.remove_user(str(user))
# Delete the user from database
conn = conn_db(cfg)
# Crea un cursore
cur = conn.cursor()
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = now() WHERE ftpuser = '{user}'")
conn.commit()
conn.close()
logging.info(f"User {user} deleted.")
self.respond('200 SITE DISU successful.')
except Exception as e:
self.respond('501 SITE DISU failed.')
print(e)
def ftp_SITE_ENAU(self, line):
"""Restores a virtual user by updating their status in the database and adding them back to the authorizer."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Restore the user into database
conn = conn_db(cfg)
# Crea un cursore
cur = conn.cursor()
try:
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = null WHERE ftpuser = '{user}'")
conn.commit()
except Exception as e:
logging.error(f"Update DB failed: {e}")
cur.execute(f"SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE ftpuser = '{user}'")
ftpuser, hash, virtpath, perm = cur.fetchone()
self.authorizer.add_user(ftpuser, hash, virtpath, perm)
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
conn.close()
logging.info(f"User {user} restored.")
self.respond('200 SITE ENAU successful.')
except Exception as e:
self.respond('501 SITE ENAU failed.')
print(e)
def ftp_SITE_LSTU(self, line):
"""Lists all virtual users from the database."""
cfg = self.cfg
users_list = []
try:
# Connect to the SQLite database to fetch users
conn = conn_db(cfg)
# Crea un cursore
cur = conn.cursor()
self.push("214-The following virtual users are defined:\r\n")
cur.execute(f'SELECT ftpuser, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL ')
[users_list.append(f'Username: {ftpuser}\tPerms: {perm}\r\n') for ftpuser, perm in cur.fetchall()]
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
except Exception as e:
self.respond(f'501 list users failed: {e}')
def main():
"""Main function to start the FTP server."""
# Load the configuration settings
cfg = setting.config()
try:
# Initialize the authorizer and handler
authorizer = DummySha256Authorizer(cfg)
handler = ASEHandler
handler.cfg = cfg
handler.authorizer = authorizer
handler.masquerade_address = cfg.proxyaddr
# Set the range of passive ports for the FTP server
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
handler.passive_ports = _range
# Configure logging
logging.basicConfig(
format="%(asctime)s %(message)s",
filename=cfg.logfilename,
level=logging.INFO,
)
# Create and start the FTP server
server = FTPServer(("0.0.0.0", 2121), handler)
server.serve_forever()
except KeyboardInterrupt:
logging.info(
"Info: Shutdown requested...exiting"
)
except Exception:
print(
f"{ts.timestamp("log")} - PID {os.getpid():>5} >> Error: {sys.exc_info()[1]}."
)
if __name__ == "__main__":
main()

38
dbddl/dataraw.ddl
View File

@@ -1,38 +0,0 @@
CREATE TABLE public.dataraw
(
id serial4 NOT NULL,
unit_name text NULL,
unit_type text NULL,
tool_name text NULL,
tool_type text NULL,
unit_ip text NULL,
unit_subnet text NULL,
unit_gateway text NULL,
event_timestamp timestamp NULL,
battery_level float8 NULL,
temperature float8 NULL,
nodes_jsonb jsonb NULL,
created_at timestamp DEFAULT CURRENT_TIMESTAMP NULL,
updated_at timestamp NULL,
CONSTRAINT dataraw_pk PRIMARY KEY (id),
CONSTRAINT dataraw_unique UNIQUE (unit_name, tool_name, event_timestamp)
);
CREATE OR REPLACE FUNCTION public.update_updated_at_column()
RETURNS trigger
LANGUAGE plpgsql
AS $function$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$function$
;
CREATE TRIGGER update_updated_at BEFORE
UPDATE
ON dataraw FOR EACH ROW
EXECUTE PROCEDURE
update_updated_at_column();

34
dbddl/rawdatacor.ddl Normal file
View File

@@ -0,0 +1,34 @@
CREATE TABLE `RAWDATACOR` (
`id` int NOT NULL AUTO_INCREMENT,
`UnitName` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ToolNameID` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
`NodeNum` int NOT NULL,
`EventDate` date NOT NULL,
`EventTime` time NOT NULL,
`BatLevel` decimal(4,2) NOT NULL,
`Temperature` decimal(5,2) NOT NULL,
`Val0` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val1` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val2` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val3` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val4` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val5` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val6` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val7` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val8` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`Val9` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValA` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValB` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValC` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValD` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValE` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`ValF` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`BatLevelModule` decimal(4,2) DEFAULT NULL,
`TemperatureModule` decimal(5,2) DEFAULT NULL,
`RssiModule` int DEFAULT NULL,
PRIMARY KEY (`id`,`EventDate`),
UNIQUE KEY `idx_ToolNodeDateTime` (`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`),
KEY `UnitToolName` (`UnitName`,`ToolNameID`) USING BTREE,
KEY `ToolNameNameNode` (`ToolNameID`,`NodeNum`,`UnitName`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci

31
dbddl/received.ddl
View File

@@ -1,17 +1,16 @@
DROP TABLE public.received;
DROP TABLE ase_lar.received;
CREATE TABLE public.received
(
id serial4 NOT NULL,
filename text NULL,
unit_name text NULL,
unit_type text NULL,
tool_name text NULL,
tool_type text NULL,
tool_data text NULL,
"locked" int2 DEFAULT 0 NULL,
status int2 DEFAULT 0 NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
loaded_at timestamptz NULL,
CONSTRAINT received_pk PRIMARY KEY (id)
);
CREATE TABLE `received` (
`id` int NOT NULL AUTO_INCREMENT,
`filename` varchar(100) COLLATE utf8mb4_general_ci NOT NULL,
`unit_name` varchar(10) COLLATE utf8mb4_general_ci NOT NULL,
`unit_type` varchar(20) COLLATE utf8mb4_general_ci NOT NULL,
`tool_name` varchar(10) COLLATE utf8mb4_general_ci NOT NULL,
`tool_type` varchar(20) COLLATE utf8mb4_general_ci NOT NULL,
`tool_data` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`locked` int DEFAULT '0',
`status` int DEFAULT '0',
`inserted_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`loaded_at` datetime DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=694 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

25
dbddl/virtusers.ddl
View File

@@ -1,14 +1,13 @@
DROP TABLE public.virtusers
DROP TABLE ase_lar.virtusers
CREATE TABLE public.virtusers
(
id serial4 NOT NULL,
ftpuser text NOT NULL,
hash text NOT NULL,
virtpath text NOT NULL,
perm text NOT NULL,
defined_at timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
deleted_at timestamptz NULL,
CONSTRAINT virtusers_pk PRIMARY KEY (id),
CONSTRAINT virtusers_unique UNIQUE (ftpuser)
);
CREATE TABLE `virtusers` (
`id` int NOT NULL AUTO_INCREMENT,
`ftpuser` varchar(20) COLLATE utf8mb4_general_ci NOT NULL,
`hash` varchar(100) COLLATE utf8mb4_general_ci NOT NULL,
`virtpath` varchar(100) COLLATE utf8mb4_general_ci NOT NULL,
`perm` varchar(20) COLLATE utf8mb4_general_ci NOT NULL,
`defined_at` datetime DEFAULT CURRENT_TIMESTAMP,
`disabled_at` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `virtusers_unique` (`ftpuser`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

145
ftp_csv_receiver.py Executable file
View File

@@ -0,0 +1,145 @@
#!.venv/bin/python
"""This module implements an FTP server with custom commands for managing virtual users and handling CSV file uploads."""
import os
import logging
from hashlib import sha256
from pathlib import Path
from utils.config import loader
from utils.database.connection import connetti_db
from utils.ftp import user_admin, file_management
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
logger = logging.getLogger(__name__)
class DummySha256Authorizer(DummyAuthorizer):
"""Custom authorizer that uses SHA256 for password hashing and manages users from a database."""
def __init__(self, cfg):
"""Initializes the authorizer, adds the admin user, and loads users from the database.
Args:
cfg: The configuration object.
"""
super().__init__()
self.add_user(
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
# Define the database connection
conn = connetti_db(cfg)
# Create a cursor
cur = conn.cursor()
cur.execute(f'SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL')
for ftpuser, hash, virtpath, perm in cur.fetchall():
self.add_user(ftpuser, hash, virtpath, perm)
"""
Create the user's directory if it does not exist.
"""
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
def validate_authentication(self, username, password, handler):
# Validate the user's password against the stored hash
hash = sha256(password.encode("UTF-8")).hexdigest()
try:
if self.user_table[username]["pwd"] != hash:
raise KeyError
except KeyError:
raise AuthenticationFailed
class ASEHandler(FTPHandler):
"""Custom FTP handler that extends FTPHandler with custom commands and file handling."""
def __init__(self, conn, server, ioloop=None):
"""Initializes the handler, adds custom commands, and sets up command permissions.
Args:
conn: The connection object.
server: The FTP server object.
ioloop: The I/O loop object.
"""
super().__init__(conn, server, ioloop)
self.proto_cmds = FTPHandler.proto_cmds.copy()
# Add custom FTP commands for managing virtual users - command in lowercase
self.proto_cmds.update(
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
)
self.proto_cmds.update(
{'SITE DISU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> DISU USERNAME (disable virtual user).')}
)
self.proto_cmds.update(
{'SITE ENAU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ENAU USERNAME (enable virtual user).')}
)
self.proto_cmds.update(
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
help='Syntax: SITE <SP> LSTU (list virtual users).')}
)
def on_file_received(self, file):
return file_management.on_file_received(self, file)
def on_incomplete_file_received(self, file):
"""Removes partially uploaded files.
Args:
file: The path to the incomplete file.
"""
os.remove(file)
def ftp_SITE_ADDU(self, line):
return user_admin.ftp_SITE_ADDU(self, line)
def ftp_SITE_DISU(self, line):
return user_admin.ftp_SITE_DISU(self, line)
def ftp_SITE_ENAU(self, line):
return user_admin.ftp_SITE_ENAU(self, line)
def ftp_SITE_LSTU(self, line):
return user_admin.ftp_SITE_LSTU(self, line)
def main():
"""Main function to start the FTP server."""
# Load the configuration settings
cfg = loader.Config()
try:
# Initialize the authorizer and handler
authorizer = DummySha256Authorizer(cfg)
handler = ASEHandler
handler.cfg = cfg
handler.authorizer = authorizer
handler.masquerade_address = cfg.proxyaddr
# Set the range of passive ports for the FTP server
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
handler.passive_ports = _range
# Configure logging
logging.basicConfig(
format="%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s ",
filename=cfg.logfilename,
level=logging.INFO,
)
# Create and start the FTP server
server = FTPServer(("0.0.0.0", 2121), handler)
server.serve_forever()
except Exception as e:
logger.error(
f"Exit with error: {e}."
)
if __name__ == "__main__":
main()

2
ftpcsvreceiver.ini
View File

@@ -29,7 +29,7 @@
dbSchema = public
userTableName = virtusers
recTableName = received
rawTableName = dataraw
rawTableName = rawdatacor
[unit]
Types = G801|G201|G301|G802|D2W|GFLOW|CR1000X|TLP|GS1

79
orchestrator.py Executable file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env python3
import mysql.connector
import os
import sys
import logging
import importlib
from utils.time import timestamp_fmt as ts
from utils.config import loader as setting
#from unit_tool_mod import g801_mums, g801_mux
def conn_db(cfg):
"""Establishes a connection to the MySQL database.
Args:
cfg: The configuration object containing database connection details.
Returns:
A MySQL database connection object.
"""
try:
conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
conn.autocommit = True
return conn
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
return None
def elab_csv(cfg):
try:
with conn_db(cfg) as conn:
cur = conn.cursor()
cur.execute(f'select id, unit_name, unit_type, tool_name, tool_type, tool_data from {cfg.dbname}.{cfg.dbrectable} where locked = 0 and status = 0 limit 1')
id, unit_name, unit_type, tool_name, tool_type, tool_data = cur.fetchone()
cur.execute(f'update {cfg.dbname}.{cfg.dbrectable} set locked = 1 where id = {id}')
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
module_name = f'unit_tool_mod.{unit_type.lower()}_{tool_type.lower()}'
modulo = importlib.import_module(module_name)
funzione = getattr(modulo, "chi_sono")
# Chiamare la funzione
risultato = funzione(unit_name, tool_name)
print(f'risultato: {risultato}')
def main():
# Load the configuration settings
cfg = setting.config()
try:
# Configura la connessione al database PostgreSQL
# Configure logging
logging.basicConfig(
format="%(asctime)s %(message)s",
filename=cfg.elablog,
level=logging.INFO,
)
elab_csv(cfg)
except KeyboardInterrupt:
logging.info(
"Info: {}.".format("Shutdown requested...exiting")
)
except Exception:
print(
"{} - PID {:>5} >> Error: {}.".format(
ts.timestamp("log"), os.getpid(), sys.exc_info()[1]
)
)
if __name__ == "__main__":
main()

15
prova_no_pd.py
View File

@@ -1,11 +1,14 @@
import mysql.connector
import utils.datefmt.date_check as date_check
import utils.timestamp.date_check as date_check
righe = ["17/03/2022 15:10;13.7;14.8;|;401;832;17373;-8;920;469;9;|;839;133;17116;675;941;228;10;|;-302;-1252;17165;288;75;-940;10;|;739;76;17203;562;879;604;9;|;1460;751;16895;672;1462;132;10;|;-1088;-1883;16675;244;1071;518;10;|;-29;-1683;16923;384;1039;505;11;|;1309;-1095;17066;-36;324;-552;10;|;-36;-713;16701;-121;372;122;10;|;508;-1318;16833;475;1154;405;10;|;1178;878;17067;636;1114;428;10;|;1613;-573;17243;291;-234;-473;9;|;-107;-259;17287;94;421;369;10;|;-900;-647;16513;168;1330;252;10;|;1372;286;17035;202;263;469;10;|;238;-2006;17142;573;1201;492;9;|;2458;589;17695;356;187;208;11;|;827;-1085;17644;308;233;66;10;|;1;-1373;17214;557;1279;298;9;|;-281;-244;17071;209;517;-36;10;|;-486;-961;17075;467;440;367;10;|;1264;-339;16918;374;476;116;8;|;661;-1330;16789;-37;478;15;9;|;1208;-724;16790;558;1303;335;8;|;-236;-1404;16678;309;426;376;8;|;367;-1402;17308;-32;428;-957;7;|;-849;-360;17640;1;371;635;7;|;-784;90;17924;533;128;-661;5;|;-723;-1062;16413;270;-79;702;7;|;458;-1235;16925;354;-117;194;5;|;-411;-1116;17403;280;777;530;1;;;;;;;;;;;;;;",
"17/03/2022 15:13;13.6;14.8;|;398;836;17368;-3;924;472;9;|;838;125;17110;675;938;230;10;|;-298;-1253;17164;290;75;-942;10;|;749;78;17221;560;883;601;9;|;1463;752;16904;673;1467;134;10;|;-1085;-1884;16655;239;1067;520;10;|;-27;-1680;16923;393;1032;507;10;|;1308;-1095;17065;-43;328;-548;10;|;-38;-712;16704;-124;373;122;10;|;512;-1318;16830;473;1155;408;10;|;1181;879;17070;637;1113;436;10;|;1610;-567;17239;287;-240;-462;10;|;-108;-250;17297;94;420;370;10;|;-903;-652;16518;169;1326;257;9;|;1371;282;17047;198;263;471;10;|;244;-2006;17137;570;1205;487;9;|;2461;589;17689;354;199;210;11;|;823;-1081;17642;310;235;68;10;|;1;-1370;17214;560;1278;290;9;|;-280;-245;17062;209;517;-31;9;|;-484;-963;17074;463;440;374;10;|;1271;-340;16912;374;477;125;8;|;668;-1331;16786;-37;478;7;9;|;1209;-724;16784;557;1301;329;8;|;-237;-1406;16673;316;425;371;8;|;371;-1401;17307;-30;429;-961;7;|;-854;-356;17647;7;368;631;7;|;-781;85;17934;531;130;-664;5;|;-726;-1062;16400;274;-79;707;6;|;460;-1233;16931;355;-113;196;5;|;-413;-1119;17405;280;780;525;1",
"17/03/2022 15:28;13.6;14.3;|;396;832;17379;-3;919;470;10;|;837;128;17114;670;945;233;10;|;-304;-1246;17167;292;77;-931;10;|;744;70;17211;567;888;601;9;|;1459;748;16893;672;1480;141;10;|;-1084;-1887;16658;236;1068;522;10;|;-29;-1686;16912;388;1035;500;10;|;1312;-1092;17062;-35;328;-545;10;|;-40;-709;16701;-120;374;121;10;|;515;-1327;16826;475;1148;402;10;|;1179;881;17063;635;1114;430;9;|;1613;-568;17246;293;-230;-461;9;|;-103;-265;17289;96;420;363;10;|;-896;-656;16522;167;1320;250;10;|;1368;288;17039;195;263;471;9;|;239;-2003;17129;578;1203;490;9;|;2461;586;17699;356;202;209;11;|;823;-1092;17649;310;237;65;10;|;-7;-1369;17215;550;1279;288;9;|;-290;-249;17072;208;515;-33;9;|;-488;-965;17071;472;439;372;10;|;1270;-342;16923;377;476;120;8;|;671;-1337;16788;-33;482;14;9;|;1206;-725;16783;556;1306;344;9;|;-232;-1404;16681;309;423;379;8;|;364;-1400;17305;-28;432;-952;7;|;-854;-363;17644;1;369;626;8;|;-782;89;17931;529;134;-661;5;|;-723;-1057;16407;269;-82;700;6;|;459;-1235;16929;358;-119;193;5;|;-414;-1122;17400;282;775;526;2"]
righe = [
"2019/05/29 17:31:17;14.0;15.6;|;-4036;2706;-12885;-181;455;-289;-1302;|;-1074;567;-16756;-110;419;148;-1359;|;-973;718;-17951;-269;458;-187;-1481;|;-2594;517;-12813;314;407;-132;-1346;|;-1908;660;-16466;119;408;140;-1412;|;-2;634;-17249;273;418;-337;-1529;|;-44;689;-17622;-49;418;-461;-924;|;-2088;947;-16491;355;377;-101;-1634;|;-679;1280;-17315;131;405;-427;-1552;|;-1348;143;-16968;-42;400;158;-1275;|;-1229;337;-15634;312;388;-229;-1208;|;-303;1213;-17223;-231;398;-312;-1323;|;-1764;-110;-12365;-123;412;153;-1045;|;-1326;1149;-17401;-112;405;150;-937;|;-918;1595;-18541;250;417;26;-1160;|;-37;-952;-17323;-141;393;115;-1368;|;-322;705;-15059;274;389;-181;-1351;|;-139;764;-15408;-296;377;-315;-1044;|;-2243;-127;-11819;127;371;130;-1573;|;-910;81;-13546;-67;388;-442;-970;|;-87;-130;-14869;124;383;128;-1307;|;-1263;1976;-11610;-26;392;136;-1336;|;-1895;1092;-15215;-248;366;-287;-1123;|;-473;1183;-14193;-256;337;-247;-1174;|;520;493;-18394;-110;361;-430;-1584;|;-608;639;-14391;-168;361;71;-1354;|;6;263;-14750;-291;369;-102;-1163;|;-658;-208;-14123;-212;401;-330;-594;|;-1153;1045;-17704;-186;395;-38;-1465;|;-107;-4;-16871;-146;352;106;-1387;|;-639;1641;-15675;-57;355;113;-1216;|;-770;300;-14141;276;345;-69;-1632;|;-1476;1326;-16517;160;314;-469;-1291;|;-632;1362;-14201;242;348;19;-1118;|;-94;85;-14397;47;334;-473;-1292;|;853;-339;-17249;161;374;-423;-1341;|;-999;1107;-15241;339;338;-40;-1054;|;-1677;1531;-14531;-125;379;87;-1337;|;-783;955;-15564;-295;345;-178;-1215;|;1700;-768;-16200;64;365;-441;-1276;|;-1238;-442;-15508;-171;363;-294;-1102;|;-2471;1640;-15797;141;361;71;-1056;|;-1135;1410;-15452;-102;338;86;-1039;|;-2294;895;-15409;-218;352;-366;-1237;|;-1235;2429;-16575;-80;373;61;-1132;|;-1304;467;-16855;226;343;-425;-1008;|;-2340;-154;-16390;-57;327;119;-1033;|;-2288;2180;-14009;292;356;-10;-1053;|;-1106;36;-15806;-257;351;-81;-1206;|;-1778;610;-14479;157;354;116;-1126;|;-2142;1735;-14860;143;366;92;-835;|;-385;-117;-15141;-189;338;67;-1499;|;-434;950;-14577;214;359;66;-942;|;40;255;-18690;345;330;-250;-1466;|;-1235;1558;-15598;233;351;63;-1160;|;-1162;732;-14129;-176;338;52;-1424;|;-270;1318;-16862;316;324;-375;-1334;|;35;-1313;-19446;190;324;112;-1153;|;109;502;-15526;310;354;-334;-1128;|;-566;-1187;-15934;-192;349;-374;-1325;|;397;665;-15063;264;354;-99;-1501;|;-1126;2257;-15976;-266;342;-129;-1441;|;-1576;1197;-13151;324;359;-378;-1235;|;-2713;844;-15693;183;366;118;-1286;|;-2120;307;-14872;-8;361;-533;-1287;|;-2236;582;-16406;386;320;-246;-1450;|;-2549;904;-13694;368;325;-115;-1331;|;-1633;1657;-13434;375;317;-370;-1843;|;-2230;-350;-11589;66;-3;-522;-1047;|;1229;1639;-13878;106;372;-535;-1347;|;9231;1087;|;9811;1091",
"2019/05/29 17:32:58;13.6;15.7;|;-4035;2693;-12877;-181;454;-289;-1269;|;-1098;571;-16785;-110;419;149;-1332;|;-982;727;-17893;-271;457;-187;-1470;|;-2581;497;-12843;314;407;-133;-1320;|;-1913;665;-16464;115;407;140;-1386;|;2;628;-17245;274;419;-337;-1516;|;-32;672;-17624;-45;419;-460;-921;|;-2091;943;-16468;354;377;-100;-1628;|;-677;1280;-17323;133;404;-427;-1544;|;-1366;150;-16978;-43;398;158;-1500;|;-1226;329;-15636;313;386;-228;-1206;|;-311;1202;-17263;-230;402;-312;-1322;|;-1750;-107;-12353;-121;410;153;-1279;|;-1338;1147;-17418;-111;402;148;-1207;|;-939;1593;-18537;250;418;31;-1154;|;-49;-941;-17323;-141;393;114;-1343;|;-330;717;-15037;273;388;-181;-1348;|;-142;785;-15453;-296;375;-313;-1048;|;-2266;-130;-11838;127;370;129;-1558;|;-895;77;-13571;-66;389;-442;-965;|;-88;-119;-14865;125;384;129;-1284;|;-1259;1987;-11617;-25;392;136;-1336;|;-1901;1093;-15210;-248;367;-287;-1106;|;-454;1212;-14192;-269;355;-260;-1169;|;523;495;-18386;-110;362;-433;-1591;|;-632;638;-14401;-169;361;73;-1356;|;3;256;-14749;-292;370;-101;-1145;|;-659;-187;-14134;-213;401;-336;-573;|;-1157;1037;-17694;-187;394;-35;-1454;|;-111;2;-16855;-150;352;98;-1375;|;-651;1641;-15722;-60;356;120;-1198;|;-791;309;-14141;272;346;-67;-1620;|;-1462;1333;-16481;160;314;-470;-1287;|;-628;1371;-14212;244;350;23;-1107;|;-81;78;-14387;49;334;-475;-1273;|;836;-339;-17269;160;373;-429;-1343;|;-988;1110;-15227;337;339;-40;-1035;|;-1681;1533;-14534;-120;378;87;-1343;|;-813;952;-15565;-296;345;-177;-1225;|;1699;-785;-16184;69;366;-450;-1267;|;-1229;-435;-15505;-176;364;-294;-1099;|;-2471;1652;-15805;141;360;67;-1034;|;-1158;1406;-15451;-106;340;86;-1018;|;-2306;898;-15427;-220;351;-363;-1223;|;-1231;2446;-16584;-78;373;58;-1123;|;-1306;460;-16889;223;344;-433;-1007;|;-2337;-149;-16379;-55;327;120;-1026;|;-2292;2187;-14002;295;356;-10;-1046;|;-1099;57;-15790;-256;349;-76;-1222;|;-1777;629;-14490;157;354;115;-1115;|;-2145;1727;-14861;143;365;93;-843;|;-384;-118;-15147;-186;337;63;-1484;|;-461;940;-14639;214;359;66;-945;|;59;254;-18691;344;332;-249;-1217;|;-1240;1558;-15599;233;351;65;-1147;|;-1157;720;-14114;-177;337;53;-1396;|;-238;1310;-16855;313;323;-374;-1318;|;35;-1322;-19451;194;324;113;-1155;|;114;484;-15521;308;355;-338;-1115;|;-564;-1189;-15915;-188;349;-377;-1344;|;390;653;-15085;268;356;-97;-1513;|;-1127;2266;-15988;-266;342;-127;-1447;|;-1552;1195;-13171;319;359;-369;-1237;|;-2691;868;-15712;181;367;116;-1289;|;-2115;302;-14855;-19;362;-538;-1297;|;-2249;561;-16438;395;319;-236;-1461;|;-2536;915;-13684;364;323;-116;-1321;|;-1647;1659;-13423;373;318;-365;-1838;|;-2244;-351;-11568;66;-3;-526;-1017;|;1238;1628;-13803;93;375;-543;-1346;|;9211;1099;|;9702;1083",
"2019/05/29 18:20:27;13.6;19.9;|;-4026;2699;-12885;-180;455;-289;-1310;|;-1076;572;-16748;-109;418;146;-1358;|;-974;731;-17918;-270;458;-186;-1490;|;-2594;529;-12819;313;408;-132;-1333;|;-1907;662;-16461;118;407;140;-1409;|;6;619;-17243;272;419;-337;-1527;|;-63;680;-17597;-46;418;-460;-932;|;-2098;953;-16490;355;377;-101;-1637;|;-671;1283;-17317;133;405;-426;-1538;|;-1359;136;-16982;-43;400;159;-1282;|;-1228;318;-15602;315;389;-230;-1195;|;-316;1201;-17236;-198;341;-266;-1319;|;-1779;-120;-12374;-122;412;154;-1040;|;-1330;1134;-17429;-111;402;147;-1205;|;-919;1594;-18535;253;415;28;-1173;|;-45;-946;-17352;-141;393;115;-1362;|;-348;712;-15046;275;388;-181;-1358;|;-127;773;-15429;-294;376;-314;-1067;|;-2272;-117;-11825;126;371;128;-1576;|;-903;89;-13539;-66;389;-443;-960;|;-76;-144;-14870;125;383;128;-1287;|;-1255;1986;-11621;-27;392;136;-1350;|;-1893;1091;-15204;-241;365;-287;-1115;|;-469;1193;-14206;-274;356;-257;-1166;|;530;504;-18410;-109;362;-432;-1580;|;-627;640;-14386;-170;361;71;-1342;|;12;254;-14769;-293;370;-102;-1158;|;-660;-187;-14132;-212;402;-335;-593;|;-1149;1042;-17689;-186;393;-31;-1478;|;-94;-4;-16854;-142;351;101;-1383;|;-657;1650;-15715;-55;354;114;-1217;|;-779;307;-14141;273;345;-65;-1623;|;-1471;1328;-16526;162;314;-465;-1291;|;-638;1377;-14170;246;348;21;-1120;|;-82;81;-14392;46;335;-465;-1286;|;826;-324;-17271;157;373;-428;-1339;|;-996;1099;-15238;330;339;-48;-1046;|;-1664;1527;-14520;-122;378;85;-1345;|;-779;964;-15604;-296;345;-181;-1210;|;1698;-788;-16201;67;364;-447;-1247;|;-1225;-439;-15485;-178;365;-293;-1092;|;-2467;1650;-15773;140;360;73;-1047;|;-1118;1417;-15455;-107;338;86;-1023;|;-2311;904;-15445;-221;353;-361;-1234;|;-1243;2417;-16569;-79;372;60;-1109;|;-1324;471;-16891;220;343;-427;-1001;|;-2329;-169;-16371;-54;326;121;-1034;|;-2280;2190;-14002;296;355;-9;-1052;|;-1125;36;-15812;-254;351;-75;-1216;|;-1776;619;-14482;154;354;124;-1127;|;-2166;1748;-14870;144;365;95;-837;|;-392;-120;-15145;-187;338;70;-1494;|;-437;946;-14585;212;358;67;-947;|;42;247;-18665;347;332;-250;-1232;|;-1242;1560;-15596;231;352;66;-1148;|;-1156;727;-14117;-176;337;49;-1389;|;-266;1322;-16856;314;323;-374;-1318;|;40;-1337;-19450;192;325;106;-1164;|;126;504;-15528;312;354;-341;-1118;|;-567;-1204;-15918;-195;349;-377;-1321;|;399;644;-15075;272;355;-91;-1503;|;-1136;2263;-15948;-267;343;-124;-1445;|;-1556;1192;-13222;325;358;-370;-1241;|;-2707;860;-15665;179;367;103;-1298;|;-2114;319;-14877;-13;350;-521;-1270;|;-2243;560;-16368;392;316;-236;-1454;|;-2552;905;-13683;367;322;-112;-1321;|;-1654;1653;-13427;375;317;-369;-1858;|;-2268;-350;-11573;64;-3;-521;-1028;|;1239;1624;-13860;94;375;-543;-1332;|;9527;1079;|;9966;1070",
"2019/05/29 18:22:45;13.2;20.1;|;-4042;2692;-12915;-181;455;-289;-1291;|;-1087;562;-16766;-113;419;148;-1357;|;-974;730;-17893;-269;458;-187;-1468;|;-2597;510;-12822;313;408;-132;-1339;|;-1915;687;-16472;118;408;142;-1409;|;1;625;-17275;275;417;-337;-1517;|;-34;689;-17613;-44;418;-460;-929;|;-2114;958;-16451;355;377;-101;-1624;|;-670;1269;-17280;132;406;-426;-1537;|;-1365;152;-16972;-42;400;159;-1269;|;-1239;316;-15620;312;388;-227;-1198;|;-310;1204;-17245;-205;357;-275;-1321;|;-1766;-102;-12379;-123;411;154;-1039;|;-1355;1136;-17429;-110;402;148;-1194;|;-935;1591;-18524;250;416;28;-1144;|;-34;-940;-17350;-141;393;115;-1357;|;-337;709;-15038;274;387;-181;-1361;|;-132;767;-15408;-295;377;-314;-1046;|;-2270;-134;-11831;127;369;130;-1561;|;-899;96;-13575;-66;389;-443;-965;|;-80;-148;-14886;126;384;128;-1280;|;-1272;1996;-11625;-27;392;135;-1330;|;-1902;1104;-15252;-247;366;-287;-1103;|;-453;1200;-14215;-270;355;-257;-1161;|;510;489;-18401;-111;362;-433;-1583;|;-617;626;-14398;-169;361;71;-1339;|;8;264;-14744;-291;369;-101;-1148;|;-664;-202;-14128;-217;401;-336;-585;|;-1145;1030;-17697;-184;395;-38;-1464;|;-107;-12;-16862;-143;353;106;-1375;|;-664;1635;-15708;-58;356;110;-1202;|;-751;299;-14121;279;344;-65;-1616;|;-1462;1337;-16507;160;315;-468;-1302;|;-635;1372;-14208;245;350;18;-1122;|;-85;78;-14394;45;335;-469;-1283;|;834;-341;-17264;159;374;-431;-1353;|;-982;1119;-15236;332;339;-37;-1046;|;-1686;1536;-14537;-120;378;88;-1343;|;-780;958;-15548;-297;344;-178;-1193;|;1704;-795;-16187;67;365;-453;-1258;|;-1235;-436;-15499;-178;364;-290;-1092;|;-2482;1637;-15811;143;358;72;-1295;|;-1151;1414;-15433;-103;339;85;-1018;|;-2333;885;-15407;-216;353;-370;-1221;|;-1262;2438;-16611;-79;371;65;-1328;|;-1315;456;-16897;219;343;-436;-994;|;-2335;-153;-16369;-56;327;119;-1034;|;-2278;2176;-14001;294;356;-10;-1042;|;-1106;46;-15798;-251;351;-78;-1214;|;-1798;610;-14491;153;357;114;-1135;|;-2159;1730;-14888;144;365;95;-865;|;-400;-133;-15134;-184;338;72;-1497;|;-452;932;-14596;212;358;68;-937;|;45;238;-18676;349;332;-250;-1230;|;-1233;1549;-15601;233;349;65;-1380;|;-1150;732;-14105;-175;338;48;-1405;|;-257;1328;-16848;315;323;-375;-1330;|;44;-1317;-19442;194;325;116;-1158;|;113;512;-15534;307;354;-331;-1132;|;-551;-1199;-15927;-195;349;-373;-1341;|;417;647;-15088;271;354;-92;-1503;|;-1135;2259;-15976;-267;341;-124;-1459;|;-1552;1182;-13164;324;358;-370;-1246;|;-2712;851;-15670;182;366;120;-1287;|;-2116;310;-14859;-14;362;-542;-1277;|;-2225;556;-16375;392;316;-242;-1438;|;-2539;894;-13692;364;323;-108;-1335;|;-1633;1675;-13414;373;318;-366;-1841;|;-2233;-343;-11582;66;-2;-515;-1051;|;1231;1634;-13817;97;374;-538;-1324;|;9506;1070;|;9734;1010"
]
#Dividi la riga principale usando il primo delimitatore ';'
@@ -31,9 +34,9 @@ def make_matrix(righe):
for riga in righe:
timestamp, batlevel, temperature, rilevazioni = riga.split(';',3)
EventDate, EventTime = timestamp.split(' ')
valori_nodi = rilevazioni.rstrip(';').split(';|;')[1:] # Toglie eventuali ';' finali, dividi per '|' e prendi gli elementi togliendo il primo che è vuoto
valori_nodi = rilevazioni.lstrip('|;').rstrip(';').split(';|;') # Toglie '|;' iniziali, toglie eventuali ';' finali, dividi per ';|;'
for num_nodo, valori_nodo in enumerate(valori_nodi, start=1):
valori = valori_nodo.split(';')[1:-1]
valori = valori_nodo.split(';')
matrice_valori.append([UnitName, ToolNameID, num_nodo, date_check.conforma_data(EventDate), EventTime, batlevel, temperature] + valori + ([None] * (19 - len(valori))))
return matrice_valori

2
prova_pd.py
View File

@@ -4,7 +4,7 @@ from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import declarative_base, Session
from sqlalchemy.exc import IntegrityError
import pandas as pd
import utils.datefmt.date_check as date_check
import utils.timestamp.date_check as date_check
righe = ["17/03/2022 15:10;13.7;14.8;|;401;832;17373;-8;920;469;9;|;839;133;17116;675;941;228;10;|;-302;-1252;17165;288;75;-940;10;|;739;76;17203;562;879;604;9;|;1460;751;16895;672;1462;132;10;|;-1088;-1883;16675;244;1071;518;10;|;-29;-1683;16923;384;1039;505;11;|;1309;-1095;17066;-36;324;-552;10;|;-36;-713;16701;-121;372;122;10;|;508;-1318;16833;475;1154;405;10;|;1178;878;17067;636;1114;428;10;|;1613;-573;17243;291;-234;-473;9;|;-107;-259;17287;94;421;369;10;|;-900;-647;16513;168;1330;252;10;|;1372;286;17035;202;263;469;10;|;238;-2006;17142;573;1201;492;9;|;2458;589;17695;356;187;208;11;|;827;-1085;17644;308;233;66;10;|;1;-1373;17214;557;1279;298;9;|;-281;-244;17071;209;517;-36;10;|;-486;-961;17075;467;440;367;10;|;1264;-339;16918;374;476;116;8;|;661;-1330;16789;-37;478;15;9;|;1208;-724;16790;558;1303;335;8;|;-236;-1404;16678;309;426;376;8;|;367;-1402;17308;-32;428;-957;7;|;-849;-360;17640;1;371;635;7;|;-784;90;17924;533;128;-661;5;|;-723;-1062;16413;270;-79;702;7;|;458;-1235;16925;354;-117;194;5;|;-411;-1116;17403;280;777;530;1",
"19/03/2022 15:13;13.6;14.8;|;398;836;17368;-3;924;472;9;|;838;125;17110;675;938;230;10;|;-298;-1253;17164;290;75;-942;10;|;749;78;17221;560;883;601;9;|;1463;752;16904;673;1467;134;10;|;-1085;-1884;16655;239;1067;520;10;|;-27;-1680;16923;393;1032;507;10;|;1308;-1095;17065;-43;328;-548;10;|;-38;-712;16704;-124;373;122;10;|;512;-1318;16830;473;1155;408;10;|;1181;879;17070;637;1113;436;10;|;1610;-567;17239;287;-240;-462;10;|;-108;-250;17297;94;420;370;10;|;-903;-652;16518;169;1326;257;9;|;1371;282;17047;198;263;471;10;|;244;-2006;17137;570;1205;487;9;|;2461;589;17689;354;199;210;11;|;823;-1081;17642;310;235;68;10;|;1;-1370;17214;560;1278;290;9;|;-280;-245;17062;209;517;-31;9;|;-484;-963;17074;463;440;374;10;|;1271;-340;16912;374;477;125;8;|;668;-1331;16786;-37;478;7;9;|;1209;-724;16784;557;1301;329;8;|;-237;-1406;16673;316;425;371;8;|;371;-1401;17307;-30;429;-961;7;|;-854;-356;17647;7;368;631;7;|;-781;85;17934;531;130;-664;5;|;-726;-1062;16400;274;-79;707;6;|;460;-1233;16931;355;-113;196;5;|;-413;-1119;17405;280;780;525;1",

8
test/test_ftp_csv_receicer.py
View File

@@ -36,8 +36,8 @@ mock_utils_config.set_config.config.return_value = mock_config_instance
# sys.modules['pyftpdlib.servers'] = MagicMock()
# Import the module AFTER mocking dependencies
import FtpCsvReceiver
from FtpCsvReceiver import (
import ftp_csv_receiver
from ftp_csv_receiver import (
extract_value,
DummySha256Authorizer,
ASEHandler,
@@ -193,13 +193,13 @@ class TestDummySha256Authorizer(unittest.TestCase):
def test_validate_authentication_wrong_password(self):
self.mock_cursor.fetchall.return_value = []
authorizer = DummySha256Authorizer(self.mock_cfg)
with self.assertRaises(FtpCsvReceiver.AuthenticationFailed):
with self.assertRaises(ftp_csv_receiver.AuthenticationFailed):
authorizer.validate_authentication('admin', 'wrongpass', None)
def test_validate_authentication_unknown_user(self):
self.mock_cursor.fetchall.return_value = []
authorizer = DummySha256Authorizer(self.mock_cfg)
with self.assertRaises(FtpCsvReceiver.AuthenticationFailed):
with self.assertRaises(ftp_csv_receiver.AuthenticationFailed):
authorizer.validate_authentication('unknown', 'somepass', None)

2
transform_file.py
View File

@@ -11,7 +11,7 @@ import mysql.connector as mysql
import logging
from utils.time import timestamp_fmt as ts
from utils.config import set_config as setting
from utils.config import loader as setting
def conn_db(cfg):
return mysql.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport )

3
utils/config/set_config.py → utils/config/loader.py
View File

@@ -2,9 +2,8 @@
"""
from configparser import ConfigParser
import json
class config:
class Config:
def __init__(self):
c = ConfigParser()
c.read(["/etc/aseftp/ftpcsvreceiver.ini", "./ftpcsvreceiver.ini",

12
utils/config/parser.py Normal file
View File

@@ -0,0 +1,12 @@
import re
def extract_value(patterns, primary_source, secondary_source, default='Not Defined'):
"""Extracts the first match for a list of patterns from the primary source.
Falls back to the secondary source if no match is found.
"""
for source in (primary_source, secondary_source):
for pattern in patterns:
matches = re.findall(pattern, source, re.IGNORECASE)
if matches:
return matches[0] # Return the first match immediately
return default # Return default if no matches are found

0
utils/datefmt/__init__.py → utils/database/__init__.py
View File

23
utils/database/connection.py Normal file
View File

@@ -0,0 +1,23 @@
import logging
import mysql.connector
logger = logging.getLogger(__name__)
def connetti_db(cfg):
"""Establishes a connection to the MySQL database.
Args:
cfg: The configuration object containing database connection details.
Returns:
A MySQL database connection object.
"""
try:
conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
conn.autocommit = True
logger.info("Connected")
return conn
except mysql.connector.Error as e:
logger.error(f'{e}')
exit(e.errno)
return None

96
utils/database/get_nodes_type.py Normal file
View File

@@ -0,0 +1,96 @@
import logging
import datetime
import os
import mysql.connector
from utils.database.connection import connetti_db
logger = logging.getLogger(__name__)
def get_timestamp(log_type):
"""Generates a timestamp string for logging."""
now = datetime.datetime.now()
return now.strftime("%Y-%m-%d %H:%M:%S")
def get_nodes_type(db_lar, server, username, password, tool, unit, channels, nodetype, ain, din):
"""
Retrieves node type, Ain, Din, and channels from the database for a specific tool and unit.
Args:
db_lar (str): The name of the MySQL database.
server (str): The hostname or IP address of the MySQL server.
username (str): The MySQL username.
password (str): The MySQL password.
tool (str): The name of the tool.
unit (str): The name of the unit.
channels (list): An empty list to store the 'channels' values.
nodetype (list): An empty list to store the 'type' values.
ain (list): An empty list to store the 'ain' values.
din (list): An empty list to store the 'din' values.
"""
try:
conn = connetti_db(cfg)
cursor = conn.cursor(dictionary=True)
query = f"""
SELECT t.name AS name, n.seq AS seq, n.num AS num, n.channels AS channels, y.type AS type, n.ain AS ain, n.din AS din
FROM nodes AS n
INNER JOIN tools AS t ON t.id = n.tool_id
INNER JOIN units AS u ON u.id = t.unit_id
INNER JOIN nodetypes AS y ON n.nodetype_id = y.id
WHERE y.type NOT IN ('Anchor Link', 'None') AND t.name = '{tool}' AND u.name = '{unit}'
ORDER BY n.num;
"""
cursor.execute(query)
results = cursor.fetchall()
print(f"{get_timestamp('log')} - pid {os.getpid()} >> {unit} - {tool}: {cursor.rowcount} rows selected to get node type/Ain/Din/channels.")
if not results:
print(f"{get_timestamp('log')} - pid {os.getpid()} >> Node/Channels/Ain/Din not defined.")
print(f"{get_timestamp('log')} - pid {os.getpid()} >> Execution ended.")
exit()
else:
for row in results:
channels.append(row['channels'])
nodetype.append(row['type'])
ain.append(row['ain'])
din.append(row['din'])
cursor.close()
dbh.close()
except mysql.connector.Error as err:
error_message = f"{get_timestamp('log')} - pid {os.getpid()} >> Could not connect to database: {err}"
print(error_message)
raise # Re-raise the exception if you want the calling code to handle it
if __name__ == '__main__':
# Example usage (replace with your actual database credentials and values)
db_name = "your_database_name"
db_server = "your_server_address"
db_user = "your_username"
db_password = "your_password"
current_tool = "your_tool_name"
current_unit = "your_unit_name"
node_channels = []
node_types = []
node_ains = []
node_dins = []
try:
get_nodes_type(db_name, db_server, db_user, db_password, current_tool, current_unit, node_channels, node_types, node_ains, node_dins)
print("\nRetrieved Data:")
print(f"Channels: {node_channels}")
print(f"Node Types: {node_types}")
print(f"Ains: {node_ains}")
print(f"Dins: {node_dins}")
except Exception as e:
print(f"An error occurred: {e}")

0
utils/ftp/__init__.py Normal file
View File

52
utils/ftp/file_management.py Normal file
View File

@@ -0,0 +1,52 @@
import os
import logging
import mysql.connector
from utils.database.connection import connetti_db
from utils.config.parser import extract_value
logger = logging.getLogger(__name__)
def on_file_received(self, file):
"""Handles the event when a file is successfully received.
Args:
file: The path to the received file.
"""
if not os.stat(file).st_size:
os.remove(file)
logging.info(f'File {file} is empty: removed.')
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if (fileExtension.upper() in (cfg.fileext)):
with open(file, 'r') as csvfile:
lines = csvfile.readlines()
unit_name = extract_value(cfg.units_name, filename, str(lines[0:9]))
unit_type = extract_value(cfg.units_type, filename, str(lines[0:9]))
tool_name = extract_value(cfg.tools_name, filename, str(lines[0:9]))
tool_type = extract_value(cfg.tools_type, filename, str(lines[0:9]))
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
try:
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (filename, unit_name, unit_type, tool_name, tool_type, tool_data) VALUES (%s, %s, %s, %s, %s, %s)", (filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines)))
conn.commit()
conn.close()
except Exception as e:
logging.error(f'File {file} not loaded. Held in user path.')
logging.error(f'{e}')
else:
os.remove(file)
logging.info(f'File {file} loaded: removed.')

140
utils/ftp/user_admin.py Normal file
View File

@@ -0,0 +1,140 @@
import os
import mysql.connector
import logging
from hashlib import sha256
from pathlib import Path
from utils.database.connection import connetti_db
logger = logging.getLogger(__name__)
def ftp_SITE_ADDU(self, line):
"""Adds a virtual user, creates their directory, and saves their details to the database.
"""
cfg = self.cfg
try:
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = sha256(password.encode("UTF-8")).hexdigest() # Hash the password
except IndexError:
self.respond('501 SITE ADDU failed. Command needs 2 arguments')
else:
try:
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.respond(f'551 Error in create virtual user path: {e}')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm=cfg.defperm)
# Save the user to the database
# Define the database connection
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbusertable} (ftpuser, hash, virtpath, perm) VALUES ('{user}', '{hash}', '{cfg.virtpath + user}', '{cfg.defperm}')")
conn.commit()
conn.close()
logging.info(f"User {user} created.")
self.respond('200 SITE ADDU successful.')
except Exception as e:
self.respond(f'501 SITE ADDU failed: {e}.')
print(e)
def ftp_SITE_DISU(self, line):
"""Removes a virtual user from the authorizer and marks them as deleted in the database."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Remove the user from the authorizer
self.authorizer.remove_user(str(user))
# Delete the user from database
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = now() WHERE ftpuser = '{user}'")
conn.commit()
conn.close()
logging.info(f"User {user} deleted.")
self.respond('200 SITE DISU successful.')
except Exception as e:
self.respond('501 SITE DISU failed.')
print(e)
def ftp_SITE_ENAU(self, line):
"""Restores a virtual user by updating their status in the database and adding them back to the authorizer."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Restore the user into database
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
try:
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = null WHERE ftpuser = '{user}'")
conn.commit()
except Exception as e:
logging.error(f"Update DB failed: {e}")
cur.execute(f"SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE ftpuser = '{user}'")
ftpuser, hash, virtpath, perm = cur.fetchone()
self.authorizer.add_user(ftpuser, hash, virtpath, perm)
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
conn.close()
logging.info(f"User {user} restored.")
self.respond('200 SITE ENAU successful.')
except Exception as e:
self.respond('501 SITE ENAU failed.')
print(e)
def ftp_SITE_LSTU(self, line):
"""Lists all virtual users from the database."""
cfg = self.cfg
users_list = []
try:
# Connect to the SQLite database to fetch users
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
self.push("214-The following virtual users are defined:\r\n")
cur.execute(f'SELECT ftpuser, perm, disabled_at FROM {cfg.dbname}.{cfg.dbusertable}')
[users_list.append(f'Username: {ftpuser}\tPerms: {perm}\tDisabled: {disabled_at}\r\n') for ftpuser, perm, disabled_at in cur.fetchall()]
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
except Exception as e:
self.respond(f'501 list users failed: {e}')

1
utils/parsers/G801_parser.py
View File

@@ -1 +0,0 @@
locals

3
utils/parsers/g801_mums.py Normal file
View File

@@ -0,0 +1,3 @@
def chi_sono(unit, tool):
print(f'g801_mums: {unit} - {tool}')
return f'g801_mums: {unit} - {tool}'

3
utils/parsers/g801_mux.py Normal file
View File

@@ -0,0 +1,3 @@
def chi_sono(unit, tool):
print(f'g801_mux: {unit} - {tool}')
return f'g801_mux: {unit} - {tool}'

1
utils/time/__init__.py
View File

@@ -1 +0,0 @@
"""Utilità per i formati timestamp"""

28
utils/time/date_refmt.py
View File

@@ -1,28 +0,0 @@
"""Funzioni per formato data
"""
from datetime import datetime
from re import search
def dateFmt(date):
t = date.replace("/", "-")
if search('^\d\d\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%Y-%m-%d")
elif search('^\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%y-%m-%d")
elif search('^\d\d-\d\d-\d\d\d\d$', t):
d = datetime.strptime(t, "%d-%m-%Y")
return datetime.strftime(d, "%Y-%m-%d")
def dateTimeFmt(date):
t = date.replace("/", "-")
if search('^\d\d\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
elif search('^\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%y-%m-%d %H:%M:%S")
elif search('^\d\d-\d\d-\d\d\d\d$', t):
d = datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
return datetime.strftime(d, "%Y-%m-%d")

25
utils/time/dt_convert.py
View File

@@ -1,25 +0,0 @@
"""Funzioni per convertire formato data
"""
import datetime
def dateFmt(date):
t = date.replace("/", "-")
try:
datetime.datetime.strptime(t, "%Y-%m-%d")
return t
except ValueError:
d = datetime.datetime.strptime(t, "%d-%m-%Y")
return datetime.datetime.strftime(d, "%Y-%m-%d")
def dateTimeFmt(date):
t = date.replace("/", "-")
try:
datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
return t
except ValueError:
d = datetime.datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
return datetime.datetime.strftime(d, "%Y-%m-%d %H:%M:%S")

10
utils/time/timestamp_fmt.py
View File

@@ -1,10 +0,0 @@
"""Funzioni per timestamp
"""
from datetime import datetime
def timestamp(t):
fmt = {"log": "%Y-%m-%d %H:%M:%S", "tms": "%Y%m%d%H%M%S"}
return datetime.now().strftime(fmt[t])

0
utils/timestamp/__init__.py Normal file
View File

0
utils/datefmt/date_check.py → utils/timestamp/date_check.py
View File