alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactory

Browse Source
This commit is contained in:
alex
2025-04-28 22:29:35 +02:00
parent 40ef173694
commit cc7a136cf3
26 changed files with 372 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CsvLoader.py
View File

@@ -11,7 +11,7 @@ import shutil
from utils.time import timestamp_fmt as ts from utils.time import timestamp_fmt as ts
from utils.time import date_refmt as df from utils.time import date_refmt as df
from utils.config import set_config as setting from utils.config import loader as setting
class sqlraw: class sqlraw:

348
FtpCsvReceiver.py
View File

@@ -1,348 +0,0 @@
#!.venv/bin/python
"""This module implements an FTP server with custom commands for managing virtual users and handling CSV file uploads."""
import sys
import os
# import ssl
import re
import logging
import mysql.connector
from hashlib import sha256
from pathlib import Path
from utils.config import set_config as setting
from utils.datefmt import date_check as date_check
from utils.db import connect_db as connect_db
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
def conn_db(cfg):
"""Establishes a connection to the MySQL database.
Args:
cfg: The configuration object containing database connection details.
Returns:
A MySQL database connection object.
"""
try:
conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
conn.autocommit = True
return conn
except mysql.connector.Error as e:
print(f"Error: {e} - Error: {e.errno}")
print (f"Error code: {e.errno}") # error number
print (f"SQLSTATE value: {e.sqlstate}") # SQLSTATE value
print (f"Error message: {e.msg}")
logging.error(f'{e}')
exit(e.errno)
def extract_value(patterns, primary_source, secondary_source, default='Not Defined'):
"""Extracts the first match for a list of patterns from the primary source.
Falls back to the secondary source if no match is found.
"""
for source in (primary_source, secondary_source):
for pattern in patterns:
matches = re.findall(pattern, source, re.IGNORECASE)
if matches:
return matches[0] # Return the first match immediately
return default # Return default if no matches are found
class DummySha256Authorizer(DummyAuthorizer):
"""Custom authorizer that uses SHA256 for password hashing and manages users from a database."""
def __init__(self, cfg):
"""Initializes the authorizer, adds the admin user, and loads users from the database.
Args:
cfg: The configuration object.
"""
super().__init__()
self.add_user(
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
# Define the database connection
conn = conn_db(cfg)
# Create a cursor
cur = conn.cursor()
cur.execute(f'SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL')
for ftpuser, hash, virtpath, perm in cur.fetchall():
self.add_user(ftpuser, hash, virtpath, perm)
"""
Create the user's directory if it does not exist.
"""
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
def validate_authentication(self, username, password, handler):
# Validate the user's password against the stored hash
hash = sha256(password.encode("UTF-8")).hexdigest()
try:
if self.user_table[username]["pwd"] != hash:
raise KeyError
except KeyError:
raise AuthenticationFailed
class ASEHandler(FTPHandler):
"""Custom FTP handler that extends FTPHandler with custom commands and file handling."""
def __init__(self, conn, server, ioloop=None):
"""Initializes the handler, adds custom commands, and sets up command permissions.
Args:
conn: The connection object.
server: The FTP server object.
ioloop: The I/O loop object.
"""
super().__init__(conn, server, ioloop)
self.proto_cmds = FTPHandler.proto_cmds.copy()
# Add custom FTP commands for managing virtual users - command in lowercase
self.proto_cmds.update(
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
)
self.proto_cmds.update(
{'SITE DISU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> DISU USERNAME (disable virtual user).')}
)
self.proto_cmds.update(
{'SITE ENAU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ENAU USERNAME (enable virtual user).')}
)
self.proto_cmds.update(
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
help='Syntax: SITE <SP> LSTU (list virtual users).')}
)
def on_file_received(self, file):
"""Handles the event when a file is successfully received.
Args:
file: The path to the received file.
"""
if not os.stat(file).st_size:
os.remove(file)
logging.info(f'File {file} was empty: removed.')
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if (fileExtension.upper() in (cfg.fileext)):
with open(file, 'r') as csvfile:
lines = csvfile.readlines()
unit_name = extract_value(cfg.units_name, filename, str(lines[0:9]))
unit_type = extract_value(cfg.units_type, filename, str(lines[0:9]))
tool_name = extract_value(cfg.tools_name, filename, str(lines[0:9]))
tool_type = extract_value(cfg.tools_type, filename, str(lines[0:9]))
try:
conn = conn_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
try:
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (filename, unit_name, unit_type, tool_name, tool_type, tool_data) VALUES (%s, %s, %s, %s, %s, %s)", (filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines)))
conn.commit()
conn.close()
except Exception as e:
logging.error(f'File {file} not loaded. Held in user path.')
logging.error(f'{e}')
else:
os.remove(file)
logging.info(f'File {file} loaded: removed.')
def on_incomplete_file_received(self, file):
"""Removes partially uploaded files.
Args:
file: The path to the incomplete file.
"""
os.remove(file)
def ftp_SITE_ADDU(self, line):
"""Adds a virtual user, creates their directory, and saves their details to the database.
"""
cfg = self.cfg
try:
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = sha256(password.encode("UTF-8")).hexdigest() # Hash the password
except IndexError:
self.respond('501 SITE ADDU failed. Command needs 2 arguments')
else:
try:
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.respond(f'551 Error in create virtual user path: {e}')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm=cfg.defperm)
# Save the user to the database
# Define the database connection
try:
conn = conn_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbusertable} (ftpuser, hash, virtpath, perm) VALUES ('{user}', '{hash}', '{cfg.virtpath + user}', '{cfg.defperm}')")
conn.commit()
conn.close()
logging.info(f"User {user} created.")
self.respond('200 SITE ADDU successful.')
except Exception as e:
self.respond(f'501 SITE ADDU failed: {e}.')
print(e)
def ftp_SITE_DISU(self, line):
"""Removes a virtual user from the authorizer and marks them as deleted in the database."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Remove the user from the authorizer
self.authorizer.remove_user(str(user))
# Delete the user from database
try:
conn = conn_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = now() WHERE ftpuser = '{user}'")
conn.commit()
conn.close()
logging.info(f"User {user} deleted.")
self.respond('200 SITE DISU successful.')
except Exception as e:
self.respond('501 SITE DISU failed.')
print(e)
def ftp_SITE_ENAU(self, line):
"""Restores a virtual user by updating their status in the database and adding them back to the authorizer."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Restore the user into database
try:
conn = conn_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
try:
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = null WHERE ftpuser = '{user}'")
conn.commit()
except Exception as e:
logging.error(f"Update DB failed: {e}")
cur.execute(f"SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE ftpuser = '{user}'")
ftpuser, hash, virtpath, perm = cur.fetchone()
self.authorizer.add_user(ftpuser, hash, virtpath, perm)
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
conn.close()
logging.info(f"User {user} restored.")
self.respond('200 SITE ENAU successful.')
except Exception as e:
self.respond('501 SITE ENAU failed.')
print(e)
def ftp_SITE_LSTU(self, line):
"""Lists all virtual users from the database."""
cfg = self.cfg
users_list = []
try:
# Connect to the SQLite database to fetch users
try:
conn = conn_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
self.push("214-The following virtual users are defined:\r\n")
cur.execute(f'SELECT ftpuser, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL ')
[users_list.append(f'Username: {ftpuser}\tPerms: {perm}\r\n') for ftpuser, perm in cur.fetchall()]
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
except Exception as e:
self.respond(f'501 list users failed: {e}')
def main():
"""Main function to start the FTP server."""
# Load the configuration settings
cfg = setting.config()
try:
# Initialize the authorizer and handler
authorizer = DummySha256Authorizer(cfg)
handler = ASEHandler
handler.cfg = cfg
handler.authorizer = authorizer
handler.masquerade_address = cfg.proxyaddr
# Set the range of passive ports for the FTP server
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
handler.passive_ports = _range
# Configure logging
logging.basicConfig(
format="%(asctime)s %(message)s",
filename=cfg.logfilename,
level=logging.INFO,
)
# Create and start the FTP server
server = FTPServer(("0.0.0.0", 2121), handler)
server.serve_forever()
except KeyboardInterrupt:
logging.info(
"Info: Shutdown requested...exiting"
)
except Exception:
print(
f"{ts.timestamp("log")} - PID {os.getpid():>5} >> Error: {sys.exc_info()[1]}."
)
if __name__ == "__main__":
main()

145
ftp_csv_receiver.py Executable file
View File

@@ -0,0 +1,145 @@
#!.venv/bin/python
"""This module implements an FTP server with custom commands for managing virtual users and handling CSV file uploads."""
import os
import logging
from hashlib import sha256
from pathlib import Path
from utils.config import loader
from utils.database.connection import connetti_db
from utils.ftp import user_admin, file_management
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
logger = logging.getLogger(__name__)
class DummySha256Authorizer(DummyAuthorizer):
"""Custom authorizer that uses SHA256 for password hashing and manages users from a database."""
def __init__(self, cfg):
"""Initializes the authorizer, adds the admin user, and loads users from the database.
Args:
cfg: The configuration object.
"""
super().__init__()
self.add_user(
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
# Define the database connection
conn = connetti_db(cfg)
# Create a cursor
cur = conn.cursor()
cur.execute(f'SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE disabled_at IS NULL')
for ftpuser, hash, virtpath, perm in cur.fetchall():
self.add_user(ftpuser, hash, virtpath, perm)
"""
Create the user's directory if it does not exist.
"""
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
def validate_authentication(self, username, password, handler):
# Validate the user's password against the stored hash
hash = sha256(password.encode("UTF-8")).hexdigest()
try:
if self.user_table[username]["pwd"] != hash:
raise KeyError
except KeyError:
raise AuthenticationFailed
class ASEHandler(FTPHandler):
"""Custom FTP handler that extends FTPHandler with custom commands and file handling."""
def __init__(self, conn, server, ioloop=None):
"""Initializes the handler, adds custom commands, and sets up command permissions.
Args:
conn: The connection object.
server: The FTP server object.
ioloop: The I/O loop object.
"""
super().__init__(conn, server, ioloop)
self.proto_cmds = FTPHandler.proto_cmds.copy()
# Add custom FTP commands for managing virtual users - command in lowercase
self.proto_cmds.update(
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
)
self.proto_cmds.update(
{'SITE DISU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> DISU USERNAME (disable virtual user).')}
)
self.proto_cmds.update(
{'SITE ENAU': dict(perm='M', auth=True, arg=True,
help='Syntax: SITE <SP> ENAU USERNAME (enable virtual user).')}
)
self.proto_cmds.update(
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
help='Syntax: SITE <SP> LSTU (list virtual users).')}
)
def on_file_received(self, file):
return file_management.on_file_received(self, file)
def on_incomplete_file_received(self, file):
"""Removes partially uploaded files.
Args:
file: The path to the incomplete file.
"""
os.remove(file)
def ftp_SITE_ADDU(self, line):
return user_admin.ftp_SITE_ADDU(self, line)
def ftp_SITE_DISU(self, line):
return user_admin.ftp_SITE_DISU(self, line)
def ftp_SITE_ENAU(self, line):
return user_admin.ftp_SITE_ENAU(self, line)
def ftp_SITE_LSTU(self, line):
return user_admin.ftp_SITE_LSTU(self, line)
def main():
"""Main function to start the FTP server."""
# Load the configuration settings
cfg = loader.Config()
try:
# Initialize the authorizer and handler
authorizer = DummySha256Authorizer(cfg)
handler = ASEHandler
handler.cfg = cfg
handler.authorizer = authorizer
handler.masquerade_address = cfg.proxyaddr
# Set the range of passive ports for the FTP server
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
handler.passive_ports = _range
# Configure logging
logging.basicConfig(
format="%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s ",
filename=cfg.logfilename,
level=logging.INFO,
)
# Create and start the FTP server
server = FTPServer(("0.0.0.0", 2121), handler)
server.serve_forever()
except Exception as e:
logger.error(
f"Exit with error: {e}."
)
if __name__ == "__main__":
main()

2
orchestrator.py
View File

@@ -7,7 +7,7 @@ import logging
import importlib import importlib
from utils.time import timestamp_fmt as ts from utils.time import timestamp_fmt as ts
from utils.config import set_config as setting from utils.config import loader as setting
#from unit_tool_mod import g801_mums, g801_mux #from unit_tool_mod import g801_mums, g801_mux
def conn_db(cfg): def conn_db(cfg):

2
prova_no_pd.py
View File

@@ -1,6 +1,6 @@
import mysql.connector import mysql.connector
import utils.datefmt.date_check as date_check import utils.timestamp.date_check as date_check
righe = [ righe = [

2
prova_pd.py
View File

@@ -4,7 +4,7 @@ from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import declarative_base, Session from sqlalchemy.orm import declarative_base, Session
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
import pandas as pd import pandas as pd
import utils.datefmt.date_check as date_check import utils.timestamp.date_check as date_check
righe = ["17/03/2022 15:10;13.7;14.8;|;401;832;17373;-8;920;469;9;|;839;133;17116;675;941;228;10;|;-302;-1252;17165;288;75;-940;10;|;739;76;17203;562;879;604;9;|;1460;751;16895;672;1462;132;10;|;-1088;-1883;16675;244;1071;518;10;|;-29;-1683;16923;384;1039;505;11;|;1309;-1095;17066;-36;324;-552;10;|;-36;-713;16701;-121;372;122;10;|;508;-1318;16833;475;1154;405;10;|;1178;878;17067;636;1114;428;10;|;1613;-573;17243;291;-234;-473;9;|;-107;-259;17287;94;421;369;10;|;-900;-647;16513;168;1330;252;10;|;1372;286;17035;202;263;469;10;|;238;-2006;17142;573;1201;492;9;|;2458;589;17695;356;187;208;11;|;827;-1085;17644;308;233;66;10;|;1;-1373;17214;557;1279;298;9;|;-281;-244;17071;209;517;-36;10;|;-486;-961;17075;467;440;367;10;|;1264;-339;16918;374;476;116;8;|;661;-1330;16789;-37;478;15;9;|;1208;-724;16790;558;1303;335;8;|;-236;-1404;16678;309;426;376;8;|;367;-1402;17308;-32;428;-957;7;|;-849;-360;17640;1;371;635;7;|;-784;90;17924;533;128;-661;5;|;-723;-1062;16413;270;-79;702;7;|;458;-1235;16925;354;-117;194;5;|;-411;-1116;17403;280;777;530;1", righe = ["17/03/2022 15:10;13.7;14.8;|;401;832;17373;-8;920;469;9;|;839;133;17116;675;941;228;10;|;-302;-1252;17165;288;75;-940;10;|;739;76;17203;562;879;604;9;|;1460;751;16895;672;1462;132;10;|;-1088;-1883;16675;244;1071;518;10;|;-29;-1683;16923;384;1039;505;11;|;1309;-1095;17066;-36;324;-552;10;|;-36;-713;16701;-121;372;122;10;|;508;-1318;16833;475;1154;405;10;|;1178;878;17067;636;1114;428;10;|;1613;-573;17243;291;-234;-473;9;|;-107;-259;17287;94;421;369;10;|;-900;-647;16513;168;1330;252;10;|;1372;286;17035;202;263;469;10;|;238;-2006;17142;573;1201;492;9;|;2458;589;17695;356;187;208;11;|;827;-1085;17644;308;233;66;10;|;1;-1373;17214;557;1279;298;9;|;-281;-244;17071;209;517;-36;10;|;-486;-961;17075;467;440;367;10;|;1264;-339;16918;374;476;116;8;|;661;-1330;16789;-37;478;15;9;|;1208;-724;16790;558;1303;335;8;|;-236;-1404;16678;309;426;376;8;|;367;-1402;17308;-32;428;-957;7;|;-849;-360;17640;1;371;635;7;|;-784;90;17924;533;128;-661;5;|;-723;-1062;16413;270;-79;702;7;|;458;-1235;16925;354;-117;194;5;|;-411;-1116;17403;280;777;530;1",
"19/03/2022 15:13;13.6;14.8;|;398;836;17368;-3;924;472;9;|;838;125;17110;675;938;230;10;|;-298;-1253;17164;290;75;-942;10;|;749;78;17221;560;883;601;9;|;1463;752;16904;673;1467;134;10;|;-1085;-1884;16655;239;1067;520;10;|;-27;-1680;16923;393;1032;507;10;|;1308;-1095;17065;-43;328;-548;10;|;-38;-712;16704;-124;373;122;10;|;512;-1318;16830;473;1155;408;10;|;1181;879;17070;637;1113;436;10;|;1610;-567;17239;287;-240;-462;10;|;-108;-250;17297;94;420;370;10;|;-903;-652;16518;169;1326;257;9;|;1371;282;17047;198;263;471;10;|;244;-2006;17137;570;1205;487;9;|;2461;589;17689;354;199;210;11;|;823;-1081;17642;310;235;68;10;|;1;-1370;17214;560;1278;290;9;|;-280;-245;17062;209;517;-31;9;|;-484;-963;17074;463;440;374;10;|;1271;-340;16912;374;477;125;8;|;668;-1331;16786;-37;478;7;9;|;1209;-724;16784;557;1301;329;8;|;-237;-1406;16673;316;425;371;8;|;371;-1401;17307;-30;429;-961;7;|;-854;-356;17647;7;368;631;7;|;-781;85;17934;531;130;-664;5;|;-726;-1062;16400;274;-79;707;6;|;460;-1233;16931;355;-113;196;5;|;-413;-1119;17405;280;780;525;1", "19/03/2022 15:13;13.6;14.8;|;398;836;17368;-3;924;472;9;|;838;125;17110;675;938;230;10;|;-298;-1253;17164;290;75;-942;10;|;749;78;17221;560;883;601;9;|;1463;752;16904;673;1467;134;10;|;-1085;-1884;16655;239;1067;520;10;|;-27;-1680;16923;393;1032;507;10;|;1308;-1095;17065;-43;328;-548;10;|;-38;-712;16704;-124;373;122;10;|;512;-1318;16830;473;1155;408;10;|;1181;879;17070;637;1113;436;10;|;1610;-567;17239;287;-240;-462;10;|;-108;-250;17297;94;420;370;10;|;-903;-652;16518;169;1326;257;9;|;1371;282;17047;198;263;471;10;|;244;-2006;17137;570;1205;487;9;|;2461;589;17689;354;199;210;11;|;823;-1081;17642;310;235;68;10;|;1;-1370;17214;560;1278;290;9;|;-280;-245;17062;209;517;-31;9;|;-484;-963;17074;463;440;374;10;|;1271;-340;16912;374;477;125;8;|;668;-1331;16786;-37;478;7;9;|;1209;-724;16784;557;1301;329;8;|;-237;-1406;16673;316;425;371;8;|;371;-1401;17307;-30;429;-961;7;|;-854;-356;17647;7;368;631;7;|;-781;85;17934;531;130;-664;5;|;-726;-1062;16400;274;-79;707;6;|;460;-1233;16931;355;-113;196;5;|;-413;-1119;17405;280;780;525;1",

8
test/test_ftp_csv_receicer.py
View File

@@ -36,8 +36,8 @@ mock_utils_config.set_config.config.return_value = mock_config_instance
# sys.modules['pyftpdlib.servers'] = MagicMock() # sys.modules['pyftpdlib.servers'] = MagicMock()
# Import the module AFTER mocking dependencies # Import the module AFTER mocking dependencies
import FtpCsvReceiver import ftp_csv_receiver
from FtpCsvReceiver import ( from ftp_csv_receiver import (
extract_value, extract_value,
DummySha256Authorizer, DummySha256Authorizer,
ASEHandler, ASEHandler,
@@ -193,13 +193,13 @@ class TestDummySha256Authorizer(unittest.TestCase):
def test_validate_authentication_wrong_password(self): def test_validate_authentication_wrong_password(self):
self.mock_cursor.fetchall.return_value = [] self.mock_cursor.fetchall.return_value = []
authorizer = DummySha256Authorizer(self.mock_cfg) authorizer = DummySha256Authorizer(self.mock_cfg)
with self.assertRaises(FtpCsvReceiver.AuthenticationFailed): with self.assertRaises(ftp_csv_receiver.AuthenticationFailed):
authorizer.validate_authentication('admin', 'wrongpass', None) authorizer.validate_authentication('admin', 'wrongpass', None)
def test_validate_authentication_unknown_user(self): def test_validate_authentication_unknown_user(self):
self.mock_cursor.fetchall.return_value = [] self.mock_cursor.fetchall.return_value = []
authorizer = DummySha256Authorizer(self.mock_cfg) authorizer = DummySha256Authorizer(self.mock_cfg)
with self.assertRaises(FtpCsvReceiver.AuthenticationFailed): with self.assertRaises(ftp_csv_receiver.AuthenticationFailed):
authorizer.validate_authentication('unknown', 'somepass', None) authorizer.validate_authentication('unknown', 'somepass', None)

2
transform_file.py
View File

@@ -11,7 +11,7 @@ import mysql.connector as mysql
import logging import logging
from utils.time import timestamp_fmt as ts from utils.time import timestamp_fmt as ts
from utils.config import set_config as setting from utils.config import loader as setting
def conn_db(cfg): def conn_db(cfg):
return mysql.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport ) return mysql.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport )

3
utils/config/set_config.py → utils/config/loader.py
View File

@@ -2,9 +2,8 @@
""" """
from configparser import ConfigParser from configparser import ConfigParser
import json
class config: class Config:
def __init__(self): def __init__(self):
c = ConfigParser() c = ConfigParser()
c.read(["/etc/aseftp/ftpcsvreceiver.ini", "./ftpcsvreceiver.ini", c.read(["/etc/aseftp/ftpcsvreceiver.ini", "./ftpcsvreceiver.ini",

12
utils/config/parser.py Normal file
View File

@@ -0,0 +1,12 @@
import re
def extract_value(patterns, primary_source, secondary_source, default='Not Defined'):
"""Extracts the first match for a list of patterns from the primary source.
Falls back to the secondary source if no match is found.
"""
for source in (primary_source, secondary_source):
for pattern in patterns:
matches = re.findall(pattern, source, re.IGNORECASE)
if matches:
return matches[0] # Return the first match immediately
return default # Return default if no matches are found

0
ase_db/__init__.py → utils/database/__init__.py
View File

3
utils/db/connect_db → utils/database/connection.py
View File

@@ -3,7 +3,7 @@ import mysql.connector
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def conn_db(cfg): def connetti_db(cfg):
"""Establishes a connection to the MySQL database. """Establishes a connection to the MySQL database.
Args: Args:
@@ -15,6 +15,7 @@ def conn_db(cfg):
try: try:
conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport) conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport)
conn.autocommit = True conn.autocommit = True
logger.info("Connected")
return conn return conn
except mysql.connector.Error as e: except mysql.connector.Error as e:
logger.error(f'{e}') logger.error(f'{e}')

19
ase_db/get_nodes_type.py → utils/database/get_nodes_type.py
View File

@@ -1,6 +1,11 @@
import mysql.connector import logging
import datetime import datetime
import os import os
import mysql.connector
from utils.database.connection import connetti_db
logger = logging.getLogger(__name__)
def get_timestamp(log_type): def get_timestamp(log_type):
"""Generates a timestamp string for logging.""" """Generates a timestamp string for logging."""
@@ -23,14 +28,12 @@ def get_nodes_type(db_lar, server, username, password, tool, unit, channels, nod
ain (list): An empty list to store the 'ain' values. ain (list): An empty list to store the 'ain' values.
din (list): An empty list to store the 'din' values. din (list): An empty list to store the 'din' values.
""" """
try: try:
dbh = mysql.connector.connect( conn = connetti_db(cfg)
host=server,
database=db_lar, cursor = conn.cursor(dictionary=True)
user=username,
password=password
)
cursor = dbh.cursor(dictionary=True)
query = f""" query = f"""
SELECT t.name AS name, n.seq AS seq, n.num AS num, n.channels AS channels, y.type AS type, n.ain AS ain, n.din AS din SELECT t.name AS name, n.seq AS seq, n.num AS num, n.channels AS channels, y.type AS type, n.ain AS ain, n.din AS din

0
utils/db/__init__.py
View File

0
unit_tool_mod/__init__.py → utils/ftp/__init__.py
View File

52
utils/ftp/file_management.py Normal file
View File

@@ -0,0 +1,52 @@
import os
import logging
import mysql.connector
from utils.database.connection import connetti_db
from utils.config.parser import extract_value
logger = logging.getLogger(__name__)
def on_file_received(self, file):
"""Handles the event when a file is successfully received.
Args:
file: The path to the received file.
"""
if not os.stat(file).st_size:
os.remove(file)
logging.info(f'File {file} is empty: removed.')
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if (fileExtension.upper() in (cfg.fileext)):
with open(file, 'r') as csvfile:
lines = csvfile.readlines()
unit_name = extract_value(cfg.units_name, filename, str(lines[0:9]))
unit_type = extract_value(cfg.units_type, filename, str(lines[0:9]))
tool_name = extract_value(cfg.tools_name, filename, str(lines[0:9]))
tool_type = extract_value(cfg.tools_type, filename, str(lines[0:9]))
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
try:
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (filename, unit_name, unit_type, tool_name, tool_type, tool_data) VALUES (%s, %s, %s, %s, %s, %s)", (filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines)))
conn.commit()
conn.close()
except Exception as e:
logging.error(f'File {file} not loaded. Held in user path.')
logging.error(f'{e}')
else:
os.remove(file)
logging.info(f'File {file} loaded: removed.')

140
utils/ftp/user_admin.py Normal file
View File

@@ -0,0 +1,140 @@
import os
import mysql.connector
import logging
from hashlib import sha256
from pathlib import Path
from utils.database.connection import connetti_db
logger = logging.getLogger(__name__)
def ftp_SITE_ADDU(self, line):
"""Adds a virtual user, creates their directory, and saves their details to the database.
"""
cfg = self.cfg
try:
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = sha256(password.encode("UTF-8")).hexdigest() # Hash the password
except IndexError:
self.respond('501 SITE ADDU failed. Command needs 2 arguments')
else:
try:
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.respond(f'551 Error in create virtual user path: {e}')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm=cfg.defperm)
# Save the user to the database
# Define the database connection
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Create a cursor
cur = conn.cursor()
cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbusertable} (ftpuser, hash, virtpath, perm) VALUES ('{user}', '{hash}', '{cfg.virtpath + user}', '{cfg.defperm}')")
conn.commit()
conn.close()
logging.info(f"User {user} created.")
self.respond('200 SITE ADDU successful.')
except Exception as e:
self.respond(f'501 SITE ADDU failed: {e}.')
print(e)
def ftp_SITE_DISU(self, line):
"""Removes a virtual user from the authorizer and marks them as deleted in the database."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Remove the user from the authorizer
self.authorizer.remove_user(str(user))
# Delete the user from database
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = now() WHERE ftpuser = '{user}'")
conn.commit()
conn.close()
logging.info(f"User {user} deleted.")
self.respond('200 SITE DISU successful.')
except Exception as e:
self.respond('501 SITE DISU failed.')
print(e)
def ftp_SITE_ENAU(self, line):
"""Restores a virtual user by updating their status in the database and adding them back to the authorizer."""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
try:
# Restore the user into database
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
try:
cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = null WHERE ftpuser = '{user}'")
conn.commit()
except Exception as e:
logging.error(f"Update DB failed: {e}")
cur.execute(f"SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE ftpuser = '{user}'")
ftpuser, hash, virtpath, perm = cur.fetchone()
self.authorizer.add_user(ftpuser, hash, virtpath, perm)
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except Exception as e:
self.responde(f'551 Error in create virtual user path: {e}')
conn.close()
logging.info(f"User {user} restored.")
self.respond('200 SITE ENAU successful.')
except Exception as e:
self.respond('501 SITE ENAU failed.')
print(e)
def ftp_SITE_LSTU(self, line):
"""Lists all virtual users from the database."""
cfg = self.cfg
users_list = []
try:
# Connect to the SQLite database to fetch users
try:
conn = connetti_db(cfg)
except mysql.connector.Error as e:
print(f"Error: {e}")
logging.error(f'{e}')
# Crea un cursore
cur = conn.cursor()
self.push("214-The following virtual users are defined:\r\n")
cur.execute(f'SELECT ftpuser, perm, disabled_at FROM {cfg.dbname}.{cfg.dbusertable}')
[users_list.append(f'Username: {ftpuser}\tPerms: {perm}\tDisabled: {disabled_at}\r\n') for ftpuser, perm, disabled_at in cur.fetchall()]
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
except Exception as e:
self.respond(f'501 list users failed: {e}')

1
utils/parsers/G801_parser.py
View File

@@ -1 +0,0 @@
locals

0
unit_tool_mod/g801_mums.py → utils/parsers/g801_mums.py
View File

0
unit_tool_mod/g801_mux.py → utils/parsers/g801_mux.py
View File

1
utils/time/__init__.py
View File

@@ -1 +0,0 @@
"""Utilità per i formati timestamp"""

28
utils/time/date_refmt.py
View File

@@ -1,28 +0,0 @@
"""Funzioni per formato data
"""
from datetime import datetime
from re import search
def dateFmt(date):
t = date.replace("/", "-")
if search('^\d\d\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%Y-%m-%d")
elif search('^\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%y-%m-%d")
elif search('^\d\d-\d\d-\d\d\d\d$', t):
d = datetime.strptime(t, "%d-%m-%Y")
return datetime.strftime(d, "%Y-%m-%d")
def dateTimeFmt(date):
t = date.replace("/", "-")
if search('^\d\d\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
elif search('^\d\d-\d\d-\d\d$', t):
d = datetime.strptime(t, "%y-%m-%d %H:%M:%S")
elif search('^\d\d-\d\d-\d\d\d\d$', t):
d = datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
return datetime.strftime(d, "%Y-%m-%d")

25
utils/time/dt_convert.py
View File

@@ -1,25 +0,0 @@
"""Funzioni per convertire formato data
"""
import datetime
def dateFmt(date):
t = date.replace("/", "-")
try:
datetime.datetime.strptime(t, "%Y-%m-%d")
return t
except ValueError:
d = datetime.datetime.strptime(t, "%d-%m-%Y")
return datetime.datetime.strftime(d, "%Y-%m-%d")
def dateTimeFmt(date):
t = date.replace("/", "-")
try:
datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
return t
except ValueError:
d = datetime.datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
return datetime.datetime.strftime(d, "%Y-%m-%d %H:%M:%S")

10
utils/time/timestamp_fmt.py
View File

@@ -1,10 +0,0 @@
"""Funzioni per timestamp
"""
from datetime import datetime
def timestamp(t):
fmt = {"log": "%Y-%m-%d %H:%M:%S", "tms": "%Y%m%d%H%M%S"}
return datetime.now().strftime(fmt[t])

0
utils/datefmt/__init__.py → utils/timestamp/__init__.py
View File

0
utils/datefmt/date_check.py → utils/timestamp/date_check.py
View File