diff --git a/ftpReceiver/FtpCsvReceiver.py b/ftpReceiver/FtpCsvReceiver.py index 39f1e72..876b4b5 100755 --- a/ftpReceiver/FtpCsvReceiver.py +++ b/ftpReceiver/FtpCsvReceiver.py @@ -7,7 +7,8 @@ import shutil import re import logging -import sqlite3 + +import psycopg2 from hashlib import md5 from pathlib import Path @@ -28,20 +29,25 @@ class DummyMD5Authorizer(DummyAuthorizer): self.add_user( cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3]) - # Connect to the SQLite database for virtual users - con = sqlite3.connect(cfg.virtusersdb) - cur = con.cursor() - # Create the virtusers table if it doesn't exist - cur.execute( - '''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''') - # Create an index on the user column for faster lookups - cur.execute( - '''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''') + # Definisci la connessione al database + conn = psycopg2.connect( + dbname=cfg.dbname, + user=cfg.dbuser, + password=cfg.dbpass, + host=cfg.dbhost, + port=cfg.dbport + ) - # Load existing virtual users from the database - for row in cur.execute('SELECT * FROM virtusers'): - self.add_user(row[0], row[1], row[2], perm=row[3]) - con.close() + # Crea un cursore + cur = conn.cursor() + cur.execute("SELECT ftpuser, hash, virtpath, perm FROM virtusers") + + for ftpuser, hash, virtpath, perm in cur.fetchall(): + self.add_user(ftpuser, hash, virtpath, perm) + try: + Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True) + except: + self.responde('551 Error in create virtual user path.') def validate_authentication(self, username, password, handler): # Validate the user's password against the stored hash @@ -74,121 +80,36 @@ class ASEHandler(FTPHandler): ) def on_file_received(self, file): - # Handle the event when a file is received - unitType = "" - unitName = "" - toolName = "" - toolType = "" - fileDate = "" - fileTime = "" - queue = "" - # Check if the file is empty and remove it if so if not os.stat(file).st_size: os.remove(file) - logging.info( - "File {} was empty: removed.".format(file)) + logging.info(f'File {file} was empty: removed.') else: cfg = self.cfg path, filenameExt = os.path.split(file) filename, fileExtension = os.path.splitext(filenameExt) - # Process the file if it has a valid extension if (fileExtension.upper() in (cfg.fileext)): - # Match the filename against a specific pattern - if m := re.match( - r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$", - filename, - re.I, - ): - unitType = m.group(1).upper() - unitName = m.group(2).upper() - toolName = m.group(3).upper() - toolType = "N/A" - fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4) - fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9) - - # Match against another pattern for different file formats - elif re.match( - r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I - ): - with open(file, "r") as fileCsv: - try: - # Read the CSV file line by line - for i, line in enumerate(fileCsv.readlines(4096), 1): - # Extract creation date and time - if m1 := re.match( - r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$", - line, - re.I, - ): - fileDate = m1.group(2) - fileTime = m1.group(3) - - # Extract unit type and name - elif m2 := re.match( - r"^(\w+\d+)\s(\w+\d+)\;*\n?$", - line, - re.I, - ): - unitType = m2.group(1).upper() - unitName = m2.group(2).upper() - - # Extract tool type and name from the path - elif m3 := re.match( - r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$", - line, - re.I, - ): - if m3.group(2): - toolType = m3.group(1).upper() - toolName = m3.group(2).upper() - else: - toolType = "".join( - re.findall( - "^[a-zA-Z]+", m3.group(1)) - ).upper() - toolName = m3.group(1).upper() - break - except: - logging.error( - "Error: {}.".format(sys.exc_info()[1])) - fileCsv.close - - # Log the extracted information - logging.info( - "{} - {} - {} - {} - {} {}.".format( - unitType, - unitName, - toolName, - toolType, - df.dateFmt(fileDate), - fileTime, + with open(file, 'r') as csvfile: + lines = csvfile.readlines() + conn = psycopg2.connect( + dbname=cfg.dbname, + user=cfg.dbuser, + password=cfg.dbpass, + host=cfg.dbhost, + port=cfg.dbport ) - ) - # Prepare the new path for the received file - newPath = cfg.csvfs + "/" + self.username + "/received/" + \ - unitName.upper() + "/" - newFilename = ( - newPath + filename + "_" + - str(ts.timestamp("tms") + fileExtension) - ) - fileRenamed = file + "_" + str(ts.timestamp("tms")) - # Rename the original file to avoid conflicts - os.rename(file, fileRenamed) - try: - # Create the new directory for the user if it doesn't exist - os.makedirs(newPath) - logging.info("Path {} created.".format(newPath)) - except FileExistsError: - logging.info("Path {} already exists.".format(newPath)) - try: - # Move the renamed file to the new location - shutil.move(fileRenamed, newFilename) - logging.info("{} moved into {}.".format( - filenameExt, newFilename)) - except OSError: - logging.error("Error to move {} into {}.".format( - filenameExt, newFilename)) - now = datetime.now() + + # Crea un cursore + print(file, lines) + cur = conn.cursor() + try: + cur.execute("INSERT INTO received (filename, content) VALUES (%s,%s)" , (filename, lines)) + conn.commit() + conn.close() + except: + logging.error(f'File {file} not loaded. Held in user path.') + else: + os.remove(file) + logging.info(f'File {file} loaded: removed.') def on_incomplete_file_received(self, file): # Remove partially uploaded files @@ -200,32 +121,43 @@ class ASEHandler(FTPHandler): Create a directory for the virtual user in the specified virtpath. """ cfg = self.cfg - parms = line.split() - user = os.path.basename(parms[0]) # Extract the username - password = parms[1] # Get the password - hash = md5(password.encode("UTF-8")).hexdigest() # Hash the password - try: - # Create the user's directory - Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True) + parms = line.split() + user = os.path.basename(parms[0]) # Extract the username + password = parms[1] # Get the password + hash = md5(password.encode("UTF-8")).hexdigest() # Hash the password except: - self.respond('551 Error in create virtual user path.') + self.respond('501 SITE ADDU failed. Command needs 2 arguments') else: try: - # Add the user to the authorizer - self.authorizer.add_user(str(user), - hash, cfg.virtpath + "/" + user, perm="lmw") - # Save the user to the SQLite database - con = sqlite3.connect(cfg.virtusersdb) - cur = con.cursor() - cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)", - (user, hash, cfg.virtpath + user, 'elmw')) - con.commit() - con.close() - logging.info("User {} created.".format(user)) - self.respond('200 SITE ADDU successful.') + # Create the user's directory + Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True) except: - self.respond('501 SITE ADDU failed.') + self.respond('551 Error in create virtual user path.') + else: + try: + # Add the user to the authorizer + self.authorizer.add_user(str(user), + hash, cfg.virtpath + "/" + user, perm="lmw") + # Save the user to the SQLite database + # Definisci la connessione al database + conn = psycopg2.connect( + dbname=cfg.dbname, + user=cfg.dbuser, + password=cfg.dbpass, + host=cfg.dbhost, + port=cfg.dbport + ) + + # Crea un cursore + cur = conn.cursor() + cur.execute("INSERT INTO virtusers (ftpuser, hash, virtpath, perm) VALUES (%s,%s,%s,%s)" , (user, hash, cfg.virtpath + user, 'elmw')) + conn.commit() + conn.close() + logging.info("User {} created.".format(user)) + self.respond('200 SITE ADDU successful.') + except: + self.respond('501 SITE ADDU failed.') def ftp_SITE_DELU(self, line): """ @@ -238,11 +170,21 @@ class ASEHandler(FTPHandler): # Remove the user from the authorizer self.authorizer.remove_user(str(user)) # Delete the user from the SQLite database - con = sqlite3.connect(cfg.virtusersdb) - cur = con.cursor() - cur.execute("DELETE FROM virtusers WHERE user = ?", (user,)) - con.commit() - con.close() + + conn = psycopg2.connect( + dbname=cfg.dbname, + user=cfg.dbuser, + password=cfg.dbpass, + host=cfg.dbhost, + port=cfg.dbport + ) + + # Crea un cursore + cur = conn.cursor() + cur.execute("DELETE FROM virtusers WHERE ftpuser = %s", (user, )) + conn.commit() + conn.close() + logging.info("User {} deleted.".format(user)) self.respond('200 SITE DELU successful.') @@ -257,21 +199,25 @@ class ASEHandler(FTPHandler): users_list = [] try: # Connect to the SQLite database to fetch users - con = sqlite3.connect(cfg.virtusersdb) - cur = con.cursor() + conn = psycopg2.connect( + dbname=cfg.dbname, + user=cfg.dbuser, + password=cfg.dbpass, + host=cfg.dbhost, + port=cfg.dbport + ) + + # Crea un cursore + cur = conn.cursor() self.push("214-The following virtual users are defined:\r\n") - # Fetch and list all virtual users - for row in cur.execute("SELECT * FROM virtusers").fetchall(): - users_list.append( - " Username: " + row[0] + "\tPerms: " + row[3] + "\r\n") - con.close() + cur.execute("SELECT ftpuser, perm FROM virtusers") + [users_list.append(f'Username: {ftpuser}\tPerms: {perm}\r\n') for ftpuser, perm in cur.fetchall()] self.push(''.join(users_list)) self.respond("214 LSTU SITE command successful.") except: self.respond('501 list users failed.') - def main(): # Load the configuration settings cfg = setting.config() @@ -308,6 +254,5 @@ def main(): ) ) - if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/ftpReceiver/ftpcsvreceiver.ini b/ftpReceiver/ftpcsvreceiver.ini index c44d6aa..90d7968 100644 --- a/ftpReceiver/ftpcsvreceiver.ini +++ b/ftpReceiver/ftpcsvreceiver.ini @@ -5,36 +5,13 @@ logFilename = ./ftppylog.log proxyAddr = 0.0.0.0 portRangeWidth = 500 - virtusersdb = /home/alex/aseftp/virtusers.db virtpath = /home/alex/aseftp/ adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT servertype = FTPHandler certfile = /home/alex/aseftp/keycert.pem - fileext = .CSV|.txt + fileext = .CSV|.TXT #servertype = FTPHandler/TLS_FTPHandler - -[mailserver] - hostname = smtps.aruba.it - port = 465 - sender = alessandro.battilani@aseltd.eu - password = taylor1964NFL! - receivers = alessandro.battilani@gmail.com - message = prova messaggio - - bbbbb - ccccc - subject = ciao a domani - debug = 0 - -[mqserver] - hostname = galera1 - port = 5672 - user = asemq - password = Ase2021 - csvQueue = task_queue - elabQueue = elab_queue - [csvfs] path = /home/alex/aseftp/csvfs/ @@ -42,10 +19,9 @@ logFilename = csvElab.log [db] - hostname = 192.168.1.241 - user = root + hostname = 10.211.114.101 + port = 5432 + user = asepg password = batt1l0 - dbName = ase - tableName = rawdata - maxInsertRow = 20000 - valueNum = 16 \ No newline at end of file + dbName = asedb + tableName = virtusers \ No newline at end of file diff --git a/ftpReceiver/utils/config/set_config.py b/ftpReceiver/utils/config/set_config.py index c19175f..78ab160 100644 --- a/ftpReceiver/utils/config/set_config.py +++ b/ftpReceiver/utils/config/set_config.py @@ -14,31 +14,12 @@ class config: self.logfilename = c.get("ftpserver", "logFilename") self.proxyaddr = c.get("ftpserver", "proxyAddr") self.portrangewidth = c.getint("ftpserver", "portRangeWidth") - self.virtusersdb = c.get("ftpserver", "virtusersdb") self.virtpath = c.get("ftpserver", "virtpath") self.adminuser = c.get("ftpserver", "adminuser").split("|") self.servertype = c.get("ftpserver", "servertype") self.certfile = c.get("ftpserver", "certfile") self.fileext = c.get("ftpserver", "fileext").upper().split("|") - # MAIL setting - self.smtphost = c.get("mailserver", "hostname") - self.smtpport = c.getint("mailserver", "port") - self.sender = c.get("mailserver", "sender") - self.password = c.get("mailserver", "password") - self.receivers = c.get("mailserver", "receivers") - self.message = c.get("mailserver", "message") - self.subject = c.get("mailserver", "subject") - self.debuglevel = c.getint("mailserver", "debug") - - # MQ setting - self.mqhost = c.get("mqserver", "hostname") - self.mqport = c.get("mqserver", "port") - self.mquser = c.get("mqserver", "user") - self.mqpass = c.get("mqserver", "password") - self.csv_queue = c.get("mqserver", "csvQueue") - self.elab_queue = c.get("mqserver", "elabQueue") - # CSV FILE setting self.csvfs = c.get("csvfs", "path") @@ -47,9 +28,8 @@ class config: # DB setting self.dbhost = c.get("db", "hostname") + self.dbport = c.getint("db", "port") self.dbuser = c.get("db", "user") self.dbpass = c.get("db", "password") self.dbname = c.get("db", "dbName") - self.table = c.get("db", "tableName") - self.valueNum = c.getint("db", "valueNum") - self.maxInsertRow = c.getint("db", "maxInsertRow") + self.dbtable = c.get("db", "tableName")