diff --git a/ase-receiver/ase-receiver/FtpCsvReceiver.py b/ase-receiver/ase-receiver/FtpCsvReceiver.py index 5decc22..134fde6 100755 --- a/ase-receiver/ase-receiver/FtpCsvReceiver.py +++ b/ase-receiver/ase-receiver/FtpCsvReceiver.py @@ -3,7 +3,7 @@ import sys import os import shutil -import ssl +# import ssl import pika import re import logging @@ -42,13 +42,11 @@ def send_mail(sev, msg, cfg): conn.sendmail(cfg.sender, cfg.receivers, msg.as_string()) except SMTPAuthenticationError: logging.error( - "PID {:>5} >> Mail failed: {}.".format( - os.getpid(), "SMTP authentication error" - ) + "Mail failed: {}.".format("SMTP authentication error") ) except: logging.info( - "PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR") + "Mail failed: {}.".format("CUSTOM_ERROR") ) finally: conn.quit() @@ -82,14 +80,10 @@ class mq: ), ) logging.info( - "PID {:>5} >> write message {} in queue".format( - os.getpid(), msg) - ) + "Write message {} in queue".format(msg)) except: logging.error( - "PID {:>5} >> error write message {} in queue".format( - os.getpid(), msg) - ) + "Error write message {} in queue".format(msg)) def close(self): self.channel.close() @@ -150,142 +144,120 @@ class ASEHandler(FTPHandler): if not os.stat(file).st_size: os.remove(file) logging.info( - "PID {:>5} >> file {} was empty: removed.".format( - os.getpid(), file) - ) + "File {} was empty: removed.".format(file)) else: cfg = self.cfg path, filenameExt = os.path.split(file) filename, fileExtension = os.path.splitext(filenameExt) + if (fileExtension.upper() in (cfg.fileext)): + if m := re.match( + r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$", + filename, + re.I, + ): + unitType = m.group(1).upper() + unitName = m.group(2).upper() + toolName = m.group(3).upper() + toolType = "N/A" + fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4) + fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9) + elif re.match( + r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I + ): + with open(file, "r") as fileCsv: + try: + for i, line in enumerate(fileCsv.readlines(4096), 1): + if m1 := re.match( + r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$", + line, + re.I, + ): + fileDate = m1.group(2) + fileTime = m1.group(3) - if m := re.match( - r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$", - filename, - re.I, - ): - unitType = m.group(1).upper() - unitName = m.group(2).upper() - toolName = m.group(3).upper() - toolType = "N/A" - fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4) - fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9) - elif re.match( - r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I - ): - with open(file, "r") as fileCsv: - try: - for i, line in enumerate(fileCsv.readlines(4096), 1): - if m1 := re.match( - r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$", - line, - re.I, - ): - fileDate = m1.group(2) - fileTime = m1.group(3) + elif m2 := re.match( + r"^(\w+\d+)\s(\w+\d+)\;*\n?$", + line, + re.I, + ): + unitType = m2.group(1).upper() + unitName = m2.group(2).upper() - elif m2 := re.match( - r"^(\w+\d+)\s(\w+\d+)\;*\n?$", - line, - re.I, - ): - unitType = m2.group(1).upper() - unitName = m2.group(2).upper() + elif m3 := re.match( + r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$", + line, + re.I, + ): + if m3.group(2): + toolType = m3.group(1).upper() + toolName = m3.group(2).upper() + else: + toolType = "".join( + re.findall( + "^[a-zA-Z]+", m3.group(1)) + ).upper() + toolName = m3.group(1).upper() + break + except: + logging.error( + "Error: {}.".format(sys.exc_info()[1])) + fileCsv.close - elif m3 := re.match( - r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$", - line, - re.I, - ): - if m3.group(2): - toolType = m3.group(1).upper() - toolName = m3.group(2).upper() - else: - toolType = "".join( - re.findall("^[a-zA-Z]+", m3.group(1)) - ).upper() - toolName = m3.group(1).upper() - break - except: - logging.error( - "PID {:>5} >> Error: {}.".format( - os.getpid(), sys.exc_info()[1] - ) - ) - fileCsv.close + logging.info( + "{} - {} - {} - {} - {} {}.".format( + unitType, + unitName, + toolName, + toolType, + df.dateFmt(fileDate), + fileTime, + ) + ) + newPath = cfg.csvfs + "/" + self.username + "/received/" + \ + unitName.upper() + "/" + newFilename = ( + newPath + filename + "_" + + str(ts.timestamp("tms") + fileExtension) + ) + fileRenamed = file + "_" + str(ts.timestamp("tms")) + os.rename(file, fileRenamed) + try: + os.makedirs(newPath) + logging.info("Path {} created.".format(newPath)) + except FileExistsError: + logging.info("Path {} already exists.".format(newPath)) + try: + shutil.move(fileRenamed, newFilename) + logging.info("{} moved into {}.".format( + filenameExt, newFilename)) + except OSError: + logging.error("Error to move {} into {}.".format( + filenameExt, newFilename)) + send_mail( + "Error", "OS error move " + filenameExt + " to " + newFilename, cfg + ) - logging.info( - "PID {:>5} >> {} - {} - {} - {} - {} {}.".format( - os.getpid(), + mq_message = "{};{};{};{};{};{};{}".format( unitType, unitName, toolName, toolType, df.dateFmt(fileDate), fileTime, + newFilename, ) - ) - newPath = cfg.csvfs + self.username + "/received/" + unitName.upper() + "/" - newFilename = ( - newPath + filename + "_" + - str(ts.timestamp("tms") + fileExtension) - ) - fileRenamed = file + "_" + str(ts.timestamp("tms")) - os.rename(file, fileRenamed) - try: - os.makedirs(newPath) - logging.info( - "PID {:>5} >> path {} created.".format( - os.getpid(), newPath) - ) - except FileExistsError: - logging.info( - "PID {:>5} >> path {} already exists.".format( - os.getpid(), newPath) - ) - try: - shutil.move(fileRenamed, newFilename) - logging.info( - "PID {:>5} >> {} moved into {}.".format( - os.getpid(), filenameExt, newFilename + try: + queue = mq(cfg) + queue.write(mq_message, cfg) + logging.info("Queue message: {}.".format(mq_message)) + except: + logging.error( + "Error to put message in queue: {}.".format(mq_message)) + send_mail( + "Error", "Error to put message " + mq_message + " in queue.", cfg ) - ) - except OSError: - logging.error( - "PID {:>5} >> Error to move {} into {}.".format( - os.getpid(), filenameExt, newFilename - ) - ) - send_mail( - "Error", "OS error move " + filenameExt + " to " + newFilename, cfg - ) - - mq_message = "{};{};{};{};{};{};{}".format( - unitType, - unitName, - toolName, - toolType, - df.dateFmt(fileDate), - fileTime, - newFilename, - ) - try: - queue = mq(cfg) - queue.write(mq_message, cfg) - logging.info( - "PID {:>5} >> queue message: {}.".format( - os.getpid(), mq_message) - ) - except: - logging.error( - "PID {:>5} >> Error to put message in queue: {}.".format( - os.getpid(), mq_message - ) - ) - send_mail( - "Error", "Error to put message " + mq_message + " in queue.", cfg - ) - finally: - queue.close() + finally: + queue.close() def on_incomplete_file_received(self, file): # remove partially uploaded files @@ -316,11 +288,7 @@ class ASEHandler(FTPHandler): (user, hash, cfg.virtpath + user, 'elmw')) con.commit() con.close() - logging.info( - "PID {:>5} >> User {} created.".format( - os.getpid(), user - ) - ) + logging.info("User {} created.".format(user)) self.respond('200 SITE ADDU successful.') except: self.respond('501 SITE ADDU failed.') @@ -339,11 +307,8 @@ class ASEHandler(FTPHandler): cur.execute("DELETE FROM virtusers WHERE user = ?", (user,)) con.commit() con.close() - logging.info( - "PID {:>5} >> User {} deleted.".format( - os.getpid(), user - ) - ) + logging.info("User {} deleted.".format(user)) + # self.push(' The user path has not been removed!\r\n') self.respond('200 SITE DELU successful.') except: @@ -361,7 +326,7 @@ class ASEHandler(FTPHandler): self.push("214-The following virtual users are defined:\r\n") for row in cur.execute("SELECT * FROM virtusers").fetchall(): users_list.append( - " Username: " + row[0] + " - Perms: " + row[3] + "\r\n") + " Username: " + row[0] + "\tPerms: " + row[3] + "\r\n") con.close() self.push(''.join(users_list)) self.respond("214 LSTU SITE command successful.") @@ -393,8 +358,7 @@ def main(): server.serve_forever() except KeyboardInterrupt: logging.info( - "PID {:>5} >> Info: {}.".format( - os.getpid(), "Shutdown requested...exiting") + "Info: {}.".format("Shutdown requested...exiting") ) except Exception: print( diff --git a/ase-receiver/ase-receiver/ftpcsvreceiver.ini b/ase-receiver/ase-receiver/ftpcsvreceiver.ini index 251440a..e42371c 100644 --- a/ase-receiver/ase-receiver/ftpcsvreceiver.ini +++ b/ase-receiver/ase-receiver/ftpcsvreceiver.ini @@ -1,13 +1,16 @@ +# to generete adminuser password hash: +# python3 -c 'from hashlib import md5;print(md5("????admin-password???".encode("UTF-8")).hexdigest())' [ftpserver] firstPort = 40000 logFilename = ./ftppylog.log proxyAddr = 0.0.0.0 - portRangeWidth = 50 + portRangeWidth = 500 virtusersdb = /home/alex/aseftp/virtusers.db virtpath = /home/alex/aseftp/ adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT servertype = FTPHandler certfile = /home/alex/aseftp/keycert.pem + fileext = .CSV|.txt #servertype = FTPHandler/TLS_FTPHandler @@ -22,7 +25,7 @@ bbbbb ccccc subject = ciao a domani - debug = 1 + debug = 0 [mqserver] hostname = 192.168.1.241 @@ -33,7 +36,7 @@ elabQueue = elab_queue [csvfs] - path = /home/ + path = /home/alex/aseftp/csvfs/ [csvelab] logFilename = csvElab.log diff --git a/ase-receiver/ase-receiver/utils/config/set_config.py b/ase-receiver/ase-receiver/utils/config/set_config.py index bb9a811..77b77b8 100644 --- a/ase-receiver/ase-receiver/utils/config/set_config.py +++ b/ase-receiver/ase-receiver/utils/config/set_config.py @@ -18,6 +18,7 @@ class config: self.adminuser = c.get("ftpserver", "adminuser").split("|") self.servertype = c.get("ftpserver", "servertype") self.certfile = c.get("ftpserver", "certfile") + self.fileext = c.get("ftpserver", "fileext").upper().split("|") # MAIL setting self.smtphost = c.get("mailserver", "hostname")