diff --git a/.gitignore b/.gitignore index 54af45d..07d3e2b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ pyftpdlib_example.py prova.py dist/FtpCsvReceiver prova1.py +prova_mail.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..500dc13 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Python: File corrente", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index cfdb08c..d82c837 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "python.pythonPath": "/usr/bin/python3.6" + "python.pythonPath": "/usr/bin/python3.8", + "python.formatting.provider": "yapf" } \ No newline at end of file diff --git a/FtpCsvReceiver.py b/FtpCsvReceiver.py old mode 100755 new mode 100644 index 55daac2..d6efb7f --- a/FtpCsvReceiver.py +++ b/FtpCsvReceiver.py @@ -2,10 +2,13 @@ import sys import os +import shutil +import ssl +import pika import re import logging -from smtplib import SMTP_SSL as SMTP # SMTP_SSL port 465 +from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError from email.mime.text import MIMEText from configparser import ConfigParser @@ -20,98 +23,169 @@ from pyftpdlib.filesystems import UnixFilesystem config = {} +def read_config(): + c = ConfigParser() + c.read([ + "/etc/ase/ftpcsvreceiver.ini", + "./ftpcsvreceiver.ini" + ]) + + config["firstport"] = c.getint("ftpserver", "firstport") + config["logfilename"] = c.get("ftpserver", "logfilename") + config["proxyaddr"] = c.get("ftpserver", "proxyaddr") + + config["smtphost"] = c.get("mailserver", "hostname") + config["smtpport"] = c.getint("mailserver", "port") + config["sender"] = c.get("mailserver", "sender") + config["password"] = c.get("mailserver", "password") + config["receivers"] = c.get("mailserver", "receivers") + config["message"] = c.get("mailserver", "message") + config["subject"] = c.get("mailserver", "subject") + config["debuglevel"] = c.getint("mailserver", "debug") + + config["mqhost"] = c.get("mqserver", "hostname") + config["mquser"] = c.get("mqserver", "user") + config["mqpass"] = c.get("mqserver", "password") + + config["csvfs"] = c.get("csvfs", "path") + +def send_mail(sev, msg): + msg = MIMEText(config["message"] + "\n" + msg) + msg["Subject"] = config["subject"] + " " + sev + msg["From"] = config["sender"] + msg["To"] = config["receivers"] + conn = SMTP(host=config["smtphost"], port=config["smtpport"], local_hostname=None, timeout=5, source_address=None ) + conn.set_debuglevel(config["debuglevel"]) + try: + conn.login(config["sender"], config["password"]) + conn.sendmail(config["sender"], config["receivers"], msg.as_string()) + except SMTPAuthenticationError: + logging.error("PID {:>5} >> Mail failed: {}.".format( + os.getpid(), "SMTP authentication error")) + except: + logging.info("PID {:>5} >> Mail failed: {}.".format( + os.getpid(), "CUSTOM_ERROR")) + finally: + conn.quit() + + +def write_queue(msg): + parameters = pika.URLParameters('amqp://' + config["mquser"] + ':' + + config["mqpass"] + '@' + config["mqhost"] + + ':5672/%2F') + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + channel.queue_declare(queue='task_queue', durable=True) + channel.basic_publish( + exchange='', + routing_key='task_queue', + body=msg, + properties=pika.BasicProperties( + delivery_mode=2, # make message persistent + )) + logging.info("PID {:>5} >> write message {} in queue".format( + os.getpid(), msg)) + connection.close() + + class ASEHandler(FTPHandler): def on_file_received(self, file): path, filenameExt = os.path.split(file) filename, fileExtension = os.path.splitext(filenameExt) - if re.match( r'^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$', filename, re.I) : - m = re.match( r'^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$', filename, re.I) + if (m := re.match( + r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$", + filename, + re.I, + )): unitType = m.group(1).upper() unitName = m.group(2).upper() toolName = m.group(3).upper() toolType = "N/A" fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4) fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9) - elif re.match( r'^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$', filename, re.I) : + elif re.match(r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", + filename, re.I): with open(file, "r") as fileCsv: try: for i, line in enumerate(fileCsv.readlines(4096), 1): if i == 1: - m1 = re.match( r'File Creation Date: (\d*\/\d*\/\d*)\s(\d*:\d*:\d*).*', line, re.I ) + m1 = re.match( + r"File Creation Date: (\d*\/\d*\/\d*)\s(\d*:\d*:\d*).*", + line, + re.I, + ) fileDate = m1.group(1) fileTime = m1.group(2) if i == 2: - unitType, unitName = line.strip('\n').replace(";","").replace(",","").split(" ") + unitType, unitName = (line.strip("\n").replace( + ";", "").replace(",", "").split(" ")) if i == 6: - m6 = re.match( r'SD path: .*\/.*\/(.*)\/(.*)\..*', line, re.I ) + m6 = re.match(r"SD path: .*\/.*\/(.*)\/(.*)\..*", + line, re.I) toolType = m6.group(1) toolName = m6.group(2) break except: - logging.error("PID {:>5} >> Error: {}.".format( os.getpid(), sys.exc_info()[1])) + logging.error("PID {:>5} >> Error: {}.".format( + os.getpid(), + sys.exc_info()[1])) fileCsv.close - - logging.info("PID {:>5} >> {} - {} - {} - {} - {} {}.".format(os.getpid(), unitType.upper(), unitName.upper(), toolName.upper(), toolType.upper(), df.dateFmt(fileDate), fileTime )) - newPath = '/home/' + self.username + '/received/' + unitName.upper() + '/' - newFilename = newPath + filename + '_' + str(ts.timestamp("tms") + fileExtension) + + logging.info("PID {:>5} >> {} - {} - {} - {} - {} {}.".format( + os.getpid(), + unitType.upper(), + unitName.upper(), + toolName.upper(), + toolType.upper(), + df.dateFmt(fileDate), + fileTime, + )) + newPath = config["csvfs"] + self.username + "/received/" + unitName.upper( + ) + "/" + newFilename = (newPath + filename + "_" + + str(ts.timestamp("tms") + fileExtension)) try: os.makedirs(newPath) - logging.info("PID {:>5} >> path {} created.".format(os.getpid(), newPath)) + logging.info("PID {:>5} >> path {} created.".format( + os.getpid(), newPath)) except FileExistsError: - logging.info("PID {:>5} >> path {} already exists.".format(os.getpid(), newPath)) - try: - os.rename(file, newFilename) - logging.info("PID {:>5} >> {} moved into {}.".format(os.getpid(), filenameExt, newFilename)) + logging.info("PID {:>5} >> path {} already exists.".format( + os.getpid(), newPath)) + try: + shutil.move(file, newFilename) + logging.info("PID {:>5} >> {} moved into {}.".format( + os.getpid(), filenameExt, newFilename)) except OSError: - logging.error("PID {:>5} >> Error to move {} into {}.".format(os.getpid(), filenameExt, newFilename)) - send_mail("Error", "OS error move " + filenameExt + " to " + newFilename) + logging.error("PID {:>5} >> Error to move {} into {}.".format( + os.getpid(), filenameExt, newFilename)) + # send_mail("Error", + # "OS error move " + filenameExt + " to " + newFilename) + + + mq_message = "{};{};{};{};{};{};{}".format( + unitType.upper(), + unitName.upper(), + toolName.upper(), + toolType.upper(), + df.dateFmt(fileDate), + fileTime, + newFilename, + ) + write_queue(mq_message) + logging.info("PID {:>5} >> queue message: {}.".format( + os.getpid(), mq_message)) def on_incomplete_file_received(self, file): # remove partially uploaded files os.remove(file) -def read_config(): - c = ConfigParser() - c.read(['/etc/ase/ftpcsvreceiver.ini', - '/home/alex/devel/python/ftpcsvreceiver.ini']) - - config["firstport"] = c.getint('ftpserver', 'firstport') - config["logfilename"] = c.get('ftpserver', 'logfilename') - config["proxyaddr"] = c.get('ftpserver', 'proxyaddr') - config["smtphost"] = c.get('mailserver', 'hostname') - config["smtpport"] = c.getint('mailserver', 'port') - config["sender"] = c.get('mailserver', 'sender') - config["password"] = c.get('mailserver', 'password') - config["receivers"] = c.get('mailserver', 'receivers') - config["message"] = c.get('mailserver', 'message') - config["subject"] = c.get('mailserver', 'subject') - config["debuglevel"] = c.getint('mailserver', 'debug') - -def send_mail(sev, msg): - msg = MIMEText(config["message"] + " " + msg) - msg['Subject'] = config["subject"] + " " + sev - msg['From'] = config["sender"] - msg['To'] = config["receivers"] - - conn = SMTP(config["smtphost"]) - conn.set_debuglevel(config["debuglevel"]) - conn.login(config["sender"], config["password"]) - try: - conn.sendmail(config["sender"], config["receivers"], msg.as_string()) - except: - logging.info("PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR")) - finally: - conn.quit() - def main(): - read_config() - try: - authorizer = UnixAuthorizer( - rejected_users=["root"], require_valid_shell=True) + authorizer = UnixAuthorizer(rejected_users=["root"], + require_valid_shell=True) handler = ASEHandler handler.authorizer = authorizer handler.abstracted_fs = UnixFilesystem @@ -119,16 +193,23 @@ def main(): _range = list(range(config["firstport"], config["firstport"] + 20)) handler.passive_ports = _range - logging.basicConfig(format='%(asctime)s %(message)s', - filename="/var/log/" + config["logfilename"], level=logging.INFO) + logging.basicConfig( + format="%(asctime)s %(message)s", + filename="/var/log/" + config["logfilename"], + level=logging.INFO, + ) + + server = FTPServer(("0.0.0.0", 21), handler) - server = FTPServer(('0.0.0.0', 21), handler) - server.serve_forever() except KeyboardInterrupt: - logging.info("PID {:>5} >> Info: {}.".format(os.getpid(), "Shutdown requested...exiting")) + logging.info("PID {:>5} >> Info: {}.".format( + os.getpid(), "Shutdown requested...exiting")) except Exception: - print("{} - PID {:>5} >> Error: {}.".format(ts.timestamp("log"), os.getpid(), sys.exc_info()[1])) - + print("{} - PID {:>5} >> Error: {}.".format(ts.timestamp("log"), + os.getpid(), + sys.exc_info()[1])) + + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/ftpcsvreceiver.ini b/ftpcsvreceiver.ini index 2424252..6cde5e9 100644 --- a/ftpcsvreceiver.ini +++ b/ftpcsvreceiver.ini @@ -14,5 +14,12 @@ bbbbb ccccc subject = ciao a domani - debug = 0 - \ No newline at end of file + debug = 1 + +[mqserver] + hostname = 192.168.1.242 + user = alex + password = batt1l0 + +[csvfs] + path = /home/ \ No newline at end of file