#!/usr/bin/env python3 import sys import os import shutil # import ssl import pika import re import logging import sqlite3 from hashlib import md5 from pathlib import Path from datetime import datetime from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError from email.mime.text import MIMEText from utils.time import timestamp_fmt as ts from utils.time import date_refmt as df from utils.config import set_config as setting from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler from pyftpdlib.servers import FTPServer from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed def send_mail(sev, msg, cfg): msg = MIMEText(cfg.message + "\n" + msg) msg["Subject"] = cfg.subject + " " + sev msg["From"] = cfg.sender msg["To"] = cfg.receivers conn = SMTP( host=cfg.smtphost, port=cfg.smtpport, local_hostname=None, timeout=5, source_address=None, ) conn.set_debuglevel(cfg.debuglevel) try: conn.login(cfg.sender, cfg.password) conn.sendmail(cfg.sender, cfg.receivers, msg.as_string()) except SMTPAuthenticationError: logging.error( "Mail failed: {}.".format("SMTP authentication error") ) except: logging.info( "Mail failed: {}.".format("CUSTOM_ERROR") ) finally: conn.quit() class mq: def __init__(self, cfg): parameters = pika.URLParameters( "amqp://" + cfg.mquser + ":" + cfg.mqpass + "@" + cfg.mqhost + ":" + cfg.mqport + "/%2F" ) connection = pika.BlockingConnection(parameters) self.channel = connection.channel() self.channel.queue_declare(queue=cfg.csv_queue, durable=True) def write(self, msg, cfg): try: props = pika.BasicProperties( delivery_mode=2, content_encoding='utf-8', timestamp=msg["timestamp"],) self.channel.basic_publish( exchange="", routing_key=cfg.csv_queue, body=msg["payload"], properties=props ) logging.info( "Write message {} in queue".format(msg)) except: logging.error( "Error write message {} in queue".format(msg)) def close(self): self.channel.close() class DummyMD5Authorizer(DummyAuthorizer): def __init__(self, cfg): super().__init__() self.add_user( cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3]) con = sqlite3.connect(cfg.virtusersdb) cur = con.cursor() cur.execute( '''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''') cur.execute( '''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''') for row in cur.execute('SELECT * FROM virtusers'): self.add_user(row[0], row[1], row[2], perm=row[3]) con.close() def validate_authentication(self, username, password, handler): hash = md5(password.encode("UTF-8")).hexdigest() try: if self.user_table[username]["pwd"] != hash: raise KeyError except KeyError: raise AuthenticationFailed class ASEHandler(FTPHandler): def __init__(self, conn, server, ioloop=None): super().__init__(conn, server, ioloop) self.proto_cmds = FTPHandler.proto_cmds.copy() self.proto_cmds.update( {'SITE ADDU': dict(perm='M', auth=True, arg=True, help='Syntax: SITE ADDU USERNAME PASSWORD (add virtual user).')} ) self.proto_cmds.update( {'SITE DELU': dict(perm='M', auth=True, arg=True, help='Syntax: SITE DELU USERNAME (remove virtual user).')} ) self.proto_cmds.update( {'SITE LSTU': dict(perm='M', auth=True, arg=None, help='Syntax: SITE LSTU (list virtual users).')} ) def on_file_received(self, file): unitType = "" unitName = "" toolName = "" toolType = "" fileDate = "" fileTime = "" queue = "" if not os.stat(file).st_size: os.remove(file) logging.info( "File {} was empty: removed.".format(file)) else: cfg = self.cfg path, filenameExt = os.path.split(file) filename, fileExtension = os.path.splitext(filenameExt) if (fileExtension.upper() in (cfg.fileext)): if m := re.match( r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$", filename, re.I, ): unitType = m.group(1).upper() unitName = m.group(2).upper() toolName = m.group(3).upper() toolType = "N/A" fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4) fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9) elif re.match( r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I ): with open(file, "r") as fileCsv: try: for i, line in enumerate(fileCsv.readlines(4096), 1): if m1 := re.match( r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$", line, re.I, ): fileDate = m1.group(2) fileTime = m1.group(3) elif m2 := re.match( r"^(\w+\d+)\s(\w+\d+)\;*\n?$", line, re.I, ): unitType = m2.group(1).upper() unitName = m2.group(2).upper() elif m3 := re.match( r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$", line, re.I, ): if m3.group(2): toolType = m3.group(1).upper() toolName = m3.group(2).upper() else: toolType = "".join( re.findall( "^[a-zA-Z]+", m3.group(1)) ).upper() toolName = m3.group(1).upper() break except: logging.error( "Error: {}.".format(sys.exc_info()[1])) fileCsv.close logging.info( "{} - {} - {} - {} - {} {}.".format( unitType, unitName, toolName, toolType, df.dateFmt(fileDate), fileTime, ) ) newPath = cfg.csvfs + "/" + self.username + "/received/" + \ unitName.upper() + "/" newFilename = ( newPath + filename + "_" + str(ts.timestamp("tms") + fileExtension) ) fileRenamed = file + "_" + str(ts.timestamp("tms")) os.rename(file, fileRenamed) try: os.makedirs(newPath) logging.info("Path {} created.".format(newPath)) except FileExistsError: logging.info("Path {} already exists.".format(newPath)) try: shutil.move(fileRenamed, newFilename) logging.info("{} moved into {}.".format( filenameExt, newFilename)) except OSError: logging.error("Error to move {} into {}.".format( filenameExt, newFilename)) send_mail( "Error", "OS error move " + filenameExt + " to " + newFilename, cfg ) now = datetime.now() mq_message = {"payload": "{};{};{};{};{};{};{}".format( unitType, unitName, toolName, toolType, df.dateFmt(fileDate), fileTime, newFilename), "timestamp": int(datetime.timestamp(now)*1000000) } try: queue = mq(cfg) queue.write(mq_message, cfg) logging.info("Queue message: {}.".format(mq_message)) except: logging.error( "Error to put message in queue: {}.".format(mq_message)) send_mail( "Error", "Error to put message " + mq_message + " in queue.", cfg ) finally: queue.close() def on_incomplete_file_received(self, file): # remove partially uploaded files os.remove(file) def ftp_SITE_ADDU(self, line): """ add virtual user and save virtuser cfg file create virtuser dir in virtpath cfg path """ cfg = self.cfg parms = line.split() user = os.path.basename(parms[0]) password = parms[1] hash = md5(password.encode("UTF-8")).hexdigest() try: Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True) except: self.responde('551 Error in create virtual user path.') else: try: self.authorizer.add_user(str(user), hash, cfg.virtpath + "/" + user, perm="lmw") con = sqlite3.connect(cfg.virtusersdb) cur = con.cursor() cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)", (user, hash, cfg.virtpath + user, 'elmw')) con.commit() con.close() logging.info("User {} created.".format(user)) self.respond('200 SITE ADDU successful.') except: self.respond('501 SITE ADDU failed.') def ftp_SITE_DELU(self, line): """ remove virtual user and save virtuser cfg file """ cfg = self.cfg parms = line.split() user = os.path.basename(parms[0]) try: self.authorizer.remove_user(str(user)) con = sqlite3.connect(cfg.virtusersdb) cur = con.cursor() cur.execute("DELETE FROM virtusers WHERE user = ?", (user,)) con.commit() con.close() logging.info("User {} deleted.".format(user)) # self.push(' The user path has not been removed!\r\n') self.respond('200 SITE DELU successful.') except: self.respond('501 SITE DELU failed.') def ftp_SITE_LSTU(self, line): """ list virtual user """ cfg = self.cfg users_list = [] try: con = sqlite3.connect(cfg.virtusersdb) cur = con.cursor() self.push("214-The following virtual users are defined:\r\n") for row in cur.execute("SELECT * FROM virtusers").fetchall(): users_list.append( " Username: " + row[0] + "\tPerms: " + row[3] + "\r\n") con.close() self.push(''.join(users_list)) self.respond("214 LSTU SITE command successful.") except: self.respond('501 list users failed.') def main(): cfg = setting.config() try: authorizer = DummyMD5Authorizer(cfg) handler = ASEHandler handler.cfg = cfg handler.authorizer = authorizer handler.masquerade_address = cfg.proxyaddr _range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth)) handler.passive_ports = _range logging.basicConfig( format="%(asctime)s %(message)s", filename=cfg.logfilename, level=logging.INFO, ) server = FTPServer(("0.0.0.0", 2121), handler) server.serve_forever() except KeyboardInterrupt: logging.info( "Info: {}.".format("Shutdown requested...exiting") )F except Exception: print( "{} - PID {:>5} >> Error: {}.".format( ts.timestamp("log"), os.getpid(), sys.exc_info()[1] ) ) if __name__ == "__main__": main()