initial
This commit is contained in:
378
ftpReceiver/FtpCsvReceiver.py
Executable file
378
ftpReceiver/FtpCsvReceiver.py
Executable file
@@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
# import ssl
|
||||
import pika
|
||||
import re
|
||||
import logging
|
||||
import sqlite3
|
||||
|
||||
from hashlib import md5
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
from utils.time import timestamp_fmt as ts
|
||||
from utils.time import date_refmt as df
|
||||
from utils.config import set_config as setting
|
||||
|
||||
from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler
|
||||
from pyftpdlib.servers import FTPServer
|
||||
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
|
||||
|
||||
|
||||
def send_mail(sev, msg, cfg):
|
||||
msg = MIMEText(cfg.message + "\n" + msg)
|
||||
msg["Subject"] = cfg.subject + " " + sev
|
||||
msg["From"] = cfg.sender
|
||||
msg["To"] = cfg.receivers
|
||||
conn = SMTP(
|
||||
host=cfg.smtphost,
|
||||
port=cfg.smtpport,
|
||||
local_hostname=None,
|
||||
timeout=5,
|
||||
source_address=None,
|
||||
)
|
||||
conn.set_debuglevel(cfg.debuglevel)
|
||||
try:
|
||||
conn.login(cfg.sender, cfg.password)
|
||||
conn.sendmail(cfg.sender, cfg.receivers, msg.as_string())
|
||||
except SMTPAuthenticationError:
|
||||
logging.error(
|
||||
"Mail failed: {}.".format("SMTP authentication error")
|
||||
)
|
||||
except:
|
||||
logging.info(
|
||||
"Mail failed: {}.".format("CUSTOM_ERROR")
|
||||
)
|
||||
finally:
|
||||
conn.quit()
|
||||
|
||||
|
||||
class mq:
|
||||
def __init__(self, cfg):
|
||||
parameters = pika.URLParameters(
|
||||
"amqp://"
|
||||
+ cfg.mquser
|
||||
+ ":"
|
||||
+ cfg.mqpass
|
||||
+ "@"
|
||||
+ cfg.mqhost
|
||||
+ ":"
|
||||
+ cfg.mqport
|
||||
+ "/%2F"
|
||||
)
|
||||
connection = pika.BlockingConnection(parameters)
|
||||
self.channel = connection.channel()
|
||||
self.channel.queue_declare(queue=cfg.csv_queue, durable=True)
|
||||
|
||||
def write(self, msg, cfg):
|
||||
try:
|
||||
props = pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_encoding='utf-8',
|
||||
timestamp=msg["timestamp"],)
|
||||
self.channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=cfg.csv_queue,
|
||||
body=msg["payload"],
|
||||
properties=props
|
||||
)
|
||||
logging.info(
|
||||
"Write message {} in queue".format(msg))
|
||||
except:
|
||||
logging.error(
|
||||
"Error write message {} in queue".format(msg))
|
||||
|
||||
def close(self):
|
||||
self.channel.close()
|
||||
|
||||
|
||||
class DummyMD5Authorizer(DummyAuthorizer):
|
||||
def __init__(self, cfg):
|
||||
super().__init__()
|
||||
self.add_user(
|
||||
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
|
||||
|
||||
con = sqlite3.connect(cfg.virtusersdb)
|
||||
cur = con.cursor()
|
||||
cur.execute(
|
||||
'''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''')
|
||||
cur.execute(
|
||||
'''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''')
|
||||
|
||||
for row in cur.execute('SELECT * FROM virtusers'):
|
||||
self.add_user(row[0], row[1], row[2], perm=row[3])
|
||||
con.close()
|
||||
|
||||
def validate_authentication(self, username, password, handler):
|
||||
hash = md5(password.encode("UTF-8")).hexdigest()
|
||||
try:
|
||||
if self.user_table[username]["pwd"] != hash:
|
||||
raise KeyError
|
||||
except KeyError:
|
||||
raise AuthenticationFailed
|
||||
|
||||
|
||||
class ASEHandler(FTPHandler):
|
||||
|
||||
def __init__(self, conn, server, ioloop=None):
|
||||
super().__init__(conn, server, ioloop)
|
||||
self.proto_cmds = FTPHandler.proto_cmds.copy()
|
||||
self.proto_cmds.update(
|
||||
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
|
||||
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
|
||||
)
|
||||
self.proto_cmds.update(
|
||||
{'SITE DELU': dict(perm='M', auth=True, arg=True,
|
||||
help='Syntax: SITE <SP> DELU USERNAME (remove virtual user).')}
|
||||
)
|
||||
self.proto_cmds.update(
|
||||
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
|
||||
help='Syntax: SITE <SP> LSTU (list virtual users).')}
|
||||
)
|
||||
|
||||
def on_file_received(self, file):
|
||||
unitType = ""
|
||||
unitName = ""
|
||||
toolName = ""
|
||||
toolType = ""
|
||||
fileDate = ""
|
||||
fileTime = ""
|
||||
queue = ""
|
||||
if not os.stat(file).st_size:
|
||||
os.remove(file)
|
||||
logging.info(
|
||||
"File {} was empty: removed.".format(file))
|
||||
else:
|
||||
cfg = self.cfg
|
||||
path, filenameExt = os.path.split(file)
|
||||
filename, fileExtension = os.path.splitext(filenameExt)
|
||||
if (fileExtension.upper() in (cfg.fileext)):
|
||||
if m := re.match(
|
||||
r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$",
|
||||
filename,
|
||||
re.I,
|
||||
):
|
||||
unitType = m.group(1).upper()
|
||||
unitName = m.group(2).upper()
|
||||
toolName = m.group(3).upper()
|
||||
toolType = "N/A"
|
||||
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
|
||||
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
|
||||
|
||||
elif re.match(
|
||||
r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
|
||||
):
|
||||
with open(file, "r") as fileCsv:
|
||||
try:
|
||||
for i, line in enumerate(fileCsv.readlines(4096), 1):
|
||||
if m1 := re.match(
|
||||
r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
|
||||
line,
|
||||
re.I,
|
||||
):
|
||||
fileDate = m1.group(2)
|
||||
fileTime = m1.group(3)
|
||||
|
||||
elif m2 := re.match(
|
||||
r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
|
||||
line,
|
||||
re.I,
|
||||
):
|
||||
unitType = m2.group(1).upper()
|
||||
unitName = m2.group(2).upper()
|
||||
|
||||
elif m3 := re.match(
|
||||
r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
|
||||
line,
|
||||
re.I,
|
||||
):
|
||||
if m3.group(2):
|
||||
toolType = m3.group(1).upper()
|
||||
toolName = m3.group(2).upper()
|
||||
else:
|
||||
toolType = "".join(
|
||||
re.findall(
|
||||
"^[a-zA-Z]+", m3.group(1))
|
||||
).upper()
|
||||
toolName = m3.group(1).upper()
|
||||
break
|
||||
except:
|
||||
logging.error(
|
||||
"Error: {}.".format(sys.exc_info()[1]))
|
||||
fileCsv.close
|
||||
|
||||
logging.info(
|
||||
"{} - {} - {} - {} - {} {}.".format(
|
||||
unitType,
|
||||
unitName,
|
||||
toolName,
|
||||
toolType,
|
||||
df.dateFmt(fileDate),
|
||||
fileTime,
|
||||
)
|
||||
)
|
||||
newPath = cfg.csvfs + "/" + self.username + "/received/" + \
|
||||
unitName.upper() + "/"
|
||||
newFilename = (
|
||||
newPath + filename + "_" +
|
||||
str(ts.timestamp("tms") + fileExtension)
|
||||
)
|
||||
fileRenamed = file + "_" + str(ts.timestamp("tms"))
|
||||
os.rename(file, fileRenamed)
|
||||
try:
|
||||
os.makedirs(newPath)
|
||||
logging.info("Path {} created.".format(newPath))
|
||||
except FileExistsError:
|
||||
logging.info("Path {} already exists.".format(newPath))
|
||||
try:
|
||||
shutil.move(fileRenamed, newFilename)
|
||||
logging.info("{} moved into {}.".format(
|
||||
filenameExt, newFilename))
|
||||
except OSError:
|
||||
logging.error("Error to move {} into {}.".format(
|
||||
filenameExt, newFilename))
|
||||
send_mail(
|
||||
"Error", "OS error move " + filenameExt + " to " + newFilename, cfg
|
||||
)
|
||||
now = datetime.now()
|
||||
|
||||
mq_message = {"payload": "{};{};{};{};{};{};{}".format(
|
||||
unitType,
|
||||
unitName,
|
||||
toolName,
|
||||
toolType,
|
||||
df.dateFmt(fileDate),
|
||||
fileTime,
|
||||
newFilename),
|
||||
"timestamp": int(datetime.timestamp(now)*1000000)
|
||||
}
|
||||
try:
|
||||
queue = mq(cfg)
|
||||
queue.write(mq_message, cfg)
|
||||
logging.info("Queue message: {}.".format(mq_message))
|
||||
except:
|
||||
logging.error(
|
||||
"Error to put message in queue: {}.".format(mq_message))
|
||||
send_mail(
|
||||
"Error", "Error to put message " + mq_message + " in queue.", cfg
|
||||
)
|
||||
finally:
|
||||
queue.close()
|
||||
|
||||
def on_incomplete_file_received(self, file):
|
||||
# remove partially uploaded files
|
||||
os.remove(file)
|
||||
|
||||
def ftp_SITE_ADDU(self, line):
|
||||
"""
|
||||
add virtual user and save virtuser cfg file
|
||||
create virtuser dir in virtpath cfg path
|
||||
"""
|
||||
cfg = self.cfg
|
||||
parms = line.split()
|
||||
user = os.path.basename(parms[0])
|
||||
password = parms[1]
|
||||
hash = md5(password.encode("UTF-8")).hexdigest()
|
||||
|
||||
try:
|
||||
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
|
||||
except:
|
||||
self.responde('551 Error in create virtual user path.')
|
||||
else:
|
||||
try:
|
||||
self.authorizer.add_user(str(user),
|
||||
hash, cfg.virtpath + "/" + user, perm="lmw")
|
||||
con = sqlite3.connect(cfg.virtusersdb)
|
||||
cur = con.cursor()
|
||||
cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)",
|
||||
(user, hash, cfg.virtpath + user, 'elmw'))
|
||||
con.commit()
|
||||
con.close()
|
||||
logging.info("User {} created.".format(user))
|
||||
self.respond('200 SITE ADDU successful.')
|
||||
except:
|
||||
self.respond('501 SITE ADDU failed.')
|
||||
|
||||
def ftp_SITE_DELU(self, line):
|
||||
"""
|
||||
remove virtual user and save virtuser cfg file
|
||||
"""
|
||||
cfg = self.cfg
|
||||
parms = line.split()
|
||||
user = os.path.basename(parms[0])
|
||||
try:
|
||||
self.authorizer.remove_user(str(user))
|
||||
con = sqlite3.connect(cfg.virtusersdb)
|
||||
cur = con.cursor()
|
||||
cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
|
||||
con.commit()
|
||||
con.close()
|
||||
logging.info("User {} deleted.".format(user))
|
||||
# self.push(' The user path has not been removed!\r\n')
|
||||
self.respond('200 SITE DELU successful.')
|
||||
|
||||
except:
|
||||
self.respond('501 SITE DELU failed.')
|
||||
|
||||
def ftp_SITE_LSTU(self, line):
|
||||
"""
|
||||
list virtual user
|
||||
"""
|
||||
cfg = self.cfg
|
||||
users_list = []
|
||||
try:
|
||||
con = sqlite3.connect(cfg.virtusersdb)
|
||||
cur = con.cursor()
|
||||
self.push("214-The following virtual users are defined:\r\n")
|
||||
for row in cur.execute("SELECT * FROM virtusers").fetchall():
|
||||
users_list.append(
|
||||
" Username: " + row[0] + "\tPerms: " + row[3] + "\r\n")
|
||||
con.close()
|
||||
self.push(''.join(users_list))
|
||||
self.respond("214 LSTU SITE command successful.")
|
||||
|
||||
except:
|
||||
self.respond('501 list users failed.')
|
||||
|
||||
|
||||
def main():
|
||||
cfg = setting.config()
|
||||
|
||||
try:
|
||||
authorizer = DummyMD5Authorizer(cfg)
|
||||
handler = ASEHandler
|
||||
handler.cfg = cfg
|
||||
handler.authorizer = authorizer
|
||||
handler.masquerade_address = cfg.proxyaddr
|
||||
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
|
||||
handler.passive_ports = _range
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s %(message)s",
|
||||
filename=cfg.logfilename,
|
||||
level=logging.INFO,
|
||||
)
|
||||
|
||||
server = FTPServer(("0.0.0.0", 2121), handler)
|
||||
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logging.info(
|
||||
"Info: {}.".format("Shutdown requested...exiting")
|
||||
)F
|
||||
except Exception:
|
||||
print(
|
||||
"{} - PID {:>5} >> Error: {}.".format(
|
||||
ts.timestamp("log"), os.getpid(), sys.exc_info()[1]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user