409 lines
14 KiB
Python
Executable File
409 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import sys
|
|
import os
|
|
import shutil
|
|
import ssl
|
|
import pika
|
|
import re
|
|
import logging
|
|
import sqlite3
|
|
|
|
from hashlib import md5
|
|
from pathlib import Path
|
|
|
|
from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError
|
|
from email.mime.text import MIMEText
|
|
|
|
from utils.time import timestamp_fmt as ts
|
|
from utils.time import date_refmt as df
|
|
from utils.config import set_config as setting
|
|
|
|
from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler
|
|
from pyftpdlib.servers import FTPServer
|
|
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
|
|
|
|
|
|
def send_mail(sev, msg, cfg):
|
|
msg = MIMEText(cfg.message + "\n" + msg)
|
|
msg["Subject"] = cfg.subject + " " + sev
|
|
msg["From"] = cfg.sender
|
|
msg["To"] = cfg.receivers
|
|
conn = SMTP(
|
|
host=cfg.smtphost,
|
|
port=cfg.smtpport,
|
|
local_hostname=None,
|
|
timeout=5,
|
|
source_address=None,
|
|
)
|
|
conn.set_debuglevel(cfg.debuglevel)
|
|
try:
|
|
conn.login(cfg.sender, cfg.password)
|
|
conn.sendmail(cfg.sender, cfg.receivers, msg.as_string())
|
|
except SMTPAuthenticationError:
|
|
logging.error(
|
|
"PID {:>5} >> Mail failed: {}.".format(
|
|
os.getpid(), "SMTP authentication error"
|
|
)
|
|
)
|
|
except:
|
|
logging.info(
|
|
"PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR")
|
|
)
|
|
finally:
|
|
conn.quit()
|
|
|
|
|
|
class mq:
|
|
def __init__(self, cfg):
|
|
parameters = pika.URLParameters(
|
|
"amqp://"
|
|
+ cfg.mquser
|
|
+ ":"
|
|
+ cfg.mqpass
|
|
+ "@"
|
|
+ cfg.mqhost
|
|
+ ":"
|
|
+ cfg.mqport
|
|
+ "/%2F"
|
|
)
|
|
connection = pika.BlockingConnection(parameters)
|
|
self.channel = connection.channel()
|
|
self.channel.queue_declare(queue=cfg.csv_queue, durable=True)
|
|
|
|
def write(self, msg, cfg):
|
|
try:
|
|
self.channel.basic_publish(
|
|
exchange="",
|
|
routing_key=cfg.csv_queue,
|
|
body=msg,
|
|
properties=pika.BasicProperties(
|
|
delivery_mode=2, # make message persistent
|
|
),
|
|
)
|
|
logging.info(
|
|
"PID {:>5} >> write message {} in queue".format(
|
|
os.getpid(), msg)
|
|
)
|
|
except:
|
|
logging.error(
|
|
"PID {:>5} >> error write message {} in queue".format(
|
|
os.getpid(), msg)
|
|
)
|
|
|
|
def close(self):
|
|
self.channel.close()
|
|
|
|
|
|
class DummyMD5Authorizer(DummyAuthorizer):
|
|
def __init__(self, cfg):
|
|
super().__init__()
|
|
self.add_user(
|
|
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
|
|
|
|
con = sqlite3.connect(cfg.virtusersdb)
|
|
cur = con.cursor()
|
|
cur.execute(
|
|
'''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''')
|
|
cur.execute(
|
|
'''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''')
|
|
|
|
for row in cur.execute('SELECT * FROM virtusers'):
|
|
self.add_user(row[0], row[1], row[2], perm=row[3])
|
|
con.close()
|
|
|
|
def validate_authentication(self, username, password, handler):
|
|
hash = md5(password.encode("UTF-8")).hexdigest()
|
|
try:
|
|
if self.user_table[username]["pwd"] != hash:
|
|
raise KeyError
|
|
except KeyError:
|
|
raise AuthenticationFailed
|
|
|
|
|
|
class ASEHandler(FTPHandler):
|
|
|
|
def __init__(self, conn, server, ioloop=None):
|
|
super().__init__(conn, server, ioloop)
|
|
self.proto_cmds = FTPHandler.proto_cmds.copy()
|
|
self.proto_cmds.update(
|
|
{'SITE ADDU': dict(perm='M', auth=True, arg=True,
|
|
help='Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).')}
|
|
)
|
|
self.proto_cmds.update(
|
|
{'SITE DELU': dict(perm='M', auth=True, arg=True,
|
|
help='Syntax: SITE <SP> DELU USERNAME (remove virtual user).')}
|
|
)
|
|
self.proto_cmds.update(
|
|
{'SITE LSTU': dict(perm='M', auth=True, arg=None,
|
|
help='Syntax: SITE <SP> LSTU (list virtual users).')}
|
|
)
|
|
|
|
def on_file_received(self, file):
|
|
unitType = ""
|
|
unitName = ""
|
|
toolName = ""
|
|
toolType = ""
|
|
fileDate = ""
|
|
fileTime = ""
|
|
queue = ""
|
|
if not os.stat(file).st_size:
|
|
os.remove(file)
|
|
logging.info(
|
|
"PID {:>5} >> file {} was empty: removed.".format(
|
|
os.getpid(), file)
|
|
)
|
|
else:
|
|
cfg = self.cfg
|
|
path, filenameExt = os.path.split(file)
|
|
filename, fileExtension = os.path.splitext(filenameExt)
|
|
|
|
if m := re.match(
|
|
r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$",
|
|
filename,
|
|
re.I,
|
|
):
|
|
unitType = m.group(1).upper()
|
|
unitName = m.group(2).upper()
|
|
toolName = m.group(3).upper()
|
|
toolType = "N/A"
|
|
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
|
|
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
|
|
elif re.match(
|
|
r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
|
|
):
|
|
with open(file, "r") as fileCsv:
|
|
try:
|
|
for i, line in enumerate(fileCsv.readlines(4096), 1):
|
|
if m1 := re.match(
|
|
r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
|
|
line,
|
|
re.I,
|
|
):
|
|
fileDate = m1.group(2)
|
|
fileTime = m1.group(3)
|
|
|
|
elif m2 := re.match(
|
|
r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
|
|
line,
|
|
re.I,
|
|
):
|
|
unitType = m2.group(1).upper()
|
|
unitName = m2.group(2).upper()
|
|
|
|
elif m3 := re.match(
|
|
r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
|
|
line,
|
|
re.I,
|
|
):
|
|
if m3.group(2):
|
|
toolType = m3.group(1).upper()
|
|
toolName = m3.group(2).upper()
|
|
else:
|
|
toolType = "".join(
|
|
re.findall("^[a-zA-Z]+", m3.group(1))
|
|
).upper()
|
|
toolName = m3.group(1).upper()
|
|
break
|
|
except:
|
|
logging.error(
|
|
"PID {:>5} >> Error: {}.".format(
|
|
os.getpid(), sys.exc_info()[1]
|
|
)
|
|
)
|
|
fileCsv.close
|
|
|
|
logging.info(
|
|
"PID {:>5} >> {} - {} - {} - {} - {} {}.".format(
|
|
os.getpid(),
|
|
unitType,
|
|
unitName,
|
|
toolName,
|
|
toolType,
|
|
df.dateFmt(fileDate),
|
|
fileTime,
|
|
)
|
|
)
|
|
newPath = cfg.csvfs + self.username + "/received/" + unitName.upper() + "/"
|
|
newFilename = (
|
|
newPath + filename + "_" +
|
|
str(ts.timestamp("tms") + fileExtension)
|
|
)
|
|
fileRenamed = file + "_" + str(ts.timestamp("tms"))
|
|
os.rename(file, fileRenamed)
|
|
try:
|
|
os.makedirs(newPath)
|
|
logging.info(
|
|
"PID {:>5} >> path {} created.".format(
|
|
os.getpid(), newPath)
|
|
)
|
|
except FileExistsError:
|
|
logging.info(
|
|
"PID {:>5} >> path {} already exists.".format(
|
|
os.getpid(), newPath)
|
|
)
|
|
try:
|
|
shutil.move(fileRenamed, newFilename)
|
|
logging.info(
|
|
"PID {:>5} >> {} moved into {}.".format(
|
|
os.getpid(), filenameExt, newFilename
|
|
)
|
|
)
|
|
except OSError:
|
|
logging.error(
|
|
"PID {:>5} >> Error to move {} into {}.".format(
|
|
os.getpid(), filenameExt, newFilename
|
|
)
|
|
)
|
|
send_mail(
|
|
"Error", "OS error move " + filenameExt + " to " + newFilename, cfg
|
|
)
|
|
|
|
mq_message = "{};{};{};{};{};{};{}".format(
|
|
unitType,
|
|
unitName,
|
|
toolName,
|
|
toolType,
|
|
df.dateFmt(fileDate),
|
|
fileTime,
|
|
newFilename,
|
|
)
|
|
try:
|
|
queue = mq(cfg)
|
|
queue.write(mq_message, cfg)
|
|
logging.info(
|
|
"PID {:>5} >> queue message: {}.".format(
|
|
os.getpid(), mq_message)
|
|
)
|
|
except:
|
|
logging.error(
|
|
"PID {:>5} >> Error to put message in queue: {}.".format(
|
|
os.getpid(), mq_message
|
|
)
|
|
)
|
|
send_mail(
|
|
"Error", "Error to put message " + mq_message + " in queue.", cfg
|
|
)
|
|
finally:
|
|
queue.close()
|
|
|
|
def on_incomplete_file_received(self, file):
|
|
# remove partially uploaded files
|
|
os.remove(file)
|
|
|
|
def ftp_SITE_ADDU(self, line):
|
|
"""
|
|
add virtual user and save virtuser cfg file
|
|
create virtuser dir in virtpath cfg path
|
|
"""
|
|
cfg = self.cfg
|
|
parms = line.split()
|
|
user = os.path.basename(parms[0])
|
|
password = parms[1]
|
|
hash = md5(password.encode("UTF-8")).hexdigest()
|
|
|
|
try:
|
|
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
|
|
except:
|
|
self.responde('551 Error in create virtual user path.')
|
|
else:
|
|
try:
|
|
self.authorizer.add_user(str(user),
|
|
hash, cfg.virtpath + "/" + user, perm="lmw")
|
|
con = sqlite3.connect(cfg.virtusersdb)
|
|
cur = con.cursor()
|
|
cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)",
|
|
(user, hash, cfg.virtpath + user, 'elmw'))
|
|
con.commit()
|
|
con.close()
|
|
logging.info(
|
|
"PID {:>5} >> User {} created.".format(
|
|
os.getpid(), user
|
|
)
|
|
)
|
|
self.respond('200 SITE ADDU successful.')
|
|
except:
|
|
self.respond('501 SITE ADDU failed.')
|
|
|
|
def ftp_SITE_DELU(self, line):
|
|
"""
|
|
remove virtual user and save virtuser cfg file
|
|
"""
|
|
cfg = self.cfg
|
|
parms = line.split()
|
|
user = os.path.basename(parms[0])
|
|
try:
|
|
self.authorizer.remove_user(str(user))
|
|
con = sqlite3.connect(cfg.virtusersdb)
|
|
cur = con.cursor()
|
|
cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
|
|
con.commit()
|
|
con.close()
|
|
logging.info(
|
|
"PID {:>5} >> User {} deleted.".format(
|
|
os.getpid(), user
|
|
)
|
|
)
|
|
self.respond('200 SITE DELU successful.')
|
|
|
|
except:
|
|
self.respond('501 SITE DELU failed.')
|
|
|
|
def ftp_SITE_LSTU(self, line):
|
|
"""
|
|
list virtual user
|
|
"""
|
|
cfg = self.cfg
|
|
users_list = []
|
|
try:
|
|
con = sqlite3.connect(cfg.virtusersdb)
|
|
cur = con.cursor()
|
|
self.push("214-The following virtual users are defined:\r\n")
|
|
for row in cur.execute("SELECT * FROM virtusers").fetchall():
|
|
users_list.append(
|
|
" Username: " + row[0] + " - Perms: " + row[3] + "\r\n")
|
|
con.close()
|
|
self.push(''.join(users_list))
|
|
self.respond("214 LSTU SITE command successful.")
|
|
|
|
except:
|
|
self.respond('501 list users failed.')
|
|
|
|
|
|
def main():
|
|
cfg = setting.config()
|
|
|
|
try:
|
|
authorizer = DummyMD5Authorizer(cfg)
|
|
handler = ASEHandler
|
|
handler.cfg = cfg
|
|
handler.authorizer = authorizer
|
|
handler.masquerade_address = cfg.proxyaddr
|
|
_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
|
|
handler.passive_ports = _range
|
|
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(message)s",
|
|
filename=cfg.logfilename,
|
|
level=logging.INFO,
|
|
)
|
|
|
|
server = FTPServer(("0.0.0.0", 2121), handler)
|
|
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
logging.info(
|
|
"PID {:>5} >> Info: {}.".format(
|
|
os.getpid(), "Shutdown requested...exiting")
|
|
)
|
|
except Exception:
|
|
print(
|
|
"{} - PID {:>5} >> Error: {}.".format(
|
|
ts.timestamp("log"), os.getpid(), sys.exc_info()[1]
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|