alex/python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prove 2

Browse Source
This commit is contained in:
Alessandro Battilani
2020-04-26 17:09:52 +02:00
parent 914dca790f
commit d7b0f79597
5 changed files with 173 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -11,3 +11,4 @@ pyftpdlib_example.py
prova.py
dist/FtpCsvReceiver
prova1.py
prova_mail.py

16
.vscode/launch.json vendored Normal file
View File

@@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: File corrente",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal"
}
]
}

3
.vscode/settings.json vendored
View File

@@ -1,3 +1,4 @@
{
"python.pythonPath": "/usr/bin/python3.6"
"python.pythonPath": "/usr/bin/python3.8",
"python.formatting.provider": "yapf"
}

201
FtpCsvReceiver.py Executable file → Normal file
View File

@@ -2,10 +2,13 @@
import sys
import os
import shutil
import ssl
import pika
import re
import logging
from smtplib import SMTP_SSL as SMTP # SMTP_SSL port 465
from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError
from email.mime.text import MIMEText
from configparser import ConfigParser
@@ -20,98 +23,169 @@ from pyftpdlib.filesystems import UnixFilesystem
config = {}
def read_config():
c = ConfigParser()
c.read([
"/etc/ase/ftpcsvreceiver.ini",
"./ftpcsvreceiver.ini"
])
config["firstport"] = c.getint("ftpserver", "firstport")
config["logfilename"] = c.get("ftpserver", "logfilename")
config["proxyaddr"] = c.get("ftpserver", "proxyaddr")
config["smtphost"] = c.get("mailserver", "hostname")
config["smtpport"] = c.getint("mailserver", "port")
config["sender"] = c.get("mailserver", "sender")
config["password"] = c.get("mailserver", "password")
config["receivers"] = c.get("mailserver", "receivers")
config["message"] = c.get("mailserver", "message")
config["subject"] = c.get("mailserver", "subject")
config["debuglevel"] = c.getint("mailserver", "debug")
config["mqhost"] = c.get("mqserver", "hostname")
config["mquser"] = c.get("mqserver", "user")
config["mqpass"] = c.get("mqserver", "password")
config["csvfs"] = c.get("csvfs", "path")
def send_mail(sev, msg):
msg = MIMEText(config["message"] + "\n" + msg)
msg["Subject"] = config["subject"] + " " + sev
msg["From"] = config["sender"]
msg["To"] = config["receivers"]
conn = SMTP(host=config["smtphost"], port=config["smtpport"], local_hostname=None, timeout=5, source_address=None )
conn.set_debuglevel(config["debuglevel"])
try:
conn.login(config["sender"], config["password"])
conn.sendmail(config["sender"], config["receivers"], msg.as_string())
except SMTPAuthenticationError:
logging.error("PID {:>5} >> Mail failed: {}.".format(
os.getpid(), "SMTP authentication error"))
except:
logging.info("PID {:>5} >> Mail failed: {}.".format(
os.getpid(), "CUSTOM_ERROR"))
finally:
conn.quit()
def write_queue(msg):
parameters = pika.URLParameters('amqp://' + config["mquser"] + ':' +
config["mqpass"] + '@' + config["mqhost"] +
':5672/%2F')
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
channel.basic_publish(
exchange='',
routing_key='task_queue',
body=msg,
properties=pika.BasicProperties(
delivery_mode=2, # make message persistent
))
logging.info("PID {:>5} >> write message {} in queue".format(
os.getpid(), msg))
connection.close()
class ASEHandler(FTPHandler):
def on_file_received(self, file):
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if re.match( r'^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$', filename, re.I) :
m = re.match( r'^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$', filename, re.I)
if (m := re.match(
r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$",
filename,
re.I,
)):
unitType = m.group(1).upper()
unitName = m.group(2).upper()
toolName = m.group(3).upper()
toolType = "N/A"
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
elif re.match( r'^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$', filename, re.I) :
elif re.match(r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$",
filename, re.I):
with open(file, "r") as fileCsv:
try:
for i, line in enumerate(fileCsv.readlines(4096), 1):
if i == 1:
m1 = re.match( r'File Creation Date: (\d*\/\d*\/\d*)\s(\d*:\d*:\d*).*', line, re.I )
m1 = re.match(
r"File Creation Date: (\d*\/\d*\/\d*)\s(\d*:\d*:\d*).*",
line,
re.I,
)
fileDate = m1.group(1)
fileTime = m1.group(2)
if i == 2:
unitType, unitName = line.strip('\n').replace(";","").replace(",","").split(" ")
unitType, unitName = (line.strip("\n").replace(
";", "").replace(",", "").split(" "))
if i == 6:
m6 = re.match( r'SD path: .*\/.*\/(.*)\/(.*)\..*', line, re.I )
m6 = re.match(r"SD path: .*\/.*\/(.*)\/(.*)\..*",
line, re.I)
toolType = m6.group(1)
toolName = m6.group(2)
break
except:
logging.error("PID {:>5} >> Error: {}.".format( os.getpid(), sys.exc_info()[1]))
logging.error("PID {:>5} >> Error: {}.".format(
os.getpid(),
sys.exc_info()[1]))
fileCsv.close
logging.info("PID {:>5} >> {} - {} - {} - {} - {} {}.".format(os.getpid(), unitType.upper(), unitName.upper(), toolName.upper(), toolType.upper(), df.dateFmt(fileDate), fileTime ))
newPath = '/home/' + self.username + '/received/' + unitName.upper() + '/'
newFilename = newPath + filename + '_' + str(ts.timestamp("tms") + fileExtension)
logging.info("PID {:>5} >> {} - {} - {} - {} - {} {}.".format(
os.getpid(),
unitType.upper(),
unitName.upper(),
toolName.upper(),
toolType.upper(),
df.dateFmt(fileDate),
fileTime,
))
newPath = config["csvfs"] + self.username + "/received/" + unitName.upper(
) + "/"
newFilename = (newPath + filename + "_" +
str(ts.timestamp("tms") + fileExtension))
try:
os.makedirs(newPath)
logging.info("PID {:>5} >> path {} created.".format(os.getpid(), newPath))
logging.info("PID {:>5} >> path {} created.".format(
os.getpid(), newPath))
except FileExistsError:
logging.info("PID {:>5} >> path {} already exists.".format(os.getpid(), newPath))
logging.info("PID {:>5} >> path {} already exists.".format(
os.getpid(), newPath))
try:
os.rename(file, newFilename)
logging.info("PID {:>5} >> {} moved into {}.".format(os.getpid(), filenameExt, newFilename))
shutil.move(file, newFilename)
logging.info("PID {:>5} >> {} moved into {}.".format(
os.getpid(), filenameExt, newFilename))
except OSError:
logging.error("PID {:>5} >> Error to move {} into {}.".format(os.getpid(), filenameExt, newFilename))
send_mail("Error", "OS error move " + filenameExt + " to " + newFilename)
logging.error("PID {:>5} >> Error to move {} into {}.".format(
os.getpid(), filenameExt, newFilename))
# send_mail("Error",
# "OS error move " + filenameExt + " to " + newFilename)
mq_message = "{};{};{};{};{};{};{}".format(
unitType.upper(),
unitName.upper(),
toolName.upper(),
toolType.upper(),
df.dateFmt(fileDate),
fileTime,
newFilename,
)
write_queue(mq_message)
logging.info("PID {:>5} >> queue message: {}.".format(
os.getpid(), mq_message))
def on_incomplete_file_received(self, file):
# remove partially uploaded files
os.remove(file)
def read_config():
c = ConfigParser()
c.read(['/etc/ase/ftpcsvreceiver.ini',
'/home/alex/devel/python/ftpcsvreceiver.ini'])
config["firstport"] = c.getint('ftpserver', 'firstport')
config["logfilename"] = c.get('ftpserver', 'logfilename')
config["proxyaddr"] = c.get('ftpserver', 'proxyaddr')
config["smtphost"] = c.get('mailserver', 'hostname')
config["smtpport"] = c.getint('mailserver', 'port')
config["sender"] = c.get('mailserver', 'sender')
config["password"] = c.get('mailserver', 'password')
config["receivers"] = c.get('mailserver', 'receivers')
config["message"] = c.get('mailserver', 'message')
config["subject"] = c.get('mailserver', 'subject')
config["debuglevel"] = c.getint('mailserver', 'debug')
def send_mail(sev, msg):
msg = MIMEText(config["message"] + " " + msg)
msg['Subject'] = config["subject"] + " " + sev
msg['From'] = config["sender"]
msg['To'] = config["receivers"]
conn = SMTP(config["smtphost"])
conn.set_debuglevel(config["debuglevel"])
conn.login(config["sender"], config["password"])
try:
conn.sendmail(config["sender"], config["receivers"], msg.as_string())
except:
logging.info("PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR"))
finally:
conn.quit()
def main():
read_config()
try:
authorizer = UnixAuthorizer(
rejected_users=["root"], require_valid_shell=True)
authorizer = UnixAuthorizer(rejected_users=["root"],
require_valid_shell=True)
handler = ASEHandler
handler.authorizer = authorizer
handler.abstracted_fs = UnixFilesystem
@@ -119,16 +193,23 @@ def main():
_range = list(range(config["firstport"], config["firstport"] + 20))
handler.passive_ports = _range
logging.basicConfig(format='%(asctime)s %(message)s',
filename="/var/log/" + config["logfilename"], level=logging.INFO)
logging.basicConfig(
format="%(asctime)s %(message)s",
filename="/var/log/" + config["logfilename"],
level=logging.INFO,
)
server = FTPServer(('0.0.0.0', 21), handler)
server = FTPServer(("0.0.0.0", 21), handler)
server.serve_forever()
except KeyboardInterrupt:
logging.info("PID {:>5} >> Info: {}.".format(os.getpid(), "Shutdown requested...exiting"))
logging.info("PID {:>5} >> Info: {}.".format(
os.getpid(), "Shutdown requested...exiting"))
except Exception:
print("{} - PID {:>5} >> Error: {}.".format(ts.timestamp("log"), os.getpid(), sys.exc_info()[1]))
print("{} - PID {:>5} >> Error: {}.".format(ts.timestamp("log"),
os.getpid(),
sys.exc_info()[1]))
if __name__ == "__main__":
main()
main()

9
ftpcsvreceiver.ini
View File

@@ -14,5 +14,12 @@
bbbbb
ccccc
subject = ciao a domani
debug = 0
debug = 1
[mqserver]
hostname = 192.168.1.242
user = alex
password = batt1l0
[csvfs]
path = /home/