alex/python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

readjust path

Browse Source
This commit is contained in:
Alessandro Battilani
2022-01-23 16:37:30 +01:00
parent dc58a0efbb
commit 8a2d4d3ee6
3 changed files with 115 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
ase-receiver/ase-receiver/FtpCsvReceiver.py
View File

@@ -3,7 +3,7 @@
import sys
import os
import shutil
import ssl
# import ssl
import pika
import re
import logging
@@ -42,13 +42,11 @@ def send_mail(sev, msg, cfg):
conn.sendmail(cfg.sender, cfg.receivers, msg.as_string())
except SMTPAuthenticationError:
logging.error(
"PID {:>5} >> Mail failed: {}.".format(
os.getpid(), "SMTP authentication error"
)
"Mail failed: {}.".format("SMTP authentication error")
)
except:
logging.info(
"PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR")
"Mail failed: {}.".format("CUSTOM_ERROR")
)
finally:
conn.quit()
@@ -82,14 +80,10 @@ class mq:
),
)
logging.info(
"PID {:>5} >> write message {} in queue".format(
os.getpid(), msg)
)
"Write message {} in queue".format(msg))
except:
logging.error(
"PID {:>5} >> error write message {} in queue".format(
os.getpid(), msg)
)
"Error write message {} in queue".format(msg))
def close(self):
self.channel.close()
@@ -150,142 +144,120 @@ class ASEHandler(FTPHandler):
if not os.stat(file).st_size:
os.remove(file)
logging.info(
"PID {:>5} >> file {} was empty: removed.".format(
os.getpid(), file)
)
"File {} was empty: removed.".format(file))
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
if (fileExtension.upper() in (cfg.fileext)):
if m := re.match(
r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$",
filename,
re.I,
):
unitType = m.group(1).upper()
unitName = m.group(2).upper()
toolName = m.group(3).upper()
toolType = "N/A"
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
elif re.match(
r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
):
with open(file, "r") as fileCsv:
try:
for i, line in enumerate(fileCsv.readlines(4096), 1):
if m1 := re.match(
r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
line,
re.I,
):
fileDate = m1.group(2)
fileTime = m1.group(3)
if m := re.match(
r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$",
filename,
re.I,
):
unitType = m.group(1).upper()
unitName = m.group(2).upper()
toolName = m.group(3).upper()
toolType = "N/A"
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
elif re.match(
r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
):
with open(file, "r") as fileCsv:
try:
for i, line in enumerate(fileCsv.readlines(4096), 1):
if m1 := re.match(
r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
line,
re.I,
):
fileDate = m1.group(2)
fileTime = m1.group(3)
elif m2 := re.match(
r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
line,
re.I,
):
unitType = m2.group(1).upper()
unitName = m2.group(2).upper()
elif m2 := re.match(
r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
line,
re.I,
):
unitType = m2.group(1).upper()
unitName = m2.group(2).upper()
elif m3 := re.match(
r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
line,
re.I,
):
if m3.group(2):
toolType = m3.group(1).upper()
toolName = m3.group(2).upper()
else:
toolType = "".join(
re.findall(
"^[a-zA-Z]+", m3.group(1))
).upper()
toolName = m3.group(1).upper()
break
except:
logging.error(
"Error: {}.".format(sys.exc_info()[1]))
fileCsv.close
elif m3 := re.match(
r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
line,
re.I,
):
if m3.group(2):
toolType = m3.group(1).upper()
toolName = m3.group(2).upper()
else:
toolType = "".join(
re.findall("^[a-zA-Z]+", m3.group(1))
).upper()
toolName = m3.group(1).upper()
break
except:
logging.error(
"PID {:>5} >> Error: {}.".format(
os.getpid(), sys.exc_info()[1]
)
)
fileCsv.close
logging.info(
"{} - {} - {} - {} - {} {}.".format(
unitType,
unitName,
toolName,
toolType,
df.dateFmt(fileDate),
fileTime,
)
)
newPath = cfg.csvfs + "/" + self.username + "/received/" + \
unitName.upper() + "/"
newFilename = (
newPath + filename + "_" +
str(ts.timestamp("tms") + fileExtension)
)
fileRenamed = file + "_" + str(ts.timestamp("tms"))
os.rename(file, fileRenamed)
try:
os.makedirs(newPath)
logging.info("Path {} created.".format(newPath))
except FileExistsError:
logging.info("Path {} already exists.".format(newPath))
try:
shutil.move(fileRenamed, newFilename)
logging.info("{} moved into {}.".format(
filenameExt, newFilename))
except OSError:
logging.error("Error to move {} into {}.".format(
filenameExt, newFilename))
send_mail(
"Error", "OS error move " + filenameExt + " to " + newFilename, cfg
)
logging.info(
"PID {:>5} >> {} - {} - {} - {} - {} {}.".format(
os.getpid(),
mq_message = "{};{};{};{};{};{};{}".format(
unitType,
unitName,
toolName,
toolType,
df.dateFmt(fileDate),
fileTime,
newFilename,
)
)
newPath = cfg.csvfs + self.username + "/received/" + unitName.upper() + "/"
newFilename = (
newPath + filename + "_" +
str(ts.timestamp("tms") + fileExtension)
)
fileRenamed = file + "_" + str(ts.timestamp("tms"))
os.rename(file, fileRenamed)
try:
os.makedirs(newPath)
logging.info(
"PID {:>5} >> path {} created.".format(
os.getpid(), newPath)
)
except FileExistsError:
logging.info(
"PID {:>5} >> path {} already exists.".format(
os.getpid(), newPath)
)
try:
shutil.move(fileRenamed, newFilename)
logging.info(
"PID {:>5} >> {} moved into {}.".format(
os.getpid(), filenameExt, newFilename
try:
queue = mq(cfg)
queue.write(mq_message, cfg)
logging.info("Queue message: {}.".format(mq_message))
except:
logging.error(
"Error to put message in queue: {}.".format(mq_message))
send_mail(
"Error", "Error to put message " + mq_message + " in queue.", cfg
)
)
except OSError:
logging.error(
"PID {:>5} >> Error to move {} into {}.".format(
os.getpid(), filenameExt, newFilename
)
)
send_mail(
"Error", "OS error move " + filenameExt + " to " + newFilename, cfg
)
mq_message = "{};{};{};{};{};{};{}".format(
unitType,
unitName,
toolName,
toolType,
df.dateFmt(fileDate),
fileTime,
newFilename,
)
try:
queue = mq(cfg)
queue.write(mq_message, cfg)
logging.info(
"PID {:>5} >> queue message: {}.".format(
os.getpid(), mq_message)
)
except:
logging.error(
"PID {:>5} >> Error to put message in queue: {}.".format(
os.getpid(), mq_message
)
)
send_mail(
"Error", "Error to put message " + mq_message + " in queue.", cfg
)
finally:
queue.close()
finally:
queue.close()
def on_incomplete_file_received(self, file):
# remove partially uploaded files
@@ -316,11 +288,7 @@ class ASEHandler(FTPHandler):
(user, hash, cfg.virtpath + user, 'elmw'))
con.commit()
con.close()
logging.info(
"PID {:>5} >> User {} created.".format(
os.getpid(), user
)
)
logging.info("User {} created.".format(user))
self.respond('200 SITE ADDU successful.')
except:
self.respond('501 SITE ADDU failed.')
@@ -339,11 +307,8 @@ class ASEHandler(FTPHandler):
cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
con.commit()
con.close()
logging.info(
"PID {:>5} >> User {} deleted.".format(
os.getpid(), user
)
)
logging.info("User {} deleted.".format(user))
# self.push(' The user path has not been removed!\r\n')
self.respond('200 SITE DELU successful.')
except:
@@ -361,7 +326,7 @@ class ASEHandler(FTPHandler):
self.push("214-The following virtual users are defined:\r\n")
for row in cur.execute("SELECT * FROM virtusers").fetchall():
users_list.append(
" Username: " + row[0] + " - Perms: " + row[3] + "\r\n")
" Username: " + row[0] + "\tPerms: " + row[3] + "\r\n")
con.close()
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
@@ -393,8 +358,7 @@ def main():
server.serve_forever()
except KeyboardInterrupt:
logging.info(
"PID {:>5} >> Info: {}.".format(
os.getpid(), "Shutdown requested...exiting")
"Info: {}.".format("Shutdown requested...exiting")
)
except Exception:
print(

9
ase-receiver/ase-receiver/ftpcsvreceiver.ini
View File

@@ -1,13 +1,16 @@
# to generete adminuser password hash:
# python3 -c 'from hashlib import md5;print(md5("????admin-password???".encode("UTF-8")).hexdigest())'
[ftpserver]
firstPort = 40000
logFilename = ./ftppylog.log
proxyAddr = 0.0.0.0
portRangeWidth = 50
portRangeWidth = 500
virtusersdb = /home/alex/aseftp/virtusers.db
virtpath = /home/alex/aseftp/
adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT
servertype = FTPHandler
certfile = /home/alex/aseftp/keycert.pem
fileext = .CSV|.txt
#servertype = FTPHandler/TLS_FTPHandler
@@ -22,7 +25,7 @@
bbbbb
ccccc
subject = ciao a domani
debug = 1
debug = 0
[mqserver]
hostname = 192.168.1.241
@@ -33,7 +36,7 @@
elabQueue = elab_queue
[csvfs]
path = /home/
path = /home/alex/aseftp/csvfs/
[csvelab]
logFilename = csvElab.log

1
ase-receiver/ase-receiver/utils/config/set_config.py
View File

@@ -18,6 +18,7 @@ class config:
self.adminuser = c.get("ftpserver", "adminuser").split("|")
self.servertype = c.get("ftpserver", "servertype")
self.certfile = c.get("ftpserver", "certfile")
self.fileext = c.get("ftpserver", "fileext").upper().split("|")
# MAIL setting
self.smtphost = c.get("mailserver", "hostname")