alex/ASE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

con postgres

Browse Source
This commit is contained in:
alex
2024-11-17 15:09:37 +01:00
parent 21493d2fa3
commit 4ddf232021
3 changed files with 110 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
ftpReceiver/FtpCsvReceiver.py
View File

@@ -7,7 +7,8 @@ import shutil
import re
import logging
import sqlite3
import psycopg2
from hashlib import md5
from pathlib import Path
@@ -28,20 +29,25 @@ class DummyMD5Authorizer(DummyAuthorizer):
self.add_user(
cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
# Connect to the SQLite database for virtual users
con = sqlite3.connect(cfg.virtusersdb)
cur = con.cursor()
# Create the virtusers table if it doesn't exist
cur.execute(
'''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''')
# Create an index on the user column for faster lookups
cur.execute(
'''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''')
# Definisci la connessione al database
conn = psycopg2.connect(
dbname=cfg.dbname,
user=cfg.dbuser,
password=cfg.dbpass,
host=cfg.dbhost,
port=cfg.dbport
)
# Load existing virtual users from the database
for row in cur.execute('SELECT * FROM virtusers'):
self.add_user(row[0], row[1], row[2], perm=row[3])
con.close()
# Crea un cursore
cur = conn.cursor()
cur.execute("SELECT ftpuser, hash, virtpath, perm FROM virtusers")
for ftpuser, hash, virtpath, perm in cur.fetchall():
self.add_user(ftpuser, hash, virtpath, perm)
try:
Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
except:
self.responde('551 Error in create virtual user path.')
def validate_authentication(self, username, password, handler):
# Validate the user's password against the stored hash
@@ -74,121 +80,36 @@ class ASEHandler(FTPHandler):
)
def on_file_received(self, file):
# Handle the event when a file is received
unitType = ""
unitName = ""
toolName = ""
toolType = ""
fileDate = ""
fileTime = ""
queue = ""
# Check if the file is empty and remove it if so
if not os.stat(file).st_size:
os.remove(file)
logging.info(
"File {} was empty: removed.".format(file))
logging.info(f'File {file} was empty: removed.')
else:
cfg = self.cfg
path, filenameExt = os.path.split(file)
filename, fileExtension = os.path.splitext(filenameExt)
# Process the file if it has a valid extension
if (fileExtension.upper() in (cfg.fileext)):
# Match the filename against a specific pattern
if m := re.match(
r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$",
filename,
re.I,
):
unitType = m.group(1).upper()
unitName = m.group(2).upper()
toolName = m.group(3).upper()
toolType = "N/A"
fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
# Match against another pattern for different file formats
elif re.match(
r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
):
with open(file, "r") as fileCsv:
try:
# Read the CSV file line by line
for i, line in enumerate(fileCsv.readlines(4096), 1):
# Extract creation date and time
if m1 := re.match(
r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
line,
re.I,
):
fileDate = m1.group(2)
fileTime = m1.group(3)
# Extract unit type and name
elif m2 := re.match(
r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
line,
re.I,
):
unitType = m2.group(1).upper()
unitName = m2.group(2).upper()
# Extract tool type and name from the path
elif m3 := re.match(
r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
line,
re.I,
):
if m3.group(2):
toolType = m3.group(1).upper()
toolName = m3.group(2).upper()
else:
toolType = "".join(
re.findall(
"^[a-zA-Z]+", m3.group(1))
).upper()
toolName = m3.group(1).upper()
break
except:
logging.error(
"Error: {}.".format(sys.exc_info()[1]))
fileCsv.close
# Log the extracted information
logging.info(
"{} - {} - {} - {} - {} {}.".format(
unitType,
unitName,
toolName,
toolType,
df.dateFmt(fileDate),
fileTime,
with open(file, 'r') as csvfile:
lines = csvfile.readlines()
conn = psycopg2.connect(
dbname=cfg.dbname,
user=cfg.dbuser,
password=cfg.dbpass,
host=cfg.dbhost,
port=cfg.dbport
)
)
# Prepare the new path for the received file
newPath = cfg.csvfs + "/" + self.username + "/received/" + \
unitName.upper() + "/"
newFilename = (
newPath + filename + "_" +
str(ts.timestamp("tms") + fileExtension)
)
fileRenamed = file + "_" + str(ts.timestamp("tms"))
# Rename the original file to avoid conflicts
os.rename(file, fileRenamed)
try:
# Create the new directory for the user if it doesn't exist
os.makedirs(newPath)
logging.info("Path {} created.".format(newPath))
except FileExistsError:
logging.info("Path {} already exists.".format(newPath))
try:
# Move the renamed file to the new location
shutil.move(fileRenamed, newFilename)
logging.info("{} moved into {}.".format(
filenameExt, newFilename))
except OSError:
logging.error("Error to move {} into {}.".format(
filenameExt, newFilename))
now = datetime.now()
# Crea un cursore
print(file, lines)
cur = conn.cursor()
try:
cur.execute("INSERT INTO received (filename, content) VALUES (%s,%s)" , (filename, lines))
conn.commit()
conn.close()
except:
logging.error(f'File {file} not loaded. Held in user path.')
else:
os.remove(file)
logging.info(f'File {file} loaded: removed.')
def on_incomplete_file_received(self, file):
# Remove partially uploaded files
@@ -200,32 +121,43 @@ class ASEHandler(FTPHandler):
Create a directory for the virtual user in the specified virtpath.
"""
cfg = self.cfg
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = md5(password.encode("UTF-8")).hexdigest() # Hash the password
try:
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
parms = line.split()
user = os.path.basename(parms[0]) # Extract the username
password = parms[1] # Get the password
hash = md5(password.encode("UTF-8")).hexdigest() # Hash the password
except:
self.respond('551 Error in create virtual user path.')
self.respond('501 SITE ADDU failed. Command needs 2 arguments')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm="lmw")
# Save the user to the SQLite database
con = sqlite3.connect(cfg.virtusersdb)
cur = con.cursor()
cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)",
(user, hash, cfg.virtpath + user, 'elmw'))
con.commit()
con.close()
logging.info("User {} created.".format(user))
self.respond('200 SITE ADDU successful.')
# Create the user's directory
Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
except:
self.respond('501 SITE ADDU failed.')
self.respond('551 Error in create virtual user path.')
else:
try:
# Add the user to the authorizer
self.authorizer.add_user(str(user),
hash, cfg.virtpath + "/" + user, perm="lmw")
# Save the user to the SQLite database
# Definisci la connessione al database
conn = psycopg2.connect(
dbname=cfg.dbname,
user=cfg.dbuser,
password=cfg.dbpass,
host=cfg.dbhost,
port=cfg.dbport
)
# Crea un cursore
cur = conn.cursor()
cur.execute("INSERT INTO virtusers (ftpuser, hash, virtpath, perm) VALUES (%s,%s,%s,%s)" , (user, hash, cfg.virtpath + user, 'elmw'))
conn.commit()
conn.close()
logging.info("User {} created.".format(user))
self.respond('200 SITE ADDU successful.')
except:
self.respond('501 SITE ADDU failed.')
def ftp_SITE_DELU(self, line):
"""
@@ -238,11 +170,21 @@ class ASEHandler(FTPHandler):
# Remove the user from the authorizer
self.authorizer.remove_user(str(user))
# Delete the user from the SQLite database
con = sqlite3.connect(cfg.virtusersdb)
cur = con.cursor()
cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
con.commit()
con.close()
conn = psycopg2.connect(
dbname=cfg.dbname,
user=cfg.dbuser,
password=cfg.dbpass,
host=cfg.dbhost,
port=cfg.dbport
)
# Crea un cursore
cur = conn.cursor()
cur.execute("DELETE FROM virtusers WHERE ftpuser = %s", (user, ))
conn.commit()
conn.close()
logging.info("User {} deleted.".format(user))
self.respond('200 SITE DELU successful.')
@@ -257,21 +199,25 @@ class ASEHandler(FTPHandler):
users_list = []
try:
# Connect to the SQLite database to fetch users
con = sqlite3.connect(cfg.virtusersdb)
cur = con.cursor()
conn = psycopg2.connect(
dbname=cfg.dbname,
user=cfg.dbuser,
password=cfg.dbpass,
host=cfg.dbhost,
port=cfg.dbport
)
# Crea un cursore
cur = conn.cursor()
self.push("214-The following virtual users are defined:\r\n")
# Fetch and list all virtual users
for row in cur.execute("SELECT * FROM virtusers").fetchall():
users_list.append(
" Username: " + row[0] + "\tPerms: " + row[3] + "\r\n")
con.close()
cur.execute("SELECT ftpuser, perm FROM virtusers")
[users_list.append(f'Username: {ftpuser}\tPerms: {perm}\r\n') for ftpuser, perm in cur.fetchall()]
self.push(''.join(users_list))
self.respond("214 LSTU SITE command successful.")
except:
self.respond('501 list users failed.')
def main():
# Load the configuration settings
cfg = setting.config()
@@ -308,6 +254,5 @@ def main():
)
)
if __name__ == "__main__":
main()

36
ftpReceiver/ftpcsvreceiver.ini
View File

@@ -5,36 +5,13 @@
logFilename = ./ftppylog.log
proxyAddr = 0.0.0.0
portRangeWidth = 500
virtusersdb = /home/alex/aseftp/virtusers.db
virtpath = /home/alex/aseftp/
adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT
servertype = FTPHandler
certfile = /home/alex/aseftp/keycert.pem
fileext = .CSV|.txt
fileext = .CSV|.TXT
#servertype = FTPHandler/TLS_FTPHandler
[mailserver]
hostname = smtps.aruba.it
port = 465
sender = alessandro.battilani@aseltd.eu
password = taylor1964NFL!
receivers = alessandro.battilani@gmail.com
message = prova messaggio
bbbbb
ccccc
subject = ciao a domani
debug = 0
[mqserver]
hostname = galera1
port = 5672
user = asemq
password = Ase2021
csvQueue = task_queue
elabQueue = elab_queue
[csvfs]
path = /home/alex/aseftp/csvfs/
@@ -42,10 +19,9 @@
logFilename = csvElab.log
[db]
hostname = 192.168.1.241
user = root
hostname = 10.211.114.101
port = 5432
user = asepg
password = batt1l0
dbName = ase
tableName = rawdata
maxInsertRow = 20000
valueNum = 16
dbName = asedb
tableName = virtusers

24
ftpReceiver/utils/config/set_config.py
View File

@@ -14,31 +14,12 @@ class config:
self.logfilename = c.get("ftpserver", "logFilename")
self.proxyaddr = c.get("ftpserver", "proxyAddr")
self.portrangewidth = c.getint("ftpserver", "portRangeWidth")
self.virtusersdb = c.get("ftpserver", "virtusersdb")
self.virtpath = c.get("ftpserver", "virtpath")
self.adminuser = c.get("ftpserver", "adminuser").split("|")
self.servertype = c.get("ftpserver", "servertype")
self.certfile = c.get("ftpserver", "certfile")
self.fileext = c.get("ftpserver", "fileext").upper().split("|")
# MAIL setting
self.smtphost = c.get("mailserver", "hostname")
self.smtpport = c.getint("mailserver", "port")
self.sender = c.get("mailserver", "sender")
self.password = c.get("mailserver", "password")
self.receivers = c.get("mailserver", "receivers")
self.message = c.get("mailserver", "message")
self.subject = c.get("mailserver", "subject")
self.debuglevel = c.getint("mailserver", "debug")
# MQ setting
self.mqhost = c.get("mqserver", "hostname")
self.mqport = c.get("mqserver", "port")
self.mquser = c.get("mqserver", "user")
self.mqpass = c.get("mqserver", "password")
self.csv_queue = c.get("mqserver", "csvQueue")
self.elab_queue = c.get("mqserver", "elabQueue")
# CSV FILE setting
self.csvfs = c.get("csvfs", "path")
@@ -47,9 +28,8 @@ class config:
# DB setting
self.dbhost = c.get("db", "hostname")
self.dbport = c.getint("db", "port")
self.dbuser = c.get("db", "user")
self.dbpass = c.get("db", "password")
self.dbname = c.get("db", "dbName")
self.table = c.get("db", "tableName")
self.valueNum = c.getint("db", "valueNum")
self.maxInsertRow = c.getint("db", "maxInsertRow")
self.dbtable = c.get("db", "tableName")