con postgres

ftpReceiver/FtpCsvReceiver.py

@@ -7,7 +7,8 @@ import shutil
 
 import re
 import logging
-import sqlite3
+
+import psycopg2
 
 from hashlib import md5
 from pathlib import Path
@@ -28,20 +29,25 @@ class DummyMD5Authorizer(DummyAuthorizer):
         self.add_user(
             cfg.adminuser[0], cfg.adminuser[1], cfg.adminuser[2], perm=cfg.adminuser[3])
 
-        # Connect to the SQLite database for virtual users
+        # Definisci la connessione al database
-        con = sqlite3.connect(cfg.virtusersdb)
+        conn = psycopg2.connect(
-        cur = con.cursor()
+            dbname=cfg.dbname,
-        # Create the virtusers table if it doesn't exist
+            user=cfg.dbuser,
-        cur.execute(
+            password=cfg.dbpass,
-            '''CREATE TABLE IF NOT EXISTS virtusers (user text, hash text, virtpath text, perm text)''')
+            host=cfg.dbhost,
-        # Create an index on the user column for faster lookups
+            port=cfg.dbport
-        cur.execute(
+        )
-            '''CREATE INDEX IF NOT EXISTS user_idx on virtusers(user)''')
 
-        # Load existing virtual users from the database
+        # Crea un cursore
-        for row in cur.execute('SELECT * FROM virtusers'):
+        cur = conn.cursor()
-            self.add_user(row[0], row[1], row[2], perm=row[3])
+        cur.execute("SELECT ftpuser, hash, virtpath, perm FROM virtusers")
-        con.close()
+
+        for ftpuser, hash, virtpath, perm in cur.fetchall():
+            self.add_user(ftpuser, hash, virtpath, perm)
+            try:
+                Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
+            except:
+                self.responde('551 Error in create virtual user path.')
 
     def validate_authentication(self, username, password, handler):
         # Validate the user's password against the stored hash
@@ -74,121 +80,36 @@ class ASEHandler(FTPHandler):
         )
 
     def on_file_received(self, file):
-        # Handle the event when a file is received
-        unitType = ""
-        unitName = ""
-        toolName = ""
-        toolType = ""
-        fileDate = ""
-        fileTime = ""
-        queue = ""
-        # Check if the file is empty and remove it if so
         if not os.stat(file).st_size:
             os.remove(file)
-            logging.info(
+            logging.info(f'File {file} was empty: removed.')
-                "File {} was empty: removed.".format(file))
         else:
             cfg = self.cfg
             path, filenameExt = os.path.split(file)
             filename, fileExtension = os.path.splitext(filenameExt)
-            # Process the file if it has a valid extension
             if (fileExtension.upper() in (cfg.fileext)):
-                # Match the filename against a specific pattern
+                with open(file, 'r') as csvfile:
-                if m := re.match(
+                    lines = csvfile.readlines()
-                    r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$",
+                    conn = psycopg2.connect(
-                    filename,
+                        dbname=cfg.dbname,
-                    re.I,
+                        user=cfg.dbuser,
-                ):
+                        password=cfg.dbpass,
-                    unitType = m.group(1).upper()
+                        host=cfg.dbhost,
-                    unitName = m.group(2).upper()
+                        port=cfg.dbport
-                    toolName = m.group(3).upper()
-                    toolType = "N/A"
-                    fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
-                    fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
-
-                # Match against another pattern for different file formats
-                elif re.match(
-                    r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
-                ):
-                    with open(file, "r") as fileCsv:
-                        try:
-                            # Read the CSV file line by line
-                            for i, line in enumerate(fileCsv.readlines(4096), 1):
-                                # Extract creation date and time
-                                if m1 := re.match(
-                                    r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
-                                    line,
-                                    re.I,
-                                ):
-                                    fileDate = m1.group(2)
-                                    fileTime = m1.group(3)
-
-                                # Extract unit type and name
-                                elif m2 := re.match(
-                                    r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
-                                    line,
-                                    re.I,
-                                ):
-                                    unitType = m2.group(1).upper()
-                                    unitName = m2.group(2).upper()
-
-                                # Extract tool type and name from the path
-                                elif m3 := re.match(
-                                    r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
-                                    line,
-                                    re.I,
-                                ):
-                                    if m3.group(2):
-                                        toolType = m3.group(1).upper()
-                                        toolName = m3.group(2).upper()
-                                    else:
-                                        toolType = "".join(
-                                            re.findall(
-                                                "^[a-zA-Z]+", m3.group(1))
-                                        ).upper()
-                                        toolName = m3.group(1).upper()
-                                    break
-                        except:
-                            logging.error(
-                                "Error: {}.".format(sys.exc_info()[1]))
-                        fileCsv.close
-
-                # Log the extracted information
-                logging.info(
-                    "{} - {} - {} - {} - {} {}.".format(
-                        unitType,
-                        unitName,
-                        toolName,
-                        toolType,
-                        df.dateFmt(fileDate),
-                        fileTime,
                     )
-                )
+
-                # Prepare the new path for the received file
+                    # Crea un cursore
-                newPath = cfg.csvfs + "/" + self.username + "/received/" + \
+                    print(file, lines)
-                    unitName.upper() + "/"
+                    cur = conn.cursor()
-                newFilename = (
+                    try:
-                    newPath + filename + "_" +
+                        cur.execute("INSERT INTO received (filename, content) VALUES (%s,%s)" , (filename, lines))
-                    str(ts.timestamp("tms") + fileExtension)
+                        conn.commit()
-                )
+                        conn.close()
-                fileRenamed = file + "_" + str(ts.timestamp("tms"))
+                    except:
-                # Rename the original file to avoid conflicts
+                        logging.error(f'File {file} not loaded. Held in user path.')
-                os.rename(file, fileRenamed)
+                    else:
-                try:
+                        os.remove(file)
-                    # Create the new directory for the user if it doesn't exist
+                        logging.info(f'File {file} loaded: removed.')
-                    os.makedirs(newPath)
-                    logging.info("Path {} created.".format(newPath))
-                except FileExistsError:
-                    logging.info("Path {} already exists.".format(newPath))
-                try:
-                    # Move the renamed file to the new location
-                    shutil.move(fileRenamed, newFilename)
-                    logging.info("{} moved into {}.".format(
-                        filenameExt, newFilename))
-                except OSError:
-                    logging.error("Error to move {} into {}.".format(
-                        filenameExt, newFilename))
-                now = datetime.now()
 
     def on_incomplete_file_received(self, file):
         # Remove partially uploaded files
@@ -200,32 +121,43 @@ class ASEHandler(FTPHandler):
             Create a directory for the virtual user in the specified virtpath.
         """
         cfg = self.cfg
-        parms = line.split()
-        user = os.path.basename(parms[0])  # Extract the username
-        password = parms[1]  # Get the password
-        hash = md5(password.encode("UTF-8")).hexdigest()  # Hash the password
-
         try:
-            # Create the user's directory
+            parms = line.split()
-            Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
+            user = os.path.basename(parms[0])  # Extract the username
+            password = parms[1]  # Get the password
+            hash = md5(password.encode("UTF-8")).hexdigest()  # Hash the password
         except:
-            self.respond('551 Error in create virtual user path.')
+            self.respond('501 SITE ADDU failed. Command needs 2 arguments')
         else:
             try:
-                # Add the user to the authorizer
+                # Create the user's directory
-                self.authorizer.add_user(str(user),
+                Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
-                                         hash, cfg.virtpath + "/" + user, perm="lmw")
-                # Save the user to the SQLite database
-                con = sqlite3.connect(cfg.virtusersdb)
-                cur = con.cursor()
-                cur.execute("INSERT INTO virtusers VALUES (?,?,?,?)",
-                            (user, hash, cfg.virtpath + user, 'elmw'))
-                con.commit()
-                con.close()
-                logging.info("User {} created.".format(user))
-                self.respond('200 SITE ADDU successful.')
             except:
-                self.respond('501 SITE ADDU failed.')
+                self.respond('551 Error in create virtual user path.')
+            else:
+                try:
+                    # Add the user to the authorizer
+                    self.authorizer.add_user(str(user),
+                                            hash, cfg.virtpath + "/" + user, perm="lmw")
+                    # Save the user to the SQLite database
+                            # Definisci la connessione al database
+                    conn = psycopg2.connect(
+                        dbname=cfg.dbname,
+                        user=cfg.dbuser,
+                        password=cfg.dbpass,
+                        host=cfg.dbhost,
+                        port=cfg.dbport
+                    )
+
+                    # Crea un cursore
+                    cur = conn.cursor()
+                    cur.execute("INSERT INTO virtusers (ftpuser, hash, virtpath, perm) VALUES (%s,%s,%s,%s)" , (user, hash, cfg.virtpath + user, 'elmw'))
+                    conn.commit()
+                    conn.close()
+                    logging.info("User {} created.".format(user))
+                    self.respond('200 SITE ADDU successful.')
+                except:
+                    self.respond('501 SITE ADDU failed.')
 
     def ftp_SITE_DELU(self, line):
         """
@@ -238,11 +170,21 @@ class ASEHandler(FTPHandler):
             # Remove the user from the authorizer
             self.authorizer.remove_user(str(user))
             # Delete the user from the SQLite database
-            con = sqlite3.connect(cfg.virtusersdb)
+
-            cur = con.cursor()
+            conn = psycopg2.connect(
-            cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
+                dbname=cfg.dbname,
-            con.commit()
+                user=cfg.dbuser,
-            con.close()
+                password=cfg.dbpass,
+                host=cfg.dbhost,
+                port=cfg.dbport
+            )
+
+            # Crea un cursore
+            cur = conn.cursor()
+            cur.execute("DELETE FROM virtusers WHERE ftpuser = %s", (user, ))
+            conn.commit()
+            conn.close()
+
             logging.info("User {} deleted.".format(user))
             self.respond('200 SITE DELU successful.')
 
@@ -257,21 +199,25 @@ class ASEHandler(FTPHandler):
         users_list = []
         try:
             # Connect to the SQLite database to fetch users
-            con = sqlite3.connect(cfg.virtusersdb)
+            conn = psycopg2.connect(
-            cur = con.cursor()
+                dbname=cfg.dbname,
+                user=cfg.dbuser,
+                password=cfg.dbpass,
+                host=cfg.dbhost,
+                port=cfg.dbport
+            )
+
+            # Crea un cursore
+            cur = conn.cursor()
             self.push("214-The following virtual users are defined:\r\n")
-            # Fetch and list all virtual users
+            cur.execute("SELECT ftpuser, perm FROM virtusers")
-            for row in cur.execute("SELECT * FROM virtusers").fetchall():
+            [users_list.append(f'Username: {ftpuser}\tPerms: {perm}\r\n') for ftpuser, perm in cur.fetchall()]
-                users_list.append(
-                    " Username: " + row[0] + "\tPerms: " + row[3] + "\r\n")
-            con.close()
             self.push(''.join(users_list))
             self.respond("214 LSTU SITE command successful.")
 
         except:
             self.respond('501 list users failed.')
 
-
 def main():
     # Load the configuration settings
     cfg = setting.config()
@@ -308,6 +254,5 @@ def main():
             )
         )
 
-
 if __name__ == "__main__":
     main()

ftpReceiver/ftpcsvreceiver.ini

@@ -5,36 +5,13 @@
   logFilename = ./ftppylog.log
   proxyAddr = 0.0.0.0
   portRangeWidth = 500
-  virtusersdb = /home/alex/aseftp/virtusers.db
   virtpath = /home/alex/aseftp/
   adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT
   servertype = FTPHandler
   certfile = /home/alex/aseftp/keycert.pem
-  fileext = .CSV|.txt
+  fileext = .CSV|.TXT
   #servertype = FTPHandler/TLS_FTPHandler
 
-
-[mailserver]
-  hostname = smtps.aruba.it
-  port = 465
-  sender = alessandro.battilani@aseltd.eu
-  password = taylor1964NFL!
-  receivers = alessandro.battilani@gmail.com
-  message = prova messaggio
-
-                bbbbb
-                ccccc
-  subject = ciao a domani
-  debug = 0
-
-[mqserver]
-  hostname = galera1
-  port = 5672
-  user = asemq
-  password = Ase2021
-  csvQueue = task_queue
-  elabQueue = elab_queue
-
 [csvfs]
   path = /home/alex/aseftp/csvfs/
 
@@ -42,10 +19,9 @@
   logFilename = csvElab.log
 
 [db]
-  hostname = 192.168.1.241
+  hostname = 10.211.114.101
-  user = root
+  port = 5432
+  user = asepg
   password = batt1l0
-  dbName = ase
+  dbName = asedb
-  tableName = rawdata
+  tableName = virtusers
-  maxInsertRow = 20000
-  valueNum = 16

ftpReceiver/utils/config/set_config.py

@@ -14,31 +14,12 @@ class config:
         self.logfilename = c.get("ftpserver", "logFilename")
         self.proxyaddr = c.get("ftpserver", "proxyAddr")
         self.portrangewidth = c.getint("ftpserver", "portRangeWidth")
-        self.virtusersdb = c.get("ftpserver", "virtusersdb")
         self.virtpath = c.get("ftpserver", "virtpath")
         self.adminuser = c.get("ftpserver", "adminuser").split("|")
         self.servertype = c.get("ftpserver", "servertype")
         self.certfile = c.get("ftpserver", "certfile")
         self.fileext = c.get("ftpserver", "fileext").upper().split("|")
 
-        # MAIL setting
-        self.smtphost = c.get("mailserver", "hostname")
-        self.smtpport = c.getint("mailserver", "port")
-        self.sender = c.get("mailserver", "sender")
-        self.password = c.get("mailserver", "password")
-        self.receivers = c.get("mailserver", "receivers")
-        self.message = c.get("mailserver", "message")
-        self.subject = c.get("mailserver", "subject")
-        self.debuglevel = c.getint("mailserver", "debug")
-
-        # MQ setting
-        self.mqhost = c.get("mqserver", "hostname")
-        self.mqport = c.get("mqserver", "port")
-        self.mquser = c.get("mqserver", "user")
-        self.mqpass = c.get("mqserver", "password")
-        self.csv_queue = c.get("mqserver", "csvQueue")
-        self.elab_queue = c.get("mqserver", "elabQueue")
-
         # CSV FILE setting
         self.csvfs = c.get("csvfs", "path")
 
@@ -47,9 +28,8 @@ class config:
 
         # DB setting
         self.dbhost = c.get("db", "hostname")
+        self.dbport = c.getint("db", "port")
         self.dbuser = c.get("db", "user")
         self.dbpass = c.get("db", "password")
         self.dbname = c.get("db", "dbName")
-        self.table = c.get("db", "tableName")
+        self.dbtable = c.get("db", "tableName")
-        self.valueNum = c.getint("db", "valueNum")
-        self.maxInsertRow = c.getint("db", "maxInsertRow")