nn

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Usare IntelliSense per informazioni sui possibili attributi.
+    // Al passaggio del mouse vengono visualizzate le descrizioni degli attributi esistenti.
+    // Per altre informazioni, visitare: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: File corrente",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

ase-receiver/ase-receiver/FtpCsvReceiver.py

@@ -3,7 +3,7 @@
 import sys
 import os
 import shutil
-import ssl
+# import ssl
 import pika
 import re
 import logging
@@ -11,6 +11,7 @@ import sqlite3
 
 from hashlib import md5
 from pathlib import Path
+from datetime import datetime
 
 from smtplib import SMTP_SSL as SMTP, SMTPException, SMTPAuthenticationError
 from email.mime.text import MIMEText
@@ -42,13 +43,11 @@ def send_mail(sev, msg, cfg):
         conn.sendmail(cfg.sender, cfg.receivers, msg.as_string())
     except SMTPAuthenticationError:
         logging.error(
-            "PID {:>5} >> Mail failed: {}.".format(
-                os.getpid(), "SMTP authentication error"
-            )
+            "Mail failed: {}.".format("SMTP authentication error")
         )
     except:
         logging.info(
-            "PID {:>5} >> Mail failed: {}.".format(os.getpid(), "CUSTOM_ERROR")
+            "Mail failed: {}.".format("CUSTOM_ERROR")
         )
     finally:
         conn.quit()
@@ -73,23 +72,21 @@ class mq:
 
     def write(self, msg, cfg):
         try:
+            props = pika.BasicProperties(
+                delivery_mode=2,
+                content_encoding='utf-8',
+                timestamp=msg["timestamp"],)
             self.channel.basic_publish(
                 exchange="",
                 routing_key=cfg.csv_queue,
-                body=msg,
-                properties=pika.BasicProperties(
-                    delivery_mode=2,  # make message persistent
-                ),
+                body=msg["payload"],
+                properties=props
             )
             logging.info(
-                "PID {:>5} >> write message {} in queue".format(
-                    os.getpid(), msg)
-            )
+                "Write message {} in queue".format(msg))
         except:
             logging.error(
-                "PID {:>5} >> error write message {} in queue".format(
-                    os.getpid(), msg)
-            )
+                "Error write message {} in queue".format(msg))
 
     def close(self):
         self.channel.close()
@@ -150,142 +147,123 @@ class ASEHandler(FTPHandler):
         if not os.stat(file).st_size:
             os.remove(file)
             logging.info(
-                "PID {:>5} >> file {} was empty: removed.".format(
-                    os.getpid(), file)
-            )
+                "File {} was empty: removed.".format(file))
         else:
             cfg = self.cfg
             path, filenameExt = os.path.split(file)
             filename, fileExtension = os.path.splitext(filenameExt)
+            if (fileExtension.upper() in (cfg.fileext)):
+                if m := re.match(
+                    r"^(G\d\d\d|GFLOW)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d|\d\d)(\d\d)(\d\d)(\d\d)$",
+                    filename,
+                    re.I,
+                ):
+                    unitType = m.group(1).upper()
+                    unitName = m.group(2).upper()
+                    toolName = m.group(3).upper()
+                    toolType = "N/A"
+                    fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
+                    fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
 
-            if m := re.match(
-                r"^(G\d\d\d)_(ID\d\d\d\d)_(DT\d\d\d\d)_(\d\d)(\d\d)(\d\d\d\d)(\d\d)(\d\d)(\d\d)$",
-                filename,
-                re.I,
-            ):
-                unitType = m.group(1).upper()
-                unitName = m.group(2).upper()
-                toolName = m.group(3).upper()
-                toolType = "N/A"
-                fileDate = m.group(6) + "/" + m.group(5) + "/" + m.group(4)
-                fileTime = m.group(7) + ":" + m.group(8) + ":" + m.group(9)
-            elif re.match(
-                r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
-            ):
-                with open(file, "r") as fileCsv:
-                    try:
-                        for i, line in enumerate(fileCsv.readlines(4096), 1):
-                            if m1 := re.match(
-                                r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
-                                line,
-                                re.I,
-                            ):
-                                fileDate = m1.group(2)
-                                fileTime = m1.group(3)
+                elif re.match(
+                    r"^(\d\d_\d\d\d\d|)(DT\d\d\d\d|LOC\d\d\d\d|GD\d\d\d\d)$", filename, re.I
+                ):
+                    with open(file, "r") as fileCsv:
+                        try:
+                            for i, line in enumerate(fileCsv.readlines(4096), 1):
+                                if m1 := re.match(
+                                    r"^(File Creation Date:\s)?(\d*\/\d*\/\d*)\s(\d*:\d*:\d*)\;*\n?$",
+                                    line,
+                                    re.I,
+                                ):
+                                    fileDate = m1.group(2)
+                                    fileTime = m1.group(3)
 
-                            elif m2 := re.match(
-                                r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
-                                line,
-                                re.I,
-                            ):
-                                unitType = m2.group(1).upper()
-                                unitName = m2.group(2).upper()
+                                elif m2 := re.match(
+                                    r"^(\w+\d+)\s(\w+\d+)\;*\n?$",
+                                    line,
+                                    re.I,
+                                ):
+                                    unitType = m2.group(1).upper()
+                                    unitName = m2.group(2).upper()
 
-                            elif m3 := re.match(
-                                r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
-                                line,
-                                re.I,
-                            ):
-                                if m3.group(2):
-                                    toolType = m3.group(1).upper()
-                                    toolName = m3.group(2).upper()
-                                else:
-                                    toolType = "".join(
-                                        re.findall("^[a-zA-Z]+", m3.group(1))
-                                    ).upper()
-                                    toolName = m3.group(1).upper()
-                                break
-                    except:
-                        logging.error(
-                            "PID {:>5} >> Error: {}.".format(
-                                os.getpid(), sys.exc_info()[1]
-                            )
-                        )
-                    fileCsv.close
+                                elif m3 := re.match(
+                                    r"^SD path: a:\/\w+\/(\w+)(?:\.\w+)?\/*(\w*)(?:\.\w+)?\;*\n?$",
+                                    line,
+                                    re.I,
+                                ):
+                                    if m3.group(2):
+                                        toolType = m3.group(1).upper()
+                                        toolName = m3.group(2).upper()
+                                    else:
+                                        toolType = "".join(
+                                            re.findall(
+                                                "^[a-zA-Z]+", m3.group(1))
+                                        ).upper()
+                                        toolName = m3.group(1).upper()
+                                    break
+                        except:
+                            logging.error(
+                                "Error: {}.".format(sys.exc_info()[1]))
+                        fileCsv.close
 
-            logging.info(
-                "PID {:>5} >> {} - {} - {} - {} - {} {}.".format(
-                    os.getpid(),
+                logging.info(
+                    "{} - {} - {} - {} - {} {}.".format(
+                        unitType,
+                        unitName,
+                        toolName,
+                        toolType,
+                        df.dateFmt(fileDate),
+                        fileTime,
+                    )
+                )
+                newPath = cfg.csvfs + "/" + self.username + "/received/" + \
+                    unitName.upper() + "/"
+                newFilename = (
+                    newPath + filename + "_" +
+                    str(ts.timestamp("tms") + fileExtension)
+                )
+                fileRenamed = file + "_" + str(ts.timestamp("tms"))
+                os.rename(file, fileRenamed)
+                try:
+                    os.makedirs(newPath)
+                    logging.info("Path {} created.".format(newPath))
+                except FileExistsError:
+                    logging.info("Path {} already exists.".format(newPath))
+                try:
+                    shutil.move(fileRenamed, newFilename)
+                    logging.info("{} moved into {}.".format(
+                        filenameExt, newFilename))
+                except OSError:
+                    logging.error("Error to move {} into {}.".format(
+                        filenameExt, newFilename))
+                    send_mail(
+                        "Error", "OS error move " + filenameExt + " to " + newFilename, cfg
+                    )
+                now = datetime.now()
+
+                mq_message = {"payload": "{};{};{};{};{};{};{}".format(
                     unitType,
                     unitName,
                     toolName,
                     toolType,
                     df.dateFmt(fileDate),
                     fileTime,
-                )
-            )
-            newPath = cfg.csvfs + self.username + "/received/" + unitName.upper() + "/"
-            newFilename = (
-                newPath + filename + "_" +
-                str(ts.timestamp("tms") + fileExtension)
-            )
-            fileRenamed = file + "_" + str(ts.timestamp("tms"))
-            os.rename(file, fileRenamed)
-            try:
-                os.makedirs(newPath)
-                logging.info(
-                    "PID {:>5} >> path {} created.".format(
-                        os.getpid(), newPath)
-                )
-            except FileExistsError:
-                logging.info(
-                    "PID {:>5} >> path {} already exists.".format(
-                        os.getpid(), newPath)
-                )
-            try:
-                shutil.move(fileRenamed, newFilename)
-                logging.info(
-                    "PID {:>5} >> {} moved into {}.".format(
-                        os.getpid(), filenameExt, newFilename
+                    newFilename),
+                    "timestamp": int(datetime.timestamp(now)*1000000)
+                }
+                try:
+                    queue = mq(cfg)
+                    queue.write(mq_message, cfg)
+                    logging.info("Queue message: {}.".format(mq_message))
+                except:
+                    logging.error(
+                        "Error to put message in queue: {}.".format(mq_message))
+                    send_mail(
+                        "Error", "Error to put message " + mq_message + " in queue.", cfg
                     )
-                )
-            except OSError:
-                logging.error(
-                    "PID {:>5} >> Error to move {} into {}.".format(
-                        os.getpid(), filenameExt, newFilename
-                    )
-                )
-                send_mail(
-                    "Error", "OS error move " + filenameExt + " to " + newFilename, cfg
-                )
-
-            mq_message = "{};{};{};{};{};{};{}".format(
-                unitType,
-                unitName,
-                toolName,
-                toolType,
-                df.dateFmt(fileDate),
-                fileTime,
-                newFilename,
-            )
-            try:
-                queue = mq(cfg)
-                queue.write(mq_message, cfg)
-                logging.info(
-                    "PID {:>5} >> queue message: {}.".format(
-                        os.getpid(), mq_message)
-                )
-            except:
-                logging.error(
-                    "PID {:>5} >> Error to put message in queue: {}.".format(
-                        os.getpid(), mq_message
-                    )
-                )
-                send_mail(
-                    "Error", "Error to put message " + mq_message + " in queue.", cfg
-                )
-            finally:
-                queue.close()
+                finally:
+                    queue.close()
 
     def on_incomplete_file_received(self, file):
         # remove partially uploaded files
@@ -316,11 +294,7 @@ class ASEHandler(FTPHandler):
                             (user, hash, cfg.virtpath + user, 'elmw'))
                 con.commit()
                 con.close()
-                logging.info(
-                    "PID {:>5} >> User {} created.".format(
-                        os.getpid(), user
-                    )
-                )
+                logging.info("User {} created.".format(user))
                 self.respond('200 SITE ADDU successful.')
             except:
                 self.respond('501 SITE ADDU failed.')
@@ -339,11 +313,8 @@ class ASEHandler(FTPHandler):
             cur.execute("DELETE FROM virtusers WHERE user = ?", (user,))
             con.commit()
             con.close()
-            logging.info(
-                "PID {:>5} >> User {} deleted.".format(
-                    os.getpid(), user
-                )
-            )
+            logging.info("User {} deleted.".format(user))
+            # self.push(' The user path has not been removed!\r\n')
             self.respond('200 SITE DELU successful.')
 
         except:
@@ -361,7 +332,7 @@ class ASEHandler(FTPHandler):
             self.push("214-The following virtual users are defined:\r\n")
             for row in cur.execute("SELECT * FROM virtusers").fetchall():
                 users_list.append(
-                    " Username: " + row[0] + " - Perms: " + row[3] + "\r\n")
+                    " Username: " + row[0] + "\tPerms: " + row[3] + "\r\n")
             con.close()
             self.push(''.join(users_list))
             self.respond("214 LSTU SITE command successful.")
@@ -393,8 +364,7 @@ def main():
         server.serve_forever()
     except KeyboardInterrupt:
         logging.info(
-            "PID {:>5} >> Info: {}.".format(
-                os.getpid(), "Shutdown requested...exiting")
+            "Info: {}.".format("Shutdown requested...exiting")
         )
     except Exception:
         print(

ase-receiver/ase-receiver/demonize_ftpd.py

@@ -28,6 +28,7 @@ Parts you might want to customize are:
 Authors:
  - Ben Timby - btimby <at> gmail.com
  - Giampaolo Rodola' - g.rodola <at> gmail.com
+ 
 """
 
 import atexit

ase-receiver/ase-receiver/ftpcsvreceiver.ini

@@ -1,13 +1,16 @@
+# to generete adminuser password hash:
+# python3 -c 'from hashlib import md5;print(md5("????admin-password???".encode("UTF-8")).hexdigest())'
 [ftpserver]
   firstPort = 40000
   logFilename = ./ftppylog.log
   proxyAddr = 0.0.0.0
-  portRangeWidth = 50
+  portRangeWidth = 500
   virtusersdb = /home/alex/aseftp/virtusers.db
   virtpath = /home/alex/aseftp/
   adminuser = admin|c8cf955bd8b8a78419013b831e627eb2|/home/alex/aseftp/|elradfmwMT
   servertype = FTPHandler
   certfile = /home/alex/aseftp/keycert.pem
+  fileext = .CSV|.txt
   #servertype = FTPHandler/TLS_FTPHandler
 
 
@@ -22,18 +25,18 @@
                 bbbbb
                 ccccc
   subject = ciao a domani
-  debug = 1
+  debug = 0
 
 [mqserver]
-  hostname = 192.168.1.241
+  hostname = galera1
   port = 5672
-  user = alex
-  password = batt1l0
+  user = asemq
+  password = Ase2021
   csvQueue = task_queue
   elabQueue = elab_queue
 
 [csvfs]
-  path = /home/
+  path = /home/alex/aseftp/csvfs/
 
 [csvelab]
   logFilename = csvElab.log

ase-receiver/ase-receiver/utils/config/set_config.py

@@ -7,7 +7,8 @@ from configparser import ConfigParser
 class config:
     def __init__(self):
         c = ConfigParser()
-        c.read(["/etc/aseftp/ftpcsvreceiver.ini", "./ftpcsvreceiver.ini"])
+        c.read(["/etc/aseftp/ftpcsvreceiver.ini", "./ftpcsvreceiver.ini",
+               "./ase-receiver/ase-receiver/ftpcsvreceiver.ini"])
         # FTP setting
         self.firstport = c.getint("ftpserver", "firstPort")
         self.logfilename = c.get("ftpserver", "logFilename")
@@ -18,6 +19,7 @@ class config:
         self.adminuser = c.get("ftpserver", "adminuser").split("|")
         self.servertype = c.get("ftpserver", "servertype")
         self.certfile = c.get("ftpserver", "certfile")
+        self.fileext = c.get("ftpserver", "fileext").upper().split("|")
 
         # MAIL setting
         self.smtphost = c.get("mailserver", "hostname")

ase-receiver/ase-receiver/utils/time/date_refmt.py

@@ -2,24 +2,27 @@
 
 """
 
-import datetime
+from datetime import datetime
+from re import search
 
 
 def dateFmt(date):
     t = date.replace("/", "-")
-    try:
-        datetime.datetime.strptime(t, "%Y-%m-%d")
-        return t
-    except ValueError:
-        d = datetime.datetime.strptime(t, "%d-%m-%Y")
-        return datetime.datetime.strftime(d, "%Y-%m-%d")
+    if search('^\d\d\d\d-\d\d-\d\d$', t):
+        d = datetime.strptime(t, "%Y-%m-%d")
+    elif search('^\d\d-\d\d-\d\d$', t):
+        d = datetime.strptime(t, "%y-%m-%d")
+    elif search('^\d\d-\d\d-\d\d\d\d$', t):
+        d = datetime.strptime(t, "%d-%m-%Y")
+    return datetime.strftime(d, "%Y-%m-%d")
 
 
 def dateTimeFmt(date):
     t = date.replace("/", "-")
-    try:
-        datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
-        return t
-    except ValueError:
-        d = datetime.datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
-        return datetime.datetime.strftime(d, "%Y-%m-%d %H:%M:%S")
+    if search('^\d\d\d\d-\d\d-\d\d$', t):
+        d = datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
+    elif search('^\d\d-\d\d-\d\d$', t):
+        d = datetime.strptime(t, "%y-%m-%d %H:%M:%S")
+    elif search('^\d\d-\d\d-\d\d\d\d$', t):
+        d = datetime.strptime(t, "%d-%m-%Y %H:%M:%S")
+    return datetime.strftime(d, "%Y-%m-%d")

ase-receiver/ase-receiver/utils/time/timestamp_fmt.py

@@ -2,9 +2,9 @@
 
 """
 
-import datetime
+from datetime import datetime
 
 
 def timestamp(t):
     fmt = {"log": "%Y-%m-%d %H:%M:%S", "tms": "%Y%m%d%H%M%S"}
-    return datetime.datetime.now().strftime(fmt[t])
+    return datetime.now().strftime(fmt[t])

pivot_pianificazione.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+from dataclasses import dataclass
+
+import pandas as pd
+
+dm = {"gennaio ": "01/", "febbraio" }
+di = {1: "A", 2: "B"}
+
+pianificazioni = pd.read_csv("/home/alex/Documenti/Cdig_Dettaglio_Pianificazione_Piatta.csv")
+
+pianificazioni.pivot_table(index=['Attività','Tipo_Progetto','Descrizione_Progetto'],columns='Mese',values='perc_pianificazione',aggfunc = "sum", margins = True, fill_value = "")
+print(pianificazioni.pivot_table(index='Attività', values='GG pianificati'))