initial

csvLoader/CsvLoader.py

@@ -0,0 +1,197 @@
+#!/usr/bin/python3
+
+import sys
+import os
+import pika
+import logging
+import csv
+import re
+import mariadb
+import shutil
+
+from utils.timefmt import timestamp_fmt as ts
+from utils.timefmt import date_refmt as df
+from utils.config import set_config as setting
+
+
+class sqlraw:
+    def __init__(self, cfg):
+        self.config = {"host": cfg.dbhost, "user": cfg.dbuser, "password": cfg.dbpass}
+        self.dbname = cfg.dbname
+        self.table = cfg.table
+        self.sql_head = (
+            "INSERT IGNORE INTO "
+            + self.dbname
+            + "."
+            + self.table
+            + " (`UnitName`,`ToolName`,`eventDT`,`BatteryLevel`,`Temperature`,`NodeNum`,"
+            + "`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,"
+            + "`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`) VALUES "
+        )
+
+    def add_data(self, values):
+        self.sql = self.sql_head + "(" + "),(".join(values) + ");"
+
+    def write_db(self):
+        try:
+            conn = mariadb.connect(**self.config, database=self.dbname)
+        except mariadb.Error as err:
+            logging.error(
+                "PID {:>5} >> Error to connet to DB {} - System error {}.".format(
+                    os.getpid(), self.dbname, err
+                )
+            )
+            sys.exit(1)
+
+        cur = conn.cursor()
+        try:
+            cur.execute(self.sql)
+        except mariadb.ProgrammingError as err:
+            logging.error(
+                "PID {:>5} >> Error write into DB {} - System error {}.".format(
+                    os.getpid(), self.dbname, err
+                )
+            )
+            print(err)
+            sys.exit(1)
+
+        finally:
+            conn.close()
+
+
+def callback_ase(ch, method, properties, body, config):  # body è di tipo byte
+    logging.info(
+        "PID {0:>5} >> Read message {1}".format(os.getpid(), body.decode("utf-8"))
+    )
+    msg = body.decode("utf-8").split(";")
+    sql = sqlraw(config)
+    stmlst = []
+    commonData = '"{0}","{1}"'.format(msg[1], msg[2])
+    tooltype = msg[3]
+    with open(msg[6], "r") as csvfile:
+        lines = csvfile.read().splitlines()
+        for line in lines:
+
+            fields = line.split(";|;")
+
+            if mG501 := re.match(
+                r"^(\d\d\d\d\/\d\d\/\d\d\s\d\d:\d\d:\d\d);(.+);(.+)$", fields[0]
+            ):
+                rowData = ',"{0}",{1},{2}'.format(
+                    mG501.group(1), mG501.group(2), mG501.group(3)
+                )
+                fields.pop(0)
+
+            elif mG201 := re.match(
+                r"^(\d\d\/\d\d\/\d\d\d\d\s\d\d:\d\d:\d\d)$", fields[0]
+            ):
+                mbtG201 = re.match(r"^(.+);(.+)$", fields[1])
+
+                rowData = ',"{0}",{1},{2}'.format(
+                    df.dateTimeFmt(mG201.group(1)), mbtG201.group(1), mbtG201.group(2)
+                )
+                fields.pop(0)
+                fields.pop(0)
+
+            else:
+                continue
+
+            nodeNum = 0
+            for field in fields:
+                nodeNum += 1
+                vals = field.split(";")
+                stmlst.append(
+                    commonData
+                    + rowData
+                    + ",{0},".format(nodeNum)
+                    + ",".join('"{0}"'.format(d) for d in vals)
+                    + ","
+                    + ",".join(["null"] * (config.valueNum - len(vals)))
+                )
+
+            if config.maxInsertRow < len(stmlst):
+                sql.add_data(stmlst)
+                try:
+                    sql.write_db()
+                    stmlst.clear()
+                except:
+                    print("errore nell'inserimento")
+                    sys.exit(1)
+
+        if len(stmlst) > 0:
+            sql.add_data(stmlst)
+            try:
+                sql.write_db()
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+            except:
+                print("errore nell'inserimento")
+                sys.exit(1)
+    newFilename = msg[6].replace("received", "loaded")
+    newPath, filenameExt = os.path.split(newFilename)
+    try:
+        os.makedirs(newPath)
+        logging.info("PID {:>5} >> path {} created.".format(os.getpid(), newPath))
+    except FileExistsError:
+        logging.info(
+            "PID {:>5} >> path {} already exists.".format(os.getpid(), newPath)
+        )
+    try:
+        shutil.move(msg[6], newFilename)
+        logging.info(
+            "PID {:>5} >> {} moved into {}.".format(
+                os.getpid(), filenameExt, newFilename
+            )
+        )
+    except OSError:
+        logging.error(
+            "PID {:>5} >> Error to move {} into {}.".format(
+                os.getpid(), filenameExt, newFilename
+            )
+        )
+
+
+def main():
+    cfg = setting.config()
+
+    logging.basicConfig(
+        format="%(asctime)s %(message)s",
+        filename="/var/log/" + cfg.elablog,
+        level=logging.INFO,
+    )
+
+    parameters = pika.URLParameters(
+        "amqp://"
+        + cfg.mquser
+        + ":"
+        + cfg.mqpass
+        + "@"
+        + cfg.mqhost
+        + ":"
+        + cfg.mqport
+        + "/%2F"
+    )
+    connection = pika.BlockingConnection(parameters)
+    channel = connection.channel()
+    channel.queue_declare(queue=cfg.csv_queue, durable=True)
+
+    channel.basic_qos(prefetch_count=1)
+    channel.basic_consume(
+        queue=cfg.csv_queue,
+        on_message_callback=lambda ch, method, properties, body: callback_ase(
+            ch, method, properties, body, config=cfg
+        ),
+    )
+
+    # channel.basic_consume(queue=cfg.csv_queue, on_message_callback=callback,arguments=cfg)
+    try:
+        channel.start_consuming()
+    except KeyboardInterrupt:
+        logging.info(
+            "PID {0:>5} >> Info: {1}.".format(
+                os.getpid(), "Shutdown requested...exiting"
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()