ASE/csvLoader/CsvLoader.py

#!/usr/bin/python3

import sys
import os
import pika
import logging
import csv
import re
import mariadb
import shutil

from utils.timefmt import timestamp_fmt as ts
from utils.timefmt import date_refmt as df
from utils.config import set_config as setting


class sqlraw:
    def __init__(self, cfg):
        self.config = {"host": cfg.dbhost, "user": cfg.dbuser, "password": cfg.dbpass}
        self.dbname = cfg.dbname
        self.table = cfg.table
        self.sql_head = (
            "INSERT IGNORE INTO "
            + self.dbname
            + "."
            + self.table
            + " (`UnitName`,`ToolName`,`eventDT`,`BatteryLevel`,`Temperature`,`NodeNum`,"
            + "`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,"
            + "`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`) VALUES "
        )

    def add_data(self, values):
        self.sql = self.sql_head + "(" + "),(".join(values) + ");"

    def write_db(self):
        try:
            conn = mariadb.connect(**self.config, database=self.dbname)
        except mariadb.Error as err:
            logging.error(
                "PID {:>5} >> Error to connet to DB {} - System error {}.".format(
                    os.getpid(), self.dbname, err
                )
            )
            sys.exit(1)

        cur = conn.cursor()
        try:
            cur.execute(self.sql)
        except mariadb.ProgrammingError as err:
            logging.error(
                "PID {:>5} >> Error write into DB {} - System error {}.".format(
                    os.getpid(), self.dbname, err
                )
            )
            print(err)
            sys.exit(1)

        finally:
            conn.close()


def callback_ase(ch, method, properties, body, config):  # body è di tipo byte
    logging.info(
        "PID {0:>5} >> Read message {1}".format(os.getpid(), body.decode("utf-8"))
    )
    msg = body.decode("utf-8").split(";")
    sql = sqlraw(config)
    stmlst = []
    commonData = '"{0}","{1}"'.format(msg[1], msg[2])
    tooltype = msg[3]
    with open(msg[6], "r") as csvfile:
        lines = csvfile.read().splitlines()
        for line in lines:

            fields = line.split(";|;")

            if mG501 := re.match(
                r"^(\d\d\d\d\/\d\d\/\d\d\s\d\d:\d\d:\d\d);(.+);(.+)$", fields[0]
            ):
                rowData = ',"{0}",{1},{2}'.format(
                    mG501.group(1), mG501.group(2), mG501.group(3)
                )
                fields.pop(0)

            elif mG201 := re.match(
                r"^(\d\d\/\d\d\/\d\d\d\d\s\d\d:\d\d:\d\d)$", fields[0]
            ):
                mbtG201 = re.match(r"^(.+);(.+)$", fields[1])

                rowData = ',"{0}",{1},{2}'.format(
                    df.dateTimeFmt(mG201.group(1)), mbtG201.group(1), mbtG201.group(2)
                )
                fields.pop(0)
                fields.pop(0)

            else:
                continue

            nodeNum = 0
            for field in fields:
                nodeNum += 1
                vals = field.split(";")
                stmlst.append(
                    commonData
                    + rowData
                    + ",{0},".format(nodeNum)
                    + ",".join('"{0}"'.format(d) for d in vals)
                    + ","
                    + ",".join(["null"] * (config.valueNum - len(vals)))
                )

            if config.maxInsertRow < len(stmlst):
                sql.add_data(stmlst)
                try:
                    sql.write_db()
                    stmlst.clear()
                except:
                    print("errore nell'inserimento")
                    sys.exit(1)

        if len(stmlst) > 0:
            sql.add_data(stmlst)
            try:
                sql.write_db()
                ch.basic_ack(delivery_tag=method.delivery_tag)
            except:
                print("errore nell'inserimento")
                sys.exit(1)
    newFilename = msg[6].replace("received", "loaded")
    newPath, filenameExt = os.path.split(newFilename)
    try:
        os.makedirs(newPath)
        logging.info("PID {:>5} >> path {} created.".format(os.getpid(), newPath))
    except FileExistsError:
        logging.info(
            "PID {:>5} >> path {} already exists.".format(os.getpid(), newPath)
        )
    try:
        shutil.move(msg[6], newFilename)
        logging.info(
            "PID {:>5} >> {} moved into {}.".format(
                os.getpid(), filenameExt, newFilename
            )
        )
    except OSError:
        logging.error(
            "PID {:>5} >> Error to move {} into {}.".format(
                os.getpid(), filenameExt, newFilename
            )
        )


def main():
    cfg = setting.config()

    logging.basicConfig(
        format="%(asctime)s %(message)s",
        filename="/var/log/" + cfg.elablog,
        level=logging.INFO,
    )

    parameters = pika.URLParameters(
        "amqp://"
        + cfg.mquser
        + ":"
        + cfg.mqpass
        + "@"
        + cfg.mqhost
        + ":"
        + cfg.mqport
        + "/%2F"
    )
    connection = pika.BlockingConnection(parameters)
    channel = connection.channel()
    channel.queue_declare(queue=cfg.csv_queue, durable=True)

    channel.basic_qos(prefetch_count=1)
    channel.basic_consume(
        queue=cfg.csv_queue,
        on_message_callback=lambda ch, method, properties, body: callback_ase(
            ch, method, properties, body, config=cfg
        ),
    )

    # channel.basic_consume(queue=cfg.csv_queue, on_message_callback=callback,arguments=cfg)
    try:
        channel.start_consuming()
    except KeyboardInterrupt:
        logging.info(
            "PID {0:>5} >> Info: {1}.".format(
                os.getpid(), "Shutdown requested...exiting"
            )
        )


if __name__ == "__main__":
    main()