diff --git a/.gitignore b/.gitignore index 7c1d0d6..bdb744f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,11 +8,9 @@ README.md prova*.* .codegpt build/ -docs/ LoadCSVData.pl matlab_elab.py doc_carri.txt ase.egg-info/ -mkdocs.yml site/ site.zip diff --git a/docs/gen_ref_pages.py b/docs/gen_ref_pages.py new file mode 100644 index 0000000..432c554 --- /dev/null +++ b/docs/gen_ref_pages.py @@ -0,0 +1,91 @@ +"""Genera le pagine di riferimento per l'API.""" + +from pathlib import Path +import mkdocs_gen_files + +nav = mkdocs_gen_files.Nav() + +# File e directory da escludere +EXCLUDE_PATTERNS = { + ".env", + ".env.*", + "__pycache__", + ".git", + ".pytest_cache", + ".venv", + "venv", + "node_modules", + "docs", # Escludi tutta la directory docs + "build", + "dist", + "*.egg-info", + ".mypy_cache", + ".coverage", + "htmlcov" +} + +def should_exclude(path: Path) -> bool: + """Verifica se un percorso deve essere escluso.""" + # Escludi file .env + if path.name.startswith('.env'): + return True + + # Escludi lo script stesso + if path.name == "gen_ref_pages.py": + return True + + # Escludi tutta la directory docs + if "old_script" in path.parts: + return True + + # Escludi tutta la directory docs + if "docs" in path.parts: + return True + + # Escludi pattern comuni + for pattern in EXCLUDE_PATTERNS: + if pattern in str(path): + return True + + return False + +# Cerca i file Python nella directory corrente +for path in sorted(Path(".").rglob("*.py")): + # Salta i file esclusi + if should_exclude(path): + continue + + # Salta i file che iniziano con un punto + if any(part.startswith('.') for part in path.parts): + continue + + # Salta i file che iniziano con prova + if any(part.startswith('prova') for part in path.parts): + continue + + if any(part.startswith('matlab_elab') for part in path.parts): + continue + + module_path = path.with_suffix("") + doc_path = path.with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = tuple(module_path.parts) + + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + + nav[parts] = doc_path.as_posix() + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + ident = ".".join(parts) + fd.write(f"::: {ident}") + + mkdocs_gen_files.set_edit_path(full_doc_path, path) + +with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..007e1fd --- /dev/null +++ b/docs/index.md @@ -0,0 +1,36 @@ +# Benvenuto nella documentazione + +Questa è la documentazione automatica dell'applicazione Python ASE per la gestione delle file CSV ricevuti via FTP. + +## Funzionalità + + - Ricezione di file csv via FTP e salvataggio in database. + - Caricamnento dei dati in database con moduli dedicati per: + - tipologia di centralina e sensore + - nome di centralina e sensore + + - Esecuzione elaborazione MatLab. + + - Gestione utenti FTP + - Caricamento massivo utenti FTP da database + +## Setup + + - personalizzazione dei file env: + - env/db.ini + - env/ftp.ini + - env/load.ini + - env/elab.ini + + - esecuzione del server FTP -> "python ftp_csv_receiver.py" + - esecuzione dell'orchestratore del caricamenti dei file csv -> "python load_orchestrator.py" + - esecuzione dell'orchestratore delle elaborazioni MatLab -> "python elab_orchestrator.py" + +E' possibile creare servizi systemd per gestire l'esecuzione automatica delle funzionalità. +Viene usato il virtualenv quindi python deve essere eseguito con i dovuti setting + +## Installazione + +Installare il pacchetto ase-x.x.x-py3-none-any.whl + + - pip install ase-x.x.x-py3-none-any.whl diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..d408f88 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,66 @@ +site_name: Ase receiver +site_description: Documentazione automatica della app Python ASE + + +theme: + name: material + features: + - navigation.tabs + - navigation.sections + - toc.integrate + - navigation.top + - search.suggest + - search.highlight + - content.tabs.link + - content.code.annotation + - content.code.copy + +plugins: + - offline + - search + - mkdocstrings: + handlers: + python: + paths: ["."] + options: + docstring_style: google + show_source: true + show_root_heading: true + show_root_toc_entry: true + show_symbol_type_heading: true + show_symbol_type_toc: true + filters: + - "!^docs" # Escludi tutto ciò che inizia con "docs" + - gen-files: + scripts: + - docs/gen_ref_pages.py + - literate-nav: + nav_file: SUMMARY.md + +nav: + - Home: index.md + - API Reference: reference/ + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - admonition + - pymdownx.details + - attr_list + - md_in_html + +# Escludi file dalla build +exclude_docs: | + .env* + __pycache__/ + .git/ + .pytest_cache/ + .venv/ + venv/ + test/ + .vscode/ \ No newline at end of file diff --git a/src/load_orchestrator.py b/src/load_orchestrator.py index aacf5be..1e1e3ad 100755 --- a/src/load_orchestrator.py +++ b/src/load_orchestrator.py @@ -12,6 +12,7 @@ import asyncio from utils.config import loader_load_data as setting from utils.database import WorkflowFlags from utils.csv.loaders import get_next_csv_atomic +from utils.database.action_query import check_flag_elab from utils.orchestrator_utils import run_orchestrator, worker_context # Initialize the logger for this module @@ -42,21 +43,24 @@ async def worker(worker_id: int, cfg: object, pool: object) -> None: while True: try: logger.info("Inizio elaborazione") + if not await check_flag_elab(): + record = await get_next_csv_atomic( + pool, + cfg.dbrectable, + WorkflowFlags.CSV_RECEIVED, + WorkflowFlags.DATA_LOADED, + ) - record = await get_next_csv_atomic( - pool, - cfg.dbrectable, - WorkflowFlags.CSV_RECEIVED, - WorkflowFlags.DATA_LOADED, - ) - - if record: - success = await load_csv(record, cfg, pool) - if not success: - logger.error("Errore durante l'elaborazione") - await asyncio.sleep(CSV_PROCESSING_DELAY) + if record: + success = await load_csv(record, cfg, pool) + if not success: + logger.error("Errore durante l'elaborazione") + await asyncio.sleep(CSV_PROCESSING_DELAY) + else: + logger.info("Nessun record disponibile") + await asyncio.sleep(NO_RECORD_SLEEP) else: - logger.info("Nessun record disponibile") + logger.info("Flag fermo elaborazione attivato") await asyncio.sleep(NO_RECORD_SLEEP) except Exception as e: # pylint: disable=broad-except diff --git a/src/utils/connect/file_management.py b/src/utils/connect/file_management.py index 191a6f0..d976ab9 100644 --- a/src/utils/connect/file_management.py +++ b/src/utils/connect/file_management.py @@ -1,10 +1,10 @@ import os +from datetime import datetime import logging import re import mysql.connector from utils.database.connection import connetti_db - from utils.csv.parser import extract_value logger = logging.getLogger(__name__) @@ -27,6 +27,9 @@ def on_file_received(self: object, file: str) -> None: cfg = self.cfg path, filenameExt = os.path.split(file) filename, fileExtension = os.path.splitext(filenameExt) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + new_filename = f"{filename}_{timestamp}{fileExtension}" + os.rename(file, f"{path}/{new_filename}") if (fileExtension.upper() in (cfg.fileext)): with open(file, 'r', encoding='utf-8', errors='ignore') as csvfile: lines = csvfile.readlines() @@ -74,13 +77,15 @@ def on_file_received(self: object, file: str) -> None: tool_info = f'{{"Stazione": "{cfg.ts_pini_path_match.get(stazione)}"}}' try: - cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (username, filename, unit_name, unit_type, tool_name, tool_type, tool_data, tool_info) VALUES (%s,%s, %s, %s, %s, %s, %s, %s)", (self.username, filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines), tool_info)) + cur.execute(f"INSERT INTO {cfg.dbname}.{cfg.dbrectable} (username, filename, unit_name, unit_type, tool_name, tool_type, tool_data, tool_info) VALUES (%s,%s, %s, %s, %s, %s, %s, %s)", (self.username, new_filename, unit_name.upper(), unit_type.upper(), tool_name.upper(), tool_type.upper(), ''.join(lines), tool_info)) conn.commit() conn.close() except Exception as e: - logger.error(f'File {file} not loaded. Held in user path.') + logger.error(f'File {new_filename} not loaded. Held in user path.') logger.error(f'{e}') + """ else: os.remove(file) - logger.info(f'File {file} loaded: removed.') \ No newline at end of file + logger.info(f'File {new_filename} removed.') + """ \ No newline at end of file diff --git a/src/utils/database/action_query.py b/src/utils/database/action_query.py index 3db3cbf..2de359f 100644 --- a/src/utils/database/action_query.py +++ b/src/utils/database/action_query.py @@ -132,4 +132,16 @@ async def get_elab_timestamp(id_recv: int, pool: object) -> float: except Exception as e: logger.error(f"id {id_recv} - Errore nella query timestamp elaborazione: {e}") - return None \ No newline at end of file + return None + +async def check_flag_elab(pool: object) -> None: + async with pool.acquire() as conn: + async with conn.cursor() as cur: + try: + await cur.execute("SELECT ferma_elab from admin_panel") + results = await cur.fetchone() + return results[0] + + except Exception as e: + logger.error(f"Errore nella query check flag stop elaborazioni: {e}") + return None