build image

.gitignore

@@ -12,4 +12,5 @@ doc_carri.txt
 ase.egg-info/
 site/
 site.zip
+*/src/
 .vscode/extensions.json

scripts/DOCKER_REGISTRY_GUIDE.md

@@ -0,0 +1,610 @@
+# Guida Docker Registry Privato
+
+Questa guida spiega come configurare un registry Docker privato e utilizzare gli script per buildare e distribuire le immagini degli orchestrator.
+
+## Indice
+
+1. [Setup Registry Docker Privato](#setup-registry-docker-privato)
+2. [Protezione Codice Sorgente (Bytecode)](#protezione-codice-sorgente-bytecode)
+3. [Build e Push dell'Immagine](#build-e-push-dellimmagine)
+4. [Aggiornamento Docker Compose](#aggiornamento-docker-compose)
+5. [Deploy sulle VM](#deploy-sulle-vm)
+
+---
+
+## Setup Registry Docker Privato
+
+### Opzione 1: Registry Locale Semplice (HTTP - Solo per sviluppo)
+
+```bash
+# Avvia un registry Docker locale sulla porta 5000
+docker run -d -p 5000:5000 --name registry --restart=always registry:2
+
+# Verifica che sia in esecuzione
+curl http://localhost:5000/v2/_catalog
+```
+
+### Opzione 2: Registry Locale con Persistenza
+
+```bash
+# Crea directory per i dati
+mkdir -p /opt/docker-registry/data
+
+# Avvia registry con volume persistente
+docker run -d \
+  -p 5000:5000 \
+  --name registry \
+  --restart=always \
+  -v /opt/docker-registry/data:/var/lib/registry \
+  registry:2
+```
+
+### Opzione 3: Registry con HTTPS (Certificato Self-Signed con SANs)
+
+⚠️ **IMPORTANTE**: I certificati moderni devono usare Subject Alternative Names (SANs) invece del campo Common Name.
+
+```bash
+# Usa lo script per generare certificati corretti
+./scripts/generate-registry-cert.sh 192.168.1.204 registry.local
+
+# Oppure manualmente:
+mkdir -p /opt/docker-registry/{data,certs}
+
+# Crea file di configurazione OpenSSL con SANs
+cat > /opt/docker-registry/certs/openssl.cnf << 'EOF'
+[req]
+default_bits       = 4096
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+x509_extensions    = v3_ca
+prompt             = no
+
+[req_distinguished_name]
+C  = IT
+ST = Italy
+L  = City
+O  = Docker Registry
+CN = registry.local
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_ca]
+subjectAltName = @alt_names
+basicConstraints = critical, CA:TRUE
+keyUsage = critical, digitalSignature, keyEncipherment, keyCertSign
+extendedKeyUsage = serverAuth
+
+[alt_names]
+DNS.1 = registry.local
+DNS.2 = localhost
+IP.1  = 192.168.1.204
+IP.2  = 127.0.0.1
+EOF
+
+# Genera certificato con SANs
+openssl req -newkey rsa:4096 \
+  -nodes -sha256 \
+  -keyout /opt/docker-registry/certs/domain.key \
+  -x509 -days 365 \
+  -config /opt/docker-registry/certs/openssl.cnf \
+  -out /opt/docker-registry/certs/domain.crt
+
+# Verifica SANs nel certificato
+openssl x509 -in /opt/docker-registry/certs/domain.crt -text -noout | grep -A 3 "Subject Alternative Name"
+
+# Avvia registry con HTTPS
+docker run -d \
+  -p 5000:5000 \
+  --name registry \
+  --restart=always \
+  -v /opt/docker-registry/data:/var/lib/registry \
+  -v /opt/docker-registry/certs:/certs \
+  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
+  -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
+  registry:2
+```
+
+### Opzione 4: Registry con Autenticazione + HTTPS
+
+```bash
+# Genera certificati (vedi Opzione 3)
+./scripts/generate-registry-cert.sh 192.168.1.204 registry.local
+
+# Crea utente e password
+mkdir -p /opt/docker-registry/auth
+docker run --rm --entrypoint htpasswd httpd:2 \
+  -Bbn myuser mypassword > /opt/docker-registry/auth/htpasswd
+
+# Avvia registry con autenticazione e HTTPS
+docker run -d \
+  -p 5000:5000 \
+  --name registry \
+  --restart=always \
+  -v /opt/docker-registry/data:/var/lib/registry \
+  -v /opt/docker-registry/auth:/auth \
+  -v /opt/docker-registry/certs:/certs \
+  -e REGISTRY_AUTH=htpasswd \
+  -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" \
+  -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
+  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
+  -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
+  registry:2
+
+# Login al registry
+docker login 192.168.1.204:5000
+```
+
+### Configurazione Client Docker per Registry HTTPS (Self-Signed)
+
+Se usi certificati self-signed, devi installare il certificato sui client Docker:
+
+#### Metodo Automatico (Raccomandato)
+
+```bash
+# Sul tuo computer di sviluppo o sulle VM client
+./scripts/setup-registry-client.sh 192.168.1.204
+
+# Lo script scaricherà e installerà automaticamente il certificato
+```
+
+#### Metodo Manuale
+
+```bash
+# 1. Crea directory per i certificati Docker
+sudo mkdir -p /etc/docker/certs.d/192.168.1.204:5000
+
+# 2. Copia il certificato dal registry
+scp root@192.168.1.204:/opt/docker-registry/certs/domain.crt /tmp/registry.crt
+sudo cp /tmp/registry.crt /etc/docker/certs.d/192.168.1.204:5000/ca.crt
+
+# 3. Riavvia Docker
+sudo systemctl restart docker
+
+# 4. (Opzionale) Installa anche nel sistema per curl/wget
+sudo cp /tmp/registry.crt /usr/local/share/ca-certificates/registry.crt
+sudo update-ca-certificates
+```
+
+#### Test Connessione con Curl
+
+```bash
+# Test con certificato insecure (disabilita verifica)
+curl -k https://192.168.1.204:5000/v2/
+
+# Test con certificato installato nel sistema
+curl https://192.168.1.204:5000/v2/_catalog
+
+# Test con certificato specificato
+curl --cacert /etc/docker/certs.d/192.168.1.204:5000/ca.crt https://192.168.1.204:5000/v2/_catalog
+
+# Con autenticazione
+curl -k -u myuser:mypassword https://192.168.1.204:5000/v2/_catalog
+```
+
+### Configurazione Client Docker per Registry Insicuro (HTTP)
+
+Se usi un registry HTTP locale, devi configurare Docker per accettare connessioni insicure:
+
+```bash
+# Edita /etc/docker/daemon.json
+sudo nano /etc/docker/daemon.json
+
+# Aggiungi:
+{
+  "insecure-registries": ["localhost:5000", "192.168.1.X:5000"]
+}
+
+# Riavvia Docker
+sudo systemctl restart docker
+```
+
+---
+
+## Protezione Codice Sorgente (Bytecode)
+
+Le immagini Docker vengono buildare con una configurazione speciale che protegge il codice sorgente Python.
+
+### Come Funziona
+
+Il Dockerfile è configurato per:
+
+1. **Compilare** tutti i file Python in bytecode (`.pyc`) usando Python con ottimizzazione `-OO`
+2. **Rimuovere** tutti i file sorgente `.py`
+3. **Mantenere** solo i file bytecode compilati in `__pycache__/`
+
+```dockerfile
+# Compila tutti i file Python in bytecode
+# Usa -OO per rimuovere docstring e assert (ottimizzazione massima)
+RUN python -OO -m compileall /app/src /app/env || true
+
+# Rimuovi tutti i file sorgente .py, lasciando solo i .pyc compilati in __pycache__
+RUN find /app/src -type f -name "*.py" -delete && \
+    find /app/env -type f -name "*.py" -delete || true
+```
+
+### Vantaggi
+
+✅ **Protezione del Codice**: I file sorgente non sono accessibili nell'immagine
+✅ **Dimensione Ridotta**: I file `.pyc` sono più compatti dei `.py`
+✅ **Performance**: Leggero miglioramento delle prestazioni (no compilazione a runtime)
+✅ **Rimozione Docstring**: Con `-OO` vengono rimossi anche commenti e docstring
+
+### Limitazioni
+
+⚠️ **Debug Difficile**: I traceback mostrano solo numeri di riga, non il codice sorgente
+⚠️ **No Reverse Engineering**: Impossibile vedere la logica senza decompilare
+⚠️ **Serve Backup**: Mantieni sempre i sorgenti in un repository git separato
+
+### Test dell'Immagine Bytecode
+
+Usa lo script `test-pyc-image.sh` per verificare che l'immagine funzioni correttamente:
+
+```bash
+# Test immagine locale
+./scripts/test-pyc-image.sh orchestrator-app:latest
+
+# Test immagine dal registry
+./scripts/test-pyc-image.sh 192.168.1.203:5000/orchestrator-app:v1.0.0
+```
+
+Il test verifica:
+- ✅ Assenza di file `.py` in `/app/src`
+- ✅ Presenza di file `.pyc` compilati
+- ✅ Importazione corretta dei moduli
+- ✅ Esecuzione dei moduli orchestrator
+- ✅ Configurazione ambiente Python
+
+### Struttura File nell'Immagine
+
+**Prima (con sorgenti):**
+```
+/app/src/
+├── load_orchestrator.py
+├── elab_orchestrator.py
+├── send_orchestrator.py
+└── ftp_csv_receiver.py
+```
+
+**Dopo (solo bytecode):**
+```
+/app/src/
+├── __pycache__/
+│   ├── load_orchestrator.cpython-312.opt-2.pyc
+│   ├── elab_orchestrator.cpython-312.opt-2.pyc
+│   ├── send_orchestrator.cpython-312.opt-2.pyc
+│   └── ftp_csv_receiver.cpython-312.opt-2.pyc
+```
+
+---
+
+## Build e Push dell'Immagine
+
+### Script: `build-and-push-image.sh`
+
+Lo script automatizza il processo di build e push dell'immagine Docker.
+
+#### Sintassi
+
+```bash
+./scripts/build-and-push-image.sh [registry_url] [image_name] [tag] [dockerfile_type]
+```
+
+#### Parametri
+
+- `registry_url` - URL del registry privato (default: `192.168.1.204:5000`)
+- `image_name` - Nome dell'immagine (default: `orchestrator-app`)
+- `tag` - Tag dell'immagine (default: `latest`)
+- `dockerfile_type` - Tipo di Dockerfile: `standard` o `distroless` (default: `standard`)
+
+#### Tipi di Dockerfile
+
+**`standard`** - Usa `Dockerfile` (python:3.12-slim)
+- Dimensione: ~333MB
+- Include shell per debugging
+- Ideale per sviluppo e staging
+
+**`distroless`** - Usa `Dockerfile.distroless` (gcr.io/distroless/python3)
+- Dimensione: ~180MB (50% più piccola!)
+- Nessuna shell (massima sicurezza)
+- Ideale per produzione
+
+#### Esempi
+
+```bash
+# Build standard (default)
+./scripts/build-and-push-image.sh
+
+# Build distroless per produzione
+./scripts/build-and-push-image.sh 192.168.1.204:5000 orchestrator-app latest distroless
+
+# Build distroless con versione specifica
+./scripts/build-and-push-image.sh 192.168.1.204:5000 orchestrator-app v1.2.3 distroless
+
+# Build standard per staging
+./scripts/build-and-push-image.sh 192.168.1.204:5000 orchestrator-app staging standard
+```
+
+#### Cosa fa lo script
+
+1. ✅ Verifica prerequisiti (Docker)
+2. ✅ Controlla che esistano Dockerfile e directory progetto
+3. ✅ Mostra configurazione e chiede conferma
+4. ✅ Builda l'immagine Docker
+5. ✅ Tagga l'immagine per il registry
+6. ✅ Mostra informazioni sull'immagine (dimensione, layer)
+7. ✅ Opzionalmente testa l'immagine localmente
+8. ✅ Pusha l'immagine al registry
+9. ✅ Opzionalmente rimuove l'immagine locale
+
+---
+
+## Aggiornamento Docker Compose
+
+### Script: `update-compose-image.sh`
+
+Questo script aggiorna i file `docker-compose.yml` in `vm1/` e `vm2/` per usare l'immagine dal registry invece di buildarla localmente.
+
+#### Sintassi
+
+```bash
+./scripts/update-compose-image.sh <registry_url> <image_name> <tag>
+```
+
+#### Esempio
+
+```bash
+# Aggiorna docker-compose.yml per usare l'immagine dal registry
+./scripts/update-compose-image.sh registry.example.com:5000 orchestrator-app latest
+```
+
+#### Cosa fa lo script
+
+1. ✅ Crea backup dei file docker-compose.yml (`.backup`)
+2. ✅ Sostituisce `build: .` con `image: registry_url/image_name:tag`
+3. ✅ Aggiorna entrambi i file vm1/ e vm2/
+
+#### Prima e Dopo
+
+**Prima:**
+```yaml
+orchestrator-1-load:
+  build: .
+  container_name: orchestrator-1-load
+  ...
+```
+
+**Dopo:**
+```yaml
+orchestrator-1-load:
+  image: registry.example.com:5000/orchestrator-app:latest
+  container_name: orchestrator-1-load
+  ...
+```
+
+---
+
+## Deploy sulle VM
+
+### 1. Setup Registry su una VM Proxmox (Opzionale)
+
+Se vuoi hostare il registry su una delle VM Proxmox:
+
+```bash
+# Crea una VM per il registry
+./scripts/create-vm.sh 203 registry-server 2 4096 50G
+
+# SSH nella VM
+ssh root@192.168.1.203
+
+# Avvia il registry
+docker run -d \
+  -p 5000:5000 \
+  --name registry \
+  --restart=always \
+  -v /opt/registry-data:/var/lib/registry \
+  registry:2
+```
+
+### 2. Build e Push dell'Immagine
+
+```bash
+# Sul tuo computer locale, dalla directory del progetto
+cd /home/alex/devel/proxmox-ha-setup
+
+# Build e push al registry
+./scripts/build-and-push-image.sh 192.168.1.203:5000 orchestrator-app v1.0.0
+```
+
+### 3. Aggiorna Docker Compose Files
+
+```bash
+# Aggiorna i file docker-compose.yml
+./scripts/update-compose-image.sh 192.168.1.203:5000 orchestrator-app v1.0.0
+```
+
+### 4. Deploy su VM1 e VM2
+
+```bash
+# Copia i file aggiornati sulle VM
+scp -r vm1/* root@192.168.1.201:/opt/myapp/
+scp -r vm2/* root@192.168.1.202:/opt/myapp/
+
+# Su VM1
+ssh root@192.168.1.201
+cd /opt/myapp
+docker pull 192.168.1.203:5000/orchestrator-app:v1.0.0
+docker compose up -d
+
+# Su VM2
+ssh root@192.168.1.202
+cd /opt/myapp
+docker pull 192.168.1.203:5000/orchestrator-app:v1.0.0
+docker compose up -d
+```
+
+### 5. Verifica Deploy
+
+```bash
+# Controlla i container in esecuzione
+ssh root@192.168.1.201 'docker ps'
+ssh root@192.168.1.202 'docker ps'
+
+# Verifica i log
+ssh root@192.168.1.201 'docker logs orchestrator-1-load'
+ssh root@192.168.1.202 'docker logs orchestrator-1-load'
+```
+
+---
+
+## Workflow Completo di Esempio
+
+### Scenario: Deploy Iniziale
+
+```bash
+# 1. Setup registry su VM dedicata
+./scripts/create-vm.sh 203 registry 2 4096 50G
+ssh root@192.168.1.203 'docker run -d -p 5000:5000 --restart=always -v /opt/registry:/var/lib/registry --name registry registry:2'
+
+# 2. Configura Docker per accettare il registry insicuro (su tutte le VM)
+for vm in 201 202; do
+  ssh root@192.168.1.$vm 'echo "{\"insecure-registries\": [\"192.168.1.203:5000\"]}" > /etc/docker/daemon.json && systemctl restart docker'
+done
+
+# 3. Build e push immagine
+./scripts/build-and-push-image.sh 192.168.1.203:5000 orchestrator-app v1.0.0
+
+# 4. Aggiorna docker-compose files
+./scripts/update-compose-image.sh 192.168.1.203:5000 orchestrator-app v1.0.0
+
+# 5. Deploy sulle VM
+for vm in 201 202; do
+  scp -r vm$((vm-200))/* root@192.168.1.$vm:/opt/myapp/
+  ssh root@192.168.1.$vm 'cd /opt/myapp && docker compose pull && docker compose up -d'
+done
+```
+
+### Scenario: Aggiornamento dell'Applicazione
+
+```bash
+# 1. Build nuova versione
+./scripts/build-and-push-image.sh 192.168.1.203:5000 orchestrator-app v1.1.0
+
+# 2. Aggiorna docker-compose (se necessario)
+./scripts/update-compose-image.sh 192.168.1.203:5000 orchestrator-app v1.1.0
+
+# 3. Update sulle VM con zero downtime
+# Prima aggiorna VM2 (backup)
+ssh root@192.168.1.202 'cd /opt/myapp && docker compose pull && docker compose up -d'
+
+# Attendi che sia stabile
+sleep 30
+
+# Poi aggiorna VM1 (master)
+ssh root@192.168.1.201 'cd /opt/myapp && docker compose pull && docker compose up -d'
+```
+
+---
+
+## Comandi Utili
+
+### Gestione Registry
+
+```bash
+# Lista immagini nel registry
+curl http://192.168.1.203:5000/v2/_catalog
+
+# Lista tag di un'immagine
+curl http://192.168.1.203:5000/v2/orchestrator-app/tags/list
+
+# Verifica dimensione del registry
+du -sh /opt/registry-data
+```
+
+### Gestione Immagini
+
+```bash
+# Pull immagine dal registry
+docker pull 192.168.1.203:5000/orchestrator-app:v1.0.0
+
+# Lista immagini locali
+docker images | grep orchestrator-app
+
+# Rimuovi immagini vecchie
+docker image prune -a
+```
+
+### Debug
+
+```bash
+# Testa connessione al registry
+telnet 192.168.1.203 5000
+
+# Verifica log del registry
+docker logs registry
+
+# Test pull senza deploy
+docker pull 192.168.1.203:5000/orchestrator-app:latest
+```
+
+---
+
+## Best Practices
+
+1. **Versioning**: Usa tag semantici (v1.0.0, v1.1.0) invece di `latest` per produzione
+2. **Backup**: Monta un volume per `/var/lib/registry` per persistenza
+3. **Security**: Usa HTTPS e autenticazione per registri in produzione
+4. **Cleanup**: Rimuovi regolarmente immagini vecchie dal registry
+5. **Testing**: Testa sempre l'immagine localmente prima del push
+6. **Documentation**: Documenta le modifiche in ogni versione
+
+---
+
+## Troubleshooting
+
+### Errore: "connection refused" durante il push
+
+**Soluzione:**
+```bash
+# Verifica che il registry sia in esecuzione
+docker ps | grep registry
+
+# Verifica connettività
+curl http://192.168.1.203:5000/v2/
+```
+
+### Errore: "http: server gave HTTP response to HTTPS client"
+
+**Soluzione:**
+Aggiungi il registry come insicuro in `/etc/docker/daemon.json`:
+```json
+{
+  "insecure-registries": ["192.168.1.203:5000"]
+}
+```
+Poi riavvia Docker: `systemctl restart docker`
+
+### Errore: "denied: requested access to the resource is denied"
+
+**Soluzione:**
+Fai login al registry:
+```bash
+docker login 192.168.1.203:5000
+```
+
+### Le immagini non si aggiornano sulle VM
+
+**Soluzione:**
+```bash
+# Forza il pull della nuova immagine
+docker compose pull
+docker compose up -d --force-recreate
+```
+
+---
+
+## Note
+
+- Lo script `build-and-push-image.sh` builderà l'immagine dalla directory `/home/alex/devel/proxmox-ha-setup/vm1`
+- I file docker-compose.yml in `vm1/` e `vm2/` devono usare `build: .` per i servizi orchestrator
+- Puoi usare più tag per la stessa immagine (es: `latest`, `v1.0.0`, `production`)

scripts/build-and-push-image.sh

@@ -0,0 +1,275 @@
+#!/bin/bash
+# build-and-push-image.sh
+# Script per buildare l'immagine Docker degli orchestrator e pusharla su un registry privato
+# Uso: ./build-and-push-image.sh [registry_url] [image_name] [tag] [dockerfile_type]
+# Esempio: ./build-and-push-image.sh registry.example.com:5000 orchestrator-app latest distroless
+
+set -e
+
+# ==================== CONFIGURAZIONE ====================
+
+# Directory del progetto (dove si trova il Dockerfile)
+PROJECT_DIR="/home/alex/devel/proxmox-ha-setup/vm1"
+
+# Parametri da riga di comando con valori di default
+REGISTRY_URL=${1:-"192.168.1.204:5000"}
+IMAGE_NAME=${2:-"orchestrator-app"}
+TAG=${3:-"latest"}
+DOCKERFILE_TYPE=${4:-"standard"}
+
+# Determina quale Dockerfile usare
+if [[ "$DOCKERFILE_TYPE" == "distroless" ]]; then
+    DOCKERFILE="Dockerfile.distroless"
+    BUILD_TYPE="Distroless (Multi-stage)"
+else
+    DOCKERFILE="Dockerfile"
+    BUILD_TYPE="Standard (python:3.12-slim)"
+fi
+
+# Nome completo dell'immagine
+FULL_IMAGE_NAME="${REGISTRY_URL}/${IMAGE_NAME}:${TAG}"
+LOCAL_IMAGE_NAME="${IMAGE_NAME}:${TAG}"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# ==================== FUNZIONI ====================
+
+print_header() {
+    echo -e "${BLUE}================================================${NC}"
+    echo -e "${BLUE}$1${NC}"
+    echo -e "${BLUE}================================================${NC}"
+}
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${CYAN}ℹ $1${NC}"; }
+
+check_command() {
+    if ! command -v $1 &> /dev/null; then
+        print_error "$1 non trovato. Installalo prima di continuare."
+        exit 1
+    fi
+}
+
+show_usage() {
+    echo "Uso: $0 [registry_url] [image_name] [tag] [dockerfile_type]"
+    echo ""
+    echo "Parametri opzionali (valori di default):"
+    echo "  registry_url    - URL del registry privato (default: 192.168.1.204:5000)"
+    echo "  image_name      - Nome dell'immagine (default: orchestrator-app)"
+    echo "  tag             - Tag dell'immagine (default: latest)"
+    echo "  dockerfile_type - Tipo di Dockerfile: 'standard' o 'distroless' (default: standard)"
+    echo ""
+    echo "Dockerfile Types:"
+    echo "  standard   - Usa Dockerfile (python:3.12-slim, ~333MB)"
+    echo "  distroless - Usa Dockerfile.distroless (gcr.io/distroless, ~180MB, più sicuro)"
+    echo ""
+    echo "Esempi:"
+    echo "  $0                                                       # Build standard locale"
+    echo "  $0 registry.example.com:5000                             # Registry custom, standard"
+    echo "  $0 registry.example.com:5000 my-app latest distroless    # Build distroless"
+    echo "  $0 192.168.1.204:5000 orchestrator-app v1.0.0 distroless # Produzione distroless"
+    echo ""
+}
+
+# ==================== VALIDAZIONE ====================
+
+# Mostra help se richiesto
+if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then
+    show_usage
+    exit 0
+fi
+
+# ==================== SCRIPT PRINCIPALE ====================
+
+print_header "BUILD E PUSH IMMAGINE DOCKER"
+
+# Check prerequisites
+print_info "Verifica prerequisiti..."
+check_command "docker"
+print_success "Docker trovato"
+
+# Verifica che la directory del progetto esista
+if [ ! -d "$PROJECT_DIR" ]; then
+    print_error "Directory del progetto non trovata: $PROJECT_DIR"
+    exit 1
+fi
+
+# Verifica che il Dockerfile richiesto esista
+if [ ! -f "$PROJECT_DIR/$DOCKERFILE" ]; then
+    print_error "Dockerfile non trovato: $PROJECT_DIR/$DOCKERFILE"
+    if [[ "$DOCKERFILE_TYPE" == "distroless" ]]; then
+        print_warning "Il Dockerfile.distroless non esiste. Usa 'standard' o crea il file."
+    fi
+    exit 1
+fi
+
+print_info ""
+print_info "Configurazione build:"
+print_info "  Directory progetto: $PROJECT_DIR"
+print_info "  Dockerfile: ${CYAN}$DOCKERFILE${NC}"
+print_info "  Build type: ${CYAN}$BUILD_TYPE${NC}"
+print_info "  Registry: $REGISTRY_URL"
+print_info "  Nome immagine: $IMAGE_NAME"
+print_info "  Tag: $TAG"
+print_info "  Nome completo: ${CYAN}$FULL_IMAGE_NAME${NC}"
+print_info ""
+
+# Conferma dall'utente
+read -p "Procedere con la build? (Y/n) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Nn]$ ]]; then
+    print_warning "Build annullata dall'utente"
+    exit 0
+fi
+
+# ==================== BUILD ====================
+
+print_header "STEP 1: Build dell'immagine Docker"
+
+print_info "Inizio build dell'immagine..."
+print_info "Comando: docker build -f $DOCKERFILE -t $LOCAL_IMAGE_NAME $PROJECT_DIR"
+echo ""
+
+# Build dell'immagine con Dockerfile specificato
+if docker build -f "$PROJECT_DIR/$DOCKERFILE" -t "$LOCAL_IMAGE_NAME" "$PROJECT_DIR"; then
+    print_success "Build completata con successo"
+else
+    print_error "Build fallita"
+    exit 1
+fi
+
+# Tag dell'immagine per il registry
+print_info "Tagging dell'immagine per il registry..."
+docker tag "$LOCAL_IMAGE_NAME" "$FULL_IMAGE_NAME"
+print_success "Immagine taggata: $FULL_IMAGE_NAME"
+
+# ==================== INFORMAZIONI IMMAGINE ====================
+
+print_header "STEP 2: Informazioni immagine"
+
+# Mostra le informazioni sull'immagine
+IMAGE_SIZE=$(docker images "$LOCAL_IMAGE_NAME" --format "{{.Size}}")
+IMAGE_ID=$(docker images "$LOCAL_IMAGE_NAME" --format "{{.ID}}")
+
+print_info "ID Immagine: $IMAGE_ID"
+print_info "Dimensione: $IMAGE_SIZE"
+print_info ""
+
+# Mostra i layer dell'immagine (opzionale)
+print_info "Layer dell'immagine:"
+docker history "$LOCAL_IMAGE_NAME" --human --format "table {{.Size}}\t{{.CreatedBy}}" | head -10
+
+# ==================== TEST OPZIONALE ====================
+
+print_header "STEP 3: Test immagine (opzionale)"
+
+read -p "Vuoi testare l'immagine localmente prima del push? (y/N) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+    print_info "Avvio container di test..."
+    print_info "Premere Ctrl+C per interrompere il test"
+    echo ""
+
+    # Test con un comando semplice
+    if docker run --rm "$LOCAL_IMAGE_NAME" python --version; then
+        print_success "Test completato con successo"
+    else
+        print_warning "Test fallito - ma puoi comunque procedere con il push"
+    fi
+    echo ""
+fi
+
+# ==================== PUSH AL REGISTRY ====================
+
+print_header "STEP 4: Push al registry"
+
+print_info "Verifica connessione al registry..."
+
+# Verifica se il registry è raggiungibile (solo per http/https)
+if [[ "$REGISTRY_URL" =~ ^(localhost|127\.0\.0\.1) ]]; then
+    print_warning "Registry locale detectato - assicurati che sia in esecuzione"
+else
+    print_info "Registry remoto: $REGISTRY_URL"
+fi
+
+read -p "Procedere con il push dell'immagine? (Y/n) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Nn]$ ]]; then
+    print_warning "Push annullato dall'utente"
+    print_info "L'immagine locale è disponibile come: $LOCAL_IMAGE_NAME"
+    exit 0
+fi
+
+print_info "Push dell'immagine al registry..."
+echo ""
+
+# Push dell'immagine
+if docker push "$FULL_IMAGE_NAME"; then
+    print_success "Push completato con successo!"
+else
+    print_error "Push fallito"
+    print_warning "Verifica che:"
+    print_warning "  1. Il registry sia in esecuzione e raggiungibile"
+    print_warning "  2. Hai eseguito 'docker login $REGISTRY_URL'"
+    print_warning "  3. Il registry accetti connessioni insicure se necessario"
+    exit 1
+fi
+
+# ==================== PULIZIA OPZIONALE ====================
+
+print_header "STEP 5: Pulizia (opzionale)"
+
+read -p "Vuoi rimuovere l'immagine locale dopo il push? (y/N) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+    print_info "Rimozione immagine locale..."
+    docker rmi "$LOCAL_IMAGE_NAME" "$FULL_IMAGE_NAME" 2>/dev/null || true
+    print_success "Immagine locale rimossa"
+else
+    print_info "Immagine locale mantenuta: $LOCAL_IMAGE_NAME"
+fi
+
+# ==================== RIEPILOGO ====================
+
+print_header "BUILD E PUSH COMPLETATI! 🎉"
+
+print_info ""
+print_info "Riepilogo:"
+print_info "  Tipo build: ${CYAN}$BUILD_TYPE${NC}"
+print_info "  Dockerfile: ${CYAN}$DOCKERFILE${NC}"
+print_info "  Immagine: ${GREEN}$FULL_IMAGE_NAME${NC}"
+print_info "  Dimensione: $IMAGE_SIZE"
+print_info "  Registry: $REGISTRY_URL"
+print_info ""
+
+# Messaggio specifico per il tipo di build
+if [[ "$DOCKERFILE_TYPE" == "distroless" ]]; then
+    print_info "Note Distroless:"
+    print_info "  ✓ Immagine più piccola e sicura"
+    print_info "  ✓ Solo bytecode Python (.pyc)"
+    print_info "  ✓ Nessuna shell (debug solo tramite log)"
+    print_info "  ✓ Ideale per produzione"
+else
+    print_info "Note Standard:"
+    print_info "  ✓ Include shell per debugging"
+    print_info "  ✓ Bytecode Python (.pyc)"
+    print_info "  ✓ Ideale per sviluppo e staging"
+fi
+
+print_info ""
+print_info "Per utilizzare l'immagine nei docker-compose.yml:"
+print_info "  Sostituisci 'build: .' con:"
+print_info "  ${CYAN}image: $FULL_IMAGE_NAME${NC}"
+print_info ""
+print_info "Per pullare l'immagine su un'altra macchina:"
+print_info "  ${CYAN}docker pull $FULL_IMAGE_NAME${NC}"
+print_info ""
+print_success "Operazione completata con successo!"

scripts/generate-registry-cert.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+# generate-registry-cert.sh
+# Script per generare certificati self-signed corretti per Docker Registry
+# con Subject Alternative Names (SANs)
+
+set -e
+
+# Parametri
+REGISTRY_IP=${1:-"192.168.1.204"}
+REGISTRY_HOSTNAME=${2:-"registry.local"}
+CERT_DIR=${3:-"/opt/docker-registry/certs"}
+CERT_DAYS=${4:-365}
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${BLUE}ℹ $1${NC}"; }
+
+if [ "$#" -eq 0 ]; then
+    echo "Uso: $0 [registry_ip] [registry_hostname] [cert_dir] [days]"
+    echo ""
+    echo "Parametri opzionali:"
+    echo "  registry_ip       - IP del registry (default: 192.168.1.204)"
+    echo "  registry_hostname - Hostname del registry (default: registry.local)"
+    echo "  cert_dir          - Directory certificati (default: /opt/docker-registry/certs)"
+    echo "  days              - Validità in giorni (default: 365)"
+    echo ""
+    echo "Esempio:"
+    echo "  $0 192.168.1.204 registry.local"
+    exit 0
+fi
+
+print_info "Generazione certificato self-signed per Docker Registry"
+echo ""
+print_info "Configurazione:"
+print_info "  IP: $REGISTRY_IP"
+print_info "  Hostname: $REGISTRY_HOSTNAME"
+print_info "  Directory: $CERT_DIR"
+print_info "  Validità: $CERT_DAYS giorni"
+echo ""
+
+# Crea directory se non esiste
+mkdir -p "$CERT_DIR"
+
+# File di output
+KEY_FILE="$CERT_DIR/domain.key"
+CERT_FILE="$CERT_DIR/domain.crt"
+CSR_FILE="$CERT_DIR/domain.csr"
+CNF_FILE="$CERT_DIR/openssl.cnf"
+
+# Crea file di configurazione OpenSSL con SANs
+cat > "$CNF_FILE" << EOF
+[req]
+default_bits       = 4096
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+x509_extensions    = v3_ca
+prompt             = no
+
+[req_distinguished_name]
+C  = IT
+ST = Italy
+L  = City
+O  = Docker Registry
+OU = IT Department
+CN = $REGISTRY_HOSTNAME
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_ca]
+subjectAltName = @alt_names
+basicConstraints = critical, CA:TRUE
+keyUsage = critical, digitalSignature, keyEncipherment, keyCertSign
+extendedKeyUsage = serverAuth
+
+[alt_names]
+DNS.1 = $REGISTRY_HOSTNAME
+DNS.2 = localhost
+IP.1  = $REGISTRY_IP
+IP.2  = 127.0.0.1
+EOF
+
+print_info "File di configurazione creato: $CNF_FILE"
+
+# Genera chiave privata e certificato
+print_info "Generazione chiave privata e certificato..."
+
+openssl req -newkey rsa:4096 \
+    -nodes \
+    -sha256 \
+    -keyout "$KEY_FILE" \
+    -x509 \
+    -days "$CERT_DAYS" \
+    -config "$CNF_FILE" \
+    -out "$CERT_FILE"
+
+print_success "Certificato generato con successo!"
+echo ""
+
+# Verifica certificato
+print_info "Verifica certificato:"
+openssl x509 -in "$CERT_FILE" -text -noout | grep -A 3 "Subject Alternative Name"
+
+echo ""
+print_info "File generati:"
+print_info "  Chiave privata: $KEY_FILE"
+print_info "  Certificato:    $CERT_FILE"
+print_info "  Config OpenSSL: $CNF_FILE"
+echo ""
+
+print_info "Prossimi passi:"
+print_info ""
+print_info "1. Riavvia il registry Docker con i nuovi certificati:"
+echo "   docker stop registry && docker rm registry"
+echo "   docker run -d \\"
+echo "     -p 5000:5000 \\"
+echo "     --name registry \\"
+echo "     --restart=always \\"
+echo "     -v $CERT_DIR:/certs \\"
+echo "     -v /opt/docker-registry/data:/var/lib/registry \\"
+echo "     -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \\"
+echo "     -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \\"
+echo "     registry:2"
+echo ""
+
+print_info "2. Installa il certificato sui client Docker:"
+echo "   # Su ogni VM che deve accedere al registry:"
+echo "   sudo mkdir -p /etc/docker/certs.d/$REGISTRY_IP:5000"
+echo "   sudo scp root@$REGISTRY_IP:$CERT_FILE /etc/docker/certs.d/$REGISTRY_IP:5000/ca.crt"
+echo "   sudo systemctl restart docker"
+echo ""
+
+print_info "3. (Opzionale) Installa nel sistema per curl/wget:"
+echo "   sudo cp $CERT_FILE /usr/local/share/ca-certificates/registry.crt"
+echo "   sudo update-ca-certificates"
+echo ""
+
+print_success "Setup completato!"

scripts/install-registry-cert.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+# install-registry-cert.sh
+# Script per installare il certificato del registry Docker
+# Supporta sia Docker standard che Docker Snap
+
+set -e
+
+CERT_FILE="/tmp/registry-new.crt"
+REGISTRY_URL="192.168.1.204:5000"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${BLUE}ℹ $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠ $1${NC}"; }
+
+echo -e "${BLUE}================================================${NC}"
+echo -e "${BLUE}INSTALLAZIONE CERTIFICATO REGISTRY${NC}"
+echo -e "${BLUE}================================================${NC}"
+echo ""
+
+# Verifica che il certificato esista
+if [ ! -f "$CERT_FILE" ]; then
+    print_error "Certificato non trovato: $CERT_FILE"
+    print_info "Scaricalo prima con: scp root@192.168.1.204:/opt/docker-registry/certs/domain.crt $CERT_FILE"
+    exit 1
+fi
+
+print_success "Certificato trovato: $CERT_FILE"
+echo ""
+
+# Verifica SANs
+print_info "Verifica Subject Alternative Names nel certificato:"
+if openssl x509 -in "$CERT_FILE" -text -noout | grep -q "Subject Alternative Name"; then
+    openssl x509 -in "$CERT_FILE" -text -noout | grep -A 2 "Subject Alternative Name"
+    print_success "Certificato ha SANs corretti"
+else
+    print_error "Certificato non ha Subject Alternative Names!"
+    print_warning "Il certificato deve essere rigenerato sul registry"
+    exit 1
+fi
+echo ""
+
+# Rileva tipo di installazione Docker
+DOCKER_TYPE="standard"
+if snap list docker &>/dev/null; then
+    DOCKER_TYPE="snap"
+    print_info "Rilevato: Docker Snap"
+elif command -v docker &>/dev/null; then
+    print_info "Rilevato: Docker Standard"
+else
+    print_error "Docker non trovato"
+    exit 1
+fi
+echo ""
+
+# Installazione certificato nel sistema (funziona per entrambi)
+print_info "Installazione certificato nel sistema..."
+sudo cp "$CERT_FILE" /usr/local/share/ca-certificates/registry-192.168.1.204.crt
+sudo chmod 644 /usr/local/share/ca-certificates/registry-192.168.1.204.crt
+sudo update-ca-certificates
+
+print_success "Certificato installato in /usr/local/share/ca-certificates/"
+echo ""
+
+# Per Docker standard, installa anche in /etc/docker/certs.d
+if [ "$DOCKER_TYPE" = "standard" ]; then
+    print_info "Installazione certificato per Docker daemon..."
+    sudo mkdir -p /etc/docker/certs.d/$REGISTRY_URL
+    sudo cp "$CERT_FILE" /etc/docker/certs.d/$REGISTRY_URL/ca.crt
+    sudo chmod 644 /etc/docker/certs.d/$REGISTRY_URL/ca.crt
+    print_success "Certificato installato in /etc/docker/certs.d/$REGISTRY_URL/"
+    echo ""
+fi
+
+# Docker Snap usa i certificati di sistema, quindi non serve altro
+
+# Verifica installazione
+print_info "Verifica installazione..."
+if ls /etc/ssl/certs/ | grep -q registry; then
+    CERT_LINK=$(ls -la /etc/ssl/certs/ | grep registry | awk '{print $9}')
+    print_success "Certificato symlink trovato: $CERT_LINK"
+else
+    print_warning "Symlink non trovato (potrebbe essere normale)"
+fi
+echo ""
+
+# Test connessione
+echo -e "${BLUE}================================================${NC}"
+echo -e "${BLUE}TEST CONNESSIONE${NC}"
+echo -e "${BLUE}================================================${NC}"
+echo ""
+
+print_info "Test connessione HTTPS al registry..."
+if curl -s https://$REGISTRY_URL/v2/ | grep -q '{}'; then
+    print_success "Connessione HTTPS: OK"
+else
+    print_warning "Test connessione fallito (verifica che il registry sia in esecuzione)"
+fi
+echo ""
+
+print_info "Test API registry..."
+if curl -s https://$REGISTRY_URL/v2/_catalog >/dev/null 2>&1; then
+    print_success "API registry: OK"
+    echo ""
+    print_info "Immagini nel registry:"
+    curl -s https://$REGISTRY_URL/v2/_catalog | python3 -m json.tool 2>/dev/null || curl -s https://$REGISTRY_URL/v2/_catalog
+else
+    print_warning "API registry non risponde"
+fi
+echo ""
+
+# Istruzioni finali
+echo -e "${BLUE}================================================${NC}"
+echo -e "${GREEN}INSTALLAZIONE COMPLETATA!${NC}"
+echo -e "${BLUE}================================================${NC}"
+echo ""
+
+print_info "Comandi per testare:"
+echo ""
+echo "  # Test con curl"
+echo "  curl https://$REGISTRY_URL/v2/_catalog"
+echo ""
+echo "  # Push immagine"
+echo "  docker push $REGISTRY_URL/orchestrator-app:latest"
+echo ""
+echo "  # Pull immagine"
+echo "  docker pull $REGISTRY_URL/orchestrator-app:latest"
+echo ""
+
+if [ "$DOCKER_TYPE" = "snap" ]; then
+    print_info "Note per Docker Snap:"
+    print_info "  - Docker Snap usa i certificati di sistema (/etc/ssl/certs/)"
+    print_info "  - Non serve riavviare il daemon Docker"
+    print_info "  - Se il push fallisce ancora, riavvia Snap: sudo snap restart docker"
+fi
+echo ""

scripts/setup-registry-client.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+# setup-registry-client.sh
+# Script per configurare un client Docker per accedere a un registry con certificato self-signed
+
+set -e
+
+REGISTRY_IP=${1:-"192.168.1.204"}
+REGISTRY_PORT=${2:-5000}
+CERT_SOURCE=${3}
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${BLUE}ℹ $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠ $1${NC}"; }
+
+if [ "$#" -lt 1 ]; then
+    echo "Uso: $0 <registry_ip> [port] [cert_file]"
+    echo ""
+    echo "Parametri:"
+    echo "  registry_ip  - IP del registry (es: 192.168.1.204)"
+    echo "  port         - Porta del registry (default: 5000)"
+    echo "  cert_file    - File certificato (opzionale, verrà scaricato se non specificato)"
+    echo ""
+    echo "Esempi:"
+    echo "  $0 192.168.1.204"
+    echo "  $0 192.168.1.204 5000"
+    echo "  $0 192.168.1.204 5000 /path/to/domain.crt"
+    exit 1
+fi
+
+REGISTRY_URL="$REGISTRY_IP:$REGISTRY_PORT"
+DOCKER_CERTS_DIR="/etc/docker/certs.d/$REGISTRY_URL"
+
+print_info "Configurazione client Docker per registry: $REGISTRY_URL"
+echo ""
+
+# Verifica se Docker è installato
+if ! command -v docker &> /dev/null; then
+    print_error "Docker non trovato. Installalo prima di continuare."
+    exit 1
+fi
+
+# Crea directory per i certificati Docker
+print_info "Creazione directory certificati Docker..."
+sudo mkdir -p "$DOCKER_CERTS_DIR"
+print_success "Directory creata: $DOCKER_CERTS_DIR"
+
+# Scarica o copia il certificato
+if [ -z "$CERT_SOURCE" ]; then
+    print_info "Scaricamento certificato dal registry..."
+
+    # Tenta di scaricare il certificato usando openssl
+    if timeout 5 openssl s_client -showcerts -connect "$REGISTRY_URL" </dev/null 2>/dev/null | \
+       openssl x509 -outform PEM > /tmp/registry-cert.pem 2>/dev/null; then
+        sudo mv /tmp/registry-cert.pem "$DOCKER_CERTS_DIR/ca.crt"
+        print_success "Certificato scaricato con successo"
+    else
+        print_error "Impossibile scaricare il certificato automaticamente"
+        print_warning "Opzioni:"
+        print_info "  1. Copia manualmente il certificato dal registry:"
+        print_info "     scp root@$REGISTRY_IP:/opt/docker-registry/certs/domain.crt /tmp/registry.crt"
+        print_info "     sudo cp /tmp/registry.crt $DOCKER_CERTS_DIR/ca.crt"
+        print_info ""
+        print_info "  2. Riavvia questo script specificando il file certificato:"
+        print_info "     $0 $REGISTRY_IP $REGISTRY_PORT /path/to/cert.crt"
+        exit 1
+    fi
+else
+    if [ ! -f "$CERT_SOURCE" ]; then
+        print_error "File certificato non trovato: $CERT_SOURCE"
+        exit 1
+    fi
+
+    print_info "Copia certificato da: $CERT_SOURCE"
+    sudo cp "$CERT_SOURCE" "$DOCKER_CERTS_DIR/ca.crt"
+    print_success "Certificato copiato"
+fi
+
+# Verifica permessi
+sudo chmod 644 "$DOCKER_CERTS_DIR/ca.crt"
+
+# Verifica certificato
+print_info "Verifica certificato installato..."
+if sudo openssl x509 -in "$DOCKER_CERTS_DIR/ca.crt" -text -noout > /dev/null 2>&1; then
+    print_success "Certificato valido"
+
+    print_info "Dettagli certificato:"
+    sudo openssl x509 -in "$DOCKER_CERTS_DIR/ca.crt" -noout -subject -issuer -dates
+
+    # Mostra SANs se presenti
+    if sudo openssl x509 -in "$DOCKER_CERTS_DIR/ca.crt" -text -noout | grep -q "Subject Alternative Name"; then
+        print_info "Subject Alternative Names:"
+        sudo openssl x509 -in "$DOCKER_CERTS_DIR/ca.crt" -text -noout | grep -A 3 "Subject Alternative Name"
+    else
+        print_warning "Certificato non ha Subject Alternative Names (SANs)"
+        print_warning "Considera di rigenerare il certificato con: ./scripts/generate-registry-cert.sh"
+    fi
+else
+    print_error "Certificato non valido"
+    exit 1
+fi
+
+# Riavvia Docker
+print_info "Riavvio Docker daemon..."
+sudo systemctl restart docker
+sleep 3
+
+if sudo systemctl is-active --quiet docker; then
+    print_success "Docker riavviato con successo"
+else
+    print_error "Errore nel riavvio di Docker"
+    exit 1
+fi
+
+# Test connessione al registry
+echo ""
+print_info "Test connessione al registry..."
+
+# Test HTTPS
+if timeout 5 openssl s_client -connect "$REGISTRY_URL" -CAfile "$DOCKER_CERTS_DIR/ca.crt" </dev/null 2>&1 | grep -q "Verify return code: 0"; then
+    print_success "Connessione HTTPS con verifica certificato: OK"
+elif timeout 5 openssl s_client -connect "$REGISTRY_URL" </dev/null 2>&1 | grep -q "CONNECTED"; then
+    print_warning "Connessione HTTPS: OK ma verifica certificato fallita"
+    print_info "Il certificato potrebbe essere self-signed o avere SANs non corretti"
+else
+    print_error "Impossibile connettersi al registry"
+fi
+
+# Test Docker login
+echo ""
+print_info "Per testare Docker con il registry, esegui:"
+echo "  docker pull $REGISTRY_URL/orchestrator-app:latest"
+echo ""
+
+print_info "Se il registry richiede autenticazione:"
+echo "  docker login $REGISTRY_URL"
+echo ""
+
+# Opzionale: installa anche nel sistema
+read -p "Vuoi installare il certificato anche nel sistema per curl/wget? (y/N) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+    print_info "Installazione certificato nel sistema..."
+    sudo cp "$DOCKER_CERTS_DIR/ca.crt" /usr/local/share/ca-certificates/registry-$REGISTRY_IP.crt
+    sudo update-ca-certificates
+    print_success "Certificato installato nel sistema"
+
+    # Test con curl
+    if curl -I "https://$REGISTRY_URL/v2/" 2>&1 | grep -q "200 OK"; then
+        print_success "Test curl: OK"
+    else
+        print_warning "Test curl fallito (normale se registry richiede autenticazione)"
+    fi
+fi
+
+echo ""
+print_success "Setup client completato!"
+print_info "Il Docker daemon ora può accedere a: $REGISTRY_URL"

scripts/sync_modifche_vm.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-rsync -avz --exclude='*.pyc' --exclude '__pycache__/' /home/alex/devel/ASE/src /home/alex/devel/proxmox-ha-setup/vm1/
+#rsync -avz --exclude='*.pyc' --exclude '__pycache__/' /home/alex/devel/ASE/src /home/alex/devel/proxmox-ha-setup/vm1/
-rsync -avz --exclude='*.pyc' --exclude '__pycache__/' /home/alex/devel/ASE/src /home/alex/devel/proxmox-ha-setup/vm2/
+#rsync -avz --exclude='*.pyc' --exclude '__pycache__/' /home/alex/devel/ASE/src /home/alex/devel/proxmox-ha-setup/vm2/
-rsync -avz -e "ssh -p 2222" --exclude='*.pyc' /home/alex/devel/proxmox-ha-setup/vm1/ root@192.168.1.201:/opt/myapp/
+rsync -avz -e "ssh -p 2222" /home/alex/devel/proxmox-ha-setup/vm1/ root@192.168.1.201:/opt/ase/
-rsync -avz -e "ssh -p 2222" --exclude='*.pyc' /home/alex/devel/proxmox-ha-setup/vm2/ root@192.168.1.202:/opt/myapp/
+rsync -avz -e "ssh -p 2222" /home/alex/devel/proxmox-ha-setup/vm2/ root@192.168.1.202:/opt/ase/

scripts/test-pyc-image.sh

@@ -0,0 +1,190 @@
+#!/bin/bash
+# test-pyc-image.sh
+# Script per testare che l'immagine Docker funzioni correttamente con solo file .pyc
+# Uso: ./test-pyc-image.sh [image_name]
+# Esempio: ./test-pyc-image.sh orchestrator-app:latest
+
+set -e
+
+# ==================== CONFIGURAZIONE ====================
+
+IMAGE_NAME=${1:-"orchestrator-app:latest"}
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# ==================== FUNZIONI ====================
+
+print_header() {
+    echo -e "${BLUE}================================================${NC}"
+    echo -e "${BLUE}$1${NC}"
+    echo -e "${BLUE}================================================${NC}"
+}
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${CYAN}ℹ $1${NC}"; }
+
+# ==================== SCRIPT PRINCIPALE ====================
+
+print_header "TEST IMMAGINE DOCKER CON BYTECODE"
+
+print_info "Immagine da testare: ${CYAN}$IMAGE_NAME${NC}"
+echo ""
+
+# Verifica che l'immagine esista
+if ! docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then
+    print_error "Immagine non trovata: $IMAGE_NAME"
+    print_info "Esegui prima: ./scripts/build-and-push-image.sh"
+    exit 1
+fi
+
+print_success "Immagine trovata"
+
+# ==================== TEST 1: Verifica assenza file .py ====================
+
+print_header "TEST 1: Verifica rimozione file sorgente .py"
+
+print_info "Controllo file .py in /app/src..."
+PY_FILES=$(docker run --rm "$IMAGE_NAME" find /app/src -type f -name "*.py" 2>/dev/null | wc -l)
+
+if [ "$PY_FILES" -eq 0 ]; then
+    print_success "Nessun file .py trovato in /app/src - OK!"
+else
+    print_warning "Trovati $PY_FILES file .py in /app/src"
+    docker run --rm "$IMAGE_NAME" find /app/src -type f -name "*.py"
+fi
+
+# ==================== TEST 2: Verifica presenza file .pyc ====================
+
+print_header "TEST 2: Verifica presenza file bytecode .pyc"
+
+print_info "Controllo file .pyc in /app/src..."
+PYC_FILES=$(docker run --rm "$IMAGE_NAME" find /app/src -type f -name "*.pyc" 2>/dev/null | wc -l)
+
+if [ "$PYC_FILES" -gt 0 ]; then
+    print_success "Trovati $PYC_FILES file .pyc - OK!"
+else
+    print_error "Nessun file .pyc trovato - qualcosa è andato storto"
+    exit 1
+fi
+
+# ==================== TEST 3: Verifica importazione moduli ====================
+
+print_header "TEST 3: Test importazione moduli Python"
+
+print_info "Test import di src..."
+if docker run --rm "$IMAGE_NAME" python -c "import src; print('Import src: OK')" 2>/dev/null; then
+    print_success "Import src funziona correttamente"
+else
+    print_error "Import src fallito"
+    exit 1
+fi
+
+# ==================== TEST 4: Lista moduli disponibili ====================
+
+print_header "TEST 4: Verifica moduli disponibili"
+
+print_info "Moduli in src:"
+docker run --rm "$IMAGE_NAME" python -c "
+import pkgutil
+import src
+for importer, modname, ispkg in pkgutil.walk_packages(path=src.__path__, prefix='src.'):
+    print(f'  - {modname} (package)' if ispkg else f'  - {modname}')
+" 2>/dev/null || print_warning "Impossibile elencare i moduli"
+
+# ==================== TEST 5: Verifica dimensione immagine ====================
+
+print_header "TEST 5: Analisi dimensione immagine"
+
+IMAGE_SIZE=$(docker images "$IMAGE_NAME" --format "{{.Size}}")
+print_info "Dimensione immagine: ${CYAN}$IMAGE_SIZE${NC}"
+
+# Conta i file per tipo
+print_info "Statistiche file in /app/src:"
+docker run --rm "$IMAGE_NAME" sh -c '
+echo "  File .py:  $(find /app/src -type f -name "*.py" 2>/dev/null | wc -l)"
+echo "  File .pyc: $(find /app/src -type f -name "*.pyc" 2>/dev/null | wc -l)"
+echo "  Totale:    $(find /app/src -type f 2>/dev/null | wc -l)"
+'
+
+# ==================== TEST 6: Test esecuzione moduli ====================
+
+print_header "TEST 6: Test esecuzione moduli orchestrator (dry-run)"
+
+print_info "Test esecuzione load_orchestrator..."
+if docker run --rm "$IMAGE_NAME" timeout 2 python -c "
+try:
+    import src.load_orchestrator
+    print('Module src.load_orchestrator: OK')
+except ImportError as e:
+    print(f'Import error: {e}')
+    exit(1)
+" 2>/dev/null; then
+    print_success "Modulo load_orchestrator importabile"
+else
+    print_warning "Test load_orchestrator timeout o errore (normale se richiede DB)"
+fi
+
+print_info "Test esecuzione elab_orchestrator..."
+if docker run --rm "$IMAGE_NAME" timeout 2 python -c "
+try:
+    import src.elab_orchestrator
+    print('Module src.elab_orchestrator: OK')
+except ImportError as e:
+    print(f'Import error: {e}')
+    exit(1)
+" 2>/dev/null; then
+    print_success "Modulo elab_orchestrator importabile"
+else
+    print_warning "Test elab_orchestrator timeout o errore (normale se richiede DB)"
+fi
+
+# ==================== TEST 7: Verifica variabili ambiente ====================
+
+print_header "TEST 7: Verifica configurazione ambiente"
+
+docker run --rm "$IMAGE_NAME" python -c "
+import os
+import sys
+
+print('Python version:', sys.version)
+print('PYTHONUNBUFFERED:', os.getenv('PYTHONUNBUFFERED', 'not set'))
+print('PYTHONDONTWRITEBYTECODE:', os.getenv('PYTHONDONTWRITEBYTECODE', 'not set'))
+print('PYTHONPATH:', os.getenv('PYTHONPATH', 'not set'))
+print('Python path:', sys.path)
+"
+
+# ==================== RIEPILOGO ====================
+
+print_header "TEST COMPLETATI! ✅"
+
+echo ""
+print_success "L'immagine contiene solo bytecode Python (.pyc)"
+print_success "I file sorgente .py sono stati rimossi correttamente"
+print_success "I moduli sono importabili e funzionanti"
+echo ""
+
+print_info "Vantaggi:"
+print_info "  ✓ Protezione del codice sorgente"
+print_info "  ✓ Riduzione dimensione immagine"
+print_info "  ✓ Leggero miglioramento delle performance"
+print_info "  ✓ Rimozione docstring e assert (con -OO)"
+echo ""
+
+print_warning "Note:"
+print_warning "  - I traceback mostreranno solo numeri di riga, non il codice"
+print_warning "  - Il debugging sarà più difficile senza codice sorgente"
+print_warning "  - Mantieni i sorgenti in un repository separato per sviluppo"
+echo ""
+
+print_info "Prossimi passi:"
+print_info "  1. Push dell'immagine al registry: ./scripts/build-and-push-image.sh"
+print_info "  2. Deploy sulle VM: docker compose pull && docker compose up -d"
+echo ""

scripts/update-compose-image.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+# update-compose-image.sh
+# Script per aggiornare i file docker-compose.yml per usare l'immagine dal registry
+# Uso: ./update-compose-image.sh <registry_url> <image_name> <tag>
+# Esempio: ./update-compose-image.sh registry.example.com:5000 orchestrator-app latest
+
+set -e
+
+# ==================== CONFIGURAZIONE ====================
+
+if [ "$#" -ne 3 ]; then
+    echo "Uso: $0 <registry_url> <image_name> <tag>"
+    echo ""
+    echo "Parametri:"
+    echo "  registry_url  - URL del registry privato"
+    echo "  image_name    - Nome dell'immagine"
+    echo "  tag           - Tag dell'immagine"
+    echo ""
+    echo "Esempio:"
+    echo "  $0 registry.example.com:5000 orchestrator-app latest"
+    echo ""
+    echo "Lo script aggiornerà i file docker-compose.yml in vm1/ e vm2/"
+    echo "sostituendo 'build: .' con 'image: registry_url/image_name:tag'"
+    exit 1
+fi
+
+REGISTRY_URL=$1
+IMAGE_NAME=$2
+TAG=$3
+FULL_IMAGE_NAME="${REGISTRY_URL}/${IMAGE_NAME}:${TAG}"
+
+# Directory dei progetti
+VM1_DIR="/home/alex/devel/proxmox-ha-setup/vm1"
+VM2_DIR="/home/alex/devel/proxmox-ha-setup/vm2"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# ==================== FUNZIONI ====================
+
+print_success() { echo -e "${GREEN}✓ $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠ $1${NC}"; }
+print_error() { echo -e "${RED}✗ $1${NC}"; }
+print_info() { echo -e "${CYAN}ℹ $1${NC}"; }
+
+update_compose_file() {
+    local compose_file=$1
+    local backup_file="${compose_file}.backup"
+
+    if [ ! -f "$compose_file" ]; then
+        print_warning "File non trovato: $compose_file"
+        return 1
+    fi
+
+    # Crea backup
+    cp "$compose_file" "$backup_file"
+    print_info "Backup creato: $backup_file"
+
+    # Sostituisci 'build: .' con 'image: ...'
+    sed -i "s|build: \.|image: ${FULL_IMAGE_NAME}|g" "$compose_file"
+
+    print_success "Aggiornato: $compose_file"
+    return 0
+}
+
+# ==================== SCRIPT PRINCIPALE ====================
+
+echo -e "${BLUE}================================================${NC}"
+echo -e "${BLUE}AGGIORNAMENTO DOCKER-COMPOSE FILES${NC}"
+echo -e "${BLUE}================================================${NC}"
+echo ""
+
+print_info "Immagine da utilizzare: ${CYAN}$FULL_IMAGE_NAME${NC}"
+echo ""
+
+# Aggiorna vm1/docker-compose.yml
+if [ -f "$VM1_DIR/docker-compose.yml" ]; then
+    print_info "Aggiornamento $VM1_DIR/docker-compose.yml..."
+    update_compose_file "$VM1_DIR/docker-compose.yml"
+else
+    print_warning "File non trovato: $VM1_DIR/docker-compose.yml"
+fi
+
+echo ""
+
+# Aggiorna vm2/docker-compose.yml
+if [ -f "$VM2_DIR/docker-compose.yml" ]; then
+    print_info "Aggiornamento $VM2_DIR/docker-compose.yml..."
+    update_compose_file "$VM2_DIR/docker-compose.yml"
+else
+    print_warning "File non trovato: $VM2_DIR/docker-compose.yml"
+fi
+
+echo ""
+echo -e "${BLUE}================================================${NC}"
+echo -e "${GREEN}✓ Aggiornamento completato!${NC}"
+echo -e "${BLUE}================================================${NC}"
+echo ""
+
+print_info "I file docker-compose.yml ora usano: ${CYAN}$FULL_IMAGE_NAME${NC}"
+print_info "I file di backup sono stati creati con estensione .backup"
+echo ""
+print_warning "Ricordati di fare il pull dell'immagine sulle VM:"
+print_info "  ssh root@192.168.1.201 'cd /opt/myapp && docker pull $FULL_IMAGE_NAME'"
+print_info "  ssh root@192.168.1.202 'cd /opt/myapp && docker pull $FULL_IMAGE_NAME'"
+echo ""

vm1/DOCKERFILE_COMPARISON.md

@@ -0,0 +1,208 @@
+# Confronto Dockerfile: Standard vs Distroless
+
+## File Disponibili
+
+1. **`Dockerfile`** - Versione standard con `python:3.12-slim`
+2. **`Dockerfile.distroless`** - Versione con immagine distroless
+
+---
+
+## Dockerfile Standard (python:3.12-slim)
+
+### Vantaggi
+✅ Più semplice da gestire
+✅ Include shell e tool di sistema
+✅ Facile debugging (puoi fare `docker exec -it container bash`)
+✅ Compatibile con tutti i tool esistenti
+✅ Build più veloce (single-stage)
+
+### Svantaggi
+⚠️ Immagine più grande (~333MB)
+⚠️ Più superficie d'attacco (shell, package manager, etc.)
+⚠️ Include tool non necessari in produzione
+
+### Dimensione
+- Base image: ~125MB
+- Con dipendenze: ~333MB
+
+### Uso
+```bash
+docker build -t orchestrator-app:latest -f Dockerfile .
+```
+
+---
+
+## Dockerfile Distroless (gcr.io/distroless/python3-debian12)
+
+### Vantaggi
+✅ **Immagine molto più piccola** (~180MB stimato)
+✅ **Massima sicurezza** - niente shell, niente package manager
+✅ Superficie d'attacco minimale
+✅ Solo runtime Python e dipendenze
+✅ **Best practice per produzione**
+
+### Svantaggi
+⚠️ **NO debugging** - non puoi fare exec nel container
+⚠️ Build più complesso (multi-stage)
+⚠️ Richiede gestione attenta delle dipendenze
+⚠️ Debugging solo tramite log
+
+### Caratteristiche
+- **Multi-stage build**: compila in python:slim, copia in distroless
+- **Nessuna shell**: impossibile fare `docker exec -it bash`
+- **Solo bytecode**: i file .py sono rimossi anche dalle dipendenze
+- **Immutabile**: modifiche solo ricostruendo l'immagine
+
+### Dimensione Stimata
+- Base distroless: ~50MB
+- Con dipendenze Python: ~180MB (risparmio ~150MB)
+
+### Uso
+```bash
+docker build -t orchestrator-app:distroless -f Dockerfile.distroless .
+```
+
+---
+
+## Confronto Tecnico
+
+| Caratteristica | Standard | Distroless |
+|----------------|----------|------------|
+| Base image | python:3.12-slim | distroless/python3 |
+| Dimensione | ~333MB | ~180MB |
+| Shell | ✅ Si | ❌ No |
+| Debug exec | ✅ Si | ❌ No |
+| Package manager | ✅ Si | ❌ No |
+| Sicurezza | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
+| Complessità | Semplice | Media |
+| Build stages | 1 | 2 |
+| Debugging | Facile | Solo log |
+| Produzione | ✅ OK | ✅ Ideale |
+
+---
+
+## Come Funziona il Multi-Stage Build (Distroless)
+
+### Stage 1: Builder (python:3.12-slim)
+```dockerfile
+FROM python:3.12-slim AS builder
+# Installa dipendenze
+# Compila bytecode
+# Rimuove .py
+```
+
+### Stage 2: Runtime (distroless)
+```dockerfile
+FROM gcr.io/distroless/python3-debian12
+# Copia SOLO:
+# - Dipendenze compilate
+# - File .pyc
+# - File statici (certs, env)
+```
+
+**Risultato**: Immagine finale ha solo Python runtime + bytecode, niente altro!
+
+---
+
+## Quando Usare Quale?
+
+### Usa **Dockerfile Standard** se:
+- ✅ Sviluppo e testing locale
+- ✅ Hai bisogno di debugging interattivo
+- ✅ Vuoi semplicità
+- ✅ La dimensione non è critica
+
+### Usa **Dockerfile.distroless** se:
+- ✅ Deploy in produzione
+- ✅ Massima sicurezza è prioritaria
+- ✅ Vuoi ridurre la superficie d'attacco
+- ✅ Dimensione immagine è importante
+- ✅ Hai buoni log e monitoring
+
+---
+
+## Debugging con Distroless
+
+Dato che non c'è shell, il debugging è diverso:
+
+### ❌ Non Funziona
+```bash
+docker exec -it container bash  # No shell!
+docker exec -it container sh    # No shell!
+```
+
+### ✅ Funziona
+```bash
+# Usa i log
+docker logs container -f
+
+# Usa debug image temporanea (per sviluppo)
+FROM gcr.io/distroless/python3-debian12:debug  # Include busybox
+
+# Debug da fuori
+docker cp container:/app/logs/error.log .
+```
+
+---
+
+## Cambio tra le Due Versioni
+
+### Per usare Standard
+```bash
+# docker-compose.yml
+services:
+  orchestrator:
+    build:
+      context: .
+      dockerfile: Dockerfile  # ← Standard
+```
+
+### Per usare Distroless
+```bash
+# docker-compose.yml
+services:
+  orchestrator:
+    build:
+      context: .
+      dockerfile: Dockerfile.distroless  # ← Distroless
+```
+
+---
+
+## Raccomandazione
+
+### Sviluppo
+Usa **Dockerfile** (standard) per facilità di debug
+
+### Staging
+Testa **Dockerfile.distroless** per validare
+
+### Produzione
+Usa **Dockerfile.distroless** per sicurezza e dimensione
+
+---
+
+## Note Importanti
+
+1. **Volumi**: Funzionano ugualmente in entrambe le versioni
+2. **Network**: Nessuna differenza
+3. **ENV vars**: Funzionano ugualmente
+4. **CMD/ENTRYPOINT**: Funzionano ugualmente
+5. **Logs**: Funzionano ugualmente (STDOUT/STDERR)
+
+## Build & Test
+
+```bash
+# Build standard
+docker build -t orchestrator:standard -f Dockerfile .
+
+# Build distroless
+docker build -t orchestrator:distroless -f Dockerfile.distroless .
+
+# Confronta dimensioni
+docker images | grep orchestrator
+
+# Test entrambe
+docker run --rm orchestrator:standard python --version
+docker run --rm orchestrator:distroless python --version
+```

vm1/DOCKERFILE_NOTES.md

@@ -0,0 +1,89 @@
+# Note sul Dockerfile
+
+## Protezione del Codice Sorgente
+
+Questo Dockerfile è configurato per **proteggere il codice sorgente Python** rimuovendo i file `.py` e mantenendo solo il bytecode compilato (`.pyc`).
+
+### Processo di Build
+
+1. **Copia dei sorgenti**: I file Python vengono copiati nell'immagine
+2. **Installazione dipendenze**: Usa `uv` per installare le dipendenze Python
+3. **Compilazione bytecode**: Compila tutti i file con `python -OO -m compileall`
+4. **Rimozione sorgenti**: Elimina tutti i file `.py` lasciando solo i `.pyc`
+
+### Ottimizzazione `-OO`
+
+L'opzione `-OO` di Python:
+- Rimuove le istruzioni `assert`
+- Rimuove le docstring (`__doc__`)
+- Rimuove il flag `__debug__`
+- Produce file `.opt-2.pyc` invece di `.pyc`
+
+### Variabili Ambiente
+
+- `PYTHONUNBUFFERED=1`: Output immediato dei log (no buffering)
+- `PYTHONDONTWRITEBYTECODE=1`: Non crea nuovi `.pyc` a runtime (già compilati)
+- `PYTHONPATH=/app`: Permette import diretti da `/app`
+
+### Verifica Immagine
+
+Per verificare che l'immagine sia stata buildata correttamente:
+
+```bash
+# Build locale
+docker build -t orchestrator-app:test .
+
+# Test immagine
+../scripts/test-pyc-image.sh orchestrator-app:test
+
+# Verifica manualmente
+docker run --rm orchestrator-app:test find /app/src -name "*.py"  # Deve restituire nulla
+docker run --rm orchestrator-app:test find /app/src -name "*.pyc" # Deve mostrare i .pyc
+```
+
+### Limitazioni
+
+⚠️ **Debug**: I traceback non mostrano il codice sorgente
+⚠️ **Sviluppo**: Non usare questa configurazione per sviluppo locale
+⚠️ **Repository**: Mantieni sempre i sorgenti in git
+
+### Alternative
+
+Se vuoi **disabilitare** la rimozione dei sorgenti (per debug), commenta queste righe nel Dockerfile:
+
+```dockerfile
+# Commenta queste righe per mantenere i file .py
+# RUN python -OO -m compileall /app/src /app/env || true
+# RUN find /app/src -type f -name "*.py" -delete && \
+#     find /app/env -type f -name "*.py" -delete || true
+```
+
+### Build con Sorgenti (Debug)
+
+Per build temporanee con codice sorgente visibile:
+
+```dockerfile
+# Dockerfile.debug
+FROM python:3.12-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+WORKDIR /app
+COPY pyproject.toml ./
+COPY src/ ./src/
+COPY env/ ./env/
+COPY certs/ ./certs/
+COPY matlab_func/ ./matlab_func/
+RUN uv pip install --system -e .
+# NO COMPILATION - NO DELETION
+RUN mkdir -p /app/logs /app/aseftp/csvfs /app/certs /app/matlab_runtime /app/matlab_func
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+CMD ["python", "-m", "src.elab_orchestrator"]
+```
+
+## Best Practices
+
+1. **Versioning**: Usa tag semantici per le immagini di produzione
+2. **Registry**: Pusha le immagini compilate su un registry privato
+3. **Backup**: Tieni sempre i sorgenti in un repository git sicuro
+4. **Testing**: Testa l'immagine con `test-pyc-image.sh` prima del deploy
+5. **Security**: Combina con autenticazione registry per massima sicurezza

vm1/Dockerfile

@@ -5,21 +5,30 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 WORKDIR /app
 
-# Copia pyproject.toml, codice sorgente e file di configurazione
+# Copia pyproject.toml, codice sorgente e file statici
+# NOTA: env/ NON viene copiato - sarà montato come volume esterno
 COPY pyproject.toml ./
 COPY src/ ./src/
-COPY env/ ./env/
 COPY certs/ ./certs/
 COPY matlab_func/ ./matlab_func/
 
 # Installa le dipendenze
 RUN uv pip install --system -e .
 
-# Crea directory per i log, FTP e MATLAB
+# Compila tutti i file Python in bytecode
-RUN mkdir -p /app/logs /app/aseftp/csvfs /app/certs /app/matlab_runtime /app/matlab_func
+# Usa -OO per rimuovere docstring e assert (ottimizzazione massima)
+RUN python -OO -m compileall /app/src || true
+
+# Rimuovi tutti i file sorgente .py, lasciando solo i .pyc compilati in __pycache__
+RUN find /app/src -type f -name "*.py" -delete
+
+# Crea directory per i log, FTP, MATLAB e ENV (sarà montata)
+RUN mkdir -p /app/logs /app/aseftp/csvfs /app/certs /app/matlab_runtime /app/matlab_func /app/env
 
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONPATH=/app
+# Disabilita la creazione di nuovi file .pyc a runtime (non necessari dato che abbiamo già i .pyc)
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # Il comando verrà specificato nel docker-compose.yml per ogni servizio
 CMD ["python", "-m", "src.elab_orchestrator"]

vm1/Dockerfile.distroless

@@ -0,0 +1,58 @@
+# Multi-stage build per distroless
+# Stage 1: Build con immagine completa
+FROM python:3.12-slim AS builder
+
+# Installa uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app
+
+# Copia i file necessari per la build
+# NOTA: env/ NON viene copiato - sarà montato come volume esterno
+COPY pyproject.toml ./
+COPY src/ ./src/
+COPY certs/ ./certs/
+COPY matlab_func/ ./matlab_func/
+
+# Installa le dipendenze in una directory separata
+RUN uv pip install --python=/usr/local/bin/python3 --target=/app/deps .
+
+# Compila tutti i file Python in bytecode
+RUN python -OO -m compileall /app/src || true
+
+# Rimuovi tutti i file sorgente .py, lasciando solo i .pyc compilati
+RUN find /app/src -type f -name "*.py" -delete
+
+# Rimuovi anche i .py dalle dipendenze
+RUN find /app/deps -type f -name "*.py" -delete || true
+
+# Crea directory vuote per runtime (saranno montate come volumi)
+RUN mkdir -p /app/logs /app/aseftp/csvfs /app/matlab_runtime /app/env
+
+# Stage 2: Immagine distroless finale
+FROM gcr.io/distroless/python3-debian12
+
+WORKDIR /app
+
+# Copia le dipendenze installate
+COPY --from=builder /app/deps /app/deps
+
+# Copia il codice compilato (solo .pyc)
+# NOTA: env/ NON viene copiato - sarà montato come volume dal docker-compose
+COPY --from=builder /app/src ./src/
+COPY --from=builder /app/certs ./certs/
+COPY --from=builder /app/matlab_func ./matlab_func/
+
+# Copia le directory vuote (i volumi le sovrascriveranno)
+COPY --from=builder /app/logs ./logs/
+COPY --from=builder /app/aseftp ./aseftp/
+COPY --from=builder /app/matlab_runtime ./matlab_runtime/
+COPY --from=builder /app/env ./env/
+
+# Variabili ambiente
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app:/app/deps
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Comando di default (sarà sovrascritto da docker-compose)
+CMD ["python", "-m", "src.elab_orchestrator"]

vm1/docker-compose.yml

@@ -36,6 +36,7 @@ services:
       TZ: Europe/Rome
     volumes:
       - app-logs:/app/logs
+      - ./env:/app/env:ro
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro
     networks:
@@ -53,6 +54,7 @@ services:
       TZ: Europe/Rome
     volumes:
       - app-logs:/app/logs
+      - ./env:/app/env:ro
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro
     networks:
@@ -70,6 +72,7 @@ services:
       TZ: Europe/Rome
     volumes:
       - app-logs:/app/logs
+      - ./env:/app/env:ro
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro
     networks:
@@ -93,6 +96,7 @@ services:
     volumes:
       - app-logs:/app/logs
       - ./aseftp:/app/aseftp
+      - ./env:/app/env:ro
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro
     networks:
@@ -119,6 +123,7 @@ services:
     volumes:
       - app-logs:/app/logs
       - ./aseftp:/app/aseftp
+      - ./env:/app/env:ro
       - ./ssh_host_key:/app/ssh_host_key:ro
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro

vm1/src

@@ -0,0 +1 @@
+/home/alex/devel/ASE/src/

vm1/src/elab_orchestrator.py

@@ -1,137 +0,0 @@
-#!.venv/bin/python
-"""
-Orchestratore dei worker che lanciano le elaborazioni
-"""
-
-# Import necessary libraries
-import asyncio
-import logging
-
-# Import custom modules for configuration and database connection
-from utils.config import loader_matlab_elab as setting
-from utils.connect.send_email import send_error_email
-from utils.csv.loaders import get_next_csv_atomic
-from utils.database import WorkflowFlags
-from utils.database.action_query import check_flag_elab, get_tool_info
-from utils.database.loader_action import unlock, update_status
-from utils.general import read_error_lines_from_logs
-from utils.orchestrator_utils import run_orchestrator, shutdown_event, worker_context
-
-# Initialize the logger for this module
-logger = logging.getLogger()
-
-# Delay tra un processamento CSV e il successivo (in secondi)
-ELAB_PROCESSING_DELAY = 0.2
-# Tempo di attesa se non ci sono record da elaborare
-NO_RECORD_SLEEP = 60
-
-
-async def worker(worker_id: int, cfg: object, pool: object) -> None:
-    """Esegue il ciclo di lavoro per l'elaborazione dei dati caricati.
-
-    Il worker preleva un record dal database che indica dati pronti per
-    l'elaborazione, esegue un comando Matlab associato e attende
-    prima di iniziare un nuovo ciclo.
-
-    Supporta graceful shutdown controllando il shutdown_event tra le iterazioni.
-
-    Args:
-        worker_id (int): L'ID univoco del worker.
-        cfg (object): L'oggetto di configurazione.
-        pool (object): Il pool di connessioni al database.
-    """
-    # Imposta il context per questo worker
-    worker_context.set(f"W{worker_id:02d}")
-
-    debug_mode = logging.getLogger().getEffectiveLevel() == logging.DEBUG
-    logger.info("Avviato")
-
-    try:
-        while not shutdown_event.is_set():
-            try:
-                logger.info("Inizio elaborazione")
-                if not await check_flag_elab(pool):
-                    record = await get_next_csv_atomic(pool, cfg.dbrectable, WorkflowFlags.DATA_LOADED, WorkflowFlags.DATA_ELABORATED)
-                    if record:
-                        rec_id, _, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
-                        if tool_type.lower() != "gd":  # i tool GD non devono essere elaborati ???
-                            tool_elab_info = await get_tool_info(WorkflowFlags.DATA_ELABORATED, unit_name.upper(), tool_name.upper(), pool)
-                            if tool_elab_info:
-                                if tool_elab_info["statustools"].lower() in cfg.elab_status:
-                                    logger.info("Elaborazione ID %s per %s %s", rec_id, unit_name, tool_name)
-                                    await update_status(cfg, rec_id, WorkflowFlags.START_ELAB, pool)
-                                    matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh \
-                                        {cfg.matlab_runtime} {unit_name.upper()} {tool_name.upper()}"
-                                    proc = await asyncio.create_subprocess_shell(
-                                        matlab_cmd, cwd=cfg.matlab_func_path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-                                    )
-
-                                    stdout, stderr = await proc.communicate()
-
-                                    if proc.returncode != 0:
-                                        logger.error("Errore durante l'elaborazione")
-                                        logger.error(stderr.decode().strip())
-
-                                        if proc.returncode == 124:
-                                            error_type = f"Matlab elab excessive duration: killed after {cfg.matlab_timeout} seconds."
-                                        else:
-                                            error_type = f"Matlab elab failed: {proc.returncode}."
-
-                                        # da verificare i log dove prenderli
-                                        # with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f:
-                                        #    f.write(stderr.decode().strip())
-                                        # errors = [line for line in stderr.decode().strip() if line.startswith("Error")]
-                                        # warnings = [line for line in stderr.decode().strip() if not line.startswith("Error")]
-
-                                        errors, warnings = await read_error_lines_from_logs(
-                                            cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt"
-                                        )
-                                        await send_error_email(
-                                            unit_name.upper(), tool_name.upper(), tool_elab_info["matcall"], error_type, errors, warnings
-                                        )
-
-                                    else:
-                                        logger.info(stdout.decode().strip())
-                                        await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                                    await unlock(cfg, rec_id, pool)
-                                    await asyncio.sleep(ELAB_PROCESSING_DELAY)
-                                else:
-                                    logger.info(
-                                        "ID %s %s - %s %s: MatLab calc by-passed.", rec_id, unit_name, tool_name, tool_elab_info["statustools"]
-                                    )
-                                    await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                                    await update_status(cfg, rec_id, WorkflowFlags.DUMMY_ELABORATED, pool)
-                                    await unlock(cfg, rec_id, pool)
-                        else:
-                            await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                            await update_status(cfg, rec_id, WorkflowFlags.DUMMY_ELABORATED, pool)
-                            await unlock(cfg, rec_id, pool)
-
-                    else:
-                        logger.info("Nessun record disponibile")
-                        await asyncio.sleep(NO_RECORD_SLEEP)
-                else:
-                    logger.info("Flag fermo elaborazione attivato")
-                    await asyncio.sleep(NO_RECORD_SLEEP)
-
-            except asyncio.CancelledError:
-                logger.info("Worker cancellato. Uscita in corso...")
-                raise
-
-            except Exception as e:  # pylint: disable=broad-except
-                logger.error("Errore durante l'esecuzione: %s", e, exc_info=debug_mode)
-                await asyncio.sleep(1)
-
-    except asyncio.CancelledError:
-        logger.info("Worker terminato per shutdown graceful")
-    finally:
-        logger.info("Worker terminato")
-
-
-async def main():
-    """Funzione principale che avvia l'elab_orchestrator."""
-    await run_orchestrator(setting.Config, worker)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm1/src/ftp_csv_receiver.py

@@ -1,245 +0,0 @@
-#!.venv/bin/python
-"""
-This module implements an FTP/SFTP server with custom commands for
-managing virtual users and handling CSV file uploads.
-
-Server mode is controlled by FTP_MODE environment variable:
-- FTP_MODE=ftp (default): Traditional FTP server
-- FTP_MODE=sftp: SFTP (SSH File Transfer Protocol) server
-"""
-
-import asyncio
-import logging
-import os
-import sys
-from hashlib import sha256
-from logging.handlers import RotatingFileHandler
-from pathlib import Path
-
-from pyftpdlib.handlers import FTPHandler
-from pyftpdlib.servers import FTPServer
-
-from utils.authorizers.database_authorizer import DatabaseAuthorizer
-from utils.config import loader_ftp_csv as setting
-from utils.connect import file_management, user_admin
-
-# Configure logging (moved inside main function)
-
-logger = logging.getLogger(__name__)
-
-
-# Legacy authorizer kept for reference (not used anymore)
-# The DatabaseAuthorizer is now used for real-time database synchronization
-
-
-class ASEHandler(FTPHandler):
-    """Custom FTP handler that extends FTPHandler with custom commands and file handling."""
-
-    # Permetti connessioni dati da indirizzi IP diversi (importante per NAT/proxy)
-    permit_foreign_addresses = True
-
-    def __init__(self: object, conn: object, server: object, ioloop: object = None) -> None:
-        """Initializes the handler, adds custom commands, and sets up command permissions.
-
-        Args:
-            conn (object): The connection object.
-            server (object): The FTP server object.
-            ioloop (object): The I/O loop object.
-        """
-        super().__init__(conn, server, ioloop)
-        self.proto_cmds = FTPHandler.proto_cmds.copy()
-        # Add custom FTP commands for managing virtual users - command in lowercase
-        self.proto_cmds.update(
-            {
-                "SITE ADDU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE DISU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> DISU USERNAME (disable virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE ENAU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> ENAU USERNAME (enable virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE LSTU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": None,
-                    "help": "Syntax: SITE <SP> LSTU (list virtual users).",
-                }
-            }
-        )
-
-    def on_file_received(self: object, file: str) -> None:
-        return file_management.on_file_received(self, file)
-
-    def on_incomplete_file_received(self: object, file: str) -> None:
-        """Removes partially uploaded files.
-        Args:
-            file: The path to the incomplete file.
-        """
-        os.remove(file)
-
-    def ftp_SITE_ADDU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_ADDU(self, line)
-
-    def ftp_SITE_DISU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_DISU(self, line)
-
-    def ftp_SITE_ENAU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_ENAU(self, line)
-
-    def ftp_SITE_LSTU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_LSTU(self, line)
-
-
-def setup_logging(log_filename: str):
-    """
-    Configura il logging per il server FTP con rotation e output su console.
-
-    Args:
-        log_filename (str): Percorso del file di log.
-    """
-    root_logger = logging.getLogger()
-    formatter = logging.Formatter("%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s")
-
-    # Rimuovi eventuali handler esistenti
-    if root_logger.hasHandlers():
-        root_logger.handlers.clear()
-
-    # Handler per file con rotation (max 10MB per file, mantiene 5 backup)
-    file_handler = RotatingFileHandler(
-        log_filename,
-        maxBytes=10 * 1024 * 1024,  # 10 MB
-        backupCount=5,  # Mantiene 5 file di backup
-        encoding="utf-8"
-    )
-    file_handler.setFormatter(formatter)
-    root_logger.addHandler(file_handler)
-
-    # Handler per console (utile per Docker)
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(formatter)
-    root_logger.addHandler(console_handler)
-
-    root_logger.setLevel(logging.INFO)
-    root_logger.info("Logging FTP configurato con rotation (10MB, 5 backup) e console output")
-
-
-def start_ftp_server(cfg):
-    """Start traditional FTP server."""
-    try:
-        # Initialize the authorizer with database support
-        # This authorizer checks the database on every login, ensuring
-        # all FTP server instances stay synchronized without restarts
-        authorizer = DatabaseAuthorizer(cfg)
-
-        # Initialize handler
-        handler = ASEHandler
-        handler.cfg = cfg
-        handler.authorizer = authorizer
-
-        # Set masquerade address only if configured (importante per HA con VIP)
-        # Questo è l'IP che il server FTP pubblicherà ai client per le connessioni passive
-        if cfg.proxyaddr and cfg.proxyaddr.strip():
-            handler.masquerade_address = cfg.proxyaddr
-            logger.info(f"FTP masquerade address configured: {cfg.proxyaddr}")
-        else:
-            logger.info("FTP masquerade address not configured - using server's default IP")
-
-        # Set the range of passive ports for the FTP server
-        passive_ports_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
-        handler.passive_ports = passive_ports_range
-
-        # Log configuration
-        logger.info(f"Starting FTP server on port {cfg.service_port} with DatabaseAuthorizer")
-        logger.info(
-            f"FTP passive ports configured: {cfg.firstport}-{cfg.firstport + cfg.portrangewidth - 1} "
-            f"({len(passive_ports_range)} ports)"
-        )
-        logger.info(f"FTP permit_foreign_addresses: {handler.permit_foreign_addresses}")
-        logger.info(f"Database connection: {cfg.dbuser}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbname}")
-
-        # Create and start the FTP server
-        server = FTPServer(("0.0.0.0", cfg.service_port), handler)
-        server.serve_forever()
-
-    except Exception as e:
-        logger.error("FTP server error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-async def start_sftp_server_async(cfg):
-    """Start SFTP server (async)."""
-    try:
-        from utils.servers.sftp_server import start_sftp_server
-
-        logger.info(f"Starting SFTP server on port {cfg.service_port}")
-        logger.info(f"Database connection: {cfg.dbuser}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbname}")
-
-        # Start SFTP server
-        server = await start_sftp_server(cfg, host="0.0.0.0", port=cfg.service_port)
-
-        # Keep server running
-        await asyncio.Event().wait()
-
-    except ImportError as e:
-        logger.error("SFTP mode requires 'asyncssh' library. Install with: pip install asyncssh")
-        logger.error(f"Error: {e}")
-        sys.exit(1)
-    except Exception as e:
-        logger.error("SFTP server error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-def main():
-    """Main function to start FTP or SFTP server based on FTP_MODE environment variable."""
-    # Load the configuration settings
-    cfg = setting.Config()
-
-    # Configure logging first
-    setup_logging(cfg.logfilename)
-
-    # Get server mode from environment variable (default: ftp)
-    server_mode = os.getenv("FTP_MODE", "ftp").lower()
-
-    if server_mode not in ["ftp", "sftp"]:
-        logger.error(f"Invalid FTP_MODE: {server_mode}. Valid values: ftp, sftp")
-        sys.exit(1)
-
-    logger.info(f"Server mode: {server_mode.upper()}")
-
-    try:
-        if server_mode == "ftp":
-            start_ftp_server(cfg)
-        elif server_mode == "sftp":
-            asyncio.run(start_sftp_server_async(cfg))
-    except KeyboardInterrupt:
-        logger.info("Server stopped by user")
-    except Exception as e:
-        logger.error("Unexpected error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

vm1/src/load_ftp_users.py

@@ -1,211 +0,0 @@
-#!.venv/bin/python
-"""
-Script per prelevare dati da MySQL e inviare comandi SITE FTP
-"""
-
-import logging
-import sys
-from ftplib import FTP
-
-import mysql.connector
-
-from utils.config import users_loader as setting
-from utils.database.connection import connetti_db
-
-# Configurazione logging
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-logger = logging.getLogger(__name__)
-
-# Configurazione server FTP
-FTP_CONFIG = {"host": "localhost", "user": "admin", "password": "batt1l0", "port": 2121}
-
-
-def connect_ftp() -> FTP:
-    """
-    Establishes a connection to the FTP server using the predefined configuration.
-    Returns:
-        FTP: An active FTP connection object.
-    """
-    try:
-        ftp = FTP()
-        ftp.connect(FTP_CONFIG["host"], FTP_CONFIG["port"])
-        ftp.login(FTP_CONFIG["user"], FTP_CONFIG["password"])
-        logger.info("Connessione FTP stabilita")
-        return ftp
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore connessione FTP: %s", e)
-        sys.exit(1)
-
-
-def fetch_data_from_db(connection: mysql.connector.MySQLConnection) -> list[tuple]:
-    """
-    Fetches username and password data from the 'ftp_accounts' table in the database.
-
-    Args:
-        connection (mysql.connector.MySQLConnection): The database connection object.
-    Returns:
-        List[Tuple]: A list of tuples, where each tuple contains (username, password).
-    """
-    try:
-        cursor = connection.cursor()
-
-        # Modifica questa query secondo le tue esigenze
-        query = """
-        SELECT username, password
-        FROM ase_lar.ftp_accounts
-        """
-
-        cursor.execute(query)
-        results = cursor.fetchall()
-
-        logger.info("Prelevate %s righe dal database", len(results))
-        return results
-
-    except mysql.connector.Error as e:
-        logger.error("Errore query database: %s", e)
-        return []
-    finally:
-        cursor.close()
-
-
-def fetch_existing_users(connection: mysql.connector.MySQLConnection) -> dict[str, tuple]:
-    """
-    Fetches existing FTP users from virtusers table.
-
-    Args:
-        connection (mysql.connector.MySQLConnection): The database connection object.
-    Returns:
-        dict: Dictionary mapping username to (is_enabled, has_matching_password).
-              is_enabled is True if disabled_at is NULL.
-    """
-    try:
-        cursor = connection.cursor()
-        query = """
-        SELECT ftpuser, disabled_at
-        FROM ase_lar.virtusers
-        """
-        cursor.execute(query)
-        results = cursor.fetchall()
-
-        # Create dictionary: username -> is_enabled
-        users_dict = {username: (disabled_at is None) for username, disabled_at in results}
-
-        logger.info("Trovati %s utenti esistenti in virtusers", len(users_dict))
-        return users_dict
-
-    except mysql.connector.Error as e:
-        logger.error("Errore query database virtusers: %s", e)
-        return {}
-    finally:
-        cursor.close()
-
-
-def send_site_command(ftp: FTP, command: str) -> bool:
-    """
-    Sends a SITE command to the FTP server.
-
-    Args:
-        ftp (FTP): The FTP connection object.
-        command (str): The SITE command string to send (e.g., "ADDU username password").
-    Returns:
-        bool: True if the command was sent successfully, False otherwise.
-    """
-    try:
-        # Il comando SITE viene inviato usando sendcmd
-        response = ftp.sendcmd(f"SITE {command}")
-        logger.info("Comando SITE %s inviato. Risposta: %s", command, response)
-        return True
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore invio comando SITE %s: %s", command, e)
-        return False
-
-
-def main():
-    """
-    Main function to connect to the database, fetch FTP user data, and synchronize users to FTP server.
-    This function is idempotent - it can be run multiple times safely:
-    - If user exists and is enabled: skips
-    - If user exists but is disabled: enables it (SITE ENAU)
-    - If user doesn't exist: creates it (SITE ADDU)
-    """
-    logger.info("Avvio script caricamento utenti FTP (idempotente)")
-    cfg = setting.Config()
-
-    # Connessioni
-    db_connection = connetti_db(cfg)
-    ftp_connection = connect_ftp()
-
-    try:
-        # Preleva utenti da sincronizzare
-        users_to_sync = fetch_data_from_db(db_connection)
-
-        if not users_to_sync:
-            logger.warning("Nessun utente da sincronizzare nel database ftp_accounts")
-            return
-
-        # Preleva utenti già esistenti
-        existing_users = fetch_existing_users(db_connection)
-
-        added_count = 0
-        enabled_count = 0
-        skipped_count = 0
-        error_count = 0
-
-        # Processa ogni utente
-        for row in users_to_sync:
-            username, password = row
-
-            if username in existing_users:
-                is_enabled = existing_users[username]
-
-                if is_enabled:
-                    # Utente già esiste ed è abilitato - skip
-                    logger.info("Utente %s già esiste ed è abilitato - skip", username)
-                    skipped_count += 1
-                else:
-                    # Utente esiste ma è disabilitato - riabilita
-                    logger.info("Utente %s esiste ma è disabilitato - riabilito con SITE ENAU", username)
-                    ftp_site_command = f"enau {username}"
-
-                    if send_site_command(ftp_connection, ftp_site_command):
-                        enabled_count += 1
-                    else:
-                        error_count += 1
-            else:
-                # Utente non esiste - crea
-                logger.info("Utente %s non esiste - creazione con SITE ADDU", username)
-                ftp_site_command = f"addu {username} {password}"
-
-                if send_site_command(ftp_connection, ftp_site_command):
-                    added_count += 1
-                else:
-                    error_count += 1
-
-        logger.info(
-            "Elaborazione completata. Aggiunti: %s, Riabilitati: %s, Saltati: %s, Errori: %s",
-            added_count,
-            enabled_count,
-            skipped_count,
-            error_count
-        )
-
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore generale: %s", e)
-
-    finally:
-        # Chiudi connessioni
-        try:
-            ftp_connection.quit()
-            logger.info("Connessione FTP chiusa")
-        except Exception as e:  # pylint: disable=broad-except
-            logger.error("Errore chiusura connessione FTP: %s", e)
-
-        try:
-            db_connection.close()
-            logger.info("Connessione MySQL chiusa")
-        except Exception as e:  # pylint: disable=broad-except
-            logger.error("Errore chiusura connessione MySQL: %s", e)
-
-
-if __name__ == "__main__":
-    main()

vm1/src/load_orchestrator.py

@@ -1,166 +0,0 @@
-#!.venv/bin/python
-"""
-Orchestratore dei worker che caricano i dati su dataraw
-"""
-
-# Import necessary libraries
-import asyncio
-import importlib
-import logging
-
-# Import custom modules for configuration and database connection
-from utils.config import loader_load_data as setting
-from utils.csv.loaders import get_next_csv_atomic
-from utils.database import WorkflowFlags
-from utils.orchestrator_utils import run_orchestrator, shutdown_event, worker_context
-
-# Initialize the logger for this module
-logger = logging.getLogger()
-
-# Delay tra un processamento CSV e il successivo (in secondi)
-CSV_PROCESSING_DELAY = 0.2
-# Tempo di attesa se non ci sono record da elaborare
-NO_RECORD_SLEEP = 60
-
-# Module import cache to avoid repeated imports (performance optimization)
-_module_cache = {}
-
-
-async def worker(worker_id: int, cfg: dict, pool: object) -> None:
-    """Esegue il ciclo di lavoro per l'elaborazione dei file CSV.
-
-    Il worker preleva un record CSV dal database, ne elabora il contenuto
-    e attende prima di iniziare un nuovo ciclo.
-
-    Supporta graceful shutdown controllando il shutdown_event tra le iterazioni.
-
-    Args:
-        worker_id (int): L'ID univoco del worker.
-        cfg (dict): L'oggetto di configurazione.
-        pool (object): Il pool di connessioni al database.
-    """
-    # Imposta il context per questo worker
-    worker_context.set(f"W{worker_id:02d}")
-
-    logger.info("Avviato")
-
-    try:
-        while not shutdown_event.is_set():
-            try:
-                logger.info("Inizio elaborazione")
-                record = await get_next_csv_atomic(
-                    pool,
-                    cfg.dbrectable,
-                    WorkflowFlags.CSV_RECEIVED,
-                    WorkflowFlags.DATA_LOADED,
-                )
-
-                if record:
-                    success = await load_csv(record, cfg, pool)
-                    if not success:
-                        logger.error("Errore durante l'elaborazione")
-                    await asyncio.sleep(CSV_PROCESSING_DELAY)
-                else:
-                    logger.info("Nessun record disponibile")
-                    await asyncio.sleep(NO_RECORD_SLEEP)
-
-            except asyncio.CancelledError:
-                logger.info("Worker cancellato. Uscita in corso...")
-                raise
-
-            except Exception as e:  # pylint: disable=broad-except
-                logger.error("Errore durante l'esecuzione: %s", e, exc_info=1)
-                await asyncio.sleep(1)
-
-    except asyncio.CancelledError:
-        logger.info("Worker terminato per shutdown graceful")
-    finally:
-        logger.info("Worker terminato")
-
-
-async def load_csv(record: tuple, cfg: object, pool: object) -> bool:
-    """Carica ed elabora un record CSV utilizzando il modulo di parsing appropriato.
-
-    Args:
-        record: Una tupla contenente i dettagli del record CSV da elaborare
-                (rec_id, unit_type, tool_type, unit_name, tool_name).
-        cfg: L'oggetto di configurazione contenente i parametri del sistema.
-        pool (object): Il pool di connessioni al database.
-
-    Returns:
-        True se l'elaborazione del CSV è avvenuta con successo, False altrimenti.
-    """
-
-    debug_mode = logging.getLogger().getEffectiveLevel() == logging.DEBUG
-    logger.debug("Inizio ricerca nuovo CSV da elaborare")
-
-    rec_id, unit_type, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
-    logger.info(
-        "Trovato CSV da elaborare: ID=%s, Tipo=%s_%s, Nome=%s_%s",
-        rec_id,
-        unit_type,
-        tool_type,
-        unit_name,
-        tool_name,
-    )
-
-    # Costruisce il nome del modulo da caricare dinamicamente
-    module_names = [
-        f"utils.parsers.by_name.{unit_name}_{tool_name}",
-        f"utils.parsers.by_name.{unit_name}_{tool_type}",
-        f"utils.parsers.by_name.{unit_name}_all",
-        f"utils.parsers.by_type.{unit_type}_{tool_type}",
-    ]
-
-    # Try to get from cache first (performance optimization)
-    modulo = None
-    cache_key = None
-
-    for module_name in module_names:
-        if module_name in _module_cache:
-            # Cache hit! Use cached module
-            modulo = _module_cache[module_name]
-            cache_key = module_name
-            logger.info("Modulo caricato dalla cache: %s", module_name)
-            break
-
-    # If not in cache, import dynamically
-    if not modulo:
-        for module_name in module_names:
-            try:
-                logger.debug("Caricamento dinamico del modulo: %s", module_name)
-                modulo = importlib.import_module(module_name)
-                # Store in cache for future use
-                _module_cache[module_name] = modulo
-                cache_key = module_name
-                logger.info("Modulo caricato per la prima volta: %s", module_name)
-                break
-            except (ImportError, AttributeError) as e:
-                logger.debug(
-                    "Modulo %s non presente o non valido. %s",
-                    module_name,
-                    e,
-                    exc_info=debug_mode,
-                )
-
-    if not modulo:
-        logger.error("Nessun modulo trovato %s", module_names)
-        return False
-
-    # Ottiene la funzione 'main_loader' dal modulo
-    funzione = modulo.main_loader
-
-    # Esegui la funzione
-    logger.info("Elaborazione con modulo %s per ID=%s", modulo, rec_id)
-    await funzione(cfg, rec_id, pool)
-    logger.info("Elaborazione completata per ID=%s", rec_id)
-    return True
-
-
-async def main():
-    """Funzione principale che avvia il load_orchestrator."""
-    await run_orchestrator(setting.Config, worker)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm1/src/old_scripts/dbconfig.py

@@ -1,16 +0,0 @@
-from configparser import ConfigParser
-
-
-def read_db_config(filename='../env/config.ini', section='mysql'):
-    parser = ConfigParser()
-    parser.read(filename)
-
-    db = {}
-    if parser.has_section(section):
-        items = parser.items(section)
-        for item in items:
-            db[item[0]] = item[1]
-    else:
-        raise Exception(f'{section} not found in the {filename} file')
-
-    return db

vm1/src/old_scripts/hirpiniaLoadScript.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-from datetime import datetime
-
-import ezodf
-from dbconfig import read_db_config
-from mysql.connector import Error, MySQLConnection
-
-
-def getDataFromCsv(pathFile):
-    try:
-        folder_path, file_with_extension = os.path.split(pathFile)
-        unit_name = os.path.basename(folder_path)#unitname
-        tool_name, _ = os.path.splitext(file_with_extension)#toolname
-        tool_name = tool_name.replace("HIRPINIA_", "")
-        tool_name = tool_name.split("_")[0]
-        print(unit_name, tool_name)
-        datiRaw = []
-        doc = ezodf.opendoc(pathFile)
-        for sheet in doc.sheets:
-            node_num = sheet.name.replace("S-", "")
-            print(f"Sheet Name: {sheet.name}")
-            rows_to_skip = 2
-            for i, row in enumerate(sheet.rows()):
-                if i < rows_to_skip:
-                    continue
-                row_data = [cell.value for cell in row]
-                date_time = datetime.strptime(row_data[0], "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S").split(" ")
-                date = date_time[0]
-                time = date_time[1]
-                val0 = row_data[2]
-                val1 = row_data[4]
-                val2 = row_data[6]
-                val3 = row_data[8]
-                datiRaw.append((unit_name, tool_name, node_num, date, time, -1, -273, val0, val1, val2, val3))
-        try:
-            db_config = read_db_config()
-            conn = MySQLConnection(**db_config)
-            cursor = conn.cursor(dictionary=True)
-            queryRaw = "insert ignore into RAWDATACOR(UnitName,ToolNameID,NodeNum,EventDate,EventTime,BatLevel,Temperature,Val0,Val1,Val2,Val3) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-            cursor.executemany(queryRaw, datiRaw)
-            conn.commit()
-        except Error as e:
-            print('Error:', e)
-        finally:
-            queryMatlab = "select m.matcall from tools as t join units as u on u.id=t.unit_id join matfuncs as m on m.id=t.matfunc where u.name=%s and t.name=%s"
-            cursor.execute(queryMatlab, [unit_name, tool_name])
-            resultMatlab = cursor.fetchall()
-            if(resultMatlab):
-                print("Avvio "+str(resultMatlab[0]["matcall"]))
-                os.system("cd /usr/local/matlab_func/; ./run_"+str(resultMatlab[0]["matcall"])+".sh /usr/local/MATLAB/MATLAB_Runtime/v93/ "+str(unit_name)+" "+str(tool_name)+"")
-            cursor.close()
-            conn.close()
-    except Exception as e:
-        print(f"An unexpected error occurred: {str(e)}\n")
-
-def main():
-    print("Avviato.")
-    getDataFromCsv(sys.argv[1])
-    print("Finito.")
-
-if __name__ == '__main__':
-    main()

vm1/src/old_scripts/sisgeoLoadScript.py

@@ -1,306 +0,0 @@
-#!/usr/bin/env python3
-import sys
-from datetime import datetime
-from decimal import Decimal
-
-from dbconfig import read_db_config
-from mysql.connector import Error, MySQLConnection
-
-
-def insertData(dati):
-    #print(dati)
-    #print(len(dati))
-    if(len(dati) > 0):
-        db_config = read_db_config()
-        conn = MySQLConnection(**db_config)
-        cursor = conn.cursor()
-        if(len(dati) == 2):
-            u = ""
-            t = ""
-            rawdata = dati[0]
-            elabdata = dati[1]
-            if(len(rawdata) > 0):
-                for r in rawdata:
-                    #print(r)
-                    #print(len(r))
-                    if(len(r) == 6):#nodo1
-                        unitname = r[0]
-                        toolname = r[1]
-                        nodenum = r[2]
-                        pressure = Decimal(r[3])*100
-                        date = r[4]
-                        time = r[5]
-                        query = "SELECT * from RAWDATACOR WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s ORDER BY EventDate desc,EventTime desc limit 1"
-                        try:
-                            cursor.execute(query, [unitname, toolname, nodenum])
-                            result = cursor.fetchall()
-                            if(result):
-                                if(result[0][8] is None):
-                                    datetimeOld = datetime.strptime(str(result[0][4]) + " " + str(result[0][5]), "%Y-%m-%d %H:%M:%S")
-                                    datetimeNew = datetime.strptime(str(date) + " " + str(time), "%Y-%m-%d %H:%M:%S")
-                                    dateDiff = datetimeNew - datetimeOld
-                                    if(dateDiff.total_seconds() / 3600 >= 5):
-                                        query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                        try:
-                                            cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, pressure, -1, -273])
-                                            conn.commit()
-                                        except Error as e:
-                                            print('Error:', e)
-                                    else:
-                                        query = "UPDATE RAWDATACOR SET val0=%s, EventDate=%s, EventTime=%s WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s AND val0 is NULL ORDER BY EventDate desc,EventTime desc limit 1"
-                                        try:
-                                            cursor.execute(query, [pressure, date, time, unitname, toolname, nodenum])
-                                            conn.commit()
-                                        except Error as e:
-                                            print('Error:', e)
-                                elif(result[0][8] is not None):
-                                    query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                    try:
-                                        cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, pressure, -1, -273])
-                                        conn.commit()
-                                    except Error as e:
-                                        print('Error:', e)
-                            else:
-                                query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                try:
-                                    cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, pressure, -1, -273])
-                                    conn.commit()
-                                except Error as e:
-                                    print('Error:', e)
-                        except Error as e:
-                            print('Error:', e)
-                    else:#altri 2->5
-                        unitname = r[0]
-                        toolname = r[1]
-                        nodenum = r[2]
-                        freqinhz = r[3]
-                        therminohms = r[4]
-                        freqindigit = r[5]
-                        date = r[6]
-                        time = r[7]
-                        query = "SELECT * from RAWDATACOR WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s ORDER BY EventDate desc,EventTime desc limit 1"
-                        try:
-                            cursor.execute(query, [unitname, toolname, nodenum])
-                            result = cursor.fetchall()
-                            if(result):
-                                if(result[0][8] is None):
-                                    query = "UPDATE RAWDATACOR SET val0=%s, val1=%s, val2=%s, EventDate=%s, EventTime=%s WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s AND val0 is NULL ORDER BY EventDate desc,EventTime desc limit 1"
-                                    try:
-                                        cursor.execute(query, [freqinhz, therminohms, freqindigit, date, time, unitname, toolname, nodenum])
-                                        conn.commit()
-                                    except Error as e:
-                                        print('Error:', e)
-                                elif(result[0][8] is not None):
-                                    query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, val1, val2, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                    try:
-                                        cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, freqinhz, therminohms, freqindigit, -1, -273])
-                                        conn.commit()
-                                    except Error as e:
-                                        print('Error:', e)
-                            else:
-                                query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, val1, val2, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                try:
-                                    cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, freqinhz, therminohms, freqindigit, -1, -273])
-                                    conn.commit()
-                                except Error as e:
-                                    print('Error:', e)
-                        except Error as e:
-                            print('Error:', e)
-
-            if(len(elabdata) > 0):
-                for e in elabdata:
-                    #print(e)
-                    #print(len(e))
-                    if(len(e) == 6):#nodo1
-                        unitname = e[0]
-                        toolname = e[1]
-                        nodenum = e[2]
-                        pressure = Decimal(e[3])*100
-                        date = e[4]
-                        time = e[5]
-                        try:
-                            query = "INSERT INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, pressure) VALUES(%s,%s,%s,%s,%s,%s)"
-                            cursor.execute(query, [unitname, toolname, nodenum, date, time, pressure])
-                            conn.commit()
-                        except Error as e:
-                            print('Error:', e)
-                    else:#altri 2->5
-                        unitname = e[0]
-                        toolname = e[1]
-                        u = unitname
-                        t = toolname
-                        nodenum = e[2]
-                        pch = e[3]
-                        tch = e[4]
-                        date = e[5]
-                        time = e[6]
-                        try:
-                            query = "INSERT INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, XShift, T_node) VALUES(%s,%s,%s,%s,%s,%s,%s)"
-                            cursor.execute(query, [unitname, toolname, nodenum, date, time, pch, tch])
-                            conn.commit()
-                        except Error as e:
-                            print('Error:', e)
-                #os.system("cd /usr/local/matlab_func/; ./run_ATD_lnx.sh /usr/local/MATLAB/MATLAB_Runtime/v93/ "+u+" "+t+"")
-        else:
-            for r in dati:
-                #print(r)
-                unitname = r[0]
-                toolname = r[1]
-                nodenum = r[2]
-                date = r[3]
-                time = r[4]
-                battery = r[5]
-                temperature = r[6]
-                query = "SELECT * from RAWDATACOR WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s ORDER BY EventDate desc,EventTime desc limit 1"
-                try:
-                    cursor.execute(query, [unitname, toolname, nodenum])
-                    result = cursor.fetchall()
-                    if(result):
-                        if(result[0][25] is None or result[0][25] == -1.00):
-                            datetimeOld = datetime.strptime(str(result[0][4]) + " " + str(result[0][5]), "%Y-%m-%d %H:%M:%S")
-                            datetimeNew = datetime.strptime(str(date) + " " + str(time), "%Y-%m-%d %H:%M:%S")
-                            dateDiff = datetimeNew - datetimeOld
-                            #print(dateDiff.total_seconds() / 3600)
-                            if(dateDiff.total_seconds() / 3600 >= 5):
-                                query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                                try:
-                                    cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, battery, temperature])
-                                    conn.commit()
-                                except Error as e:
-                                    print('Error:', e)
-                            else:
-                                query = "UPDATE RAWDATACOR SET BatLevelModule=%s, TemperatureModule=%s WHERE UnitName=%s AND ToolNameID=%s AND NodeNum=%s AND (BatLevelModule is NULL or BatLevelModule = -1.00) ORDER BY EventDate desc,EventTime desc limit 1"
-                                try:
-                                    cursor.execute(query, [battery, temperature, unitname, toolname, nodenum])
-                                    conn.commit()
-                                except Error as e:
-                                    print('Error:', e)
-                        elif(result[0][25] is not None and result[0][25] != -1.00):
-                            query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                            try:
-                                cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, battery, temperature])
-                                conn.commit()
-                            except Error as e:
-                                print('Error:', e)
-                    else:
-                        query = "INSERT INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, BatLevelModule, TemperatureModule) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-                        try:
-                            cursor.execute(query, [unitname, toolname, nodenum, date, time, -1, -273, battery, temperature])
-                            conn.commit()
-                        except Error as e:
-                            print('Error:', e)
-                except Error as e:
-                    print('Error:', e)
-        cursor.close()
-        conn.close()
-
-def getDataFromCsv(pathFile):
-    with open(pathFile) as file:
-        data = file.readlines()
-        data = [row.rstrip() for row in data]
-    serial_number = data[0].split(",")[1]
-    data = data[10:] #rimuove righe header
-    dati = []
-    rawDatiReadings = []#tmp
-    elabDatiReadings = []#tmp
-    datiReadings = []
-    i = 0
-    unit = ""
-    tool = ""
-    #row = data[0]#quando non c'era il for solo 1 riga
-    for row in data:#se ci sono righe multiple
-        row = row.split(",")
-        if i == 0:
-            query = "SELECT unit_name, tool_name FROM sisgeo_tools WHERE serial_number='"+serial_number+"'"
-            try:
-                db_config = read_db_config()
-                conn = MySQLConnection(**db_config)
-                cursor = conn.cursor()
-                cursor.execute(query)
-                result = cursor.fetchall()
-            except Error as e:
-                print('Error:', e)
-            unit = result[0][0]
-            tool = result[0][1]
-            #print(result[0][0])
-            #print(result[0][1])
-        if("health" in pathFile):
-            datetime = str(row[0]).replace("\"", "").split(" ")
-            date = datetime[0]
-            time = datetime[1]
-            battery = row[1]
-            temperature = row[2]
-            dati.append((unit, tool, 1, date, time, battery, temperature))
-            dati.append((unit, tool, 2, date, time, battery, temperature))
-            dati.append((unit, tool, 3, date, time, battery, temperature))
-            dati.append((unit, tool, 4, date, time, battery, temperature))
-            dati.append((unit, tool, 5, date, time, battery, temperature))
-        else:
-            datetime = str(row[0]).replace("\"", "").split(" ")
-            date = datetime[0]
-            time = datetime[1]
-            atmpressure = row[1]#nodo1
-            #raw
-            freqinhzch1 = row[2]#nodo2
-            freqindigitch1 = row[3]#nodo2
-            thermResInOhmsch1 = row[4]#nodo2
-            freqinhzch2 = row[5]#nodo3
-            freqindigitch2 = row[6]#nodo3
-            thermResInOhmsch2 = row[7]#nodo3
-            freqinhzch3 = row[8]#nodo4
-            freqindigitch3 = row[9]#nodo4
-            thermResInOhmsch3 = row[10]#nodo4
-            freqinhzch4 = row[11]#nodo5
-            freqindigitch4 = row[12]#nodo5
-            thermResInOhmsch4 = row[13]#nodo5
-            #elab
-            pch1 = row[18]#nodo2
-            tch1 = row[19]#nodo2
-            pch2 = row[20]#nodo3
-            tch2 = row[21]#nodo3
-            pch3 = row[22]#nodo4
-            tch3 = row[23]#nodo4
-            pch4 = row[24]#nodo5
-            tch4 = row[25]#nodo5
-
-            rawDatiReadings.append((unit, tool, 1, atmpressure, date, time))
-            rawDatiReadings.append((unit, tool, 2, freqinhzch1, thermResInOhmsch1, freqindigitch1, date, time))
-            rawDatiReadings.append((unit, tool, 3, freqinhzch2, thermResInOhmsch2, freqindigitch2, date, time))
-            rawDatiReadings.append((unit, tool, 4, freqinhzch3, thermResInOhmsch3, freqindigitch3, date, time))
-            rawDatiReadings.append((unit, tool, 5, freqinhzch4, thermResInOhmsch4, freqindigitch4, date, time))
-
-            elabDatiReadings.append((unit, tool, 1, atmpressure, date, time))
-            elabDatiReadings.append((unit, tool, 2, pch1, tch1, date, time))
-            elabDatiReadings.append((unit, tool, 3, pch2, tch2, date, time))
-            elabDatiReadings.append((unit, tool, 4, pch3, tch3, date, time))
-            elabDatiReadings.append((unit, tool, 5, pch4, tch4, date, time))
-
-            #[ram],[elab]#quando c'era solo 1 riga
-            #dati = [
-            #    [
-            #        (unit, tool, 1, atmpressure, date, time),
-            #        (unit, tool, 2, freqinhzch1, thermResInOhmsch1, freqindigitch1, date, time),
-            #        (unit, tool, 3, freqinhzch2, thermResInOhmsch2, freqindigitch2, date, time),
-            #        (unit, tool, 4, freqinhzch3, thermResInOhmsch3, freqindigitch3, date, time),
-            #        (unit, tool, 5, freqinhzch4, thermResInOhmsch4, freqindigitch4, date, time),
-            #    ], [
-            #        (unit, tool, 1, atmpressure, date, time),
-            #        (unit, tool, 2, pch1, tch1, date, time),
-            #        (unit, tool, 3, pch2, tch2, date, time),
-            #        (unit, tool, 4, pch3, tch3, date, time),
-            #        (unit, tool, 5, pch4, tch4, date, time),
-            #        ]
-            #    ]
-        i+=1
-        #print(dati)
-    if(len(rawDatiReadings) > 0 or len(elabDatiReadings) > 0):
-        datiReadings = [rawDatiReadings, elabDatiReadings]
-    if(len(datiReadings) > 0):
-        return datiReadings
-    return dati
-
-def main():
-    insertData(getDataFromCsv(sys.argv[1]))
-
-if __name__ == '__main__':
-    main()

vm1/src/old_scripts/sorotecPini.py

@@ -1,304 +0,0 @@
-#!/usr/bin/env python3
-import sys
-
-from dbconfig import read_db_config
-from mysql.connector import Error, MySQLConnection
-
-
-def removeDuplicates(lst):
-    return list(set([i for i in lst]))
-
-def getDataFromCsvAndInsert(pathFile):
-    try:
-        print(pathFile)
-        folder_name = pathFile.split("/")[-2]#cartella
-        with open(pathFile) as file:
-            data = file.readlines()
-            data = [row.rstrip() for row in data]
-        if(len(data) > 0 and data is not None):
-            if(folder_name == "ID0247"):
-                unit_name = "ID0247"
-                tool_name = "DT0001"
-                data.pop(0) #rimuove header
-                data.pop(0)
-                data.pop(0)
-                data.pop(0)
-                data = [element for element in data if element != ""]
-                try:
-                    db_config = read_db_config()
-                    conn = MySQLConnection(**db_config)
-                    cursor = conn.cursor()
-                    queryElab = "insert ignore into ELABDATADISP(UnitName,ToolNameID,NodeNum,EventDate,EventTime,load_value) values (%s,%s,%s,%s,%s,%s)"
-                    queryRaw = "insert ignore into RAWDATACOR(UnitName,ToolNameID,NodeNum,EventDate,EventTime,BatLevel,Temperature,Val0) values (%s,%s,%s,%s,%s,%s,%s,%s)"
-                    if("_1_" in pathFile):
-                        print("File tipo 1.\n")
-                        #print(unit_name, tool_name)
-                        dataToInsertElab = []
-                        dataToInsertRaw = []
-                        for row in data:
-                            rowSplitted = row.replace("\"","").split(";")
-                            eventTimestamp = rowSplitted[0].split(" ")
-                            date = eventTimestamp[0].split("-")
-                            date = date[2]+"-"+date[1]+"-"+date[0]
-                            time = eventTimestamp[1]
-                            an3 = rowSplitted[1]
-                            an4 = rowSplitted[2]#V unit battery
-                            OUTREG2 = rowSplitted[3]
-                            E8_181_CH2 = rowSplitted[4]#2
-                            E8_181_CH3 = rowSplitted[5]#3
-                            E8_181_CH4 = rowSplitted[6]#4
-                            E8_181_CH5 = rowSplitted[7]#5
-                            E8_181_CH6 = rowSplitted[8]#6
-                            E8_181_CH7 = rowSplitted[9]#7
-                            E8_181_CH8 = rowSplitted[10]#8
-                            E8_182_CH1 = rowSplitted[11]#9
-                            E8_182_CH2 = rowSplitted[12]#10
-                            E8_182_CH3 = rowSplitted[13]#11
-                            E8_182_CH4 = rowSplitted[14]#12
-                            E8_182_CH5 = rowSplitted[15]#13
-                            E8_182_CH6 = rowSplitted[16]#14
-                            E8_182_CH7 = rowSplitted[17]#15
-                            E8_182_CH8 = rowSplitted[18]#16
-                            E8_183_CH1 = rowSplitted[19]#17
-                            E8_183_CH2 = rowSplitted[20]#18
-                            E8_183_CH3 = rowSplitted[21]#19
-                            E8_183_CH4 = rowSplitted[22]#20
-                            E8_183_CH5 = rowSplitted[23]#21
-                            E8_183_CH6 = rowSplitted[24]#22
-                            E8_183_CH7 = rowSplitted[25]#23
-                            E8_183_CH8 = rowSplitted[26]#24
-                            E8_184_CH1 = rowSplitted[27]#25
-                            E8_184_CH2 = rowSplitted[28]#26
-                            E8_184_CH3 = rowSplitted[29]#27 mv/V
-                            E8_184_CH4 = rowSplitted[30]#28 mv/V
-                            E8_184_CH5 = rowSplitted[31]#29 mv/V
-                            E8_184_CH6 = rowSplitted[32]#30 mv/V
-                            E8_184_CH7 = rowSplitted[33]#31 mv/V
-                            E8_184_CH8 = rowSplitted[34]#32 mv/V
-                            E8_181_CH1 = rowSplitted[35]#1
-                            an1 = rowSplitted[36]
-                            an2 = rowSplitted[37]
-                            #print(unit_name, tool_name, 1, E8_181_CH1)
-                            #print(unit_name, tool_name, 2, E8_181_CH2)
-                            #print(unit_name, tool_name, 3, E8_181_CH3)
-                            #print(unit_name, tool_name, 4, E8_181_CH4)
-                            #print(unit_name, tool_name, 5, E8_181_CH5)
-                            #print(unit_name, tool_name, 6, E8_181_CH6)
-                            #print(unit_name, tool_name, 7, E8_181_CH7)
-                            #print(unit_name, tool_name, 8, E8_181_CH8)
-                            #print(unit_name, tool_name, 9, E8_182_CH1)
-                            #print(unit_name, tool_name, 10, E8_182_CH2)
-                            #print(unit_name, tool_name, 11, E8_182_CH3)
-                            #print(unit_name, tool_name, 12, E8_182_CH4)
-                            #print(unit_name, tool_name, 13, E8_182_CH5)
-                            #print(unit_name, tool_name, 14, E8_182_CH6)
-                            #print(unit_name, tool_name, 15, E8_182_CH7)
-                            #print(unit_name, tool_name, 16, E8_182_CH8)
-                            #print(unit_name, tool_name, 17, E8_183_CH1)
-                            #print(unit_name, tool_name, 18, E8_183_CH2)
-                            #print(unit_name, tool_name, 19, E8_183_CH3)
-                            #print(unit_name, tool_name, 20, E8_183_CH4)
-                            #print(unit_name, tool_name, 21, E8_183_CH5)
-                            #print(unit_name, tool_name, 22, E8_183_CH6)
-                            #print(unit_name, tool_name, 23, E8_183_CH7)
-                            #print(unit_name, tool_name, 24, E8_183_CH8)
-                            #print(unit_name, tool_name, 25, E8_184_CH1)
-                            #print(unit_name, tool_name, 26, E8_184_CH2)
-                            #print(unit_name, tool_name, 27, E8_184_CH3)
-                            #print(unit_name, tool_name, 28, E8_184_CH4)
-                            #print(unit_name, tool_name, 29, E8_184_CH5)
-                            #print(unit_name, tool_name, 30, E8_184_CH6)
-                            #print(unit_name, tool_name, 31, E8_184_CH7)
-                            #print(unit_name, tool_name, 32, E8_184_CH8)
-                            #---------------------------------------------------------------------------------------
-                            dataToInsertRaw.append((unit_name, tool_name, 1, date, time, an4, -273, E8_181_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 2, date, time, an4, -273, E8_181_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 3, date, time, an4, -273, E8_181_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 4, date, time, an4, -273, E8_181_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 5, date, time, an4, -273, E8_181_CH5))
-                            dataToInsertRaw.append((unit_name, tool_name, 6, date, time, an4, -273, E8_181_CH6))
-                            dataToInsertRaw.append((unit_name, tool_name, 7, date, time, an4, -273, E8_181_CH7))
-                            dataToInsertRaw.append((unit_name, tool_name, 8, date, time, an4, -273, E8_181_CH8))
-                            dataToInsertRaw.append((unit_name, tool_name, 9, date, time, an4, -273, E8_182_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 10, date, time, an4, -273, E8_182_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 11, date, time, an4, -273, E8_182_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 12, date, time, an4, -273, E8_182_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 13, date, time, an4, -273, E8_182_CH5))
-                            dataToInsertRaw.append((unit_name, tool_name, 14, date, time, an4, -273, E8_182_CH6))
-                            dataToInsertRaw.append((unit_name, tool_name, 15, date, time, an4, -273, E8_182_CH7))
-                            dataToInsertRaw.append((unit_name, tool_name, 16, date, time, an4, -273, E8_182_CH8))
-                            dataToInsertRaw.append((unit_name, tool_name, 17, date, time, an4, -273, E8_183_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 18, date, time, an4, -273, E8_183_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 19, date, time, an4, -273, E8_183_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 20, date, time, an4, -273, E8_183_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 21, date, time, an4, -273, E8_183_CH5))
-                            dataToInsertRaw.append((unit_name, tool_name, 22, date, time, an4, -273, E8_183_CH6))
-                            dataToInsertRaw.append((unit_name, tool_name, 23, date, time, an4, -273, E8_183_CH7))
-                            dataToInsertRaw.append((unit_name, tool_name, 24, date, time, an4, -273, E8_183_CH8))
-                            dataToInsertRaw.append((unit_name, tool_name, 25, date, time, an4, -273, E8_184_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 26, date, time, an4, -273, E8_184_CH2))
-                            #---------------------------------------------------------------------------------------
-                            dataToInsertElab.append((unit_name, tool_name, 1, date, time, E8_181_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 2, date, time, E8_181_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 3, date, time, E8_181_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 4, date, time, E8_181_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 5, date, time, E8_181_CH5))
-                            dataToInsertElab.append((unit_name, tool_name, 6, date, time, E8_181_CH6))
-                            dataToInsertElab.append((unit_name, tool_name, 7, date, time, E8_181_CH7))
-                            dataToInsertElab.append((unit_name, tool_name, 8, date, time, E8_181_CH8))
-                            dataToInsertElab.append((unit_name, tool_name, 9, date, time, E8_182_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 10, date, time, E8_182_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 11, date, time, E8_182_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 12, date, time, E8_182_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 13, date, time, E8_182_CH5))
-                            dataToInsertElab.append((unit_name, tool_name, 14, date, time, E8_182_CH6))
-                            dataToInsertElab.append((unit_name, tool_name, 15, date, time, E8_182_CH7))
-                            dataToInsertElab.append((unit_name, tool_name, 16, date, time, E8_182_CH8))
-                            dataToInsertElab.append((unit_name, tool_name, 17, date, time, E8_183_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 18, date, time, E8_183_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 19, date, time, E8_183_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 20, date, time, E8_183_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 21, date, time, E8_183_CH5))
-                            dataToInsertElab.append((unit_name, tool_name, 22, date, time, E8_183_CH6))
-                            dataToInsertElab.append((unit_name, tool_name, 23, date, time, E8_183_CH7))
-                            dataToInsertElab.append((unit_name, tool_name, 24, date, time, E8_183_CH8))
-                            dataToInsertElab.append((unit_name, tool_name, 25, date, time, E8_184_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 26, date, time, E8_184_CH2))
-                            #---------------------------------------------------------------------------------------
-                        cursor.executemany(queryElab, dataToInsertElab)
-                        cursor.executemany(queryRaw, dataToInsertRaw)
-                        conn.commit()
-                        #print(dataToInsertElab)
-                        #print(dataToInsertRaw)
-                    elif("_2_" in pathFile):
-                        print("File tipo 2.\n")
-                        #print(unit_name, tool_name)
-                        dataToInsertElab = []
-                        dataToInsertRaw = []
-                        for row in data:
-                            rowSplitted = row.replace("\"","").split(";")
-                            eventTimestamp = rowSplitted[0].split(" ")
-                            date = eventTimestamp[0].split("-")
-                            date = date[2]+"-"+date[1]+"-"+date[0]
-                            time = eventTimestamp[1]
-                            an2 = rowSplitted[1]
-                            an3 = rowSplitted[2]
-                            an1 = rowSplitted[3]
-                            OUTREG2 = rowSplitted[4]
-                            E8_181_CH1 = rowSplitted[5]#33 mv/V
-                            E8_181_CH2 = rowSplitted[6]#34 mv/V
-                            E8_181_CH3 = rowSplitted[7]#35 mv/V
-                            E8_181_CH4 = rowSplitted[8]#36 mv/V
-                            E8_181_CH5 = rowSplitted[9]#37 mv/V
-                            E8_181_CH6 = rowSplitted[10]#38 mv/V
-                            E8_181_CH7 = rowSplitted[11]#39 mv/V
-                            E8_181_CH8 = rowSplitted[12]#40 mv/V
-                            E8_182_CH1 = rowSplitted[13]#41
-                            E8_182_CH2 = rowSplitted[14]#42
-                            E8_182_CH3 = rowSplitted[15]#43
-                            E8_182_CH4 = rowSplitted[16]#44
-                            E8_182_CH5 = rowSplitted[17]#45 mv/V
-                            E8_182_CH6 = rowSplitted[18]#46 mv/V
-                            E8_182_CH7 = rowSplitted[19]#47 mv/V
-                            E8_182_CH8 = rowSplitted[20]#48 mv/V
-                            E8_183_CH1 = rowSplitted[21]#49
-                            E8_183_CH2 = rowSplitted[22]#50
-                            E8_183_CH3 = rowSplitted[23]#51
-                            E8_183_CH4 = rowSplitted[24]#52
-                            E8_183_CH5 = rowSplitted[25]#53 mv/V
-                            E8_183_CH6 = rowSplitted[26]#54 mv/V
-                            E8_183_CH7 = rowSplitted[27]#55 mv/V
-                            E8_183_CH8 = rowSplitted[28]#56
-                            E8_184_CH1 = rowSplitted[29]#57
-                            E8_184_CH2 = rowSplitted[30]#58
-                            E8_184_CH3 = rowSplitted[31]#59
-                            E8_184_CH4 = rowSplitted[32]#60
-                            E8_184_CH5 = rowSplitted[33]#61
-                            E8_184_CH6 = rowSplitted[34]#62
-                            E8_184_CH7 = rowSplitted[35]#63 mv/V
-                            E8_184_CH8 = rowSplitted[36]#64 mv/V
-                            an4 = rowSplitted[37]#V unit battery
-                            #print(unit_name, tool_name, 33, E8_181_CH1)
-                            #print(unit_name, tool_name, 34, E8_181_CH2)
-                            #print(unit_name, tool_name, 35, E8_181_CH3)
-                            #print(unit_name, tool_name, 36, E8_181_CH4)
-                            #print(unit_name, tool_name, 37, E8_181_CH5)
-                            #print(unit_name, tool_name, 38, E8_181_CH6)
-                            #print(unit_name, tool_name, 39, E8_181_CH7)
-                            #print(unit_name, tool_name, 40, E8_181_CH8)
-                            #print(unit_name, tool_name, 41, E8_182_CH1)
-                            #print(unit_name, tool_name, 42, E8_182_CH2)
-                            #print(unit_name, tool_name, 43, E8_182_CH3)
-                            #print(unit_name, tool_name, 44, E8_182_CH4)
-                            #print(unit_name, tool_name, 45, E8_182_CH5)
-                            #print(unit_name, tool_name, 46, E8_182_CH6)
-                            #print(unit_name, tool_name, 47, E8_182_CH7)
-                            #print(unit_name, tool_name, 48, E8_182_CH8)
-                            #print(unit_name, tool_name, 49, E8_183_CH1)
-                            #print(unit_name, tool_name, 50, E8_183_CH2)
-                            #print(unit_name, tool_name, 51, E8_183_CH3)
-                            #print(unit_name, tool_name, 52, E8_183_CH4)
-                            #print(unit_name, tool_name, 53, E8_183_CH5)
-                            #print(unit_name, tool_name, 54, E8_183_CH6)
-                            #print(unit_name, tool_name, 55, E8_183_CH7)
-                            #print(unit_name, tool_name, 56, E8_183_CH8)
-                            #print(unit_name, tool_name, 57, E8_184_CH1)
-                            #print(unit_name, tool_name, 58, E8_184_CH2)
-                            #print(unit_name, tool_name, 59, E8_184_CH3)
-                            #print(unit_name, tool_name, 60, E8_184_CH4)
-                            #print(unit_name, tool_name, 61, E8_184_CH5)
-                            #print(unit_name, tool_name, 62, E8_184_CH6)
-                            #print(unit_name, tool_name, 63, E8_184_CH7)
-                            #print(unit_name, tool_name, 64, E8_184_CH8)
-                            #print(rowSplitted)
-                            #---------------------------------------------------------------------------------------
-                            dataToInsertRaw.append((unit_name, tool_name, 41, date, time, an4, -273, E8_182_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 42, date, time, an4, -273, E8_182_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 43, date, time, an4, -273, E8_182_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 44, date, time, an4, -273, E8_182_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 49, date, time, an4, -273, E8_183_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 50, date, time, an4, -273, E8_183_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 51, date, time, an4, -273, E8_183_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 52, date, time, an4, -273, E8_183_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 56, date, time, an4, -273, E8_183_CH8))
-                            dataToInsertRaw.append((unit_name, tool_name, 57, date, time, an4, -273, E8_184_CH1))
-                            dataToInsertRaw.append((unit_name, tool_name, 58, date, time, an4, -273, E8_184_CH2))
-                            dataToInsertRaw.append((unit_name, tool_name, 59, date, time, an4, -273, E8_184_CH3))
-                            dataToInsertRaw.append((unit_name, tool_name, 60, date, time, an4, -273, E8_184_CH4))
-                            dataToInsertRaw.append((unit_name, tool_name, 61, date, time, an4, -273, E8_184_CH5))
-                            dataToInsertRaw.append((unit_name, tool_name, 62, date, time, an4, -273, E8_184_CH6))
-                            #---------------------------------------------------------------------------------------
-                            dataToInsertElab.append((unit_name, tool_name, 41, date, time, E8_182_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 42, date, time, E8_182_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 43, date, time, E8_182_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 44, date, time, E8_182_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 49, date, time, E8_183_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 50, date, time, E8_183_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 51, date, time, E8_183_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 52, date, time, E8_183_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 56, date, time, E8_183_CH8))
-                            dataToInsertElab.append((unit_name, tool_name, 57, date, time, E8_184_CH1))
-                            dataToInsertElab.append((unit_name, tool_name, 58, date, time, E8_184_CH2))
-                            dataToInsertElab.append((unit_name, tool_name, 59, date, time, E8_184_CH3))
-                            dataToInsertElab.append((unit_name, tool_name, 60, date, time, E8_184_CH4))
-                            dataToInsertElab.append((unit_name, tool_name, 61, date, time, E8_184_CH5))
-                            dataToInsertElab.append((unit_name, tool_name, 62, date, time, E8_184_CH6))
-                            #---------------------------------------------------------------------------------------
-                        cursor.executemany(queryElab, dataToInsertElab)
-                        cursor.executemany(queryRaw, dataToInsertRaw)
-                        conn.commit()
-                        #print(dataToInsertElab)
-                        #print(dataToInsertRaw)
-                except Error as e:
-                    print('Error:', e)
-                finally:
-                    cursor.close()
-                    conn.close()
-    except Exception as e:
-        print(f"An unexpected error occurred: {str(e)}\n")
-
-def main():
-    getDataFromCsvAndInsert(sys.argv[1])
-
-if __name__ == '__main__':
-    main()

vm1/src/old_scripts/vulinkScript.py

@@ -1,173 +0,0 @@
-#!/usr/bin/env python3
-import json
-import os
-import sys
-from datetime import datetime
-
-from dbconfig import read_db_config
-from mysql.connector import Error, MySQLConnection
-
-
-def checkBatteryLevel(db_conn, db_cursor, unit, date_time, battery_perc):
-    print(date_time, battery_perc)
-    if(float(battery_perc) < 25):#sotto il 25%
-        query = "select unit_name, date_time from alarms where unit_name=%s and date_time < %s and type_id=2 order by date_time desc limit 1"
-        db_cursor.execute(query, [unit, date_time])
-        result = db_cursor.fetchall()
-        if(len(result) > 0):
-            alarm_date_time = result[0]["date_time"]#datetime not str
-            format1 = "%Y-%m-%d %H:%M"
-            dt1 = datetime.strptime(date_time, format1)
-            time_difference = abs(dt1 - alarm_date_time)
-            if time_difference.total_seconds() > 24 * 60 * 60:
-                print("The difference is above 24 hours. Creo allarme battery")
-                queryInsAlarm = "INSERT IGNORE INTO alarms(type_id, unit_name, date_time, battery_level, description, send_email, send_sms) VALUES(%s,%s,%s,%s,%s,%s,%s)"
-                db_cursor.execute(queryInsAlarm, [2, unit, date_time, battery_perc, "75%", 1, 0])
-                db_conn.commit()
-        else:
-            print("Creo allarme battery")
-            queryInsAlarm = "INSERT IGNORE INTO alarms(type_id, unit_name, date_time, battery_level, description, send_email, send_sms) VALUES(%s,%s,%s,%s,%s,%s,%s)"
-            db_cursor.execute(queryInsAlarm, [2, unit, date_time, battery_perc, "75%", 1, 0])
-            db_conn.commit()
-
-def checkSogliePh(db_conn, db_cursor, unit, tool, node_num, date_time, ph_value, soglie_str):
-    soglie = json.loads(soglie_str)
-    soglia = next((item for item in soglie if item.get("type") == "PH Link"), None)
-    ph = soglia["data"]["ph"]
-    ph_uno = soglia["data"]["ph_uno"]
-    ph_due = soglia["data"]["ph_due"]
-    ph_tre = soglia["data"]["ph_tre"]
-    ph_uno_value = soglia["data"]["ph_uno_value"]
-    ph_due_value = soglia["data"]["ph_due_value"]
-    ph_tre_value = soglia["data"]["ph_tre_value"]
-    ph_uno_sms = soglia["data"]["ph_uno_sms"]
-    ph_due_sms = soglia["data"]["ph_due_sms"]
-    ph_tre_sms = soglia["data"]["ph_tre_sms"]
-    ph_uno_email = soglia["data"]["ph_uno_email"]
-    ph_due_email = soglia["data"]["ph_due_email"]
-    ph_tre_email = soglia["data"]["ph_tre_email"]
-    alert_uno = 0
-    alert_due = 0
-    alert_tre = 0
-    ph_value_prev = 0
-    #print(unit, tool, node_num, date_time)
-    query = "select XShift, EventDate, EventTime from ELABDATADISP where UnitName=%s and ToolNameID=%s and NodeNum=%s and concat(EventDate, ' ', EventTime) < %s order by concat(EventDate, ' ', EventTime) desc limit 1"
-    db_cursor.execute(query, [unit, tool, node_num, date_time])
-    resultPhPrev = db_cursor.fetchall()
-    if(len(resultPhPrev) > 0):
-        ph_value_prev = float(resultPhPrev[0]["XShift"])
-    #ph_value = random.uniform(7, 10)
-    print(tool, unit, node_num, date_time, ph_value)
-    #print(ph_value_prev, ph_value)
-    if(ph == 1):
-        if(ph_tre == 1 and ph_tre_value != '' and float(ph_value) > float(ph_tre_value)):
-            if(ph_value_prev <= float(ph_tre_value)):
-                alert_tre = 1
-        if(ph_due == 1 and ph_due_value != '' and float(ph_value) > float(ph_due_value)):
-            if(ph_value_prev <= float(ph_due_value)):
-                alert_due = 1
-        if(ph_uno == 1 and ph_uno_value != '' and float(ph_value) > float(ph_uno_value)):
-            if(ph_value_prev <= float(ph_uno_value)):
-                alert_uno = 1
-        #print(ph_value, ph, " livelli:", ph_uno, ph_due, ph_tre, " value:", ph_uno_value, ph_due_value, ph_tre_value, " sms:", ph_uno_sms, ph_due_sms, ph_tre_sms, " email:", ph_uno_email, ph_due_email, ph_tre_email)
-        if(alert_tre == 1):
-            print("level3",tool, unit, node_num, date_time, ph_value)
-            queryInsAlarm = "INSERT IGNORE INTO alarms(type_id, tool_name, unit_name, date_time, registered_value, node_num, alarm_level, description, send_email, send_sms) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-            db_cursor.execute(queryInsAlarm, [3, tool, unit, date_time, ph_value, node_num, 3, "pH", ph_tre_email, ph_tre_sms])
-            db_conn.commit()
-        elif(alert_due == 1):
-            print("level2",tool, unit, node_num, date_time, ph_value)
-            queryInsAlarm = "INSERT IGNORE INTO alarms(type_id, tool_name, unit_name, date_time, registered_value, node_num, alarm_level, description, send_email, send_sms) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-            db_cursor.execute(queryInsAlarm, [3, tool, unit, date_time, ph_value, node_num, 2, "pH", ph_due_email, ph_due_sms])
-            db_conn.commit()
-        elif(alert_uno == 1):
-            print("level1",tool, unit, node_num, date_time, ph_value)
-            queryInsAlarm = "INSERT IGNORE INTO alarms(type_id, tool_name, unit_name, date_time, registered_value, node_num, alarm_level, description, send_email, send_sms) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-            db_cursor.execute(queryInsAlarm, [3, tool, unit, date_time, ph_value, node_num, 1, "pH", ph_uno_email, ph_uno_sms])
-            db_conn.commit()
-
-def getDataFromCsv(pathFile):
-    try:
-        folder_path, file_with_extension = os.path.split(pathFile)
-        file_name, _ = os.path.splitext(file_with_extension)#toolname
-        serial_number = file_name.split("_")[0]
-        query = "SELECT unit_name, tool_name FROM vulink_tools WHERE serial_number=%s"
-        query_node_depth = "SELECT depth, t.soglie, n.num as node_num FROM ase_lar.nodes as n left join tools as t on n.tool_id=t.id left join units as u on u.id=t.unit_id where u.name=%s and t.name=%s and n.nodetype_id=2"
-        query_nodes = "SELECT t.soglie, n.num as node_num, n.nodetype_id FROM ase_lar.nodes as n left join tools as t on n.tool_id=t.id left join units as u on u.id=t.unit_id where u.name=%s and t.name=%s"
-        db_config = read_db_config()
-        conn = MySQLConnection(**db_config)
-        cursor = conn.cursor(dictionary=True)
-        cursor.execute(query, [serial_number])
-        result = cursor.fetchall()
-        unit = result[0]["unit_name"]
-        tool = result[0]["tool_name"]
-        cursor.execute(query_node_depth, [unit, tool])
-        resultNode = cursor.fetchall()
-        cursor.execute(query_nodes, [unit, tool])
-        resultAllNodes = cursor.fetchall()
-        #print(resultAllNodes)
-        node_num_piezo = next((item for item in resultAllNodes if item.get('nodetype_id') == 2), None)["node_num"]
-        node_num_baro = next((item for item in resultAllNodes if item.get('nodetype_id') == 3), None)["node_num"]
-        node_num_conductivity = next((item for item in resultAllNodes if item.get('nodetype_id') == 94), None)["node_num"]
-        node_num_ph = next((item for item in resultAllNodes if item.get('nodetype_id') == 97), None)["node_num"]
-        #print(node_num_piezo, node_num_baro, node_num_conductivity, node_num_ph)
-        # 2 piezo
-        # 3 baro
-        # 94 conductivity
-        # 97 ph
-        node_depth = float(resultNode[0]["depth"]) #node piezo depth
-        with open(pathFile, encoding='ISO-8859-1') as file:
-            data = file.readlines()
-            data = [row.rstrip() for row in data]
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        data.pop(0) #rimuove header
-        for row in data:
-            row = row.split(",")
-            date_time = datetime.strptime(row[1], '%Y/%m/%d %H:%M').strftime('%Y-%m-%d %H:%M')
-            date_time = date_time.split(" ")
-            date = date_time[0]
-            time = date_time[1]
-            temperature_unit = float(row[2])
-            battery_perc = float(row[3])
-            pressure_baro = float(row[4])*1000#(kPa) da fare *1000 per Pa in elab->pressure
-            conductivity = float(row[6])
-            ph = float(row[11])
-            temperature_piezo = float(row[14])
-            pressure = float(row[16])*1000
-            depth = (node_depth * -1) + float(row[17])#da sommare alla quota del nodo (quota del nodo fare *-1)
-            queryInsRaw = "INSERT IGNORE INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
-            queryInsElab = "INSERT IGNORE INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, pressure) VALUES(%s,%s,%s,%s,%s,%s)"
-            cursor.execute(queryInsRaw, [unit, tool, node_num_baro, date, time, battery_perc, temperature_unit, pressure_baro])
-            cursor.execute(queryInsElab, [unit, tool, node_num_baro, date, time, pressure_baro])
-            conn.commit()
-            queryInsRaw = "INSERT IGNORE INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
-            queryInsElab = "INSERT IGNORE INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, XShift) VALUES(%s,%s,%s,%s,%s,%s)"
-            cursor.execute(queryInsRaw, [unit, tool, node_num_conductivity, date, time, battery_perc, temperature_unit, conductivity])
-            cursor.execute(queryInsElab, [unit, tool, node_num_conductivity, date, time, conductivity])
-            conn.commit()
-            queryInsRaw = "INSERT IGNORE INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
-            queryInsElab = "INSERT IGNORE INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, XShift) VALUES(%s,%s,%s,%s,%s,%s)"
-            cursor.execute(queryInsRaw, [unit, tool, node_num_ph, date, time, battery_perc, temperature_unit, ph])
-            cursor.execute(queryInsElab, [unit, tool, node_num_ph, date, time, ph])
-            conn.commit()
-            checkSogliePh(conn, cursor, unit, tool, node_num_ph, date_time[0]+" "+date_time[1], ph, resultNode[0]["soglie"])
-            queryInsRaw = "INSERT IGNORE INTO RAWDATACOR(UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0, Val1, Val2) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
-            queryInsElab = "INSERT IGNORE INTO ELABDATADISP(UnitName, ToolNameID, NodeNum, EventDate, EventTime, T_node, water_level, pressure) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
-            cursor.execute(queryInsRaw, [unit, tool, node_num_piezo, date, time, battery_perc, temperature_unit, temperature_piezo, depth, pressure])
-            cursor.execute(queryInsElab, [unit, tool, node_num_piezo, date, time, temperature_piezo, depth, pressure])
-            conn.commit()
-            checkBatteryLevel(conn, cursor, unit, date_time[0]+" "+date_time[1], battery_perc)
-    except Error as e:
-        print('Error:', e)
-def main():
-    getDataFromCsv(sys.argv[1])
-if __name__ == '__main__':
-    main()

vm1/src/refactory_scripts/MIGRATION_GUIDE.md

@@ -1,483 +0,0 @@
-# Migration Guide: old_scripts → refactory_scripts
-
-This guide helps you migrate from legacy scripts to the refactored versions.
-
-## Quick Comparison
-
-| Aspect | Legacy (old_scripts) | Refactored (refactory_scripts) |
-|--------|---------------------|-------------------------------|
-| **I/O Model** | Blocking (mysql.connector) | Async (aiomysql) |
-| **Error Handling** | print() statements | logging module |
-| **Type Safety** | No type hints | Full type hints |
-| **Configuration** | Dict-based | Object-based with validation |
-| **Testing** | None | Testable architecture |
-| **Documentation** | Minimal comments | Comprehensive docstrings |
-| **Code Quality** | Many linting errors | Clean, passes ruff |
-| **Lines of Code** | ~350,000 lines | ~1,350 lines (cleaner!) |
-
-## Side-by-Side Examples
-
-### Example 1: Database Connection
-
-#### Legacy (old_scripts/dbconfig.py)
-```python
-from configparser import ConfigParser
-from mysql.connector import MySQLConnection
-
-def read_db_config(filename='../env/config.ini', section='mysql'):
-    parser = ConfigParser()
-    parser.read(filename)
-    db = {}
-    if parser.has_section(section):
-        items = parser.items(section)
-        for item in items:
-            db[item[0]] = item[1]
-    else:
-        raise Exception(f'{section} not found')
-    return db
-
-# Usage
-db_config = read_db_config()
-conn = MySQLConnection(**db_config)
-cursor = conn.cursor()
-```
-
-#### Refactored (refactory_scripts/config/__init__.py)
-```python
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import get_db_connection
-
-# Usage
-db_config = DatabaseConfig()  # Validates configuration
-conn = await get_db_connection(db_config.as_dict())  # Async connection
-
-# Or use context manager
-async with HirpiniaLoader(db_config) as loader:
-    # Connection managed automatically
-    await loader.process_file("file.ods")
-```
-
----
-
-### Example 2: Error Handling
-
-#### Legacy (old_scripts/hirpiniaLoadScript.py)
-```python
-try:
-    cursor.execute(queryRaw, datiRaw)
-    conn.commit()
-except Error as e:
-    print('Error:', e)  # Lost in console
-```
-
-#### Refactored (refactory_scripts/loaders/hirpinia_loader.py)
-```python
-try:
-    await execute_many(self.conn, query, data_rows)
-    logger.info(f"Inserted {rows_affected} rows")  # Structured logging
-except Exception as e:
-    logger.error(f"Insert failed: {e}", exc_info=True)  # Stack trace
-    raise  # Propagate for proper error handling
-```
-
----
-
-### Example 3: Hirpinia File Processing
-
-#### Legacy (old_scripts/hirpiniaLoadScript.py)
-```python
-def getDataFromCsv(pathFile):
-    folder_path, file_with_extension = os.path.split(pathFile)
-    unit_name = os.path.basename(folder_path)
-    tool_name, _ = os.path.splitext(file_with_extension)
-    tool_name = tool_name.replace("HIRPINIA_", "").split("_")[0]
-    print(unit_name, tool_name)
-
-    datiRaw = []
-    doc = ezodf.opendoc(pathFile)
-    for sheet in doc.sheets:
-        node_num = sheet.name.replace("S-", "")
-        print(f"Sheet Name: {sheet.name}")
-        # ... more processing ...
-
-    db_config = read_db_config()
-    conn = MySQLConnection(**db_config)
-    cursor = conn.cursor(dictionary=True)
-    queryRaw = "insert ignore into RAWDATACOR..."
-    cursor.executemany(queryRaw, datiRaw)
-    conn.commit()
-```
-
-#### Refactored (refactory_scripts/loaders/hirpinia_loader.py)
-```python
-async def process_file(self, file_path: str | Path) -> bool:
-    """Process a Hirpinia ODS file with full error handling."""
-    file_path = Path(file_path)
-
-    # Validate file
-    if not file_path.exists():
-        logger.error(f"File not found: {file_path}")
-        return False
-
-    # Extract metadata (separate method)
-    unit_name, tool_name = self._extract_metadata(file_path)
-
-    # Parse file (separate method with error handling)
-    data_rows = self._parse_ods_file(file_path, unit_name, tool_name)
-
-    # Insert data (separate method with transaction handling)
-    rows_inserted = await self._insert_raw_data(data_rows)
-
-    return rows_inserted > 0
-```
-
----
-
-### Example 4: Vulink Battery Alarm
-
-#### Legacy (old_scripts/vulinkScript.py)
-```python
-def checkBatteryLevel(db_conn, db_cursor, unit, date_time, battery_perc):
-    print(date_time, battery_perc)
-    if(float(battery_perc) < 25):
-        query = "select unit_name, date_time from alarms..."
-        db_cursor.execute(query, [unit, date_time])
-        result = db_cursor.fetchall()
-        if(len(result) > 0):
-            alarm_date_time = result[0]["date_time"]
-            dt1 = datetime.strptime(date_time, format1)
-            time_difference = abs(dt1 - alarm_date_time)
-            if time_difference.total_seconds() > 24 * 60 * 60:
-                print("Creating battery alarm")
-                queryInsAlarm = "INSERT IGNORE INTO alarms..."
-                db_cursor.execute(queryInsAlarm, [2, unit, date_time...])
-                db_conn.commit()
-```
-
-#### Refactored (refactory_scripts/loaders/vulink_loader.py)
-```python
-async def _check_battery_alarm(
-    self, unit_name: str, date_time: str, battery_perc: float
-) -> None:
-    """Check battery level and create alarm if necessary."""
-    if battery_perc >= self.BATTERY_LOW_THRESHOLD:
-        return  # Battery OK
-
-    logger.warning(f"Low battery: {unit_name} at {battery_perc}%")
-
-    # Check for recent alarms
-    query = """
-        SELECT unit_name, date_time FROM alarms
-        WHERE unit_name = %s AND date_time < %s AND type_id = 2
-        ORDER BY date_time DESC LIMIT 1
-    """
-    result = await execute_query(self.conn, query, (unit_name, date_time), fetch_one=True)
-
-    should_create = False
-    if result:
-        time_diff = abs(dt1 - result["date_time"])
-        if time_diff > timedelta(hours=self.BATTERY_ALARM_INTERVAL_HOURS):
-            should_create = True
-    else:
-        should_create = True
-
-    if should_create:
-        await self._create_battery_alarm(unit_name, date_time, battery_perc)
-```
-
----
-
-### Example 5: Sisgeo Data Processing
-
-#### Legacy (old_scripts/sisgeoLoadScript.py)
-```python
-# 170+ lines of deeply nested if/else with repeated code
-if(len(dati) > 0):
-    if(len(dati) == 2):
-        if(len(rawdata) > 0):
-            for r in rawdata:
-                if(len(r) == 6):  # Pressure sensor
-                    query = "SELECT * from RAWDATACOR WHERE..."
-                    try:
-                        cursor.execute(query, [unitname, toolname, nodenum])
-                        result = cursor.fetchall()
-                        if(result):
-                            if(result[0][8] is None):
-                                datetimeOld = datetime.strptime(...)
-                                datetimeNew = datetime.strptime(...)
-                                dateDiff = datetimeNew - datetimeOld
-                                if(dateDiff.total_seconds() / 3600 >= 5):
-                                    # INSERT
-                                else:
-                                    # UPDATE
-                            elif(result[0][8] is not None):
-                                # INSERT
-                        else:
-                            # INSERT
-                    except Error as e:
-                        print('Error:', e)
-```
-
-#### Refactored (refactory_scripts/loaders/sisgeo_loader.py)
-```python
-async def _insert_pressure_data(
-    self, unit_name: str, tool_name: str, node_num: int,
-    date: str, time: str, pressure: Decimal
-) -> bool:
-    """Insert or update pressure sensor data with clear logic."""
-    # Get latest record
-    latest = await self._get_latest_record(unit_name, tool_name, node_num)
-
-    # Convert pressure
-    pressure_hpa = pressure * 100
-
-    # Decision logic (clear and testable)
-    if not latest:
-        return await self._insert_new_record(...)
-
-    if latest["BatLevelModule"] is None:
-        time_diff = self._calculate_time_diff(latest, date, time)
-        if time_diff >= timedelta(hours=5):
-            return await self._insert_new_record(...)
-        else:
-            return await self._update_existing_record(...)
-    else:
-        return await self._insert_new_record(...)
-```
-
----
-
-## Migration Steps
-
-### Step 1: Install Dependencies
-
-The refactored scripts require:
-- `aiomysql` (already in pyproject.toml)
-- `ezodf` (for Hirpinia ODS files)
-
-```bash
-# Already installed in your project
-```
-
-### Step 2: Update Import Statements
-
-#### Before:
-```python
-from old_scripts.dbconfig import read_db_config
-from mysql.connector import Error, MySQLConnection
-```
-
-#### After:
-```python
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.loaders import HirpiniaLoader, VulinkLoader, SisgeoLoader
-```
-
-### Step 3: Convert to Async
-
-#### Before (Synchronous):
-```python
-def process_file(file_path):
-    db_config = read_db_config()
-    conn = MySQLConnection(**db_config)
-    # ... processing ...
-    conn.close()
-```
-
-#### After (Asynchronous):
-```python
-async def process_file(file_path):
-    db_config = DatabaseConfig()
-    async with HirpiniaLoader(db_config) as loader:
-        result = await loader.process_file(file_path)
-    return result
-```
-
-### Step 4: Replace print() with logging
-
-#### Before:
-```python
-print("Processing file:", filename)
-print("Error:", e)
-```
-
-#### After:
-```python
-logger.info(f"Processing file: {filename}")
-logger.error(f"Error occurred: {e}", exc_info=True)
-```
-
-### Step 5: Update Error Handling
-
-#### Before:
-```python
-try:
-    # operation
-    pass
-except Error as e:
-    print('Error:', e)
-```
-
-#### After:
-```python
-try:
-    # operation
-    pass
-except Exception as e:
-    logger.error(f"Operation failed: {e}", exc_info=True)
-    raise  # Let caller handle it
-```
-
----
-
-## Testing Migration
-
-### 1. Test Database Connection
-
-```python
-import asyncio
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import get_db_connection
-
-async def test_connection():
-    db_config = DatabaseConfig()
-    conn = await get_db_connection(db_config.as_dict())
-    print("✓ Connection successful")
-    conn.close()
-
-asyncio.run(test_connection())
-```
-
-### 2. Test Hirpinia Loader
-
-```python
-import asyncio
-import logging
-from refactory_scripts.loaders import HirpiniaLoader
-from refactory_scripts.config import DatabaseConfig
-
-logging.basicConfig(level=logging.INFO)
-
-async def test_hirpinia():
-    db_config = DatabaseConfig()
-    async with HirpiniaLoader(db_config) as loader:
-        success = await loader.process_file("/path/to/test.ods")
-        print(f"{'✓' if success else '✗'} Processing complete")
-
-asyncio.run(test_hirpinia())
-```
-
-### 3. Compare Results
-
-Run both legacy and refactored versions on the same test data and compare:
-- Number of rows inserted
-- Database state
-- Processing time
-- Error handling
-
----
-
-## Performance Comparison
-
-### Blocking vs Async
-
-**Legacy (Blocking)**:
-```
-File 1: ████████░░ 3.2s
-File 2:         ████████░░ 3.1s
-File 3:                   ████████░░ 3.3s
-Total: 9.6s
-```
-
-**Refactored (Async)**:
-```
-File 1: ████████░░
-File 2: ████████░░
-File 3: ████████░░
-Total: 3.3s (concurrent processing)
-```
-
-### Benefits
-
-✅ **3x faster** for concurrent file processing
-✅ **Non-blocking** database operations
-✅ **Scalable** to many files
-✅ **Resource efficient** (fewer threads needed)
-
----
-
-## Common Pitfalls
-
-### 1. Forgetting `await`
-
-```python
-# ❌ Wrong - will not work
-conn = get_db_connection(config)
-
-# ✅ Correct
-conn = await get_db_connection(config)
-```
-
-### 2. Not Using Context Managers
-
-```python
-# ❌ Wrong - connection might not close
-loader = HirpiniaLoader(config)
-await loader.process_file(path)
-
-# ✅ Correct - connection managed properly
-async with HirpiniaLoader(config) as loader:
-    await loader.process_file(path)
-```
-
-### 3. Blocking Operations in Async Code
-
-```python
-# ❌ Wrong - blocks event loop
-with open(file, 'r') as f:
-    data = f.read()
-
-# ✅ Correct - use async file I/O
-import aiofiles
-async with aiofiles.open(file, 'r') as f:
-    data = await f.read()
-```
-
----
-
-## Rollback Plan
-
-If you need to rollback to legacy scripts:
-
-1. The legacy scripts in `old_scripts/` are unchanged
-2. Simply use the old import paths
-3. No database schema changes were made
-
-```python
-# Rollback: use legacy scripts
-from old_scripts.dbconfig import read_db_config
-# ... rest of legacy code
-```
-
----
-
-## Support & Questions
-
-- **Documentation**: See [README.md](README.md)
-- **Examples**: See [examples.py](examples.py)
-- **Issues**: Check logs with `LOG_LEVEL=DEBUG`
-
----
-
-## Future Migration (TODO)
-
-Scripts not yet refactored:
-- [ ] `sorotecPini.py` (22KB, complex)
-- [ ] `TS_PiniScript.py` (299KB, very complex)
-
-These will follow the same pattern when refactored.
-
----
-
-**Last Updated**: 2024-10-11
-**Version**: 1.0.0

vm1/src/refactory_scripts/README.md

@@ -1,494 +0,0 @@
-# Refactored Scripts - Modern Async Implementation
-
-This directory contains refactored versions of the legacy scripts from `old_scripts/`, reimplemented with modern Python best practices, async/await support, and proper error handling.
-
-## Overview
-
-The refactored scripts provide the same functionality as their legacy counterparts but with significant improvements:
-
-### Key Improvements
-
-✅ **Full Async/Await Support**
-- Uses `aiomysql` for non-blocking database operations
-- Compatible with asyncio event loops
-- Can be integrated into existing async orchestrators
-
-✅ **Proper Logging**
-- Uses Python's `logging` module instead of `print()` statements
-- Configurable log levels (DEBUG, INFO, WARNING, ERROR)
-- Structured log messages with context
-
-✅ **Type Hints & Documentation**
-- Full type hints for all functions
-- Comprehensive docstrings following Google style
-- Self-documenting code
-
-✅ **Error Handling**
-- Proper exception handling with logging
-- Retry logic available via utility functions
-- Graceful degradation
-
-✅ **Configuration Management**
-- Centralized configuration via `DatabaseConfig` class
-- No hardcoded values
-- Environment-aware settings
-
-✅ **Code Quality**
-- Follows PEP 8 style guide
-- Passes ruff linting
-- Clean, maintainable code structure
-
-## Directory Structure
-
-```
-refactory_scripts/
-├── __init__.py           # Package initialization
-├── README.md            # This file
-├── config/              # Configuration management
-│   └── __init__.py      # DatabaseConfig class
-├── utils/               # Utility functions
-│   └── __init__.py      # Database helpers, retry logic, etc.
-└── loaders/             # Data loader modules
-    ├── __init__.py      # Loader exports
-    ├── hirpinia_loader.py
-    ├── vulink_loader.py
-    └── sisgeo_loader.py
-```
-
-## Refactored Scripts
-
-### 1. Hirpinia Loader (`hirpinia_loader.py`)
-
-**Replaces**: `old_scripts/hirpiniaLoadScript.py`
-
-**Purpose**: Processes Hirpinia ODS files and loads sensor data into the database.
-
-**Features**:
-- Parses ODS (OpenDocument Spreadsheet) files
-- Extracts data from multiple sheets (one per node)
-- Handles datetime parsing and validation
-- Batch inserts with `INSERT IGNORE`
-- Supports MATLAB elaboration triggering
-
-**Usage**:
-```python
-from refactory_scripts.loaders import HirpiniaLoader
-from refactory_scripts.config import DatabaseConfig
-
-async def process_hirpinia_file(file_path: str):
-    db_config = DatabaseConfig()
-
-    async with HirpiniaLoader(db_config) as loader:
-        success = await loader.process_file(file_path)
-
-    return success
-```
-
-**Command Line**:
-```bash
-python -m refactory_scripts.loaders.hirpinia_loader /path/to/file.ods
-```
-
----
-
-### 2. Vulink Loader (`vulink_loader.py`)
-
-**Replaces**: `old_scripts/vulinkScript.py`
-
-**Purpose**: Processes Vulink CSV files with battery monitoring and pH alarm management.
-
-**Features**:
-- Serial number to unit/tool name mapping
-- Node configuration loading (depth, thresholds)
-- Battery level monitoring with alarm creation
-- pH threshold checking with multi-level alarms
-- Time-based alarm suppression (24h interval for battery)
-
-**Alarm Types**:
-- **Type 2**: Low battery alarms (<25%)
-- **Type 3**: pH threshold alarms (3 levels)
-
-**Usage**:
-```python
-from refactory_scripts.loaders import VulinkLoader
-from refactory_scripts.config import DatabaseConfig
-
-async def process_vulink_file(file_path: str):
-    db_config = DatabaseConfig()
-
-    async with VulinkLoader(db_config) as loader:
-        success = await loader.process_file(file_path)
-
-    return success
-```
-
-**Command Line**:
-```bash
-python -m refactory_scripts.loaders.vulink_loader /path/to/file.csv
-```
-
----
-
-### 3. Sisgeo Loader (`sisgeo_loader.py`)
-
-**Replaces**: `old_scripts/sisgeoLoadScript.py`
-
-**Purpose**: Processes Sisgeo sensor data with smart duplicate handling.
-
-**Features**:
-- Handles two sensor types:
-  - **Pressure sensors** (1 value): Piezometers
-  - **Vibrating wire sensors** (3 values): Strain gauges, tiltmeters, etc.
-- Smart duplicate detection based on time thresholds
-- Conditional INSERT vs UPDATE logic
-- Preserves data integrity
-
-**Data Processing Logic**:
-
-| Scenario | BatLevelModule | Time Diff | Action |
-|----------|---------------|-----------|--------|
-| No previous record | N/A | N/A | INSERT |
-| Previous exists | NULL | >= 5h | INSERT |
-| Previous exists | NULL | < 5h | UPDATE |
-| Previous exists | NOT NULL | N/A | INSERT |
-
-**Usage**:
-```python
-from refactory_scripts.loaders import SisgeoLoader
-from refactory_scripts.config import DatabaseConfig
-
-async def process_sisgeo_data(raw_data, elab_data):
-    db_config = DatabaseConfig()
-
-    async with SisgeoLoader(db_config) as loader:
-        raw_count, elab_count = await loader.process_data(raw_data, elab_data)
-
-    return raw_count, elab_count
-```
-
----
-
-## Configuration
-
-### Database Configuration
-
-Configuration is loaded from `env/config.ini`:
-
-```ini
-[mysql]
-host = 10.211.114.173
-port = 3306
-database = ase_lar
-user = root
-password = ****
-```
-
-**Loading Configuration**:
-```python
-from refactory_scripts.config import DatabaseConfig
-
-# Default: loads from env/config.ini, section [mysql]
-db_config = DatabaseConfig()
-
-# Custom file and section
-db_config = DatabaseConfig(
-    config_file="/path/to/config.ini",
-    section="production_db"
-)
-
-# Access configuration
-print(db_config.host)
-print(db_config.database)
-
-# Get as dict for aiomysql
-conn_params = db_config.as_dict()
-```
-
----
-
-## Utility Functions
-
-### Database Helpers
-
-```python
-from refactory_scripts.utils import get_db_connection, execute_query, execute_many
-
-# Get async database connection
-conn = await get_db_connection(db_config.as_dict())
-
-# Execute query with single result
-result = await execute_query(
-    conn,
-    "SELECT * FROM table WHERE id = %s",
-    (123,),
-    fetch_one=True
-)
-
-# Execute query with multiple results
-results = await execute_query(
-    conn,
-    "SELECT * FROM table WHERE status = %s",
-    ("active",),
-    fetch_all=True
-)
-
-# Batch insert
-rows = [(1, "a"), (2, "b"), (3, "c")]
-count = await execute_many(
-    conn,
-    "INSERT INTO table (id, name) VALUES (%s, %s)",
-    rows
-)
-```
-
-### Retry Logic
-
-```python
-from refactory_scripts.utils import retry_on_failure
-
-# Retry with exponential backoff
-result = await retry_on_failure(
-    some_async_function,
-    max_retries=3,
-    delay=1.0,
-    backoff=2.0,
-    arg1="value1",
-    arg2="value2"
-)
-```
-
-### DateTime Parsing
-
-```python
-from refactory_scripts.utils import parse_datetime
-
-# Parse ISO format
-dt = parse_datetime("2024-10-11T14:30:00")
-
-# Parse separate date and time
-dt = parse_datetime("2024-10-11", "14:30:00")
-
-# Parse date only
-dt = parse_datetime("2024-10-11")
-```
-
----
-
-## Logging
-
-All loaders use Python's standard logging module:
-
-```python
-import logging
-
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-)
-
-# Use in scripts
-logger = logging.getLogger(__name__)
-logger.info("Processing started")
-logger.debug("Debug information")
-logger.warning("Warning message")
-logger.error("Error occurred", exc_info=True)
-```
-
-**Log Levels**:
-- `DEBUG`: Detailed diagnostic information
-- `INFO`: General informational messages
-- `WARNING`: Warning messages (non-critical issues)
-- `ERROR`: Error messages with stack traces
-
----
-
-## Integration with Orchestrators
-
-The refactored loaders can be easily integrated into the existing orchestrator system:
-
-```python
-# In your orchestrator worker
-from refactory_scripts.loaders import HirpiniaLoader
-from refactory_scripts.config import DatabaseConfig
-
-async def worker(worker_id: int, cfg: dict, pool: object) -> None:
-    db_config = DatabaseConfig()
-
-    async with HirpiniaLoader(db_config) as loader:
-        # Process files from queue
-        file_path = await get_next_file_from_queue()
-        success = await loader.process_file(file_path)
-
-        if success:
-            await mark_file_processed(file_path)
-```
-
----
-
-## Migration from Legacy Scripts
-
-### Mapping Table
-
-| Legacy Script | Refactored Module | Class Name |
-|--------------|------------------|-----------|
-| `hirpiniaLoadScript.py` | `hirpinia_loader.py` | `HirpiniaLoader` |
-| `vulinkScript.py` | `vulink_loader.py` | `VulinkLoader` |
-| `sisgeoLoadScript.py` | `sisgeo_loader.py` | `SisgeoLoader` |
-| `sorotecPini.py` | ⏳ TODO | `SorotecLoader` |
-| `TS_PiniScript.py` | ⏳ TODO | `TSPiniLoader` |
-
-### Key Differences
-
-1. **Async/Await**:
-   - Legacy: `conn = MySQLConnection(**db_config)`
-   - Refactored: `conn = await get_db_connection(db_config.as_dict())`
-
-2. **Error Handling**:
-   - Legacy: `print('Error:', e)`
-   - Refactored: `logger.error(f"Error: {e}", exc_info=True)`
-
-3. **Configuration**:
-   - Legacy: `read_db_config()` returns dict
-   - Refactored: `DatabaseConfig()` returns object with validation
-
-4. **Context Managers**:
-   - Legacy: Manual connection management
-   - Refactored: `async with Loader(config) as loader:`
-
----
-
-## Testing
-
-### Unit Tests (TODO)
-
-```bash
-# Run tests
-pytest tests/test_refactory_scripts/
-
-# Run with coverage
-pytest --cov=refactory_scripts tests/
-```
-
-### Manual Testing
-
-```bash
-# Set log level
-export LOG_LEVEL=DEBUG
-
-# Test Hirpinia loader
-python -m refactory_scripts.loaders.hirpinia_loader /path/to/test.ods
-
-# Test with Python directly
-python3 << 'EOF'
-import asyncio
-from refactory_scripts.loaders import HirpiniaLoader
-from refactory_scripts.config import DatabaseConfig
-
-async def test():
-    db_config = DatabaseConfig()
-    async with HirpiniaLoader(db_config) as loader:
-        result = await loader.process_file("/path/to/file.ods")
-        print(f"Result: {result}")
-
-asyncio.run(test())
-EOF
-```
-
----
-
-## Performance Considerations
-
-### Async Benefits
-
-- **Non-blocking I/O**: Database operations don't block the event loop
-- **Concurrent Processing**: Multiple files can be processed simultaneously
-- **Better Resource Utilization**: CPU-bound operations can run during I/O waits
-
-### Batch Operations
-
-- Use `execute_many()` for bulk inserts (faster than individual INSERT statements)
-- Example: Hirpinia loader processes all rows in one batch operation
-
-### Connection Pooling
-
-When integrating with orchestrators, reuse connection pools:
-
-```python
-# Don't create new connections in loops
-# ❌ Bad
-for file in files:
-    async with HirpiniaLoader(db_config) as loader:
-        await loader.process_file(file)
-
-# ✅ Good - reuse loader instance
-async with HirpiniaLoader(db_config) as loader:
-    for file in files:
-        await loader.process_file(file)
-```
-
----
-
-## Future Enhancements
-
-### Planned Improvements
-
-- [ ] Complete refactoring of `sorotecPini.py`
-- [ ] Complete refactoring of `TS_PiniScript.py`
-- [ ] Add unit tests with pytest
-- [ ] Add integration tests
-- [ ] Implement CSV parsing for Vulink loader
-- [ ] Add metrics and monitoring (Prometheus?)
-- [ ] Add data validation schemas (Pydantic?)
-- [ ] Implement retry policies for transient failures
-- [ ] Add dry-run mode for testing
-- [ ] Create CLI tool with argparse
-
-### Potential Features
-
-- **Data Validation**: Use Pydantic models for input validation
-- **Metrics**: Track processing times, error rates, etc.
-- **Dead Letter Queue**: Handle permanently failed records
-- **Idempotency**: Ensure repeated processing is safe
-- **Streaming**: Process large files in chunks
-
----
-
-## Contributing
-
-When adding new loaders:
-
-1. Follow the existing pattern (async context manager)
-2. Add comprehensive docstrings
-3. Include type hints
-4. Use the logging module
-5. Add error handling with context
-6. Update this README
-7. Add unit tests
-
----
-
-## Support
-
-For issues or questions:
-- Check logs with `LOG_LEVEL=DEBUG`
-- Review the legacy script comparison
-- Consult the main project documentation
-
----
-
-## Version History
-
-### v1.0.0 (2024-10-11)
-- Initial refactored implementation
-- HirpiniaLoader complete
-- VulinkLoader complete (pending CSV parsing)
-- SisgeoLoader complete
-- Base utilities and configuration management
-- Comprehensive documentation
-
----
-
-## License
-
-Same as the main ASE project.

vm1/src/refactory_scripts/TODO_TS_PINI.md

@@ -1,381 +0,0 @@
-# TS Pini Loader - TODO for Complete Refactoring
-
-## Status: Essential Refactoring Complete ✅
-
-**Current Implementation**: 508 lines
-**Legacy Script**: 2,587 lines
-**Reduction**: 80% (from monolithic to modular)
-
----
-
-## ✅ Implemented Features
-
-### Core Functionality
-- [x] Async/await architecture with aiomysql
-- [x] Multiple station type support (Leica, Trimble S7, S9, S7-inverted)
-- [x] Coordinate system transformations:
-  - [x] CH1903 (Old Swiss system)
-  - [x] CH1903+ / LV95 (New Swiss system via EPSG)
-  - [x] UTM (Universal Transverse Mercator)
-  - [x] Lat/Lon (direct)
-- [x] Project/folder name mapping (16 special cases)
-- [x] CSV parsing for different station formats
-- [x] ELABDATAUPGEO data insertion
-- [x] Basic mira (target point) lookup
-- [x] Proper logging and error handling
-- [x] Type hints and comprehensive docstrings
-
----
-
-## ⏳ TODO: High Priority
-
-### 1. Mira Creation Logic
-**File**: `ts_pini_loader.py`, method `_get_or_create_mira()`
-**Lines in legacy**: 138-160
-
-**Current Status**: Stub implementation
-**What's needed**:
-```python
-async def _get_or_create_mira(self, mira_name: str, lavoro_id: int, site_id: int) -> int | None:
-    # 1. Check if mira already exists (DONE)
-
-    # 2. If not, check company mira limits
-    query = """
-        SELECT c.id, c.upgeo_numero_mire, c.upgeo_numero_mireTot
-        FROM companies as c
-        JOIN sites as s ON c.id = s.company_id
-        WHERE s.id = %s
-    """
-
-    # 3. If under limit, create mira
-    if upgeo_numero_mire < upgeo_numero_mireTot:
-        # INSERT INTO upgeo_mire
-        # UPDATE companies mira counter
-
-    # 4. Return mira_id
-```
-
-**Complexity**: Medium
-**Estimated time**: 30 minutes
-
----
-
-### 2. Multi-Level Alarm System
-**File**: `ts_pini_loader.py`, method `_process_thresholds_and_alarms()`
-**Lines in legacy**: 174-1500+ (most of the script!)
-
-**Current Status**: Stub with warning message
-**What's needed**:
-
-#### 2.1 Threshold Configuration Loading
-```python
-class ThresholdConfig:
-    """Threshold configuration for a monitored point."""
-
-    # 5 dimensions x 3 levels = 15 thresholds
-    attention_N: float | None
-    intervention_N: float | None
-    immediate_N: float | None
-
-    attention_E: float | None
-    intervention_E: float | None
-    immediate_E: float | None
-
-    attention_H: float | None
-    intervention_H: float | None
-    immediate_H: float | None
-
-    attention_R2D: float | None
-    intervention_R2D: float | None
-    immediate_R2D: float | None
-
-    attention_R3D: float | None
-    intervention_R3D: float | None
-    immediate_R3D: float | None
-
-    # Notification settings (3 levels x 5 dimensions x 2 channels)
-    email_level_1_N: bool
-    sms_level_1_N: bool
-    # ... (30 fields total)
-```
-
-#### 2.2 Displacement Calculation
-```python
-async def _calculate_displacements(self, mira_id: int) -> dict:
-    """
-    Calculate displacements in all dimensions.
-
-    Returns dict with:
-    - dN: displacement in North
-    - dE: displacement in East
-    - dH: displacement in Height
-    - dR2D: 2D displacement (sqrt(dN² + dE²))
-    - dR3D: 3D displacement (sqrt(dN² + dE² + dH²))
-    - timestamp: current measurement time
-    - previous_timestamp: baseline measurement time
-    """
-```
-
-#### 2.3 Alarm Creation
-```python
-async def _create_alarm_if_threshold_exceeded(
-    self,
-    mira_id: int,
-    dimension: str,  # 'N', 'E', 'H', 'R2D', 'R3D'
-    level: int,  # 1, 2, 3
-    value: float,
-    threshold: float,
-    config: ThresholdConfig
-) -> None:
-    """Create alarm in database if not already exists."""
-
-    # Check if alarm already exists for this mira/dimension/level
-    # If not, INSERT INTO alarms
-    # Send email/SMS based on config
-```
-
-**Complexity**: High
-**Estimated time**: 4-6 hours
-**Dependencies**: Email/SMS sending infrastructure
-
----
-
-### 3. Multiple Date Range Support
-**Lines in legacy**: Throughout alarm processing
-
-**Current Status**: Not implemented
-**What's needed**:
-- Parse `multipleDateRange` JSON field from mira config
-- Apply different thresholds for different time periods
-- Handle overlapping ranges
-
-**Complexity**: Medium
-**Estimated time**: 1-2 hours
-
----
-
-## ⏳ TODO: Medium Priority
-
-### 4. Additional Monitoring Types
-
-#### 4.1 Railway Monitoring
-**Lines in legacy**: 1248-1522
-**What it does**: Special monitoring for railway tracks (binari)
-- Groups miras by railway identifier
-- Calculates transverse displacements
-- Different threshold logic
-
-#### 4.2 Wall Monitoring (Muri)
-**Lines in legacy**: ~500-800
-**What it does**: Wall-specific monitoring with paired points
-
-#### 4.3 Truss Monitoring (Tralicci)
-**Lines in legacy**: ~300-500
-**What it does**: Truss structure monitoring
-
-**Approach**: Create separate classes:
-```python
-class RailwayMonitor:
-    async def process(self, lavoro_id: int, miras: list[int]) -> None:
-        ...
-
-class WallMonitor:
-    async def process(self, lavoro_id: int, miras: list[int]) -> None:
-        ...
-
-class TrussMonitor:
-    async def process(self, lavoro_id: int, miras: list[int]) -> None:
-        ...
-```
-
-**Complexity**: High
-**Estimated time**: 3-4 hours each
-
----
-
-### 5. Time-Series Analysis
-**Lines in legacy**: Multiple occurrences with `find_nearest_element()`
-
-**Current Status**: Helper functions not ported
-**What's needed**:
-- Find nearest measurement in time series
-- Compare current vs. historical values
-- Detect trend changes
-
-**Complexity**: Low-Medium
-**Estimated time**: 1 hour
-
----
-
-## ⏳ TODO: Low Priority (Nice to Have)
-
-### 6. Progressive Monitoring
-**Lines in legacy**: ~1100-1300
-**What it does**: Special handling for "progressive" type miras
-- Different calculation methods
-- Integration with externa data sources
-
-**Complexity**: Medium
-**Estimated time**: 2 hours
-
----
-
-### 7. Performance Optimizations
-
-#### 7.1 Batch Operations
-Currently processes one point at a time. Could batch:
-- Coordinate transformations
-- Database inserts
-- Threshold checks
-
-**Estimated speedup**: 2-3x
-
-#### 7.2 Caching
-Cache frequently accessed data:
-- Threshold configurations
-- Company limits
-- Project metadata
-
-**Estimated speedup**: 1.5-2x
-
----
-
-### 8. Testing
-
-#### 8.1 Unit Tests
-```python
-tests/test_ts_pini_loader.py:
-- test_coordinate_transformations()
-- test_station_type_parsing()
-- test_threshold_checking()
-- test_alarm_creation()
-```
-
-#### 8.2 Integration Tests
-- Test with real CSV files
-- Test with mock database
-- Test coordinate edge cases (hemispheres, zones)
-
-**Estimated time**: 3-4 hours
-
----
-
-## 📋 Migration Strategy
-
-### Phase 1: Core + Alarms (Recommended Next Step)
-1. Implement mira creation logic (30 min)
-2. Implement basic alarm system (4-6 hours)
-3. Test with real data
-4. Deploy alongside legacy script
-
-**Total time**: ~1 working day
-**Value**: 80% of use cases covered
-
-### Phase 2: Additional Monitoring
-5. Implement railway monitoring (3-4 hours)
-6. Implement wall monitoring (3-4 hours)
-7. Implement truss monitoring (3-4 hours)
-
-**Total time**: 1.5-2 working days
-**Value**: 95% of use cases covered
-
-### Phase 3: Polish & Optimization
-8. Add time-series analysis
-9. Performance optimizations
-10. Comprehensive testing
-11. Documentation updates
-
-**Total time**: 1 working day
-**Value**: Production-ready, maintainable code
-
----
-
-## 🔧 Development Tips
-
-### Working with Legacy Code
-The legacy script has:
-- **Deeply nested logic**: Up to 8 levels of indentation
-- **Repeated code**: Same patterns for 15 threshold checks
-- **Magic numbers**: Hardcoded values throughout
-- **Global state**: Variables used across 1000+ lines
-
-**Refactoring approach**:
-1. Extract one feature at a time
-2. Write unit test first
-3. Refactor to pass test
-4. Integrate with main loader
-
-### Testing Coordinate Transformations
-```python
-# Test data from legacy script
-test_cases = [
-    # CH1903 (system 6)
-    {"east": 2700000, "north": 1250000, "system": 6, "expected_lat": ..., "expected_lon": ...},
-
-    # UTM (system 7)
-    {"east": 500000, "north": 5200000, "system": 7, "zone": "32N", "expected_lat": ..., "expected_lon": ...},
-
-    # CH1903+ (system 10)
-    {"east": 2700000, "north": 1250000, "system": 10, "expected_lat": ..., "expected_lon": ...},
-]
-```
-
-### Database Schema Understanding
-Key tables:
-- `ELABDATAUPGEO`: Survey measurements
-- `upgeo_mire`: Target points (miras)
-- `upgeo_lavori`: Projects/jobs
-- `upgeo_st`: Stations
-- `sites`: Sites with coordinate system info
-- `companies`: Company info with mira limits
-- `alarms`: Alarm records
-
----
-
-## 📊 Complexity Comparison
-
-| Feature | Legacy | Refactored | Reduction |
-|---------|--------|-----------|-----------|
-| **Lines of code** | 2,587 | 508 (+TODO) | 80% |
-| **Functions** | 5 (1 huge) | 10+ modular | +100% |
-| **Max nesting** | 8 levels | 3 levels | 63% |
-| **Type safety** | None | Full hints | ∞ |
-| **Testability** | Impossible | Easy | ∞ |
-| **Maintainability** | Very low | High | ∞ |
-
----
-
-## 📚 References
-
-### Coordinate Systems
-- **CH1903**: https://www.swisstopo.admin.ch/en/knowledge-facts/surveying-geodesy/reference-systems/local/lv03.html
-- **CH1903+/LV95**: https://www.swisstopo.admin.ch/en/knowledge-facts/surveying-geodesy/reference-systems/local/lv95.html
-- **UTM**: https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system
-
-### Libraries Used
-- **utm**: UTM <-> lat/lon conversions
-- **pyproj**: Swiss coordinate system transformations (EPSG:21781 -> EPSG:4326)
-
----
-
-## 🎯 Success Criteria
-
-Phase 1 complete when:
-- [ ] All CSV files process without errors
-- [ ] Coordinate transformations match legacy output
-- [ ] Miras are created/updated correctly
-- [ ] Basic alarms are generated for threshold violations
-- [ ] No regressions in data quality
-
-Full refactoring complete when:
-- [ ] All TODO items implemented
-- [ ] Test coverage > 80%
-- [ ] Performance >= legacy script
-- [ ] All additional monitoring types work
-- [ ] Legacy script can be retired
-
----
-
-**Version**: 1.0 (Essential Refactoring)
-**Last Updated**: 2024-10-11
-**Status**: Ready for Phase 1 implementation

vm1/src/refactory_scripts/__init__.py

@@ -1,15 +0,0 @@
-"""
-Refactored scripts with async/await, proper logging, and modern Python practices.
-
-This package contains modernized versions of the legacy scripts from old_scripts/,
-with the following improvements:
-- Full async/await support using aiomysql
-- Proper logging instead of print statements
-- Type hints and comprehensive docstrings
-- Error handling and retry logic
-- Configuration management
-- No hardcoded values
-- Follows PEP 8 and modern Python best practices
-"""
-
-__version__ = "1.0.0"

vm1/src/refactory_scripts/config/__init__.py

@@ -1,80 +0,0 @@
-"""Configuration management for refactored scripts."""
-
-import logging
-from configparser import ConfigParser
-from pathlib import Path
-from typing import Dict
-
-logger = logging.getLogger(__name__)
-
-
-class DatabaseConfig:
-    """Database configuration loader with validation."""
-
-    def __init__(self, config_file: Path | str = None, section: str = "mysql"):
-        """
-        Initialize database configuration.
-
-        Args:
-            config_file: Path to the configuration file. Defaults to env/config.ini
-            section: Configuration section name. Defaults to 'mysql'
-        """
-        if config_file is None:
-            # Default to env/config.ini relative to project root
-            config_file = Path(__file__).resolve().parent.parent.parent.parent / "env" / "config.ini"
-
-        self.config_file = Path(config_file)
-        self.section = section
-        self._config = self._load_config()
-
-    def _load_config(self) -> dict[str, str]:
-        """Load and validate configuration from file."""
-        if not self.config_file.exists():
-            raise FileNotFoundError(f"Configuration file not found: {self.config_file}")
-
-        parser = ConfigParser()
-        parser.read(self.config_file)
-
-        if not parser.has_section(self.section):
-            raise ValueError(f"Section '{self.section}' not found in {self.config_file}")
-
-        config = dict(parser.items(self.section))
-        logger.info(f"Configuration loaded from {self.config_file}, section [{self.section}]")
-
-        return config
-
-    @property
-    def host(self) -> str:
-        """Database host."""
-        return self._config.get("host", "localhost")
-
-    @property
-    def port(self) -> int:
-        """Database port."""
-        return int(self._config.get("port", "3306"))
-
-    @property
-    def database(self) -> str:
-        """Database name."""
-        return self._config["database"]
-
-    @property
-    def user(self) -> str:
-        """Database user."""
-        return self._config["user"]
-
-    @property
-    def password(self) -> str:
-        """Database password."""
-        return self._config["password"]
-
-    def as_dict(self) -> dict[str, any]:
-        """Return configuration as dictionary compatible with aiomysql."""
-        return {
-            "host": self.host,
-            "port": self.port,
-            "db": self.database,
-            "user": self.user,
-            "password": self.password,
-            "autocommit": True,
-        }

vm1/src/refactory_scripts/examples.py

@@ -1,233 +0,0 @@
-"""
-Example usage of the refactored loaders.
-
-This file demonstrates how to use the refactored scripts in various scenarios.
-"""
-
-import asyncio
-import logging
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.loaders import HirpiniaLoader, SisgeoLoader, VulinkLoader
-
-
-async def example_hirpinia():
-    """Example: Process a Hirpinia ODS file."""
-    print("\n=== Hirpinia Loader Example ===")
-
-    db_config = DatabaseConfig()
-
-    async with HirpiniaLoader(db_config) as loader:
-        # Process a single file
-        success = await loader.process_file("/path/to/hirpinia_file.ods")
-
-        if success:
-            print("✓ File processed successfully")
-        else:
-            print("✗ File processing failed")
-
-
-async def example_vulink():
-    """Example: Process a Vulink CSV file with alarm management."""
-    print("\n=== Vulink Loader Example ===")
-
-    db_config = DatabaseConfig()
-
-    async with VulinkLoader(db_config) as loader:
-        # Process a single file
-        success = await loader.process_file("/path/to/vulink_file.csv")
-
-        if success:
-            print("✓ File processed successfully")
-        else:
-            print("✗ File processing failed")
-
-
-async def example_sisgeo():
-    """Example: Process Sisgeo data (typically called by another module)."""
-    print("\n=== Sisgeo Loader Example ===")
-
-    db_config = DatabaseConfig()
-
-    # Example raw data
-    # Pressure sensor (6 fields): unit, tool, node, pressure, date, time
-    # Vibrating wire (8 fields): unit, tool, node, freq_hz, therm_ohms, freq_digit, date, time
-
-    raw_data = [
-        # Pressure sensor data
-        ("UNIT1", "TOOL1", 1, 101325.0, "2024-10-11", "14:30:00"),
-        # Vibrating wire data
-        ("UNIT1", "TOOL1", 2, 850.5, 1250.3, 12345, "2024-10-11", "14:30:00"),
-    ]
-
-    elab_data = []  # Elaborated data (if any)
-
-    async with SisgeoLoader(db_config) as loader:
-        raw_count, elab_count = await loader.process_data(raw_data, elab_data)
-
-        print(f"✓ Processed {raw_count} raw records, {elab_count} elaborated records")
-
-
-async def example_batch_processing():
-    """Example: Process multiple Hirpinia files efficiently."""
-    print("\n=== Batch Processing Example ===")
-
-    db_config = DatabaseConfig()
-
-    files = [
-        "/path/to/file1.ods",
-        "/path/to/file2.ods",
-        "/path/to/file3.ods",
-    ]
-
-    # Efficient: Reuse the same loader instance
-    async with HirpiniaLoader(db_config) as loader:
-        for file_path in files:
-            print(f"Processing: {file_path}")
-            success = await loader.process_file(file_path)
-            print(f"  {'✓' if success else '✗'} {file_path}")
-
-
-async def example_concurrent_processing():
-    """Example: Process multiple files concurrently."""
-    print("\n=== Concurrent Processing Example ===")
-
-    db_config = DatabaseConfig()
-
-    files = [
-        "/path/to/file1.ods",
-        "/path/to/file2.ods",
-        "/path/to/file3.ods",
-    ]
-
-    async def process_file(file_path):
-        """Process a single file."""
-        async with HirpiniaLoader(db_config) as loader:
-            return await loader.process_file(file_path)
-
-    # Process all files concurrently
-    results = await asyncio.gather(*[process_file(f) for f in files], return_exceptions=True)
-
-    for file_path, result in zip(files, results, strict=False):
-        if isinstance(result, Exception):
-            print(f"✗ {file_path}: {result}")
-        elif result:
-            print(f"✓ {file_path}")
-        else:
-            print(f"✗ {file_path}: Failed")
-
-
-async def example_with_error_handling():
-    """Example: Proper error handling and logging."""
-    print("\n=== Error Handling Example ===")
-
-    # Configure logging
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    logger = logging.getLogger(__name__)
-
-    db_config = DatabaseConfig()
-
-    try:
-        async with HirpiniaLoader(db_config) as loader:
-            success = await loader.process_file("/path/to/file.ods")
-
-            if success:
-                logger.info("Processing completed successfully")
-            else:
-                logger.error("Processing failed")
-
-    except FileNotFoundError as e:
-        logger.error(f"File not found: {e}")
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-
-
-async def example_integration_with_orchestrator():
-    """Example: Integration with orchestrator pattern."""
-    print("\n=== Orchestrator Integration Example ===")
-
-    db_config = DatabaseConfig()
-
-    async def worker(worker_id: int):
-        """Simulated worker that processes files."""
-        logger = logging.getLogger(f"Worker-{worker_id}")
-
-        async with HirpiniaLoader(db_config) as loader:
-            while True:
-                # In real implementation, get file from queue
-                file_path = await get_next_file_from_queue()
-
-                if not file_path:
-                    await asyncio.sleep(60)  # No files to process
-                    continue
-
-                logger.info(f"Processing: {file_path}")
-                success = await loader.process_file(file_path)
-
-                if success:
-                    await mark_file_as_processed(file_path)
-                    logger.info(f"Completed: {file_path}")
-                else:
-                    await mark_file_as_failed(file_path)
-                    logger.error(f"Failed: {file_path}")
-
-    # Dummy functions for demonstration
-    async def get_next_file_from_queue():
-        """Get next file from processing queue."""
-        return None  # Placeholder
-
-    async def mark_file_as_processed(file_path):
-        """Mark file as successfully processed."""
-        pass
-
-    async def mark_file_as_failed(file_path):
-        """Mark file as failed."""
-        pass
-
-    # Start multiple workers
-    workers = [asyncio.create_task(worker(i)) for i in range(3)]
-
-    print("Workers started (simulated)")
-    # await asyncio.gather(*workers)
-
-
-async def example_custom_configuration():
-    """Example: Using custom configuration."""
-    print("\n=== Custom Configuration Example ===")
-
-    # Load from custom config file
-    db_config = DatabaseConfig(config_file="/custom/path/config.ini", section="production_db")
-
-    print(f"Connected to: {db_config.host}:{db_config.port}/{db_config.database}")
-
-    async with HirpiniaLoader(db_config) as loader:
-        success = await loader.process_file("/path/to/file.ods")
-        print(f"{'✓' if success else '✗'} Processing complete")
-
-
-async def main():
-    """Run all examples."""
-    print("=" * 60)
-    print("Refactored Scripts - Usage Examples")
-    print("=" * 60)
-
-    # Note: These are just examples showing the API
-    # They won't actually run without real files and database
-
-    print("\n📝 These examples demonstrate the API.")
-    print("   To run them, replace file paths with real data.")
-
-    # Uncomment to run specific examples:
-    # await example_hirpinia()
-    # await example_vulink()
-    # await example_sisgeo()
-    # await example_batch_processing()
-    # await example_concurrent_processing()
-    # await example_with_error_handling()
-    # await example_integration_with_orchestrator()
-    # await example_custom_configuration()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm1/src/refactory_scripts/loaders/__init__.py

@@ -1,9 +0,0 @@
-"""Data loaders for various sensor types."""
-
-from refactory_scripts.loaders.hirpinia_loader import HirpiniaLoader
-from refactory_scripts.loaders.sisgeo_loader import SisgeoLoader
-from refactory_scripts.loaders.sorotec_loader import SorotecLoader
-from refactory_scripts.loaders.ts_pini_loader import TSPiniLoader
-from refactory_scripts.loaders.vulink_loader import VulinkLoader
-
-__all__ = ["HirpiniaLoader", "SisgeoLoader", "SorotecLoader", "TSPiniLoader", "VulinkLoader"]

vm1/src/refactory_scripts/loaders/hirpinia_loader.py

@@ -1,264 +0,0 @@
-"""
-Hirpinia data loader - Refactored version with async support.
-
-This script processes Hirpinia ODS files and loads data into the database.
-Replaces the legacy hirpiniaLoadScript.py with modern async/await patterns.
-"""
-
-import asyncio
-import logging
-import sys
-from datetime import datetime
-from pathlib import Path
-
-import ezodf
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import execute_many, execute_query, get_db_connection
-
-logger = logging.getLogger(__name__)
-
-
-class HirpiniaLoader:
-    """Loads Hirpinia sensor data from ODS files into the database."""
-
-    def __init__(self, db_config: DatabaseConfig):
-        """
-        Initialize the Hirpinia loader.
-
-        Args:
-            db_config: Database configuration object
-        """
-        self.db_config = db_config
-        self.conn = None
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        self.conn = await get_db_connection(self.db_config.as_dict())
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        if self.conn:
-            self.conn.close()
-
-    def _extract_metadata(self, file_path: Path) -> tuple[str, str]:
-        """
-        Extract unit name and tool name from file path.
-
-        Args:
-            file_path: Path to the ODS file
-
-        Returns:
-            Tuple of (unit_name, tool_name)
-        """
-        folder_path = file_path.parent
-        unit_name = folder_path.name
-
-        file_name = file_path.stem  # Filename without extension
-        tool_name = file_name.replace("HIRPINIA_", "")
-        tool_name = tool_name.split("_")[0]
-
-        logger.debug(f"Extracted metadata - Unit: {unit_name}, Tool: {tool_name}")
-        return unit_name, tool_name
-
-    def _parse_ods_file(self, file_path: Path, unit_name: str, tool_name: str) -> list[tuple]:
-        """
-        Parse ODS file and extract raw data.
-
-        Args:
-            file_path: Path to the ODS file
-            unit_name: Unit name
-            tool_name: Tool name
-
-        Returns:
-            List of tuples ready for database insertion
-        """
-        data_rows = []
-        doc = ezodf.opendoc(str(file_path))
-
-        for sheet in doc.sheets:
-            node_num = sheet.name.replace("S-", "")
-            logger.debug(f"Processing sheet: {sheet.name} (Node: {node_num})")
-
-            rows_to_skip = 2  # Skip header rows
-
-            for i, row in enumerate(sheet.rows()):
-                if i < rows_to_skip:
-                    continue
-
-                row_data = [cell.value for cell in row]
-
-                # Parse datetime
-                try:
-                    dt = datetime.strptime(row_data[0], "%Y-%m-%dT%H:%M:%S")
-                    date = dt.strftime("%Y-%m-%d")
-                    time = dt.strftime("%H:%M:%S")
-                except (ValueError, TypeError) as e:
-                    logger.warning(f"Failed to parse datetime in row {i}: {row_data[0]} - {e}")
-                    continue
-
-                # Extract values
-                val0 = row_data[2] if len(row_data) > 2 else None
-                val1 = row_data[4] if len(row_data) > 4 else None
-                val2 = row_data[6] if len(row_data) > 6 else None
-                val3 = row_data[8] if len(row_data) > 8 else None
-
-                # Create tuple for database insertion
-                data_rows.append((unit_name, tool_name, node_num, date, time, -1, -273, val0, val1, val2, val3))
-
-        logger.info(f"Parsed {len(data_rows)} data rows from {file_path.name}")
-        return data_rows
-
-    async def _insert_raw_data(self, data_rows: list[tuple]) -> int:
-        """
-        Insert raw data into the database.
-
-        Args:
-            data_rows: List of data tuples
-
-        Returns:
-            Number of rows inserted
-        """
-        if not data_rows:
-            logger.warning("No data rows to insert")
-            return 0
-
-        query = """
-            INSERT IGNORE INTO RAWDATACOR
-            (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0, Val1, Val2, Val3)
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-        """
-
-        rows_affected = await execute_many(self.conn, query, data_rows)
-        logger.info(f"Inserted {rows_affected} rows into RAWDATACOR")
-
-        return rows_affected
-
-    async def _get_matlab_function(self, unit_name: str, tool_name: str) -> str | None:
-        """
-        Get the MATLAB function name for this unit/tool combination.
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-
-        Returns:
-            MATLAB function name or None if not found
-        """
-        query = """
-            SELECT m.matcall
-            FROM tools AS t
-            JOIN units AS u ON u.id = t.unit_id
-            JOIN matfuncs AS m ON m.id = t.matfunc
-            WHERE u.name = %s AND t.name = %s
-        """
-
-        result = await execute_query(self.conn, query, (unit_name, tool_name), fetch_one=True)
-
-        if result and result.get("matcall"):
-            matlab_func = result["matcall"]
-            logger.info(f"MATLAB function found: {matlab_func}")
-            return matlab_func
-
-        logger.warning(f"No MATLAB function found for {unit_name}/{tool_name}")
-        return None
-
-    async def process_file(self, file_path: str | Path, trigger_matlab: bool = True) -> bool:
-        """
-        Process a Hirpinia ODS file and load data into the database.
-
-        Args:
-            file_path: Path to the ODS file to process
-            trigger_matlab: Whether to trigger MATLAB elaboration after loading
-
-        Returns:
-            True if processing was successful, False otherwise
-        """
-        file_path = Path(file_path)
-
-        if not file_path.exists():
-            logger.error(f"File not found: {file_path}")
-            return False
-
-        if file_path.suffix.lower() not in [".ods"]:
-            logger.error(f"Invalid file type: {file_path.suffix}. Expected .ods")
-            return False
-
-        try:
-            # Extract metadata
-            unit_name, tool_name = self._extract_metadata(file_path)
-
-            # Parse ODS file
-            data_rows = self._parse_ods_file(file_path, unit_name, tool_name)
-
-            # Insert data
-            rows_inserted = await self._insert_raw_data(data_rows)
-
-            if rows_inserted > 0:
-                logger.info(f"Successfully loaded {rows_inserted} rows from {file_path.name}")
-
-                # Optionally trigger MATLAB elaboration
-                if trigger_matlab:
-                    matlab_func = await self._get_matlab_function(unit_name, tool_name)
-                    if matlab_func:
-                        logger.warning(
-                            f"MATLAB elaboration would be triggered: {matlab_func} for {unit_name}/{tool_name}"
-                        )
-                        logger.warning("Note: Direct MATLAB execution not implemented in refactored version")
-                        # In production, this should integrate with elab_orchestrator instead
-                        # of calling MATLAB directly via os.system()
-
-                return True
-            else:
-                logger.warning(f"No new rows inserted from {file_path.name}")
-                return False
-
-        except Exception as e:
-            logger.error(f"Failed to process file {file_path}: {e}", exc_info=True)
-            return False
-
-
-async def main(file_path: str):
-    """
-    Main entry point for the Hirpinia loader.
-
-    Args:
-        file_path: Path to the ODS file to process
-    """
-    # Setup logging
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    logger.info("Hirpinia Loader started")
-    logger.info(f"Processing file: {file_path}")
-
-    try:
-        # Load configuration
-        db_config = DatabaseConfig()
-
-        # Process file
-        async with HirpiniaLoader(db_config) as loader:
-            success = await loader.process_file(file_path)
-
-        if success:
-            logger.info("Processing completed successfully")
-            return 0
-        else:
-            logger.error("Processing failed")
-            return 1
-
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-        return 1
-
-    finally:
-        logger.info("Hirpinia Loader finished")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python hirpinia_loader.py <path_to_ods_file>")
-        sys.exit(1)
-
-    exit_code = asyncio.run(main(sys.argv[1]))
-    sys.exit(exit_code)

vm1/src/refactory_scripts/loaders/sisgeo_loader.py

@@ -1,413 +0,0 @@
-"""
-Sisgeo data loader - Refactored version with async support.
-
-This script processes Sisgeo sensor data and loads it into the database.
-Handles different node types with different data formats.
-Replaces the legacy sisgeoLoadScript.py with modern async/await patterns.
-"""
-
-import asyncio
-import logging
-from datetime import datetime, timedelta
-from decimal import Decimal
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import execute_query, get_db_connection
-
-logger = logging.getLogger(__name__)
-
-
-class SisgeoLoader:
-    """Loads Sisgeo sensor data into the database with smart duplicate handling."""
-
-    # Node configuration constants
-    NODE_TYPE_PRESSURE = 1  # Node type 1: Pressure sensor (single value)
-    NODE_TYPE_VIBRATING_WIRE = 2  # Node type 2-5: Vibrating wire sensors (three values)
-
-    # Time threshold for duplicate detection (hours)
-    DUPLICATE_TIME_THRESHOLD_HOURS = 5
-
-    # Default values for missing data
-    DEFAULT_BAT_LEVEL = -1
-    DEFAULT_TEMPERATURE = -273
-
-    def __init__(self, db_config: DatabaseConfig):
-        """
-        Initialize the Sisgeo loader.
-
-        Args:
-            db_config: Database configuration object
-        """
-        self.db_config = db_config
-        self.conn = None
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        self.conn = await get_db_connection(self.db_config.as_dict())
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        if self.conn:
-            self.conn.close()
-
-    async def _get_latest_record(
-        self, unit_name: str, tool_name: str, node_num: int
-    ) -> dict | None:
-        """
-        Get the latest record for a specific node.
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-            node_num: Node number
-
-        Returns:
-            Latest record dict or None if not found
-        """
-        query = """
-            SELECT *
-            FROM RAWDATACOR
-            WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s
-            ORDER BY EventDate DESC, EventTime DESC
-            LIMIT 1
-        """
-
-        result = await execute_query(
-            self.conn, query, (unit_name, tool_name, node_num), fetch_one=True
-        )
-
-        return result
-
-    async def _insert_pressure_data(
-        self,
-        unit_name: str,
-        tool_name: str,
-        node_num: int,
-        date: str,
-        time: str,
-        pressure: Decimal,
-    ) -> bool:
-        """
-        Insert or update pressure sensor data (Node type 1).
-
-        Logic:
-        - If no previous record exists, insert new record
-        - If previous record has NULL BatLevelModule:
-            - Check time difference
-            - If >= 5 hours: insert new record
-            - If < 5 hours: update existing record
-        - If previous record has non-NULL BatLevelModule: insert new record
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-            node_num: Node number
-            date: Date string (YYYY-MM-DD)
-            time: Time string (HH:MM:SS)
-            pressure: Pressure value (in Pa, will be converted to hPa)
-
-        Returns:
-            True if operation was successful
-        """
-        # Get latest record
-        latest = await self._get_latest_record(unit_name, tool_name, node_num)
-
-        # Convert pressure from Pa to hPa (*100)
-        pressure_hpa = pressure * 100
-
-        if not latest:
-            # No previous record, insert new
-            query = """
-                INSERT INTO RAWDATACOR
-                (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule)
-                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-            """
-            params = (
-                unit_name,
-                tool_name,
-                node_num,
-                date,
-                time,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-                pressure_hpa,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-            )
-
-            await execute_query(self.conn, query, params)
-            logger.debug(
-                f"Inserted new pressure record: {unit_name}/{tool_name}/node{node_num}"
-            )
-            return True
-
-        # Check BatLevelModule status
-        if latest["BatLevelModule"] is None:
-            # Calculate time difference
-            old_datetime = datetime.strptime(
-                f"{latest['EventDate']} {latest['EventTime']}", "%Y-%m-%d %H:%M:%S"
-            )
-            new_datetime = datetime.strptime(f"{date} {time}", "%Y-%m-%d %H:%M:%S")
-            time_diff = new_datetime - old_datetime
-
-            if time_diff >= timedelta(hours=self.DUPLICATE_TIME_THRESHOLD_HOURS):
-                # Time difference >= 5 hours, insert new record
-                query = """
-                    INSERT INTO RAWDATACOR
-                    (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule)
-                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-                """
-                params = (
-                    unit_name,
-                    tool_name,
-                    node_num,
-                    date,
-                    time,
-                    self.DEFAULT_BAT_LEVEL,
-                    self.DEFAULT_TEMPERATURE,
-                    pressure_hpa,
-                    self.DEFAULT_BAT_LEVEL,
-                    self.DEFAULT_TEMPERATURE,
-                )
-
-                await execute_query(self.conn, query, params)
-                logger.debug(
-                    f"Inserted new pressure record (time diff: {time_diff}): {unit_name}/{tool_name}/node{node_num}"
-                )
-            else:
-                # Time difference < 5 hours, update existing record
-                query = """
-                    UPDATE RAWDATACOR
-                    SET val0 = %s, EventDate = %s, EventTime = %s
-                    WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s AND val0 IS NULL
-                    ORDER BY EventDate DESC, EventTime DESC
-                    LIMIT 1
-                """
-                params = (pressure_hpa, date, time, unit_name, tool_name, node_num)
-
-                await execute_query(self.conn, query, params)
-                logger.debug(
-                    f"Updated existing pressure record (time diff: {time_diff}): {unit_name}/{tool_name}/node{node_num}"
-                )
-
-        else:
-            # BatLevelModule is not NULL, insert new record
-            query = """
-                INSERT INTO RAWDATACOR
-                (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, BatLevelModule, TemperatureModule)
-                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-            """
-            params = (
-                unit_name,
-                tool_name,
-                node_num,
-                date,
-                time,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-                pressure_hpa,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-            )
-
-            await execute_query(self.conn, query, params)
-            logger.debug(
-                f"Inserted new pressure record (BatLevelModule not NULL): {unit_name}/{tool_name}/node{node_num}"
-            )
-
-        return True
-
-    async def _insert_vibrating_wire_data(
-        self,
-        unit_name: str,
-        tool_name: str,
-        node_num: int,
-        date: str,
-        time: str,
-        freq_hz: float,
-        therm_ohms: float,
-        freq_digit: float,
-    ) -> bool:
-        """
-        Insert or update vibrating wire sensor data (Node types 2-5).
-
-        Logic:
-        - If no previous record exists, insert new record
-        - If previous record has NULL BatLevelModule: update existing record
-        - If previous record has non-NULL BatLevelModule: insert new record
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-            node_num: Node number
-            date: Date string (YYYY-MM-DD)
-            time: Time string (HH:MM:SS)
-            freq_hz: Frequency in Hz
-            therm_ohms: Thermistor in Ohms
-            freq_digit: Frequency in digits
-
-        Returns:
-            True if operation was successful
-        """
-        # Get latest record
-        latest = await self._get_latest_record(unit_name, tool_name, node_num)
-
-        if not latest:
-            # No previous record, insert new
-            query = """
-                INSERT INTO RAWDATACOR
-                (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, val1, val2, BatLevelModule, TemperatureModule)
-                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-            """
-            params = (
-                unit_name,
-                tool_name,
-                node_num,
-                date,
-                time,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-                freq_hz,
-                therm_ohms,
-                freq_digit,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-            )
-
-            await execute_query(self.conn, query, params)
-            logger.debug(
-                f"Inserted new vibrating wire record: {unit_name}/{tool_name}/node{node_num}"
-            )
-            return True
-
-        # Check BatLevelModule status
-        if latest["BatLevelModule"] is None:
-            # Update existing record
-            query = """
-                UPDATE RAWDATACOR
-                SET val0 = %s, val1 = %s, val2 = %s, EventDate = %s, EventTime = %s
-                WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s AND val0 IS NULL
-                ORDER BY EventDate DESC, EventTime DESC
-                LIMIT 1
-            """
-            params = (freq_hz, therm_ohms, freq_digit, date, time, unit_name, tool_name, node_num)
-
-            await execute_query(self.conn, query, params)
-            logger.debug(
-                f"Updated existing vibrating wire record: {unit_name}/{tool_name}/node{node_num}"
-            )
-
-        else:
-            # BatLevelModule is not NULL, insert new record
-            query = """
-                INSERT INTO RAWDATACOR
-                (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, val0, val1, val2, BatLevelModule, TemperatureModule)
-                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-            """
-            params = (
-                unit_name,
-                tool_name,
-                node_num,
-                date,
-                time,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-                freq_hz,
-                therm_ohms,
-                freq_digit,
-                self.DEFAULT_BAT_LEVEL,
-                self.DEFAULT_TEMPERATURE,
-            )
-
-            await execute_query(self.conn, query, params)
-            logger.debug(
-                f"Inserted new vibrating wire record (BatLevelModule not NULL): {unit_name}/{tool_name}/node{node_num}"
-            )
-
-        return True
-
-    async def process_data(
-        self, raw_data: list[tuple], elab_data: list[tuple]
-    ) -> tuple[int, int]:
-        """
-        Process raw and elaborated data from Sisgeo sensors.
-
-        Args:
-            raw_data: List of raw data tuples
-            elab_data: List of elaborated data tuples
-
-        Returns:
-            Tuple of (raw_records_processed, elab_records_processed)
-        """
-        raw_count = 0
-        elab_count = 0
-
-        # Process raw data
-        for record in raw_data:
-            try:
-                if len(record) == 6:
-                    # Pressure sensor data (node type 1)
-                    unit_name, tool_name, node_num, pressure, date, time = record
-                    success = await self._insert_pressure_data(
-                        unit_name, tool_name, node_num, date, time, Decimal(pressure)
-                    )
-                    if success:
-                        raw_count += 1
-
-                elif len(record) == 8:
-                    # Vibrating wire sensor data (node types 2-5)
-                    (
-                        unit_name,
-                        tool_name,
-                        node_num,
-                        freq_hz,
-                        therm_ohms,
-                        freq_digit,
-                        date,
-                        time,
-                    ) = record
-                    success = await self._insert_vibrating_wire_data(
-                        unit_name,
-                        tool_name,
-                        node_num,
-                        date,
-                        time,
-                        freq_hz,
-                        therm_ohms,
-                        freq_digit,
-                    )
-                    if success:
-                        raw_count += 1
-                else:
-                    logger.warning(f"Unknown record format: {len(record)} fields")
-
-            except Exception as e:
-                logger.error(f"Failed to process raw record: {e}", exc_info=True)
-                logger.debug(f"Record: {record}")
-
-        # Process elaborated data (if needed)
-        # Note: The legacy script had elab_data parameter but didn't use it
-        # This can be implemented if elaborated data processing is needed
-
-        logger.info(f"Processed {raw_count} raw records, {elab_count} elaborated records")
-        return raw_count, elab_count
-
-
-async def main():
-    """
-    Main entry point for the Sisgeo loader.
-
-    Note: This is a library module, typically called by other scripts.
-    Direct execution is provided for testing purposes.
-    """
-    logging.basicConfig(
-        level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-
-    logger.info("Sisgeo Loader module loaded")
-    logger.info("This is a library module. Use SisgeoLoader class in your scripts.")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm1/src/refactory_scripts/loaders/sorotec_loader.py

@@ -1,396 +0,0 @@
-"""
-Sorotec Pini data loader - Refactored version with async support.
-
-This script processes Sorotec Pini CSV files and loads multi-channel sensor data.
-Handles two different file formats (_1_ and _2_) with different channel mappings.
-Replaces the legacy sorotecPini.py with modern async/await patterns.
-"""
-
-import asyncio
-import logging
-import sys
-from pathlib import Path
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import execute_many, get_db_connection
-
-logger = logging.getLogger(__name__)
-
-
-class SorotecLoader:
-    """Loads Sorotec Pini multi-channel sensor data from CSV files."""
-
-    # File type identifiers
-    FILE_TYPE_1 = "_1_"
-    FILE_TYPE_2 = "_2_"
-
-    # Default values
-    DEFAULT_TEMPERATURE = -273
-    DEFAULT_UNIT_NAME = "ID0247"
-    DEFAULT_TOOL_NAME = "DT0001"
-
-    # Channel mappings for File Type 1 (nodes 1-26)
-    CHANNELS_TYPE_1 = list(range(1, 27))  # Nodes 1 to 26
-
-    # Channel mappings for File Type 2 (selective nodes)
-    CHANNELS_TYPE_2 = [41, 42, 43, 44, 49, 50, 51, 52, 56, 57, 58, 59, 60, 61, 62]  # 15 nodes
-
-    def __init__(self, db_config: DatabaseConfig):
-        """
-        Initialize the Sorotec loader.
-
-        Args:
-            db_config: Database configuration object
-        """
-        self.db_config = db_config
-        self.conn = None
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        self.conn = await get_db_connection(self.db_config.as_dict())
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        if self.conn:
-            self.conn.close()
-
-    def _extract_metadata(self, file_path: Path) -> tuple[str, str]:
-        """
-        Extract unit name and tool name from file path.
-
-        For Sorotec, metadata is determined by folder name.
-
-        Args:
-            file_path: Path to the CSV file
-
-        Returns:
-            Tuple of (unit_name, tool_name)
-        """
-        # Get folder name (second to last part of path)
-        folder_name = file_path.parent.name
-
-        # Currently hardcoded for ID0247
-        # TODO: Make this configurable if more units are added
-        if folder_name == "ID0247":
-            unit_name = self.DEFAULT_UNIT_NAME
-            tool_name = self.DEFAULT_TOOL_NAME
-        else:
-            logger.warning(f"Unknown folder: {folder_name}, using defaults")
-            unit_name = self.DEFAULT_UNIT_NAME
-            tool_name = self.DEFAULT_TOOL_NAME
-
-        logger.debug(f"Metadata: Unit={unit_name}, Tool={tool_name}")
-        return unit_name, tool_name
-
-    def _determine_file_type(self, file_path: Path) -> str | None:
-        """
-        Determine file type based on filename pattern.
-
-        Args:
-            file_path: Path to the CSV file
-
-        Returns:
-            File type identifier ("_1_" or "_2_") or None if unknown
-        """
-        filename = file_path.name
-
-        if self.FILE_TYPE_1 in filename:
-            return self.FILE_TYPE_1
-        elif self.FILE_TYPE_2 in filename:
-            return self.FILE_TYPE_2
-        else:
-            logger.error(f"Unknown file type: {filename}")
-            return None
-
-    def _parse_datetime(self, timestamp_str: str) -> tuple[str, str]:
-        """
-        Parse datetime string and convert to database format.
-
-        Converts from "DD-MM-YYYY HH:MM:SS" to ("YYYY-MM-DD", "HH:MM:SS")
-
-        Args:
-            timestamp_str: Timestamp string in format "DD-MM-YYYY HH:MM:SS"
-
-        Returns:
-            Tuple of (date, time) strings
-
-        Examples:
-            >>> _parse_datetime("11-10-2024 14:30:00")
-            ("2024-10-11", "14:30:00")
-        """
-        parts = timestamp_str.split(" ")
-        date_parts = parts[0].split("-")
-
-        # Convert DD-MM-YYYY to YYYY-MM-DD
-        date = f"{date_parts[2]}-{date_parts[1]}-{date_parts[0]}"
-        time = parts[1]
-
-        return date, time
-
-    def _parse_csv_type_1(self, lines: list[str], unit_name: str, tool_name: str) -> tuple[list, list]:
-        """
-        Parse CSV file of type 1 (_1_).
-
-        File Type 1 has 38 columns and maps to nodes 1-26.
-
-        Args:
-            lines: List of CSV lines
-            unit_name: Unit name
-            tool_name: Tool name
-
-        Returns:
-            Tuple of (raw_data_rows, elab_data_rows)
-        """
-        raw_data = []
-        elab_data = []
-
-        for line in lines:
-            # Parse CSV row
-            row = line.replace('"', "").split(";")
-
-            # Extract timestamp
-            date, time = self._parse_datetime(row[0])
-
-            # Extract battery voltage (an4 = column 2)
-            battery = row[2]
-
-            # Extract channel values (E8_xxx_CHx)
-            # Type 1 mapping: columns 4-35 map to channels
-            ch_values = [
-                row[35],  # E8_181_CH1 (node 1)
-                row[4],  # E8_181_CH2 (node 2)
-                row[5],  # E8_181_CH3 (node 3)
-                row[6],  # E8_181_CH4 (node 4)
-                row[7],  # E8_181_CH5 (node 5)
-                row[8],  # E8_181_CH6 (node 6)
-                row[9],  # E8_181_CH7 (node 7)
-                row[10],  # E8_181_CH8 (node 8)
-                row[11],  # E8_182_CH1 (node 9)
-                row[12],  # E8_182_CH2 (node 10)
-                row[13],  # E8_182_CH3 (node 11)
-                row[14],  # E8_182_CH4 (node 12)
-                row[15],  # E8_182_CH5 (node 13)
-                row[16],  # E8_182_CH6 (node 14)
-                row[17],  # E8_182_CH7 (node 15)
-                row[18],  # E8_182_CH8 (node 16)
-                row[19],  # E8_183_CH1 (node 17)
-                row[20],  # E8_183_CH2 (node 18)
-                row[21],  # E8_183_CH3 (node 19)
-                row[22],  # E8_183_CH4 (node 20)
-                row[23],  # E8_183_CH5 (node 21)
-                row[24],  # E8_183_CH6 (node 22)
-                row[25],  # E8_183_CH7 (node 23)
-                row[26],  # E8_183_CH8 (node 24)
-                row[27],  # E8_184_CH1 (node 25)
-                row[28],  # E8_184_CH2 (node 26)
-            ]
-
-            # Create data rows for each channel
-            for node_num, value in enumerate(ch_values, start=1):
-                # Raw data (with battery info)
-                raw_data.append((unit_name, tool_name, node_num, date, time, battery, self.DEFAULT_TEMPERATURE, value))
-
-                # Elaborated data (just the load value)
-                elab_data.append((unit_name, tool_name, node_num, date, time, value))
-
-        logger.info(f"Parsed Type 1: {len(elab_data)} channel readings ({len(elab_data)//26} timestamps x 26 channels)")
-        return raw_data, elab_data
-
-    def _parse_csv_type_2(self, lines: list[str], unit_name: str, tool_name: str) -> tuple[list, list]:
-        """
-        Parse CSV file of type 2 (_2_).
-
-        File Type 2 has 38 columns and maps to selective nodes (41-62).
-
-        Args:
-            lines: List of CSV lines
-            unit_name: Unit name
-            tool_name: Tool name
-
-        Returns:
-            Tuple of (raw_data_rows, elab_data_rows)
-        """
-        raw_data = []
-        elab_data = []
-
-        for line in lines:
-            # Parse CSV row
-            row = line.replace('"', "").split(";")
-
-            # Extract timestamp
-            date, time = self._parse_datetime(row[0])
-
-            # Extract battery voltage (an4 = column 37)
-            battery = row[37]
-
-            # Extract channel values for Type 2
-            # Type 2 mapping: specific columns to specific nodes
-            channel_mapping = [
-                (41, row[13]),  # E8_182_CH1
-                (42, row[14]),  # E8_182_CH2
-                (43, row[15]),  # E8_182_CH3
-                (44, row[16]),  # E8_182_CH4
-                (49, row[21]),  # E8_183_CH1
-                (50, row[22]),  # E8_183_CH2
-                (51, row[23]),  # E8_183_CH3
-                (52, row[24]),  # E8_183_CH4
-                (56, row[28]),  # E8_183_CH8
-                (57, row[29]),  # E8_184_CH1
-                (58, row[30]),  # E8_184_CH2
-                (59, row[31]),  # E8_184_CH3
-                (60, row[32]),  # E8_184_CH4
-                (61, row[33]),  # E8_184_CH5
-                (62, row[34]),  # E8_184_CH6
-            ]
-
-            # Create data rows for each channel
-            for node_num, value in channel_mapping:
-                # Raw data (with battery info)
-                raw_data.append((unit_name, tool_name, node_num, date, time, battery, self.DEFAULT_TEMPERATURE, value))
-
-                # Elaborated data (just the load value)
-                elab_data.append((unit_name, tool_name, node_num, date, time, value))
-
-        logger.info(f"Parsed Type 2: {len(elab_data)} channel readings ({len(elab_data)//15} timestamps x 15 channels)")
-        return raw_data, elab_data
-
-    async def _insert_data(self, raw_data: list, elab_data: list) -> tuple[int, int]:
-        """
-        Insert raw and elaborated data into the database.
-
-        Args:
-            raw_data: List of raw data tuples
-            elab_data: List of elaborated data tuples
-
-        Returns:
-            Tuple of (raw_rows_inserted, elab_rows_inserted)
-        """
-        raw_query = """
-            INSERT IGNORE INTO RAWDATACOR
-            (UnitName, ToolNameID, NodeNum, EventDate, EventTime, BatLevel, Temperature, Val0)
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
-        """
-
-        elab_query = """
-            INSERT IGNORE INTO ELABDATADISP
-            (UnitName, ToolNameID, NodeNum, EventDate, EventTime, load_value)
-            VALUES (%s, %s, %s, %s, %s, %s)
-        """
-
-        # Insert elaborated data first
-        elab_count = await execute_many(self.conn, elab_query, elab_data)
-        logger.info(f"Inserted {elab_count} elaborated records")
-
-        # Insert raw data
-        raw_count = await execute_many(self.conn, raw_query, raw_data)
-        logger.info(f"Inserted {raw_count} raw records")
-
-        return raw_count, elab_count
-
-    async def process_file(self, file_path: str | Path) -> bool:
-        """
-        Process a Sorotec CSV file and load data into the database.
-
-        Args:
-            file_path: Path to the CSV file to process
-
-        Returns:
-            True if processing was successful, False otherwise
-        """
-        file_path = Path(file_path)
-
-        if not file_path.exists():
-            logger.error(f"File not found: {file_path}")
-            return False
-
-        if file_path.suffix.lower() not in [".csv", ".txt"]:
-            logger.error(f"Invalid file type: {file_path.suffix}")
-            return False
-
-        try:
-            logger.info(f"Processing file: {file_path.name}")
-
-            # Extract metadata
-            unit_name, tool_name = self._extract_metadata(file_path)
-
-            # Determine file type
-            file_type = self._determine_file_type(file_path)
-            if not file_type:
-                return False
-
-            logger.info(f"File type detected: {file_type}")
-
-            # Read file
-            with open(file_path, encoding="utf-8") as f:
-                lines = [line.rstrip() for line in f.readlines()]
-
-            # Remove empty lines and header rows
-            lines = [line for line in lines if line]
-            if len(lines) > 4:
-                lines = lines[4:]  # Skip first 4 header lines
-
-            if not lines:
-                logger.warning(f"No data lines found in {file_path.name}")
-                return False
-
-            # Parse based on file type
-            if file_type == self.FILE_TYPE_1:
-                raw_data, elab_data = self._parse_csv_type_1(lines, unit_name, tool_name)
-            else:  # FILE_TYPE_2
-                raw_data, elab_data = self._parse_csv_type_2(lines, unit_name, tool_name)
-
-            # Insert into database
-            raw_count, elab_count = await self._insert_data(raw_data, elab_data)
-
-            logger.info(f"Successfully processed {file_path.name}: {raw_count} raw, {elab_count} elab records")
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to process file {file_path}: {e}", exc_info=True)
-            return False
-
-
-async def main(file_path: str):
-    """
-    Main entry point for the Sorotec loader.
-
-    Args:
-        file_path: Path to the CSV file to process
-    """
-    # Setup logging
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    logger.info("Sorotec Loader started")
-    logger.info(f"Processing file: {file_path}")
-
-    try:
-        # Load configuration
-        db_config = DatabaseConfig()
-
-        # Process file
-        async with SorotecLoader(db_config) as loader:
-            success = await loader.process_file(file_path)
-
-        if success:
-            logger.info("Processing completed successfully")
-            return 0
-        else:
-            logger.error("Processing failed")
-            return 1
-
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-        return 1
-
-    finally:
-        logger.info("Sorotec Loader finished")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python sorotec_loader.py <path_to_csv_file>")
-        sys.exit(1)
-
-    exit_code = asyncio.run(main(sys.argv[1]))
-    sys.exit(exit_code)

vm1/src/refactory_scripts/loaders/ts_pini_loader.py

@@ -1,508 +0,0 @@
-"""
-TS Pini (Total Station) data loader - Refactored version with async support.
-
-This script processes Total Station survey data from multiple instrument types
-(Leica, Trimble S7, S9) and manages complex monitoring with multi-level alarms.
-
-**STATUS**: Essential refactoring - Base structure with coordinate transformations.
-**TODO**: Complete alarm management, threshold checking, and additional monitoring.
-
-Replaces the legacy TS_PiniScript.py (2,587 lines) with a modular, maintainable architecture.
-"""
-
-import asyncio
-import logging
-import sys
-from datetime import datetime
-from enum import IntEnum
-from pathlib import Path
-
-import utm
-from pyproj import Transformer
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import execute_query, get_db_connection
-
-logger = logging.getLogger(__name__)
-
-
-class StationType(IntEnum):
-    """Total Station instrument types."""
-
-    LEICA = 1
-    TRIMBLE_S7 = 4
-    TRIMBLE_S9 = 7
-    TRIMBLE_S7_INVERTED = 10  # x-y coordinates inverted
-
-
-class CoordinateSystem(IntEnum):
-    """Coordinate system types for transformations."""
-
-    CH1903 = 6  # Swiss coordinate system (old)
-    UTM = 7  # Universal Transverse Mercator
-    CH1903_PLUS = 10  # Swiss coordinate system LV95 (new)
-    LAT_LON = 0  # Default: already in lat/lon
-
-
-class TSPiniLoader:
-    """
-    Loads Total Station Pini survey data with coordinate transformations and alarm management.
-
-    This loader handles:
-    - Multiple station types (Leica, Trimble S7/S9)
-    - Coordinate system transformations (CH1903, UTM, lat/lon)
-    - Target point (mira) management
-    - Multi-level alarm system (TODO: complete implementation)
-    - Additional monitoring for railways, walls, trusses (TODO)
-    """
-
-    # Folder name mappings for special cases
-    FOLDER_MAPPINGS = {
-        "[276_208_TS0003]": "TS0003",
-        "[Neuchatel_CDP]": "TS7",
-        "[TS0006_EP28]": "TS0006_EP28",
-        "[TS0007_ChesaArcoiris]": "TS0007_ChesaArcoiris",
-        "[TS0006_EP28_3]": "TS0006_EP28_3",
-        "[TS0006_EP28_4]": "TS0006_EP28_4",
-        "[TS0006_EP28_5]": "TS0006_EP28_5",
-        "[TS18800]": "TS18800",
-        "[Granges_19 100]": "Granges_19 100",
-        "[Granges_19 200]": "Granges_19 200",
-        "[Chesa_Arcoiris_2]": "Chesa_Arcoiris_2",
-        "[TS0006_EP28_1]": "TS0006_EP28_1",
-        "[TS_PS_Petites_Croisettes]": "TS_PS_Petites_Croisettes",
-        "[_Chesa_Arcoiris_1]": "_Chesa_Arcoiris_1",
-        "[TS_test]": "TS_test",
-        "[TS-VIME]": "TS-VIME",
-    }
-
-    def __init__(self, db_config: DatabaseConfig):
-        """
-        Initialize the TS Pini loader.
-
-        Args:
-            db_config: Database configuration object
-        """
-        self.db_config = db_config
-        self.conn = None
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        self.conn = await get_db_connection(self.db_config.as_dict())
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        if self.conn:
-            self.conn.close()
-
-    def _extract_folder_name(self, file_path: Path) -> str:
-        """
-        Extract and normalize folder name from file path.
-
-        Handles special folder name mappings for specific projects.
-
-        Args:
-            file_path: Path to the CSV file
-
-        Returns:
-            Normalized folder name
-        """
-        # Get folder name from path
-        folder_name = file_path.parent.name
-
-        # Check for special mappings in filename
-        filename = file_path.name
-        for pattern, mapped_name in self.FOLDER_MAPPINGS.items():
-            if pattern in filename:
-                logger.debug(f"Mapped folder: {pattern} -> {mapped_name}")
-                return mapped_name
-
-        return folder_name
-
-    async def _get_project_info(self, folder_name: str) -> dict | None:
-        """
-        Get project information from database based on folder name.
-
-        Args:
-            folder_name: Folder/station name
-
-        Returns:
-            Dictionary with project info or None if not found
-        """
-        query = """
-            SELECT
-                l.id as lavoro_id,
-                s.id as site_id,
-                st.type_id,
-                s.upgeo_sist_coordinate,
-                s.upgeo_utmzone,
-                s.upgeo_utmhemisphere
-            FROM upgeo_st as st
-            LEFT JOIN upgeo_lavori as l ON st.lavoro_id = l.id
-            LEFT JOIN sites as s ON s.id = l.site_id
-            WHERE st.name = %s
-        """
-
-        result = await execute_query(self.conn, query, (folder_name,), fetch_one=True)
-
-        if not result:
-            logger.error(f"Project not found for folder: {folder_name}")
-            return None
-
-        return {
-            "lavoro_id": result["lavoro_id"],
-            "site_id": result["site_id"],
-            "station_type": result["type_id"],
-            "coordinate_system": int(result["upgeo_sist_coordinate"]),
-            "utm_zone": result["upgeo_utmzone"],
-            "utm_hemisphere": result["upgeo_utmhemisphere"] != "S",  # True for North
-        }
-
-    def _parse_csv_row(self, row: list[str], station_type: int) -> tuple[str, str, str, str, str]:
-        """
-        Parse CSV row based on station type.
-
-        Different station types have different column orders.
-
-        Args:
-            row: List of CSV values
-            station_type: Station type identifier
-
-        Returns:
-            Tuple of (mira_name, easting, northing, height, timestamp)
-        """
-        if station_type == StationType.LEICA:
-            # Leica format: name, easting, northing, height, timestamp
-            mira_name = row[0]
-            easting = row[1]
-            northing = row[2]
-            height = row[3]
-            # Convert timestamp: DD.MM.YYYY HH:MM:SS.fff -> YYYY-MM-DD HH:MM:SS
-            timestamp = datetime.strptime(row[4], "%d.%m.%Y %H:%M:%S.%f").strftime("%Y-%m-%d %H:%M:%S")
-
-        elif station_type in (StationType.TRIMBLE_S7, StationType.TRIMBLE_S9):
-            # Trimble S7/S9 format: timestamp, name, northing, easting, height
-            timestamp = row[0]
-            mira_name = row[1]
-            northing = row[2]
-            easting = row[3]
-            height = row[4]
-
-        elif station_type == StationType.TRIMBLE_S7_INVERTED:
-            # Trimble S7 inverted: timestamp, name, easting(row[2]), northing(row[3]), height
-            timestamp = row[0]
-            mira_name = row[1]
-            northing = row[3]  # Inverted!
-            easting = row[2]  # Inverted!
-            height = row[4]
-
-        else:
-            raise ValueError(f"Unknown station type: {station_type}")
-
-        return mira_name, easting, northing, height, timestamp
-
-    def _transform_coordinates(
-        self, easting: float, northing: float, coord_system: int, utm_zone: str = None, utm_hemisphere: bool = True
-    ) -> tuple[float, float]:
-        """
-        Transform coordinates to lat/lon based on coordinate system.
-
-        Args:
-            easting: Easting coordinate
-            northing: Northing coordinate
-            coord_system: Coordinate system type
-            utm_zone: UTM zone (required for UTM system)
-            utm_hemisphere: True for Northern, False for Southern
-
-        Returns:
-            Tuple of (latitude, longitude)
-        """
-        if coord_system == CoordinateSystem.CH1903:
-            # Old Swiss coordinate system transformation
-            y = easting
-            x = northing
-            y_ = (y - 2600000) / 1000000
-            x_ = (x - 1200000) / 1000000
-
-            lambda_ = 2.6779094 + 4.728982 * y_ + 0.791484 * y_ * x_ + 0.1306 * y_ * x_**2 - 0.0436 * y_**3
-            phi_ = 16.9023892 + 3.238272 * x_ - 0.270978 * y_**2 - 0.002528 * x_**2 - 0.0447 * y_**2 * x_ - 0.0140 * x_**3
-
-            lat = phi_ * 100 / 36
-            lon = lambda_ * 100 / 36
-
-        elif coord_system == CoordinateSystem.UTM:
-            # UTM to lat/lon
-            if not utm_zone:
-                raise ValueError("UTM zone required for UTM coordinate system")
-
-            result = utm.to_latlon(easting, northing, utm_zone, northern=utm_hemisphere)
-            lat = result[0]
-            lon = result[1]
-
-        elif coord_system == CoordinateSystem.CH1903_PLUS:
-            # New Swiss coordinate system (LV95) using EPSG:21781 -> EPSG:4326
-            transformer = Transformer.from_crs("EPSG:21781", "EPSG:4326")
-            lat, lon = transformer.transform(easting, northing)
-
-        else:
-            # Already in lat/lon
-            lon = easting
-            lat = northing
-
-        logger.debug(f"Transformed coordinates: ({easting}, {northing}) -> ({lat:.6f}, {lon:.6f})")
-        return lat, lon
-
-    async def _get_or_create_mira(self, mira_name: str, lavoro_id: int) -> int | None:
-        """
-        Get existing mira (target point) ID or create new one if allowed.
-
-        Args:
-            mira_name: Name of the target point
-            lavoro_id: Project ID
-
-        Returns:
-            Mira ID or None if creation not allowed
-        """
-        # Check if mira exists
-        query = """
-            SELECT m.id as mira_id, m.name
-            FROM upgeo_mire as m
-            JOIN upgeo_lavori as l ON m.lavoro_id = l.id
-            WHERE m.name = %s AND m.lavoro_id = %s
-        """
-
-        result = await execute_query(self.conn, query, (mira_name, lavoro_id), fetch_one=True)
-
-        if result:
-            return result["mira_id"]
-
-        # Mira doesn't exist - check if we can create it
-        logger.info(f"Mira '{mira_name}' not found, attempting to create...")
-
-        # TODO: Implement mira creation logic
-        # This requires checking company limits and updating counters
-        # For now, return None to skip
-        logger.warning("Mira creation not yet implemented in refactored version")
-        return None
-
-    async def _insert_survey_data(
-        self,
-        mira_id: int,
-        timestamp: str,
-        northing: float,
-        easting: float,
-        height: float,
-        lat: float,
-        lon: float,
-        coord_system: int,
-    ) -> bool:
-        """
-        Insert survey data into ELABDATAUPGEO table.
-
-        Args:
-            mira_id: Target point ID
-            timestamp: Survey timestamp
-            northing: Northing coordinate
-            easting: Easting coordinate
-            height: Elevation
-            lat: Latitude
-            lon: Longitude
-            coord_system: Coordinate system type
-
-        Returns:
-            True if insert was successful
-        """
-        query = """
-            INSERT IGNORE INTO ELABDATAUPGEO
-            (mira_id, EventTimestamp, north, east, elevation, lat, lon, sist_coordinate)
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
-        """
-
-        params = (mira_id, timestamp, northing, easting, height, lat, lon, coord_system)
-
-        try:
-            await execute_query(self.conn, query, params)
-            logger.debug(f"Inserted survey data for mira_id {mira_id} at {timestamp}")
-            return True
-        except Exception as e:
-            logger.error(f"Failed to insert survey data: {e}")
-            return False
-
-    async def _process_thresholds_and_alarms(self, lavoro_id: int, processed_miras: list[int]) -> None:
-        """
-        Process thresholds and create alarms for monitored points.
-
-        **TODO**: This is a stub for the complex alarm system.
-        The complete implementation requires:
-        - Multi-level threshold checking (3 levels: attention, intervention, immediate)
-        - 5 dimensions: N, E, H, R2D, R3D
-        - Email and SMS notifications
-        - Time-series analysis
-        - Railway/wall/truss specific monitoring
-
-        Args:
-            lavoro_id: Project ID
-            processed_miras: List of mira IDs that were processed
-        """
-        logger.warning("Threshold and alarm processing is not yet implemented")
-        logger.info(f"Would process alarms for {len(processed_miras)} miras in lavoro {lavoro_id}")
-
-        # TODO: Implement alarm system
-        # 1. Load threshold configurations from upgeo_lavori and upgeo_mire tables
-        # 2. Query latest survey data for each mira
-        # 3. Calculate displacements (N, E, H, R2D, R3D)
-        # 4. Check against 3-level thresholds
-        # 5. Create alarms if thresholds exceeded
-        # 6. Handle additional monitoring (railways, walls, trusses)
-
-    async def process_file(self, file_path: str | Path) -> bool:
-        """
-        Process a Total Station CSV file and load data into the database.
-
-        **Current Implementation**: Core data loading with coordinate transformations.
-        **TODO**: Complete alarm and additional monitoring implementation.
-
-        Args:
-            file_path: Path to the CSV file to process
-
-        Returns:
-            True if processing was successful, False otherwise
-        """
-        file_path = Path(file_path)
-
-        if not file_path.exists():
-            logger.error(f"File not found: {file_path}")
-            return False
-
-        try:
-            logger.info(f"Processing Total Station file: {file_path.name}")
-
-            # Extract folder name
-            folder_name = self._extract_folder_name(file_path)
-            logger.info(f"Station/Project: {folder_name}")
-
-            # Get project information
-            project_info = await self._get_project_info(folder_name)
-            if not project_info:
-                return False
-
-            station_type = project_info["station_type"]
-            coord_system = project_info["coordinate_system"]
-            lavoro_id = project_info["lavoro_id"]
-
-            logger.info(f"Station type: {station_type}, Coordinate system: {coord_system}")
-
-            # Read and parse CSV file
-            with open(file_path, encoding="utf-8") as f:
-                lines = [line.rstrip() for line in f.readlines()]
-
-            # Skip header
-            if lines:
-                lines = lines[1:]
-
-            processed_count = 0
-            processed_miras = []
-
-            # Process each survey point
-            for line in lines:
-                if not line:
-                    continue
-
-                row = line.split(",")
-
-                try:
-                    # Parse row based on station type
-                    mira_name, easting, northing, height, timestamp = self._parse_csv_row(row, station_type)
-
-                    # Transform coordinates to lat/lon
-                    lat, lon = self._transform_coordinates(
-                        float(easting),
-                        float(northing),
-                        coord_system,
-                        project_info.get("utm_zone"),
-                        project_info.get("utm_hemisphere"),
-                    )
-
-                    # Get or create mira
-                    mira_id = await self._get_or_create_mira(mira_name, lavoro_id)
-
-                    if not mira_id:
-                        logger.warning(f"Skipping mira '{mira_name}' - not found and creation not allowed")
-                        continue
-
-                    # Insert survey data
-                    success = await self._insert_survey_data(
-                        mira_id, timestamp, float(northing), float(easting), float(height), lat, lon, coord_system
-                    )
-
-                    if success:
-                        processed_count += 1
-                        if mira_id not in processed_miras:
-                            processed_miras.append(mira_id)
-
-                except Exception as e:
-                    logger.error(f"Failed to process row: {e}")
-                    logger.debug(f"Row data: {row}")
-                    continue
-
-            logger.info(f"Processed {processed_count} survey points for {len(processed_miras)} miras")
-
-            # Process thresholds and alarms (TODO: complete implementation)
-            if processed_miras:
-                await self._process_thresholds_and_alarms(lavoro_id, processed_miras)
-
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to process file {file_path}: {e}", exc_info=True)
-            return False
-
-
-async def main(file_path: str):
-    """
-    Main entry point for the TS Pini loader.
-
-    Args:
-        file_path: Path to the CSV file to process
-    """
-    # Setup logging
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    logger.info("TS Pini Loader started")
-    logger.info(f"Processing file: {file_path}")
-    logger.warning("NOTE: Alarm system not yet fully implemented in this refactored version")
-
-    try:
-        # Load configuration
-        db_config = DatabaseConfig()
-
-        # Process file
-        async with TSPiniLoader(db_config) as loader:
-            success = await loader.process_file(file_path)
-
-        if success:
-            logger.info("Processing completed successfully")
-            return 0
-        else:
-            logger.error("Processing failed")
-            return 1
-
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-        return 1
-
-    finally:
-        logger.info("TS Pini Loader finished")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python ts_pini_loader.py <path_to_csv_file>")
-        print("\nNOTE: This is an essential refactoring of the legacy TS_PiniScript.py")
-        print("      Core functionality (data loading, coordinates) is implemented.")
-        print("      Alarm system and additional monitoring require completion.")
-        sys.exit(1)
-
-    exit_code = asyncio.run(main(sys.argv[1]))
-    sys.exit(exit_code)

vm1/src/refactory_scripts/loaders/vulink_loader.py

@@ -1,392 +0,0 @@
-"""
-Vulink data loader - Refactored version with async support.
-
-This script processes Vulink CSV files and loads data into the database.
-Handles battery level monitoring and pH threshold alarms.
-Replaces the legacy vulinkScript.py with modern async/await patterns.
-"""
-
-import asyncio
-import json
-import logging
-import sys
-from datetime import datetime, timedelta
-from pathlib import Path
-
-from refactory_scripts.config import DatabaseConfig
-from refactory_scripts.utils import execute_query, get_db_connection
-
-logger = logging.getLogger(__name__)
-
-
-class VulinkLoader:
-    """Loads Vulink sensor data from CSV files into the database with alarm management."""
-
-    # Node type constants
-    NODE_TYPE_PIEZO = 2
-    NODE_TYPE_BARO = 3
-    NODE_TYPE_CONDUCTIVITY = 4
-    NODE_TYPE_PH = 5
-
-    # Battery threshold
-    BATTERY_LOW_THRESHOLD = 25.0
-    BATTERY_ALARM_INTERVAL_HOURS = 24
-
-    def __init__(self, db_config: DatabaseConfig):
-        """
-        Initialize the Vulink loader.
-
-        Args:
-            db_config: Database configuration object
-        """
-        self.db_config = db_config
-        self.conn = None
-
-    async def __aenter__(self):
-        """Async context manager entry."""
-        self.conn = await get_db_connection(self.db_config.as_dict())
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        if self.conn:
-            self.conn.close()
-
-    def _extract_metadata(self, file_path: Path) -> str:
-        """
-        Extract serial number from filename.
-
-        Args:
-            file_path: Path to the CSV file
-
-        Returns:
-            Serial number string
-        """
-        file_name = file_path.stem
-        serial_number = file_name.split("_")[0]
-        logger.debug(f"Extracted serial number: {serial_number}")
-        return serial_number
-
-    async def _get_unit_and_tool(self, serial_number: str) -> tuple[str, str] | None:
-        """
-        Get unit name and tool name from serial number.
-
-        Args:
-            serial_number: Device serial number
-
-        Returns:
-            Tuple of (unit_name, tool_name) or None if not found
-        """
-        query = "SELECT unit_name, tool_name FROM vulink_tools WHERE serial_number = %s"
-        result = await execute_query(self.conn, query, (serial_number,), fetch_one=True)
-
-        if result:
-            unit_name = result["unit_name"]
-            tool_name = result["tool_name"]
-            logger.info(f"Serial {serial_number} -> Unit: {unit_name}, Tool: {tool_name}")
-            return unit_name, tool_name
-
-        logger.error(f"Serial number {serial_number} not found in vulink_tools table")
-        return None
-
-    async def _get_node_configuration(
-        self, unit_name: str, tool_name: str
-    ) -> dict[int, dict]:
-        """
-        Get node configuration including depth and thresholds.
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-
-        Returns:
-            Dictionary mapping node numbers to their configuration
-        """
-        query = """
-            SELECT t.soglie, n.num as node_num, n.nodetype_id, n.depth
-            FROM nodes AS n
-            LEFT JOIN tools AS t ON n.tool_id = t.id
-            LEFT JOIN units AS u ON u.id = t.unit_id
-            WHERE u.name = %s AND t.name = %s
-        """
-
-        results = await execute_query(self.conn, query, (unit_name, tool_name), fetch_all=True)
-
-        node_config = {}
-        for row in results:
-            node_num = row["node_num"]
-            node_config[node_num] = {
-                "nodetype_id": row["nodetype_id"],
-                "depth": row.get("depth"),
-                "thresholds": row.get("soglie"),
-            }
-
-        logger.debug(f"Loaded configuration for {len(node_config)} nodes")
-        return node_config
-
-    async def _check_battery_alarm(self, unit_name: str, date_time: str, battery_perc: float) -> None:
-        """
-        Check battery level and create alarm if necessary.
-
-        Args:
-            unit_name: Unit name
-            date_time: Current datetime string
-            battery_perc: Battery percentage
-        """
-        if battery_perc >= self.BATTERY_LOW_THRESHOLD:
-            return  # Battery level is fine
-
-        logger.warning(f"Low battery detected for {unit_name}: {battery_perc}%")
-
-        # Check if we already have a recent battery alarm
-        query = """
-            SELECT unit_name, date_time
-            FROM alarms
-            WHERE unit_name = %s AND date_time < %s AND type_id = 2
-            ORDER BY date_time DESC
-            LIMIT 1
-        """
-
-        result = await execute_query(self.conn, query, (unit_name, date_time), fetch_one=True)
-
-        should_create_alarm = False
-
-        if result:
-            alarm_date_time = result["date_time"]
-            dt1 = datetime.strptime(date_time, "%Y-%m-%d %H:%M")
-
-            time_difference = abs(dt1 - alarm_date_time)
-
-            if time_difference > timedelta(hours=self.BATTERY_ALARM_INTERVAL_HOURS):
-                logger.info(f"Previous alarm was more than {self.BATTERY_ALARM_INTERVAL_HOURS}h ago, creating new alarm")
-                should_create_alarm = True
-        else:
-            logger.info("No previous battery alarm found, creating new alarm")
-            should_create_alarm = True
-
-        if should_create_alarm:
-            await self._create_battery_alarm(unit_name, date_time, battery_perc)
-
-    async def _create_battery_alarm(self, unit_name: str, date_time: str, battery_perc: float) -> None:
-        """
-        Create a battery level alarm.
-
-        Args:
-            unit_name: Unit name
-            date_time: Datetime string
-            battery_perc: Battery percentage
-        """
-        query = """
-            INSERT IGNORE INTO alarms
-            (type_id, unit_name, date_time, battery_level, description, send_email, send_sms)
-            VALUES (%s, %s, %s, %s, %s, %s, %s)
-        """
-
-        params = (2, unit_name, date_time, battery_perc, "Low battery <25%", 1, 0)
-
-        await execute_query(self.conn, query, params)
-        logger.warning(f"Battery alarm created for {unit_name} at {date_time}: {battery_perc}%")
-
-    async def _check_ph_threshold(
-        self,
-        unit_name: str,
-        tool_name: str,
-        node_num: int,
-        date_time: str,
-        ph_value: float,
-        thresholds_json: str,
-    ) -> None:
-        """
-        Check pH value against thresholds and create alarm if necessary.
-
-        Args:
-            unit_name: Unit name
-            tool_name: Tool name
-            node_num: Node number
-            date_time: Datetime string
-            ph_value: Current pH value
-            thresholds_json: JSON string with threshold configuration
-        """
-        if not thresholds_json:
-            return
-
-        try:
-            thresholds = json.loads(thresholds_json)
-            ph_config = next((item for item in thresholds if item.get("type") == "PH Link"), None)
-
-            if not ph_config or not ph_config["data"].get("ph"):
-                return  # pH monitoring not enabled
-
-            data = ph_config["data"]
-
-            # Get previous pH value
-            query = """
-                SELECT XShift, EventDate, EventTime
-                FROM ELABDATADISP
-                WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s
-                AND CONCAT(EventDate, ' ', EventTime) < %s
-                ORDER BY CONCAT(EventDate, ' ', EventTime) DESC
-                LIMIT 1
-            """
-
-            result = await execute_query(self.conn, query, (unit_name, tool_name, node_num, date_time), fetch_one=True)
-
-            ph_value_prev = float(result["XShift"]) if result else 0.0
-
-            # Check each threshold level (3 = highest, 1 = lowest)
-            for level, level_name in [(3, "tre"), (2, "due"), (1, "uno")]:
-                enabled_key = f"ph_{level_name}"
-                value_key = f"ph_{level_name}_value"
-                email_key = f"ph_{level_name}_email"
-                sms_key = f"ph_{level_name}_sms"
-
-                if (
-                    data.get(enabled_key)
-                    and data.get(value_key)
-                    and float(ph_value) > float(data[value_key])
-                    and ph_value_prev <= float(data[value_key])
-                ):
-                    # Threshold crossed, create alarm
-                    await self._create_ph_alarm(
-                        tool_name,
-                        unit_name,
-                        node_num,
-                        date_time,
-                        ph_value,
-                        level,
-                        data[email_key],
-                        data[sms_key],
-                    )
-                    logger.info(f"pH alarm level {level} triggered for {unit_name}/{tool_name}/node{node_num}")
-                    break  # Only trigger highest level alarm
-
-        except (json.JSONDecodeError, KeyError, TypeError) as e:
-            logger.error(f"Failed to parse pH thresholds: {e}")
-
-    async def _create_ph_alarm(
-        self,
-        tool_name: str,
-        unit_name: str,
-        node_num: int,
-        date_time: str,
-        ph_value: float,
-        level: int,
-        send_email: bool,
-        send_sms: bool,
-    ) -> None:
-        """
-        Create a pH threshold alarm.
-
-        Args:
-            tool_name: Tool name
-            unit_name: Unit name
-            node_num: Node number
-            date_time: Datetime string
-            ph_value: pH value
-            level: Alarm level (1-3)
-            send_email: Whether to send email
-            send_sms: Whether to send SMS
-        """
-        query = """
-            INSERT IGNORE INTO alarms
-            (type_id, tool_name, unit_name, date_time, registered_value, node_num, alarm_level, description, send_email, send_sms)
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-        """
-
-        params = (3, tool_name, unit_name, date_time, ph_value, node_num, level, "pH", send_email, send_sms)
-
-        await execute_query(self.conn, query, params)
-        logger.warning(
-            f"pH alarm level {level} created for {unit_name}/{tool_name}/node{node_num}: {ph_value} at {date_time}"
-        )
-
-    async def process_file(self, file_path: str | Path) -> bool:
-        """
-        Process a Vulink CSV file and load data into the database.
-
-        Args:
-            file_path: Path to the CSV file to process
-
-        Returns:
-            True if processing was successful, False otherwise
-        """
-        file_path = Path(file_path)
-
-        if not file_path.exists():
-            logger.error(f"File not found: {file_path}")
-            return False
-
-        try:
-            # Extract serial number
-            serial_number = self._extract_metadata(file_path)
-
-            # Get unit and tool names
-            unit_tool = await self._get_unit_and_tool(serial_number)
-            if not unit_tool:
-                return False
-
-            unit_name, tool_name = unit_tool
-
-            # Get node configuration
-            node_config = await self._get_node_configuration(unit_name, tool_name)
-
-            if not node_config:
-                logger.error(f"No node configuration found for {unit_name}/{tool_name}")
-                return False
-
-            # Parse CSV file (implementation depends on CSV format)
-            logger.info(f"Processing Vulink file: {file_path.name}")
-            logger.info(f"Unit: {unit_name}, Tool: {tool_name}")
-            logger.info(f"Nodes configured: {len(node_config)}")
-
-            # Note: Actual CSV parsing and data insertion logic would go here
-            # This requires knowledge of the specific Vulink CSV format
-            logger.warning("CSV parsing not fully implemented - requires Vulink CSV format specification")
-
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to process file {file_path}: {e}", exc_info=True)
-            return False
-
-
-async def main(file_path: str):
-    """
-    Main entry point for the Vulink loader.
-
-    Args:
-        file_path: Path to the CSV file to process
-    """
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    logger.info("Vulink Loader started")
-    logger.info(f"Processing file: {file_path}")
-
-    try:
-        db_config = DatabaseConfig()
-
-        async with VulinkLoader(db_config) as loader:
-            success = await loader.process_file(file_path)
-
-        if success:
-            logger.info("Processing completed successfully")
-            return 0
-        else:
-            logger.error("Processing failed")
-            return 1
-
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-        return 1
-
-    finally:
-        logger.info("Vulink Loader finished")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python vulink_loader.py <path_to_csv_file>")
-        sys.exit(1)
-
-    exit_code = asyncio.run(main(sys.argv[1]))
-    sys.exit(exit_code)

vm1/src/refactory_scripts/utils/__init__.py

@@ -1,178 +0,0 @@
-"""Utility functions for refactored scripts."""
-
-import asyncio
-import logging
-from datetime import datetime
-from typing import Any, Optional
-
-import aiomysql
-
-logger = logging.getLogger(__name__)
-
-
-async def get_db_connection(config: dict) -> aiomysql.Connection:
-    """
-    Create an async database connection.
-
-    Args:
-        config: Database configuration dictionary
-
-    Returns:
-        aiomysql.Connection: Async database connection
-
-    Raises:
-        Exception: If connection fails
-    """
-    try:
-        conn = await aiomysql.connect(**config)
-        logger.debug("Database connection established")
-        return conn
-    except Exception as e:
-        logger.error(f"Failed to connect to database: {e}")
-        raise
-
-
-async def execute_query(
-    conn: aiomysql.Connection,
-    query: str,
-    params: tuple | list = None,
-    fetch_one: bool = False,
-    fetch_all: bool = False,
-) -> Any | None:
-    """
-    Execute a database query safely with proper error handling.
-
-    Args:
-        conn: Database connection
-        query: SQL query string
-        params: Query parameters
-        fetch_one: Whether to fetch one result
-        fetch_all: Whether to fetch all results
-
-    Returns:
-        Query results or None
-
-    Raises:
-        Exception: If query execution fails
-    """
-    async with conn.cursor(aiomysql.DictCursor) as cursor:
-        try:
-            await cursor.execute(query, params or ())
-
-            if fetch_one:
-                return await cursor.fetchone()
-            elif fetch_all:
-                return await cursor.fetchall()
-
-            return None
-
-        except Exception as e:
-            logger.error(f"Query execution failed: {e}")
-            logger.debug(f"Query: {query}")
-            logger.debug(f"Params: {params}")
-            raise
-
-
-async def execute_many(conn: aiomysql.Connection, query: str, params_list: list) -> int:
-    """
-    Execute a query with multiple parameter sets (batch insert).
-
-    Args:
-        conn: Database connection
-        query: SQL query string
-        params_list: List of parameter tuples
-
-    Returns:
-        Number of affected rows
-
-    Raises:
-        Exception: If query execution fails
-    """
-    if not params_list:
-        logger.warning("execute_many called with empty params_list")
-        return 0
-
-    async with conn.cursor() as cursor:
-        try:
-            await cursor.executemany(query, params_list)
-            affected_rows = cursor.rowcount
-            logger.debug(f"Batch insert completed: {affected_rows} rows affected")
-            return affected_rows
-
-        except Exception as e:
-            logger.error(f"Batch query execution failed: {e}")
-            logger.debug(f"Query: {query}")
-            logger.debug(f"Number of parameter sets: {len(params_list)}")
-            raise
-
-
-def parse_datetime(date_str: str, time_str: str = None) -> datetime:
-    """
-    Parse date and optional time strings into datetime object.
-
-    Args:
-        date_str: Date string (various formats supported)
-        time_str: Optional time string
-
-    Returns:
-        datetime object
-
-    Examples:
-        >>> parse_datetime("2024-10-11", "14:30:00")
-        datetime(2024, 10, 11, 14, 30, 0)
-
-        >>> parse_datetime("2024-10-11T14:30:00")
-        datetime(2024, 10, 11, 14, 30, 0)
-    """
-    # Handle ISO format with T separator
-    if "T" in date_str:
-        return datetime.fromisoformat(date_str.replace("T", " "))
-
-    # Handle separate date and time
-    if time_str:
-        return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
-
-    # Handle date only
-    return datetime.strptime(date_str, "%Y-%m-%d")
-
-
-async def retry_on_failure(
-    coro_func,
-    max_retries: int = 3,
-    delay: float = 1.0,
-    backoff: float = 2.0,
-    *args,
-    **kwargs,
-):
-    """
-    Retry an async function on failure with exponential backoff.
-
-    Args:
-        coro_func: Async function to retry
-        max_retries: Maximum number of retry attempts
-        delay: Initial delay between retries (seconds)
-        backoff: Backoff multiplier for delay
-        *args: Arguments to pass to coro_func
-        **kwargs: Keyword arguments to pass to coro_func
-
-    Returns:
-        Result from coro_func
-
-    Raises:
-        Exception: If all retries fail
-    """
-    last_exception = None
-
-    for attempt in range(max_retries):
-        try:
-            return await coro_func(*args, **kwargs)
-        except Exception as e:
-            last_exception = e
-            if attempt < max_retries - 1:
-                wait_time = delay * (backoff**attempt)
-                logger.warning(f"Attempt {attempt + 1}/{max_retries} failed: {e}. Retrying in {wait_time}s...")
-                await asyncio.sleep(wait_time)
-            else:
-                logger.error(f"All {max_retries} attempts failed")
-
-    raise last_exception

vm1/src/send_orchestrator.py

@@ -1,92 +0,0 @@
-#!.venv/bin/python
-"""
-Orchestratore dei worker che inviano i dati ai clienti
-"""
-
-# Import necessary libraries
-import asyncio
-import logging
-
-# Import custom modules for configuration and database connection
-from utils.config import loader_send_data as setting
-from utils.connect.send_data import process_workflow_record
-from utils.csv.loaders import get_next_csv_atomic
-from utils.database import WorkflowFlags
-from utils.general import alterna_valori
-from utils.orchestrator_utils import run_orchestrator, shutdown_event, worker_context
-
-# from utils.ftp.send_data import   ftp_send_elab_csv_to_customer, api_send_elab_csv_to_customer, \
-#                                   ftp_send_raw_csv_to_customer, api_send_raw_csv_to_customer
-
-
-# Initialize the logger for this module
-logger = logging.getLogger()
-
-# Delay tra un processamento CSV e il successivo (in secondi)
-ELAB_PROCESSING_DELAY = 0.2
-# Tempo di attesa se non ci sono record da elaborare
-NO_RECORD_SLEEP = 30
-
-
-async def worker(worker_id: int, cfg: dict, pool: object) -> None:
-    """Esegue il ciclo di lavoro per l'invio dei dati.
-
-    Il worker preleva un record dal database che indica dati pronti per
-    l'invio (sia raw che elaborati), li processa e attende prima di
-    iniziare un nuovo ciclo.
-
-    Supporta graceful shutdown controllando il shutdown_event tra le iterazioni.
-
-    Args:
-        worker_id (int): L'ID univoco del worker.
-        cfg (dict): L'oggetto di configurazione.
-        pool (object): Il pool di connessioni al database.
-    """
-
-    # Imposta il context per questo worker
-    worker_context.set(f"W{worker_id:02d}")
-
-    debug_mode = logging.getLogger().getEffectiveLevel() == logging.DEBUG
-    logger.info("Avviato")
-
-    alternatore = alterna_valori(
-        [WorkflowFlags.CSV_RECEIVED, WorkflowFlags.SENT_RAW_DATA],
-        [WorkflowFlags.DATA_ELABORATED, WorkflowFlags.SENT_ELAB_DATA],
-    )
-
-    try:
-        while not shutdown_event.is_set():
-            try:
-                logger.info("Inizio elaborazione")
-
-                status, fase = next(alternatore)
-                record = await get_next_csv_atomic(pool, cfg.dbrectable, status, fase)
-
-                if record:
-                    await process_workflow_record(record, fase, cfg, pool)
-                    await asyncio.sleep(ELAB_PROCESSING_DELAY)
-                else:
-                    logger.info("Nessun record disponibile")
-                    await asyncio.sleep(NO_RECORD_SLEEP)
-
-            except asyncio.CancelledError:
-                logger.info("Worker cancellato. Uscita in corso...")
-                raise
-
-            except Exception as e:  # pylint: disable=broad-except
-                logger.error("Errore durante l'esecuzione: %s", e, exc_info=debug_mode)
-                await asyncio.sleep(1)
-
-    except asyncio.CancelledError:
-        logger.info("Worker terminato per shutdown graceful")
-    finally:
-        logger.info("Worker terminato")
-
-
-async def main():
-    """Funzione principale che avvia il send_orchestrator."""
-    await run_orchestrator(setting.Config, worker)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm1/src/utils/__init__.py

@@ -1 +0,0 @@
-"""Utilità"""

vm1/src/utils/authorizers/database_authorizer.py

@@ -1,177 +0,0 @@
-"""
-Database-backed authorizer for FTP server that checks authentication against database in real-time.
-This ensures multiple FTP server instances stay synchronized without needing restarts.
-"""
-
-import logging
-from hashlib import sha256
-from pathlib import Path
-
-from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
-
-from utils.database.connection import connetti_db
-
-logger = logging.getLogger(__name__)
-
-
-class DatabaseAuthorizer(DummyAuthorizer):
-    """
-    Custom authorizer that validates users against the database on every login.
-
-    This approach ensures that:
-    - Multiple FTP server instances stay synchronized
-    - User changes (add/remove/disable) are reflected immediately
-    - No server restart is needed when users are modified
-    """
-
-    def __init__(self, cfg: dict) -> None:
-        """
-        Initializes the authorizer with admin user only.
-        Regular users are validated against database at login time.
-
-        Args:
-            cfg: The configuration object.
-        """
-        super().__init__()
-        self.cfg = cfg
-
-        # Add admin user to in-memory authorizer (always available)
-        self.add_user(
-            cfg.adminuser[0],  # username
-            cfg.adminuser[1],  # password hash
-            cfg.adminuser[2],  # home directory
-            perm=cfg.adminuser[3]  # permissions
-        )
-
-        logger.info("DatabaseAuthorizer initialized with admin user")
-
-    def validate_authentication(self, username: str, password: str, handler: object) -> None:
-        """
-        Validates user authentication against the database.
-
-        This method is called on every login attempt and checks:
-        1. If user is admin, use in-memory credentials
-        2. Otherwise, query database for user credentials
-        3. Verify password hash matches
-        4. Ensure user is not disabled
-
-        Args:
-            username: The username attempting to login.
-            password: The plain-text password provided.
-            handler: The FTP handler object.
-
-        Raises:
-            AuthenticationFailed: If authentication fails for any reason.
-        """
-        # Hash the provided password
-        password_hash = sha256(password.encode("UTF-8")).hexdigest()
-
-        # Check if user is admin (stored in memory)
-        if username == self.cfg.adminuser[0]:
-            if self.user_table[username]["pwd"] != password_hash:
-                logger.warning(f"Failed admin login attempt for user: {username}")
-                raise AuthenticationFailed("Invalid credentials")
-            return
-
-        # For regular users, check database
-        try:
-            conn = connetti_db(self.cfg)
-            cur = conn.cursor()
-
-            # Query user from database
-            cur.execute(
-                f"SELECT ftpuser, hash, virtpath, perm, disabled_at FROM {self.cfg.dbname}.{self.cfg.dbusertable} WHERE ftpuser = %s",
-                (username,)
-            )
-
-            result = cur.fetchone()
-            cur.close()
-            conn.close()
-
-            if not result:
-                logger.warning(f"Login attempt for non-existent user: {username}")
-                raise AuthenticationFailed("Invalid credentials")
-
-            ftpuser, stored_hash, virtpath, perm, disabled_at = result
-
-            # Check if user is disabled
-            if disabled_at is not None:
-                logger.warning(f"Login attempt for disabled user: {username}")
-                raise AuthenticationFailed("User account is disabled")
-
-            # Verify password
-            if stored_hash != password_hash:
-                logger.warning(f"Invalid password for user: {username}")
-                raise AuthenticationFailed("Invalid credentials")
-
-            # Authentication successful - ensure user directory exists
-            try:
-                Path(virtpath).mkdir(parents=True, exist_ok=True)
-            except Exception as e:
-                logger.error(f"Failed to create directory for user {username}: {e}")
-                raise AuthenticationFailed("System error")
-
-            # Temporarily add user to in-memory table for this session
-            # This allows pyftpdlib to work correctly for the duration of the session
-            # We add/update directly to avoid issues with add_user() checking if user exists
-            if username in self.user_table:
-                # User already exists, just update credentials
-                self.user_table[username]['pwd'] = stored_hash
-                self.user_table[username]['home'] = virtpath
-                self.user_table[username]['perm'] = perm
-                self.user_table[username]['operms'] = {}
-            else:
-                # User doesn't exist, add to table directly with all required fields
-                self.user_table[username] = {
-                    'pwd': stored_hash,
-                    'home': virtpath,
-                    'perm': perm,
-                    'operms': {},  # Optional per-directory permissions
-                    'msg_login': '230 Login successful.',
-                    'msg_quit': '221 Goodbye.'
-                }
-
-            logger.info(f"Successful login for user: {username}")
-
-        except AuthenticationFailed:
-            raise
-        except Exception as e:
-            logger.error(f"Database error during authentication for user {username}: {e}", exc_info=True)
-            raise AuthenticationFailed("System error")
-
-    def has_user(self, username: str) -> bool:
-        """
-        Check if a user exists in the database or in-memory table.
-
-        This is called by pyftpdlib for various checks. We override it to check
-        the database as well as the in-memory table.
-
-        Args:
-            username: The username to check.
-
-        Returns:
-            True if user exists and is enabled, False otherwise.
-        """
-        # Check in-memory first (for admin and active sessions)
-        if username in self.user_table:
-            return True
-
-        # Check database for regular users
-        try:
-            conn = connetti_db(self.cfg)
-            cur = conn.cursor()
-
-            cur.execute(
-                f"SELECT COUNT(*) FROM {self.cfg.dbname}.{self.cfg.dbusertable} WHERE ftpuser = %s AND disabled_at IS NULL",
-                (username,)
-            )
-
-            count = cur.fetchone()[0]
-            cur.close()
-            conn.close()
-
-            return count > 0
-
-        except Exception as e:
-            logger.error(f"Database error checking user existence for {username}: {e}")
-            return False

vm1/src/utils/config/__init__.py

@@ -1,4 +0,0 @@
-"""Config ini setting"""
-from pathlib import Path
-
-ENV_PARENT_PATH = Path(__file__).resolve().parent.parent.parent.parent

vm1/src/utils/config/loader_email.py

@@ -1,25 +0,0 @@
-"""set configurations"""
-
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    def __init__(self):
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/email.ini"])
-
-        # email setting
-        self.from_addr = c.get("address", "from")
-        self.to_addr = c.get("address", "to")
-        self.cc_addr = c.get("address", "cc")
-        self.bcc_addr = c.get("address", "bcc")
-
-        self.subject = c.get("msg", "subject")
-        self.body = c.get("msg", "body")
-
-        self.smtp_addr = c.get("smtp", "address")
-        self.smtp_port = c.getint("smtp", "port")
-        self.smtp_user = c.get("smtp", "user")
-        self.smtp_passwd = c.get("smtp", "password")

vm1/src/utils/config/loader_ftp_csv.py

@@ -1,83 +0,0 @@
-"""set configurations"""
-
-import os
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    def __init__(self):
-        """
-        Initializes the Config class by reading configuration files.
-        It loads settings from 'ftp.ini' and 'db.ini' for FTP server, CSV, logging, and database.
-        Environment variables override INI file settings for Docker deployments.
-        """
-
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/ftp.ini", f"{ENV_PARENT_PATH}/env/db.ini"])
-
-        # FTP setting (with environment variable override for Docker)
-        self.service_port = int(os.getenv("FTP_PORT", c.getint("ftpserver", "service_port")))
-
-        # FTP_PASSIVE_PORTS: override della porta iniziale del range passivo
-        self.firstport = int(os.getenv("FTP_PASSIVE_PORTS", c.getint("ftpserver", "firstPort")))
-
-        # FTP_EXTERNAL_IP: override dell'IP pubblicizzato (VIP per HA)
-        self.proxyaddr = os.getenv("FTP_EXTERNAL_IP", c.get("ftpserver", "proxyAddr"))
-
-        self.portrangewidth = c.getint("ftpserver", "portRangeWidth")
-        self.virtpath = c.get("ftpserver", "virtpath")
-        self.adminuser = c.get("ftpserver", "adminuser").split("|")
-        self.servertype = c.get("ftpserver", "servertype")
-        self.certfile = c.get("ftpserver", "certfile")
-        self.fileext = c.get("ftpserver", "fileext").upper().split("|")
-        self.defperm = c.get("ftpserver", "defaultUserPerm")
-
-        # File processing behavior: delete files after successful processing
-        # Set DELETE_AFTER_PROCESSING=true in docker-compose to enable
-        self.delete_after_processing = os.getenv("DELETE_AFTER_PROCESSING", "false").lower() in ("true", "1", "yes")
-
-        # CSV FILE setting
-        self.csvfs = c.get("csvfs", "path")
-
-        # LOG setting
-        self.logfilename = c.get("logging", "logFilename")
-
-        # DB setting (with environment variable override for Docker)
-        self.dbhost = os.getenv("DB_HOST", c.get("db", "hostname"))
-        self.dbport = int(os.getenv("DB_PORT", c.getint("db", "port")))
-        self.dbuser = os.getenv("DB_USER", c.get("db", "user"))
-        self.dbpass = os.getenv("DB_PASSWORD", c.get("db", "password"))
-        self.dbname = os.getenv("DB_NAME", c.get("db", "dbName"))
-        self.max_retries = c.getint("db", "maxRetries")
-
-        # Tables
-        self.dbusertable = c.get("tables", "userTableName")
-        self.dbrectable = c.get("tables", "recTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbnodes = c.get("tables", "nodesTableName")
-
-        # unit setting
-        self.units_name = list(c.get("unit", "Names").split("|"))
-        self.units_type = list(c.get("unit", "Types").split("|"))
-        self.units_alias = {key: value for item in c.get("unit", "Alias").split("|") for key, value in [item.split(":", 1)]}
-        # self.units_header = {key: int(value) for pair in c.get("unit", "Headers").split('|') for key, value in [pair.split(':')]}
-
-        # tool setting
-        self.tools_name = list(c.get("tool", "Names").split("|"))
-        self.tools_type = list(c.get("tool", "Types").split("|"))
-        self.tools_alias = {
-            key: key if value == "=" else value for item in c.get("tool", "Alias").split("|") for key, value in [item.split(":", 1)]
-        }
-
-        # csv info
-        self.csv_infos = list(c.get("csv", "Infos").split("|"))
-
-        # TS pini path match
-        self.ts_pini_path_match = {
-            key: key[1:-1] if value == "=" else value
-            for item in c.get("ts_pini", "path_match").split("|")
-            for key, value in [item.split(":", 1)]
-        }

vm1/src/utils/config/loader_load_data.py

@@ -1,37 +0,0 @@
-"""set configurations"""
-
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    def __init__(self):
-        """
-        Initializes the Config class by reading configuration files.
-        It loads settings from 'load.ini' and 'db.ini' for logging, worker, database, and table configurations.
-        """
-
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/load.ini", f"{ENV_PARENT_PATH}/env/db.ini"])
-
-        # LOG setting
-        self.logfilename = c.get("logging", "logFilename")
-
-        # Worker setting
-        self.max_threads = c.getint("threads", "max_num")
-
-        # DB setting
-        self.dbhost = c.get("db", "hostname")
-        self.dbport = c.getint("db", "port")
-        self.dbuser = c.get("db", "user")
-        self.dbpass = c.get("db", "password")
-        self.dbname = c.get("db", "dbName")
-        self.max_retries = c.getint("db", "maxRetries")
-
-        # Tables
-        self.dbusertable = c.get("tables", "userTableName")
-        self.dbrectable = c.get("tables", "recTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbnodes = c.get("tables", "nodesTableName")

vm1/src/utils/config/loader_matlab_elab.py

@@ -1,47 +0,0 @@
-"""set configurations"""
-
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    def __init__(self):
-        """
-        Initializes the Config class by reading configuration files.
-        It loads settings from 'elab.ini' and 'db.ini' for logging, worker, database, table, tool, and Matlab configurations.
-        """
-
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/elab.ini", f"{ENV_PARENT_PATH}/env/db.ini"])
-
-        # LOG setting
-        self.logfilename = c.get("logging", "logFilename")
-
-        # Worker setting
-        self.max_threads = c.getint("threads", "max_num")
-
-        # DB setting
-        self.dbhost = c.get("db", "hostname")
-        self.dbport = c.getint("db", "port")
-        self.dbuser = c.get("db", "user")
-        self.dbpass = c.get("db", "password")
-        self.dbname = c.get("db", "dbName")
-        self.max_retries = c.getint("db", "maxRetries")
-
-        # Tables
-        self.dbusertable = c.get("tables", "userTableName")
-        self.dbrectable = c.get("tables", "recTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbnodes = c.get("tables", "nodesTableName")
-
-        # Tool
-        self.elab_status = list(c.get("tool", "elab_status").split("|"))
-
-        # Matlab
-        self.matlab_runtime = c.get("matlab", "runtime")
-        self.matlab_func_path = c.get("matlab", "func_path")
-        self.matlab_timeout = c.getint("matlab", "timeout")
-        self.matlab_error = c.get("matlab", "error")
-        self.matlab_error_path = c.get("matlab", "error_path")

vm1/src/utils/config/loader_send_data.py

@@ -1,37 +0,0 @@
-"""set configurations"""
-
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    def __init__(self):
-        """
-        Initializes the Config class by reading configuration files.
-        It loads settings from 'send.ini' and 'db.ini' for logging, worker, database, and table configurations.
-        """
-
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/send.ini", f"{ENV_PARENT_PATH}/env/db.ini"])
-
-        # LOG setting
-        self.logfilename = c.get("logging", "logFilename")
-
-        # Worker setting
-        self.max_threads = c.getint("threads", "max_num")
-
-        # DB setting
-        self.dbhost = c.get("db", "hostname")
-        self.dbport = c.getint("db", "port")
-        self.dbuser = c.get("db", "user")
-        self.dbpass = c.get("db", "password")
-        self.dbname = c.get("db", "dbName")
-        self.max_retries = c.getint("db", "maxRetries")
-
-        # Tables
-        self.dbusertable = c.get("tables", "userTableName")
-        self.dbrectable = c.get("tables", "recTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbrawdata = c.get("tables", "rawTableName")
-        self.dbnodes = c.get("tables", "nodesTableName")

vm1/src/utils/config/users_loader.py

@@ -1,23 +0,0 @@
-"""set configurations"""
-
-from configparser import ConfigParser
-
-from . import ENV_PARENT_PATH
-
-
-class Config:
-    """
-    Handles configuration loading for database settings to load ftp users.
-    """
-
-    def __init__(self):
-        c = ConfigParser()
-        c.read([f"{ENV_PARENT_PATH}/env/db.ini"])
-
-        # DB setting
-        self.dbhost = c.get("db", "hostname")
-        self.dbport = c.getint("db", "port")
-        self.dbuser = c.get("db", "user")
-        self.dbpass = c.get("db", "password")
-        self.dbname = c.get("db", "dbName")
-        self.max_retries = c.getint("db", "maxRetries")

vm1/src/utils/connect/file_management.py

@@ -1,131 +0,0 @@
-import asyncio
-import logging
-import os
-import re
-from datetime import datetime
-
-from utils.csv.parser import extract_value
-from utils.database.connection import connetti_db_async
-
-logger = logging.getLogger(__name__)
-
-
-def on_file_received(self: object, file: str) -> None:
-    """
-    Wrapper sincrono per on_file_received_async.
-
-    Questo wrapper permette di mantenere la compatibilità con il server FTP
-    che si aspetta una funzione sincrona, mentre internamente usa asyncio.
-    """
-    asyncio.run(on_file_received_async(self, file))
-
-
-async def on_file_received_async(self: object, file: str) -> None:
-    """
-    Processes a received file, extracts relevant information, and inserts it into the database.
-
-    If the file is empty, it is removed. Otherwise, it extracts unit and tool
-    information from the filename and the first few lines of the CSV, handles
-    aliases, and then inserts the data into the configured database table.
-
-    Args:
-        file (str): The path to the received file."""
-
-    if not os.stat(file).st_size:
-        os.remove(file)
-        logger.info(f"File {file} is empty: removed.")
-    else:
-        cfg = self.cfg
-        path, filenameExt = os.path.split(file)
-        filename, fileExtension = os.path.splitext(filenameExt)
-        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
-        new_filename = f"{filename}_{timestamp}{fileExtension}"
-        os.rename(file, f"{path}/{new_filename}")
-        if fileExtension.upper() in (cfg.fileext):
-            with open(f"{path}/{new_filename}", encoding="utf-8", errors="ignore") as csvfile:
-                lines = csvfile.readlines()
-
-                unit_name = extract_value(cfg.units_name, filename, str(lines[0:10]))
-                unit_type = extract_value(cfg.units_type, filename, str(lines[0:10]))
-                tool_name = extract_value(cfg.tools_name, filename, str(lines[0:10]))
-                tool_type = extract_value(cfg.tools_type, filename, str(lines[0:10]))
-                tool_info = "{}"
-
-                # se esiste l'alias in alias_unit_type, allora prende il valore dell'alias
-                # verifica sia lo unit_type completo che i primi 3 caratteri per CO_xxxxx
-                upper_unit_type = unit_type.upper()
-                unit_type = cfg.units_alias.get(upper_unit_type) or cfg.units_alias.get(upper_unit_type[:3]) or upper_unit_type
-                upper_tool_type = tool_type.upper()
-                tool_type = cfg.tools_alias.get(upper_tool_type) or cfg.tools_alias.get(upper_tool_type[:3]) or upper_tool_type
-
-                try:
-                    # Use async database connection to avoid blocking
-                    conn = await connetti_db_async(cfg)
-                except Exception as e:
-                    logger.error(f"Database connection error: {e}")
-                    return
-
-                try:
-                    # Create a cursor
-                    async with conn.cursor() as cur:
-                        # da estrarre in un modulo
-                        if unit_type.upper() == "ISI CSV LOG" and tool_type.upper() == "VULINK":
-                            serial_number = filename.split("_")[0]
-                            tool_info = f'{{"serial_number": {serial_number}}}'
-                            try:
-                                # Use parameterized query to prevent SQL injection
-                                await cur.execute(
-                                    f"SELECT unit_name, tool_name FROM {cfg.dbname}.vulink_tools WHERE serial_number = %s", (serial_number,)
-                                )
-                                result = await cur.fetchone()
-                                if result:
-                                    unit_name, tool_name = result
-                            except Exception as e:
-                                logger.warning(f"{tool_type} serial number {serial_number} not found in table vulink_tools. {e}")
-
-                        # da estrarre in un modulo
-                        if unit_type.upper() == "STAZIONETOTALE" and tool_type.upper() == "INTEGRITY MONITOR":
-                            escaped_keys = [re.escape(key) for key in cfg.ts_pini_path_match.keys()]
-                            stazione = extract_value(escaped_keys, filename)
-                            if stazione:
-                                tool_info = f'{{"Stazione": "{cfg.ts_pini_path_match.get(stazione)}"}}'
-
-                        # Insert file data into database
-                        await cur.execute(
-                            f"""INSERT INTO {cfg.dbname}.{cfg.dbrectable}
-                            (username, filename, unit_name, unit_type, tool_name, tool_type, tool_data, tool_info)
-                            VALUES (%s,%s, %s, %s, %s, %s, %s, %s)""",
-                            (
-                                self.username,
-                                new_filename,
-                                unit_name.upper(),
-                                unit_type.upper(),
-                                tool_name.upper(),
-                                tool_type.upper(),
-                                "".join(lines),
-                                tool_info,
-                            ),
-                        )
-                        # Note: autocommit=True in connection, no need for explicit commit
-                        logger.info(f"File {new_filename} loaded successfully")
-
-                        # Delete file after successful processing if configured
-                        if getattr(cfg, 'delete_after_processing', False):
-                            try:
-                                os.remove(f"{path}/{new_filename}")
-                                logger.info(f"File {new_filename} deleted after successful processing")
-                            except Exception as e:
-                                logger.warning(f"Failed to delete file {new_filename}: {e}")
-
-                except Exception as e:
-                    logger.error(f"File {new_filename} not loaded. Held in user path.")
-                    logger.error(f"{e}")
-
-                finally:
-                    # Always close the connection
-                    conn.close()
-                """
-                else:
-                    os.remove(file)
-                    logger.info(f'File {new_filename} removed.')
-                """

vm1/src/utils/connect/send_data.py

@@ -1,655 +0,0 @@
-import logging
-import ssl
-from datetime import datetime
-from io import BytesIO
-
-import aioftp
-import aiomysql
-
-from utils.database import WorkflowFlags
-from utils.database.action_query import get_data_as_csv, get_elab_timestamp, get_tool_info
-from utils.database.loader_action import unlock, update_status
-
-logger = logging.getLogger(__name__)
-
-
-class AsyncFTPConnection:
-    """
-    Manages an async FTP or FTPS (TLS) connection with context manager support.
-
-    This class provides a fully asynchronous FTP client using aioftp, replacing
-    the blocking ftplib implementation for better performance in async workflows.
-
-    Args:
-        host (str): FTP server hostname or IP address
-        port (int): FTP server port (default: 21)
-        use_tls (bool): Use FTPS with TLS encryption (default: False)
-        user (str): Username for authentication (default: "")
-        passwd (str): Password for authentication (default: "")
-        passive (bool): Use passive mode (default: True)
-        timeout (float): Connection timeout in seconds (default: None)
-
-    Example:
-        async with AsyncFTPConnection(host="ftp.example.com", user="user", passwd="pass") as ftp:
-            await ftp.change_directory("/uploads")
-            await ftp.upload(data, "filename.csv")
-    """
-
-    def __init__(self, host: str, port: int = 21, use_tls: bool = False, user: str = "",
-                 passwd: str = "", passive: bool = True, timeout: float = None):
-        self.host = host
-        self.port = port
-        self.use_tls = use_tls
-        self.user = user
-        self.passwd = passwd
-        self.passive = passive
-        self.timeout = timeout
-        self.client = None
-
-    async def __aenter__(self):
-        """Async context manager entry: connect and login"""
-        # Create SSL context for FTPS if needed
-        ssl_context = None
-        if self.use_tls:
-            ssl_context = ssl.create_default_context()
-            ssl_context.check_hostname = False
-            ssl_context.verify_mode = ssl.CERT_NONE  # For compatibility with self-signed certs
-
-        # Create client with appropriate socket timeout
-        self.client = aioftp.Client(socket_timeout=self.timeout)
-
-        # Connect with optional TLS
-        if self.use_tls:
-            await self.client.connect(self.host, self.port, ssl=ssl_context)
-        else:
-            await self.client.connect(self.host, self.port)
-
-        # Login
-        await self.client.login(self.user, self.passwd)
-
-        # Set passive mode (aioftp uses passive by default, but we can configure if needed)
-        # Note: aioftp doesn't have explicit passive mode setting like ftplib
-
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit: disconnect gracefully"""
-        if self.client:
-            try:
-                await self.client.quit()
-            except Exception as e:
-                logger.warning(f"Error during FTP disconnect: {e}")
-
-    async def change_directory(self, path: str):
-        """Change working directory on FTP server"""
-        await self.client.change_directory(path)
-
-    async def upload(self, data: bytes, filename: str) -> bool:
-        """
-        Upload data to FTP server.
-
-        Args:
-            data (bytes): Data to upload
-            filename (str): Remote filename
-
-        Returns:
-            bool: True if upload successful, False otherwise
-        """
-        try:
-            # aioftp expects a stream or path, so we use BytesIO
-            stream = BytesIO(data)
-            await self.client.upload_stream(stream, filename)
-            return True
-        except Exception as e:
-            logger.error(f"FTP upload error: {e}")
-            return False
-
-
-async def ftp_send_raw_csv_to_customer(cfg: dict, id: int, unit: str, tool: str, pool: object) -> bool:
-    """
-    Sends raw CSV data to a customer via FTP (async implementation).
-
-    Retrieves raw CSV data from the database (received.tool_data column),
-    then sends it to the customer via FTP using the unit's FTP configuration.
-
-    Args:
-        cfg (dict): Configuration dictionary.
-        id (int): The ID of the record being processed (used for logging and DB query).
-        unit (str): The name of the unit associated with the data.
-        tool (str): The name of the tool associated with the data.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the CSV data was sent successfully, False otherwise.
-    """
-    # Query per ottenere il CSV raw dal database
-    raw_data_query = f"""
-    SELECT tool_data
-    FROM {cfg.dbname}.{cfg.dbrectable}
-    WHERE id = %s
-    """
-
-    # Query per ottenere le info FTP
-    ftp_info_query = """
-    SELECT ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename_raw, ftp_target_raw, duedate
-    FROM units
-    WHERE name = %s
-    """
-
-    async with pool.acquire() as conn:
-        async with conn.cursor(aiomysql.DictCursor) as cur:
-            try:
-                # 1. Recupera il CSV raw dal database
-                await cur.execute(raw_data_query, (id,))
-                raw_data_result = await cur.fetchone()
-
-                if not raw_data_result or not raw_data_result.get("tool_data"):
-                    logger.error(f"id {id} - {unit} - {tool}: nessun dato raw (tool_data) trovato nel database")
-                    return False
-
-                csv_raw_data = raw_data_result["tool_data"]
-                logger.info(f"id {id} - {unit} - {tool}: estratto CSV raw dal database ({len(csv_raw_data)} bytes)")
-
-                # 2. Recupera configurazione FTP
-                await cur.execute(ftp_info_query, (unit,))
-                send_ftp_info = await cur.fetchone()
-
-                if not send_ftp_info:
-                    logger.error(f"id {id} - {unit} - {tool}: nessuna configurazione FTP trovata per unit")
-                    return False
-
-                # Verifica che ci siano configurazioni per raw data
-                if not send_ftp_info.get("ftp_filename_raw"):
-                    logger.warning(f"id {id} - {unit} - {tool}: ftp_filename_raw non configurato. Uso ftp_filename standard se disponibile")
-                    # Fallback al filename standard se raw non è configurato
-                    if not send_ftp_info.get("ftp_filename"):
-                        logger.error(f"id {id} - {unit} - {tool}: nessun filename FTP configurato")
-                        return False
-                    ftp_filename = send_ftp_info["ftp_filename"]
-                else:
-                    ftp_filename = send_ftp_info["ftp_filename_raw"]
-
-                # Target directory (con fallback)
-                ftp_target = send_ftp_info.get("ftp_target_raw") or send_ftp_info.get("ftp_target") or "/"
-
-                logger.info(f"id {id} - {unit} - {tool}: configurazione FTP raw estratta")
-
-            except Exception as e:
-                logger.error(f"id {id} - {unit} - {tool} - errore nella query per invio ftp raw: {e}")
-                return False
-
-            try:
-                # 3. Converti in bytes se necessario
-                if isinstance(csv_raw_data, str):
-                    csv_bytes = csv_raw_data.encode("utf-8")
-                else:
-                    csv_bytes = csv_raw_data
-
-                # 4. Parse parametri FTP
-                ftp_parms = await parse_ftp_parms(send_ftp_info["ftp_parm"] or "")
-                use_tls = "ssl_version" in ftp_parms
-                passive = ftp_parms.get("passive", True)
-                port = ftp_parms.get("port", 21)
-                timeout = ftp_parms.get("timeout", 30.0)
-
-                # 5. Async FTP connection e upload
-                async with AsyncFTPConnection(
-                    host=send_ftp_info["ftp_addrs"],
-                    port=port,
-                    use_tls=use_tls,
-                    user=send_ftp_info["ftp_user"],
-                    passwd=send_ftp_info["ftp_passwd"],
-                    passive=passive,
-                    timeout=timeout,
-                ) as ftp:
-                    # Change directory se necessario
-                    if ftp_target and ftp_target != "/":
-                        await ftp.change_directory(ftp_target)
-
-                    # Upload raw data
-                    success = await ftp.upload(csv_bytes, ftp_filename)
-
-                    if success:
-                        logger.info(f"id {id} - {unit} - {tool}: File raw {ftp_filename} inviato con successo via FTP")
-                        return True
-                    else:
-                        logger.error(f"id {id} - {unit} - {tool}: Errore durante l'upload FTP raw")
-                        return False
-
-            except Exception as e:
-                logger.error(f"id {id} - {unit} - {tool} - Errore FTP raw: {e}", exc_info=True)
-                return False
-
-
-async def ftp_send_elab_csv_to_customer(cfg: dict, id: int, unit: str, tool: str, csv_data: str, pool: object) -> bool:
-    """
-    Sends elaborated CSV data to a customer via FTP (async implementation).
-
-    Retrieves FTP connection details from the database based on the unit name,
-    then establishes an async FTP connection and uploads the CSV data.
-
-    This function now uses aioftp for fully asynchronous FTP operations,
-    eliminating blocking I/O that previously affected event loop performance.
-
-    Args:
-        cfg (dict): Configuration dictionary (not directly used in this function but passed for consistency).
-        id (int): The ID of the record being processed (used for logging).
-        unit (str): The name of the unit associated with the data.
-        tool (str): The name of the tool associated with the data.
-        csv_data (str): The CSV data as a string to be sent.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the CSV data was sent successfully, False otherwise.
-    """
-    query = """
-    SELECT ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename, ftp_target, duedate
-    FROM units
-    WHERE name = %s
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor(aiomysql.DictCursor) as cur:
-            try:
-                await cur.execute(query, (unit,))
-                send_ftp_info = await cur.fetchone()
-
-                if not send_ftp_info:
-                    logger.error(f"id {id} - {unit} - {tool}: nessun dato FTP trovato per unit")
-                    return False
-
-                logger.info(f"id {id} - {unit} - {tool}: estratti i dati per invio via ftp")
-
-            except Exception as e:
-                logger.error(f"id {id} - {unit} - {tool} - errore nella query per invio ftp: {e}")
-                return False
-
-            try:
-                # Convert to bytes
-                csv_bytes = csv_data.encode("utf-8")
-
-                # Parse FTP parameters
-                ftp_parms = await parse_ftp_parms(send_ftp_info["ftp_parm"])
-                use_tls = "ssl_version" in ftp_parms
-                passive = ftp_parms.get("passive", True)
-                port = ftp_parms.get("port", 21)
-                timeout = ftp_parms.get("timeout", 30.0)  # Default 30 seconds
-
-                # Async FTP connection
-                async with AsyncFTPConnection(
-                    host=send_ftp_info["ftp_addrs"],
-                    port=port,
-                    use_tls=use_tls,
-                    user=send_ftp_info["ftp_user"],
-                    passwd=send_ftp_info["ftp_passwd"],
-                    passive=passive,
-                    timeout=timeout,
-                ) as ftp:
-                    # Change directory if needed
-                    if send_ftp_info["ftp_target"] and send_ftp_info["ftp_target"] != "/":
-                        await ftp.change_directory(send_ftp_info["ftp_target"])
-
-                    # Upload file
-                    success = await ftp.upload(csv_bytes, send_ftp_info["ftp_filename"])
-
-                    if success:
-                        logger.info(f"id {id} - {unit} - {tool}: File {send_ftp_info['ftp_filename']} inviato con successo via FTP")
-                        return True
-                    else:
-                        logger.error(f"id {id} - {unit} - {tool}: Errore durante l'upload FTP")
-                        return False
-
-            except Exception as e:
-                logger.error(f"id {id} - {unit} - {tool} - Errore FTP: {e}", exc_info=True)
-                return False
-
-
-async def parse_ftp_parms(ftp_parms: str) -> dict:
-    """
-    Parses a string of FTP parameters into a dictionary.
-
-    Args:
-        ftp_parms (str): A string containing key-value pairs separated by commas,
-                         with keys and values separated by '=>'.
-
-    Returns:
-        dict: A dictionary where keys are parameter names (lowercase) and values are their parsed values.
-    """
-    # Rimuovere spazi e dividere per virgola
-    pairs = ftp_parms.split(",")
-    result = {}
-
-    for pair in pairs:
-        if "=>" in pair:
-            key, value = pair.split("=>", 1)
-            key = key.strip().lower()
-            value = value.strip().lower()
-
-            # Convertire i valori appropriati
-            if value.isdigit():
-                value = int(value)
-            elif value == "":
-                value = None
-
-            result[key] = value
-
-    return result
-
-
-async def process_workflow_record(record: tuple, fase: int, cfg: dict, pool: object):
-    """
-    Elabora un singolo record del workflow in base alla fase specificata.
-
-    Args:
-        record: Tupla contenente i dati del record
-        fase: Fase corrente del workflow
-        cfg: Configurazione
-        pool: Pool di connessioni al database
-    """
-    # Estrazione e normalizzazione dei dati del record
-    id, unit_type, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
-
-    try:
-        # Recupero informazioni principali
-        tool_elab_info = await get_tool_info(fase, unit_name.upper(), tool_name.upper(), pool)
-        if tool_elab_info:
-            timestamp_matlab_elab = await get_elab_timestamp(id, pool)
-
-            # Verifica se il processing può essere eseguito
-            if not _should_process(tool_elab_info, timestamp_matlab_elab):
-                logger.info(
-                    f"id {id} - {unit_name} - {tool_name} {tool_elab_info['duedate']}: invio dati non eseguito - due date raggiunta."
-                )
-
-                await update_status(cfg, id, fase, pool)
-                return
-
-            # Routing basato sulla fase
-            success = await _route_by_phase(fase, tool_elab_info, cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-
-            if success:
-                await update_status(cfg, id, fase, pool)
-        else:
-            await update_status(cfg, id, fase, pool)
-
-    except Exception as e:
-        logger.error(f"Errore durante elaborazione id {id} - {unit_name} - {tool_name}: {e}")
-        raise
-    finally:
-        await unlock(cfg, id, pool)
-
-
-def _should_process(tool_elab_info: dict, timestamp_matlab_elab: datetime) -> bool:
-    """
-    Determines if a record should be processed based on its due date.
-
-    Args:
-        tool_elab_info (dict): A dictionary containing information about the tool and its due date.
-        timestamp_matlab_elab (datetime): The timestamp of the last MATLAB elaboration.
-
-    Returns:
-        bool: True if the record should be processed, False otherwise."""
-    """Verifica se il record può essere processato basandosi sulla due date."""
-    duedate = tool_elab_info.get("duedate")
-
-    # Se non c'è duedate o è vuota/nulla, può essere processato
-    if not duedate or duedate in ("0000-00-00 00:00:00", ""):
-        return True
-
-    # Se timestamp_matlab_elab è None/null, usa il timestamp corrente
-    comparison_timestamp = timestamp_matlab_elab if timestamp_matlab_elab is not None else datetime.now()
-
-    # Converti duedate in datetime se è una stringa
-    if isinstance(duedate, str):
-        duedate = datetime.strptime(duedate, "%Y-%m-%d %H:%M:%S")
-
-    # Assicurati che comparison_timestamp sia datetime
-    if isinstance(comparison_timestamp, str):
-        comparison_timestamp = datetime.strptime(comparison_timestamp, "%Y-%m-%d %H:%M:%S")
-
-    return duedate > comparison_timestamp
-
-
-async def _route_by_phase(
-    fase: int, tool_elab_info: dict, cfg: dict, id: int, unit_name: str, tool_name: str, timestamp_matlab_elab: datetime, pool: object
-) -> bool:
-    """
-    Routes the processing of a workflow record based on the current phase.
-
-    This function acts as a dispatcher, calling the appropriate handler function
-    for sending elaborated data or raw data based on the `fase` (phase) parameter.
-
-    Args:
-    fase (int): The current phase of the workflow (e.g., WorkflowFlags.SENT_ELAB_DATA, WorkflowFlags.SENT_RAW_DATA).
-    tool_elab_info (dict): A dictionary containing information about the tool and its elaboration status.
-    cfg (dict): The configuration dictionary.
-    id (int): The ID of the record being processed.
-    unit_name (str): The name of the unit associated with the data.
-    tool_name (str): The name of the tool associated with the data.
-    timestamp_matlab_elab (datetime): The timestamp of the last MATLAB elaboration.
-    pool (object): The database connection pool.
-
-    Returns:
-    bool: True if the data sending operation was successful or no action was needed, False otherwise.
-    """
-    if fase == WorkflowFlags.SENT_ELAB_DATA:
-        return await _handle_elab_data_phase(tool_elab_info, cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-
-    elif fase == WorkflowFlags.SENT_RAW_DATA:
-        return await _handle_raw_data_phase(tool_elab_info, cfg, id, unit_name, tool_name, pool)
-
-    else:
-        logger.info(f"id {id} - {unit_name} - {tool_name}: nessuna azione da eseguire.")
-        return True
-
-
-async def _handle_elab_data_phase(
-    tool_elab_info: dict, cfg: dict, id: int, unit_name: str, tool_name: str, timestamp_matlab_elab: datetime, pool: object
-) -> bool:
-    """
-    Handles the phase of sending elaborated data.
-
-    This function checks if elaborated data needs to be sent via FTP or API
-    based on the `tool_elab_info` and calls the appropriate sending function.
-
-    Args:
-        tool_elab_info (dict): A dictionary containing information about the tool and its elaboration status,
-                               including flags for FTP and API sending.
-        cfg (dict): The configuration dictionary.
-        id (int): The ID of the record being processed.
-        unit_name (str): The name of the unit associated with the data.
-        tool_name (str): The name of the tool associated with the data.
-        timestamp_matlab_elab (datetime): The timestamp of the last MATLAB elaboration.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the data sending operation was successful or no action was needed, False otherwise.
-    """
-    # FTP send per dati elaborati
-    if tool_elab_info.get("ftp_send"):
-        return await _send_elab_data_ftp(cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-
-    # API send per dati elaborati
-    elif _should_send_elab_api(tool_elab_info):
-        return await _send_elab_data_api(cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-
-    return True
-
-
-async def _handle_raw_data_phase(tool_elab_info: dict, cfg: dict, id: int, unit_name: str, tool_name: str, pool: object) -> bool:
-    """
-    Handles the phase of sending raw data.
-
-    This function checks if raw data needs to be sent via FTP or API
-    based on the `tool_elab_info` and calls the appropriate sending function.
-
-    Args:
-    tool_elab_info (dict): A dictionary containing information about the tool and its raw data sending status,
-    including flags for FTP and API sending.
-    cfg (dict): The configuration dictionary.
-    id (int): The ID of the record being processed.
-    unit_name (str): The name of the unit associated with the data.
-    tool_name (str): The name of the tool associated with the data.
-    pool (object): The database connection pool.
-
-    Returns:
-    bool: True if the data sending operation was successful or no action was needed, False otherwise.
-    """
-
-    # FTP send per dati raw
-    if tool_elab_info.get("ftp_send_raw"):
-        return await _send_raw_data_ftp(cfg, id, unit_name, tool_name, pool)
-
-    # API send per dati raw
-    elif _should_send_raw_api(tool_elab_info):
-        return await _send_raw_data_api(cfg, id, unit_name, tool_name, pool)
-
-    return True
-
-
-def _should_send_elab_api(tool_elab_info: dict) -> bool:
-    """Verifica se i dati elaborati devono essere inviati via API."""
-    return tool_elab_info.get("inoltro_api") and tool_elab_info.get("api_send") and tool_elab_info.get("inoltro_api_url", "").strip()
-
-
-def _should_send_raw_api(tool_elab_info: dict) -> bool:
-    """Verifica se i dati raw devono essere inviati via API."""
-    return (
-        tool_elab_info.get("inoltro_api_raw")
-        and tool_elab_info.get("api_send_raw")
-        and tool_elab_info.get("inoltro_api_url_raw", "").strip()
-    )
-
-
-async def _send_elab_data_ftp(cfg: dict, id: int, unit_name: str, tool_name: str, timestamp_matlab_elab: datetime, pool: object) -> bool:
-    """
-    Sends elaborated data via FTP.
-
-    This function retrieves the elaborated CSV data and attempts to send it
-    to the customer via FTP using async operations. It logs success or failure.
-
-    Args:
-        cfg (dict): The configuration dictionary.
-        id (int): The ID of the record being processed.
-        unit_name (str): The name of the unit associated with the data.
-        tool_name (str): The name of the tool associated with the data.
-        timestamp_matlab_elab (datetime): The timestamp of the last MATLAB elaboration.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the FTP sending was successful, False otherwise.
-    """
-    try:
-        elab_csv = await get_data_as_csv(cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-        if not elab_csv:
-            logger.warning(f"id {id} - {unit_name} - {tool_name}: nessun dato CSV elaborato trovato")
-            return False
-
-        # Send via async FTP
-        if await ftp_send_elab_csv_to_customer(cfg, id, unit_name, tool_name, elab_csv, pool):
-            logger.info(f"id {id} - {unit_name} - {tool_name}: invio FTP completato con successo")
-            return True
-        else:
-            logger.error(f"id {id} - {unit_name} - {tool_name}: invio FTP fallito")
-            return False
-
-    except Exception as e:
-        logger.error(f"Errore invio FTP elab data id {id}: {e}", exc_info=True)
-        return False
-
-
-async def _send_elab_data_api(cfg: dict, id: int, unit_name: str, tool_name: str, timestamp_matlab_elab: datetime, pool: object) -> bool:
-    """
-    Sends elaborated data via API.
-
-    This function retrieves the elaborated CSV data and attempts to send it
-    to the customer via an API. It logs success or failure.
-
-    Args:
-        cfg (dict): The configuration dictionary.
-        id (int): The ID of the record being processed.
-        unit_name (str): The name of the unit associated with the data.
-        tool_name (str): The name of the tool associated with the data.
-        timestamp_matlab_elab (datetime): The timestamp of the last MATLAB elaboration.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the API sending was successful, False otherwise.
-    """
-    try:
-        elab_csv = await get_data_as_csv(cfg, id, unit_name, tool_name, timestamp_matlab_elab, pool)
-        if not elab_csv:
-            return False
-
-        logger.debug(f"id {id} - {unit_name} - {tool_name}: CSV elaborato pronto per invio API (size: {len(elab_csv)} bytes)")
-        # if await send_elab_csv_to_customer(cfg, id, unit_name, tool_name, elab_csv, pool):
-        if True:  # Placeholder per test
-            return True
-        else:
-            logger.error(f"id {id} - {unit_name} - {tool_name}: invio API fallito.")
-            return False
-
-    except Exception as e:
-        logger.error(f"Errore invio API elab data id {id}: {e}")
-        return False
-
-
-async def _send_raw_data_ftp(cfg: dict, id: int, unit_name: str, tool_name: str, pool: object) -> bool:
-    """
-    Sends raw data via FTP.
-
-    This function attempts to send raw CSV data to the customer via FTP
-    using async operations. It retrieves the raw data from the database
-    and uploads it to the configured FTP server.
-
-    Args:
-        cfg (dict): The configuration dictionary.
-        id (int): The ID of the record being processed.
-        unit_name (str): The name of the unit associated with the data.
-        tool_name (str): The name of the tool associated with the data.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the FTP sending was successful, False otherwise.
-    """
-    try:
-        # Send raw CSV via async FTP
-        if await ftp_send_raw_csv_to_customer(cfg, id, unit_name, tool_name, pool):
-            logger.info(f"id {id} - {unit_name} - {tool_name}: invio FTP raw completato con successo")
-            return True
-        else:
-            logger.error(f"id {id} - {unit_name} - {tool_name}: invio FTP raw fallito")
-            return False
-
-    except Exception as e:
-        logger.error(f"Errore invio FTP raw data id {id}: {e}", exc_info=True)
-        return False
-
-
-async def _send_raw_data_api(cfg: dict, id: int, unit_name: str, tool_name: str, pool: object) -> bool:
-    """
-    Sends raw data via API.
-
-    This function attempts to send raw CSV data to the customer via an API.
-    It logs success or failure.
-
-    Args:
-        cfg (dict): The configuration dictionary.
-        id (int): The ID of the record being processed.
-        unit_name (str): The name of the unit associated with the data.
-        tool_name (str): The name of the tool associated with the data.
-        pool (object): The database connection pool.
-
-    Returns:
-        bool: True if the API sending was successful, False otherwise.
-    """
-    try:
-        # if await api_send_raw_csv_to_customer(cfg, id, unit_name, tool_name, pool):
-        if True:  # Placeholder per test
-            return True
-        else:
-            logger.error(f"id {id} - {unit_name} - {tool_name}: invio API raw fallito.")
-            return False
-
-    except Exception as e:
-        logger.error(f"Errore invio API raw data id {id}: {e}")
-        return False

vm1/src/utils/connect/send_email.py

@@ -1,63 +0,0 @@
-import logging
-from email.message import EmailMessage
-
-import aiosmtplib
-
-from utils.config import loader_email as setting
-
-cfg = setting.Config()
-logger = logging.getLogger(__name__)
-
-
-async def send_error_email(unit_name: str, tool_name: str, matlab_cmd: str, matlab_error: str, errors: list, warnings: list) -> None:
-    """
-    Sends an error email containing details about a MATLAB processing failure.
-
-    The email includes information about the unit, tool, MATLAB command, error message,
-    and lists of specific errors and warnings encountered.
-
-    Args:
-        unit_name (str): The name of the unit involved in the processing.
-        tool_name (str): The name of the tool involved in the processing.
-        matlab_cmd (str): The MATLAB command that was executed.
-        matlab_error (str): The main MATLAB error message.
-        errors (list): A list of detailed error messages from MATLAB.
-        warnings (list): A list of detailed warning messages from MATLAB.
-    """
-
-    # Creazione dell'oggetto messaggio
-    msg = EmailMessage()
-    msg["Subject"] = cfg.subject
-    msg["From"] = cfg.from_addr
-    msg["To"] = cfg.to_addr
-    msg["Cc"] = cfg.cc_addr
-    msg["Bcc"] = cfg.bcc_addr
-
-    MatlabErrors = "<br/>".join(errors)
-    MatlabWarnings = "<br/>".join(dict.fromkeys(warnings))
-
-    # Imposta il contenuto del messaggio come HTML
-    msg.add_alternative(
-        cfg.body.format(
-            unit=unit_name,
-            tool=tool_name,
-            matlab_cmd=matlab_cmd,
-            matlab_error=matlab_error,
-            MatlabErrors=MatlabErrors,
-            MatlabWarnings=MatlabWarnings,
-        ),
-        subtype="html",
-    )
-    try:
-        # Use async SMTP to prevent blocking the event loop
-        await aiosmtplib.send(
-            msg,
-            hostname=cfg.smtp_addr,
-            port=cfg.smtp_port,
-            username=cfg.smtp_user,
-            password=cfg.smtp_passwd,
-            start_tls=True,
-        )
-        logger.info("Email inviata con successo!")
-    except Exception as e:
-        logger.error(f"Errore durante l'invio dell'email: {e}")

vm1/src/utils/connect/user_admin.py

@@ -1,228 +0,0 @@
-import asyncio
-import logging
-import os
-from hashlib import sha256
-from pathlib import Path
-
-from utils.database.connection import connetti_db_async
-
-logger = logging.getLogger(__name__)
-
-
-# Sync wrappers for FTP commands (required by pyftpdlib)
-
-
-def ftp_SITE_ADDU(self: object, line: str) -> None:
-    """Sync wrapper for ftp_SITE_ADDU_async."""
-    asyncio.run(ftp_SITE_ADDU_async(self, line))
-
-
-def ftp_SITE_DISU(self: object, line: str) -> None:
-    """Sync wrapper for ftp_SITE_DISU_async."""
-    asyncio.run(ftp_SITE_DISU_async(self, line))
-
-
-def ftp_SITE_ENAU(self: object, line: str) -> None:
-    """Sync wrapper for ftp_SITE_ENAU_async."""
-    asyncio.run(ftp_SITE_ENAU_async(self, line))
-
-
-def ftp_SITE_LSTU(self: object, line: str) -> None:
-    """Sync wrapper for ftp_SITE_LSTU_async."""
-    asyncio.run(ftp_SITE_LSTU_async(self, line))
-
-
-# Async implementations
-
-
-async def ftp_SITE_ADDU_async(self: object, line: str) -> None:
-    """
-    Adds a virtual user, creates their directory, and saves their details to the database.
-
-    Args:
-        line (str): A string containing the username and password separated by a space.
-    """
-    cfg = self.cfg
-    try:
-        parms = line.split()
-        user = os.path.basename(parms[0])  # Extract the username
-        password = parms[1]  # Get the password
-        hash_value = sha256(password.encode("UTF-8")).hexdigest()  # Hash the password
-    except IndexError:
-        self.respond("501 SITE ADDU failed. Command needs 2 arguments")
-    else:
-        try:
-            # Create the user's directory
-            Path(cfg.virtpath + user).mkdir(parents=True, exist_ok=True)
-        except Exception as e:
-            self.respond(f"551 Error in create virtual user path: {e}")
-        else:
-            try:
-                # Add the user to the authorizer
-                self.authorizer.add_user(str(user), hash_value, cfg.virtpath + "/" + user, perm=cfg.defperm)
-
-                # Save the user to the database using async connection
-                try:
-                    conn = await connetti_db_async(cfg)
-                except Exception as e:
-                    logger.error(f"Database connection error: {e}")
-                    self.respond("501 SITE ADDU failed: Database error")
-                    return
-
-                try:
-                    async with conn.cursor() as cur:
-                        # Use parameterized query to prevent SQL injection
-                        await cur.execute(
-                            f"INSERT INTO {cfg.dbname}.{cfg.dbusertable} (ftpuser, hash, virtpath, perm) VALUES (%s, %s, %s, %s)",
-                            (user, hash_value, cfg.virtpath + user, cfg.defperm),
-                        )
-                        # autocommit=True in connection
-                    logger.info(f"User {user} created.")
-                    self.respond("200 SITE ADDU successful.")
-                except Exception as e:
-                    self.respond(f"501 SITE ADDU failed: {e}.")
-                    logger.error(f"Error creating user {user}: {e}")
-                finally:
-                    conn.close()
-
-            except Exception as e:
-                self.respond(f"501 SITE ADDU failed: {e}.")
-                logger.error(f"Error in ADDU: {e}")
-
-
-async def ftp_SITE_DISU_async(self: object, line: str) -> None:
-    """
-    Removes a virtual user from the authorizer and marks them as deleted in the database.
-
-    Args:
-        line (str): A string containing the username to be disabled.
-    """
-    cfg = self.cfg
-    parms = line.split()
-    user = os.path.basename(parms[0])  # Extract the username
-    try:
-        # Remove the user from the authorizer
-        self.authorizer.remove_user(str(user))
-
-        # Delete the user from database
-        try:
-            conn = await connetti_db_async(cfg)
-        except Exception as e:
-            logger.error(f"Database connection error: {e}")
-            self.respond("501 SITE DISU failed: Database error")
-            return
-
-        try:
-            async with conn.cursor() as cur:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = NOW() WHERE ftpuser = %s", (user,))
-                # autocommit=True in connection
-            logger.info(f"User {user} deleted.")
-            self.respond("200 SITE DISU successful.")
-        except Exception as e:
-            logger.error(f"Error disabling user {user}: {e}")
-            self.respond("501 SITE DISU failed.")
-        finally:
-            conn.close()
-
-    except Exception as e:
-        self.respond("501 SITE DISU failed.")
-        logger.error(f"Error in DISU: {e}")
-
-
-async def ftp_SITE_ENAU_async(self: object, line: str) -> None:
-    """
-    Restores a virtual user by updating their status in the database and adding them back to the authorizer.
-
-    Args:
-        line (str): A string containing the username to be enabled.
-    """
-    cfg = self.cfg
-    parms = line.split()
-    user = os.path.basename(parms[0])  # Extract the username
-    try:
-        # Restore the user into database
-        try:
-            conn = await connetti_db_async(cfg)
-        except Exception as e:
-            logger.error(f"Database connection error: {e}")
-            self.respond("501 SITE ENAU failed: Database error")
-            return
-
-        try:
-            async with conn.cursor() as cur:
-                # Enable the user
-                await cur.execute(f"UPDATE {cfg.dbname}.{cfg.dbusertable} SET disabled_at = NULL WHERE ftpuser = %s", (user,))
-
-                # Fetch user details
-                await cur.execute(
-                    f"SELECT ftpuser, hash, virtpath, perm FROM {cfg.dbname}.{cfg.dbusertable} WHERE ftpuser = %s", (user,)
-                )
-                result = await cur.fetchone()
-
-                if not result:
-                    self.respond(f"501 SITE ENAU failed: User {user} not found")
-                    return
-
-                ftpuser, hash_value, virtpath, perm = result
-                self.authorizer.add_user(ftpuser, hash_value, virtpath, perm)
-
-                try:
-                    Path(cfg.virtpath + ftpuser).mkdir(parents=True, exist_ok=True)
-                except Exception as e:
-                    self.respond(f"551 Error in create virtual user path: {e}")
-                    return
-
-            logger.info(f"User {user} restored.")
-            self.respond("200 SITE ENAU successful.")
-
-        except Exception as e:
-            logger.error(f"Error enabling user {user}: {e}")
-            self.respond("501 SITE ENAU failed.")
-        finally:
-            conn.close()
-
-    except Exception as e:
-        self.respond("501 SITE ENAU failed.")
-        logger.error(f"Error in ENAU: {e}")
-
-
-async def ftp_SITE_LSTU_async(self: object, line: str) -> None:
-    """
-    Lists all virtual users from the database.
-
-    Args:
-        line (str): An empty string (no arguments needed for this command).
-    """
-    cfg = self.cfg
-    users_list = []
-    try:
-        # Connect to the database to fetch users
-        try:
-            conn = await connetti_db_async(cfg)
-        except Exception as e:
-            logger.error(f"Database connection error: {e}")
-            self.respond("501 SITE LSTU failed: Database error")
-            return
-
-        try:
-            async with conn.cursor() as cur:
-                self.push("214-The following virtual users are defined:\r\n")
-                await cur.execute(f"SELECT ftpuser, perm, disabled_at FROM {cfg.dbname}.{cfg.dbusertable}")
-                results = await cur.fetchall()
-
-                for ftpuser, perm, disabled_at in results:
-                    users_list.append(f"Username: {ftpuser}\tPerms: {perm}\tDisabled: {disabled_at}\r\n")
-
-                self.push("".join(users_list))
-                self.respond("214 LSTU SITE command successful.")
-
-        except Exception as e:
-            self.respond(f"501 list users failed: {e}")
-            logger.error(f"Error listing users: {e}")
-        finally:
-            conn.close()
-
-    except Exception as e:
-        self.respond(f"501 list users failed: {e}")
-        logger.error(f"Error in LSTU: {e}")

vm1/src/utils/csv/__init__.py

@@ -1 +0,0 @@
-"""Parser delle centraline"""

vm1/src/utils/csv/data_preparation.py

@@ -1,309 +0,0 @@
-#!.venv/bin/python
-import logging
-import re
-from datetime import datetime, timedelta
-from itertools import islice
-
-from utils.database.loader_action import find_nearest_timestamp
-from utils.database.nodes_query import get_nodes_type
-from utils.timestamp.date_check import normalizza_data, normalizza_orario
-
-logger = logging.getLogger(__name__)
-
-
-async def get_data(cfg: object, id: int, pool: object) -> tuple:
-    """
-    Retrieves unit name, tool name, and tool data for a given record ID from the database.
-
-    Args:
-        cfg (object): Configuration object containing database table name.
-        id (int): The ID of the record to retrieve.
-        pool (object): The database connection pool.
-    Returns:
-        tuple: A tuple containing unit_name, tool_name, and tool_data.
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            # Use parameterized query to prevent SQL injection
-            await cur.execute(f"SELECT filename, unit_name, tool_name, tool_data FROM {cfg.dbrectable} WHERE id = %s", (id,))
-            filename, unit_name, tool_name, tool_data = await cur.fetchone()
-
-            return filename, unit_name, tool_name, tool_data
-
-
-async def make_pipe_sep_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes pipe-separated data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    righe = ToolData.splitlines()
-    matrice_valori = []
-    """
-    Ciclo su tutte le righe del file CSV, escludendo quelle che:
-        non hanno il pattern ';|;' perché non sono dati ma è la header
-        che hanno il pattern 'No RX' perché sono letture non pervenute o in errore
-        che hanno il pattern '.-' perché sono letture con un numero errato - negativo dopo la virgola
-        che hanno il pattern 'File Creation' perché vuol dire che c'è stato un errore della centralina
-    """
-    for riga in [
-        riga
-        for riga in righe
-        if ";|;" in riga and "No RX" not in riga and ".-" not in riga and "File Creation" not in riga and riga.isprintable()
-    ]:
-        timestamp, batlevel, temperature, rilevazioni = riga.split(";", 3)
-        EventDate, EventTime = timestamp.split(" ")
-        if batlevel == "|":
-            batlevel = temperature
-            temperature, rilevazioni = rilevazioni.split(";", 1)
-        """ in alcune letture mancano temperatura e livello batteria"""
-        if temperature == "":
-            temperature = 0
-        if batlevel == "":
-            batlevel = 0
-        valori_nodi = (
-            rilevazioni.lstrip("|;").rstrip(";").split(";|;")
-        )  # Toglie '|;' iniziali, toglie eventuali ';' finali, dividi per ';|;'
-        for num_nodo, valori_nodo in enumerate(valori_nodi, start=1):
-            valori = valori_nodo.split(";")
-            matrice_valori.append(
-                [UnitName, ToolNameID, num_nodo, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                + valori
-                + ([None] * (19 - len(valori)))
-            )
-
-    return matrice_valori
-
-
-async def make_ain_din_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes analog and digital input data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    node_channels, node_types, node_ains, node_dins = await get_nodes_type(cfg, ToolNameID, UnitName, pool)
-    righe = ToolData.splitlines()
-    matrice_valori = []
-    pattern = r"^(?:\d{4}\/\d{2}\/\d{2}|\d{2}\/\d{2}\/\d{4}) \d{2}:\d{2}:\d{2}(?:;\d+\.\d+){2}(?:;\d+){4}$"
-    if node_ains or node_dins:
-        for riga in [riga for riga in righe if re.match(pattern, riga)]:
-            timestamp, batlevel, temperature, analog_input1, analog_input2, digital_input1, digital_input2 = riga.split(";")
-            EventDate, EventTime = timestamp.split(" ")
-            if any(node_ains):
-                for node_num, analog_act in enumerate([analog_input1, analog_input2], start=1):
-                    matrice_valori.append(
-                        [UnitName, ToolNameID, node_num, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                        + [analog_act]
-                        + ([None] * (19 - 1))
-                    )
-            else:
-                logger.info(f"Nessun Ingresso analogico per {UnitName} {ToolNameID}")
-            if any(node_dins):
-                start_node = 3 if any(node_ains) else 1
-                for node_num, digital_act in enumerate([digital_input1, digital_input2], start=start_node):
-                    matrice_valori.append(
-                        [UnitName, ToolNameID, node_num, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                        + [digital_act]
-                        + ([None] * (19 - 1))
-                    )
-            else:
-                logger.info(f"Nessun Ingresso digitale per {UnitName} {ToolNameID}")
-
-    return matrice_valori
-
-
-async def make_channels_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes channel-based data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    node_channels, node_types, node_ains, node_dins = await get_nodes_type(cfg, ToolNameID, UnitName, pool)
-    righe = ToolData.splitlines()
-    matrice_valori = []
-    for riga in [
-        riga
-        for riga in righe
-        if ";|;" in riga and "No RX" not in riga and ".-" not in riga and "File Creation" not in riga and riga.isprintable()
-    ]:
-        timestamp, batlevel, temperature, rilevazioni = riga.replace(";|;", ";").split(";", 3)
-        EventDate, EventTime = timestamp.split(" ")
-        valori_splitted = [valore for valore in rilevazioni.split(";") if valore != "|"]
-        valori_iter = iter(valori_splitted)
-
-        valori_nodi = [list(islice(valori_iter, channels)) for channels in node_channels]
-
-        for num_nodo, valori in enumerate(valori_nodi, start=1):
-            matrice_valori.append(
-                [UnitName, ToolNameID, num_nodo, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                + valori
-                + ([None] * (19 - len(valori)))
-            )
-
-    return matrice_valori
-
-
-async def make_musa_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes 'Musa' specific data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    node_channels, node_types, node_ains, node_dins = await get_nodes_type(cfg, ToolNameID, UnitName, pool)
-    righe = ToolData.splitlines()
-    matrice_valori = []
-    for riga in [
-        riga
-        for riga in righe
-        if ";|;" in riga and "No RX" not in riga and ".-" not in riga and "File Creation" not in riga and riga.isprintable()
-    ]:
-        timestamp, batlevel, rilevazioni = riga.replace(";|;", ";").split(";", 2)
-        if timestamp == "":
-            continue
-        EventDate, EventTime = timestamp.split(" ")
-        temperature = rilevazioni.split(";")[0]
-        logger.info(f"{temperature}, {rilevazioni}")
-        valori_splitted = [valore for valore in rilevazioni.split(";") if valore != "|"]
-        valori_iter = iter(valori_splitted)
-
-        valori_nodi = [list(islice(valori_iter, channels)) for channels in node_channels]
-
-        for num_nodo, valori in enumerate(valori_nodi, start=1):
-            matrice_valori.append(
-                [UnitName, ToolNameID, num_nodo, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                + valori
-                + ([None] * (19 - len(valori)))
-            )
-
-    return matrice_valori
-
-
-async def make_tlp_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes 'TLP' specific data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    righe = ToolData.splitlines()
-    valori_x_nodo = 2
-    matrice_valori = []
-    for riga in righe:
-        timestamp, batlevel, temperature, barometer, rilevazioni = riga.split(";", 4)
-        EventDate, EventTime = timestamp.split(" ")
-        lista_rilevazioni = rilevazioni.strip(";").split(";")
-        lista_rilevazioni.append(barometer)
-        valori_nodi = [lista_rilevazioni[i : i + valori_x_nodo] for i in range(0, len(lista_rilevazioni), valori_x_nodo)]
-        for num_nodo, valori in enumerate(valori_nodi, start=1):
-            matrice_valori.append(
-                [UnitName, ToolNameID, num_nodo, normalizza_data(EventDate), normalizza_orario(EventTime), batlevel, temperature]
-                + valori
-                + ([None] * (19 - len(valori)))
-            )
-    return matrice_valori
-
-
-async def make_gd_matrix(cfg: object, id: int, pool: object) -> list:
-    """
-    Processes 'GD' specific data from a CSV record into a structured matrix.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record.
-        pool (object): The database connection pool.
-    Returns:
-        list: A list of lists, where each inner list represents a row in the matrix.
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    righe = ToolData.splitlines()
-    matrice_valori = []
-    pattern = r";-?\d+dB$"
-    for riga in [
-        riga
-        for riga in righe
-        if ";|;" in riga and "No RX" not in riga and ".-" not in riga and "File Creation" not in riga and riga.isprintable()
-    ]:
-        timestamp, rilevazioni = riga.split(";|;", 1)
-        EventDate, EventTime = timestamp.split(" ")
-        # logger.debug(f"GD id {id}: {pattern} {rilevazioni}")
-        if re.search(pattern, rilevazioni):
-            if len(matrice_valori) == 0:
-                matrice_valori.append(["RSSI"])
-            batlevel, temperature, rssi = rilevazioni.split(";")
-            # logger.debug(f"GD id {id}: {EventDate}, {EventTime}, {batlevel}, {temperature}, {rssi}")
-
-            gd_timestamp = datetime.strptime(f"{normalizza_data(EventDate)} {normalizza_orario(EventTime)}", "%Y-%m-%d %H:%M:%S")
-            start_timestamp = gd_timestamp - timedelta(seconds=45)
-            end_timestamp = gd_timestamp + timedelta(seconds=45)
-            matrice_valori.append(
-                [
-                    UnitName,
-                    ToolNameID.replace("GD", "DT"),
-                    1,
-                    f"{start_timestamp:%Y-%m-%d %H:%M:%S}",
-                    f"{end_timestamp:%Y-%m-%d %H:%M:%S}",
-                    f"{gd_timestamp:%Y-%m-%d %H:%M:%S}",
-                    batlevel,
-                    temperature,
-                    int(rssi[:-2]),
-                ]
-            )
-
-        elif all(char == ";" for char in rilevazioni):
-            pass
-        elif ";|;" in rilevazioni:
-            unit_metrics, data = rilevazioni.split(";|;")
-            batlevel, temperature = unit_metrics.split(";")
-            # logger.debug(f"GD id {id}: {EventDate}, {EventTime}, {batlevel}, {temperature}, {data}")
-
-            dt_timestamp, dt_batlevel, dt_temperature = await find_nearest_timestamp(
-                cfg,
-                {
-                    "timestamp": f"{normalizza_data(EventDate)} {normalizza_orario(EventTime)}",
-                    "unit": UnitName,
-                    "tool": ToolNameID.replace("GD", "DT"),
-                    "node_num": 1,
-                },
-                pool,
-            )
-            EventDate, EventTime = dt_timestamp.strftime("%Y-%m-%d %H:%M:%S").split(" ")
-            valori = data.split(";")
-            matrice_valori.append(
-                [UnitName, ToolNameID.replace("GD", "DT"), 2, EventDate, EventTime, float(dt_batlevel), float(dt_temperature)]
-                + valori
-                + ([None] * (16 - len(valori)))
-                + [batlevel, temperature, None]
-            )
-        else:
-            logger.warning(f"GD id {id}: dati non trattati - {rilevazioni}")
-
-    return matrice_valori

vm1/src/utils/csv/loaders.py

@@ -1,153 +0,0 @@
-import asyncio
-import logging
-import os
-import tempfile
-
-from utils.csv.data_preparation import (
-    get_data,
-    make_ain_din_matrix,
-    make_channels_matrix,
-    make_gd_matrix,
-    make_musa_matrix,
-    make_pipe_sep_matrix,
-    make_tlp_matrix,
-)
-from utils.database import WorkflowFlags
-from utils.database.loader_action import load_data, unlock, update_status
-
-logger = logging.getLogger(__name__)
-
-
-async def main_loader(cfg: object, id: int, pool: object, action: str) -> None:
-    """
-    Main loader function to process CSV data based on the specified action.
-
-    Args:
-        cfg (object): Configuration object.
-        id (int): The ID of the CSV record to process.
-        pool (object): The database connection pool.
-        action (str): The type of data processing to perform (e.g., "pipe_separator", "analogic_digital").
-    """
-    type_matrix_mapping = {
-        "pipe_separator": make_pipe_sep_matrix,
-        "analogic_digital": make_ain_din_matrix,
-        "channels": make_channels_matrix,
-        "tlp": make_tlp_matrix,
-        "gd": make_gd_matrix,
-        "musa": make_musa_matrix,
-    }
-    if action in type_matrix_mapping:
-        function_to_call = type_matrix_mapping[action]
-        # Create a matrix of values from the data
-        matrice_valori = await function_to_call(cfg, id, pool)
-
-        logger.info("matrice valori creata")
-        # Load the data into the database
-        if await load_data(cfg, matrice_valori, pool, type=action):
-            await update_status(cfg, id, WorkflowFlags.DATA_LOADED, pool)
-            await unlock(cfg, id, pool)
-    else:
-        logger.warning(f"Action '{action}' non riconosciuta.")
-
-
-async def get_next_csv_atomic(pool: object, table_name: str, status: int, next_status: int) -> tuple:
-    """
-    Retrieves the next available CSV record for processing in an atomic manner.
-
-    This function acquires a database connection from the pool, begins a transaction,
-    and attempts to select and lock a single record from the specified table that
-    matches the given status and has not yet reached the next_status. It uses
-    `SELECT FOR UPDATE SKIP LOCKED` to ensure atomicity and prevent other workers
-    from processing the same record concurrently.
-
-    Args:
-        pool (object): The database connection pool.
-        table_name (str): The name of the table to query.
-        status (int): The current status flag that the record must have.
-        next_status (int): The status flag that the record should NOT have yet.
-    Returns:
-        tuple: The next available received record if found, otherwise None.
-    """
-    async with pool.acquire() as conn:
-        # IMPORTANTE: Disabilita autocommit per questa transazione
-        await conn.begin()
-
-        try:
-            async with conn.cursor() as cur:
-                # Usa SELECT FOR UPDATE per lock atomico
-
-                await cur.execute(
-                    f"""
-                    SELECT id, unit_type, tool_type, unit_name, tool_name
-                    FROM {table_name}
-                    WHERE locked = 0
-                        AND ((status & %s) > 0 OR %s = 0)
-                        AND (status & %s) = 0
-                    ORDER BY id
-                    LIMIT 1
-                    FOR UPDATE SKIP LOCKED
-                """,
-                    (status, status, next_status),
-                )
-
-                result = await cur.fetchone()
-                if result:
-                    await cur.execute(
-                        f"""
-                        UPDATE {table_name}
-                        SET locked = 1
-                        WHERE id = %s
-                    """,
-                        (result[0],),
-                    )
-
-                # Commit esplicito per rilasciare il lock
-                await conn.commit()
-                return result
-
-        except Exception as e:
-            # Rollback in caso di errore
-            await conn.rollback()
-            raise e
-
-
-async def main_old_script_loader(cfg: object, id: int, pool: object, script_name: str) -> None:
-    """
-    This function retrieves CSV data, writes it to a temporary file,
-    executes an external Python script to process it,
-    and then updates the workflow status in the database.
-    Args:
-    cfg (object): The configuration object.
-    id (int): The ID of the CSV record to process.
-    pool (object): The database connection pool.
-    script_name (str): The name of the script to execute (without the .py extension).
-    """
-    filename, UnitName, ToolNameID, ToolData = await get_data(cfg, id, pool)
-    # Creare un file temporaneo
-    with tempfile.NamedTemporaryFile(mode="w", prefix=filename, suffix=".csv", delete=False) as temp_file:
-        temp_file.write(ToolData)
-        temp_filename = temp_file.name
-
-    try:
-        # Usa asyncio.subprocess per vero async
-        process = await asyncio.create_subprocess_exec(
-            "python3", f"old_scripts/{script_name}.py", temp_filename, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-        )
-        stdout, stderr = await process.communicate()
-
-        result_stdout = stdout.decode("utf-8")
-        result_stderr = stderr.decode("utf-8")
-
-    finally:
-        # Pulire il file temporaneo
-        os.unlink(temp_filename)
-
-    if process.returncode != 0:
-        logger.error(f"Errore nell'esecuzione del programma {script_name}.py: {result_stderr}")
-        raise Exception(f"Errore nel programma: {result_stderr}")
-    else:
-        logger.info(f"Programma {script_name}.py eseguito con successo.")
-        logger.debug(f"Stdout: {result_stdout}")
-        await update_status(cfg, id, WorkflowFlags.DATA_LOADED, pool)
-        await update_status(cfg, id, WorkflowFlags.DATA_ELABORATED, pool)
-        await unlock(cfg, id, pool)

vm1/src/utils/csv/parser.py

@@ -1,28 +0,0 @@
-import re
-
-
-def extract_value(patterns: list, primary_source: str, secondary_source: str = None, default: str = "Not Defined") -> str:
-    """
-    Extracts a value from a given source (or sources) based on a list of regex patterns.
-
-    It iterates through the provided patterns and attempts to find a match in the
-    primary source first, then in the secondary source if provided. The first
-    successful match is returned. If no match is found after checking all sources
-    with all patterns, a default value is returned.
-
-    Args:
-        patterns (list): A list of regular expression strings to search for.
-        primary_source (str): The main string to search within.
-        secondary_source (str, optional): An additional string to search within if no match is found in the primary source.
-                                          Defaults to None.
-        default (str, optional): The value to return if no match is found. Defaults to 'Not Defined'.
-
-    Returns:
-        str: The first matched value, or the default value if no match is found.
-    """
-    for source in [source for source in (primary_source, secondary_source) if source is not None]:
-        for pattern in patterns:
-            matches = re.findall(pattern, source, re.IGNORECASE)
-            if matches:
-                return matches[0]  # Return the first match immediately
-    return default  # Return default if no matches are found

vm1/src/utils/database/__init__.py

@@ -1,37 +0,0 @@
-class WorkflowFlags:
-    """
-    Defines integer flags representing different stages in a data processing workflow.
-    Each flag is a power of 2, allowing them to be combined using bitwise operations
-    to represent multiple states simultaneously.
-    """
-
-    CSV_RECEIVED = 0  #   0000
-    DATA_LOADED = 1  #   0001
-    START_ELAB = 2  #   0010
-    DATA_ELABORATED = 4  #   0100
-    SENT_RAW_DATA = 8  #   1000
-    SENT_ELAB_DATA = 16  #  10000
-    DUMMY_ELABORATED = 32  # 100000 (Used for testing or specific dummy elaborations)
-
-
-# Mappatura flag -> colonna timestamp
-FLAG_TO_TIMESTAMP = {
-    WorkflowFlags.CSV_RECEIVED: "inserted_at",
-    WorkflowFlags.DATA_LOADED: "loaded_at",
-    WorkflowFlags.START_ELAB: "start_elab_at",
-    WorkflowFlags.DATA_ELABORATED: "elaborated_at",
-    WorkflowFlags.SENT_RAW_DATA: "sent_raw_at",
-    WorkflowFlags.SENT_ELAB_DATA: "sent_elab_at",
-    WorkflowFlags.DUMMY_ELABORATED: "elaborated_at",  # Shares the same timestamp column as DATA_ELABORATED
-}
-"""
-A dictionary mapping each WorkflowFlag to the corresponding database column
-name that stores the timestamp when that workflow stage was reached.
-"""
-
-# Dimensione degli split della matrice per il caricamento
-BATCH_SIZE = 1000
-"""
-The number of records to process in a single batch when loading data into the database.
-This helps manage memory usage and improve performance for large datasets.
-"""

vm1/src/utils/database/action_query.py

@@ -1,152 +0,0 @@
-import csv
-import logging
-from io import StringIO
-
-import aiomysql
-
-from utils.database import WorkflowFlags
-
-logger = logging.getLogger(__name__)
-
-sub_select = {
-    WorkflowFlags.DATA_ELABORATED: """m.matcall, s.`desc` AS statustools""",
-    WorkflowFlags.SENT_RAW_DATA: """t.ftp_send, t.api_send, u.inoltro_api, u.inoltro_api_url, u.inoltro_api_bearer_token,
-                                    s.`desc` AS statustools, IFNULL(u.duedate, "") AS duedate""",
-    WorkflowFlags.SENT_ELAB_DATA: """t.ftp_send_raw, IFNULL(u.ftp_mode_raw, "") AS ftp_mode_raw,
-    IFNULL(u.ftp_addrs_raw, "") AS ftp_addrs_raw, IFNULL(u.ftp_user_raw, "") AS ftp_user_raw,
-    IFNULL(u.ftp_passwd_raw, "") AS ftp_passwd_raw, IFNULL(u.ftp_filename_raw, "") AS ftp_filename_raw,
-    IFNULL(u.ftp_parm_raw, "") AS ftp_parm_raw, IFNULL(u.ftp_target_raw, "") AS ftp_target_raw,
-    t.unit_id, s.`desc` AS statustools, u.inoltro_ftp_raw, u.inoltro_api_raw,
-    IFNULL(u.inoltro_api_url_raw, "") AS inoltro_api_url_raw,
-    IFNULL(u.inoltro_api_bearer_token_raw, "") AS inoltro_api_bearer_token_raw,
-    t.api_send_raw, IFNULL(u.duedate, "") AS duedate
-    """,
-}
-
-
-async def get_tool_info(next_status: int, unit: str, tool: str, pool: object) -> tuple:
-    """
-    Retrieves tool-specific information from the database based on the next workflow status,
-    unit name, and tool name.
-
-    This function dynamically selects columns based on the `next_status` provided,
-    joining `matfuncs`, `tools`, `units`, and `statustools` tables.
-
-    Args:
-        next_status (int): The next workflow status flag (e.g., WorkflowFlags.DATA_ELABORATED).
-                           This determines which set of columns to select from the database.
-        unit (str): The name of the unit associated with the tool.
-        tool (str): The name of the tool.
-        pool (object): The database connection pool.
-
-    Returns:
-        tuple: A dictionary-like object (aiomysql.DictCursor result) containing the tool information,
-               or None if no information is found for the given unit and tool.
-    """
-
-    async with pool.acquire() as conn:
-        async with conn.cursor(aiomysql.DictCursor) as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute(f"""
-                SELECT {sub_select[next_status]}
-                FROM matfuncs AS m
-                    INNER JOIN tools AS t ON t.matfunc = m.id
-                    INNER JOIN units AS u ON u.id = t.unit_id
-                    INNER JOIN statustools AS s ON t.statustool_id = s.id
-                WHERE t.name = %s AND u.name = %s;
-                """, (tool, unit))
-
-                result = await cur.fetchone()
-
-                if not result:
-                    logger.warning(f"{unit} - {tool}: Tool info not found.")
-                    return None
-                else:
-                    return result
-            except Exception as e:
-                logger.error(f"Error: {e}")
-
-
-async def get_data_as_csv(cfg: dict, id_recv: int, unit: str, tool: str, matlab_timestamp: float, pool: object) -> str:
-    """
-    Retrieves elaborated data from the database and formats it as a CSV string.
-
-    The query selects data from the `ElabDataView` based on `UnitName`, `ToolNameID`,
-    and a `updated_at` timestamp, then orders it. The first row of the CSV will be
-    the column headers.
-
-    Args:
-        cfg (dict): Configuration dictionary (not directly used in the query but passed for consistency).
-        id_recv (int): The ID of the record being processed (used for logging).
-        pool (object): The database connection pool.
-        unit (str): The name of the unit to filter the data.
-        tool (str): The ID of the tool to filter the data.
-        matlab_timestamp (float): A timestamp used to filter data updated after this time.
-
-    Returns:
-        str: A string containing the elaborated data in CSV format.
-    """
-    query = """
-    select * from (
-        select 'ToolNameID', 'EventDate', 'EventTime', 'NodeNum', 'NodeType', 'NodeDepth',
-        'XShift', 'YShift', 'ZShift' , 'X', 'Y', 'Z', 'HShift', 'HShiftDir', 'HShift_local',
-        'speed', 'speed_local', 'acceleration', 'acceleration_local', 'T_node', 'water_level',
-        'pressure', 'load_value', 'AlfaX', 'AlfaY', 'CalcErr'
-        union all
-        select ToolNameID, EventDate, EventTime, NodeNum, NodeType, NodeDepth,
-        XShift, YShift, ZShift , X, Y, Z, HShift, HShiftDir, HShift_local,
-        speed, speed_local, acceleration, acceleration_local, T_node, water_level, pressure, load_value, AlfaX, AlfaY, calcerr
-        from ElabDataView
-        where UnitName = %s and ToolNameID = %s and updated_at > %s
-        order by ToolNameID DESC, concat(EventDate, EventTime), convert(`NodeNum`, UNSIGNED INTEGER) DESC
-    ) resulting_set
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                await cur.execute(query, (unit, tool, matlab_timestamp))
-                results = await cur.fetchall()
-                logger.info(f"id {id_recv} - {unit} - {tool}: estratti i dati per invio CSV")
-                logger.info(f"Numero di righe estratte: {len(results)}")
-
-                # Creare CSV in memoria
-                output = StringIO()
-                writer = csv.writer(output, delimiter=",", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
-                for row in results:
-                    writer.writerow(row)
-                csv_data = output.getvalue()
-                output.close()
-
-                return csv_data
-
-            except Exception as e:
-                logger.error(f"id {id_recv} - {unit} - {tool} - errore nel query creazione csv: {e}")
-                return None
-
-
-async def get_elab_timestamp(id_recv: int, pool: object) -> float:
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute("SELECT start_elab_at FROM received WHERE id = %s", (id_recv,))
-                results = await cur.fetchone()
-                return results[0]
-
-            except Exception as e:
-                logger.error(f"id {id_recv} - Errore nella query timestamp elaborazione: {e}")
-                return None
-
-
-async def check_flag_elab(pool: object) -> None:
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                await cur.execute("SELECT stop_elab from admin_panel")
-                results = await cur.fetchone()
-                return results[0]
-
-            except Exception as e:
-                logger.error(f"Errore nella query check flag stop elaborazioni: {e}")
-                return None

vm1/src/utils/database/connection.py

@@ -1,80 +0,0 @@
-import logging
-
-import aiomysql
-import mysql.connector
-from mysql.connector import Error
-
-logger = logging.getLogger(__name__)
-
-
-def connetti_db(cfg: object) -> object:
-    """
-    Establishes a synchronous connection to a MySQL database.
-
-    DEPRECATED: Use connetti_db_async() for async code.
-    This function is kept for backward compatibility with synchronous code
-    (e.g., ftp_csv_receiver.py which uses pyftpdlib).
-
-    Args:
-        cfg: A configuration object containing database connection parameters.
-             It should have the following attributes:
-                 - dbuser: The database username.
-                 - dbpass: The database password.
-                 - dbhost: The database host address.
-                 - dbport: The database port number.
-                 - dbname: The name of the database to connect to.
-
-    Returns:
-        A MySQL connection object if the connection is successful, otherwise None.
-    """
-    try:
-        conn = mysql.connector.connect(user=cfg.dbuser, password=cfg.dbpass, host=cfg.dbhost, port=cfg.dbport, database=cfg.dbname)
-        conn.autocommit = True
-        logger.info("Connected")
-        return conn
-    except Error as e:
-        logger.error(f"Database connection error: {e}")
-        raise  # Re-raise the exception to be handled by the caller
-
-
-async def connetti_db_async(cfg: object) -> aiomysql.Connection:
-    """
-    Establishes an asynchronous connection to a MySQL database.
-
-    This is the preferred method for async code. Use this instead of connetti_db()
-    in all async contexts to avoid blocking the event loop.
-
-    Args:
-        cfg: A configuration object containing database connection parameters.
-             It should have the following attributes:
-                 - dbuser: The database username.
-                 - dbpass: The database password.
-                 - dbhost: The database host address.
-                 - dbport: The database port number.
-                 - dbname: The name of the database to connect to.
-
-    Returns:
-        An aiomysql Connection object if the connection is successful.
-
-    Raises:
-        Exception: If the connection fails.
-
-    Example:
-        async with await connetti_db_async(cfg) as conn:
-            async with conn.cursor() as cur:
-                await cur.execute("SELECT * FROM table")
-    """
-    try:
-        conn = await aiomysql.connect(
-            user=cfg.dbuser,
-            password=cfg.dbpass,
-            host=cfg.dbhost,
-            port=cfg.dbport,
-            db=cfg.dbname,
-            autocommit=True,
-        )
-        logger.info("Connected (async)")
-        return conn
-    except Exception as e:
-        logger.error(f"Database connection error (async): {e}")
-        raise

vm1/src/utils/database/loader_action.py

@@ -1,242 +0,0 @@
-#!.venv/bin/python
-import asyncio
-import logging
-from datetime import datetime, timedelta
-
-from utils.database import BATCH_SIZE, FLAG_TO_TIMESTAMP
-
-logger = logging.getLogger(__name__)
-
-
-async def load_data(cfg: object, matrice_valori: list, pool: object, type: str) -> bool:
-    """Carica una lista di record di dati grezzi nel database.
-
-    Esegue un'operazione di inserimento massivo (executemany) per caricare i dati.
-    Utilizza la clausola 'ON DUPLICATE KEY UPDATE' per aggiornare i record esistenti.
-    Implementa una logica di re-tentativo in caso di deadlock.
-
-    Args:
-        cfg (object): L'oggetto di configurazione contenente i nomi delle tabelle e i parametri di re-tentativo.
-        matrice_valori (list): Una lista di tuple, dove ogni tupla rappresenta una riga da inserire.
-        pool (object): Il pool di connessioni al database.
-        type (str): tipo di caricamento dati. Per GD fa l'update del tool DT corrispondente
-
-    Returns:
-        bool: True se il caricamento ha avuto successo, False altrimenti.
-    """
-    if not matrice_valori:
-        logger.info("Nulla da caricare.")
-        return True
-
-    if type == "gd" and matrice_valori[0][0] == "RSSI":
-        matrice_valori.pop(0)
-        sql_load_RAWDATA = f"""
-        UPDATE {cfg.dbrawdata} t1
-        JOIN (
-            SELECT id
-            FROM {cfg.dbrawdata}
-            WHERE UnitName = %s AND ToolNameID = %s AND NodeNum = %s
-                AND TIMESTAMP(`EventDate`, `EventTime`) BETWEEN %s AND %s
-            ORDER BY ABS(TIMESTAMPDIFF(SECOND, TIMESTAMP(`EventDate`, `EventTime`), %s))
-            LIMIT 1
-        ) t2 ON t1.id = t2.id
-        SET t1.BatLevelModule = %s, t1.TemperatureModule = %s, t1.RssiModule = %s
-        """
-    else:
-        sql_load_RAWDATA = f"""
-        INSERT INTO {cfg.dbrawdata} (
-            `UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,
-            `Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,
-            `Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`,
-            `BatLevelModule`,`TemperatureModule`, `RssiModule`
-        )
-        VALUES (
-            %s, %s, %s, %s, %s, %s, %s,
-            %s, %s, %s, %s, %s, %s, %s, %s,
-            %s, %s, %s, %s, %s, %s, %s, %s,
-            %s, %s, %s
-        ) as new_data
-        ON DUPLICATE KEY UPDATE
-            `BatLevel` = IF({cfg.dbrawdata}.`BatLevel` != new_data.`BatLevel`, new_data.`BatLevel`, {cfg.dbrawdata}.`BatLevel`),
-            `Temperature` = IF({cfg.dbrawdata}.`Temperature` != new_data.Temperature, new_data.Temperature, {cfg.dbrawdata}.`Temperature`),
-            `Val0` = IF({cfg.dbrawdata}.`Val0` != new_data.Val0 AND new_data.`Val0` IS NOT NULL, new_data.Val0, {cfg.dbrawdata}.`Val0`),
-            `Val1` = IF({cfg.dbrawdata}.`Val1` != new_data.Val1 AND new_data.`Val1` IS NOT NULL, new_data.Val1, {cfg.dbrawdata}.`Val1`),
-            `Val2` = IF({cfg.dbrawdata}.`Val2` != new_data.Val2 AND new_data.`Val2` IS NOT NULL, new_data.Val2, {cfg.dbrawdata}.`Val2`),
-            `Val3` = IF({cfg.dbrawdata}.`Val3` != new_data.Val3 AND new_data.`Val3` IS NOT NULL, new_data.Val3, {cfg.dbrawdata}.`Val3`),
-            `Val4` = IF({cfg.dbrawdata}.`Val4` != new_data.Val4 AND new_data.`Val4` IS NOT NULL, new_data.Val4, {cfg.dbrawdata}.`Val4`),
-            `Val5` = IF({cfg.dbrawdata}.`Val5` != new_data.Val5 AND new_data.`Val5` IS NOT NULL, new_data.Val5, {cfg.dbrawdata}.`Val5`),
-            `Val6` = IF({cfg.dbrawdata}.`Val6` != new_data.Val6 AND new_data.`Val6` IS NOT NULL, new_data.Val6, {cfg.dbrawdata}.`Val6`),
-            `Val7` = IF({cfg.dbrawdata}.`Val7` != new_data.Val7 AND new_data.`Val7` IS NOT NULL, new_data.Val7, {cfg.dbrawdata}.`Val7`),
-            `Val8` = IF({cfg.dbrawdata}.`Val8` != new_data.Val8 AND new_data.`Val8` IS NOT NULL, new_data.Val8, {cfg.dbrawdata}.`Val8`),
-            `Val9` = IF({cfg.dbrawdata}.`Val9` != new_data.Val9 AND new_data.`Val9` IS NOT NULL, new_data.Val9, {cfg.dbrawdata}.`Val9`),
-            `ValA` = IF({cfg.dbrawdata}.`ValA` != new_data.ValA AND new_data.`ValA` IS NOT NULL, new_data.ValA, {cfg.dbrawdata}.`ValA`),
-            `ValB` = IF({cfg.dbrawdata}.`ValB` != new_data.ValB AND new_data.`ValB` IS NOT NULL, new_data.ValB, {cfg.dbrawdata}.`ValB`),
-            `ValC` = IF({cfg.dbrawdata}.`ValC` != new_data.ValC AND new_data.`ValC` IS NOT NULL, new_data.ValC, {cfg.dbrawdata}.`ValC`),
-            `ValD` = IF({cfg.dbrawdata}.`ValD` != new_data.ValD AND new_data.`ValD` IS NOT NULL, new_data.ValD, {cfg.dbrawdata}.`ValD`),
-            `ValE` = IF({cfg.dbrawdata}.`ValE` != new_data.ValE AND new_data.`ValE` IS NOT NULL, new_data.ValE, {cfg.dbrawdata}.`ValE`),
-            `ValF` = IF({cfg.dbrawdata}.`ValF` != new_data.ValF AND new_data.`ValF` IS NOT NULL, new_data.ValF, {cfg.dbrawdata}.`ValF`),
-            `BatLevelModule` = IF({cfg.dbrawdata}.`BatLevelModule` != new_data.BatLevelModule, new_data.BatLevelModule,
-                                  {cfg.dbrawdata}.`BatLevelModule`),
-            `TemperatureModule` = IF({cfg.dbrawdata}.`TemperatureModule` != new_data.TemperatureModule, new_data.TemperatureModule,
-                                  {cfg.dbrawdata}.`TemperatureModule`),
-            `RssiModule` = IF({cfg.dbrawdata}.`RssiModule` != new_data.RssiModule, new_data.RssiModule, {cfg.dbrawdata}.`RssiModule`),
-            `Created_at` = NOW()
-        """
-    # logger.info(f"Query insert: {sql_load_RAWDATA}.")
-    # logger.info(f"Matrice valori da inserire: {matrice_valori}.")
-    rc = False
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            for attempt in range(cfg.max_retries):
-                try:
-                    logger.info(f"Loading data attempt {attempt + 1}.")
-
-                    for i in range(0, len(matrice_valori), BATCH_SIZE):
-                        batch = matrice_valori[i : i + BATCH_SIZE]
-
-                        await cur.executemany(sql_load_RAWDATA, batch)
-                        await conn.commit()
-
-                        logger.info(f"Completed batch {i // BATCH_SIZE + 1}/{(len(matrice_valori) - 1) // BATCH_SIZE + 1}")
-
-                    logger.info("Data loaded.")
-                    rc = True
-                    break
-                except Exception as e:
-                    await conn.rollback()
-                    logger.error(f"Error: {e}.")
-                    # logger.error(f"Matrice valori da inserire: {batch}.")
-
-                    if e.args[0] == 1213:  # Deadlock detected
-                        logger.warning(f"Deadlock detected, attempt {attempt + 1}/{cfg.max_retries}")
-
-                        if attempt < cfg.max_retries - 1:
-                            delay = 2 * attempt
-                            await asyncio.sleep(delay)
-                            continue
-                        else:
-                            logger.error("Max retry attempts reached for deadlock")
-                    raise
-    return rc
-
-
-async def update_status(cfg: object, id: int, status: str, pool: object) -> None:
-    """Aggiorna lo stato di un record nella tabella dei record CSV.
-
-    Args:
-        cfg (object): L'oggetto di configurazione contenente il nome della tabella.
-        id (int): L'ID del record da aggiornare.
-        status (int): Il nuovo stato da impostare.
-        pool (object): Il pool di connessioni al database.
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                timestamp_field = FLAG_TO_TIMESTAMP[status]
-                await cur.execute(
-                    f"""UPDATE {cfg.dbrectable} SET
-                        status = status | %s,
-                        {timestamp_field} = NOW()
-                        WHERE id = %s
-                    """,
-                    (status, id)
-                )
-                await conn.commit()
-                logger.info(f"Status updated id {id}.")
-            except Exception as e:
-                await conn.rollback()
-                logger.error(f"Error: {e}")
-
-
-async def unlock(cfg: object, id: int, pool: object) -> None:
-    """Sblocca un record nella tabella dei record CSV.
-
-    Imposta il campo 'locked' a 0 per un dato ID.
-
-    Args:
-        cfg (object): L'oggetto di configurazione contenente il nome della tabella.
-        id (int): L'ID del record da sbloccare.
-        pool (object): Il pool di connessioni al database.
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute(f"UPDATE {cfg.dbrectable} SET locked = 0 WHERE id = %s", (id,))
-                await conn.commit()
-                logger.info(f"id {id} unlocked.")
-            except Exception as e:
-                await conn.rollback()
-                logger.error(f"Error: {e}")
-
-
-async def get_matlab_cmd(cfg: object, unit: str, tool: str, pool: object) -> tuple:
-    """Recupera le informazioni per l'esecuzione di un comando Matlab dal database.
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        unit (str): Il nome dell'unità.
-        tool (str): Il nome dello strumento.
-        pool (object): Il pool di connessioni al database.
-
-    Returns:
-        tuple: Una tupla contenente le informazioni del comando Matlab, o None in caso di errore.
-    """
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute('''SELECT m.matcall, t.ftp_send, t.unit_id, s.`desc` AS statustools, t.api_send, u.inoltro_api,
-                            u.inoltro_api_url, u.inoltro_api_bearer_token, IFNULL(u.duedate, "") AS duedate
-                            FROM matfuncs AS m
-                            INNER JOIN tools AS t ON t.matfunc = m.id
-                            INNER JOIN units AS u ON u.id = t.unit_id
-                            INNER JOIN statustools AS s ON t.statustool_id = s.id
-                            WHERE t.name = %s AND u.name = %s''',
-                            (tool, unit))
-                return await cur.fetchone()
-            except Exception as e:
-                logger.error(f"Error: {e}")
-
-
-async def find_nearest_timestamp(cfg: object, unit_tool_data: dict, pool: object) -> tuple:
-    """
-    Finds the nearest timestamp in the raw data table based on a reference timestamp
-    and unit/tool/node information.
-
-    Args:
-        cfg (object): Configuration object containing database table name (`cfg.dbrawdata`).
-        unit_tool_data (dict): A dictionary containing:
-            - "timestamp" (str): The reference timestamp string in "%Y-%m-%d %H:%M:%S" format.
-            - "unit" (str): The UnitName to filter by.
-            - "tool" (str): The ToolNameID to filter by.
-            - "node_num" (int): The NodeNum to filter by.
-        pool (object): The database connection pool.
-
-    Returns:
-        tuple: A tuple containing the event timestamp, BatLevel, and Temperature of the
-               nearest record, or None if an error occurs or no record is found.
-    """
-
-    ref_timestamp = datetime.strptime(unit_tool_data["timestamp"], "%Y-%m-%d %H:%M:%S")
-    start_timestamp = ref_timestamp - timedelta(seconds=45)
-    end_timestamp = ref_timestamp + timedelta(seconds=45)
-    logger.info(f"Find nearest timestamp: {ref_timestamp}")
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cur:
-            try:
-                # Use parameterized query to prevent SQL injection
-                await cur.execute(f'''SELECT TIMESTAMP(`EventDate`, `EventTime`) AS event_timestamp, BatLevel, Temperature
-                                  FROM {cfg.dbrawdata}
-                                  WHERE UnitName = %s AND ToolNameID = %s
-                                  AND NodeNum = %s
-                                    AND TIMESTAMP(`EventDate`, `EventTime`) BETWEEN %s AND %s
-                                  ORDER BY ABS(TIMESTAMPDIFF(SECOND, TIMESTAMP(`EventDate`, `EventTime`), %s))
-                                  LIMIT 1
-                                  ''',
-                                  (unit_tool_data["unit"], unit_tool_data["tool"], unit_tool_data["node_num"],
-                                   start_timestamp, end_timestamp, ref_timestamp))
-                return await cur.fetchone()
-            except Exception as e:
-                logger.error(f"Error: {e}")

vm1/src/utils/database/nodes_query.py

@@ -1,48 +0,0 @@
-import logging
-
-import aiomysql
-
-logger = logging.getLogger(__name__)
-
-
-async def get_nodes_type(cfg: object, tool: str, unit: str, pool: object) -> tuple:
-    """Recupera le informazioni sui nodi (tipo, canali, input) per un dato strumento e unità.
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        tool (str): Il nome dello strumento.
-        unit (str): Il nome dell'unità.
-        pool (object): Il pool di connessioni al database.
-
-    Returns:
-        tuple: Una tupla contenente quattro liste: canali, tipi, ain, din.
-               Se non vengono trovati risultati, restituisce (None, None, None, None).
-    """
-
-    async with pool.acquire() as conn:
-        async with conn.cursor(aiomysql.DictCursor) as cur:
-            # Use parameterized query to prevent SQL injection
-            await cur.execute(f"""
-                SELECT t.name AS name, n.seq AS seq, n.num AS num, n.channels AS channels, y.type AS type, n.ain AS ain, n.din AS din
-                FROM {cfg.dbname}.{cfg.dbnodes} AS n
-                INNER JOIN tools AS t ON t.id = n.tool_id
-                INNER JOIN units AS u ON u.id = t.unit_id
-                INNER JOIN nodetypes AS y ON n.nodetype_id = y.id
-                WHERE y.type NOT IN ('Anchor Link', 'None') AND t.name = %s AND u.name = %s
-                ORDER BY n.num;
-            """, (tool, unit))
-
-            results = await cur.fetchall()
-            logger.info(f"{unit} - {tool}: {cur.rowcount} rows selected to get node type/Ain/Din/channels.")
-
-            if not results:
-                logger.info(f"{unit} - {tool}: Node/Channels/Ain/Din not defined.")
-                return None, None, None, None
-            else:
-                channels, types, ains, dins = [], [], [], []
-                for row in results:
-                    channels.append(row["channels"])
-                    types.append(row["type"])
-                    ains.append(row["ain"])
-                    dins.append(row["din"])
-                return channels, types, ains, dins

vm1/src/utils/general.py

@@ -1,89 +0,0 @@
-import glob
-import logging
-import os
-from itertools import chain, cycle
-
-logger = logging.getLogger()
-
-
-def alterna_valori(*valori: any, ping_pong: bool = False) -> any:
-    """
-    Genera una sequenza ciclica di valori, con opzione per una sequenza "ping-pong".
-
-    Args:
-        *valori (any): Uno o più valori da ciclare.
-        ping_pong (bool, optional): Se True, la sequenza sarà valori -> valori al contrario.
-                                    Ad esempio, per (1, 2, 3) diventa 1, 2, 3, 2, 1, 2, 3, ...
-                                    Se False, la sequenza è semplicemente ciclica.
-                                    Defaults to False.
-
-    Yields:
-        any: Il prossimo valore nella sequenza ciclica.
-
-    """
-    if not valori:
-        return
-
-    if ping_pong:
-        # Crea la sequenza ping-pong: valori + valori al contrario (senza ripetere primo e ultimo)
-        forward = valori
-        backward = valori[-2:0:-1]  # Esclude ultimo e primo elemento
-        ping_pong_sequence = chain(forward, backward)
-        yield from cycle(ping_pong_sequence)
-    else:
-        yield from cycle(valori)
-
-
-async def read_error_lines_from_logs(base_path: str, pattern: str) -> tuple[list[str], list[str]]:
-    """
-    Reads error and warning lines from log files matching a given pattern within a base path.
-
-    This asynchronous function searches for log files, reads their content, and categorizes
-    lines starting with 'Error' as errors and all other non-empty lines as warnings.
-
-    Args:
-        base_path (str): The base directory where log files are located.
-        pattern (str): The glob-style pattern to match log filenames (e.g., "*.txt", "prefix_*_output_error.txt").
-
-    Returns:
-        tuple[list[str], list[str]]: A tuple containing two lists:
-                                     - The first list contains all extracted error messages.
-                                     - The second list contains all extracted warning messages."""
-    import aiofiles
-
-    # Costruisce il path completo con il pattern
-    search_pattern = os.path.join(base_path, pattern)
-
-    # Trova tutti i file che corrispondono al pattern
-    matching_files = glob.glob(search_pattern)
-
-    if not matching_files:
-        logger.warning(f"Nessun file trovato per il pattern: {search_pattern}")
-        return [], []
-
-    all_errors = []
-    all_warnings = []
-
-    for file_path in matching_files:
-        try:
-            # Use async file I/O to prevent blocking the event loop
-            async with aiofiles.open(file_path, encoding="utf-8") as file:
-                content = await file.read()
-                lines = content.splitlines()
-                # Usando dict.fromkeys() per mantenere l'ordine e togliere le righe duplicate per i warnings
-                non_empty_lines = [line.strip() for line in lines if line.strip()]
-
-                # Fix: Accumulate errors and warnings from all files instead of overwriting
-                file_errors = [line for line in non_empty_lines if line.startswith("Error")]
-                file_warnings = [line for line in non_empty_lines if not line.startswith("Error")]
-
-                all_errors.extend(file_errors)
-                all_warnings.extend(file_warnings)
-
-        except Exception as e:
-            logger.error(f"Errore durante la lettura del file {file_path}: {e}")
-
-    # Remove duplicates from warnings while preserving order
-    unique_warnings = list(dict.fromkeys(all_warnings))
-
-    return all_errors, unique_warnings

vm1/src/utils/orchestrator_utils.py

@@ -1,179 +0,0 @@
-import asyncio
-import contextvars
-import logging
-import os
-import signal
-from collections.abc import Callable, Coroutine
-from logging.handlers import RotatingFileHandler
-from typing import Any
-
-import aiomysql
-
-# Crea una context variable per identificare il worker
-worker_context = contextvars.ContextVar("worker_id", default="^-^")
-
-# Global shutdown event
-shutdown_event = asyncio.Event()
-
-
-# Formatter personalizzato che include il worker_id
-class WorkerFormatter(logging.Formatter):
-    """Formatter personalizzato per i log che include l'ID del worker."""
-
-    def format(self, record: logging.LogRecord) -> str:
-        """Formatta il record di log includendo l'ID del worker.
-
-        Args:
-            record (str): Il record di log da formattare.
-
-        Returns:
-            La stringa formattata del record di log.
-        """
-        record.worker_id = worker_context.get()
-        return super().format(record)
-
-
-def setup_logging(log_filename: str, log_level_str: str):
-    """Configura il logging globale con rotation automatica.
-
-    Args:
-        log_filename (str): Percorso del file di log.
-        log_level_str (str): Livello di log (es. "INFO", "DEBUG").
-    """
-    logger = logging.getLogger()
-    formatter = WorkerFormatter("%(asctime)s - PID: %(process)d.Worker-%(worker_id)s.%(name)s.%(funcName)s.%(levelname)s: %(message)s")
-
-    # Rimuovi eventuali handler esistenti
-    if logger.hasHandlers():
-        logger.handlers.clear()
-
-    # Handler per file con rotation (max 10MB per file, mantiene 5 backup)
-    file_handler = RotatingFileHandler(
-        log_filename,
-        maxBytes=10 * 1024 * 1024,  # 10 MB
-        backupCount=5,  # Mantiene 5 file di backup
-        encoding="utf-8"
-    )
-    file_handler.setFormatter(formatter)
-    logger.addHandler(file_handler)
-
-    # Handler per console (utile per Docker)
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(formatter)
-    logger.addHandler(console_handler)
-
-    log_level = getattr(logging, log_level_str.upper(), logging.INFO)
-    logger.setLevel(log_level)
-    logger.info("Logging configurato correttamente con rotation (10MB, 5 backup)")
-
-
-def setup_signal_handlers(logger: logging.Logger):
-    """Setup signal handlers for graceful shutdown.
-
-    Handles both SIGTERM (from systemd/docker) and SIGINT (Ctrl+C).
-
-    Args:
-        logger: Logger instance for logging shutdown events.
-    """
-
-    def signal_handler(signum, frame):
-        """Handle shutdown signals."""
-        sig_name = signal.Signals(signum).name
-        logger.info(f"Ricevuto segnale {sig_name} ({signum}). Avvio shutdown graceful...")
-        shutdown_event.set()
-
-    # Register handlers for graceful shutdown
-    signal.signal(signal.SIGTERM, signal_handler)
-    signal.signal(signal.SIGINT, signal_handler)
-    logger.info("Signal handlers configurati (SIGTERM, SIGINT)")
-
-
-async def run_orchestrator(
-    config_class: Any,
-    worker_coro: Callable[[int, Any, Any], Coroutine[Any, Any, None]],
-):
-    """Funzione principale che inizializza e avvia un orchestratore.
-
-    Gestisce graceful shutdown su SIGTERM e SIGINT, permettendo ai worker
-    di completare le operazioni in corso prima di terminare.
-
-    Args:
-        config_class: La classe di configurazione da istanziare.
-        worker_coro: La coroutine del worker da eseguire in parallelo.
-    """
-    logger = logging.getLogger()
-    logger.info("Avvio del sistema...")
-
-    cfg = config_class()
-    logger.info("Configurazione caricata correttamente")
-
-    debug_mode = False
-    pool = None
-
-    try:
-        log_level = os.getenv("LOG_LEVEL", "INFO").upper()
-        setup_logging(cfg.logfilename, log_level)
-        debug_mode = logger.getEffectiveLevel() == logging.DEBUG
-
-        # Setup signal handlers for graceful shutdown
-        setup_signal_handlers(logger)
-
-        logger.info(f"Avvio di {cfg.max_threads} worker concorrenti")
-
-        pool = await aiomysql.create_pool(
-            host=cfg.dbhost,
-            user=cfg.dbuser,
-            password=cfg.dbpass,
-            db=cfg.dbname,
-            minsize=cfg.max_threads,
-            maxsize=cfg.max_threads * 2,  # Optimized: 2x instead of 4x (more efficient)
-            pool_recycle=3600,
-            # Note: aiomysql doesn't support pool_pre_ping like SQLAlchemy
-            # Connection validity is checked via pool_recycle
-        )
-
-        tasks = [asyncio.create_task(worker_coro(i, cfg, pool)) for i in range(cfg.max_threads)]
-
-        logger.info("Sistema avviato correttamente. In attesa di nuovi task...")
-
-        # Wait for either tasks to complete or shutdown signal
-        shutdown_task = asyncio.create_task(shutdown_event.wait())
-        done, pending = await asyncio.wait(
-            [shutdown_task, *tasks], return_when=asyncio.FIRST_COMPLETED
-        )
-
-        if shutdown_event.is_set():
-            logger.info("Shutdown event rilevato. Cancellazione worker in corso...")
-
-            # Cancel all pending tasks
-            for task in pending:
-                if not task.done():
-                    task.cancel()
-
-            # Wait for tasks to finish with timeout
-            if pending:
-                logger.info(f"In attesa della terminazione di {len(pending)} worker...")
-                try:
-                    await asyncio.wait_for(
-                        asyncio.gather(*pending, return_exceptions=True),
-                        timeout=30.0,  # Grace period for workers to finish
-                    )
-                    logger.info("Tutti i worker terminati correttamente")
-                except TimeoutError:
-                    logger.warning("Timeout raggiunto. Alcuni worker potrebbero non essere terminati correttamente")
-
-    except KeyboardInterrupt:
-        logger.info("Info: Shutdown richiesto da KeyboardInterrupt... chiusura in corso")
-
-    except Exception as e:
-        logger.error(f"Errore principale: {e}", exc_info=debug_mode)
-
-    finally:
-        # Always cleanup pool
-        if pool:
-            logger.info("Chiusura pool di connessioni database...")
-            pool.close()
-            await pool.wait_closed()
-            logger.info("Pool database chiuso correttamente")
-
-        logger.info("Shutdown completato")

vm1/src/utils/parsers/__init__.py

@@ -1 +0,0 @@
-"""Parser delle centraline con le tipologie di unit e tool"""

vm1/src/utils/parsers/by_name/__init__.py

@@ -1 +0,0 @@
-"""Parser delle centraline con nomi di unit e tool"""

vm1/src/utils/parsers/by_type/__init__.py

@@ -1 +0,0 @@
-"""Parser delle centraline"""

vm1/src/utils/parsers/by_type/cr1000x_cr1000x.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'cr1000x_cr1000x'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/d2w_d2w.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'd2w_d2w'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g201_g201.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as channels_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g201_g201'.
-
-    Questa funzione è un wrapper per `channels_main_loader` e passa il tipo
-    di elaborazione come "channels".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await channels_main_loader(cfg, id, pool, "channels")

vm1/src/utils/parsers/by_type/g301_g301.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g301_g301'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g801_iptm.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g801_iptm'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g801_loc.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as analog_dig_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g801_loc'.
-
-    Questa funzione è un wrapper per `analog_dig_main_loader` e passa il tipo
-    di elaborazione come "analogic_digital".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await analog_dig_main_loader(cfg, id, pool, "analogic_digital")

vm1/src/utils/parsers/by_type/g801_mums.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g801_mums'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g801_musa.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as musa_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g801_musa'.
-
-    Questa funzione è un wrapper per `musa_main_loader` e passa il tipo
-    di elaborazione come "musa".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await musa_main_loader(cfg, id, pool, "musa")

vm1/src/utils/parsers/by_type/g801_mux.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as channels_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g801_mux'.
-
-    Questa funzione è un wrapper per `channels_main_loader` e passa il tipo
-    di elaborazione come "channels".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await channels_main_loader(cfg, id, pool, "channels")

vm1/src/utils/parsers/by_type/g802_dsas.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_dsas'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g802_gd.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as gd_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_gd'.
-
-    Questa funzione è un wrapper per `gd_main_loader` e passa il tipo
-    di elaborazione come "gd".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await gd_main_loader(cfg, id, pool, "gd")

vm1/src/utils/parsers/by_type/g802_loc.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as analog_dig_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_loc'.
-
-    Questa funzione è un wrapper per `analog_dig_main_loader` e passa il tipo
-    di elaborazione come "analogic_digital".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await analog_dig_main_loader(cfg, id, pool, "analogic_digital")

vm1/src/utils/parsers/by_type/g802_modb.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_modb'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g802_mums.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_mums'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/g802_mux.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as channels_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'g802_mux'.
-
-    Questa funzione è un wrapper per `channels_main_loader` e passa il tipo
-    di elaborazione come "channels".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await channels_main_loader(cfg, id, pool, "channels")

vm1/src/utils/parsers/by_type/gs1_gs1.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as tlp_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'gs1_gs1'.
-
-    Questa funzione è un wrapper per `tlp_main_loader` e passa il tipo
-    di elaborazione come "tlp".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await tlp_main_loader(cfg, id, pool, "tlp")

vm1/src/utils/parsers/by_type/hirpinia_hirpinia.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as hirpinia_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'hirpinia_hirpinia'.
-
-    Questa funzione è un wrapper per `main_old_script_loader` e passa il nome
-    dello script di elaborazione come "hirpiniaLoadScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await hirpinia_main_loader(cfg, id, pool, "hirpiniaLoadScript")

vm1/src/utils/parsers/by_type/hortus_hortus.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as pipe_sep_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'hortus_hortus'.
-
-    Questa funzione è un wrapper per `pipe_sep_main_loader` e passa il tipo
-    di elaborazione come "pipe_separator".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await pipe_sep_main_loader(cfg, id, pool, "pipe_separator")

vm1/src/utils/parsers/by_type/isi_csv_log_vulink.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as vulink_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'isi_csv_log_vulink'.
-
-    Questa funzione è un wrapper per `vulink_main_loader` e passa il nome
-    dello script di elaborazione come "vulinkScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await vulink_main_loader(cfg, id, pool, "vulinkScript")

vm1/src/utils/parsers/by_type/sisgeo_health.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as sisgeo_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'sisgeo_health'.
-
-    Questa funzione è un wrapper per `main_old_script_loader` e passa il nome
-    dello script di elaborazione come "sisgeoLoadScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await sisgeo_main_loader(cfg, id, pool, "sisgeoLoadScript")

vm1/src/utils/parsers/by_type/sisgeo_readings.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as sisgeo_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'sisgeo_readings'.
-
-    Questa funzione è un wrapper per `main_old_script_loader` e passa il nome
-    dello script di elaborazione come "sisgeoLoadScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await sisgeo_main_loader(cfg, id, pool, "sisgeoLoadScript")

vm1/src/utils/parsers/by_type/sorotecpini_co.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as sorotecPini_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'sorotecpini_co'.
-
-    Questa funzione è un wrapper per `sorotecPini_main_loader` e passa il nome
-    dello script di elaborazione come "sorotecPini".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await sorotecPini_main_loader(cfg, id, pool, "sorotecPini")

vm1/src/utils/parsers/by_type/stazionetotale_integrity_monitor.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as ts_pini_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'stazionetotale_integrity_monitor'.
-
-    Questa funzione è un wrapper per `main_old_script_loader` e passa il nome
-    dello script di elaborazione come "TS_PiniScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await ts_pini_main_loader(cfg, id, pool, "TS_PiniScript")

vm1/src/utils/parsers/by_type/stazionetotale_messpunktepini.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_old_script_loader as ts_pini_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'stazionetotale_messpunktepini'.
-
-    Questa funzione è un wrapper per `ts_pini_main_loader` e passa il nome
-    dello script di elaborazione come "TS_PiniScript".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await ts_pini_main_loader(cfg, id, pool, "TS_PiniScript")

vm1/src/utils/parsers/by_type/tlp_loc.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as analog_dig_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'tlp_loc'.
-
-    Questa funzione è un wrapper per `analog_dig_main_loader` e passa il tipo
-    di elaborazione come "analogic_digital".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await analog_dig_main_loader(cfg, id, pool, "analogic_digital")

vm1/src/utils/parsers/by_type/tlp_tlp.py

@@ -1,16 +0,0 @@
-from utils.csv.loaders import main_loader as tlp_main_loader
-
-
-async def main_loader(cfg: object, id: int, pool: object) -> None:
-    """
-    Carica ed elabora i dati CSV specifici per il tipo 'tlp_tlp'.
-
-    Questa funzione è un wrapper per `tlp_main_loader` e passa il tipo
-    di elaborazione come "tlp".
-
-    Args:
-        cfg (object): L'oggetto di configurazione.
-        id (int): L'ID del record CSV da elaborare.
-        pool (object): Il pool di connessioni al database.
-    """
-    await tlp_main_loader(cfg, id, pool, "tlp")

vm1/src/utils/servers/sftp_server.py

@@ -1,240 +0,0 @@
-"""
-SFTP Server implementation using asyncssh.
-Shares the same authentication system and file handling logic as the FTP server.
-"""
-
-import asyncio
-import logging
-from pathlib import Path
-
-import asyncssh
-
-from utils.connect import file_management
-from utils.database.connection import connetti_db_async
-
-logger = logging.getLogger(__name__)
-
-
-class ASESFTPServer(asyncssh.SFTPServer):
-    """Custom SFTP server that handles file uploads with the same logic as FTP server."""
-
-    def __init__(self, chan):
-        """Initialize SFTP server with channel."""
-        super().__init__(chan)
-        # Get config from connection (set during authentication)
-        self.cfg = chan.get_connection()._cfg
-
-    async def close(self):
-        """Called when SFTP session is closed."""
-        logger.info(f"SFTP session closed for user: {self._chan.get_connection().get_extra_info('username')}")
-        await super().close()
-
-
-class ASESSHServer(asyncssh.SSHServer):
-    """Custom SSH server for SFTP authentication using database."""
-
-    def __init__(self, cfg):
-        """Initialize SSH server with configuration."""
-        self.cfg = cfg
-        self.user_home_dirs = {}  # Store user home directories after authentication
-        super().__init__()
-
-    def connection_made(self, conn):
-        """Called when connection is established."""
-        # Store config in connection for later use
-        conn._cfg = self.cfg
-        conn._ssh_server = self  # Store reference to server for accessing user_home_dirs
-        logger.info(f"SSH connection from {conn.get_extra_info('peername')[0]}")
-
-    def connection_lost(self, exc):
-        """Called when connection is lost."""
-        if exc:
-            logger.error(f"SSH connection lost: {exc}")
-
-    async def validate_password(self, username, password):
-        """
-        Validate user credentials against database.
-        Same logic as DatabaseAuthorizer for FTP.
-        """
-        from hashlib import sha256
-
-        # Hash the provided password
-        password_hash = sha256(password.encode("UTF-8")).hexdigest()
-
-        # Check if user is admin
-        if username == self.cfg.adminuser[0]:
-            if self.cfg.adminuser[1] == password_hash:
-                # Store admin home directory
-                self.user_home_dirs[username] = self.cfg.adminuser[2]
-                logger.info(f"Admin user '{username}' authenticated successfully (home: {self.cfg.adminuser[2]})")
-                return True
-            else:
-                logger.warning(f"Failed admin login attempt for user: {username}")
-                return False
-
-        # For regular users, check database
-        try:
-            conn = await connetti_db_async(self.cfg)
-            cur = await conn.cursor()
-
-            # Query user from database
-            await cur.execute(
-                f"SELECT ftpuser, hash, virtpath, perm, disabled_at FROM {self.cfg.dbname}.{self.cfg.dbusertable} WHERE ftpuser = %s",
-                (username,)
-            )
-
-            result = await cur.fetchone()
-            await cur.close()
-            conn.close()
-
-            if not result:
-                logger.warning(f"SFTP login attempt for non-existent user: {username}")
-                return False
-
-            ftpuser, stored_hash, virtpath, perm, disabled_at = result
-
-            # Check if user is disabled
-            if disabled_at is not None:
-                logger.warning(f"SFTP login attempt for disabled user: {username}")
-                return False
-
-            # Verify password
-            if stored_hash != password_hash:
-                logger.warning(f"Invalid password for SFTP user: {username}")
-                return False
-
-            # Authentication successful - ensure user directory exists
-            try:
-                Path(virtpath).mkdir(parents=True, exist_ok=True)
-            except Exception as e:
-                logger.error(f"Failed to create directory for user {username}: {e}")
-                return False
-
-            # Store the user's home directory for chroot
-            self.user_home_dirs[username] = virtpath
-
-            logger.info(f"Successful SFTP login for user: {username} (home: {virtpath})")
-            return True
-
-        except Exception as e:
-            logger.error(f"Database error during SFTP authentication for user {username}: {e}", exc_info=True)
-            return False
-
-    def password_auth_supported(self):
-        """Enable password authentication."""
-        return True
-
-    def begin_auth(self, username):
-        """Called when authentication begins."""
-        logger.debug(f"Authentication attempt for user: {username}")
-        return True
-
-
-class SFTPFileHandler(asyncssh.SFTPServer):
-    """Extended SFTP server with file upload handling."""
-
-    def __init__(self, chan):
-        super().__init__(chan, chroot=self._get_user_home(chan))
-        self.cfg = chan.get_connection()._cfg
-        self._open_files = {}  # Track open files for processing
-
-    @staticmethod
-    def _get_user_home(chan):
-        """Get the home directory for the authenticated user."""
-        conn = chan.get_connection()
-        username = conn.get_extra_info('username')
-        ssh_server = getattr(conn, '_ssh_server', None)
-
-        if ssh_server and username in ssh_server.user_home_dirs:
-            return ssh_server.user_home_dirs[username]
-
-        # Fallback for admin user
-        if hasattr(conn, '_cfg') and username == conn._cfg.adminuser[0]:
-            return conn._cfg.adminuser[2]
-
-        return None
-
-    def open(self, path, pflags, attrs):
-        """Track files being opened for writing."""
-        result = super().open(path, pflags, attrs)
-
-        # If file is opened for writing (pflags contains FXF_WRITE)
-        if pflags & 0x02:  # FXF_WRITE flag
-            real_path = self.map_path(path)
-            # Convert bytes to str if necessary
-            if isinstance(real_path, bytes):
-                real_path = real_path.decode('utf-8')
-            self._open_files[result] = real_path
-            logger.debug(f"File opened for writing: {real_path}")
-
-        return result
-
-    async def close(self, file_obj):
-        """Process file after it's closed."""
-        # Call parent close first (this doesn't return anything useful)
-        result = super().close(file_obj)
-
-        # Check if this file was tracked
-        if file_obj in self._open_files:
-            filepath = self._open_files.pop(file_obj)
-
-            # Process CSV files
-            if filepath.lower().endswith('.csv'):
-                try:
-                    logger.info(f"CSV file closed after upload via SFTP: {filepath}")
-
-                    # Get username
-                    username = self._chan.get_connection().get_extra_info('username')
-
-                    # Create a mock handler object with required attributes
-                    mock_handler = type('obj', (object,), {
-                        'cfg': self.cfg,
-                        'username': username
-                    })()
-
-                    # Call the file processing function
-                    from utils.connect import file_management
-                    await file_management.on_file_received_async(mock_handler, filepath)
-                except Exception as e:
-                    logger.error(f"Error processing SFTP file on close: {e}", exc_info=True)
-
-        return result
-
-    async def exit(self):
-        """Handle session close."""
-        await super().exit()
-
-    # Note: File processing is handled in close() method, not here
-    # This avoids double-processing when both close and rename are called
-
-
-async def start_sftp_server(cfg, host='0.0.0.0', port=22):
-    """
-    Start SFTP server.
-
-    Args:
-        cfg: Configuration object
-        host: Host to bind to
-        port: Port to bind to
-
-    Returns:
-        asyncssh server object
-    """
-    logger.info(f"Starting SFTP server on {host}:{port}")
-
-    # Create SSH server
-    ssh_server = ASESSHServer(cfg)
-
-    # Start asyncssh server
-    server = await asyncssh.create_server(
-        lambda: ssh_server,
-        host,
-        port,
-        server_host_keys=['/app/ssh_host_key'],  # You'll need to generate this
-        sftp_factory=SFTPFileHandler,
-    )
-
-    logger.info(f"SFTP server started successfully on {host}:{port}")
-    logger.info(f"Database connection: {cfg.dbuser}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbname}")
-
-    return server

vm1/src/utils/timestamp/date_check.py

@@ -1,44 +0,0 @@
-from datetime import datetime
-
-
-def normalizza_data(data_string: str) -> str:
-    """
-    Normalizza una stringa di data al formato YYYY-MM-DD, provando diversi formati di input.
-
-    Args:
-        data_string (str): La stringa di data da normalizzare.
-
-    Returns:
-        str: La data normalizzata nel formato YYYY-MM-DD,
-             o None se la stringa non può essere interpretata come una data.
-    """
-    formato_desiderato = "%Y-%m-%d"
-    formati_input = [
-        "%Y/%m/%d",
-        "%Y-%m-%d",
-        "%d-%m-%Y",
-        "%d/%m/%Y",
-    ]  # Ordine importante: prova prima il più probabile
-
-    for formato_input in formati_input:
-        try:
-            data_oggetto = datetime.strptime(data_string, formato_input)
-            return data_oggetto.strftime(formato_desiderato)
-        except ValueError:
-            continue  # Prova il formato successivo se quello attuale fallisce
-
-    return None  # Se nessun formato ha avuto successo
-
-
-def normalizza_orario(orario_str):
-    try:
-        # Prova prima con HH:MM:SS
-        dt = datetime.strptime(orario_str, "%H:%M:%S")
-        return dt.strftime("%H:%M:%S")
-    except ValueError:
-        try:
-            # Se fallisce, prova con HH:MM
-            dt = datetime.strptime(orario_str, "%H:%M")
-            return dt.strftime("%H:%M:%S")
-        except ValueError:
-            return orario_str  # Restituisce originale se non parsabile

vm2/src/elab_orchestrator.py

@@ -1,137 +0,0 @@
-#!.venv/bin/python
-"""
-Orchestratore dei worker che lanciano le elaborazioni
-"""
-
-# Import necessary libraries
-import asyncio
-import logging
-
-# Import custom modules for configuration and database connection
-from utils.config import loader_matlab_elab as setting
-from utils.connect.send_email import send_error_email
-from utils.csv.loaders import get_next_csv_atomic
-from utils.database import WorkflowFlags
-from utils.database.action_query import check_flag_elab, get_tool_info
-from utils.database.loader_action import unlock, update_status
-from utils.general import read_error_lines_from_logs
-from utils.orchestrator_utils import run_orchestrator, shutdown_event, worker_context
-
-# Initialize the logger for this module
-logger = logging.getLogger()
-
-# Delay tra un processamento CSV e il successivo (in secondi)
-ELAB_PROCESSING_DELAY = 0.2
-# Tempo di attesa se non ci sono record da elaborare
-NO_RECORD_SLEEP = 60
-
-
-async def worker(worker_id: int, cfg: object, pool: object) -> None:
-    """Esegue il ciclo di lavoro per l'elaborazione dei dati caricati.
-
-    Il worker preleva un record dal database che indica dati pronti per
-    l'elaborazione, esegue un comando Matlab associato e attende
-    prima di iniziare un nuovo ciclo.
-
-    Supporta graceful shutdown controllando il shutdown_event tra le iterazioni.
-
-    Args:
-        worker_id (int): L'ID univoco del worker.
-        cfg (object): L'oggetto di configurazione.
-        pool (object): Il pool di connessioni al database.
-    """
-    # Imposta il context per questo worker
-    worker_context.set(f"W{worker_id:02d}")
-
-    debug_mode = logging.getLogger().getEffectiveLevel() == logging.DEBUG
-    logger.info("Avviato")
-
-    try:
-        while not shutdown_event.is_set():
-            try:
-                logger.info("Inizio elaborazione")
-                if not await check_flag_elab(pool):
-                    record = await get_next_csv_atomic(pool, cfg.dbrectable, WorkflowFlags.DATA_LOADED, WorkflowFlags.DATA_ELABORATED)
-                    if record:
-                        rec_id, _, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
-                        if tool_type.lower() != "gd":  # i tool GD non devono essere elaborati ???
-                            tool_elab_info = await get_tool_info(WorkflowFlags.DATA_ELABORATED, unit_name.upper(), tool_name.upper(), pool)
-                            if tool_elab_info:
-                                if tool_elab_info["statustools"].lower() in cfg.elab_status:
-                                    logger.info("Elaborazione ID %s per %s %s", rec_id, unit_name, tool_name)
-                                    await update_status(cfg, rec_id, WorkflowFlags.START_ELAB, pool)
-                                    matlab_cmd = f"timeout {cfg.matlab_timeout} ./run_{tool_elab_info['matcall']}.sh \
-                                        {cfg.matlab_runtime} {unit_name.upper()} {tool_name.upper()}"
-                                    proc = await asyncio.create_subprocess_shell(
-                                        matlab_cmd, cwd=cfg.matlab_func_path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-                                    )
-
-                                    stdout, stderr = await proc.communicate()
-
-                                    if proc.returncode != 0:
-                                        logger.error("Errore durante l'elaborazione")
-                                        logger.error(stderr.decode().strip())
-
-                                        if proc.returncode == 124:
-                                            error_type = f"Matlab elab excessive duration: killed after {cfg.matlab_timeout} seconds."
-                                        else:
-                                            error_type = f"Matlab elab failed: {proc.returncode}."
-
-                                        # da verificare i log dove prenderli
-                                        # with open(f"{cfg.matlab_error_path}{unit_name}{tool_name}_output_error.txt", "w") as f:
-                                        #    f.write(stderr.decode().strip())
-                                        # errors = [line for line in stderr.decode().strip() if line.startswith("Error")]
-                                        # warnings = [line for line in stderr.decode().strip() if not line.startswith("Error")]
-
-                                        errors, warnings = await read_error_lines_from_logs(
-                                            cfg.matlab_error_path, f"_{unit_name}_{tool_name}*_*_output_error.txt"
-                                        )
-                                        await send_error_email(
-                                            unit_name.upper(), tool_name.upper(), tool_elab_info["matcall"], error_type, errors, warnings
-                                        )
-
-                                    else:
-                                        logger.info(stdout.decode().strip())
-                                        await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                                    await unlock(cfg, rec_id, pool)
-                                    await asyncio.sleep(ELAB_PROCESSING_DELAY)
-                                else:
-                                    logger.info(
-                                        "ID %s %s - %s %s: MatLab calc by-passed.", rec_id, unit_name, tool_name, tool_elab_info["statustools"]
-                                    )
-                                    await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                                    await update_status(cfg, rec_id, WorkflowFlags.DUMMY_ELABORATED, pool)
-                                    await unlock(cfg, rec_id, pool)
-                        else:
-                            await update_status(cfg, rec_id, WorkflowFlags.DATA_ELABORATED, pool)
-                            await update_status(cfg, rec_id, WorkflowFlags.DUMMY_ELABORATED, pool)
-                            await unlock(cfg, rec_id, pool)
-
-                    else:
-                        logger.info("Nessun record disponibile")
-                        await asyncio.sleep(NO_RECORD_SLEEP)
-                else:
-                    logger.info("Flag fermo elaborazione attivato")
-                    await asyncio.sleep(NO_RECORD_SLEEP)
-
-            except asyncio.CancelledError:
-                logger.info("Worker cancellato. Uscita in corso...")
-                raise
-
-            except Exception as e:  # pylint: disable=broad-except
-                logger.error("Errore durante l'esecuzione: %s", e, exc_info=debug_mode)
-                await asyncio.sleep(1)
-
-    except asyncio.CancelledError:
-        logger.info("Worker terminato per shutdown graceful")
-    finally:
-        logger.info("Worker terminato")
-
-
-async def main():
-    """Funzione principale che avvia l'elab_orchestrator."""
-    await run_orchestrator(setting.Config, worker)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

vm2/src/ftp_csv_receiver.py

@@ -1,245 +0,0 @@
-#!.venv/bin/python
-"""
-This module implements an FTP/SFTP server with custom commands for
-managing virtual users and handling CSV file uploads.
-
-Server mode is controlled by FTP_MODE environment variable:
-- FTP_MODE=ftp (default): Traditional FTP server
-- FTP_MODE=sftp: SFTP (SSH File Transfer Protocol) server
-"""
-
-import asyncio
-import logging
-import os
-import sys
-from hashlib import sha256
-from logging.handlers import RotatingFileHandler
-from pathlib import Path
-
-from pyftpdlib.handlers import FTPHandler
-from pyftpdlib.servers import FTPServer
-
-from utils.authorizers.database_authorizer import DatabaseAuthorizer
-from utils.config import loader_ftp_csv as setting
-from utils.connect import file_management, user_admin
-
-# Configure logging (moved inside main function)
-
-logger = logging.getLogger(__name__)
-
-
-# Legacy authorizer kept for reference (not used anymore)
-# The DatabaseAuthorizer is now used for real-time database synchronization
-
-
-class ASEHandler(FTPHandler):
-    """Custom FTP handler that extends FTPHandler with custom commands and file handling."""
-
-    # Permetti connessioni dati da indirizzi IP diversi (importante per NAT/proxy)
-    permit_foreign_addresses = True
-
-    def __init__(self: object, conn: object, server: object, ioloop: object = None) -> None:
-        """Initializes the handler, adds custom commands, and sets up command permissions.
-
-        Args:
-            conn (object): The connection object.
-            server (object): The FTP server object.
-            ioloop (object): The I/O loop object.
-        """
-        super().__init__(conn, server, ioloop)
-        self.proto_cmds = FTPHandler.proto_cmds.copy()
-        # Add custom FTP commands for managing virtual users - command in lowercase
-        self.proto_cmds.update(
-            {
-                "SITE ADDU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> ADDU USERNAME PASSWORD (add virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE DISU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> DISU USERNAME (disable virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE ENAU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": True,
-                    "help": "Syntax: SITE <SP> ENAU USERNAME (enable virtual user).",
-                }
-            }
-        )
-        self.proto_cmds.update(
-            {
-                "SITE LSTU": {
-                    "perm": "M",
-                    "auth": True,
-                    "arg": None,
-                    "help": "Syntax: SITE <SP> LSTU (list virtual users).",
-                }
-            }
-        )
-
-    def on_file_received(self: object, file: str) -> None:
-        return file_management.on_file_received(self, file)
-
-    def on_incomplete_file_received(self: object, file: str) -> None:
-        """Removes partially uploaded files.
-        Args:
-            file: The path to the incomplete file.
-        """
-        os.remove(file)
-
-    def ftp_SITE_ADDU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_ADDU(self, line)
-
-    def ftp_SITE_DISU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_DISU(self, line)
-
-    def ftp_SITE_ENAU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_ENAU(self, line)
-
-    def ftp_SITE_LSTU(self: object, line: str) -> None:
-        return user_admin.ftp_SITE_LSTU(self, line)
-
-
-def setup_logging(log_filename: str):
-    """
-    Configura il logging per il server FTP con rotation e output su console.
-
-    Args:
-        log_filename (str): Percorso del file di log.
-    """
-    root_logger = logging.getLogger()
-    formatter = logging.Formatter("%(asctime)s - PID: %(process)d.%(name)s.%(levelname)s: %(message)s")
-
-    # Rimuovi eventuali handler esistenti
-    if root_logger.hasHandlers():
-        root_logger.handlers.clear()
-
-    # Handler per file con rotation (max 10MB per file, mantiene 5 backup)
-    file_handler = RotatingFileHandler(
-        log_filename,
-        maxBytes=10 * 1024 * 1024,  # 10 MB
-        backupCount=5,  # Mantiene 5 file di backup
-        encoding="utf-8"
-    )
-    file_handler.setFormatter(formatter)
-    root_logger.addHandler(file_handler)
-
-    # Handler per console (utile per Docker)
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(formatter)
-    root_logger.addHandler(console_handler)
-
-    root_logger.setLevel(logging.INFO)
-    root_logger.info("Logging FTP configurato con rotation (10MB, 5 backup) e console output")
-
-
-def start_ftp_server(cfg):
-    """Start traditional FTP server."""
-    try:
-        # Initialize the authorizer with database support
-        # This authorizer checks the database on every login, ensuring
-        # all FTP server instances stay synchronized without restarts
-        authorizer = DatabaseAuthorizer(cfg)
-
-        # Initialize handler
-        handler = ASEHandler
-        handler.cfg = cfg
-        handler.authorizer = authorizer
-
-        # Set masquerade address only if configured (importante per HA con VIP)
-        # Questo è l'IP che il server FTP pubblicherà ai client per le connessioni passive
-        if cfg.proxyaddr and cfg.proxyaddr.strip():
-            handler.masquerade_address = cfg.proxyaddr
-            logger.info(f"FTP masquerade address configured: {cfg.proxyaddr}")
-        else:
-            logger.info("FTP masquerade address not configured - using server's default IP")
-
-        # Set the range of passive ports for the FTP server
-        passive_ports_range = list(range(cfg.firstport, cfg.firstport + cfg.portrangewidth))
-        handler.passive_ports = passive_ports_range
-
-        # Log configuration
-        logger.info(f"Starting FTP server on port {cfg.service_port} with DatabaseAuthorizer")
-        logger.info(
-            f"FTP passive ports configured: {cfg.firstport}-{cfg.firstport + cfg.portrangewidth - 1} "
-            f"({len(passive_ports_range)} ports)"
-        )
-        logger.info(f"FTP permit_foreign_addresses: {handler.permit_foreign_addresses}")
-        logger.info(f"Database connection: {cfg.dbuser}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbname}")
-
-        # Create and start the FTP server
-        server = FTPServer(("0.0.0.0", cfg.service_port), handler)
-        server.serve_forever()
-
-    except Exception as e:
-        logger.error("FTP server error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-async def start_sftp_server_async(cfg):
-    """Start SFTP server (async)."""
-    try:
-        from utils.servers.sftp_server import start_sftp_server
-
-        logger.info(f"Starting SFTP server on port {cfg.service_port}")
-        logger.info(f"Database connection: {cfg.dbuser}@{cfg.dbhost}:{cfg.dbport}/{cfg.dbname}")
-
-        # Start SFTP server
-        server = await start_sftp_server(cfg, host="0.0.0.0", port=cfg.service_port)
-
-        # Keep server running
-        await asyncio.Event().wait()
-
-    except ImportError as e:
-        logger.error("SFTP mode requires 'asyncssh' library. Install with: pip install asyncssh")
-        logger.error(f"Error: {e}")
-        sys.exit(1)
-    except Exception as e:
-        logger.error("SFTP server error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-def main():
-    """Main function to start FTP or SFTP server based on FTP_MODE environment variable."""
-    # Load the configuration settings
-    cfg = setting.Config()
-
-    # Configure logging first
-    setup_logging(cfg.logfilename)
-
-    # Get server mode from environment variable (default: ftp)
-    server_mode = os.getenv("FTP_MODE", "ftp").lower()
-
-    if server_mode not in ["ftp", "sftp"]:
-        logger.error(f"Invalid FTP_MODE: {server_mode}. Valid values: ftp, sftp")
-        sys.exit(1)
-
-    logger.info(f"Server mode: {server_mode.upper()}")
-
-    try:
-        if server_mode == "ftp":
-            start_ftp_server(cfg)
-        elif server_mode == "sftp":
-            asyncio.run(start_sftp_server_async(cfg))
-    except KeyboardInterrupt:
-        logger.info("Server stopped by user")
-    except Exception as e:
-        logger.error("Unexpected error: %s", e, exc_info=True)
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

vm2/src/load_ftp_users.py

@@ -1,211 +0,0 @@
-#!.venv/bin/python
-"""
-Script per prelevare dati da MySQL e inviare comandi SITE FTP
-"""
-
-import logging
-import sys
-from ftplib import FTP
-
-import mysql.connector
-
-from utils.config import users_loader as setting
-from utils.database.connection import connetti_db
-
-# Configurazione logging
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-logger = logging.getLogger(__name__)
-
-# Configurazione server FTP
-FTP_CONFIG = {"host": "localhost", "user": "admin", "password": "batt1l0", "port": 2121}
-
-
-def connect_ftp() -> FTP:
-    """
-    Establishes a connection to the FTP server using the predefined configuration.
-    Returns:
-        FTP: An active FTP connection object.
-    """
-    try:
-        ftp = FTP()
-        ftp.connect(FTP_CONFIG["host"], FTP_CONFIG["port"])
-        ftp.login(FTP_CONFIG["user"], FTP_CONFIG["password"])
-        logger.info("Connessione FTP stabilita")
-        return ftp
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore connessione FTP: %s", e)
-        sys.exit(1)
-
-
-def fetch_data_from_db(connection: mysql.connector.MySQLConnection) -> list[tuple]:
-    """
-    Fetches username and password data from the 'ftp_accounts' table in the database.
-
-    Args:
-        connection (mysql.connector.MySQLConnection): The database connection object.
-    Returns:
-        List[Tuple]: A list of tuples, where each tuple contains (username, password).
-    """
-    try:
-        cursor = connection.cursor()
-
-        # Modifica questa query secondo le tue esigenze
-        query = """
-        SELECT username, password
-        FROM ase_lar.ftp_accounts
-        """
-
-        cursor.execute(query)
-        results = cursor.fetchall()
-
-        logger.info("Prelevate %s righe dal database", len(results))
-        return results
-
-    except mysql.connector.Error as e:
-        logger.error("Errore query database: %s", e)
-        return []
-    finally:
-        cursor.close()
-
-
-def fetch_existing_users(connection: mysql.connector.MySQLConnection) -> dict[str, tuple]:
-    """
-    Fetches existing FTP users from virtusers table.
-
-    Args:
-        connection (mysql.connector.MySQLConnection): The database connection object.
-    Returns:
-        dict: Dictionary mapping username to (is_enabled, has_matching_password).
-              is_enabled is True if disabled_at is NULL.
-    """
-    try:
-        cursor = connection.cursor()
-        query = """
-        SELECT ftpuser, disabled_at
-        FROM ase_lar.virtusers
-        """
-        cursor.execute(query)
-        results = cursor.fetchall()
-
-        # Create dictionary: username -> is_enabled
-        users_dict = {username: (disabled_at is None) for username, disabled_at in results}
-
-        logger.info("Trovati %s utenti esistenti in virtusers", len(users_dict))
-        return users_dict
-
-    except mysql.connector.Error as e:
-        logger.error("Errore query database virtusers: %s", e)
-        return {}
-    finally:
-        cursor.close()
-
-
-def send_site_command(ftp: FTP, command: str) -> bool:
-    """
-    Sends a SITE command to the FTP server.
-
-    Args:
-        ftp (FTP): The FTP connection object.
-        command (str): The SITE command string to send (e.g., "ADDU username password").
-    Returns:
-        bool: True if the command was sent successfully, False otherwise.
-    """
-    try:
-        # Il comando SITE viene inviato usando sendcmd
-        response = ftp.sendcmd(f"SITE {command}")
-        logger.info("Comando SITE %s inviato. Risposta: %s", command, response)
-        return True
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore invio comando SITE %s: %s", command, e)
-        return False
-
-
-def main():
-    """
-    Main function to connect to the database, fetch FTP user data, and synchronize users to FTP server.
-    This function is idempotent - it can be run multiple times safely:
-    - If user exists and is enabled: skips
-    - If user exists but is disabled: enables it (SITE ENAU)
-    - If user doesn't exist: creates it (SITE ADDU)
-    """
-    logger.info("Avvio script caricamento utenti FTP (idempotente)")
-    cfg = setting.Config()
-
-    # Connessioni
-    db_connection = connetti_db(cfg)
-    ftp_connection = connect_ftp()
-
-    try:
-        # Preleva utenti da sincronizzare
-        users_to_sync = fetch_data_from_db(db_connection)
-
-        if not users_to_sync:
-            logger.warning("Nessun utente da sincronizzare nel database ftp_accounts")
-            return
-
-        # Preleva utenti già esistenti
-        existing_users = fetch_existing_users(db_connection)
-
-        added_count = 0
-        enabled_count = 0
-        skipped_count = 0
-        error_count = 0
-
-        # Processa ogni utente
-        for row in users_to_sync:
-            username, password = row
-
-            if username in existing_users:
-                is_enabled = existing_users[username]
-
-                if is_enabled:
-                    # Utente già esiste ed è abilitato - skip
-                    logger.info("Utente %s già esiste ed è abilitato - skip", username)
-                    skipped_count += 1
-                else:
-                    # Utente esiste ma è disabilitato - riabilita
-                    logger.info("Utente %s esiste ma è disabilitato - riabilito con SITE ENAU", username)
-                    ftp_site_command = f"enau {username}"
-
-                    if send_site_command(ftp_connection, ftp_site_command):
-                        enabled_count += 1
-                    else:
-                        error_count += 1
-            else:
-                # Utente non esiste - crea
-                logger.info("Utente %s non esiste - creazione con SITE ADDU", username)
-                ftp_site_command = f"addu {username} {password}"
-
-                if send_site_command(ftp_connection, ftp_site_command):
-                    added_count += 1
-                else:
-                    error_count += 1
-
-        logger.info(
-            "Elaborazione completata. Aggiunti: %s, Riabilitati: %s, Saltati: %s, Errori: %s",
-            added_count,
-            enabled_count,
-            skipped_count,
-            error_count
-        )
-
-    except Exception as e:  # pylint: disable=broad-except
-        logger.error("Errore generale: %s", e)
-
-    finally:
-        # Chiudi connessioni
-        try:
-            ftp_connection.quit()
-            logger.info("Connessione FTP chiusa")
-        except Exception as e:  # pylint: disable=broad-except
-            logger.error("Errore chiusura connessione FTP: %s", e)
-
-        try:
-            db_connection.close()
-            logger.info("Connessione MySQL chiusa")
-        except Exception as e:  # pylint: disable=broad-except
-            logger.error("Errore chiusura connessione MySQL: %s", e)
-
-
-if __name__ == "__main__":
-    main()

vm2/src/load_orchestrator.py

@@ -1,166 +0,0 @@
-#!.venv/bin/python
-"""
-Orchestratore dei worker che caricano i dati su dataraw
-"""
-
-# Import necessary libraries
-import asyncio
-import importlib
-import logging
-
-# Import custom modules for configuration and database connection
-from utils.config import loader_load_data as setting
-from utils.csv.loaders import get_next_csv_atomic
-from utils.database import WorkflowFlags
-from utils.orchestrator_utils import run_orchestrator, shutdown_event, worker_context
-
-# Initialize the logger for this module
-logger = logging.getLogger()
-
-# Delay tra un processamento CSV e il successivo (in secondi)
-CSV_PROCESSING_DELAY = 0.2
-# Tempo di attesa se non ci sono record da elaborare
-NO_RECORD_SLEEP = 60
-
-# Module import cache to avoid repeated imports (performance optimization)
-_module_cache = {}
-
-
-async def worker(worker_id: int, cfg: dict, pool: object) -> None:
-    """Esegue il ciclo di lavoro per l'elaborazione dei file CSV.
-
-    Il worker preleva un record CSV dal database, ne elabora il contenuto
-    e attende prima di iniziare un nuovo ciclo.
-
-    Supporta graceful shutdown controllando il shutdown_event tra le iterazioni.
-
-    Args:
-        worker_id (int): L'ID univoco del worker.
-        cfg (dict): L'oggetto di configurazione.
-        pool (object): Il pool di connessioni al database.
-    """
-    # Imposta il context per questo worker
-    worker_context.set(f"W{worker_id:02d}")
-
-    logger.info("Avviato")
-
-    try:
-        while not shutdown_event.is_set():
-            try:
-                logger.info("Inizio elaborazione")
-                record = await get_next_csv_atomic(
-                    pool,
-                    cfg.dbrectable,
-                    WorkflowFlags.CSV_RECEIVED,
-                    WorkflowFlags.DATA_LOADED,
-                )
-
-                if record:
-                    success = await load_csv(record, cfg, pool)
-                    if not success:
-                        logger.error("Errore durante l'elaborazione")
-                    await asyncio.sleep(CSV_PROCESSING_DELAY)
-                else:
-                    logger.info("Nessun record disponibile")
-                    await asyncio.sleep(NO_RECORD_SLEEP)
-
-            except asyncio.CancelledError:
-                logger.info("Worker cancellato. Uscita in corso...")
-                raise
-
-            except Exception as e:  # pylint: disable=broad-except
-                logger.error("Errore durante l'esecuzione: %s", e, exc_info=1)
-                await asyncio.sleep(1)
-
-    except asyncio.CancelledError:
-        logger.info("Worker terminato per shutdown graceful")
-    finally:
-        logger.info("Worker terminato")
-
-
-async def load_csv(record: tuple, cfg: object, pool: object) -> bool:
-    """Carica ed elabora un record CSV utilizzando il modulo di parsing appropriato.
-
-    Args:
-        record: Una tupla contenente i dettagli del record CSV da elaborare
-                (rec_id, unit_type, tool_type, unit_name, tool_name).
-        cfg: L'oggetto di configurazione contenente i parametri del sistema.
-        pool (object): Il pool di connessioni al database.
-
-    Returns:
-        True se l'elaborazione del CSV è avvenuta con successo, False altrimenti.
-    """
-
-    debug_mode = logging.getLogger().getEffectiveLevel() == logging.DEBUG
-    logger.debug("Inizio ricerca nuovo CSV da elaborare")
-
-    rec_id, unit_type, tool_type, unit_name, tool_name = [x.lower().replace(" ", "_") if isinstance(x, str) else x for x in record]
-    logger.info(
-        "Trovato CSV da elaborare: ID=%s, Tipo=%s_%s, Nome=%s_%s",
-        rec_id,
-        unit_type,
-        tool_type,
-        unit_name,
-        tool_name,
-    )
-
-    # Costruisce il nome del modulo da caricare dinamicamente
-    module_names = [
-        f"utils.parsers.by_name.{unit_name}_{tool_name}",
-        f"utils.parsers.by_name.{unit_name}_{tool_type}",
-        f"utils.parsers.by_name.{unit_name}_all",
-        f"utils.parsers.by_type.{unit_type}_{tool_type}",
-    ]
-
-    # Try to get from cache first (performance optimization)
-    modulo = None
-    cache_key = None
-
-    for module_name in module_names:
-        if module_name in _module_cache:
-            # Cache hit! Use cached module
-            modulo = _module_cache[module_name]
-            cache_key = module_name
-            logger.info("Modulo caricato dalla cache: %s", module_name)
-            break
-
-    # If not in cache, import dynamically
-    if not modulo:
-        for module_name in module_names:
-            try:
-                logger.debug("Caricamento dinamico del modulo: %s", module_name)
-                modulo = importlib.import_module(module_name)
-                # Store in cache for future use
-                _module_cache[module_name] = modulo
-                cache_key = module_name
-                logger.info("Modulo caricato per la prima volta: %s", module_name)
-                break
-            except (ImportError, AttributeError) as e:
-                logger.debug(
-                    "Modulo %s non presente o non valido. %s",
-                    module_name,
-                    e,
-                    exc_info=debug_mode,
-                )
-
-    if not modulo:
-        logger.error("Nessun modulo trovato %s", module_names)
-        return False
-
-    # Ottiene la funzione 'main_loader' dal modulo
-    funzione = modulo.main_loader
-
-    # Esegui la funzione
-    logger.info("Elaborazione con modulo %s per ID=%s", modulo, rec_id)
-    await funzione(cfg, rec_id, pool)
-    logger.info("Elaborazione completata per ID=%s", rec_id)
-    return True
-
-
-async def main():
-    """Funzione principale che avvia il load_orchestrator."""
-    await run_orchestrator(setting.Config, worker)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())