Files
ase_perl/LoadCSVData.pl
2021-12-08 18:20:08 +01:00

711 lines
24 KiB
Perl
Executable File

#!/usr/bin/perl
use warnings;
use strict;
use DBI;
use Getopt::Long;
use Net::FTP;
use File::Basename qw( fileparse );
use List::Util qw( max );
use Data::Dumper qw(Dumper);
use Cwd;
use MIME::Lite;
$|++; # Autoflush
my ( $filename, $sqlout );
my $username = 'ase_lar';
my $password = 'laravel';
my $db_lar = 'ase_lar';
my $db_sp = 'ase_lar';
my $server = 'localhost';
my $db_name = "ase_lar";
my $table = "RAWDATACOR";
my ( @CSVData, @sql );
my $outcount = 0;
my $outfilenum = 0;
my $maxsqllines = 10000;
my (
$tool, $toolbatt, $tooltemp, $tooltype, @channels,
@NewData, @ain, @din, @nodetype, $GDEventDate
);
my $matlab_proxy_ssh = "localhost"; #"80.211.107.201";
my $matlab_proxy_ssh_port = 22; #2222;
my $matlab_cmd = "";
my $matlab_timestamp = "";
my $ftp_send = 1;
my $ftp_addrs;
my $ftp_user;
my $ftp_passwd;
my $ftp_parm;
my $ftp_filename;
my $ftp_target;
my $duedate = '';
my ( $unit_id, $unit, $unittype );
my ( $fileDate, $fileTime );
my $matlab_rt = "/usr/local/MATLAB/MATLAB_Runtime/v93";
my $matlab_func_dir = "/usr/local/matlab_func/";
my $matlab_error = '';
my $matlab_timeout = '1800';
my $email_addr =
'andrea.carri@aseltd.eu,alessandro.battilani@gmail.com,alessandro.valletta@aseltd.eu';
my $from = 'ASE Alert System<alert@aseltd.eu>';
my ( $email_msg, $email_obj );
my $G201_fcheck = 0;
my $D2W_fcheck = 0;
my $G301_fcheck = 0;
my $FtpToCustomerCmd = 'SendFtpElabData.pl';
my ( $scriptname, $scriptpath );
my $MatlabErrorFilename = "/tmp/";
my @matlabOutputErrorArray;
my @errors;
my @warnings;
my $MatlabErrors = '';
my $MatlabWarnings = '';
my $tool_status = '';
sub uniq {
my %seen;
grep !$seen{$_}++, @_;
}
sub getTimeStamp
{ # parm [ts] => timestamp for filename; log => timestamp for log
my $format = "%04d%02d%02d%02d%02d%02d";
my ($p1) = @_;
if ( defined $p1 and $p1 eq "log" ) {
$format = "%04d%02d%02d %02d:%02d:%02d";
}
if ( defined $p1 and $p1 eq "db_ts" ) {
$format = "%04d-%02d-%02d %02d:%02d:%02d";
}
my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
localtime(time);
my $timestamp =
sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec );
return $timestamp;
}
sub writeOutSql {
my ($tn) = @_;
my $outfile = $sqlout . "_" . $outfilenum++;
open SQLOUT, ">", $outfile
or die( getTimeStamp("log")
. " - pid $$ >> Error: opening output file "
. $outfile
. ":$!\n" );
print getTimeStamp("log") . " - pid $$ >> file $outfile created\n";
print SQLOUT "/* lock "
. $db_name
. ".$table table */\nLOCK TABLES "
. $db_name
. ".$table WRITE;\n";
print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n";
print SQLOUT
"(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n";
print SQLOUT
"`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n";
print SQLOUT "VALUES\n";
print SQLOUT @sql;
@sql = ();
print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n";
close SQLOUT
or die( getTimeStamp("log")
. " - pid $$ >> Error in closing file "
. $outfile
. "\n" );
my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile );
system(@args) == 0
or die( getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n" );
print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n";
unlink $outfile;
return;
}
sub matlabCalc {
$matlab_timestamp = getTimeStamp("db_ts");
print getTimeStamp("log")
. " - pid $$ >> $unit - $tool MatLab calc started...\n";
if ( $matlab_cmd =~ /_lnx$/ ) {
$matlab_cmd = './run_' . $matlab_cmd . '.sh';
$MatlabErrorFilename .= "${unit}${tool}_output_error.txt";
#print $ENV{"HOME"} . "\n";
$ENV{"HOME"} = "/" . $ENV{"HOME"};
my $cmd_elab =
"timeout $matlab_timeout $matlab_cmd $matlab_rt $unit $tool";
#my $args = "sh -c \'cd $matlab_func_dir; pwd; $cmd_elab\'";
my $args =
"ssh $matlab_proxy_ssh -p $matlab_proxy_ssh_port \"sh -c \'cd $matlab_func_dir; $cmd_elab\'\"";
if ( system($args) != 0 ) {
open( my $fh, '<', $MatlabErrorFilename )
or warn print getTimeStamp("log")
. " - pid $$ >> Cannot open Matlab output error file: ${MatlabErrorFilename}\n";
{
chomp( @matlabOutputErrorArray = <$fh> );
}
close($fh);
@matlabOutputErrorArray = grep( /\S/, @matlabOutputErrorArray );
@errors = grep( /^Error/, @matlabOutputErrorArray );
@warnings = grep( !/^Error/, @matlabOutputErrorArray );
$MatlabErrors = join( "<br/>", @errors );
$MatlabWarnings = join( "<br/>", uniq(@warnings) );
my $exit_value = $? >> 8;
if ( $exit_value == 124 ) {
print getTimeStamp("log")
. " - pid $$ >> system $args excessive duration: killed after $matlab_timeout seconds\n";
$matlab_error =
"Matlab elab excessive duration: killed after $matlab_timeout seconds";
}
else {
print getTimeStamp("log")
. " - pid $$ >> system $args failed: return code $exit_value - $?\n";
$matlab_error = "Matlab elab failed: $exit_value - $?";
}
makeEmailMsg();
emailSend();
print getTimeStamp("log") . " - pid $$ >> Execution ended.\n";
exit(1);
}
}
print getTimeStamp("log")
. " - pid $$ >> $unit - $tool MatLab calc executed.\n";
return;
}
sub trxelab {
my $user = getpwuid($>);
my $ftpcmd =
"$scriptpath$FtpToCustomerCmd -m \"$matlab_timestamp\" -u $unit -t $tool >> /home/$user/log/loadcsvdata.log 2>&1";
print getTimeStamp("log")
. " - pid $$ >> Fork FTP command to set GID mysql: $ftpcmd.\n";
unless ( fork() ) {
exec( '/usr/bin/sg', 'mysql', "$ftpcmd" );
exit(0);
}
}
sub getNodesType {
my $dbh =
DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare(
'select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.ain as ain, n.din as din
from nodes as n
inner join tools as t on t.id = n.tool_id
inner join units as u on u.id = t.unit_id
inner join nodetypes as y on n.nodetype_id = y.id
where y.type not in (\'Anchor Link\', \'None\') and t.name = "'
. $tool
. '" and u.name = "'
. $unit
. '" order by 3;'
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute();
print getTimeStamp("log")
. " - pid $$ >> $unit - $tool: "
. $sth->rows
. " rows selected to get node type/Ain/Din/channels.\n";
if ( $sth->rows eq 0 ) {
print getTimeStamp("log")
. " - pid $$ >> Node/Channels/Ain/Din not defined.\n";
print getTimeStamp("log") . " - pid $$ >> Execution ended.\n";
exit;
}
else {
my $row = 0;
while ( my $results = $sth->fetchrow_hashref ) {
$channels[$row] = $results->{'channels'};
$nodetype[$row] = $results->{'type'};
$ain[$row] = $results->{'ain'};
$din[ $row++ ] = $results->{'din'};
}
}
$sth->finish;
# Disconnect
$dbh->disconnect;
return;
}
sub getUdbPwd {
my $dbh =
DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare(
"SELECT db_user, db_pwd FROM ase_lar.companies where db_name = '"
. $db_name
. "';" )
or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute();
if ( $sth->rows eq 0 ) {
die getTimeStamp("log") . " - pid $$ >> No password selected.\n";
}
else {
my $results = $sth->fetchrow_hashref;
$password = $results->{'db_pwd'};
$username = $results->{'db_user'};
}
$sth->finish;
# Disconnect
$dbh->disconnect;
return;
}
sub getMatlabCmd {
my $dbh =
DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare(
'select m.matcall, t.ftp_send , t.unit_id, s.`desc` as statustools from matfuncs as m
inner join tools as t on t.matfunc = m.id
inner join units as u on u.id = t.unit_id
inner join statustools as s on t.statustool_id = s.id
where t.name = "' . $tool . '" and u.name = "' . $unit . '";'
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute();
if ( $sth->rows eq 0 ) {
die getTimeStamp("log")
. " - pid $$ >> No tool's matlab function selected.\n";
}
else {
while ( my $results = $sth->fetchrow_hashref ) {
$matlab_cmd = $results->{'matcall'};
$ftp_send = $results->{'ftp_send'};
$unit_id = $results->{'unit_id'};
$tool_status = $results->{'statustools'};
}
}
$sth->finish;
# Disconnect
$dbh->disconnect;
return;
}
sub makeEmailMsg {
$email_msg = <<"END_MSG";
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>Alert from ASE</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
</head>
<body style="margin: 0; padding: 0;">
<table bgcolor="#ffffff" border="0" cellpadding="0" cellspacing="0" width="100%">
<tr>
<td align="center">
<img src="https://www2.aseltd.eu/static/img/logo_ASE_small.png" alt="ASE" style="display: block;" />
</td>
</tr>
<tr>
<td align="center">
<h1 style="margin: 5px;">Alert from ASE:</h1>
</td>
</tr>
<tr>
<td align="center">
<h3 style="margin: 5px;">Matlab function $matlab_cmd failed on unit => $unit - tool => $tool</h3>
</td>
</tr>
<tr>
<td align="center">
<h4 style="margin: 5px;">$matlab_error</h4>
</td>
</tr>
<tr>
<td style="padding: 20px; padding-bottom: 0px; color: red">
$MatlabErrors
</td>
</tr>
<tr>
<td style="padding: 20px;">
$MatlabWarnings
</td>
</tr>
</table>
</body>
</html>
END_MSG
}
sub emailSend {
$email_obj = 'Matlab error on unit ' . $unit . ' - tool ' . $tool;
my $msg = MIME::Lite->new(
From => $from,
To => $email_addr,
Subject => $email_obj,
Data => $email_msg
);
$msg->attr( "content-type" => "text/html" );
if (
$msg->send(
'smtp', "smtps.aruba.it",
AuthUser => "alert\@aseltd.eu",
AuthPass => "Ase#2013!20\@bat",
Port => 465,
SSL => 1,
Debug => 0
)
)
{
print getTimeStamp("log") . " - pid $$ >> $email_obj\n";
print getTimeStamp("log") . " - pid $$ >> Mail sent to: $email_addr.\n";
}
else {
print getTimeStamp("log")
. " - pid $$ >> Error in sending mail to: $email_addr.\n";
}
}
my $starttime = getTimeStamp("log");
print "$starttime - pid $$ >> Start execution.\n";
GetOptions(
"filename=s" => \$filename,
"sqlout=s" => \$sqlout,
"dbname=s" => \$db_name
) or die("Error in command line arguments\n");
open FILE, $filename
or die( "Error: opening input file " . $filename . "\n" );
( $scriptname, $scriptpath ) = fileparse($0);
if ( $db_name ne 'asega' ) {
$db_name =~ s/aseu/dbn/;
$db_sp = $db_name;
getUdbPwd();
$db_lar = $db_name;
}
else {
$db_name = 'ase_lar';
}
my ( $filecsvname, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ );
if ( $filecsvname =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)_\d*$/i ) {
( $fileDate, $fileTime ) = split( /\s/, <FILE> );
( $unittype, $unit ) = split( /\s/, uc <FILE> );
$unit =~ s/;+$//;
}
elsif ( $filecsvname =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*_\d*$/i ) {
my @strings = $filecsvname =~
/(.{1,4})_(.{1,6})_(.{1,6})_(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2}).*/;
$unittype = $strings[0];
$unit = $strings[1];
$tool = $strings[2];
$fileDate = $strings[3] . "/" . $strings[4] . "/" . $strings[5];
$fileTime = $strings[6] . ":" . $strings[7] . ":" . $strings[8];
$tooltype = 'MUX';
}
elsif ( $filecsvname =~ m/^D2W_ID\d\d\d\d_DT\d\d\d\d_\d*$/i ) {
my @strings = $filecsvname =~ /(.{1,3})_(.{1,6})_(.{1,6}).*/;
$unittype = $strings[0];
$unit = $strings[1];
$tool = $strings[2];
$tooltype = 'D2W';
}
elsif ( $filecsvname =~ m/^CR1000X_ID\d\d\d\d_DT\d\d\d\d_\d*$/i ) {
my @strings = $filecsvname =~ /(.{1,7})_(.{1,6})_(.{1,6}).*/;
$unittype = $strings[0];
$unit = $strings[1];
$tool = $strings[2];
$tooltype = 'CR1000X';
}
elsif ( $filecsvname =~ m/^(\d*_|)(G301_ID\d\d\d\d_DT\d\d\d\d_\d*)$/i ) {
my $filecsvname_clean =
substr( $filecsvname, index( $filecsvname, "G301" ) );
my @strings = $filecsvname_clean =~ /(.{1,4})_(.{1,6})_(.{1,6}).*/;
$unittype = $strings[0];
$unit = $strings[1];
$tool = $strings[2];
$tooltype = 'G301';
}
while ( my $line = <FILE> ) {
if ( $line =~ m/\A [[:ascii:]]* \Z/xms ) {
$line =~ tr /\,/\;/ if ( $line =~ m/^\"\d\d\d\d.*/i ); #CR1000X
$line =~ s/\"//g;
my ( $first, $NodeData ) = split( /;/, $line, 2 );
if ( defined $first and ( $first ne '' ) ) {
$first =~ s/;+$//;
$first =~ tr /\-/\//;
if ( !defined $NodeData or ( $NodeData =~ m/^;+/ ) ) {
my @info = ( split( /[\/,\.]/, $first ) );
if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) { #G801
$tool = uc $info[3];
$tooltype = uc $info[2];
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
}
elsif ( defined $info[5] && $info[5] =~ m/^DT\d\d\d\d$/i )
{ #G802
$tool = uc $info[5];
$tooltype = uc $info[4];
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
}
elsif ( defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) { #G801
$tool = uc $info[2];
$tooltype = 'LOC';
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
}
elsif ( defined $info[4] && $info[4] =~ m/^LOC\d*$/i ) { #G802
$tool = uc $info[4];
$tooltype = 'LOC';
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
}
elsif ( defined $info[3] && $info[3] =~ m/^GD\d*$/i ) {
$tool = uc $info[3];
$tooltype = 'GD';
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
}
}
else {
$NodeData =~ s/\x0d{0,1}\x0a\Z//s;
$NodeData =~ s/;+$//;
#$NodeData =~ s/\r\n\z//;
if ( $unittype eq 'G201' and $G201_fcheck eq 0 ) {
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
$G201_fcheck = 1;
}
if ( $unittype eq 'D2W' and $D2W_fcheck eq 0 ) {
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
$D2W_fcheck = 1;
}
if ( $unittype eq 'CR1000X' and $CR1000X_fcheck eq 0 ) {
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
$CR1000X_fcheck = 1;
}
if ( $unittype eq 'G301' and $G301_fcheck eq 0 ) {
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
$G301_fcheck = 1;
}
if ( $tooltype eq "MUX"
or $tooltype eq "D2W"
or $tooltype eq "G301" )
{
#$NodeData =~ s/Dis\./N\/A/g;
@NewData = grep /\S/, split( /\||;/, $NodeData );
if ( scalar(@NewData) != 0 ) {
my $idx = 0;
$CSVData[ $idx++ ] =
join( ';', splice( @NewData, 0, 2 ) );
foreach my $ch (@channels) {
$CSVData[ $idx++ ] =
join( ';', splice( @NewData, 0, $ch ) );
}
}
#print Dumper \@CSVData;
#print Dumper \@channels;
#print Dumper \@din;
}
elsif ( $tooltype eq "CR1000X" )
{ # può essere tolto per attivarlo
@CSVData = split( /\|/, $NodeData );
print getTimeStamp("log")
. " - pid $$ >> TEST CR1000X Execution ended.\n";
exit;
}
elsif ( $tooltype eq "MUSA" ) {
#$NodeData =~ s/Dis\./N\/A/g;
my @tmpData = grep /\S/, split( /\||;/, $NodeData );
my $tmpBatt = shift @tmpData;
@NewData = ( $tmpBatt, $tmpData[0], @tmpData );
my $idx = 0;
$CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) );
foreach my $ch (@channels) {
$CSVData[ $idx++ ] =
join( ';', splice( @NewData, 0, $ch ) );
}
#print Dumper \@CSVData;
#print Dumper \@channels;
#print Dumper \@din;
}
elsif ( $tooltype eq "LOC" ) {
my ( $Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2 ) =
split( /;/, $NodeData );
my $idx = 0;
$CSVData[ $idx++ ] = $Vbat . ";" . $Tmod;
my $used_din = 0;
my $used_ain = 0;
for my $i ( 0 .. $#din ) {
if ( $din[$i] eq 1 ) {
if ( $used_din eq 0 ) {
$CSVData[ $idx++ ] = $Din1;
$used_din = 1;
}
else {
$CSVData[ $idx++ ] = $Din2;
}
}
elsif ( $ain[$i] eq 1 ) {
if ( $used_ain eq 0 ) {
$CSVData[ $idx++ ] = $Ain1;
$used_ain = 1;
}
else {
$CSVData[ $idx++ ] = $Ain2;
}
}
elsif ( $din[$i] eq 2 ) {
$CSVData[ $idx++ ] = $Din1 . ";" . $Din2;
}
elsif ( $ain[$i] eq 2 ) {
$CSVData[ $idx++ ] = $Ain1 . ";" . $Ain2;
}
else {
$CSVData[ $idx++ ] = "N/A";
}
}
}
elsif ( $tooltype eq "GD" ) {
my ( $div, $Vbat, $Tmod ) =
split( /;/, $NodeData );
$CSVData[0] = $Vbat . ";" . $Tmod;
my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first );
my ( $h, $min, $sec ) = split( /:/, $t );
$GDEventDate = $y . '-' . $m . '-' . $d;
$CSVData[1] = "DT"
. substr( $tool, 2, 4 ) . ';'
. $h . ':'
. ( $min - 1 ) . ';'
. $h . ':'
. ( $min + 1 );
}
else {
@CSVData = split( /\|/, $NodeData );
}
my $nodenum = 0;
foreach (@CSVData) {
$sql[ ( $outcount % $maxsqllines ) ] = "";
my (@data) = grep { /\S/ } split( /\;/, $_ );
if ( $nodenum eq 0 ) {
$toolbatt = $data[0];
$tooltemp = $data[1];
}
else {
my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first );
if ( length($y) != 4 )
{ # G201/G301 con la data invertita
my $w = $y;
$y = $d;
$m = sprintf( '%02d', $m );
$d = sprintf( '%02d', $w );
}
foreach (@data) {
if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) {
$sql[ ( $outcount % $maxsqllines ) ] =
"('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'";
}
$sql[ ( $outcount % $maxsqllines ) ] .=
",'" . $_ . "'";
}
my $InsCompl = ",NULL" x ( 15 - $#data );
$sql[ ( $outcount % $maxsqllines ) ] .= $InsCompl . ")";
if ( ++$outcount % $maxsqllines eq 0 ) {
$sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n";
writeOutSql($tool);
}
else {
$sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n";
}
}
$nodenum++;
}
}
}
}
}
close FILE;
$sql[ ( $outcount % $maxsqllines ) - 1 ] =~ s/,$/;/g;
writeOutSql($tool);
if ( $tooltype eq "GD" ) {
print getTimeStamp("log") . " - pid $$ >> tool GD: nothing to do.\n";
}
else {
getMatlabCmd();
print getTimeStamp("log")
. " - pid $$ >> $unit - $tool - Status $tool_status.\n";
if ( $tool_status eq 'Monitoring Completed' ) {
print getTimeStamp("log")
. " - pid $$ >> $unit - $tool - Monitoring completed: MatLab calc by-passed.\n";
}
else {
matlabCalc();
if ($ftp_send) {
if ( $tool eq 'DT0076' ) {
sleep(600);
}
trxelab();
}
}
}
print getTimeStamp("log") . " - pid $$ >> Execution ended.\n";
exit;