#!/usr/bin/perl use warnings; use strict; use DBI; use Getopt::Long; use Net::FTP; use File::Basename qw( fileparse ); use List::Util qw( max ); use Data::Dumper qw(Dumper); use Cwd; use Mail::Sender; $|++; # Autoflush my ( $filename, $sqlout ); my $username = 'ase_lar'; my $password = 'laravel'; my $db_lar = 'ase_lar'; my $db_sp = 'ase_lar'; my $server = 'localhost'; my $db_name = "ase_lar"; my $table = "RAWDATACOR"; my ( @CSVData, @sql ); my $outcount = 0; my $outfilenum = 0; my $maxsqllines = 10000; my ( $tool, $toolbatt, $tooltemp, $tooltype, @channels, @NewData, @ain, @din, @nodetype, $GDEventDate ); my $matlab_ip = "160.78.21.71"; my $matlab_cmd = "matlab.cmd"; my $matlab_timestamp = ""; my $ftp_send = 1; my $ftp_addrs; my $ftp_user; my $ftp_passwd; my $ftp_parm; my $ftp_filename; my $ftp_target; my $duedate = ''; my ( $unit_id, $unit, $unittype ); my ( $fileDate, $fileTime ); my $matlab_rt = "/usr/local/MATLAB/MATLAB_Runtime/v93"; my $matlab_func_dir = "/usr/local/matlab_func/"; my $matlab_error = ''; my $matlab_timeout = '1800'; my $email_addr = 'andrea.carri@aseltd.eu,edoardo.cavalca@aseltd.eu,alessandro.battilani@gmail.com,alessandro.valletta@aseltd.eu'; my ( $email_msg, $email_obj ); my $G201_fcheck = 0; sub getTimeStamp { # parm [ts] => timestamp for filename; log => timestamp for log my $format = "%04d%02d%02d%02d%02d%02d"; my ($p1) = @_; if ( defined $p1 and $p1 eq "log" ) { $format = "%04d%02d%02d %02d:%02d:%02d"; } if ( defined $p1 and $p1 eq "db_ts" ) { $format = "%04d-%02d-%02d %02d:%02d:%02d"; } my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = localtime(time); my $timestamp = sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec ); return $timestamp; } sub writeOutSql { my ($tn) = @_; my $outfile = $sqlout . "_" . $outfilenum++; open SQLOUT, ">", $outfile or die( getTimeStamp("log") . " - pid $$ >> Error: opening output file " . $outfile . ":$!\n" ); print getTimeStamp("log") . " - pid $$ >> file $outfile created\n"; print SQLOUT "/* lock " . $db_name . ".$table table */\nLOCK TABLES " . $db_name . ".$table WRITE;\n"; print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n"; print SQLOUT "(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n"; print SQLOUT "`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n"; print SQLOUT "VALUES\n"; print SQLOUT @sql; @sql = (); print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n"; close SQLOUT or die( getTimeStamp("log") . " - pid $$ >> Error in closing file " . $outfile . "\n" ); my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile ); system(@args) == 0 or die( getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n" ); print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n"; unlink $outfile; return; } sub matlabCalc { getMatlabCmd(); $matlab_timestamp = getTimeStamp("db_ts"); print getTimeStamp("log") . " - pid $$ >> $unit - $tool MatLab calc started...\n"; if ( $matlab_cmd =~ /_lnx$/ ) { $matlab_cmd = './run_' . $matlab_cmd . '.sh'; #print $ENV{"HOME"} . "\n"; $ENV{"HOME"} = "/" . $ENV{"HOME"}; chdir($matlab_func_dir) or die "cannot change: $!\n"; print getTimeStamp("log") . " - pid $$ >> current dir: $matlab_func_dir\n"; my @args = ( 'timeout', $matlab_timeout, $matlab_cmd, $matlab_rt, $unit, $tool ); if ( system(@args) != 0 ) { my $exit_value = $? >> 8; if ( $exit_value == 124 ) { print getTimeStamp("log") . " - pid $$ >> system @args excessive duration: killed after $matlab_timeout seconds\n"; $matlab_error = "Maltab elab excessive duration: killed after $matlab_timeout seconds"; } else { print getTimeStamp("log") . " - pid $$ >> system @args failed: return code $exit_value - $?\n"; $matlab_error = "Maltab elab failed: $exit_value - $?"; } makeEmailMsg(); emailSend(); print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; exit(1); } } else { $matlab_cmd = 'matlab_' . uc $matlab_cmd . '.cmd'; my @out_matlab = `ssh1 $matlab_ip 'cmd /c $matlab_cmd ' $tool`; if ( index( $out_matlab[-1], "Failure" ) != -1 ) { print getTimeStamp("log") . " - pid $$ >> $unit - $tool MatLab calc failed.\n"; makeEmailMsg(); emailSend(); print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; exit(1); } } print getTimeStamp("log") . " - pid $$ >> $unit - $tool MatLab calc executed.\n"; return; } sub trxelab { my $dbh = DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) or die getTimeStamp("log") . " - pid $$ >> Could not connect to database: $DBI::errstr"; my $sth = $dbh->prepare( 'select ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename, ftp_target, duedate from units where id = ' . $unit_id . ';' ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; $sth->execute(); if ( $sth->rows eq 0 ) { die getTimeStamp("log") . " - pid $$ >> No unit ftp params selected.\n"; } else { while ( my $results = $sth->fetchrow_hashref ) { $ftp_addrs = $results->{'ftp_addrs'}; $ftp_user = $results->{'ftp_user'}; $ftp_passwd = $results->{'ftp_passwd'}; $ftp_parm = $results->{'ftp_parm'}; $ftp_filename = $results->{'ftp_filename'}; $ftp_target = $results->{'ftp_target'}; $duedate = $results->{'duedate'}; } } $sth->finish; # Disconnect my $fileelab; if ( $ftp_filename eq "" ) { $fileelab = '/var/lib/mysql-files/' . $tool . '_' . getTimeStamp() . '.csv'; #mettere quello che si prende da query *** aggiungere $unit } else { $ftp_filename =~ s/(\$\w+)/$1/eeg; $ftp_filename =~ s/\s/_/g; $fileelab = '/var/lib/mysql-files/' . $ftp_filename . '_' . getTimeStamp() . '.csv'; } my $sthdo = $dbh->do( qq{ select * from ( select 'ToolNameID', 'EventDate', 'EventTime', 'NodeNum', 'NodeType', 'NodeDepth', 'XShift', 'YShift', 'ZShift' , 'X', 'Y', 'Z', 'HShift', 'HShiftDir', 'HShift_local', 'speed', 'speed_local', 'acceleration', 'acceleration_local', 'T_node', 'water_level', 'pressure', 'load_value', 'AlfaX', 'AlfaY', 'CalcErr' union all select ToolNameID, EventDate, EventTime, NodeNum, NodeType, NodeDepth, XShift, YShift, ZShift , X, Y, Z, HShift, HShiftDir, HShift_local, speed, speed_local, acceleration, acceleration_local, T_node, water_level, pressure, load_value, AlfaX, AlfaY, calcerr from elabdataview where UnitName = '$unit' and ToolNameID = '$tool' and updated_at > '$matlab_timestamp' order by ToolNameID DESC, EventDate, EventTime, convert(`NodeNum`, UNSIGNED INTEGER) DESC ) resulting_set into outfile '$fileelab' fields terminated by ',' optionally enclosed by '"' lines terminated by '\n'} ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; $dbh->disconnect; $ftp_parm //= ""; $ftp_parm =~ s/\s//g; my %ftp_parms = ( 'Host' => $ftp_addrs, ); $ftp_parms{'Debug'} = 0; $ftp_parms{'Passive'} = 1; foreach my $parm ( split( /,/, $ftp_parm ) ) { my ( $key, $val ) = split( /=>/, $parm ); $ftp_parms{$key} = $val; } print getTimeStamp("log") . " - pid $$ >> $unit - $tool ftp parm:\n" . Dumper(%ftp_parms); if ( !defined $duedate or ( $duedate eq '0000-00-00 00:00:00' ) or ( $duedate eq '' ) or ( $duedate gt getTimeStamp("db_ts") ) ) { my $ftp = Net::FTP->new(%ftp_parms) or die getTimeStamp("log") . " - pid $$ >> Cannot connect to $ftp_addrs: $@ \n"; $ftp->starttls() if exists $ftp_parms{'SSL_version'}; $ftp->login( $ftp_user, $ftp_passwd ) or die getTimeStamp("log") . " - pid $$ >> Cannot login " . $ftp->message . "\n"; $ftp_target //= ""; if ( $ftp_target ne "" ) { $ftp->cwd($ftp_target) or die getTimeStamp("log") . " - pid $$ >> Change remote dir failed " . $ftp->message . "\n"; } print getTimeStamp("log") . " - pid $$ >> ftp target: " . $ftp->pwd() . ".\n"; $ftp->put($fileelab) or die getTimeStamp("log") . " - pid $$ >> Put failed: " . $ftp->message . "\n"; print getTimeStamp("log") . " - pid $$ >> ftp target lists " . scalar @{ $ftp->ls() } . " elements.\n"; $ftp->quit; print getTimeStamp("log") . " - pid $$ >> $unit - $tool: $fileelab ftp put executed.\n"; } else { print getTimeStamp("log") . " - pid $$ >> $unit - $tool: $fileelab ftp put didn't executed because due date reached.\n"; } return; } sub getNodesType { my $dbh = DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) or die getTimeStamp("log") . " - pid $$ >> Could not connect to database: $DBI::errstr"; my $sth = $dbh->prepare( 'select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.ain as ain, n.din as din from nodes as n inner join tools as t on t.id = n.tool_id inner join units as u on u.id = t.unit_id inner join nodetypes as y on n.nodetype_id = y.id where t.name = "' . $tool . '" and u.name = "' . $unit . '" order by 3;' ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; $sth->execute(); print getTimeStamp("log") . " - pid $$ >> $unit - $tool: " . $sth->rows . " rows selected to get node type/Ain/Din/channels.\n"; if ( $sth->rows eq 0 ) { print getTimeStamp("log") . " - pid $$ >> Node/Channels/Ain/Din not defined.\n"; print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; exit; } else { my $row = 0; while ( my $results = $sth->fetchrow_hashref ) { $channels[$row] = $results->{'channels'}; $nodetype[$row] = $results->{'type'}; $ain[$row] = $results->{'ain'}; $din[ $row++ ] = $results->{'din'}; } } $sth->finish; # Disconnect $dbh->disconnect; return; } sub getUdbPwd { my $dbh = DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) or die getTimeStamp("log") . " - pid $$ >> Could not connect to database: $DBI::errstr"; my $sth = $dbh->prepare( "SELECT db_user, db_pwd FROM ase_lar.companies where db_name = '" . $db_name . "';" ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; $sth->execute(); if ( $sth->rows eq 0 ) { die getTimeStamp("log") . " - pid $$ >> No password selected.\n"; } else { my $results = $sth->fetchrow_hashref; $password = $results->{'db_pwd'}; $username = $results->{'db_user'}; } $sth->finish; # Disconnect $dbh->disconnect; return; } sub getMatlabCmd { my $dbh = DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) or die getTimeStamp("log") . " - pid $$ >> Could not connect to database: $DBI::errstr"; my $sth = $dbh->prepare( 'select m.matcall, t.ftp_send , t.unit_id from matfuncs as m inner join tools as t on t.matfunc = m.id inner join units as u on u.id = t.unit_id where t.name = "' . $tool . '" and u.name = "' . $unit . '";' ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; $sth->execute(); if ( $sth->rows eq 0 ) { die getTimeStamp("log") . " - pid $$ >> No tool's matlab function selected.\n"; } else { while ( my $results = $sth->fetchrow_hashref ) { $matlab_cmd = $results->{'matcall'}; $ftp_send = $results->{'ftp_send'}; $unit_id = $results->{'unit_id'}; } } $sth->finish; # Disconnect $dbh->disconnect; return; } sub makeEmailMsg { $email_msg = <<"END_MSG"; Alert from ASE
ASE
Alert from ASE:
Matlab function $matlab_cmd failed on unit => $unit - tool => $tool
$matlab_error
END_MSG } sub emailSend { $email_obj = 'Matlab error on unit ' . $unit . ' - tool ' . $tool; my $sender = new Mail::Sender { auth => 'PLAIN', authid => 'ase.mums.alert@gmail.com', authpwd => 'ASE@mums2013F!', smtp => 'smtp.gmail.com', port => 587, from => 'ASE Alert System', to => $email_addr, subject => $email_obj, msg => $email_msg, ctype => "text/html", encoding => "7bit" }; my $mail_res = $sender->MailMsg( { msg => $sender->{msg}, } ); if ( ref($mail_res) ne "Mail::Sender" ) { print getTimeStamp("log") . " - pid $$ >> $mail_res: $sender->{error_msg} \n"; } else { print getTimeStamp("log") . " - pid $$ >> $email_obj\n"; print getTimeStamp("log") . " - pid $$ >> Mail sent to: $email_addr.\n"; } } my $starttime = getTimeStamp("log"); print "$starttime - pid $$ >> Start execution.\n"; GetOptions( "filename=s" => \$filename, "sqlout=s" => \$sqlout, "dbname=s" => \$db_name ) or die("Error in command line arguments\n"); open FILE, $filename or die( "Error: opening input file " . $filename . "\n" ); if ( $db_name ne 'asega' ) { $db_name =~ s/aseu/dbn/; $db_sp = $db_name; getUdbPwd(); $db_lar = $db_name; } else { $db_name = 'ase_lar'; } my ( $filecsvname, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ ); if ( $filecsvname =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)_\d*$/i ) { ( $fileDate, $fileTime ) = split( /\s/, ); ( $unittype, $unit ) = split( /\s/, uc ); $unit =~ s/;+$//; } elsif ( $filecsvname =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*_\d*$/i ) { my @strings = $filecsvname =~ /(.{1,4})_(.{1,6})_(.{1,6})_(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2}).*/; $unittype = $strings[0]; $unit = $strings[1]; $tool = $strings[2]; $fileDate = $strings[3] . "/" . $strings[4] . "/" . $strings[5]; $fileTime = $strings[6] . ":" . $strings[7] . ":" . $strings[8]; $tooltype = 'MUX'; } while ( my $line = ) { my ( $first, $NodeData ) = split( /;/, $line, 2 ); if ( defined $first and ( $first ne '' ) ) { $first =~ s/;+$//; if ( !defined $NodeData or ( $NodeData =~ m/^;+/ ) ) { my @info = ( split( /[\/,\.]/, $first ) ); if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) { $tool = uc $info[3]; $tooltype = uc $info[2]; print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n"; getNodesType(); } elsif ( defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) { $tool = uc $info[2]; $tooltype = 'LOC'; print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n"; getNodesType(); } elsif ( defined $info[3] && $info[3] =~ m/^GD\d*$/i ) { $tool = uc $info[3]; $tooltype = 'GD'; print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n"; } } else { $NodeData =~ s/\x0d{0,1}\x0a\Z//s; $NodeData =~ s/;+$//; #$NodeData =~ s/\r\n\z//; if ( $unittype eq 'G201' and $G201_fcheck eq 0 ) { print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n"; getNodesType(); $G201_fcheck = 1; } if ( $tooltype eq "MUX" ) { #$NodeData =~ s/Dis\./N\/A/g; @NewData = grep /\S/, split( /\||;/, $NodeData ); my $idx = 0; $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); foreach my $ch (@channels) { $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, $ch ) ); } } elsif ( $tooltype eq "MUSA" ) { #$NodeData =~ s/Dis\./N\/A/g; my @tmpData = grep /\S/, split( /\||;/, $NodeData ); my $tmpBatt = shift @tmpData; @NewData = ( $tmpBatt, $tmpData[0], @tmpData ); my $idx = 0; $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); foreach my $ch (@channels) { $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, $ch ) ); } #print Dumper \@CSVData; #print Dumper \@channels; #print Dumper \@din; } elsif ( $tooltype eq "LOC" ) { my ( $Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2 ) = split( /;/, $NodeData ); my $idx = 0; $CSVData[ $idx++ ] = $Vbat . ";" . $Tmod; my $used_din = 0; my $used_ain = 0; for my $i ( 0 .. $#din ) { if ( $din[$i] eq 1 ) { if ( $used_din eq 0 ) { $CSVData[ $idx++ ] = $Din1; $used_din = 1; } else { $CSVData[ $idx++ ] = $Din2; } } elsif ( $ain[$i] eq 1 ) { if ( $used_ain eq 0 ) { $CSVData[ $idx++ ] = $Ain1; $used_ain = 1; } else { $CSVData[ $idx++ ] = $Ain2; } } elsif ( $din[$i] eq 2 ) { $CSVData[ $idx++ ] = $Din1 . ";" . $Din2; } elsif ( $ain[$i] eq 2 ) { $CSVData[ $idx++ ] = $Ain1 . ";" . $Ain2; } else { $CSVData[ $idx++ ] = "N/A"; } } } elsif ( $tooltype eq "GD" ) { my ( $div, $Vbat, $Tmod ) = split( /;/, $NodeData ); $CSVData[0] = $Vbat . ";" . $Tmod; my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); my ( $h, $min, $sec ) = split( /:/, $t ); $GDEventDate = $y . '-' . $m . '-' . $d; $CSVData[1] = "DT" . substr( $tool, 2, 4 ) . ';' . $h . ':' . ( $min - 1 ) . ';' . $h . ':' . ( $min + 1 ); } else { @CSVData = split( /\|/, $NodeData ); } my $nodenum = 0; foreach (@CSVData) { $sql[ ( $outcount % $maxsqllines ) ] = ""; my (@data) = grep { /\S/ } split( /\;/, $_ ); if ( $nodenum eq 0 ) { $toolbatt = $data[0]; $tooltemp = $data[1]; } else { my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); if ( length($y) != 4 ) { # G201 con la data invertita my $w = $y; $y = $d; $m = sprintf( '%02d', $m ); $d = sprintf( '%02d', $w ); } foreach (@data) { if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) { $sql[ ( $outcount % $maxsqllines ) ] = "('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'"; } $sql[ ( $outcount % $maxsqllines ) ] .= ",'" . $_ . "'"; } my $InsCompl = ",NULL" x ( 15 - $#data ); $sql[ ( $outcount % $maxsqllines ) ] .= $InsCompl . ")"; if ( ++$outcount % $maxsqllines eq 0 ) { $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n"; writeOutSql($tool); } else { $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n"; } } $nodenum++; } } } } close FILE; $sql[ ( $outcount % $maxsqllines ) - 1 ] =~ s/,$/;/g; writeOutSql($tool); if ( $tooltype eq "GD" ) { print getTimeStamp("log") . " - pid $$ >> tool GD: nothing to do.\n"; } else { matlabCalc(); if ($ftp_send) { if ( $tool eq 'DT0076' ) { sleep(600); } trxelab(); } } print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; exit;