diff --git a/LoadCSVData.pl b/LoadCSVData.pl index cdd9e4f..b1c07f4 100755 --- a/LoadCSVData.pl +++ b/LoadCSVData.pl @@ -46,9 +46,10 @@ my $matlab_func_dir = "/usr/local/matlab_func/"; my $matlab_error = ''; my $matlab_timeout = '1800'; my $email_addr = -'andrea.carri@aseltd.eu,edoardo.cavalca@aseltd.eu,alessandro.battilani@gmail.com,alessandro.valletta@aseltd.eu'; +'andrea.carri@aseltd.eu,alessandro.battilani@gmail.com,alessandro.valletta@aseltd.eu'; my $from = 'ASE Alert System'; my ( $email_msg, $email_obj ); +my $lock_elab = '/tmp/lock.loadcsvdata'; my $G201_fcheck = 0; my $D2W_fcheck = 0; my $FtpToCustomerCmd = 'SendFtpElabData.pl'; @@ -615,6 +616,10 @@ writeOutSql($tool); if ( $tooltype eq "GD" ) { print getTimeStamp("log") . " - pid $$ >> tool GD: nothing to do.\n"; } +elsif ( -e $lock_elab ) { + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool MatLab calc not executed due to the lock: $lock_elab\n"; +} else { matlabCalc(); diff --git a/mind/LoadCSVData.pl b/mind/LoadCSVData.pl new file mode 100755 index 0000000..7b154ca --- /dev/null +++ b/mind/LoadCSVData.pl @@ -0,0 +1,637 @@ +#!/usr/bin/perl +use warnings; +use strict; +use DBI; +use Getopt::Long; +use Net::FTP; +use File::Basename qw( fileparse ); +use List::Util qw( max ); +use Data::Dumper qw(Dumper); +use Cwd; +use MIME::Lite; + +$|++; # Autoflush + +my ( $filename, $sqlout ); +my $username = 'ase_lar'; +my $password = 'Laravel@22500'; +my $db_lar = 'ase_lar'; +my $db_sp = 'ase_lar'; +my $server = 'localhost'; +my $db_name = "ase_lar"; +my $table = "RAWDATACOR"; +my ( @CSVData, @sql ); +my $outcount = 0; +my $outfilenum = 0; +my $maxsqllines = 10000; +my ( + $tool, $toolbatt, $tooltemp, $tooltype, @channels, + @NewData, @ain, @din, @nodetype, $GDEventDate +); +my $matlab_ip = "160.78.21.71"; +my $matlab_cmd = "matlab.cmd"; +my $matlab_timestamp = ""; +my $ftp_send = 1; +my $ftp_addrs; +my $ftp_user; +my $ftp_passwd; +my $ftp_parm; +my $ftp_filename; +my $ftp_target; +my $duedate = ''; +my ( $unit_id, $unit, $unittype ); +my ( $fileDate, $fileTime ); +my $matlab_rt = "/usr/local/MATLAB/MATLAB_Runtime/v93"; +my $matlab_func_dir = "/usr/local/matlab_func/"; +my $matlab_error = ''; +my $matlab_timeout = '1800'; +my $email_addr = +'andrea.carri@aseltd.eu,alessandro.battilani@gmail.com,alessandro.valletta@aseltd.eu'; +my $from = 'ASE Alert System'; +my ( $email_msg, $email_obj ); +my $lock_elab = '/tmp/lock.loadcsvdata'; +my $G201_fcheck = 0; +my $D2W_fcheck = 0; +my $FtpToCustomerCmd = 'SendFtpElabData.pl'; +my ( $scriptname, $scriptpath ); +my $MatlabErrorFilename = "/tmp/"; +my @matlabOutputErrorArray; +my @errors; +my @warnings; +my $MatlabErrors = ''; +my $MatlabWarnings = ''; + +sub uniq { + my %seen; + grep !$seen{$_}++, @_; +} + +sub getTimeStamp +{ # parm [ts] => timestamp for filename; log => timestamp for log + my $format = "%04d%02d%02d%02d%02d%02d"; + my ($p1) = @_; + if ( defined $p1 and $p1 eq "log" ) { + $format = "%04d%02d%02d %02d:%02d:%02d"; + } + if ( defined $p1 and $p1 eq "db_ts" ) { + $format = "%04d-%02d-%02d %02d:%02d:%02d"; + } + my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = + localtime(time); + my $timestamp = + sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec ); + return $timestamp; +} + +sub writeOutSql { + my ($tn) = @_; + my $outfile = $sqlout . "_" . $outfilenum++; + open SQLOUT, ">", $outfile + or die( getTimeStamp("log") + . " - pid $$ >> Error: opening output file " + . $outfile + . ":$!\n" ); + print getTimeStamp("log") . " - pid $$ >> file $outfile created\n"; + print SQLOUT "/* lock " + . $db_name + . ".$table table */\nLOCK TABLES " + . $db_name + . ".$table WRITE;\n"; + print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n"; + print SQLOUT +"(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n"; + print SQLOUT +"`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n"; + print SQLOUT "VALUES\n"; + print SQLOUT @sql; + @sql = (); + print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n"; + + close SQLOUT + or die( getTimeStamp("log") + . " - pid $$ >> Error in closing file " + . $outfile + . "\n" ); + my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile ); + system(@args) == 0 + or die( getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n" ); + print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n"; + + unlink $outfile; + return; +} + +sub matlabCalc { + getMatlabCmd(); + $matlab_timestamp = getTimeStamp("db_ts"); + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool MatLab calc started...\n"; + if ( $matlab_cmd =~ /_lnx$/ ) { + $matlab_cmd = './run_' . $matlab_cmd . '.sh'; + $MatlabErrorFilename .= "${unit}${tool}_output_error.txt"; + + #print $ENV{"HOME"} . "\n"; + $ENV{"HOME"} = "/" . $ENV{"HOME"}; + chdir($matlab_func_dir) + or die "cannot change: $!\n"; + print getTimeStamp("log") + . " - pid $$ >> current dir: $matlab_func_dir\n"; + my @args = + ( 'timeout', $matlab_timeout, $matlab_cmd, $matlab_rt, $unit, $tool ); + if ( system(@args) != 0 ) { + open( my $fh, '<', $MatlabErrorFilename ) + or warn print getTimeStamp("log") + . " - pid $$ >> Cannot open Matlab output error file: ${MatlabErrorFilename}\n"; + { + chomp( @matlabOutputErrorArray = <$fh> ); + } + close($fh); + @matlabOutputErrorArray = grep( /\S/, @matlabOutputErrorArray ); + @errors = grep( /^Error/, @matlabOutputErrorArray ); + @warnings = grep( !/^Error/, @matlabOutputErrorArray ); + $MatlabErrors = join( "
", @errors ); + $MatlabWarnings = join( "
", uniq(@warnings) ); + my $exit_value = $? >> 8; + + if ( $exit_value == 124 ) { + print getTimeStamp("log") + . " - pid $$ >> system @args excessive duration: killed after $matlab_timeout seconds\n"; + $matlab_error = +"Matlab elab excessive duration: killed after $matlab_timeout seconds"; + } + else { + print getTimeStamp("log") + . " - pid $$ >> system @args failed: return code $exit_value - $?\n"; + $matlab_error = "Matlab elab failed: $exit_value - $?"; + } + + #makeEmailMsg(); + #emailSend(); + print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; + exit(1); + } + } + else { + $matlab_cmd = 'matlab_' . uc $matlab_cmd . '.cmd'; + my @out_matlab = `ssh1 $matlab_ip 'cmd /c $matlab_cmd ' $tool`; + if ( index( $out_matlab[-1], "Failure" ) != -1 ) { + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool MatLab calc failed.\n"; + + #makeEmailMsg(); + #emailSend(); + print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; + exit(1); + } + } + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool MatLab calc executed.\n"; + return; +} + +sub trxelab { + my $user = getpwuid($>); + my $ftpcmd = +"$scriptpath$FtpToCustomerCmd -m \"$matlab_timestamp\" -u $unit -t $tool >> /home/$user/log/loadcsvdata.log 2>&1"; + print getTimeStamp("log") + . " - pid $$ >> Fork FTP command to set GID mysql: $ftpcmd.\n"; + + unless ( fork() ) { + exec( '/usr/bin/sg', 'mysql', "$ftpcmd" ); + exit(0); + } + +} + +sub getNodesType { + my $dbh = + DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log") + . " - pid $$ >> Could not connect to database: $DBI::errstr"; + + my $sth = $dbh->prepare( +'select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.ain as ain, n.din as din + from nodes as n + inner join tools as t on t.id = n.tool_id + inner join units as u on u.id = t.unit_id + inner join nodetypes as y on n.nodetype_id = y.id + where t.name = "' + . $tool + . '" and u.name = "' + . $unit + . '" order by 3;' + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool: " + . $sth->rows + . " rows selected to get node type/Ain/Din/channels.\n"; + if ( $sth->rows eq 0 ) { + print getTimeStamp("log") + . " - pid $$ >> Node/Channels/Ain/Din not defined.\n"; + print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; + exit; + } + else { + my $row = 0; + while ( my $results = $sth->fetchrow_hashref ) { + $channels[$row] = $results->{'channels'}; + $nodetype[$row] = $results->{'type'}; + $ain[$row] = $results->{'ain'}; + $din[ $row++ ] = $results->{'din'}; + } + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + +sub getUdbPwd { + my $dbh = + DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log") + . " - pid $$ >> Could not connect to database: $DBI::errstr"; + my $sth = $dbh->prepare( + "SELECT db_user, db_pwd FROM ase_lar.companies where db_name = '" + . $db_name + . "';" ) + or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + if ( $sth->rows eq 0 ) { + die getTimeStamp("log") . " - pid $$ >> No password selected.\n"; + } + else { + my $results = $sth->fetchrow_hashref; + $password = $results->{'db_pwd'}; + $username = $results->{'db_user'}; + + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + +sub getMatlabCmd { + my $dbh = + DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log") + . " - pid $$ >> Could not connect to database: $DBI::errstr"; + + my $sth = $dbh->prepare( + 'select m.matcall, t.ftp_send , t.unit_id from matfuncs as m + inner join tools as t on t.matfunc = m.id + inner join units as u on u.id = t.unit_id + where t.name = "' . $tool . '" and u.name = "' . $unit . '";' + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + if ( $sth->rows eq 0 ) { + die getTimeStamp("log") + . " - pid $$ >> No tool's matlab function selected.\n"; + } + else { + while ( my $results = $sth->fetchrow_hashref ) { + $matlab_cmd = $results->{'matcall'}; + $ftp_send = $results->{'ftp_send'}; + $unit_id = $results->{'unit_id'}; + } + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + +sub makeEmailMsg { + $email_msg = <<"END_MSG"; + + + + + Alert from ASE + + + + + + + + + + + + + + + + + + + + + + +
+ ASE +
+

Alert from ASE:

+
+

Matlab function $matlab_cmd failed on unit => $unit - tool => $tool

+
+

$matlab_error

+
+ $MatlabErrors +
+ $MatlabWarnings +
+ + +END_MSG +} + +sub emailSend { + $email_obj = 'Matlab error on unit ' . $unit . ' - tool ' . $tool; + my $msg = MIME::Lite->new( + From => $from, + To => $email_addr, + Subject => $email_obj, + Data => $email_msg + ); + + $msg->attr( "content-type" => "text/html" ); + if ( + $msg->send( + 'smtp', "smtps.aruba.it", + AuthUser => "alert\@aseltd.eu", + AuthPass => "Ase#2013!20\@bat", + Port => 465, + SSL => 1, + Debug => 0 + ) + ) + { + print getTimeStamp("log") . " - pid $$ >> $email_obj\n"; + print getTimeStamp("log") . " - pid $$ >> Mail sent to: $email_addr.\n"; + } + else { + print getTimeStamp("log") + . " - pid $$ >> Error in sending mail to: $email_addr.\n"; + } +} + +my $starttime = getTimeStamp("log"); +print "$starttime - pid $$ >> Start execution.\n"; + +GetOptions( + "filename=s" => \$filename, + "sqlout=s" => \$sqlout, + "dbname=s" => \$db_name +) or die("Error in command line arguments\n"); +open FILE, $filename + or die( "Error: opening input file " . $filename . "\n" ); + +( $scriptname, $scriptpath ) = fileparse($0); + +if ( $db_name ne 'aseftp' ) { + $db_name =~ s/aseu/dbn/; + $db_sp = $db_name; + getUdbPwd(); + $db_lar = $db_name; +} +else { + $db_name = 'ase_lar'; +} + +my ( $filecsvname, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ ); +if ( $filecsvname =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)_\d*$/i ) { + ( $fileDate, $fileTime ) = split( /\s/, ); + ( $unittype, $unit ) = split( /\s/, uc ); + $unit =~ s/;+$//; +} +elsif ( $filecsvname =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*_\d*$/i ) { + my @strings = $filecsvname =~ +/(.{1,4})_(.{1,6})_(.{1,6})_(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2}).*/; + $unittype = $strings[0]; + $unit = $strings[1]; + $tool = $strings[2]; + $fileDate = $strings[3] . "/" . $strings[4] . "/" . $strings[5]; + $fileTime = $strings[6] . ":" . $strings[7] . ":" . $strings[8]; + $tooltype = 'MUX'; +} +elsif ( $filecsvname =~ m/^D2W_ID\d\d\d\d_DT\d\d\d\d_\d*$/i ) { + my @strings = $filecsvname =~ /(.{1,3})_(.{1,6})_(.{1,6}).*/; + $unittype = $strings[0]; + $unit = $strings[1]; + $tool = $strings[2]; + $tooltype = 'D2W'; +} + +while ( my $line = ) { + my ( $first, $NodeData ) = split( /;/, $line, 2 ); + if ( defined $first and ( $first ne '' ) ) { + $first =~ s/;+$//; + if ( !defined $NodeData or ( $NodeData =~ m/^;+/ ) ) { + my @info = ( split( /[\/,\.]/, $first ) ); + if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) { + $tool = uc $info[3]; + $tooltype = uc $info[2]; + print getTimeStamp("log") + . " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + } + elsif ( defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) { + $tool = uc $info[2]; + $tooltype = 'LOC'; + print getTimeStamp("log") + . " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + } + elsif ( defined $info[3] && $info[3] =~ m/^GD\d*$/i ) { + $tool = uc $info[3]; + $tooltype = 'GD'; + print getTimeStamp("log") + . " - pid $$ >> Found $tooltype tool name: $tool\n"; + } + } + else { + $NodeData =~ s/\x0d{0,1}\x0a\Z//s; + $NodeData =~ s/;+$//; + + #$NodeData =~ s/\r\n\z//; + if ( $unittype eq 'G201' and $G201_fcheck eq 0 ) { + print getTimeStamp("log") + . " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + $G201_fcheck = 1; + } + if ( $unittype eq 'D2W' and $D2W_fcheck eq 0 ) { + print getTimeStamp("log") + . " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + $D2W_fcheck = 1; + } + + if ( $tooltype eq "MUX" or $tooltype eq "D2W" ) { + + #$NodeData =~ s/Dis\./N\/A/g; + @NewData = grep /\S/, split( /\||;/, $NodeData ); + if ( scalar(@NewData) != 0 ) { + my $idx = 0; + + $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); + foreach my $ch (@channels) { + $CSVData[ $idx++ ] = + join( ';', splice( @NewData, 0, $ch ) ); + } + } + } + elsif ( $tooltype eq "MUSA" ) { + + #$NodeData =~ s/Dis\./N\/A/g; + my @tmpData = grep /\S/, split( /\||;/, $NodeData ); + my $tmpBatt = shift @tmpData; + @NewData = ( $tmpBatt, $tmpData[0], @tmpData ); + + my $idx = 0; + + $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); + foreach my $ch (@channels) { + $CSVData[ $idx++ ] = + join( ';', splice( @NewData, 0, $ch ) ); + } + + #print Dumper \@CSVData; + #print Dumper \@channels; + #print Dumper \@din; + + } + elsif ( $tooltype eq "LOC" ) { + my ( $Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2 ) = + split( /;/, $NodeData ); + + my $idx = 0; + $CSVData[ $idx++ ] = $Vbat . ";" . $Tmod; + + my $used_din = 0; + my $used_ain = 0; + for my $i ( 0 .. $#din ) { + if ( $din[$i] eq 1 ) { + if ( $used_din eq 0 ) { + $CSVData[ $idx++ ] = $Din1; + $used_din = 1; + } + else { + $CSVData[ $idx++ ] = $Din2; + } + } + elsif ( $ain[$i] eq 1 ) { + if ( $used_ain eq 0 ) { + $CSVData[ $idx++ ] = $Ain1; + $used_ain = 1; + } + else { + $CSVData[ $idx++ ] = $Ain2; + } + } + elsif ( $din[$i] eq 2 ) { + $CSVData[ $idx++ ] = $Din1 . ";" . $Din2; + } + elsif ( $ain[$i] eq 2 ) { + $CSVData[ $idx++ ] = $Ain1 . ";" . $Ain2; + } + else { + $CSVData[ $idx++ ] = "N/A"; + } + } + + } + elsif ( $tooltype eq "GD" ) { + my ( $div, $Vbat, $Tmod ) = + split( /;/, $NodeData ); + $CSVData[0] = $Vbat . ";" . $Tmod; + my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); + my ( $h, $min, $sec ) = split( /:/, $t ); + $GDEventDate = $y . '-' . $m . '-' . $d; + $CSVData[1] = "DT" + . substr( $tool, 2, 4 ) . ';' + . $h . ':' + . ( $min - 1 ) . ';' + . $h . ':' + . ( $min + 1 ); + } + else { + @CSVData = split( /\|/, $NodeData ); + } + + my $nodenum = 0; + foreach (@CSVData) { + $sql[ ( $outcount % $maxsqllines ) ] = ""; + my (@data) = grep { /\S/ } split( /\;/, $_ ); + if ( $nodenum eq 0 ) { + $toolbatt = $data[0]; + $tooltemp = $data[1]; + } + else { + my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); + if ( length($y) != 4 ) { # G201 con la data invertita + my $w = $y; + $y = $d; + $m = sprintf( '%02d', $m ); + $d = sprintf( '%02d', $w ); + } + foreach (@data) { + if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) { + $sql[ ( $outcount % $maxsqllines ) ] = +"('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'"; + } + $sql[ ( $outcount % $maxsqllines ) ] .= ",'" . $_ . "'"; + } + my $InsCompl = ",NULL" x ( 15 - $#data ); + $sql[ ( $outcount % $maxsqllines ) ] .= $InsCompl . ")"; + if ( ++$outcount % $maxsqllines eq 0 ) { + $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n"; + writeOutSql($tool); + } + else { + $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n"; + } + } + $nodenum++; + } + } + } +} +close FILE; +$sql[ ( $outcount % $maxsqllines ) - 1 ] =~ s/,$/;/g; +writeOutSql($tool); +if ( $tooltype eq "GD" ) { + print getTimeStamp("log") . " - pid $$ >> tool GD: nothing to do.\n"; +} +elsif ( -e $lock_elab ) { + print getTimeStamp("log") + . " - pid $$ >> $unit - $tool MatLab calc not executed due to the lock: $lock_elab\n"; +} +else { + matlabCalc(); + + if ($ftp_send) { + if ( $tool eq 'DT0076' ) { + sleep(600); + } + trxelab(); + } +} + +print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; +exit; diff --git a/mind/LoadCSVData_old.pl b/mind/LoadCSVData_old.pl new file mode 100755 index 0000000..5f0d310 --- /dev/null +++ b/mind/LoadCSVData_old.pl @@ -0,0 +1,561 @@ +#!/usr/bin/perl +use warnings; +use strict; +use DBI; +use Getopt::Long; +use Net::FTP; +use File::Basename qw( fileparse ); +use List::Util qw( max ); +use Data::Dumper qw(Dumper); +use Cwd; +use Mail::Sender; + +$|++; # Autoflush + +my ( $filename, $sqlout ); +my $username = 'ase_lar'; +my $password = 'Laravel@22500'; +my $db_lar = 'ase_lar'; +my $db_sp = 'ase_lar'; +my $server = 'localhost'; +my $db_name = "ase_lar"; +my $table = "RAWDATACOR"; +my ( @CSVData, @sql ); +my $outcount = 0; +my $outfilenum = 0; +my $maxsqllines = 10000; +my ($tool, $toolbatt, $tooltemp, $tooltype, @channels,@NewData, @ain, @din, @nodetype, $GDEventDate); +my $matlab_ip = "160.78.21.71"; +my $matlab_cmd = "matlab.cmd"; +my $matlab_timestamp = ""; +my $ftp_send = 1; +my $ftp_addrs; +my $ftp_user; +my $ftp_passwd; +my $ftp_parm; +my $ftp_filename; +my $ftp_target; +my $duedate = ''; +my ($unit_id, $unit, $unittype); +my ($fileDate, $fileTime); +my $matlab_rt = "/usr/local/MATLAB/MATLAB_Runtime/v93"; +my $matlab_func_dir = "/usr/local/matlab_func/"; +my $matlab_error = ''; +my $matlab_timeout = '1200'; +my $email_addr = 'andrea.carri@aseltd.eu,roberto.savi@aseltd.eu,alessandro.battilani@gmail.com'; +my ($email_msg, $email_obj); +my $lock_elab = '/tmp/lock.loadcsvdata'; + + +sub getTimeStamp{ # parm [ts] => timestamp for filename; log => timestamp for log + my $format = "%04d%02d%02d%02d%02d%02d"; + my ($p1) = @_; + if ( defined $p1 and $p1 eq "log" ) { + $format = "%04d%02d%02d %02d:%02d:%02d"; + } + if ( defined $p1 and $p1 eq "db_ts" ) { + $format = "%04d-%02d-%02d %02d:%02d:%02d"; + } + my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =localtime(time); + my $timestamp =sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec ); + return $timestamp; +} + + +sub writeOutSql { + my ($tn) = @_; + my $outfile = $sqlout . "_" . $outfilenum++; + open SQLOUT, ">", $outfile + or die( getTimeStamp("log"). " - pid $$ >> Error: opening output file ". $outfile. ":$!\n" ); + print getTimeStamp("log") . " - pid $$ >> file $outfile created\n"; + print SQLOUT "/* lock ". $db_name. ".$table table */\nLOCK TABLES ". $db_name. ".$table WRITE;\n"; + print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n"; + print SQLOUT "(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n"; + print SQLOUT "`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n"; + print SQLOUT "VALUES\n"; + print SQLOUT @sql; + @sql = (); + print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n"; + + close SQLOUT + or die( getTimeStamp("log"). " - pid $$ >> Error in closing file ". $outfile. "\n" ); + my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile ); + system(@args) == 0 + or die( getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n" ); + print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n"; + + unlink $outfile; + return; +} + + +sub matlabCalc { + getMatlabCmd(); + $matlab_timestamp = getTimeStamp("db_ts"); + print getTimeStamp("log") . " - pid $$ >> $unit - $tool MatLab calc started...\n"; + if ($matlab_cmd =~ /_lnx$/) { + $matlab_cmd = './run_' . $matlab_cmd . '.sh'; + + #print $ENV{"HOME"} . "\n"; + $ENV{"HOME"} = "/" . $ENV{"HOME"}; + chdir($matlab_func_dir) + or die "cannot change: $!\n"; + print getTimeStamp("log") . " - pid $$ >> current dir: $matlab_func_dir\n"; + my @args = ( 'timeout', $matlab_timeout, $matlab_cmd, $matlab_rt, $unit, $tool); + if (system(@args) != 0) { + my $exit_value = $? >> 8; + if ($exit_value == 124) { + print getTimeStamp("log") . " - pid $$ >> system @args excessive duration: killed after $matlab_timeout seconds\n"; + $matlab_error = "Maltab elab excessive duration: killed after $matlab_timeout seconds"; + } else { + print getTimeStamp("log") . " - pid $$ >> system @args failed: return code $exit_value - $?\n"; + $matlab_error = "Maltab elab failed: $exit_value - $?"; + } + #makeEmailMsg(); + #emailSend(); + print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; + exit(1); + } + } else { + print getTimeStamp("log") . " - pid $$ >> Wrong MatLab calc command.\n"; + print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; + exit(1); + } + print getTimeStamp("log"). " - pid $$ >> $unit - $tool MatLab calc executed.\n"; + return; +} + + +sub trxelab { + my $dbh =DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log"). " - pid $$ >> Could not connect to database: $DBI::errstr"; + + my $sth = $dbh->prepare( + 'select ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename, ftp_target, duedate from units + where id = ' . $unit_id . ';' + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + if ( $sth->rows eq 0 ) { + die getTimeStamp("log") . " - pid $$ >> No unit ftp params selected.\n"; + }else { + while ( my $results = $sth->fetchrow_hashref ) { + $ftp_addrs = $results->{'ftp_addrs'}; + $ftp_user = $results->{'ftp_user'}; + $ftp_passwd = $results->{'ftp_passwd'}; + $ftp_parm = $results->{'ftp_parm'}; + $ftp_filename = $results->{'ftp_filename'}; + $ftp_target = $results->{'ftp_target'}; + $duedate = $results->{'duedate'}; + } + } + + $sth->finish; + + # Disconnect + + my $fileelab; + if ( $ftp_filename eq "" ) { + $fileelab ='/var/lib/mysql-files/'. $tool . '_'. getTimeStamp(). '.csv'; #mettere quello che si prende da query *** aggiungere $unit + }else { + $ftp_filename =~ s/(\$\w+)/$1/eeg; + $ftp_filename =~ s/\s/_/g; + $fileelab ='/var/lib/mysql-files/'. $ftp_filename . '_'. getTimeStamp() . '.csv'; + } + + my $sthdo = $dbh->do( + qq{ + select * from ( + select 'ToolNameID', 'EventDate', 'EventTime', 'NodeNum', 'NodeType', 'NodeDepth', + 'XShift', 'YShift', 'ZShift' , 'X', 'Y', 'Z', 'HShift', 'HShiftDir', 'HShift_local', + 'speed', 'speed_local', 'acceleration', 'acceleration_local', 'T_node', 'water_level', 'pressure', 'load_value', 'AlfaX', 'AlfaY', 'CalcErr' + union all + select ToolNameID, EventDate, EventTime, NodeNum, NodeType, NodeDepth, + XShift, YShift, ZShift , X, Y, Z, HShift, HShiftDir, HShift_local, + speed, speed_local, acceleration, acceleration_local, T_node, water_level, pressure, load_value, AlfaX, AlfaY, calcerr + from elabdataview + where UnitName = '$unit' and ToolNameID = '$tool' and updated_at > '$matlab_timestamp' + order by ToolNameID DESC, EventDate, EventTime, convert(`NodeNum`, UNSIGNED INTEGER) DESC + ) resulting_set + into outfile '$fileelab' + fields terminated by ',' optionally enclosed by '"' lines terminated by '\n'} + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $dbh->disconnect; + $ftp_parm //= ""; + $ftp_parm =~ s/\s//g; + + my %ftp_parms = ('Host' => $ftp_addrs,); + + foreach my $parm (split(/,/, $ftp_parm)){ + my ($key, $val) = split(/=>/, $parm); + $ftp_parms{$key} = $val; + } + $ftp_parms{'Debug'} = 0; + $ftp_parms{'Passive'} = 1; + + print getTimeStamp("log") . " - pid $$ >> $unit - $tool ftp parm:\n" . Dumper(%ftp_parms); + + if ( !defined $duedate + or($duedate eq '0000-00-00 00:00:00') + or($duedate eq '') + or($duedate gt getTimeStamp("db_ts") )) { + my $ftp = Net::FTP->new(%ftp_parms) + or die getTimeStamp("log"). " - pid $$ >> Cannot connect to $ftp_addrs: $@ \n"; + $ftp->login( $ftp_user, $ftp_passwd ) + or die getTimeStamp("log"). " - pid $$ >> Cannot login ". $ftp->message . "\n"; + $ftp_target //= ""; + if ( $ftp_target ne "" ) { + $ftp->cwd($ftp_target) + or die getTimeStamp("log"). " - pid $$ >> Change remote dir failed ". $ftp->message . "\n"; + } + + print getTimeStamp("log"). " - pid $$ >> ftp target: ". $ftp->pwd() . ".\n"; + + $ftp->put($fileelab) + or die getTimeStamp("log"). " - pid $$ >> Put failed: ". $ftp->message . "\n"; + + print getTimeStamp("log"). " - pid $$ >> ftp target lists ". scalar @{ $ftp->ls() }. " elements.\n"; + + $ftp->quit; + print getTimeStamp("log"). " - pid $$ >> $unit - $tool: $fileelab ftp put executed.\n"; + } else { + print getTimeStamp("log"). " - pid $$ >> $unit - $tool: $fileelab ftp put didn't executed because due date reached.\n"; + } + return; +} + + +sub getNodesType { + my $dbh =DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log"). " - pid $$ >> Could not connect to database: $DBI::errstr"; + + my $sth = $dbh->prepare( + 'select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.ain as ain, n.din as din + from nodes as n + inner join tools as t on t.id = n.tool_id + inner join units as u on u.id = t.unit_id + inner join nodetypes as y on n.nodetype_id = y.id + where t.name = "' . $tool . '" and u.name = "' . $unit . '" order by 3;' + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + print getTimeStamp("log"). " - pid $$ >> $unit - $tool: " . $sth->rows . " rows selected to get node type/Ain/Din/channels.\n"; + if ( $sth->rows eq 0 ) { + print getTimeStamp("log"). " - pid $$ >> Node/Channels/Ain/Din not defined.\n"; + print getTimeStamp("log"). " - pid $$ >> Execution ended.\n"; + exit; + }else { + my $row = 0; + while ( my $results = $sth->fetchrow_hashref ) { + $channels[$row] = $results->{'channels'}; + $nodetype[$row] = $results->{'type'}; + $ain[$row] = $results->{'ain'}; + $din[ $row++ ] = $results->{'din'}; + } + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + + +sub getUdbPwd { + my $dbh =DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log"). " - pid $$ >> Could not connect to database: $DBI::errstr"; + my $sth = $dbh->prepare("SELECT db_user, db_pwd FROM ase_lar.companies where db_name = '". $db_name. "';" ) + or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + if ( $sth->rows eq 0 ) { + die getTimeStamp("log") . " - pid $$ >> No password selected.\n"; + }else { + my $results = $sth->fetchrow_hashref; + $password = $results->{'db_pwd'}; + $username = $results->{'db_user'}; + + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + + +sub getMatlabCmd { + my $dbh =DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password ) + or die getTimeStamp("log"). " - pid $$ >> Could not connect to database: $DBI::errstr"; + + my $sth = $dbh->prepare( + 'select m.matcall, t.ftp_send , t.unit_id from matfuncs as m + inner join tools as t on t.matfunc = m.id + inner join units as u on u.id = t.unit_id + where t.name = "' . $tool . '" and u.name = "' . $unit . '";' + ) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr"; + + $sth->execute(); + if ( $sth->rows eq 0 ) { + die getTimeStamp("log"). " - pid $$ >> No tool's matlab function selected.\n"; + }else { + while ( my $results = $sth->fetchrow_hashref ) { + $matlab_cmd = $results->{'matcall'}; + $ftp_send = $results->{'ftp_send'}; + $unit_id = $results->{'unit_id'}; + } + } + + $sth->finish; + + # Disconnect + $dbh->disconnect; + return; +} + + +sub makeEmailMsg { + $email_msg = <<"END_MSG"; + + + + + Alert from ASE + + + + + + + +
+ + + + + + + +
+ ASE +
+ + + + +
+ Alert from ASE:
+ Matlab function $matlab_cmd failed on unit => $unit - tool => $tool
+ $matlab_error +
+
+
+ + +END_MSG +} + + +sub emailSend { + $email_obj = 'Matlab error on unit ' . $unit . ' - tool ' . $tool; + + my $sender = new Mail::Sender { + auth => 'PLAIN', + authid => 'ase.mums.alert@gmail.com', + authpwd => '', + smtp => 'smtp.gmail.com', + port => 587, + from => 'ASE Alert System', + to => $email_addr, + subject => $email_obj, + msg => $email_msg, + ctype => "text/html", + encoding => "7bit" + }; + + my $mail_res = $sender->MailMsg({msg => $sender->{msg},}); + if (ref($mail_res) ne "Mail::Sender") { + print getTimeStamp("log") . " - pid $$ >> $mail_res: $sender->{error_msg} \n"; + } else { + print getTimeStamp("log") . " - pid $$ >> $email_obj\n"; + print getTimeStamp("log") . " - pid $$ >> Mail sent to: $email_addr.\n"; + } +} + +my $starttime = getTimeStamp("log"); +print "$starttime - pid $$ >> Start execution.\n"; + +GetOptions( + "filename=s" => \$filename, + "sqlout=s" => \$sqlout, + "dbname=s" => \$db_name +) or die("Error in command line arguments\n"); +open FILE, $filename + or die( "Error: opening input file " . $filename . "\n" ); + +if ( $db_name ne 'aseftp' ) { + $db_name =~ s/aseu/dbn/; + $db_sp = $db_name; + getUdbPwd(); + $db_lar = $db_name; +}else { + $db_name = 'ase_lar'; +} + +my ( $filecsvname, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ ); + +if ( $filecsvname =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)_\d*$/i ) { + ($fileDate, $fileTime) = split( /\s/, ); + ($unittype, $unit ) = split( /\s/, uc ); + $unit =~ s/;+$//; +} elsif ( $filecsvname =~ m/^(\d\d_\d\d\d\d_|)(\d\d\d\d\d\d\d\d\d\d\d\d\d\d_)(G201_ID\d\d\d\d_DT\d\d\d\d)_\d*$/i ) { + + # da mettere il controllo ^^^^^^^^^^^^^^^ se ha mese ed anno all'inivio + my @strings = $filecsvname =~ /(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2})_(.{1,4})_(.{1,6})_(.{1,6}).*/; + $fileDate = $strings[0] . "/" . $strings[1] . "/" . $strings[2]; + $fileTime = $strings[3] . ":" . $strings[4] . ":" . $strings[5]; + $unittype = $strings[6]; + $unit = $strings[7]; + $tool = $strings[8]; + $tooltype = 'MUX'; +} + +while ( my $line = ) { + my ( $first, $NodeData ) = split( /;/, $line, 2 ); + if ( defined $first and ( $first ne '' ) ) { + $first =~ s/;+$//; + if ( !defined $NodeData or ( $NodeData =~ m/^;+/ ) ) { + my @info = ( split( /[\/,\.]/, $first ) ); + if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) { + $tool = uc $info[3]; + $tooltype = uc $info[2]; + print getTimeStamp("log"). " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + }elsif ( defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) { + $tool = uc $info[2]; + $tooltype = 'LOC'; + print getTimeStamp("log"). " - pid $$ >> Found $tooltype tool name: $tool\n"; + getNodesType(); + }elsif ( defined $info[3] && $info[3] =~ m/^GD\d*$/i ) { + $tool = uc $info[3]; + $tooltype = 'GD'; + print getTimeStamp("log"). " - pid $$ >> Found $tooltype tool name: $tool\n"; + } + }else { + $NodeData =~ s/\x0d{0,1}\x0a\Z//s; + $NodeData =~ s/;+$//; + + #$NodeData =~ s/\r\n\z//; + + if ( $tooltype eq "MUX" ) { + + #$NodeData =~ s/Dis\./N\/A/g; + @NewData = grep /\S/, split( /\||;/, $NodeData ); + + my $idx = 0; + + $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); + foreach my $ch (@channels) { + $CSVData[ $idx++ ] = + join( ';', splice( @NewData, 0, $ch ) ); + } + + }elsif ( $tooltype eq "MUSA" ) { + + #$NodeData =~ s/Dis\./N\/A/g; + my @tmpData = grep /\S/, split( /\||;/, $NodeData ); + my $tmpBatt = shift @tmpData; + @NewData = ($tmpBatt, $tmpData[0], @tmpData); + + my $idx = 0; + + $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); + foreach my $ch (@channels) { + $CSVData[ $idx++ ] = + join( ';', splice( @NewData, 0, $ch ) ); + } + + #print Dumper \@CSVData; + #print Dumper \@channels; + #print Dumper \@din; + + }elsif ( $tooltype eq "LOC" ) { + if ( !defined $din[0] ) { + print getTimeStamp("log"). " - pid $$ >> Din not defined.\n"; + print getTimeStamp("log"). " - pid $$ >> Execution ended.\n"; + exit; + } + my ( $Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2 ) = + split( /;/, $NodeData ); + $CSVData[0] = $Vbat . ";" . $Tmod; + $CSVData[1] = $Din1; + }elsif ( $tooltype eq "GD" ) { + my ( $div, $Vbat, $Tmod ) = + split( /;/, $NodeData ); + $CSVData[0] = $Vbat . ";" . $Tmod; + my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); + my ( $h, $min, $sec ) = split( /:/, $t ); + $GDEventDate = $y . '-' . $m . '-' . $d; + $CSVData[1] = "DT". substr( $tool, 2, 4 ) . ';'. $h . ':'. ( $min - 1 ) . ';'. $h . ':'. ( $min + 1 ); + }else { + @CSVData = split( /\|/, $NodeData ); + } + + my $nodenum = 0; + foreach (@CSVData) { + $sql[ ( $outcount % $maxsqllines ) ] = ""; + my (@data) = grep { /\S/ } split( /\;/, $_ ); + if ( $nodenum eq 0 ) { + $toolbatt = $data[0]; + $tooltemp = $data[1]; + }else { + my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); + if (length($y) != 4) { # G201 con la data invertita + my $w = $y; + $y = $d; + $m = sprintf('%02d',$m); + $d = sprintf('%02d',$w); + } + foreach (@data) { + if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) { + $sql[ ( $outcount % $maxsqllines ) ] = "('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'"; + } + $sql[ ( $outcount % $maxsqllines ) ] .= ",'" . $_ . "'"; + } + my $InsCompl = ",NULL" x ( 15 - $#data ); + $sql[ ( $outcount % $maxsqllines ) ] .= $InsCompl . ")"; + if ( ++$outcount % $maxsqllines eq 0 ) { + $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n"; + writeOutSql($tool); + }else { + $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n"; + } + } + $nodenum++; + } + } + } +} +close FILE; +$sql[ ( $outcount % $maxsqllines ) - 1 ] =~ s/,$/;/g; +writeOutSql($tool); +if ( $tooltype eq "GD" ) { + print getTimeStamp("log") . " - pid $$ >> tool GD: nothing to do.\n"; +} elsif (-e $lock_elab) { + print getTimeStamp("log") . " - pid $$ >> $unit - $tool MatLab calc not executed due to the lock: $lock_elab\n"; +} else { + matlabCalc(); + + if ($ftp_send) { + if ( $tool eq 'DT0076' ) { + sleep(600); + } + trxelab(); + } +} + +print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; +exit; \ No newline at end of file diff --git a/mind/UnitCsvReceiverSmart.pl b/mind/UnitCsvReceiverSmart.pl new file mode 100644 index 0000000..c95bd80 --- /dev/null +++ b/mind/UnitCsvReceiverSmart.pl @@ -0,0 +1,177 @@ +#!/usr/bin/perl +use warnings; +use strict; +use Getopt::Long; +use File::SmartTail; +use File::Basename qw( fileparse ); +use File::Path qw( make_path ); +use File::Copy qw( move ); +use POSIX; +$SIG{CHLD} = 'IGNORE'; + +$|++; # Autoflush + +sub getTimeStamp +{ # parm [ts] => timestamp for filename; log => timestamp for log + my $format = "%04d%02d%02d%02d%02d%02d"; + my ($p1) = @_; + if ( defined $p1 and $p1 eq "log" ) { + $format = "%04d%02d%02d %02d:%02d:%02d"; + } + my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = + localtime(time); + my $timestamp = + sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec ); + return $timestamp; +} + +sub getUnitName { # parm => file received ($trfile) + my ($filename) = @_; + my ( $fileDate, $fileTime ); + my ( $unittype, $unit ); + my ( $filenamecsv, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ ); + if ( $filenamecsv =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*$/i ) { # G201 + my @strings = $filenamecsv =~ +/(.{1,4})_(.{1,6})_(.{1,6})_(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2}).*/; + $unittype = $strings[0]; + $unit = $strings[1]; + $fileDate = $strings[3] . "/" . $strings[4] . "/" . $strings[5]; + $fileTime = $strings[6] . ":" . $strings[7] . ":" . $strings[8]; + } + elsif ( $filenamecsv =~ m/^D2W_ID\d\d\d\d_DT\d\d\d\d$/i ) { + my @strings = $filenamecsv =~ /(.{1,3})_(.{1,6})_(.{1,6}).*/; + $unittype = $strings[0]; + $unit = $strings[1]; + } + else { + open FILE, $filename + or warn getTimeStamp("log") + . " >> Error: opening input file $filename\n"; + ( $fileDate, $fileTime ) = split( /\s/, ); + ( $unittype, $unit ) = split( /\s/, uc ); + $unit =~ s/;+$//; + close FILE; + } + return $unit; +} + +my $readingFile; +my $recvOKstr = "OK UPLOAD"; +my $ext = ".csv"; + +GetOptions( "file=s" => \$readingFile ) + or die("Error in command line arguments\n"); + +my ( $scriptname, $scriptpath ) = fileparse($0); + +my $tail = new File::SmartTail; +$tail->WatchFile( -file => $readingFile, -type => "UNIX", -timeout => '10' ); + +while ( my $line = $tail->GetLine() ) { + if ( index( $line, $recvOKstr ) != -1 ) { + my ( + undef, undef, undef, $truser, undef, + $trip, undef, $trfile, $trstat + ) = split( /[\"\[\]]/, $line ); + my ( $login, $pass, $uid, $gid ) = getpwnam($truser) + or warn getTimeStamp("log") . " >> $truser not in passwd file.\n"; + + my ( $filename, $path, $suffix ) = fileparse( $trfile, qr/\.[^.]*/ ); + $path = "/home/" . $truser . "/"; + $trfile = "/home/" . $truser . $trfile; + + if ( + ( ( uc $suffix ) eq ( uc $ext ) ) + and ( $filename =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d*|GD\d*)$/i + or $filename =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*$/i + or $filename =~ m/^D2W_ID\d\d\d\d_DT\d\d\d\d$/i ) + ) + { + my $unit = getUnitName($trfile); + print getTimeStamp("log") . " >> Unit $unit - Filename $trfile\n"; + + if ( !-d "$path/log" ) { + make_path "$path/log", + { mode => 0755, owner => $truser, group => $gid } + or warn getTimeStamp("log") + . " >> Failed to create path: $path/log"; + } + + my $outpath = $path . $unit; + if ( !-d "$outpath/SQL" ) { + make_path "$outpath/SQL", + { mode => 0755, owner => $truser, group => $gid } + or warn getTimeStamp("log") + . " >> Failed to create path: $outpath/SQL"; + } + + if ( !-d "$outpath/config" ) { + make_path "$outpath/config", + { mode => 0755, owner => $truser, group => $gid } + or warn getTimeStamp("log") + . " >> Failed to create path: $outpath/config"; + } + + my $timestamp = getTimeStamp(); + my $dest = $outpath . "/" . $filename . "_" . $timestamp . $suffix; + if ( !move $trfile, $dest ) { + warn getTimeStamp("log") + . " >> Move $trfile -> $dest failed: $!"; + } + else { + print getTimeStamp("log") . " >> Moved $trfile -> $dest.\n"; + chmod 0664, $dest; + my @fname = ($dest); + chown $uid, $gid, @fname; + if ( + ( + $filename =~ + m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)$/i + or $filename =~ m/^G201_ID\d\d\d\d_DT\d\d\d\d_\d*$/i + or $filename =~ m/^D2W_ID\d\d\d\d_DT\d\d\d\d$/i + ) + and ( $unit ne 'ID9999' ) + and ( $truser ne 'corra' ) + ) + { + print getTimeStamp("log") + . " >> Sender user $truser: load data into DB.\n"; + unless ( fork() ) { + setgid($gid); + setuid($uid); + $ENV{"HOME"} = 'home/' . $truser; + exec( $scriptpath + . "LoadCSVData.pl -f \"$dest\" -s \"$outpath/SQL/$filename" + . "_" + . "$timestamp.sql\" -d $truser >> /home/$truser/log/loadcsvdata.log 2>&1" + ); + exit(0); + } + if ( ( $truser ne 'aseftp' ) and ( $truser ne 'corra' ) ) { + print getTimeStamp("log") + . " >> Sender user $truser: duplicate as aseftp user -> load data into DB.\n"; + my $realtruser = $truser; + $truser = "aseftp"; + $outpath =~ s/$realtruser/$truser/; + my ( $login, $pass, $uid, $gid ) = getpwnam($truser) + or warn getTimeStamp("log") + . " >> $truser not in passwd file.\n"; + unless ( fork() ) { + setgid($gid); + setuid($uid); + $ENV{"HOME"} = 'home/' . $truser; + exec( $scriptpath + . "LoadCSVData.pl -f \"$dest\" -s \"$outpath/SQL/$filename" + . "_" + . "$timestamp.sql\" -d $truser >> /home/$truser/log/loadcsvdata.log 2>&1" + ); + exit(0); + } + } + } + } + } + } +} + +exit(0); diff --git a/mind/UnitCsvReceiverSmart_old.pl b/mind/UnitCsvReceiverSmart_old.pl new file mode 100755 index 0000000..f622f88 --- /dev/null +++ b/mind/UnitCsvReceiverSmart_old.pl @@ -0,0 +1,156 @@ +#!/usr/bin/perl +use warnings; +use strict; +use Getopt::Long; +use File::SmartTail; +use File::Basename qw( fileparse ); +use File::Path qw( make_path ); +use File::Copy qw( move ); +use POSIX; +$SIG{CHLD} = 'IGNORE'; + +$|++; # Autoflush + +sub getTimeStamp +{ # parm [ts] => timestamp for filename; log => timestamp for log + my $format = "%04d%02d%02d%02d%02d%02d"; + my ($p1) = @_; + if ( defined $p1 and $p1 eq "log" ) { + $format = "%04d%02d%02d %02d:%02d:%02d"; + } + my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = + localtime(time); + my $timestamp = + sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec ); + return $timestamp; +} + +sub getUnitName { # parm => file received ($trfile) + my ($filename) = @_; + my ( $fileDate, $fileTime ); + my ( $unittype, $unit ); + my ( $filenamecsv, $path, $suffix ) = fileparse( $filename, qr/\.[^.]*/ ); + if ( $filenamecsv =~ m/^(\d\d_\d\d\d\d_|)(\d\d\d\d\d\d\d\d\d\d\d\d\d\d_)(G201_ID\d\d\d\d_DT\d\d\d\d)$/i ) { # GD201 + my @strings = $filenamecsv =~ /(.{1,4})(.{1,2})(.{1,2})(.{1,2})(.{1,2})(.{1,2})_(.{1,4})_(.{1,6})_(.{1,6}).*/; + $fileDate = $strings[0] . "/" . $strings[1] . "/" . $strings[2];; + $fileTime = $strings[3] . ":" . $strings[4] . ":" . $strings[5]; + $unittype = $strings[6]; + $unit = $strings[7]; + } else { + open FILE, $filename + or warn getTimeStamp("log") . " >> Error: opening input file $filename\n"; + ( $fileDate, $fileTime ) = split( /\s/, ); + ( $unittype, $unit ) = split( /\s/, uc ); + $unit =~ s/;+$//; + close FILE; + } + return $unit; +} + +my $readingFile; +my $recvOKstr = "OK UPLOAD"; +my $ext = ".csv"; + +GetOptions( "file=s" => \$readingFile ) + or die("Error in command line arguments\n"); + +my ( $scriptname, $scriptpath ) = fileparse($0); + +my $tail = new File::SmartTail; +$tail->WatchFile( -file => $readingFile, -type => "UNIX", -timeout => '10' ); + +while ( my $line = $tail->GetLine() ) { + if ( index( $line, $recvOKstr ) != -1 ) { + my ( + undef, undef, undef, $truser, undef, + $trip, undef, $trfile, $trstat + ) = split( /[\"\[\]]/, $line ); + my ( $login, $pass, $uid, $gid ) = getpwnam($truser) + or warn getTimeStamp("log") . " >> $truser not in passwd file.\n"; + + my ( $filename, $path, $suffix ) = fileparse( $trfile, qr/\.[^.]*/ ); + if ( $path eq "/" ) { + $path = "/home/" . $truser . "/"; + $trfile = "/home/" . $truser . $trfile; + } + + if ( ( ( uc $suffix ) eq ( uc $ext ) ) + and + ( $filename =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d*|GD\d*)$/i + or $filename =~ m/^(\d\d_\d\d\d\d_|)(\d\d\d\d\d\d\d\d\d\d\d\d\d\d_)(G201_ID\d\d\d\d_DT\d\d\d\d)$/i) ) + { + my $unit = getUnitName($trfile); + print getTimeStamp("log") . " >> Unit $unit - Filename $trfile\n"; + + if ( !-d "$path/log" ) { + make_path "$path/log", + { mode => 0755, owner => $truser, group => $gid } + or warn getTimeStamp("log") + . " >> Failed to create path: $path/log"; + } + + my $outpath = $path . $unit; + if ( !-d "$outpath/SQL" ) { + make_path "$outpath/SQL", + { mode => 0755, owner => $truser, group => $gid } + or warn getTimeStamp("log") + . " >> Failed to create path: $outpath/SQL"; + } + + my $timestamp = getTimeStamp(); + my $dest = $outpath . "/" . $filename . "_" . $timestamp . $suffix; + if ( !move $trfile, $dest ) { + warn getTimeStamp("log") + . " >> Move $trfile -> $dest failed: $!"; + } + else { + print getTimeStamp("log") . " >> Moved $trfile -> $dest.\n"; + chmod 0664, $dest; + my @fname = ($dest); + chown $uid, $gid, @fname; + if ( ($filename =~ m/^(\d\d_\d\d\d\d_|)(DT\d\d\d\d|LOC\d.*|GD\d*)$/i + or $filename =~ m/^(\d\d_\d\d\d\d_|)(\d\d\d\d\d\d\d\d\d\d\d\d\d\d_)(G201_ID\d\d\d\d_DT\d\d\d\d)$/i) + and ( $unit ne 'ID9999' ) + and ( $truser ne 'corra' ) ) + { + print getTimeStamp("log") + . " >> Sender user $truser: load data into DB.\n"; + unless ( fork() ) { + setgid($gid); + setuid($uid); + $ENV{"HOME"} = 'home/' . $truser; + exec( $scriptpath + . "LoadCSVData.pl -f \"$dest\" -s \"$outpath/SQL/$filename" + . "_" + . "$timestamp.sql\" -d $truser >> /home/$truser/log/loadcsvdata.log 2>&1" + ); + exit(0); + } + if ( ( $truser ne 'aseftp' ) and ( $truser ne 'corra' ) ) { + print getTimeStamp("log") + . " >> Sender user $truser: duplicate as aseftp user -> load data into DB.\n"; + my $realtruser = $truser; + $truser = "aseftp"; + $outpath =~ s/$realtruser/$truser/; + my ( $login, $pass, $uid, $gid ) = getpwnam($truser) + or warn getTimeStamp("log") + . " >> $truser not in passwd file.\n"; + unless ( fork() ) { + setgid($gid); + setuid($uid); + $ENV{"HOME"} = 'home/' . $truser; + exec( $scriptpath + . "LoadCSVData.pl -f \"$dest\" -s \"$outpath/SQL/$filename" + . "_" + . "$timestamp.sql\" -d $truser >> /home/$truser/log/loadcsvdata.log 2>&1" + ); + exit(0); + } + } + } + } + } + } +} + +exit(0);