alex/ase_perl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

user e pwd x db company

Browse Source
This commit is contained in:
battilo
2017-09-17 23:23:58 +02:00
parent 980477fdfd
commit 151149c155
1 changed files with 342 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

251
LoadCSVData.pl
View File

@@ -13,6 +13,7 @@ my ($filename, $sqlout);
my $username = 'ase_lar'; my $username = 'ase_lar';
my $password = 'laravel'; my $password = 'laravel';
my $db_lar = 'ase_lar'; my $db_lar = 'ase_lar';
my $db_sp = 'ase_lar';
my $server = 'localhost'; my $server = 'localhost';
my $db_name = "ase_mums"; my $db_name = "ase_mums";
my $table = "RAWDATA"; my $table = "RAWDATA";
@@ -20,7 +21,10 @@ my (@CSVData, @sql);
my $outcount = 0; my $outcount = 0;
my $outfilenum = 0; my $outfilenum = 0;
my $maxsqllines = 10000; my $maxsqllines = 10000;
my ($tool, $toolbatt, $tooltemp, $tooltype, @channels, @NewData, @din, @nodetype); my (
$tool, $toolbatt, $tooltemp, $tooltype,
@channels, @NewData, @din, @nodetype
);
my $matlab_ip = "160.78.21.71"; my $matlab_ip = "160.78.21.71";
my $matlab_cmd = "matlab.cmd"; my $matlab_cmd = "matlab.cmd";
my $matlab_timestamp = ""; my $matlab_timestamp = "";
@@ -33,7 +37,8 @@ my $ftp_filename;
my $ftp_target; my $ftp_target;
my $unit_id; my $unit_id;
sub getTimeStamp { # parm [ts] => timestamp for filename; log => timestamp for log sub getTimeStamp
{ # parm [ts] => timestamp for filename; log => timestamp for log
my $format = "%04d%02d%02d%02d%02d%02d"; my $format = "%04d%02d%02d%02d%02d%02d";
my ($p1) = @_; my ($p1) = @_;
if ( defined $p1 and $p1 eq "log" ) { if ( defined $p1 and $p1 eq "log" ) {
@@ -42,28 +47,49 @@ sub getTimeStamp { # parm [ts] => timestamp for filename; log => timestamp for l
if ( defined $p1 and $p1 eq "db_ts" ) { if ( defined $p1 and $p1 eq "db_ts" ) {
$format = "%04d-%02d-%02d %02d:%02d:%02d"; $format = "%04d-%02d-%02d %02d:%02d:%02d";
} }
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
my $timestamp = sprintf ( $format, $year+1900,$mon+1,$mday,$hour,$min,$sec); localtime(time);
my $timestamp =
sprintf( $format, $year + 1900, $mon + 1, $mday, $hour, $min, $sec );
return $timestamp; return $timestamp;
} }
sub writeOutSql { sub writeOutSql {
my ($tn) = @_; my ($tn) = @_;
my $outfile = $sqlout . "_" . $outfilenum++; my $outfile = $sqlout . "_" . $outfilenum++;
open SQLOUT, ">", $outfile or die(getTimeStamp("log") . " - pid $$ >> Error: opening output file " . $outfile . ":$!\n"); open SQLOUT, ">", $outfile
or die( getTimeStamp("log")
. " - pid $$ >> Error: opening output file "
. $outfile
. ":$!\n" );
print getTimeStamp("log") . " - pid $$ >> file $outfile created\n"; print getTimeStamp("log") . " - pid $$ >> file $outfile created\n";
print SQLOUT "/* lock " . $db_name . ".$table table */\nLOCK TABLES " . $db_name . ".$table WRITE;\n"; print SQLOUT "/* lock "
. $db_name
. ".$table table */\nLOCK TABLES "
. $db_name
. ".$table WRITE;\n";
print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n"; print SQLOUT "INSERT IGNORE INTO " . $db_name . ".$table\n";
print SQLOUT "(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n"; print SQLOUT
print SQLOUT "`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n"; "(`UnitName`,`ToolNameID`,`NodeNum`,`EventDate`,`EventTime`,`BatLevel`,`Temperature`,\n";
print SQLOUT
"`Val0`,`Val1`,`Val2`,`Val3`,`Val4`,`Val5`,`Val6`,`Val7`,`Val8`,`Val9`,`ValA`,`ValB`,`ValC`,`ValD`,`ValE`,`ValF`)\n";
print SQLOUT "VALUES\n"; print SQLOUT "VALUES\n";
print SQLOUT @sql; print SQLOUT @sql;
@sql = (); @sql = ();
print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n"; print SQLOUT "/* unlock table */\nUNLOCK TABLES;\n";
print SQLOUT "/* call stored procedure to copy in ase_lar */\ncall ase_lar.CopyAndFill('$tn');\n"; print SQLOUT "/* call stored procedure to copy in "
close SQLOUT or die(getTimeStamp("log") . " - pid $$ >> Error in closing file " . $outfile . "\n"); . $db_sp
. "*/\ncall "
. $db_sp
. ".CopyAndFill('$tn');\n";
close SQLOUT
or die( getTimeStamp("log")
. " - pid $$ >> Error in closing file "
. $outfile
. "\n" );
my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile ); my @args = ( "mysql", "--login-path=asepath", "-e source " . $outfile );
system(@args) == 0 or die (getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n"); system(@args) == 0
or die( getTimeStamp("log") . " - pid $$ >> system @args failed: $?\n" );
print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n"; print getTimeStamp("log") . " - pid $$ >> file $outfile loaded into DB\n";
unlink $outfile; unlink $outfile;
return; return;
@@ -74,23 +100,30 @@ sub matlabCalc {
$matlab_timestamp = getTimeStamp("db_ts"); $matlab_timestamp = getTimeStamp("db_ts");
print getTimeStamp("log") . " - pid $$ >> $tool MatLab calc started...\n"; print getTimeStamp("log") . " - pid $$ >> $tool MatLab calc started...\n";
my @out_matlab = `ssh1 $matlab_ip 'cmd /c $matlab_cmd ' $tool`; my @out_matlab = `ssh1 $matlab_ip 'cmd /c $matlab_cmd ' $tool`;
print getTimeStamp("log") . " - pid $$ >> $tool MatLab calc executed: $out_matlab[-1]"; print getTimeStamp("log")
. " - pid $$ >> $tool MatLab calc executed: $out_matlab[-1]";
#print getTimeStamp("log") . " - @out_matlab"; #print getTimeStamp("log") . " - @out_matlab";
#print getTimeStamp("log") . " - pid $$ >> $out_matlab[-1]"; #print getTimeStamp("log") . " - pid $$ >> $out_matlab[-1]";
return; return;
} }
sub trxelab { sub trxelab {
my $dbh = DBI->connect("DBI:mysql:$db_lar;host=$server", $username, $password) my $dbh =
or die getTimeStamp("log") . " - Could not connect to database: $DBI::errstr"; DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare('select ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename, ftp_target from units my $sth = $dbh->prepare(
where id = ' . $unit_id . ';') or die getTimeStamp("log") . " - $DBI::errstr"; 'select ftp_addrs, ftp_user, ftp_passwd, ftp_parm, ftp_filename, ftp_target from units
where id = ' . $unit_id . ';'
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute(); $sth->execute();
if ( $sth->rows < 0 ) { if ( $sth->rows < 0 ) {
die getTimeStamp("log") . " - No unit ftp params selected.\n"; die getTimeStamp("log") . " - pid $$ >> No unit ftp params selected.\n";
} else { }
else {
while ( my $results = $sth->fetchrow_hashref ) { while ( my $results = $sth->fetchrow_hashref ) {
$ftp_addrs = $results->{'ftp_addrs'}; $ftp_addrs = $results->{'ftp_addrs'};
$ftp_user = $results->{'ftp_user'}; $ftp_user = $results->{'ftp_user'};
@@ -102,18 +135,28 @@ sub trxelab {
} }
$sth->finish; $sth->finish;
# Disconnect # Disconnect
my $fileelab; my $fileelab;
if ( $ftp_filename eq "" ) { if ( $ftp_filename eq "" ) {
$fileelab = '/var/lib/mysql-files/' . $tool . '_' . getTimeStamp() . '.csv'; #mettere quello che si prende da query $fileelab =
} else { '/var/lib/mysql-files/'
. $tool . '_'
. getTimeStamp()
. '.csv'; #mettere quello che si prende da query
}
else {
$ftp_filename =~ s/(\$\w+)/$1/eeg; $ftp_filename =~ s/(\$\w+)/$1/eeg;
$ftp_filename =~ s/\s/_/g; $ftp_filename =~ s/\s/_/g;
$fileelab = '/var/lib/mysql-files/' . $ftp_filename . '_' . getTimeStamp() . '.csv'; $fileelab =
'/var/lib/mysql-files/'
. $ftp_filename . '_'
. getTimeStamp() . '.csv';
} }
my $sthdo = $dbh->do( qq{ my $sthdo = $dbh->do(
qq{
select * from ( select * from (
( (
select 'ToolNameID', 'EventDate', 'EventTime', 'NodeNum', 'NodeType', 'NodeDepth', select 'ToolNameID', 'EventDate', 'EventTime', 'NodeNum', 'NodeType', 'NodeDepth',
@@ -132,51 +175,68 @@ sub trxelab {
) resulting_set ) resulting_set
into outfile '$fileelab' into outfile '$fileelab'
fields terminated by ',' optionally enclosed by '"' lines terminated by '\n'}) or die getTimeStamp("log") . " - $DBI::errstr"; fields terminated by ',' optionally enclosed by '"' lines terminated by '\n'}
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$dbh->disconnect; $dbh->disconnect;
$ftp_parm //= ""; $ftp_parm //= "";
if ( $ftp_parm ne "" ) { if ( $ftp_parm ne "" ) {
$ftp_parm = "debug => 0, " . $ftp_parm; $ftp_parm = "debug => 0, " . $ftp_parm;
} else { }
else {
$ftp_parm = "debug => 0"; $ftp_parm = "debug => 0";
} }
print getTimeStamp("log") . " - pid $$ >> $tool: ftp parm = $ftp_parm.\n"; print getTimeStamp("log") . " - pid $$ >> $tool: ftp parm = $ftp_parm.\n";
my $ftp = Net::FTP->new( $ftp_addrs, debug => 0 ) my $ftp = Net::FTP->new( $ftp_addrs, debug => 0 )
or die getTimeStamp("log") . " - pid $$ >> Cannot connect to $ftp_addrs: $@ \n"; or die getTimeStamp("log")
. " - pid $$ >> Cannot connect to $ftp_addrs: $@ \n";
$ftp->login( $ftp_user, $ftp_passwd ) $ftp->login( $ftp_user, $ftp_passwd )
or die getTimeStamp("log") . " - pid $$ >> Cannot login " . $ftp->message . "\n"; or die getTimeStamp("log")
. " - pid $$ >> Cannot login "
. $ftp->message . "\n";
$ftp_target //= ""; $ftp_target //= "";
if ( $ftp_target ne "" ) { if ( $ftp_target ne "" ) {
$ftp->cwd($ftp_target) $ftp->cwd($ftp_target)
or die getTimeStamp("log") . " - pid $$ >> Change remote dir failed " . $ftp->message . "\n"; or die getTimeStamp("log")
. " - pid $$ >> Change remote dir failed "
. $ftp->message . "\n";
} }
print getTimeStamp("log") . " - pid $$ >> ftp target: " . $ftp->pwd() . ".\n"; print getTimeStamp("log")
. " - pid $$ >> ftp target: "
. $ftp->pwd() . ".\n";
$ftp->put($fileelab) $ftp->put($fileelab)
or die getTimeStamp("log") . " - pid $$ >> Put failed " . $ftp->message . "\n"; or die getTimeStamp("log")
. " - pid $$ >> Put failed "
. $ftp->message . "\n";
$ftp->quit; $ftp->quit;
print getTimeStamp("log") . " - pid $$ >> $tool: $fileelab ftp put executed.\n"; print getTimeStamp("log")
. " - pid $$ >> $tool: $fileelab ftp put executed.\n";
return; return;
} }
sub getNodesType { sub getNodesType {
my $dbh = DBI->connect("DBI:mysql:$db_lar;host=$server", $username, $password) my $dbh =
or die getTimeStamp("log") . " - Could not connect to database: $DBI::errstr"; DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare('select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.din as din my $sth = $dbh->prepare(
'select t.name as name, n.seq as seq, n.num as num, n.channels as channels, y.type as type, n.din as din
from nodes as n from nodes as n
inner join tools as t on t.id = n.tool_id inner join tools as t on t.id = n.tool_id
inner join nodetypes as y on n.nodetype_id = y.id inner join nodetypes as y on n.nodetype_id = y.id
where t.name = "' . $tool . '" order by 3;') or die getTimeStamp("log") . " - $DBI::errstr"; where t.name = "' . $tool . '" order by 3;'
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute(); $sth->execute();
if ( $sth->rows < 0 ) { if ( $sth->rows < 0 ) {
die getTimeStamp("log") . " - No node channels selected.\n"; die getTimeStamp("log") . " - pid $$ >> No node channels selected.\n";
} else { }
else {
my $row = 0; my $row = 0;
while ( my $results = $sth->fetchrow_hashref ) { while ( my $results = $sth->fetchrow_hashref ) {
$channels[$row] = $results->{'channels'}; $channels[$row] = $results->{'channels'};
@@ -186,23 +246,60 @@ sub getNodesType {
} }
$sth->finish; $sth->finish;
# Disconnect
$dbh->disconnect;
return;
}
sub getUdbPwd {
my $dbh =
DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare(
"SELECT db_pwd FROM ase_lar.companies where db_name = '"
. $db_name
. "';" )
or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute();
if ( $sth->rows < 0 ) {
die getTimeStamp("log") . " - pid $$ >> No password selected.\n";
}
else {
my $mfunc;
while ( my $results = $sth->fetchrow_hashref ) {
$password = $results->{'db_pwd'};
$username = $results->{'db_user'};
}
}
$sth->finish;
# Disconnect # Disconnect
$dbh->disconnect; $dbh->disconnect;
return; return;
} }
sub getMatlabCmd { sub getMatlabCmd {
my $dbh = DBI->connect("DBI:mysql:$db_lar;host=$server", $username, $password) my $dbh =
or die getTimeStamp("log") . " - Could not connect to database: $DBI::errstr"; DBI->connect( "DBI:mysql:$db_lar;host=$server", $username, $password )
or die getTimeStamp("log")
. " - pid $$ >> Could not connect to database: $DBI::errstr";
my $sth = $dbh->prepare('select m.matcall, t.ftp_send , t.unit_id from matfuncs as m my $sth = $dbh->prepare(
'select m.matcall, t.ftp_send , t.unit_id from matfuncs as m
inner join tools as t on t.matfunc = m.id inner join tools as t on t.matfunc = m.id
where t.name = "' . $tool . '" ;') or die getTimeStamp("log") . " - $DBI::errstr"; where t.name = "' . $tool . '" ;'
) or die getTimeStamp("log") . " - pid $$ >> $DBI::errstr";
$sth->execute(); $sth->execute();
if ( $sth->rows < 0 ) { if ( $sth->rows < 0 ) {
die getTimeStamp("log") . " - No tool's matlab function selected.\n"; die getTimeStamp("log")
} else { . " - pid $$ >> No tool's matlab function selected.\n";
}
else {
my $mfunc; my $mfunc;
while ( my $results = $sth->fetchrow_hashref ) { while ( my $results = $sth->fetchrow_hashref ) {
$mfunc = uc $results->{'matcall'}; $mfunc = uc $results->{'matcall'};
@@ -213,6 +310,7 @@ sub getMatlabCmd {
} }
$sth->finish; $sth->finish;
# Disconnect # Disconnect
$dbh->disconnect; $dbh->disconnect;
return; return;
@@ -221,11 +319,22 @@ sub getMatlabCmd {
my $starttime = getTimeStamp("log"); my $starttime = getTimeStamp("log");
print "$starttime - pid $$ >> Start execution.\n"; print "$starttime - pid $$ >> Start execution.\n";
GetOptions ("filename=s" => \$filename, GetOptions(
"filename=s" => \$filename,
"sqlout=s" => \$sqlout, "sqlout=s" => \$sqlout,
"dbname=s" => \$db_name) "dbname=s" => \$db_name
or die("Error in command line arguments\n"); ) or die("Error in command line arguments\n");
open FILE, $filename or die("Error: opening input file " . $filename . "\n"); open FILE, $filename
or die( "Error: opening input file " . $filename . "\n" );
if ( $db_name ne 'asega' ) {
$db_name =~ s/dbn/dbu/;
$db_sp = $db_name;
getUdbPwd();
}
else {
$db_name = 'ase_mums';
}
my ( $fileDate, $fileTime ) = split( /\s/, <FILE> ); my ( $fileDate, $fileTime ) = split( /\s/, <FILE> );
my ( $unittype, $unit ) = split( /\s/, uc <FILE> ); my ( $unittype, $unit ) = split( /\s/, uc <FILE> );
@@ -238,38 +347,51 @@ while (my $line = <FILE>) {
if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) { if ( defined $info[3] && $info[3] =~ m/^DT\d\d\d\d$/i ) {
$tool = uc $info[3]; $tool = uc $info[3];
$tooltype = uc $info[2]; $tooltype = uc $info[2];
print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n"; print getTimeStamp("log")
getNodesType(); . " - pid $$ >> Found $tooltype tool name: $tool\n";
} elsif (defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) {
$tool= uc $info[2];
$tooltype = 'LOC';
print getTimeStamp("log") . " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType(); getNodesType();
} }
} else { elsif ( defined $info[2] && $info[2] =~ m/^LOC\d*$/i ) {
$tool = uc $info[2];
$tooltype = 'LOC';
print getTimeStamp("log")
. " - pid $$ >> Found $tooltype tool name: $tool\n";
getNodesType();
}
}
else {
$NodeData =~ s/\x0d{0,1}\x0a\Z//s; $NodeData =~ s/\x0d{0,1}\x0a\Z//s;
#$NodeData =~ s/\r\n\z//; #$NodeData =~ s/\r\n\z//;
if ( $tooltype eq "MUX" ) { if ( $tooltype eq "MUX" ) {
#$NodeData =~ s/Dis\./N\/A/g; #$NodeData =~ s/Dis\./N\/A/g;
@NewData = grep /\S/, split( /\||;/, $NodeData ); @NewData = grep /\S/, split( /\||;/, $NodeData );
my $idx = 0; my $idx = 0;
$CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) ); $CSVData[ $idx++ ] = join( ';', splice( @NewData, 0, 2 ) );
foreach my $ch (@channels) { foreach my $ch (@channels) {
$CSVData[$idx++] = join( ';', splice(@NewData, 0, $ch)); $CSVData[ $idx++ ] =
join( ';', splice( @NewData, 0, $ch ) );
} }
#print Dumper \@NewData; #print Dumper \@NewData;
#print Dumper \@din; #print Dumper \@din;
} elsif ($tooltype eq "LOC") { }
elsif ( $tooltype eq "LOC" ) {
if ( !defined $din[0] ) { if ( !defined $din[0] ) {
print getTimeStamp("log") . " - pid $$ >> Din not defined.\n"; print getTimeStamp("log")
print getTimeStamp("log") . " - pid $$ >> Execution ended.\n"; . " - pid $$ >> Din not defined.\n";
print getTimeStamp("log")
. " - pid $$ >> Execution ended.\n";
exit; exit;
}; }
my ($Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2) = split(/;/,$NodeData); my ( $Vbat, $Tmod, $Ain1, $Ain2, $Din1, $Din2 ) =
split( /;/, $NodeData );
$CSVData[0] = $Vbat . ";" . $Tmod; $CSVData[0] = $Vbat . ";" . $Tmod;
$CSVData[1] = $Din1; $CSVData[1] = $Din1;
} else { }
else {
@CSVData = split( /\|/, $NodeData ); @CSVData = split( /\|/, $NodeData );
} }
@@ -280,11 +402,13 @@ while (my $line = <FILE>) {
if ( $nodenum eq 0 ) { if ( $nodenum eq 0 ) {
$toolbatt = $data[0]; $toolbatt = $data[0];
$tooltemp = $data[1]; $tooltemp = $data[1];
} else { }
else {
my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first ); my ( $y, $m, $d, $t ) = split( /[\s\/]/, $first );
foreach (@data) { foreach (@data) {
if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) { if ( $sql[ ( $outcount % $maxsqllines ) ] eq "" ) {
$sql[($outcount%$maxsqllines)] = "('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'"; $sql[ ( $outcount % $maxsqllines ) ] =
"('$unit','$tool','$nodenum','$y-$m-$d','$t','$toolbatt','$tooltemp'";
} }
$sql[ ( $outcount % $maxsqllines ) ] .= ",'" . $_ . "'"; $sql[ ( $outcount % $maxsqllines ) ] .= ",'" . $_ . "'";
} }
@@ -293,7 +417,8 @@ while (my $line = <FILE>) {
if ( ++$outcount % $maxsqllines eq 0 ) { if ( ++$outcount % $maxsqllines eq 0 ) {
$sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n"; $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ";\n";
writeOutSql($tool); writeOutSql($tool);
} else { }
else {
$sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n"; $sql[ ( $outcount % $maxsqllines ) - 1 ] .= ",\n";
} }
} }