Commit b7369761 authored by Leigh B Stoller's avatar Leigh B Stoller

Another round of changes, close to ready.

parent 578e7233
...@@ -33,8 +33,10 @@ use Getopt::Std; ...@@ -33,8 +33,10 @@ use Getopt::Std;
use Data::Dumper; use Data::Dumper;
use Date::Parse; use Date::Parse;
use IO::Socket::INET; use IO::Socket::INET;
use File::Temp qw(tempdir);
use POSIX qw(strftime); use POSIX qw(strftime);
use POSIX ":sys_wait_h"; use POSIX ":sys_wait_h";
use JSON;
# #
# Take input from the RF monitor running on the control nucs, compare # Take input from the RF monitor running on the control nucs, compare
...@@ -45,9 +47,10 @@ sub usage() ...@@ -45,9 +47,10 @@ sub usage()
print "Usage: rfmonitor_daemon [-d] [-n] [-p port]\n"; print "Usage: rfmonitor_daemon [-d] [-n] [-p port]\n";
exit(1); exit(1);
} }
my $optlist = "dnp:"; my $optlist = "dnp:m";
my $debug = 0; my $debug = 0;
my $impotent = 0; my $impotent = 0;
my $mailonly = 0;
my $PORT = 12237; my $PORT = 12237;
my $HOST = "@POWDER_RFMONITOR_HOST@"; my $HOST = "@POWDER_RFMONITOR_HOST@";
...@@ -61,7 +64,7 @@ my $POWDER_RFMONITOR = @POWDER_RFMONITOR@; ...@@ -61,7 +64,7 @@ my $POWDER_RFMONITOR = @POWDER_RFMONITOR@;
my $LOGFILE = "$TB/log/rfmonitor_daemon.log"; my $LOGFILE = "$TB/log/rfmonitor_daemon.log";
my $POWER = "$TB/bin/power"; my $POWER = "$TB/bin/power";
my $FORMAT = "portid,timestamp,frequency,power"; my $FORMAT = "portid,timestamp,frequency,power";
my $MAILDELAY = 10; my $MAILDELAY = 60;
# un-taint path # un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin'; $ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
...@@ -95,6 +98,9 @@ if (! getopts($optlist, \%options)) { ...@@ -95,6 +98,9 @@ if (! getopts($optlist, \%options)) {
if (defined($options{"d"})) { if (defined($options{"d"})) {
$debug = 1; $debug = 1;
} }
if (defined($options{"m"})) {
$mailonly = 1;
}
if (defined($options{"n"})) { if (defined($options{"n"})) {
$impotent = 1; $impotent = 1;
} }
...@@ -126,6 +132,11 @@ if (! $impotent) { ...@@ -126,6 +132,11 @@ if (! $impotent) {
} }
} }
#
# Tempdir for storing data between messages. Removed at exit.
#
my $TEMPDIR = tempdir("/tmp/rfmonitor.XXXXX", CLEANUP => 1);
# #
# Setup a signal handler for newsyslog. # Setup a signal handler for newsyslog.
# #
...@@ -219,6 +230,49 @@ sub HandleChild($) ...@@ -219,6 +230,49 @@ sub HandleChild($)
my @lines = (); my @lines = ();
my %rflimits = (); my %rflimits = ();
my %violaters = (); my %violaters = ();
my $datafile = "$TEMPDIR/$address";
my $prevdata = {};
#
# Read in the previous run data.
#
if (-e $datafile) {
if (-s $datafile) {
my $stuff = emutil::ReadFile($datafile);
if (!defined($stuff) || $stuff eq "") {
print STDERR "$datafile has no data\n";
}
else {
my $tmp = eval { decode_json($stuff) };
if ($@) {
print STDERR "Could not decode json data: $stuff\n";
}
else {
$prevdata = $tmp;
}
}
}
# Will write a new one at the end.
unlink($datafile);
#
# Prune out anything that looks stale.
#
my $tmp = {};
foreach my $key (keys(%{$prevdata})) {
my $measurement = $prevdata->{$key};
my $stamp = $measurement->{'timestamp'};
next
if (time() - $stamp > 60);
$tmp->{$key} = $measurement;
}
$prevdata = $tmp;
}
print "Previous data:\n";
print Dumper($prevdata);
my $processLine = sub { my $processLine = sub {
my ($line) = @_; my ($line) = @_;
...@@ -257,18 +311,12 @@ sub HandleChild($) ...@@ -257,18 +311,12 @@ sub HandleChild($)
print STDERR "Improper nodeid from $address:$port: $nodeid\n"; print STDERR "Improper nodeid from $address:$port: $nodeid\n";
return undef; return undef;
} }
my $node = Node->Lookup($nodeid);
if (!$node) {
print STDERR "No such node $nodeid\n";
next;
}
return { return {
"node" => $node,
"node_id" => $nodeid, "node_id" => $nodeid,
"iface" => $iface, "iface" => $iface,
"frequency" => $frequency, "frequency" => $frequency,
"power" => $power, "power" => $power,
"timestamp" => $timestamp, "timestamp" => int($timestamp),
}; };
}; };
...@@ -277,7 +325,7 @@ sub HandleChild($) ...@@ -277,7 +325,7 @@ sub HandleChild($)
push(@lines, $line); push(@lines, $line);
print $line . "\n" print $line . "\n"
if ($debug); if ($debug > 1);
} }
# Close the connection, we do not tell the other side anything. # Close the connection, we do not tell the other side anything.
...@@ -305,12 +353,18 @@ sub HandleChild($) ...@@ -305,12 +353,18 @@ sub HandleChild($)
my $measurement = &$processLine($line); my $measurement = &$processLine($line);
next next
if (!defined($measurement)); if (!defined($measurement));
my $node = $measurement->{"node"};
my $nodeid = $measurement->{"node_id"}; my $nodeid = $measurement->{"node_id"};
my $iface = $measurement->{"iface"}; my $iface = $measurement->{"iface"};
my $frequency = $measurement->{"frequency"} + 0.0; my $frequency = $measurement->{"frequency"} + 0.0;
my $power = $measurement->{"power"} + 0.0; my $power = $measurement->{"power"} + 0.0;
my $timestamp = $measurement->{"timestamp"}; my $timestamp = $measurement->{"timestamp"};
my $key = "${nodeid}:${iface}:${frequency}";
my $node = Node->Lookup($nodeid);
if (!$node) {
print STDERR "No such node $nodeid\n";
next;
}
# #
# This is bogus, I was expecting the monitor to only send me # This is bogus, I was expecting the monitor to only send me
...@@ -331,7 +385,8 @@ sub HandleChild($) ...@@ -331,7 +385,8 @@ sub HandleChild($)
# to turn off a node that is already off, is fine, but lets avoid # to turn off a node that is already off, is fine, but lets avoid
# a bunch of email noise. # a bunch of email noise.
# #
if ($node->eventstate() eq TBDB_NODESTATE_POWEROFF()) { if (!$impotent &&
$node->eventstate() eq TBDB_NODESTATE_POWEROFF()) {
my $stamp = $node->state_timestamp(); my $stamp = $node->state_timestamp();
if (time() - $stamp < 180) { if (time() - $stamp < 180) {
print "$nodeid was just powered off, ignoring this report.\n"; print "$nodeid was just powered off, ignoring this report.\n";
...@@ -360,14 +415,28 @@ sub HandleChild($) ...@@ -360,14 +415,28 @@ sub HandleChild($)
} }
$rflimits{$nodeid} = $limits; $rflimits{$nodeid} = $limits;
} }
my $limits;
if (exists($rflimits{$nodeid}->{$iface}) &&
scalar(@{$rflimits{$nodeid}->{$iface}})) {
$limits = $rflimits{$nodeid}->{$iface};
}
# #
# If no RF limit for the interface, its a violation. # If no RF limit for the interface, its a violation.
# #
if (!exists($rflimits{$nodeid}->{$iface})) { if (!defined($limits)) {
if (!exists($violaters{$nodeid})) { # Ignore sample first time we see it.
$violaters{$nodeid} = {}; if (exists($prevdata->{$key})) {
if (!exists($violaters{$nodeid})) {
$violaters{$nodeid} = {};
}
if (!exists($violaters{$nodeid}->{$iface})) {
$violaters{$nodeid}->{$iface} = [];
}
push(@{$violaters{$nodeid}->{$iface}}, $measurement);
} }
$violaters{$nodeid}->{$iface} = $measurement; # Remember for next time
$prevdata->{$key} = $measurement;
next; next;
} }
# #
...@@ -376,24 +445,51 @@ sub HandleChild($) ...@@ -376,24 +445,51 @@ sub HandleChild($)
# #
my $inrange = 0; my $inrange = 0;
foreach my $limit (@{$rflimits{$nodeid}->{$iface}}) { if (defined($limits)) {
if ($measurement->{"frequency"} >= $limit->{"freq_low"} && foreach my $limit (@{$limits}) {
$measurement->{"frequency"} <= $limit->{"freq_high"} && if ($measurement->{"frequency"} >= $limit->{"freq_low"} &&
$measurement->{"power"} <= $limit->{"power"}) { $measurement->{"frequency"} <= $limit->{"freq_high"} &&
$inrange = 1; # Not doing power comparison yet.
last; (1 || $measurement->{"power"} <= $limit->{"power"})) {
$inrange = 1;
last;
}
} }
} }
if (!$inrange) { if (!$inrange) {
if (!exists($violaters{$nodeid})) { # Ignore sample first time we see it.
$violaters{$nodeid} = {}; if (exists($prevdata->{$key})) {
if (!exists($violaters{$nodeid})) {
$violaters{$nodeid} = {};
}
if (!exists($violaters{$nodeid}->{$iface})) {
$violaters{$nodeid}->{$iface} = [];
}
push(@{$violaters{$nodeid}->{$iface}}, $measurement);
} }
$violaters{$nodeid}->{$iface} = $measurement; # Remember for next time
$prevdata->{$key} = $measurement;
} }
} }
if (keys(%violaters)) { if (keys(%violaters)) {
HandleViolations(\%violaters, \%rflimits); HandleViolations(\%violaters, \%rflimits);
} }
#
# Write back the new previous data for next time
#
print "New previous data:\n";
print Dumper($prevdata);
my $string = eval { encode_json($prevdata) };
if ($@) {
print STDERR "Could not encode json data\n";
}
elsif (open(PREV, ">$datafile")) {
print PREV $string;
close(PREV);
}
else {
print STDERR "Could not open $datafile for writing: $!\n";
}
print "Finished with $address:$port at " . print "Finished with $address:$port at " .
POSIX::strftime("%m/%d %H:%M:%S", localtime()) . "\n"; POSIX::strftime("%m/%d %H:%M:%S", localtime()) . "\n";
exit(0); exit(0);
...@@ -453,17 +549,25 @@ sub HandleViolations($$) ...@@ -453,17 +549,25 @@ sub HandleViolations($$)
my $experiment = $node->Reservation(); my $experiment = $node->Reservation();
my $project = $experiment->GetProject(); my $project = $experiment->GetProject();
my $swapper = $experiment->GetSwapper(); my $swapper = $experiment->GetSwapper();
my $creator = $experiment->GetCreator();
my $user_name = $swapper->name(); my $user_name = $swapper->name();
my $user_email = $swapper->email(); my $user_email = $swapper->email();
my $leader = $project->GetLeader(); my $leader = $project->GetLeader();
my $pname = $leader->name(); my $pname = $leader->name();
my $pemail = $leader->email(); my $pemail = $leader->email();
#
# Nonlocal project (on the geni path), its the creator we want.
# (The swapper is always the geniuser).
#
if ($project->IsNonLocal()) {
$user_name = $creator->name();
$user_email = $creator->email();
}
if ($debug) { if ($debug) {
$pname = $user_name; $pname = $user_name;
$pemail = $user_email; $pemail = $user_email;
} }
$TO = "$user_name <$user_email>"; $TO = "$user_name <$user_email>";
$CC = "$pname <$pemail>, $TBOPS"; $CC = "$pname <$pemail>, $TBOPS";
} }
...@@ -474,33 +578,36 @@ sub HandleViolations($$) ...@@ -474,33 +578,36 @@ sub HandleViolations($$)
$body = "The following interfaces are in violation on $nodeid:\n\n"; $body = "The following interfaces are in violation on $nodeid:\n\n";
foreach my $iface (keys(%{$violations->{$nodeid}})) { foreach my $iface (keys(%{$violations->{$nodeid}})) {
my $measurement = $violations->{$nodeid}->{$iface}; foreach my $measurement (@{$violations->{$nodeid}->{$iface}}) {
my $stamp = POSIX::strftime("%m/%d %H:%M:%S %Z", my $stamp = POSIX::strftime("%m/%d %H:%M:%S %Z",
localtime($measurement->{"timestamp"})); localtime($measurement->{"timestamp"}));
if (!exists($rflimits->{$nodeid}->{$iface})) { if (!exists($rflimits->{$nodeid}->{$iface})) {
$body .= "$iface: NO LIMITS DEFINED\n"; $body .= "$iface: NO LIMITS DEFINED\n";
} }
else { else {
$body .= "$iface:\n"; $body .= "$iface:\n";
my @limits = @{$rflimits->{$nodeid}->{$iface}}; my @limits = @{$rflimits->{$nodeid}->{$iface}};
foreach my $limit (@limits) { foreach my $limit (@limits) {
my $freq_low = $limit->{"freq_low"}; my $freq_low = $limit->{"freq_low"};
my $freq_high = $limit->{"freq_high"}; my $freq_high = $limit->{"freq_high"};
my $power = $limit->{"power"}; my $power = $limit->{"power"};
$body .= $body .=
" Limits: Min Freq:$freq_low MHZ, ". " Limits: Min Freq:$freq_low MHZ, ".
"Max Freq:$freq_high MHZ, Max Power:$power dBm\n"; "Max Freq:$freq_high MHZ, Max Power:$power dBm\n";
}
} }
$body .=
" Measured at $stamp: ".
"Frequency: " . $measurement->{"frequency"} . " MHZ, ".
"Power: " . $measurement->{"power"} . " dBm\n";
} }
$body .=
" Measured at $stamp: ".
"Frequency: " . $measurement->{"frequency"} . " MHZ, ".
"Power: " . $measurement->{"power"} . " dBm\n";
} }
$body .= "\n" . "This node will be immediately powered off!\n"; if (!$mailonly) {
$body .= "\n" . "This node will be immediately powered off!\n";
}
if ($debug || $impotent) { if ($debug || $impotent) {
print $subject . "\n"; print $subject . "\n";
print $body; print $body;
...@@ -513,7 +620,7 @@ sub HandleViolations($$) ...@@ -513,7 +620,7 @@ sub HandleViolations($$)
libtestbed::SENDMAIL($TO, $subject, $body, $TBOPS, $headers); libtestbed::SENDMAIL($TO, $subject, $body, $TBOPS, $headers);
NodeNotified($nodeid, "user", $body); NodeNotified($nodeid, "user", $body);
} }
if (!$impotent) { if (!($impotent || $mailonly)) {
# #
# We are allowed to run the power command as root. # We are allowed to run the power command as root.
# #
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment