Commit caa4023a authored by Dan Gebhardt's avatar Dan Gebhardt

Initial changes to support tool generalization. All command-line

arguments and result parsing handled by separate application wrappers.
Currently fping and iperf supported.
Not fully functional at this point: user interface needs extending
to support the generalization, as well as getting the results back
to "opsrecv".

Also merged some of Mike's changes to allow for "private" managers
to use during testing.
parent 7db5d15b
......@@ -32,10 +32,10 @@ my %intersitenodes = (); #final list for fully-connected test
my @constrnodes; #test constrained to these nodes
my %sitenodes; #hash listing all sites => nodes
my $CPUUSAGETHRESHOLD = 10; #should help prevent flip-flopping between
#"best" nodes at a site
#TODO: document normal range of values for CPU
#"best" nodes at a site
#TODO: document normal range of values for CPU
my $SITEDIFFTHRESHOLD = 5; #number of site differences between period
#calculations that trigger an update
#calculations that trigger an update
my $IPERFDURATION = 5; #duration in seconds of iperf test
my %allnodes;
my %deadsites;
......@@ -677,6 +677,6 @@ sub getstatuswrapper($) {
$nodestatus{$node} = 0;
}
return $status;
}
}
......@@ -144,7 +144,7 @@ sub updateTestEvent($);
sub addCmd($$);
sub updateOutageState($);
sub updateOutageState_lossCheck($);
sub initTestEv($$);
sub initTestEv($$$$$$$);
sub getstats;
sub diffstats($$);
sub printstats($$);
......@@ -216,6 +216,21 @@ my $subtimer_reset = 10; # subtimer reaches 0 this many times thru poll loop
my $subtimer = $subtimer_reset; #decrement every poll-loop.
#
# XXX / TODO: This is a hack to keep the "outage detection"
# stuff working. Outage detection should probably be its own tool.
#
sub hacky_evalLatencyResult($){
my ($result) = @_;
my %eval_result = (split(/[=,]/, $result));
if( $eval_result{error} != 0 ){
return -1;
}else{
return $eval_result{latency};
}
}
sub handleincomingmsgs()
{
my $inmsg;
......@@ -271,7 +286,18 @@ sub handleincomingmsgs()
my $newtestper = $sockIn{testper};
my $duration =$sockIn{duration};
my $managerID=$sockIn{managerID};
initTestEv($linkdest,$testtype);
#XXX / TODO: How to handle tool parameterization if an
#EDIT comes in with a different toolname, etc..!
# -- what are the issues?
initTestEv(
$linkdest,$testtype,
$sockIn{toolname},
$sockIn{toolwrapperpath},
$sockIn{tooltype},
$sockIn{req_params},
$sockIn{opt_params}
);
my $testev = \%{ $testevents{$linkdest}{$testtype} };
# print time()." EDIT:\n";
......@@ -302,7 +328,15 @@ sub handleincomingmsgs()
}
my $offset = 0;
foreach my $linkdest (@destnodes){
initTestEv($linkdest,$testtype);
# initTestEv($linkdest,$testtype);
initTestEv(
$linkdest,$testtype,
$sockIn{toolname},
$sockIn{toolwrapperpath},
$sockIn{tooltype},
$sockIn{req_params},
$sockIn{opt_params}
);
my $testev = \%{ $testevents{$linkdest}{$testtype} };
#add new cmd to queue
addCmd( $testev,
......@@ -481,6 +515,9 @@ while (1) {
my @raw_lines;
#read raw results from temp file
# NOTE: parsing is now done in the wrapper, and we return the
# entire wrapper output
# this stuff left in for the "continuous hack" (for now)
my $filename = $testev->{"outfile"};
if (!$testev->{"flag_finished"} ||
!$testev->{"continuous"}) {
......@@ -519,10 +556,12 @@ while (1) {
}
#parse raw data
# print "RAW data = $raw\n";
my $parsedData = parsedata($testtype,
$testev->{"continuous"},
$raw);
$testev->{"results_parsed"} = $parsedData;
print "PARSED data = $parsedData\n";
my %results =
("sourceaddr" => $thismonaddr,
......@@ -762,16 +801,34 @@ sub spawnTest($$)
}elsif( defined $pid ){
#child
if( $testtype eq "latency" ||
$testtype eq "bw")
{
my $toolwrapperpath
= "$testevents{$linkdest}{$testtype}{toolwrapperpath}";
print "toolwrapperpath=$toolwrapperpath\n";
my $toolparams =
$testevents{$linkdest}{$testtype}{req_params} ." ".
# $testevents{$linkdest}{$testtype}{opt_params} .
" target $linkdest";
print "Running test: $toolwrapperpath $toolparams\n";
exec "sudo $toolwrapperpath $toolparams >$filename 2>&1"
or die "can't exec: $!\n";
}
=pod
#############################
###ADD MORE TEST TYPES HERE###
#############################
if( $testtype eq "latency" ){
#command line for "LATENCY TEST"
# print "##########latTest\n";
my $duration = 0;
my $period = $testevents{$linkdest}{$testtype}{"testper"};
if ($contin) {
$duration = int($testevents{$linkdest}{$testtype}{"limitTime"}) - time() + 1;
$duration = int($testevents{$linkdest}{$testtype}{"limitTime"})
- time() + 1;
if ($duration < 0 || $duration > (24*60*60)) {
$contin = 0;
}
......@@ -795,14 +852,15 @@ sub spawnTest($$)
exec "$workingdir".
"iperf -c $linkdest -t $iperfduration -p $iperfport >$filename"
or die "can't exec: $!";
}elsif( $testtype eq "outage" ){
}
=cut
elsif( $testtype eq "outage" ){
exec "$workingdir".
"fping -c16 -p250 $linkdest >& $filename"
or die "can't exec: $!\n";
}else{
warn "bad testtype: $testtype";
}
}elsif( $! == EAGAIN ){
#recoverable fork error, redo;
sleep 1;
......@@ -844,19 +902,7 @@ sub parsedata($$$)
#############################
#latency test
if( $type eq "latency" ){
# for fping results parsing, using -s option.
# Note, these must be in this order!
# TODO: get these regex's to work as OR logic...
=pod
if(
/ICMP Network Unreachable/ ||
/ICMP Host Unreachable from/ ||
/ICMP Protocol Unreachable/ ||
/ICMP Port Unreachable/ ||
/ICMP Unreachable/
=cut
if ($iscontin) {
# "traditional" ping output
if( /icmp_seq=\d+ ttl=\d+ time=([\d\.]+) ms/ ) {
......@@ -867,7 +913,9 @@ sub parsedata($$$)
$parsed = $ERRID{timeout};
}
} else {
$parsed = $raw;
# fping output
=pod
if( /^ICMP / )
{
$parsed = $ERRID{ICMPunreachable};
......@@ -880,6 +928,7 @@ sub parsedata($$$)
}else{
$parsed = $ERRID{unknown};
}
=cut
}
=pod
#this section of parsing is for linux ping hosts.
......@@ -906,6 +955,8 @@ sub parsedata($$$)
}
=cut
}elsif( $type eq "bw" ){
$parsed = $raw;
=pod
if( /connect failed: Connection timed out/ ){
# this one shouldn't happen, if the timeout check done by
# bgmon is set low enough.
......@@ -923,6 +974,7 @@ sub parsedata($$$)
$parsed = $ERRID{unknown};
}
# print "parsed=$parsed\n";
=cut
}elsif( $type eq "outage" ){
#print "parsing Outage data\n";
if( /loss = \d+\/\d+\/([\d.]+)%/ ){
......@@ -1153,7 +1205,9 @@ sub updateOutageState($)
my $testev = \%{ $testevents{$destaddr}{latency} };
my $curstate = $testev->{outagestate};
my $nextstate = $curstate;
if( $testev->{results_parsed} > 0 ){
# if( $testev->{results_parsed} > 0 ){
if( hacky_evalLatencyResult($testev->{results_parsed}) > 0 ){
#valid result, so note the time that this was seen
$testev->{lastValidLatTime} = time();
}
......@@ -1258,7 +1312,8 @@ sub isWorkingPathDown($)
{
my ($testev) = @_;
my $secSinceUp = time() - $testev->{lastValidLatTime};
if( $testev->{results_parsed} < 0 &&
# if( $testev->{results_parsed} < 0 &&
if( hacky_evalLatencyResult($testev->{results_parsed}) < 0 &&
$secSinceUp < $outDet_maxPastSuc*60*60 )
{
#path is down and was up recently
......@@ -1331,9 +1386,10 @@ sub addCmd($$)
updateTestEvent( $testev );
}
sub initTestEv($$)
sub initTestEv($$$$$$$)
{
my ($dest,$type) = @_;
my ($dest, $type, $toolname, $toolwrapperpath, $tooltype,
$req_params, $opt_params) = @_;
if( !defined $testevents{$dest}{$type} ){
$testevents{$dest}{$type} = {
......@@ -1350,6 +1406,11 @@ sub initTestEv($$)
fpingTimeout => $fpingTimeoutDef
};
}
$testevents{$dest}{$type}{toolname} = $toolname;
$testevents{$dest}{$type}{toolwrapperpath} = $toolwrapperpath;
$testevents{$dest}{$type}{tooltype} = $tooltype;
$testevents{$dest}{$type}{req_params} = $req_params;
$testevents{$dest}{$type}{opt_params} = $opt_params;
}
sub getstats()
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
my %params = (@ARGV);
my $execpath = "/tmp/fping";
my %ERRID = ( #kept the same as defined in libwanetmon for backwards compatibility
unknown => -3,
timeout => -1,
unknownhost => -4,
ICMPunreachable => -5
);
#
# Check for existence of all required params
#
# TODO
#
# Add optional args, if they exist
#
my %optional_exec_args; #TODO
#
# Form arguments specific to this execuable
#
my $execargs = " -t $params{timeout} -s -r $params{retries} $params{target}";
#
# Execute testing application
#
#print "FPING WRAPPER... executing= $execpath $execargs\n";
my $raw = `sudo $execpath $execargs 2>&1`;
#print "FPING WRAPPER... parsing raw= $raw\n";
#
# Parse output
#
$_ = $raw;
my $measr = 0;
my $error = 0;
if( /^ICMP / )
{
$error = $ERRID{ICMPunreachable};
}elsif( /address not found/ ){
$error = $ERRID{unknownhost};
}elsif( /2 timeouts/ ){
$error = $ERRID{timeout};
}elsif( /[\s]+([\S]*) ms \(avg round trip time\)/ ){
$measr = "$1" if( $1 ne "0.00" );
}else{
$error = $ERRID{unknown};
}
#print "MEASR=$measr, ERROR=$error\n";
#
# Output result
#
print "latency=$measr,error=$error\n";
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
my %params = (@ARGV);
my $execpath = "/tmp/iperf";
my %ERRID = ( #kept the same as defined in libwanetmon for backwards compatibility
unknown => -3,
timeout => -1,
unknownhost => -4,
iperfHostUnreachable => -6
);
#
# Check for existence of all required params
#
# TODO
#
# Add optional args, if they exist
#
my %optional_exec_args; #TODO
#
# Form arguments specific to this execuable
#
my $execargs = " -fk -c $params{target} -t $params{duration} -p $params{port}";
#
# Execute testing application
#
my $raw = `$execpath $execargs`;
#print "RAW = $raw\n";
#
# Parse output
#
$_ = $raw;
my $measr = 0;
my $error = 0;
if( /connect failed: Connection timed out/ ){
# this one shouldn't happen, if the timeout check done by
# bgmon is set low enough.
$error = $ERRID{timeout};
}elsif( /write1 failed:/ ){
$error = $ERRID{iperfHostUnreachable};
}elsif( /error: Name or service not known/ ){
$error = $ERRID{unknownhost};
}elsif( /\s+(\S*)\s+([MK])bits\/sec/ ){
$measr = $1;
if( $2 eq "M" ){
$measr *= 1000;
}
}else{
$error = $ERRID{unknown};
}
#print "MEASR=$measr, ERROR=$error\n";
#
# Output result
#
print "bw=$measr,error=$error\n";
......@@ -27,12 +27,29 @@ sub usage {
my $debug = 0;
my $impotent = 0;
my $nocap = 0;
my $bwperiod = 5;
#
# $evexpt is the experiment ("pid/eid") in which the entire bgmon framework
# (probes, manager, clients) is running. This is used for confining Emulab
# events (client/manager communication) to just that experiment. The
# '-e pid/eid' option to the manager sets the context for that instance.
#
# $bgmonexpt is an incompletely implemented feature that allows multiple
# probe experiments to share the manager. Here the client specifies the
# experiment of the probes (passed in the message) and one common manager
# muxes/demuxes messages to the different probe experiments. There are
# still seperate opsrecv processes. It isn't yet clear how useful this
# might be.
#
my $evexpt = "__none";
my $bgmonexpt;
my $default_bgmonexpt = "tbres/pelabbgmon";
my $bgmonexpt;
my ($server,$port,$cmdport);
my %opt = ();
if (!getopts("s:p:c:dih", \%opt)) {
if (!getopts("s:p:c:dNihe:", \%opt)) {
exit &usage;
}
......@@ -58,9 +75,10 @@ if ($opt{s}) { $server = $opt{s}; } else { $server = "localhost"; }
if ($opt{p}) { $port = $opt{p}; }
if ($opt{c}) { $cmdport = $opt{c};} else { $cmdport = 5052; }
if ($opt{h}) { exit &usage; }
#if ($opt{e}) { $evexpt = $opt{e}; }
if ($opt{e}) { $evexpt = $opt{e}; setevexpid($evexpt); $bgmonexpt=$evexpt;}
if ($opt{d}) { $debug = 1; }
if ($opt{i}) { $impotent = 1; }
if ($opt{N}) { $nocap = 1; print "*** No bandwidth cap enforced\n"; }
if (@ARGV !=0) { exit &usage; }
......@@ -87,6 +105,7 @@ if ($port) { $URL .= ":$port"; }
# Register for normal "manager client" notifications
my $handle_mc = event_register($URL,0);
if (!$handle_mc) { die "Unable to register with event system\n"; }
my $tuple = address_tuple_alloc();
if (!$tuple) { die "Could not allocate an address tuple\n"; }
......@@ -116,7 +135,7 @@ $sel->add($socket_cmd);
print "setting cmdport $cmdport and cmdexpt $bgmonexpt\n";
setcmdport($cmdport);
setexpid($default_bgmonexpt);
setexpid($bgmonexpt);
#main()
......@@ -181,8 +200,16 @@ sub callbackFunc($$$) {
print "EVENT: $time $objtype $eventtype\n"
if ($debug);
# XXX just to make sure we get the event
if( $eventtype eq "TEST" ){
my $managerID = event_notification_get_string($handle,
$notification,
"managerID");
print "got TEST event: managerID=$managerID\n";
}
# TODO: Does this have to be listed in lib/libtb/tbdefs.h ??
if( $eventtype eq "EDIT" ){
elsif( $eventtype eq "EDIT" ){
#
# Got a request to modify a path's test freq
#
......@@ -208,6 +235,30 @@ sub callbackFunc($$$) {
my $newexpid = event_notification_get_string($handle,
$notification,
"expid");
#
#XXX / TODO: stuff for tool generalization
# hacked in here for now, but source should be from the
# managerclient' message (?)
#
my $toolname;
my $toolwrapperpath;
my $tooltype; #one-shot or continuous
my $req_params; #params required, but universal for each tool instance
my $opt_params = "";
if( $testtype eq "bw" ){
$toolname = "iperf";
$toolwrapperpath = "/tmp/iperfwrapper";
$tooltype = "one-shot";
$req_params = "port 5002 duration 5";
}elsif( $testtype eq "latency"){
$toolname = "fping";
$toolwrapperpath = "/tmp/fpingwrapper";
$tooltype = "one-shot";
$req_params = "timeout 10000 printstats 1 retries 1";
}
if( !defined $newexpid || $newexpid eq "" ){
$newexpid = $bgmonexpt;
}
......@@ -219,6 +270,11 @@ sub callbackFunc($$$) {
testper => "$period",
duration => "$duration",
managerID => $managerID
,toolname => $toolname
,toolwrapperpath=>$toolwrapperpath
,tooltype => $tooltype
,req_params=> $req_params
,opt_params=> $opt_params
);
......@@ -254,6 +310,29 @@ sub callbackFunc($$$) {
my $newexpid = event_notification_get_string($handle,
$notification,
"expid");
#
#XXX / TODO: stuff for tool generalization
# hacked in here for now, but source should be from the
# managerclient' message (?)
#
my $toolname;
my $toolwrapperpath;
my $tooltype; #one-shot or continuous
my $req_params; #params required, but universal for each tool instance
my $opt_params = "";
if( $testtype eq "bw" ){
$toolname = "iperf";
$toolwrapperpath = "/tmp/iperfwrapper.pl";
$tooltype = "one-shot";
$req_params = "port 5002 duration 5";
}elsif( $testtype eq "latency"){
$toolname = "fping";
$toolwrapperpath = "/tmp/fpingwrapper.pl";
$tooltype = "one-shot";
$req_params = "timeout 10000 retries 1";
}
if( !defined $newexpid || $newexpid eq "" ){
$newexpid = $bgmonexpt;
}
......@@ -266,6 +345,11 @@ sub callbackFunc($$$) {
testper => "$testper",
duration => "$duration"
,managerID => $managerID
,toolname => $toolname
,toolwrapperpath=>$toolwrapperpath
,tooltype => $tooltype
,req_params=> $req_params
,opt_params=> $opt_params
);
print "got $eventtype:$srcnode,$destnodes,$testtype,".
......@@ -277,8 +361,11 @@ sub callbackFunc($$$) {
# sendcmd( $srcnode, \%cmd );
# }
if( isCmdValid(\%cmd) ){
print "sending cmd from $srcnode\n";
# print "sending cmd from $srcnode\n";
print "sending cmd to $srcnode on behalf of $managerID\n";
sendcmd( $srcnode, \%cmd );
}else{
print "rejecting $testtype cmd for $srcnode\n";
}
}
elsif( $eventtype eq "STOPALL" ){
......@@ -315,11 +402,21 @@ sub isCmdValid($)
#list of invalid conditions
if( $cmdref->{managerID} ne "automanagerclient" ){
#managerclient is not the AMC
if( $cmdref->{duration} eq "0" ){
if( $cmdref->{duration} == 0 ){
#only AMC can send "forever" commands
$valid = 0;
}elsif( $cmdref->{testtype} eq "bw" )
{
return(0);
}
if( $cmdref->{testtype} eq "bw" ){
# Cannot specify period less than test length
if ( $cmdref->{testper} < $bwperiod ){
return(0);
}
# No caps enforced
if ($nocap) {
return(1);
}
my @destnodes;
if( $cmdref->{cmdtype} eq "INIT" ){
@destnodes = split(" ",$cmdref->{destnodes});
......
......@@ -56,7 +56,7 @@ else { $settings{managerID} = "default"; }
if ($opt{s}) { $server = $opt{s}; } else { $server = "localhost"; }
if ($opt{p}) { $port = $opt{p}; }
if ($opt{h}) { exit &usage; }
if ($opt{e}) { $settings{"bgmonexpt"} = $opt{e}; }
if ($opt{e}) { $settings{"bgmonexpt"} = $opt{e}; setevexpid($opt{e});}
if ($opt{d}) { $settings{"testduration"} = $opt{d}; }
if ($opt{a}) { $settings{"allpairs"} = 1; }
if ($opt{L}) { $settings{"noLatency"} = 1; }
......
......@@ -7,7 +7,7 @@
rm bgmon.tar.gz
#tar -czf bgmon.tar.gz bgmon.pl runbgmon event.pm
tar -czf bgmon.tar.gz bgmon.pl libwanetmon.pm Cmdqueue.pm runbgmon
tar -czf bgmon.tar.gz bgmon.pl libwanetmon.pm Cmdqueue.pm runbgmon iperfwrapper.pl
cp bgmon.tar.gz /proj/tbres/gebhardt/wanetmonstuff/
#rm /proj/tbres/gebhardt/wanetmonstuff/iperfd.tar.gz
#rm /proj/tbres/gebhardt/wanetmonstuff/fping.tar.gz
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment