Commit 63a206c4 authored by Mike Hibler's avatar Mike Hibler
Browse files

Add dbmonitor, a varient of the monitor that ignores the stubs entirely

and just updates link characteristics based on the pelab DB values.
parent 02f6a090
......@@ -111,14 +111,22 @@ set clientprog "/usr/bin/iperf -t 60 -c "
#
set use_magent 0
#
# If non-zero, uses the DB-based "monitor" to control the cloud shaping
#
set use_dbmonitor 0
# End of user-settable options
##########
#
# -DNODE has fixes but is not as heavily tested
#
#set delay_os FBSD54-FUTURE
# XXX should really never use -FUTURE anymore.
set delay_os FBSD54-DNODE
#set delay_os FBSD54-FUTURE
# XXX -UPDATE for now. Contains bug fixes (progagent command lines).
set node_os PLAB-DEVBOX-UPDATE
#set node_os PLAB-DEVBOX
tb-set-delay-os $delay_os
......@@ -139,6 +147,11 @@ if {$use_magent} {
} else {
set stubcommand "/bin/sh /local/pelab/stub/auto-stub.sh"
}
if {$use_dbmonitor} {
set moncommand "/bin/sh /local/pelab/dbmonitor/auto-dbmonitor.sh"
} else {
set moncommand "/bin/sh /local/pelab/monitor/auto-monitor.sh"
}
set elan_string ""
set plan_string ""
......@@ -186,7 +199,7 @@ for {set i 1} {$i <= $num_pcs} {incr i} {
if {$fake_plab} {
set plab($i) [$ns node]
tb-set-node-os $plab($i) PLAB-DEVBOX
tb-set-node-os $plab($i) $node_os
tb-set-hardware $plab($i) $hwtype
append plan_string "$plab(${i}) "
set plabstub($i) [$plab($i) program-agent -command $stubcommand]
......@@ -207,10 +220,10 @@ for {set i 1} {$i <= $num_pcs} {incr i} {
}
set elab($i) [$ns node]
tb-set-node-os $elab($i) PLAB-DEVBOX
tb-set-node-os $elab($i) $node_os
tb-set-hardware $elab($i) $hwtype
append elan_string "$elab(${i}) "
set monitor($i) [$elab($i) program-agent -command "/bin/sh /local/pelab/monitor/auto-monitor.sh"]
set monitor($i) [$elab($i) program-agent -command $moncommand]
lappend monitorlist $monitor($i)
set server($i) [$elab($i) program-agent -command $serverprog]
......
......@@ -84,6 +84,7 @@ export STUB_DIR="${BASE}/stub/";
export MAGENT_DIR="${BASE}/magent/";
export NETMON_DIR="${BASE}/libnetmon/";
export MONITOR_DIR="${BASE}/monitor/";
export DBMONITOR_DIR="${BASE}/dbmonitor/";
export TMPDIR="/var/tmp/";
export LOGDIR="/local/logs/"
......@@ -99,6 +100,7 @@ export NETMOND="netmond"
export STUBD="stubd"
export MAGENT="magent"
export MONITOR="monitor.py"
export DBMONITOR="dbmonitor.pl"
export GENIPMAP="gen-ip-mapping.pl"
export NETMON_LIB="libnetmon.so"
......
#!/bin/sh
ARGS=$*
if [ -z "$PID" -o -z "$EID" ]; then
echo "*** experiment PID and EID are not set!"
exit 1
fi
. `dirname $0`/../common-env.sh
#
# We don't give a rat's ass about the stubs, but we have to stay in synch
#
echo "Waiting for stubs to become ready";
barrier_wait "stub"; _rval=$?
if [ $_rval -ne 0 ]; then
echo "*** WARNING: not all stubs started ($_rval)"
fi
#
# Copy over the node list
#
cp -p /proj/$PID/exp/$EID/tmp/node_list /var/tmp/node-mapping
#
# Start up our own monitor
#
echo $SH ${DBMONITOR_DIR}/run-dbmonitor.sh $ARGS
$SH ${DBMONITOR_DIR}/run-dbmonitor.sh $ARGS &
DBMONPID=$!
# Kill the monitor if we get killed - TODO: harsher kill?
trap "$AS_ROOT kill $DBMONPID" EXIT
#
# Give it time to come up
#
sleep 1
#
# Wait for all the monitors to come up
#
echo "Waiting for dbmonitors to become ready";
barrier_wait "monitor"; _rval=$?
if [ $_rval -ne 0 ]; then
echo "*** WARNING: not all dbmonitors started ($_rval)"
fi
echo "Running!";
#
# Wait for our monitor to finish
#
wait
#!/usr/bin/perl
#
# Periodically extract the latest path characteristics from the PELAB DB
# for paths from us (rather, the planetlab node we represent) to all other
# nodes in the experiment. These data are then fed to the appropriate
# delay-agents.
#
# Elab nodes are in a "cloud" allowing you to set node-to-node characteristics
# for all nodes within the cloud. To do this for node N to node M:
#
# tevc -e pid/eid now elabc-elab-N MODIFY DEST=<elab-M-elabc IP> \
# [ BANDWIDTH=<kbits/sec> ] [ DELAY=<ms> ] [ PLR=<prob> ]
#
# The characteristics are applied to the node->LAN pipe for node N.
# Since this is one-way, you will have to apply the usual tricks to
# convert round-trip delays and PLR.
#
my $TEVC = "/usr/testbed/bin/tevc";
my $NODEMAP = "/var/tmp/node-mapping";
my $pprefix = "planet";
my $eprefix = "elab";
# XXX Need to configure this stuff!
use lib '/usr/testbed/lib';
use libtbdb;
use Socket;
use Getopt::Std;
use POSIX qw(strftime);
#
# Every source host has a list of <dest-IP,bw,delay,plr> tuples, one
# element per possible destination
#
my %shapeinfo;
sub usage()
{
print STDERR
"usage: dbmonitor.pl [-1dns] [-i interval] [-S starttime] pid eid\n".
" -1 run once only (set initial values and exit)\n".
" -d turn on debugging\n".
" -n don't actually set characteristics, just print values from DB\n".
" -s silent mode, don't print any non-errors\n".
"\n".
" -i interval periodic interval at which to query DB (seconds)\n".
" -S starttime start time for values pulled from DB (seconds since epoch)\n";
exit(1);
}
my $optlist = "ndi:ps1S:";
my $showonly = 0;
my $debug = 0;
my $interval = (1 * 60);
my $verbose = 1;
my $onceonly = 0;
my $starttime = 0;
my $timeskew = 0;
# Default values. Note: delay and PLR are round trip values.
my $DEF_BW = 10000; # Kbits/sec
my $DEF_DEL = 0; # ms
my $DEF_PLR = 0.0; # prob.
my $PWDFILE = "/local/pelab/pelabdb.pwd";
my $DBHOST = "users.emulab.net";
my $DBNAME = "pelab";
my $DBUSER = "pelab";
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"n"})) {
$showonly = 1;
}
if (defined($options{"d"})) {
$debug++;
}
if (defined($options{"i"})) {
$interval = int($options{"i"});
}
# XXX compat: ignore -p option
if (defined($options{"p"})) {
;
}
if (defined($options{"s"})) {
$verbose = 0;
}
if (defined($options{"1"})) {
$onceonly = 1;
}
if (defined($options{"S"})) {
my $high = time();
my $low = $high - (23 * 60 * 60); # XXX
$starttime = $options{"S"};
if ($starttime && ($starttime < $low || $starttime > $high)) {
die("Bogus timestamp $starttime, should be in [$low - $high]\n");
}
$timeskew = $high - $starttime;
}
if (@ARGV != 2) {
usage();
}
my ($pid,$eid) = @ARGV;
# Get DB password and connect.
my $DBPWD = `cat $PWDFILE`;
if ($DBPWD =~ /^([\w]*)\s([\w]*)$/) {
$DBPWD = $1;
}
else{
die("Bad chars in password!\n");
}
#
# XXX figure out how many pairs there are, and for each, who the
# corresponding planetlab node is. This file is just the output of
# node_list run on ops. Could probably get all this easier via XMLRPC,
# if the XMLRPC interface worked in the PLAB-DEVBOX image...
#
my @nodelist = split('\s+', `cat $NODEMAP`);
chomp(@nodelist);
my $nnodes = grep(/^$pprefix-/, @nodelist);
if ($nnodes == 0) {
print STDERR "No planetlab nodes in $pid/$eid?!\n";
exit(1);
}
#
# Map nodes to site/node indicies
#
my %vnode_to_dbix = ();
my %vnode_to_pnode = ();
my %ix_mapping = ();
my %ip_mapping = ();
TBDBConnect($DBNAME, $DBUSER, $DBPWD, $DBHOST) == 0
or die("Could not connect to pelab database!\n");
foreach my $mapping (@nodelist) {
if ($mapping =~ /^($pprefix-\d+)=(\w*)$/) {
my $vnode = $1;
my $pnode = $2;
# Grab the indices.
my $query_result =
DBQueryFatal("select site_idx,node_idx from site_mapping ".
"where node_id='$pnode'");
if (!$query_result->numrows) {
die("Could not map $pnode to its site index!\n");
}
my ($site_index, $node_index) = $query_result->fetchrow_array();
$vnode_to_pnode{$vnode} = $pnode;
$vnode_to_dbix{$vnode} = $site_index;
if ($vnode =~ /^$pprefix-(\d+)/) {
$ix_mapping{$vnode} = $1;
} else {
die("Could not map $vnode to its index!\n");
}
# Grab the IP address and save.
my (undef,undef,undef,undef,@ips) = gethostbyname("$pnode");
if (!@ips) {
die("Could not map $pnode to its ipaddr\n");
}
$ip_mapping{$pnode} = inet_ntoa($ips[0]);
} elsif ($mapping =~ /^($eprefix-\d+)=(\w*)$/) {
$vnode_to_pnode{$1} = $2;
}
}
TBDBDisconnect();
undef @nodelist;
#
# Now figure out who we represent and who all the possible destinations are.
# Names are kept as virtual planetlab names.
#
my $me;
my ($elabname) = split('\.', `hostname`);
if ($elabname =~ /^$eprefix-(\d)+$/) {
$me = "$pprefix-$1";
if (!exists($vnode_to_pnode{$me})) {
die("Bogus host name $elabname");
}
if (!exists($vnode_to_dbix{$me})) {
die("No DB indices for $me");
}
if ($debug) {
print STDERR "I am $elabname (", $vnode_to_pnode{$elabname},
") representing $me (", $vnode_to_pnode{$me},
")\n";
}
} else {
die("Bogus host name $elabname");
}
my @them = ();
foreach my $vnode (keys(%vnode_to_pnode)) {
if ($vnode =~ /^$pprefix-(\d)+$/) {
my $ix = $1;
if ($vnode eq $me) {
next;
}
if (!exists($vnode_to_dbix{$vnode})) {
die("No DB indices for destination $vnode");
}
push(@them, $vnode);
if ($debug) {
my $ename = "$eprefix-$ix";
print STDERR "Destination $ename (", $vnode_to_pnode{$ename},
") representing $vnode (", $vnode_to_pnode{$vnode},
")\n";
}
}
}
my $msg = "DB Monitor starting at ";
if ($starttime) {
$msg .= "$starttime (now - $timeskew)\n";
} else {
$msg .= time() . " (now)\n";
}
logmsg($msg);
#
# Periodically get DB info for all possible targets and
# send it to the appropriate delay agents.
#
while (1) {
if (get_plabinfo($me, @them) && !$showonly) {
send_events();
}
last
if ($onceonly);
sleep($interval);
}
exit(0);
sub send_events()
{
my $msg;
foreach my $src (keys %shapeinfo) {
foreach my $rec (@{$shapeinfo{$src}}) {
my ($doit,$dst,$bw,$del,$plr) = @{$rec};
next
if (!$doit);
my $cmd = "$TEVC -e $pid/$eid now elabc-$src MODIFY ".
"DEST=$dst BANDWIDTH=$bw DELAY=$del PLR=$plr";
$msg = "elabc-$src: DEST=$dst BANDWIDTH=$bw DELAY=$del PLR=$plr...";
if (system("$cmd") != 0) {
warn("*** '$cmd' failed\n");
$msg .= "[FAILED]\n";
} else {
$msg .= "[OK]\n";
}
logmsg($msg);
}
}
}
#
# Grab data from DB.
#
sub get_plabinfo($@)
{
my ($me, @nodes) = @_;
if (TBDBConnect($DBNAME, $DBUSER, $DBPWD, $DBHOST) != 0) {
warn("Could not connect to pelab DB\n");
return(0);
}
my $src_site = $vnode_to_dbix{$me};
my $src_ename;
($src_ename = $me) =~ s/$pprefix/$eprefix/;
my $src_pname = $vnode_to_pnode{$me};
my %lastinfo;
if (exists($shapeinfo{$src_ename})) {
foreach my $rec (@{$shapeinfo{$src_ename}}) {
my ($doit,$dst,$bw,$del,$plr) = @$rec;
$lastinfo{$dst} = [ $bw, $del, $plr ];
}
}
@{$shapeinfo{$src_ename}} = ();
foreach my $dstvnode (@nodes) {
my $dst_ip = $ip_mapping{$vnode_to_pnode{$dstvnode}};
my $dst_site = $vnode_to_dbix{$dstvnode};
my $dst_ename;
my $dst_ix;
($dst_ename = $dstvnode) =~ s/$pprefix/$eprefix/;
($dst_ix = $dst_ename) =~ s/$eprefix-//;
#
# If an explict replay start time was chosen add a time qualifier.
#
# Obviously, if we are operating in "unreal" time, we could do a
# much more exact job of setting characteristics. Rather than
# periodic polling of the DB, we could compute a schedule of
# upcoming events based on the DB and apply them "when they happen".
# But, we don't.
#
my $dateclause = "";
if ($starttime) {
my $mynow = time() - $timeskew;
$dateclause = "and unixstamp <= $mynow ";
}
#
# BW and latency records are separate so we just grab the last 20
# and hope we get a legit value for both.
# Note that there are no loss measurements right now.
#
my ($del,$plr,$bw);
my ($del_stamp,$plr_stamp,$bw_stamp);
my $query_result =
DBQueryFatal("select latency,loss,bw,unixstamp from pair_data ".
" where ".
" srcsite_idx='$src_site' and ".
" dstsite_idx='$dst_site' and ".
" (latency is not null or ".
" loss is not null or bw is not null) ".
$dateclause .
" order by unixstamp desc limit 20");
if (!$query_result->numrows) {
warn("*** Could not get pair data for ".
"$me ($src_site) --> $dstvnode ($dst_site)\n".
" defaulting to ".
"${DEF_BW}bps, ${DEF_DEL}ms, ${DEF_PLR}plr\n");
($del,$plr,$bw) = ($DEF_DEL, $DEF_PLR, $DEF_BW);
$del_stamp = $plr_stamp = $bw_stamp = time();
} else {
print "$src_ename -> $dst_ename (on behalf of $dst_ip):\n"
if ($showonly || $debug);
while (my ($_del,$_plr,$_bw,$_stamp) =
$query_result->fetchrow_array()) {
print " ($_del, $_plr, $_bw, $_stamp)\n"
if ($showonly || $debug);
if (!defined($del) && defined($_del)) {
$del = $_del;
$del_stamp = $_stamp;
}
if (!defined($plr) && defined($_plr)) {
$plr = $_plr;
$plr_stamp = $_stamp;
}
if (!defined($bw) && defined($_bw)) {
$bw = $_bw;
$bw_stamp = $_stamp;
}
}
#
# XXX This needs to be modified!
#
if (!defined($del)) {
$del = $DEF_DEL;
$del_stamp = time();
}
if (!defined($plr)) {
$plr = $DEF_PLR;
$plr_stamp = time();
}
# undef or zero--zero BW is not very useful
if ($bw == 0) {
$bw = $DEF_BW;
$bw_stamp = time();
}
#
# Check for down links, reflected by either delay or bandwidth
# being set to -1. If only one is set to -1, look at the most
# recent of delay/bw to determine whether to mark the link as
# down.
#
if ($del == -1 || $bw == -1) {
if (($del == -1 && $bw == -1) ||
($del == -1 && $del_stamp >= $bw_stamp) ||
($bw == -1 && $bw_stamp >= $del_stamp)) {
print STDERR "marking as down: bw=$bw($bw_stamp), del=$del($del_stamp)\n"
if ($debug);
$plr = 1;
}
$del = $DEF_DEL
if ($del == -1);
$bw = $DEF_BW
if ($bw == -1);
}
$del = int($del / 2 + 0.5);
}
print "$src_ename -> $dst_ename: ".
"real=$dst_ip, bw=$bw, del=$del, plr=$plr\n"
if ($showonly || $debug);
# XXX need to lookup "elab-$dst_ix"
$dst_ip = "10.0.0.$dst_ix";
my ($lbw, $ldel, $lplr);
if (exists($lastinfo{$dst_ip})) {
($lbw, $ldel, $lplr) = @{$lastinfo{$dst_ip}};
print STDERR "$src_pname->$dst_ip: old values: bw=$lbw, del=$ldel, plr=$lplr\n"
if ($debug);
} else {
$lbw = $ldel = $lplr = -2;
}
my $doit;
if ($lbw != $bw || $ldel != $del || $lplr != $plr) {
$doit = 1;
} else {
$doit = 0;
print STDERR "$src_pname->$dst_ip: values unchanged\n"
if ($debug);
}
push(@{$shapeinfo{$src_ename}}, [ $doit, $dst_ip, $bw, $del, $plr ]);
}
TBDBDisconnect();
return(1);
}
sub logmsg($)
{
my ($msg) = @_;
print strftime("%b %e %H:%M:%S", localtime)." dbmonitor[$$]: $msg"
if ($verbose || $debug);
}
#!/bin/sh
#
# Script to run the monitor, collecting data from libnetmon
#
#
# Let common-env know what role we're playing
#
export HOST_ROLE="monitor"
#
# Grab common environment variables
#
. `dirname $0`/../common-env.sh
#
# Just run it!
#
echo "Running PID $$"
echo "Starting dbmonitor, Extra arguments: $*"
exec $AS_ROOT $DBMONITOR_DIR/$DBMONITOR $* $PID $EID
......@@ -8,6 +8,7 @@ my $LOGHOLE = "/usr/testbed/bin/loghole";
my $PYTHON = "/usr/local/bin/python";
my $PERL = "/usr/bin/perl";
my $EVENTSYS = "/usr/testbed/bin/eventsys_control";
my $NODELIST = "/usr/testbed/bin/node_list";
my $realplab = 0;
my $initelab = 0;
......@@ -15,7 +16,7 @@ my $initelab = 0;
my $UNKNOWN = "<unknown>";
my $stub_cmd = "/bin/sh /local/pelab/magent/auto-magent.sh";
my $stub_cmdargs = $UNKNOWN;
my $mon_cmd = "/bin/sh /local/pelab/monitor/auto-monitor.sh";
my $mon_cmd = "/bin/sh /local/pelab/dbmonitor/auto-dbmonitor.sh";
my $mon_cmdargs = $UNKNOWN;
sub get_cmdargs($$);
......@@ -155,6 +156,15 @@ if (system "$PYTHON resetlinks.py $pid $eid") {
die "Error resetting links\n";
}
#
# XXX stash node list for experiment since nodes don't have node_list
# right now. This is needed by the dbmonitor if it is in use.
#