Merge branch 'master' of git-public.flux.utah.edu:/flux/git/emulab-devel

GNUmakefile.in

@@ -238,6 +238,7 @@ endif
 	@$(MAKE) -C tmcd subboss-install
 	@$(MAKE) -C rc.d subboss-install
 	@$(MAKE) -C dhcpd subboss-install
+	@$(MAKE) -C tbsetup client-install
 	# Hack: this should probably be done by a subboss-install
 	# script, but there isn't a huge need for that yet
 	@rm -f $(INSTALL_RCDIR)/isc-dhcpd

assign/fstring.h

@@ -126,7 +126,11 @@ class fstring {
          * Output/debugging functions
          */
         friend ostream &operator<<(ostream &o, const fstring &s) {
-            return(o << s.str);
+            if (s.str != NULL) {
+                return(o << s.str);
+            } else {
+                return(o << "(null)");
+            }
         }
 		
 		

assign/score.cc

@@ -88,8 +88,18 @@ void score_link_endpoints(pedge pe);
 #define SDEBADD(amount) cerr << "SADD: " << #amount << "=" << amount << " from " << score;score+=amount;cerr << " to " << score << endl
 #define SDEBSUB(amount)  cerr << "SSUB: " << #amount << "=" << amount << " from " << score;score-=amount;cerr << " to " << score << endl
 
+#ifdef SCORE_DEBUG_MAX
+// Handy way to print only the first N debugging messages, since that's usually
+// enough to get the idea
+static unsigned long scoredebugcount = 0;
+#endif
+
 #ifdef SCORE_DEBUG
+#ifdef SCORE_DEBUG_MAX
+#define SDEBUG(a) if (scoredebugcount++ < SCORE_DEBUG_MAX) { a; }
+#else
 #define SDEBUG(a) a
+#endif
 #else
 #define SDEBUG(a) 
 #endif
@@ -207,7 +217,7 @@ void init_score()
 float find_link_resolutions(resolution_vector &resolutions, pvertex pv,
     pvertex dest_pv, tb_vlink *vlink, tb_pnode *pnode, tb_pnode *dest_pnode,
     bool flipped) {
-  SDEBUG(cerr << "   finding link resolutions" << endl);
+  SDEBUG(cerr << "   finding link resolutions from " << pnode->name << " to " << dest_pnode->name << endl);
   /* We need to calculate all possible link resolutions, stick
    * them in a nice datastructure along with their weights, and
    * then select one randomly.
@@ -245,7 +255,7 @@ float find_link_resolutions(resolution_vector &resolutions, pvertex pv,
     info.plinks.push_back(pe);
     resolutions.push_back(info);
     total_weight += LINK_RESOLVE_DIRECT;
-    SDEBUG(cerr << "    direct_link " << pe << endl);
+    SDEBUG(cerr << "    added a direct_link " << pe << endl);
   }
   // Intraswitch link
   pedge first,second;
@@ -253,6 +263,7 @@ float find_link_resolutions(resolution_vector &resolutions, pvertex pv,
       switch_it != pnode->switches.end();++switch_it) {
     if (dest_pnode->switches.find(*switch_it) !=
         dest_pnode->switches.end()) {
+      SDEBUG(cerr << "    intraswitch: both are connected to " << *switch_it << endl);
       bool first_link, second_link;
       /*
        * Check to see if either, or both, pnodes are actually
@@ -348,13 +359,17 @@ float find_link_resolutions(resolution_vector &resolutions, pvertex pv,
       if ((pv == *source_switch_it) || (pv ==
             *dest_switch_it)) {
         first_link = false;
+        SDEBUG(cerr << "    interswitch: not first link in a path" << endl);
       } else {
+        SDEBUG(cerr << "    interswitch: *is* first link in a path" << endl);
         first_link = true;
       }
       if ((dest_pv == *source_switch_it) ||
           (dest_pv == *dest_switch_it)) {
         second_link = false;
+        SDEBUG(cerr << "    interswitch: not second link in a path" << endl);
       } else {
+        SDEBUG(cerr << "    interswitch: *is* second link in a path" << endl);
         second_link = true;
       }
 
@@ -517,6 +532,7 @@ void resolve_link(vvertex vv, pvertex pv, tb_vnode *vnode, tb_pnode *pnode,
   if (vlink->src != vv) {
     SDEBUG(cerr << "  vlink is flipped" << endl);
     flipped = true;
+    assert(vlink->dst == vv);
   }
 
   /*
@@ -1490,13 +1506,30 @@ bool find_best_link(pvertex pv,pvertex switch_pv,tb_vlink *vlink,
 	  continue;
       }
 
+      // XXX: Not 100% sure it's better to do this inside find_best_link rather
+      // than in the caller
+      if (flipped) {
+          // If the endpoints are flipped, then we need to flip our notion of
+          // which ones need to be compared.
+          bool tmp = check_src_iface;
+          check_src_iface = check_dst_iface;
+          check_dst_iface = tmp;
+          SDEBUG(cerr << "         find_best_link: flipping interface comparisons" << endl;)
+      }
+
+      SDEBUG(cerr << "         find_best_link: fix_src_iface = " <<
+              vlink->fix_src_iface << " check_src_iface = " << check_src_iface
+              << " fix_dst_iface = " << vlink->fix_dst_iface
+              << " check_dst_iface = " << check_dst_iface
+              << " flipped = " << flipped << endl);
+
       // If the vlink has a fixed source interface, and it doesn't match
       // this plink, skip it
       if (vlink->fix_src_iface && check_src_iface) {
           // Whether we check the 'source' or 'destination' on the vlink against
           // the phyisical link's source interface depends on whether we're
           // traversing the link if forward or reverse (flipped) order
-          fstring compare_iface = (flipped?vlink->dst_iface:vlink->src_iface);
+          fstring compare_iface = vlink->src_iface;
           if (plink->srciface != compare_iface) {
               SDEBUG(cerr << "          find_best_link (" << vlink->name <<
                       "): Fix source: " << plink->srciface << " != " <<
@@ -1514,7 +1547,7 @@ bool find_best_link(pvertex pv,pvertex switch_pv,tb_vlink *vlink,
       // XXX: This only works because we always have the node as the 'source'
       // of a plink! Shouldn't depend on this!
       if (vlink->fix_dst_iface && check_dst_iface) {
-          fstring compare_iface = (flipped?vlink->src_iface:vlink->dst_iface);
+          fstring compare_iface = vlink->dst_iface;
           if (plink->srciface != compare_iface) {
               SDEBUG(cerr << "          find_best_link (" << vlink->name <<
                       "): Fix dst: " << plink->srciface << " != " <<

configure.in

@@ -101,7 +101,7 @@ AC_SUBST(optional_subdirs)
 # These get defined in the definitions file (--with-TBDEFS).
 # NB: See AC_DEFINE_UNQUOTED() calls below, if you want a variable
 # to appear in config.h.
-# 
+#
 AC_SUBST(TBDBNAME)
 AC_SUBST(TBERRORLOGDBNAME)
 AC_SUBST(TBADMINGROUP)
@@ -156,6 +156,7 @@ AC_SUBST(ELABINELAB)
 AC_SUBST(OUTERBOSS_NODENAME)
 AC_SUBST(OUTERBOSS_XMLRPCPORT)
 AC_SUBST(OUTERBOSS_SSLCERTNAME)
+AC_SUBST(SUBBOSS_SSLCERTNAME)
 AC_SUBST(PLABSUPPORT)
 AC_SUBST(PUBSUPPORT)
 AC_SUBST(WIKISUPPORT)
@@ -250,7 +251,7 @@ AC_SUBST(TBERRORSEMAIL_NOSLASH)
 
 #
 # Defaults for for above variables.
-# 
+#
 TBERRORLOGDBNAME="errorlog"
 TBADMINGROUP="flux"
 CLIENT_ETCDIR="/etc/emulab"
@@ -270,6 +271,7 @@ ELABINELAB=0
 OUTERBOSS_NODENAME=""
 OUTERBOSS_XMLRPCPORT=3069
 OUTERBOSS_SSLCERTNAME="/etc/outer_emulab.pem"
+SUBBOSS_SSLCERTNAME="/etc/subboss.pem"
 PLABSUPPORT=0
 PUBSUPPORT=0
 WIKISUPPORT=0
@@ -431,7 +433,7 @@ AC_SUBST(LOG_TESTBED)
 # If the private network is different then the control network, we need
 # an extra subnet decl for it in dhcpd.conf.template. This is enormously
 # crude and simplistic, but works for the local case!
-# 
+#
 if test "$PRIVATE_NETWORK" != "$CONTROL_NETWORK"; then
     DHCPD_CONTROLNET_DECL="subnet $PRIVATE_NETWORK netmask $PRIVATE_NETMASK {}"
 fi
@@ -660,7 +662,7 @@ AC_SUBST(OPT_CFLAGS)
 
 #
 # Disable event system. Relies on Elvin. On by default.
-# 
+#
 AC_ARG_ENABLE(events,
 [  --disable-events      Disable events (requires Elvin libraries)])
 
@@ -715,7 +717,7 @@ fi
 # Enable Windows support.
 # Right now this means NTFS support in imagezip, maybe more things later.
 # Relies on Linux NTFS library.
-# 
+#
 AC_ARG_ENABLE(windows,
 [  --enable-windows      Enable Windows XP support (default)])
 
@@ -868,6 +870,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/wanlinkinfo tbsetup/wanassign \
 	tbsetup/swapexp	tbsetup/endexp tbsetup/elabinelab \
 	tbsetup/frisbeelauncher tbsetup/eventsys.proxy tbsetup/frisbeeimage \
+	tbsetup/subboss_frisbeelauncher tbsetup/subboss_frisbeeimage \
 	tbsetup/snmpit.proxy tbsetup/snmpit_remote.pm \
 	tbsetup/snmpit tbsetup/ns2ir/GNUmakefile \
 	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \

tbsetup/GNUmakefile.in

@@ -48,6 +48,8 @@ ifeq ($(ISMAINSITE),1)
 SBIN_STUFF     += repos_daemon
 endif
 
+SUBBOSS_SBIN_STUFF = subboss_frisbeelauncher subboss_frisbeeimage
+
 CTRLSBIN_STUFF  = console_setup.proxy sfskey_update.proxy \
 		  savelogs.proxy eventsys.proxy
 
@@ -266,6 +268,11 @@ endif
 	chown root $(INSTALL_SBINDIR)/nfstrace
 	chmod u+s $(INSTALL_SBINDIR)/nfstrace
 
+#
+# Subboss installation
+#
+subboss-install:	$(addprefix $(INSTALL_SBINDIR)/, $(SUBBOSS_SBIN_STUFF))
+
 #
 # Control node installation (okay, plastic)
 #

tbsetup/subboss_frisbeeimage.in

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2002, 2004, 2006, 2007 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+
+#
+# Ask outer emulab for an image we do not happen to have locally.
+#
+sub usage()
+{
+    print STDOUT "Usage: frisbeeimage [-d] imageid filename\n";
+
+    exit(-1);
+}
+my $optlist  = "d";
+my $debug    = 1;
+
+#
+# Configure variables
+#
+my $TB		= "@prefix@";
+my $TBOPS       = "@TBOPSEMAIL@";
+my $RPCSERVER   = "@OUTERBOSS_NODENAME@";
+my $RPCPORT     = "@OUTERBOSS_XMLRPCPORT@";
+my $RPCCERT     = "@OUTERBOSS_SSLCERTNAME@";
+
+# un-taint path
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# We don't want to run this script unless its the real version.
+#
+if ($EUID != 0) {
+    die("Must be root! Maybe its a development version?\n");
+}
+
+
+# Load the Testbed support stuff.
+use lib "@prefix@/lib";
+use libtestbed;
+use libxmlrpc;
+
+# Locals
+my $FRISBEE     = "$TB/sbin/frisbee";
+my $SAVEUID     = $UID;
+my $loadaddr;
+my $loadport;
+my $mcastif;
+
+#
+# Parse command arguments. Once we return from getopts, all that should
+# left are the required arguments.
+#
+%options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (! @ARGV) {
+    usage();
+}
+my ($imageid, $filename) = @ARGV;
+
+#
+# Untaint the arguments.
+#
+if ($imageid =~ /^([-\@\w\+\.]+)$/) {
+    $imageid = $1;
+}
+else {
+    die("Tainted argument $imageid!\n");
+}
+
+my $mcast_ip_file = "/var/emulab/boot/myip";
+
+if (! -e $mcast_ip_file) {
+    die("$mcast_ip_file does not exist!");
+}
+
+$mcastif = `cat $mcast_ip_file`;
+chomp($mcastif);
+if ($mcastif =~ /^([\d\.]+)$/) {
+    $mcastif = $1;
+}
+else {
+    die("Could not parse outer IP: $mcastif!");
+}
+
+#
+# Ask the outer Emulab to lauch a frisbee and return the loadinfo to us.
+#
+ libxmlrpc::Config({"server"  => $RPCSERVER,
+		    "verbose" => 1,
+		    "cert"    => $RPCCERT,
+		    "portnum" => $RPCPORT});
+
+my $rval = libxmlrpc::CallMethod("subboss", "frisbeelauncher",
+				 {"imageid" => "$imageid"});
+if (!defined($rval)) {
+    die("Could not fire up frisbee on boss!");
+}
+
+#
+# The return value is the loadaddr. Parse that into something we
+# can pass to the frisbee client.
+#
+if ($rval =~ /^(.*):(\d*)$/) {
+    $loadaddr = $1;
+    $loadport = $2;
+}
+else {
+    die("Could not parse loadinfo from server: $rval!");
+}
+
+if ($debug) {
+    print "$FRISBEE -N -i $mcastif -m $loadaddr -p $loadport $filename\n";
+}
+
+# XXX HACK
+$filename =~ /(.*)/;
+$filename = $1;
+$SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;
+system($FRISBEE, '-N', '-i', $mcastif, '-m', $loadaddr,
+	'-p', $loadport, $filename);
+if ($?) {
+    die("Error downloading image data from outer Emulab!");
+}
+exit(0);
+
+sub cleanup {
+    tbwarn("Download interrupted, $filename removed");
+    $SIG{HUP} = $SIG{INT} = $SIG{TERM} = 'IGNORE';
+    kill('TERM', 0);
+    unlink($filename);
+    exit(-1);
+}

tbsetup/subboss_frisbeelauncher.in

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2007 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use strict;
+use Getopt::Std;
+use POSIX 'setsid';	 # For &daemonize
+use POSIX ":sys_wait_h"; # For &WNOHANG
+use Sys::Syslog;
+use English;
+use Socket;
+use Errno qw(EADDRINUSE);
+
+#
+# This also kills a running frisbee.
+#
+sub usage()
+{
+    print "Usage: $0 [-d] [-k] <imageid> <path> <size> <mtime>\n";
+    print "-k:	Kill running frisbee.\n";
+    print "-d:	Print debugging output.\n";
+    exit(1);
+}
+my $optlist  = "dk";
+my $debug    = 0;
+my $killmode = 0;
+
+# Configure variables
+my $TB		= "@prefix@";
+my $TBOPS	= "@TBOPSEMAIL@";
+my $MY_IP;
+my $BASEADDR    = "@FRISEBEEMCASTADDR@";
+my $BASEPORT    = "@FRISEBEEMCASTPORT@";
+
+my $RPCSERVER   = "@OUTERBOSS_NODENAME@";
+my $RPCPORT     = "@OUTERBOSS_XMLRPCPORT@";
+my $RPCCERT     = "@SUBBOSS_SSLCERTNAME@";
+
+#
+# Untaint the path
+#
+$ENV{'PATH'} = "/bin:/usr/bin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+use lib "@prefix@/lib";
+use libtestbed;
+use libxmlrpc;
+use libtmcc;
+#use libtblog;
+
+# Protos.
+sub PickAddress($$);
+sub SetPid($);
+sub SetSyncFlag($);
+sub ClearSyncFlag();
+sub TestBusy();
+sub ClearPid();
+sub ClearAddress();
+sub Fatal($);
+sub debug($);
+sub GetImageInfo($$);
+
+# Defines
+my $FRISBEED	= "$TB/sbin/frisbeed";
+my $LOGFILE	= "$TB/log/frisbeelauncher";
+my $FRISBEEIMAGE= "$TB/sbin/frisbeeimage";
+my $child_pid   = 0;
+my $dlfilename  = 0;
+
+my $STD_BW	= 72000000; 	# 71.6Mb/sec w/1000HZ kernel
+my $USR_BW	= 54000000;	# 53.7Mb/sec w/1000HZ kernel
+
+my $mcast_ip_file = "/var/emulab/boot/myip";
+
+if (! -e $mcast_ip_file) {
+    die("$mcast_ip_file does not exist!");
+}
+
+$MY_IP = `cat $mcast_ip_file`;
+chomp($MY_IP);
+if ($MY_IP =~ /^([\d\.]+)$/) {
+    $MY_IP = $1;
+}
+else {
+    die("Could not parse outer IP: $MY_IP!");
+}
+
+#
+# Ask the outer Emulab to lauch a frisbee and return the loadinfo to us.
+#
+ libxmlrpc::Config({"server"  => $RPCSERVER,
+		    "verbose" => 1,
+		    "cert"    => $RPCCERT,
+		    "portnum" => $RPCPORT});
+
+
+#
+# Parse command arguments. Once we return from getopts, all that should be
+# left are the required arguments.
+#
+my %options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (defined($options{"k"})) {
+    $killmode = 1;
+}
+usage()
+    if (! @ARGV);
+
+#
+# Grab the Image.
+#
+# XXX hack: use map + regex to get around taint check
+my ($imageid, $filename, $size, $mtime) = map { /(.*)/; $1 } @ARGV;
+
+#
+# Get node ID via tmcc
+#
+my @tmp;
+if (tmcc(TMCCCMD_NODEID, undef, \@tmp) != 0) {
+	Fatal("Error getting subboss_id!\n");
+}
+my ($subbossid) = @tmp;
+chomp $subbossid;
+
+#if (!$killmode && ! -R $filename) {
+#    Fatal("You do not have permission to read the image file for".
+#	  "$imageid: $filename\n");
+#}
+
+#
+# When running inside an inner Emulab, try to get the image from the
+# outer emulab when it does not exist locally. Of course, it could still
+# be a bogus image.
+#
+# We do this before going into the background as that is what os_setup
+# expects.  Note that we set a signal handler so we can remove any partially
+# downloaded image if interrupted.  We also record our pid so that -k will
+# work in another frisbeelauncher instance.
+#
+
+my $naddress;
+my $address;
+my $pid;
+
+if (!$killmode) {
+	#
+	# Pick an address to use before locking the DB.  Die if unsucessful,
+	# set address if sucessful.  If something goes wrong later or there is
+	# already a laucher active, we will wind up wasting this address,
+	# but hey, we can make more!
+	#
+	$naddress = PickAddress($subbossid, $imageid);
+
+	# Try to discover if some other process is handling this address
+	$address = $$naddress{"address"};
+	$pid     = $$naddress{"frisbee_pid"};
+
+
+	#
+	# Keep the current frisbeed running for another time period.
+	#
+	if ($pid && $address) {
+	    debug("A server ($address) is already running for image $imageid\n");
+	    exit(0);
+	}
+
+	if (-e $filename) {
+		my @img_stat = stat $filename;
+		if (!@img_stat) {
+			Fatal("Unable to get size and mtime for $imageid: $filename\n");
+		}
+
+		my $img_size = $img_stat[7];
+		my $img_mtime = $img_stat[9];
+
+		if (($img_size != $size) || ($img_mtime < $mtime)) {
+			if (!unlink $filename) {
+				Fatal("Unable to unlink $imageid: $filename for updating\n");
+			}
+		}
+	}
+
+	if (! -e $filename) {
+	    debug("Fetching image $filename ($imageid) from boss\n");
+	    $SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;
+
+	    SetSyncFlag(1);
+	    SetPid($PID);
+	    $EUID = $UID;
+	    $dlfilename = $filename;
+	    system($FRISBEEIMAGE, $imageid, $filename);
+	    ClearPid();
+	    ClearSyncFlag();
+	    $dlfilename = 0;
+	    $EUID = 0;
+	    if ($?) {
+		# XXX clear address if sync failed?
+		Fatal("No such image file: $filename!");
+	    }
+	}
+} else {
+    my $result = GetImageInfo($subbossid, $imageid);
+    my $address = $$result{"address"};
+    my $pid = $$result{"frisbee_pid"};
+
+    # Nothing running.
+    exit(0)
+	if (!$address && !$pid);
+
+    if (!$address && $pid) {
+	#
+	# This makes no sense. Just send email.
+	#
+	my $mesg = "Inconsistent DB state. PID ($pid) but no load address!";
+
+	SENDMAIL($TBOPS,
+		 "Frisbee Killer Failed!",
+		 "Image: $imageid\n".
+		 $mesg);
+
+	Fatal($mesg);
+    }
+    if ($address && !$pid) {
+	#
+	# Okay, minor problem. It is possible we caught the launcher between
+	# setting the address and setting the pid. Wait a moment, and then
+	# try again. If still no pid, bail.
+	#
+	sleep(1);
+	# XXX add call to get address and pid
+	$result = GetImageInfo($imageid, $subbossid);
+	$address = $$result{"address"};
+	$pid = $$result{"frisbee_pid"};
+
+	# Okay, situation resolved itself; other frisbeelauncher bailed.
+	exit(0)
+	    if (!$address && !$pid);
+
+	#
+	# Still inconsistent so bail.
+	#
+	if ($address && !$pid) {
+	    my $mesg = "Inconsistent DB state. Load address but no PID!";
+
+	    SENDMAIL($TBOPS,
+		     "Frisbee Killer Failed!",
+		     "Image: $imageid\n".
+		     $mesg);
+
+	    Fatal($mesg);
+	}
+    }
+
+    #
+    # Okay, address and pid. We could clear the pid from the DB,
+    # preventing another killer from thinking it is running, but not
+    # much point since it is not likely to happen. If it turns out to
+    # be a problem we can change the way this works.
+    #
+    if (! kill('TERM', $pid)) {
+	SENDMAIL($TBOPS,
+		 "Frisbee Killer Failed!",
+		 "Failed to stop frisbee daemon for $imageid\n".
+		 "Could not kill(TERM) process $pid: $? $!");
+
+	Fatal("Failed to stop frisbee daemon for $imageid!");
+    }
+    exit(0);
+}
+
+# Ok, no process is already running for this image, we will use the new
+# address chosen earlier.
+#
+debug("Picked address $address\n