Merge branch 'master' of git-public.flux.utah.edu:/flux/git/emulab-devel

db/Node.pm.in

@@ -592,6 +592,46 @@ sub IsReserved($)
     return 1;
 }
 
+
+sub GetSubboss($$)
+{
+    my ($self, $service, $subboss_id) = @_;
+
+    return 0
+	if (! ref($self));
+
+    my $ref;
+
+    if (defined $self->{"SUBBOSSES"}) {
+	my $ref = $self->{"SUBBOSSES"}->{$service};
+    }
+
+    if (!defined $ref) {
+	my $nodeid = $self->node_id();
+
+	my $query_result =
+	    DBQueryWarn("select * from subbosses " .
+			"where node_id='$nodeid' and " .
+			"service = '$service'");
+
+	return 0
+	    if (!$query_result);
+	return 0
+	    if (!$query_result->numrows);
+
+	if (!defined($self->{"SUBBOSSES"})) {
+	    $self->{"SUBBOSSES"} = {};
+	}
+
+	$ref = $self->{"SUBBOSSES"}->{$service} =
+	    $query_result->fetchrow_hashref();
+    }
+
+    $$subboss_id = $ref->{'subboss_id'};
+
+    return 0;
+}
+
 #
 # Flush the reserve info so it reloads.
 #

tbsetup/GNUmakefile.in

@@ -43,7 +43,7 @@ SBIN_STUFF	= resetvlans console_setup.proxy sched_reload named_setup \
 		  elabinelab snmpit.proxy panic node_attributes \
 		  nfstrace plabinelab smbpasswd_setup smbpasswd_setup.proxy \
 		  rmproj snmpit.proxynew snmpit.proxyv2 pool_daemon \
-		  checknodes_daemon
+		  checknodes_daemon subboss_frisbeelauncher_wrapper
 ifeq ($(ISMAINSITE),1)
 SBIN_STUFF     += repos_daemon
 endif
@@ -95,7 +95,7 @@ SETUID_BIN_SCRIPTS   = node_reboot eventsys_control tarfiles_setup savelogs \
 SETUID_SBIN_SCRIPTS  = mkproj rmgroup mkgroup frisbeelauncher frisbeeimage \
 			rmuser idleswap named_setup exports_setup \
 			sfskey_update setgroups newnode_reboot vnode_setup \
-			elabinelab nfstrace rmproj
+			elabinelab nfstrace rmproj subboss_frisbeelauncher_wrapper
 SETUID_LIBX_SCRIPTS  = console_setup spewlogfile
 
 ifeq ($(SYSTEM),FreeBSD)
@@ -231,6 +231,8 @@ endif
 	chmod u+s $(INSTALL_SBINDIR)/mkgroup
 	chown root $(INSTALL_SBINDIR)/frisbeelauncher
 	chmod u+s $(INSTALL_SBINDIR)/frisbeelauncher
+	chown root $(INSTALL_SBINDIR)/subboss_frisbeelauncher_wrapper
+	chmod u+s $(INSTALL_SBINDIR)/subboss_frisbeelauncher_wrapper
 	chown root $(INSTALL_SBINDIR)/frisbeeimage
 	chmod u+s $(INSTALL_SBINDIR)/frisbeeimage
 	chown root $(INSTALL_SBINDIR)/rmuser

tbsetup/libosload.pm.in

@@ -45,6 +45,7 @@ my $PROJROOT    = "@PROJROOT_DIR@";
 my $MAXRETRIES  = 1;
 
 my $FRISBEELAUNCHER = "$TB/sbin/frisbeelauncher";
+my $SUBBOSS_FRISBEELAUNCHER = "$TB/sbin/subboss_frisbeelauncher_wrapper";
 my $osselect	    = "$TB/bin/os_select";
 my $TBUISP	    = "$TB/bin/tbuisp";
 
@@ -452,6 +453,7 @@ sub osload ($$) {
 	my $class = $nodeobject->class();
 	my $isremote = $nodeobject->isremotenode();
 	my $isvirtnode = $nodeobject->isvirtnode();
+	my $subboss;
 	my $reload_mode;
 	my $reload_func;
 	my $reboot_required;
@@ -465,6 +467,7 @@ sub osload ($$) {
 	    $reload_mode = "Frisbee";
 	    $reload_func = \&SetupReloadFrisbee;
 	    $reboot_required = !$noreboot; # Reboot unless $noreboot flag set
+	    $nodeobject->GetSubboss('frisbee', \$subboss);
 
 	    if (defined($nodeflags{$node}) 
 		&& defined($nodeflags{$node}{'noreboot'})) {
@@ -506,6 +509,7 @@ sub osload ($$) {
 	    'zerofree' => $zerofree,
 	    'prepare'  => $prepare,
 	    'maxwait'  => $maxwait,
+	    'subboss' => $subboss,
 	    'isremote' => $isremote,
 	    'isvirtnode' => $isvirtnode
 	};
@@ -981,6 +985,7 @@ sub SetupReloadFrisbee($)
     my $zerofree      = $reload_info->{'zerofree'};
     my $prepare       = $reload_info->{'prepare'};
     my $isvirtnode    = $reload_info->{'isvirtnode'};
+    my $subboss       = $reload_info->{'subboss'};
     my $osid          = TBNodeDiskloadOSID($node);
 
     #
@@ -1016,6 +1021,25 @@ sub SetupReloadFrisbee($)
     }
 
     foreach my $imageid (@$imageids) {
+	if (defined $subboss) {
+		my $image = Image->Lookup($imageid);
+		my $filename = $image->filename();
+		my @image_stats = stat $filename;
+
+		my $query_result = DBQueryWarn("select sync from subboss_images where " .
+			    "subboss_id = '$subboss' and imageid = '$imageid' and sync = 1");
+
+		if (!$query_result || !$query_result->numrows) {
+			system("$SUBBOSS_FRISBEELAUNCHER " .  ($debug ? "-d ": "") .
+				"$subboss " . "$imageid " . "\"$filename\" " .
+				stat[7] . ' ' . stat[9]);
+			if ($?) {
+			    tberror "Subboss Frisbee Launcher ($imageid on $subboss) failed!";
+			    return -1;
+			}
+			return 0;
+		}
+	}
 	system("$FRISBEELAUNCHER " . ($debug ? "-d ": "") . "$imageid");
 	if ($?) {
 	    tberror "Frisbee Launcher ($imageid) failed!";

tbsetup/subboss_frisbeeimage.in

@@ -24,7 +24,7 @@ my $debug    = 1;
 #
 my $TB		= "@prefix@";
 my $TBOPS       = "@TBOPSEMAIL@";
-my $RPCSERVER   = "@OUTERBOSS_NODENAME@";
+my $RPCSERVER   = "@BOSSNODE@";
 my $RPCPORT     = "@OUTERBOSS_XMLRPCPORT@";
 my $RPCCERT     = "@OUTERBOSS_SSLCERTNAME@";
 

tbsetup/subboss_frisbeelauncher.in

@@ -12,6 +12,8 @@ use Sys::Syslog;
 use English;
 use Socket;
 use Errno qw(EADDRINUSE);
+use File::Basename;
+use File::Path;
 
 #
 # This also kills a running frisbee.
@@ -34,7 +36,7 @@ my $MY_IP;
 my $BASEADDR    = "@FRISEBEEMCASTADDR@";
 my $BASEPORT    = "@FRISEBEEMCASTPORT@";
 
-my $RPCSERVER   = "@OUTERBOSS_NODENAME@";
+my $RPCSERVER   = "@BOSSNODE@";
 my $RPCPORT     = "@OUTERBOSS_XMLRPCPORT@";
 my $RPCCERT     = "@SUBBOSS_SSLCERTNAME@";
 
@@ -65,12 +67,13 @@ sub ClearPid();
 sub ClearAddress();
 sub Fatal($);
 sub debug($);
-sub GetImageInfo($$);
+sub GetLoadAddress($$);
 
 # Defines
 my $FRISBEED	= "$TB/sbin/frisbeed";
 my $LOGFILE	= "$TB/log/frisbeelauncher";
 my $FRISBEEIMAGE= "$TB/sbin/frisbeeimage";
+my $IMAGE_CACHE = "$TB/image_cache";
 my $child_pid   = 0;
 my $dlfilename  = 0;
 
@@ -123,6 +126,7 @@ usage()
 #
 # XXX hack: use map + regex to get around taint check
 my ($imageid, $filename, $size, $mtime) = map { /(.*)/; $1 } @ARGV;
+$filename = "$IMAGE_CACHE/$filename";
 
 #
 # Get node ID via tmcc
@@ -154,6 +158,13 @@ my $naddress;
 my $address;
 my $pid;
 
+$naddress = GetLoadAddress($subbossid, $imageid);
+if (defined $naddress) {
+	# Try to discover if some other process is handling this address
+	$address = $$naddress{"address"};
+	$pid     = $$naddress{"frisbee_pid"};
+}
+
 if (!$killmode) {
 	#
 	# Pick an address to use before locking the DB.  Die if unsucessful,
@@ -161,11 +172,6 @@ if (!$killmode) {
 	# already a laucher active, we will wind up wasting this address,
 	# but hey, we can make more!
 	#
-	$naddress = PickAddress($subbossid, $imageid);
-
-	# Try to discover if some other process is handling this address
-	$address = $$naddress{"address"};
-	$pid     = $$naddress{"frisbee_pid"};
 
 
 	#
@@ -174,6 +180,8 @@ if (!$killmode) {
 	if ($pid && $address) {
 	    debug("A server ($address) is already running for image $imageid\n");
 	    exit(0);
+	} else {
+		$address = PickAddress($subbossid, $imageid);
 	}
 
 	if (-e $filename) {
@@ -200,6 +208,10 @@ if (!$killmode) {
 	    SetPid($PID);
 	    $EUID = $UID;
 	    $dlfilename = $filename;
+	    my $image_dir = dirname($filename);
+	    if (! -d $image_dir && !mkpath($image_dir)) {
+		    Fatal("Unable to create path \"$image_dir\"\n");
+	    }
 	    system($FRISBEEIMAGE, $imageid, $filename);
 	    ClearPid();
 	    ClearSyncFlag();
@@ -211,9 +223,6 @@ if (!$killmode) {
 	    }
 	}
 } else {
-    my $result = GetImageInfo($subbossid, $imageid);
-    my $address = $$result{"address"};
-    my $pid = $$result{"frisbee_pid"};
 
     # Nothing running.
     exit(0)
@@ -240,9 +249,9 @@ if (!$killmode) {
 	#
 	sleep(1);
 	# XXX add call to get address and pid
-	$result = GetImageInfo($imageid, $subbossid);
-	$address = $$result{"address"};
-	$pid = $$result{"frisbee_pid"};
+	$naddress = GetLoadAddress($imageid, $subbossid);
+	$address = $$naddress{"address"};
+	$pid = $$naddress{"frisbee_pid"};
 
 	# Okay, situation resolved itself; other frisbeelauncher bailed.
 	exit(0)
@@ -430,7 +439,7 @@ sub debug($)
 sub PickAddress($$)
 {
     my ($subbossid, $imageid) = @_;
-    my $rval = libxmlrpc::CallMethod("subboss", "alloc_load_address",
+    my $rval = libxmlrpc::CallMethod("subboss", "allocate_load_address",
 				     {"imageid" => "$imageid", "subboss_id" => $subbossid});
 
     if (not defined $rval) {
@@ -444,15 +453,15 @@ sub PickAddress($$)
 # Get current load address and frisbeed pid for the specified imageid on the specified
 # subbossid
 #
-sub GetImageInfo($$)
+sub GetLoadAddress($$)
 {
     my ($subbossid, $imageid) = @_;
-    my $rval = libxmlrpc::CallMethod("subboss", "get_image_info",
+    my $rval = libxmlrpc::CallMethod("subboss", "get_load_address",
 				     {"imageid" => $imageid, "subboss_id" => $subbossid});
 
-    if (not defined $rval) {
-	    Fatal("$$: Failed to get mcast address from server!");
-    }
+    #if (not defined $rval) {
+    #	    Fatal("$$: Failed to get mcast address from server!");
+    #}
 
     return $rval;
 }
@@ -465,7 +474,7 @@ sub ClearAddress()
 {
     debug("Clearing out registered load_address and pid\n");
 
-    my $rval = libxmlrpc::CallMethod("subboss", "dealloc_load_address",
+    my $rval = libxmlrpc::CallMethod("subboss", "clear_load_address",
 				     {"imageid" => "$imageid", "subboss_id" => $subbossid});
 
     if (not defined $rval) {
@@ -482,7 +491,7 @@ sub ClearAddress()
 sub SetPid($)
 {
     my ($pid) = @_;
-    my $rval = libxmlrpc::CallMethod("subboss", "set_pid",
+    my $rval = libxmlrpc::CallMethod("subboss", "set_frisbee_pid",
 				     {"imageid" => "$imageid", "subboss_id" => $subbossid,
 				      "frisbee_pid" => $pid});
 

tbsetup/subboss_frisbeelauncher_wrapper

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2007 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use strict;
+use Getopt::Std;
+use POSIX 'setsid';	 # For &daemonize
+use POSIX ":sys_wait_h"; # For &WNOHANG
+use Sys::Syslog;
+use English;
+use Socket;
+use Errno qw(EADDRINUSE);
+
+sub Fatal($);
+
+#
+# This also kills a running frisbee.
+#
+sub usage()
+{
+    print "Usage: $0 [-d] [-k] <imageid>\n";
+    print "-k:	Kill running frisbee.\n";
+    print "-d:	Print debugging output.\n";
+    exit(1);
+}
+my $optlist  = "dk";
+my $debug    = 0;
+my $killmode = 0;
+
+#
+# Untaint the path
+#
+$ENV{'PATH'} = "/bin:/usr/bin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+# Change to root
+$<=$>=0;
+
+# Parse command arguments. Once we return from getopts, all that should be
+# left are the required arguments.
+#
+my %options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (defined($options{"k"})) {
+    $killmode = 1;
+}
+usage()
+    if (@ARGV < 5);
+
+# XXX HACK UGLY
+# quick and dirty hack to get around taint checking
+# This isn't really an issue anyway since we use the
+# list form of system() which doesn't run the command
+# via the shell (so no globbing or interpretation of
+# metacharacters is done).
+my ($subboss, $imageid, $path, $size, $mtime) = map { /(.*)/; $1 } @ARGV;
+my @command = ('ssh', $subboss);
+
+push @command, '-d' if ($debug);
+push @command, '-k' if ($killmode);
+push @command, ('/usr/testbed/sbin/subboss_frisbeelauncher', $imageid, $path, $size, $mtime);
+
+system(@command);
+if ($?) {
+	Fatal("Failed to launch frisbee on subboss $subboss");
+}
+
+sub Fatal($)
+{
+    my ($msg) = @_;
+
+    die("*** $0:\n".
+	  "    $msg\n");
+}

tmcd/common/config/rc.mksubboss

@@ -36,6 +36,7 @@ sub doboot()
 {
     my $EXTRAFS = "/z";
     my $TFTPBOOT_DIR = "$EXTRAFS/tftpboot";
+    my $IMAGES_DIR = "$EXTRAFS/image_cache";
     my $stuffdir = "$EXTRAFS/tmp";
     my $MFSTARBALL = "tftpboot-subboss.tar.gz";
 
@@ -47,8 +48,9 @@ sub doboot()
     system("/usr/local/etc/rc.d/tftpd-hpa.sh stop");
     RecreateDir($EXTRAFS,1);
     mysystem("$BINDIR/mkextrafs.pl -f $EXTRAFS");
-    mysystem("mkdir -p $TFTPBOOT_DIR $stuffdir");
+    mysystem("mkdir -p $TFTPBOOT_DIR $IMAGES_DIR $stuffdir");
     mysystem("ln -sf $TFTPBOOT_DIR /tftpboot");
+    mysystem("ln -sf $IMAGES_DIR $TB/image_cache");
 
     print "Copying over tftpboot tar file from web server and unpacking\n";
 

tmcd/decls.h

@@ -26,4 +26,4 @@
  * NB: See ron/libsetup.pm. That is version 4! I'll merge that in. 
  */
 #define DEFAULT_VERSION		2
-#define CURRENT_VERSION		30
+#define CURRENT_VERSION		31

tmcd/freebsd/rc.frisbee

@@ -140,6 +140,8 @@ loadone() {
     echo "Loading image #$NUM"
 
     ADDRESS=`echo $LOADINFO | sed -e 's/.*ADDR=\([^[:space:]]*\).*/\1/'`
+    SERVER=`echo $LOADINFO | sed -e 's/.*SERVER=\([^[:space:]]*\).*/\1/'`
+    SERVER=${SERVER:-$BOSSIP}
     PARTITION=`echo $LOADINFO | sed -e 's/.*PART=\([^[:space:]]*\).*/\1/'`
     PARTITION=${PARTITION:-'0'}
     PARTOS=`echo $LOADINFO | sed -e 's/.*PARTOS=\([^[:space:]]*\).*/\1/'`
@@ -339,8 +341,8 @@ loadone() {
 	    echo "Running /usr/local/bin/imageunzip -o -O -W 32 $ZFILL $imagefile /dev/${DISK}s${PARTITION}"
 	    /usr/local/bin/imageunzip -o -O -W 32 $ZFILL $imagefile /dev/${DISK}s${PARTITION}
 	else
-	    echo "Running $BINDIR/frisbee $LOADIP $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK at `date`"
-	    $BINDIR/frisbee $LOADIP $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK
+	    echo "Running $BINDIR/frisbee -S $SERVER $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK at `date`"
+	    $BINDIR/frisbee -S $SERVER $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK
 	fi
         fstat=$?
 
@@ -395,9 +397,6 @@ STATUS=`$BINDIR/tmcc status`
 
 BOSSIP=`echo $BOSSINFO | awk '{ print $2 }'`
 
-# XXX should be part of loadinfo
-LOADIP="-S $BOSSIP"
-
 if [ -x /usr/sbin/ntpdate ]; then
 	/usr/sbin/ntpdate -b $BOSSIP >/dev/null 2>&1
 fi

tmcd/linux/rc.frisbee

@@ -133,6 +133,7 @@ write_image()
 	local slice=$3
 	local zfill=$4
 	local ptype=$5
+	local server=$6
 	local port=""
 	local imagefile=""
 	local frisbee_opts="$FRISBEE_OPTS"
@@ -207,7 +208,8 @@ write_image()
 	$BINDIR/tmcc state RELOADING
 
 	if [ $protocol = frisbee ]; then
-		$BINDIR/frisbee -m $address -p $port $frisbee_opts $disk
+		[ -n "$server" ] || server=$BOSSIP
+		$BINDIR/frisbee -m $address -p $port -S $server $frisbee_opts $disk
 		rc=$?
 
 		if [ $rc -ne 0 ]; then
@@ -233,6 +235,7 @@ handle_loadinfo()
 {
 	local LOADINFO="$@"
 	local ADDRESS=`get_value "$LOADINFO" ADDR`
+	local SERVER=`get_value "$LOADINFO" SERVER`
 	local PARTOS=`get_value "$LOADINFO" PARTOS`
 	local PARTITION=`get_value "$LOADINFO" PART`
 	PARTITION=${PARTITION:-'0'}
@@ -305,7 +308,7 @@ handle_loadinfo()
 		zapsuperblocks /dev/$DISK
 	fi
 
-	write_image $ADDRESS /dev/$DISK $PARTITION $ZFILL $PTYPE
+	write_image $ADDRESS /dev/$DISK $PARTITION $ZFILL $PTYPE $SERVER
 	rc=$?
 	if [ $rc -ne 0 ]; then
 		echo "Failed to write image to disk, status $rc"
@@ -397,7 +400,7 @@ $BINDIR/get_edd_map > $BOOTDIR/edd_map 2>/dev/null
 FRISBEE_OPTS="-S $BOSSIP"
 
 if [ -e $BOOTDIR/myip ]; then
-	FRISBEE_OPTS="$FRISBEE_OPTS -i `cat $BOOTDIR/myip`"
+	FRISBEE_OPTS="-i `cat $BOOTDIR/myip`"
 fi
 IMAGEUNZIP_OPTS="-o -O -W 32"
 

tmcd/tmcd.c

@@ -3841,6 +3841,7 @@ COMMAND_PROTOTYPE(doloadinfo)
 	char		buf[MYBUFSIZE];
 	char		*bufp = buf, *ebufp = &buf[sizeof(buf)];
 	char		*disktype, *useacpi, *useasf, address[MYBUFSIZE];
+	char            server_address[MYBUFSIZE];
 	char		mbrvers[51];
 	char            *loadpart, *OS, *prepare;
 	int		disknum, nrows, zfill;
@@ -3912,11 +3913,12 @@ COMMAND_PROTOTYPE(doloadinfo)
 		OS = row[2];
 		prepare = row[8];
 
-		res2 = mydb_query("select load_address,frisbee_pid from subboss_images as i "
+		res2 = mydb_query("select load_address,frisbee_pid,IP from subboss_images as i "
 				 "left join subbosses as s on s.subboss_id = i.subboss_id "
+				 "left join interfaces as n on n.node_id =  s.subboss_id "
 				 "where s.node_id = '%s' and s.service = 'frisbee' and "
 				 "i.imageid = '%s' and i.load_address != '' and "
-				 "i.sync != 1", 2, reqp->nodeid, row[7]);
+				 "n.role='ctrl' and i.sync != 1", 3, reqp->nodeid, row[7]);
 
 		if (!res2) {
 			error("doloadinfo: %s: DB Error getting subboss info!\n",
@@ -3927,6 +3929,7 @@ COMMAND_PROTOTYPE(doloadinfo)
 
 		frisbee_pid = 0;
 		address[0] = '\0';
+		server_address[0] = '\0';
 
 		if (mysql_num_rows(res2)) {
 			row2 = mysql_fetch_row(res2);
@@ -3936,12 +3939,16 @@ COMMAND_PROTOTYPE(doloadinfo)
 
 			if (row2[1] && row2[1][0])
 				frisbee_pid = atoi(row2[1]);
+			
+			strcpy(server_address, row2[2]);
 		} else {
 			if (row[0] && row[0][0])
 				strcpy(address, row[0]);
 
 			if (row[3] && row[3][0])
 				frisbee_pid = atoi(row[3]);
+			
+			strcpy(server_address, BOSSNODE_IP);
 		}
 
 		mysql_free_result(res2);
@@ -3960,6 +3967,8 @@ COMMAND_PROTOTYPE(doloadinfo)
 			OUTPUT(address, sizeof(address),
 			       "%s/spewimage.php?imageid=%s&access_key=%s",
 			       TBBASE, row[7], row[6]);
+			
+			server_address[0] = 0;
 		}
 		else {
 			/*
@@ -3984,6 +3993,11 @@ COMMAND_PROTOTYPE(doloadinfo)
 
 		bufp += OUTPUT(bufp, ebufp - bufp,
 			       "ADDR=%s PART=%s PARTOS=%s", address, loadpart, OS);
+		
+		if (server_address[0] && (vers >= 31)) {
+			bufp += OUTPUT(bufp, ebufp - bufp,
+			               " SERVER=%s", server_address);
+		}
 
 		/*
 		 * Remember zero-fill free space, mbr version fields, and access_key

xmlrpc/emulabserver.py.in

@@ -4364,6 +4364,7 @@ class subboss:
         if not re.match("^[-\@\w\+\.]*$", str(argdict["imageid"])):
             return EmulabResponse(RESPONSE_BADARGS,
                                   output="Improperly formed imageid value!")
+
         
         verifyerror = self.verifystuff();
         if (verifyerror):
@@ -4971,7 +4972,7 @@ class elabinelab:
             stuff += l
             pass
         
-        return EmulabResponse(RESPONSE_SUCCESS, output=output)
+        return EmulabResponse(RESPONSE_SUCCESS, value=stuff, output=output)
     #
     # Fire up a frisbeed for an image,
     #