Commit 1e8d93dd authored by Ryan Jackson's avatar Ryan Jackson

add libxmlrpc stuff for subboss

parent e99c2f4f
......@@ -238,6 +238,7 @@ endif
@$(MAKE) -C tmcd subboss-install
@$(MAKE) -C rc.d subboss-install
@$(MAKE) -C dhcpd subboss-install
@$(MAKE) -C tbsetup client-install
# Hack: this should probably be done by a subboss-install
# script, but there isn't a huge need for that yet
@rm -f $(INSTALL_RCDIR)/isc-dhcpd
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -101,7 +101,7 @@ AC_SUBST(optional_subdirs)
# These get defined in the definitions file (--with-TBDEFS).
# NB: See AC_DEFINE_UNQUOTED() calls below, if you want a variable
# to appear in config.h.
#
#
AC_SUBST(TBDBNAME)
AC_SUBST(TBERRORLOGDBNAME)
AC_SUBST(TBADMINGROUP)
......@@ -156,6 +156,7 @@ AC_SUBST(ELABINELAB)
AC_SUBST(OUTERBOSS_NODENAME)
AC_SUBST(OUTERBOSS_XMLRPCPORT)
AC_SUBST(OUTERBOSS_SSLCERTNAME)
AC_SUBST(SUBBOSS_SSLCERTNAME)
AC_SUBST(PLABSUPPORT)
AC_SUBST(PUBSUPPORT)
AC_SUBST(WIKISUPPORT)
......@@ -250,7 +251,7 @@ AC_SUBST(TBERRORSEMAIL_NOSLASH)
#
# Defaults for for above variables.
#
#
TBERRORLOGDBNAME="errorlog"
TBADMINGROUP="flux"
CLIENT_ETCDIR="/etc/emulab"
......@@ -270,6 +271,7 @@ ELABINELAB=0
OUTERBOSS_NODENAME=""
OUTERBOSS_XMLRPCPORT=3069
OUTERBOSS_SSLCERTNAME="/etc/outer_emulab.pem"
SUBBOSS_SSLCERTNAME="/etc/subboss.pem"
PLABSUPPORT=0
PUBSUPPORT=0
WIKISUPPORT=0
......@@ -431,7 +433,7 @@ AC_SUBST(LOG_TESTBED)
# If the private network is different then the control network, we need
# an extra subnet decl for it in dhcpd.conf.template. This is enormously
# crude and simplistic, but works for the local case!
#
#
if test "$PRIVATE_NETWORK" != "$CONTROL_NETWORK"; then
DHCPD_CONTROLNET_DECL="subnet $PRIVATE_NETWORK netmask $PRIVATE_NETMASK {}"
fi
......@@ -660,7 +662,7 @@ AC_SUBST(OPT_CFLAGS)
#
# Disable event system. Relies on Elvin. On by default.
#
#
AC_ARG_ENABLE(events,
[ --disable-events Disable events (requires Elvin libraries)])
......@@ -715,7 +717,7 @@ fi
# Enable Windows support.
# Right now this means NTFS support in imagezip, maybe more things later.
# Relies on Linux NTFS library.
#
#
AC_ARG_ENABLE(windows,
[ --enable-windows Enable Windows XP support (default)])
......@@ -868,6 +870,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/wanlinkinfo tbsetup/wanassign \
tbsetup/swapexp tbsetup/endexp tbsetup/elabinelab \
tbsetup/frisbeelauncher tbsetup/eventsys.proxy tbsetup/frisbeeimage \
tbsetup/subboss_frisbeelauncher tbsetup/subboss_frisbeeimage \
tbsetup/snmpit.proxy tbsetup/snmpit_remote.pm \
tbsetup/snmpit tbsetup/ns2ir/GNUmakefile \
tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
......
......@@ -48,6 +48,8 @@ ifeq ($(ISMAINSITE),1)
SBIN_STUFF += repos_daemon
endif
SUBBOSS_SBIN_STUFF = subboss_frisbeelauncher subboss_frisbeeimage
CTRLSBIN_STUFF = console_setup.proxy sfskey_update.proxy \
savelogs.proxy eventsys.proxy
......@@ -266,6 +268,11 @@ endif
chown root $(INSTALL_SBINDIR)/nfstrace
chmod u+s $(INSTALL_SBINDIR)/nfstrace
#
# Subboss installation
#
subboss-install: $(addprefix $(INSTALL_SBINDIR)/, $(SUBBOSS_SBIN_STUFF))
#
# Control node installation (okay, plastic)
#
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002, 2004, 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
#
# Ask outer emulab for an image we do not happen to have locally.
#
sub usage()
{
print STDOUT "Usage: frisbeeimage [-d] imageid filename\n";
exit(-1);
}
my $optlist = "d";
my $debug = 1;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $RPCSERVER = "@OUTERBOSS_NODENAME@";
my $RPCPORT = "@OUTERBOSS_XMLRPCPORT@";
my $RPCCERT = "@OUTERBOSS_SSLCERTNAME@";
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
die("Must be root! Maybe its a development version?\n");
}
# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libtestbed;
use libxmlrpc;
# Locals
my $FRISBEE = "$TB/sbin/frisbee";
my $SAVEUID = $UID;
my $loadaddr;
my $loadport;
my $mcastif;
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (! @ARGV) {
usage();
}
my ($imageid, $filename) = @ARGV;
#
# Untaint the arguments.
#
if ($imageid =~ /^([-\@\w\+\.]+)$/) {
$imageid = $1;
}
else {
die("Tainted argument $imageid!\n");
}
my $mcast_ip_file = "/var/emulab/boot/myip";
if (! -e $mcast_ip_file) {
die("$mcast_ip_file does not exist!");
}
$mcastif = `cat $mcast_ip_file`;
chomp($mcastif);
if ($mcastif =~ /^([\d\.]+)$/) {
$mcastif = $1;
}
else {
die("Could not parse outer IP: $mcastif!");
}
#
# Ask the outer Emulab to lauch a frisbee and return the loadinfo to us.
#
libxmlrpc::Config({"server" => $RPCSERVER,
"verbose" => 1,
"cert" => $RPCCERT,
"portnum" => $RPCPORT});
my $rval = libxmlrpc::CallMethod("subboss", "frisbeelauncher",
{"imageid" => "$imageid"});
if (!defined($rval)) {
die("Could not fire up frisbee on boss!");
}
#
# The return value is the loadaddr. Parse that into something we
# can pass to the frisbee client.
#
if ($rval =~ /^(.*):(\d*)$/) {
$loadaddr = $1;
$loadport = $2;
}
else {
die("Could not parse loadinfo from server: $rval!");
}
if ($debug) {
print "$FRISBEE -N -i $mcastif -m $loadaddr -p $loadport $filename\n";
}
# XXX HACK
$filename =~ /(.*)/;
$filename = $1;
$SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;
system($FRISBEE, '-N', '-i', $mcastif, '-m', $loadaddr,
'-p', $loadport, $filename);
if ($?) {
die("Error downloading image data from outer Emulab!");
}
exit(0);
sub cleanup {
tbwarn("Download interrupted, $filename removed");
$SIG{HUP} = $SIG{INT} = $SIG{TERM} = 'IGNORE';
kill('TERM', 0);
unlink($filename);
exit(-1);
}
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
use Getopt::Std;
use POSIX 'setsid'; # For &daemonize
use POSIX ":sys_wait_h"; # For &WNOHANG
use Sys::Syslog;
use English;
use Socket;
use Errno qw(EADDRINUSE);
#
# This also kills a running frisbee.
#
sub usage()
{
print "Usage: $0 [-d] [-k] <imageid> <path> <size> <mtime>\n";
print "-k: Kill running frisbee.\n";
print "-d: Print debugging output.\n";
exit(1);
}
my $optlist = "dk";
my $debug = 0;
my $killmode = 0;
# Configure variables
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $MY_IP;
my $BASEADDR = "@FRISEBEEMCASTADDR@";
my $BASEPORT = "@FRISEBEEMCASTPORT@";
my $RPCSERVER = "@OUTERBOSS_NODENAME@";
my $RPCPORT = "@OUTERBOSS_XMLRPCPORT@";
my $RPCCERT = "@SUBBOSS_SSLCERTNAME@";
#
# Untaint the path
#
$ENV{'PATH'} = "/bin:/usr/bin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
use lib "@prefix@/lib";
use libtestbed;
use libxmlrpc;
use libtmcc;
#use libtblog;
# Protos.
sub PickAddress($$);
sub SetPid($);
sub SetSyncFlag($);
sub ClearSyncFlag();
sub TestBusy();
sub ClearPid();
sub ClearAddress();
sub Fatal($);
sub debug($);
sub GetImageInfo($$);
# Defines
my $FRISBEED = "$TB/sbin/frisbeed";
my $LOGFILE = "$TB/log/frisbeelauncher";
my $FRISBEEIMAGE= "$TB/sbin/frisbeeimage";
my $child_pid = 0;
my $dlfilename = 0;
my $STD_BW = 72000000; # 71.6Mb/sec w/1000HZ kernel
my $USR_BW = 54000000; # 53.7Mb/sec w/1000HZ kernel
my $mcast_ip_file = "/var/emulab/boot/myip";
if (! -e $mcast_ip_file) {
die("$mcast_ip_file does not exist!");
}
$MY_IP = `cat $mcast_ip_file`;
chomp($MY_IP);
if ($MY_IP =~ /^([\d\.]+)$/) {
$MY_IP = $1;
}
else {
die("Could not parse outer IP: $MY_IP!");
}
#
# Ask the outer Emulab to lauch a frisbee and return the loadinfo to us.
#
libxmlrpc::Config({"server" => $RPCSERVER,
"verbose" => 1,
"cert" => $RPCCERT,
"portnum" => $RPCPORT});
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"k"})) {
$killmode = 1;
}
usage()
if (! @ARGV);
#
# Grab the Image.
#
# XXX hack: use map + regex to get around taint check
my ($imageid, $filename, $size, $mtime) = map { /(.*)/; $1 } @ARGV;
#
# Get node ID via tmcc
#
my @tmp;
if (tmcc(TMCCCMD_NODEID, undef, \@tmp) != 0) {
Fatal("Error getting subboss_id!\n");
}
my ($subbossid) = @tmp;
chomp $subbossid;
#if (!$killmode && ! -R $filename) {
# Fatal("You do not have permission to read the image file for".
# "$imageid: $filename\n");
#}
#
# When running inside an inner Emulab, try to get the image from the
# outer emulab when it does not exist locally. Of course, it could still
# be a bogus image.
#
# We do this before going into the background as that is what os_setup
# expects. Note that we set a signal handler so we can remove any partially
# downloaded image if interrupted. We also record our pid so that -k will
# work in another frisbeelauncher instance.
#
my $naddress;
my $address;
my $pid;
if (!$killmode) {
#
# Pick an address to use before locking the DB. Die if unsucessful,
# set address if sucessful. If something goes wrong later or there is
# already a laucher active, we will wind up wasting this address,
# but hey, we can make more!
#
$naddress = PickAddress($subbossid, $imageid);
# Try to discover if some other process is handling this address
$address = $$naddress{"address"};
$pid = $$naddress{"frisbee_pid"};
#
# Keep the current frisbeed running for another time period.
#
if ($pid && $address) {
debug("A server ($address) is already running for image $imageid\n");
exit(0);
}
if (-e $filename) {
my @img_stat = stat $filename;
if (!@img_stat) {
Fatal("Unable to get size and mtime for $imageid: $filename\n");
}
my $img_size = $img_stat[7];
my $img_mtime = $img_stat[9];
if (($img_size != $size) || ($img_mtime < $mtime)) {
if (!unlink $filename) {
Fatal("Unable to unlink $imageid: $filename for updating\n");
}
}
}
if (! -e $filename) {
debug("Fetching image $filename ($imageid) from boss\n");
$SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;
SetSyncFlag(1);
SetPid($PID);
$EUID = $UID;
$dlfilename = $filename;
system($FRISBEEIMAGE, $imageid, $filename);
ClearPid();
ClearSyncFlag();
$dlfilename = 0;
$EUID = 0;
if ($?) {
# XXX clear address if sync failed?
Fatal("No such image file: $filename!");
}
}
} else {
my $result = GetImageInfo($subbossid, $imageid);
my $address = $$result{"address"};
my $pid = $$result{"frisbee_pid"};
# Nothing running.
exit(0)
if (!$address && !$pid);
if (!$address && $pid) {
#
# This makes no sense. Just send email.
#
my $mesg = "Inconsistent DB state. PID ($pid) but no load address!";
SENDMAIL($TBOPS,
"Frisbee Killer Failed!",
"Image: $imageid\n".
$mesg);
Fatal($mesg);
}
if ($address && !$pid) {
#
# Okay, minor problem. It is possible we caught the launcher between
# setting the address and setting the pid. Wait a moment, and then
# try again. If still no pid, bail.
#
sleep(1);
# XXX add call to get address and pid
$result = GetImageInfo($imageid, $subbossid);
$address = $$result{"address"};
$pid = $$result{"frisbee_pid"};
# Okay, situation resolved itself; other frisbeelauncher bailed.
exit(0)
if (!$address && !$pid);
#
# Still inconsistent so bail.
#
if ($address && !$pid) {
my $mesg = "Inconsistent DB state. Load address but no PID!";
SENDMAIL($TBOPS,
"Frisbee Killer Failed!",
"Image: $imageid\n".
$mesg);
Fatal($mesg);
}
}
#
# Okay, address and pid. We could clear the pid from the DB,
# preventing another killer from thinking it is running, but not
# much point since it is not likely to happen. If it turns out to
# be a problem we can change the way this works.
#
if (! kill('TERM', $pid)) {
SENDMAIL($TBOPS,
"Frisbee Killer Failed!",
"Failed to stop frisbee daemon for $imageid\n".
"Could not kill(TERM) process $pid: $? $!");
Fatal("Failed to stop frisbee daemon for $imageid!");
}
exit(0);
}
# Ok, no process is already running for this image, we will use the new
# address chosen earlier.
#
debug("Picked address $address\n");
# Run in the background
if (my $childpid = TBBackGround($LOGFILE)) {
#
# Delay a moment, and they look for an exit status.
# This is intended to catch startup problems.
#
# XXX TBBackGround should do this!
#
sleep(1);
my $foo = waitpid($childpid, &WNOHANG);
if ($foo) {
#ClearAddress();
Fatal("Error $? backgrounding frisbeelauncher!");
}
exit(0);
}
#XXX TBdbfork();
# Set up a signal handler that will clean up in case we get killed
$SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;
# Set our pid. This happens outside the lock which could lead to races,
# but that is unlikely. Look for it above though.
SetPid($PID);
#
# Drop root permissions, if we have them
#
if ($EUID == 0) {
$EUID = $UID;
}
my @args = ('-i', $MY_IP);
#
# Select the appropriate bandwidth
#
if ($filename =~ /^$TB\/images/) {
push @args, ('-W', $STD_BW);
}
#
# Force multicast keepalives
# XXX Should we do this for subbosses?
#
push @args, ('-K', '15');
my $firsttry = 1;
# Now, we actually launch Frisbee
while (1) {
#
# Each time the server exits, test the busy bit to see if
# it should keep going. This has to be done with tables locked
# since another caller is going to bump it.
#
last if (!TestBusy());
if ($child_pid = fork()) {
# Wait for child to exit
waitpid($child_pid, 0);
if ($?) {
#
# If this is the first time we are firing up this server
# and the port is already in use, try another port.
#
# If a server was running, died, and now we can no longer
# get the port, something else might be wrong so just die
# in the usual way.
#
my $err = $?;
if ($firsttry && ($err >> 8) == EADDRINUSE()) {
warn("Frisbeed bind failed for address $address, ".
"picking another address\n");
ClearAddress();
$address = PickAddress($subbossid, $imageid);
next;
}
#SENDMAIL($TBOPS, "Frisbeed Failed!",
# "Image: $imageid\n".
# "Address: $address\n\n".
# "Process $child_pid exited with value $err.\n".
# "Please look at the syslog for frisbeed!\n\n".
# "NOTE: Another frisbeed will not start!\n");
print STDERR "Frisbeed Failed!",
"Image: $imageid\n".
"Address: $address\n\n".
"Process $child_pid exited with value $err.\n".
"Please look at the syslog for frisbeed!\n\n".
"NOTE: Another frisbeed will not start!\n";
#
# Dump early. This will leave the address in
# in the DB, so that another one will not start
# until the matter is resolved by someone.
#
ClearPid();
exit(1);
}
else {
$firsttry = 0;
}
}
else {
print STDERR "WTF???\n";
# Child branch
# The database format for address is host:port - however,
# we need to give them as seperate arguments to frisbeed.
if ($address =~ /(.*):(.*)/) {
my $addr = $1;
my $port = $2;
print STDERR "$FRISBEED @args -m $addr -p $port $filename";
if (!exec($FRISBEED, @args, '-m', $addr, '-p', $port, $filename)) {
#Fatal("$$: Unable to exec $FRISBEED");
print STDERR "$$: Unable to exec $FRISBEED\n";
}
}
Fatal("$$: Bad address format: $address!");
}
}
ClearAddress();
exit(0);
######################################################################
# Subroutines
######################################################################
# Only print if -d option was given. Also add $$ on the beginning of the
# string, to ease debugging
sub debug($)
{
my ($msg) = @_;
print "$$: $msg"
if ($debug);
}
#
# Pick out an address to use.
#
sub PickAddress($$)
{
my ($subbossid, $imageid) = @_;
my $rval = libxmlrpc::CallMethod("subboss", "alloc_load_address",
{"imageid" => "$imageid", "subboss_id" => $subbossid});
if (not defined $rval) {
Fatal("$$: Failed to get mcast address from server!");
}
return $rval;
}
#
# Get current load address and frisbeed pid for the specified imageid on the specified
# subbossid
#
sub GetImageInfo($$)
{
my ($subbossid, $imageid) = @_;
my $rval = libxmlrpc::CallMethod("subboss", "get_image_info",
{"imageid" => $imageid, "subboss_id" => $subbossid});
if (not defined $rval) {
Fatal("$$: Failed to get mcast address from server!");
}
return $rval;
}
#
# Clear out the address (and pid) registered to this process
# Might be called with tables locked.
#
sub ClearAddress()
{
debug("Clearing out registered load_address and pid\n");
my $rval = libxmlrpc::CallMethod("subboss", "dealloc_load_address",
{"imageid" => "$imageid", "subboss_id" => $subbossid});
if (not defined $rval) {
Fatal("$$: Failed to clear mcast address from server!");
}
return $rval;
}
#
# Set and clear the frisbee process ID.
# Always called with tables unlocked.
#
sub SetPid($)
{
my ($pid) = @_;
my $rval = libxmlrpc::CallMethod("subboss", "set_pid",
{"imageid" => "$imageid", "subboss_id" => $subbossid,
"frisbee_pid" => $pid});
if (not defined $rval) {
Fatal("$$: Failed to set pid to $pid!");
}
return $rval;
}
sub ClearPid()
{
SetPid(0);
}
#
# Set and clear the image sync flag for this subboss
# Always called with tables unlocked ???.
#
sub SetSyncFlag($)
{