Commit 5b52831c authored by Kirk Webb's avatar Kirk Webb

Well, here it is: The checkin implementing robust recovery/retry and

asynchronous safe termination in plab allocation/deallocation/setup.

Here are some of the more prominent changes/additions:

* Bounded plab agent communication
  Scripts should never hang waiting for plab xmlrpc commands to complete;
  they have their own internal timeouts.  Node.create() in libplab is an
  exception, but is always run under a timeout constraint in vnode_setup
  and can be changed easily if the need arises.

* Wrote functions in libplab to do the retry/recovery/timeout of remote
  command exection.

* Wrapped critical sections with a signal watcher.

* Added code to handle various error conditions properly

* Added a libtestbed function, TBForkCmd, which runs a given program in
  a child process, and can optionally catch incoming SIGTERMs and terminate
  the child (then exit itself).

* Fixed up vnode_setup to batch the 'plabnode free' operation along with
  a few other cleanups.  This should alleviate Jay's concern about how
  long it used to take to teardown a plab expt.

* Whacked plabmonitord into better shape; fixed a couple bugs, taught it how
  to daemonize, and implemented a priority list for testing broken plab nodes.
  This list causes new (as yet unseen) nodes to be tried first over ones that
  have been tested already.
parent d639dbac
......@@ -13,7 +13,7 @@ use Exporter;
@EXPORT =
qw ( SENDMAIL OPENMAIL TBTimeStamp TBBackGround TBDateTimeFSSafe
TBMakeLogname TB_BOSSNODE TB_OPSEMAIL TBGenSecretKey TBDebugTimeStamp
TBDebugTimeStampsOn );
TBDebugTimeStampsOn TBForkCmd);
# After package decl.
use English;
......@@ -263,4 +263,53 @@ sub TBGenSecretKey()
return $key;
}
#
# Fork+exec a command and return its exit value. This is similar to
# system(), but does not use a shell to invoke the command. The function
# exits with the return value from wait().
#
# If the second optional param is passed and true, then
# a signal handler for TERM will be installed, and the
# child process will be sent a SIGTERM if this (the calling)
# process gets one. The handler exits with the exit status returned by
# wait() after sending the signal.
#
sub TBForkCmd($;$) {
my ($cmd, $dokill) = @_;
my $childpid = fork();
if ($childpid) {
my $handler = sub {
kill("TERM", $childpid);
my $exstat = wait();
print STDERR "*** $0:\n".
" Command terminated: $cmd.\n";
exit($exstat);
};
local $SIG{TERM} = \&$handler if (defined($dokill) && $dokill);
my $waitpid = wait();
my $exitstatus = $?;
if ($waitpid < 0) {
die("*** $0:\n".
" Uh oh, wait() returned a negative number");
}
elsif ($waitpid != $childpid) {
warn("*** $0:\n".
" pid returned by wait() != pid ".
"from fork(): $waitpid $childpid");
}
return $exitstatus;
}
else {
exec($cmd);
die("*** $0:\n".
" exec of $cmd failed!\n");
}
# NOTREACHED
return(0);
}
1;
This diff is collapsed.
......@@ -10,6 +10,10 @@ import libplab
GETFREE_PERIOD = 2*60
RENEW_PERIOD = 60*60
# maximum number of times (in a row) the daemon loop can
# catch an exception before bailing out.
MAXCONSECEXC = 3
def usage(me):
print "Usage: %s [ -vd ] { getfree [-u] | renew }" % me
print " Passing -u to getfree will cause it to only update"
......@@ -71,12 +75,40 @@ def doDaemon(func, period, logname):
calls the given func every period seconds.
"""
import time
import traceback
consecexc = MAXCONSECEXC
if not libplab.debug:
daemonize(logname)
while True:
start = time.clock()
func()
try:
func()
except libplab.SignalInterrupt, e:
print "Received signal %s in daemon loop, exiting." % e.signum
sys.exit(0)
except KeyboardInterrupt:
print "Received keyboard interrupt in daemon loop, exiting."
sys.exit(1)
except:
print "Exception caught in plab daemon loop:"
print "".join(traceback.format_exception(*sys.exc_info()))
consecexc -= 1
if consecexc > 0:
print "Going back to sleep until next scheduled run"
else:
print "Too many consecutive exceptions seen, bailing out!"
libplab.SENDMAIL(libplab.TBOPS, "Plabdaemon Exiting",
"The plab %s daemon has seen too many "
"consecutive exceptions and is bailing out."
"Someone needs to check the log!" %
func.func_name)
raise
else:
consecexc = MAXCONSECEXC
end = time.clock()
if end - start < period:
wait = period - (end - start)
......@@ -107,7 +139,7 @@ def main(args):
elif command == "renew":
doDaemon(plab.renew, RENEW_PERIOD, "plabrenew")
else:
usage(ms)
usage(me)
except getopt.GetoptError:
usage(me)
......
......@@ -70,7 +70,7 @@ sub TimeStamp()
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
......@@ -81,12 +81,34 @@ if (defined($options{"d"})) {
$debug = 1;
}
#
# Local vars
#
my $logfile = "$TB/log/plabmonitord";
my @nodes = ();
my $SLEEPINT = 300; # five minutes between alloc retries.
#
# daemonize
#
if (!$debug) {
if (TBBackGround($logfile)) {
exit(0);
}
}
print "Plab Monitor Daemon starting... pid $$, at ".`date`;
#
# We want list of all vnodes in our special experiment, whose pnodes are
# in hwdown. These are the nodes we test, hoping to move them out of
# hwdown.
#
while (1) {
print "--------------------------------------------------------------".
"----------------\n";
my $query_result =
DBQueryWarn("select r1.node_id,n1.phys_nodeid from reserved as r1 ".
"left join nodes as n1 on n1.node_id=r1.node_id ".
......@@ -95,34 +117,67 @@ while (1) {
" r1.eid='$PLABMOND_EID' and ".
" r2.pid='$NODEDEAD_PID' and ".
" r2.eid='$NODEDEAD_EID' ".
"limit 1");
"order by rand()");
if (!$query_result) {
print "Failed to get node list from DB! Waiting a bit ...\n";
goto loop;
}
#
# Build up current node list
#
my @newnodes = ();
while (my ($vnode,$pnode) = $query_result->fetchrow_array()) {
if (search($vnode, @nodes)) {
push (@newnodes, [$vnode, $pnode]);
}
else {
unshift(@newnodes, [$vnode, $pnode]);
}
}
@nodes = @newnodes;
foreach my $nmap (@nodes) {
my ($vnode, $pnode) = @$nmap;
my $revive = 0;
sleep(5);
print "Checking $vnode on $pnode at " . TimeStamp() . "\n";
sleep(5);
#
# Make sure the node is still in $NODEDEAD_*
#
my $query_result = DBQueryWarn("select node_id from reserved where ".
"node_id = '$pnode' and ".
"pid = '$NODEDEAD_PID' and ".
"eid = '$NODEDEAD_EID'");
if (!$query_result) {
print "Node entry DB check failed! Waiting a bit ...\n";
last;
}
if (!$query_result->num_rows()) {
print "Node was removed out from under us! Continuing on ...\n";
next;
}
print "\n\#\#\# Checking $vnode on $pnode at " . TimeStamp() . "\n";
#
# Try to set it up, wait for ISUP, then tear it down.
#
system("vnode_setup -f -d $PLABMOND_PID $PLABMOND_EID $vnode");
if ($?) {
print "Leaving $pnode in hwdown!\n";
next;
}
if (! TBNodeStateWait($vnode, TBDB_NODESTATE_ISUP, time(), 120)) {
$revive = 1;
}
print "Failed to allocate $vnode on $pnode\n";
} else {
if (! TBNodeStateWait($vnode, TBDB_NODESTATE_ISUP, time(), 120)) {
$revive = 1;
}
}
system("vnode_setup -f -k -d $PLABMOND_PID $PLABMOND_EID $vnode");
if ($?) {
print "Failed to teardown $vnode on $pnode\n";
$revive = 0;
}
......@@ -138,16 +193,18 @@ while (1) {
TimeStamp() . "\n";
TBSetNodeLogEntry($pnode, "root", TB_DEFAULT_NODELOGTYPE(),
"Moved to $PLABHOLDING_EID; ".
"plab node $vnode setup okay by monitor.");
"'Moved to $PLABHOLDING_EID; ".
"plab node $vnode setup okay by monitor.'");
SENDMAIL($TBOPS, "$pnode is alive",
"$pnode has been brought back from the afterworld!".
$TBOPS);
}
} else {
print "Leaving $pnode in hwdown!\n";
}
}
loop:
sleep(60);
sleep($SLEEPINT);
}
exit(0);
......@@ -159,3 +216,14 @@ sub fatal($)
SENDMAIL($TBOPS, "Plab Monitor Died", $msg, $TBOPS);
die($msg);
}
sub search($@)
{
$target = shift;
foreach $elt (@_) {
if ($target eq $elt->[0]) {
return 1;
}
}
return 0;
}
......@@ -32,35 +32,12 @@ def main(args):
slice = plab.loadSlice(pid, eid)
if command == "alloc":
node = slice.createNode(nodeid)
node.addKey("/root/.ssh/identity.pub")
while 1:
try:
node = slice.createNode(nodeid)
except xmlrpclib.Fault, e:
print "XML-RPC Fault happened while attempting " \
"node alloc for %s" % nodeid
print "\tCode: %s, Error: %s" % (e.faultCode, e.faultString)
except (socket.error, xmlrpclib.ProtocolError):
print "Encountered problem communicating with an agent " \
"while setting up plab vnode %s" % nodeid
else:
break
alloctries = alloctries - 1
if alloctries > 0:
print "Sleeping for %s seconds, then retrying alloc on %s" % \
(SLEEPINT, nodeid)
time.sleep(SLEEPINT)
else:
print "Giving up after %s tries" % TRIES
raise
while 1:
try:
node.addKey("/root/.ssh/identity.pub")
node.emulabify()
# XXX This file is redundant
# node.putConfig("/etc/vnodeid", nodeid)
# Note that vnode_setup boots the node
except:
print "Node setup failed on %s" % nodeid
......@@ -78,12 +55,20 @@ def main(args):
node.free()
raise
elif command == "renew":
try:
node = slice.loadNode(nodeid)
except:
print "renew: Node %s isn't allocated" % nodeid
sys.exit(1)
node.renew()
elif command == "free":
try:
node = slice.loadNode(nodeid)
except:
print "Node %s wasn't really allocated" % nodeid
sys.exit(0)
print "free: Node %s wasn't really allocated" % nodeid
sys.exit(1)
node.free()
......
......@@ -273,6 +273,7 @@ foreach my $node (@nodes) {
}
my $children = 0;
my %child_vnodes = ();
while (1) {
......@@ -313,13 +314,14 @@ while (1) {
[$vnode, $pnode, $mode, $jailed, $plab, time()];
$children++;
} else {
my $exval = 0;
# Must change our real UID to root so that ssh will work.
$UID = 0;
if ($plab && $mode eq "setup") {
if (system("$TB/sbin/plabnode alloc $pid $eid $vnode")) {
print "*** $0:\n" .
" Plab node allocation failed\n";
if (TBForkCmd("$TB/sbin/plabnode alloc $pid $eid $vnode", 1)) {
print STDERR "*** $0:\n" .
" Plab node allocation failed\n";
# Should check DB state instead.
exit(99);
}
......@@ -332,12 +334,18 @@ while (1) {
$args .= ($jailed ? "-j " : " ");
$args .= ($plab ? "-p " : " ");
$args .= "$vnode ";
exec("$ssh -host $vnode $CLIENT_BIN/vnodesetup $args");
die("*** $0:\n".
" exec failed!\n");
}
exit(0);
$exval = TBForkCmd("$ssh -host $vnode ".
"$CLIENT_BIN/vnodesetup $args",1);
}
# Free the plab node lease if necessary.
if ($plab && ($mode eq "teardown" || $mode eq "cleanup")) {
exec("$TB/sbin/plabnode free $pid $eid $vnode");
die("*** $0:\n".
" exec failed!\n");
}
exit($exval);
}
} else {
#
......@@ -419,22 +427,6 @@ while (1) {
}
if ($plab) {
#
# I am totally unhappy with this mess.
#
if ((($mode eq "teardown") || ($mode eq "cleanup")) ||
($exitstatus && (($exitstatus >> 8) != 99))) {
#
# Besides teardown/cleanup, we want to do this if the
# vnode setup on the node failed, but only if the
# plabnode alloc worked (it can fail too, exits with 99).
#
if (system("$TB/sbin/plabnode free $pid $eid $vnode")) {
warn("*** $0:\n".
" Plab node free of $vnode failed");
}
}
#
# If the node was in the setup process, then mark its allocstate
# as down so os_setup knows not to bother waiting for it. DEAD
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment