Commit 1daaa992 authored by Mac Newbold's avatar Mac Newbold

The new and fully functional rebooting-via-events stuff and the

really-reboot-nodes-that-timeout stuff.

NOTE: Until the timeout/retry stuff is gone from os_load/os_setup, it is
disabled in stated. It will still only send email. But all the stuff is
there and has been tested.

NOTE: Until other things don't depend on the old behavior of node_reboot
(when it returns, all nodes are in SHUTDOWN), the event stuff is disabled.
Real mode is the default, and can be run by anyone.

In short, this commit is new versions of stated and node_reboot that act
almost exactly like the old ones. But I wanted to commit them before I go
on making a bunch more changes, to have a checkpoint that I know works.
parent 243921d5
......@@ -115,8 +115,11 @@ my $TBREBOOT = TBDB_COMMAND_REBOOT;
my $TBPOWEROFF = TBDB_COMMAND_POWEROFF;
my $TBPOWERON = TBDB_COMMAND_POWERON;
my $TBPOWERCYCLE = TBDB_COMMAND_POWERCYCLE;
my $TB_OSID_MBKERNEL = TB_OSID_MBKERNEL;
my $TBISUP = TBDB_NODESTATE_ISUP;
my $TBTIMEOUTREBOOT = TBDB_STATED_TIMEOUT_REBOOT;
my $TBTIMEOUTNOTIFY = TBDB_STATED_TIMEOUT_NOTIFY;
my $TBTIMEOUTCMDRETRY = TBDB_STATED_TIMEOUT_CMDRETRY;
my $TB_OSID_MBKERNEL = TB_OSID_MBKERNEL;
# This only gets used here, so it isn't in a lib constant.
my $TBFREENODE = "FREENODE";
......@@ -362,6 +365,9 @@ sub readStates(;@) {
$nodes{$node_id}{timestamp} = $timestamp;
$nodes{$node_id}{mode} = $mode;
$nodes{$node_id}{mode_timestamp} = $mode_timestamp;
$nodes{$node_id}{notified} = 0;
$nodes{$node_id}{timedout} = 0;
$nodes{$node_id}{noretry} = 0;
# Is there a timeout? If so, set it up!
setTimeout($mode,$state,$node_id,$timestamp);
}
......@@ -537,20 +543,22 @@ sub stateTransition($$) {
# to check if we had a pending command
if (qfind($node) &&
$timeout_tag{$node} =~ /^$TBCOMMAND:/) {
debug("TimeoutTag = '$timeout_tag{$node}'\n");
my ($str,$cmd) = split(":",$timeout_tag{$node});
debug("str=$str\tcmd=$cmd\tTBREBOOT=$TBREBOOT\tstate=$newstate\n");
if ($cmd eq $TBREBOOT) {
if ($state eq TBDB_NODESTATE_SHUTDOWN ) {
if ($newstate eq TBDB_NODESTATE_SHUTDOWN ) {
info("$node: $TBREBOOT success\n");
# Timeout will get cleared below by setTimeout call
} else {
notify("$node: $TBREBOOT in progress, but got state $state ".
"instead of ". TBDB_NODESTATE_SHUTDOWN ."!\n");
notify("$node: $TBREBOOT in progress, but got state ".
"$newstate instead of ".TBDB_NODESTATE_SHUTDOWN."!\n");
}
#} elsif ($cmd eq $FOO ) {
# Add more here...
} else {
notify("$node: Unknown command timeout '$timeout_tag{$node}' ".
"found at $mode/$state\n");
"found at $mode/$newstate\n");
}
}
......@@ -611,6 +619,9 @@ sub stateTransition($$) {
foreach ( @trigs , @nodetrigs) {
my $trig = $_;
/^$TBRESET$/ && do {
# We successfully booted, so clear some flags
$nodes{$node}{noretry} = 0;
$nodes{$node}{timedout} = 0;
# Check if we really need to do a reset
my $r = DBQueryWarn("select osid,def_boot_osid from nodes ".
"where node_id='$node'");
......@@ -790,40 +801,60 @@ sub handleCtrlEvent($$) {
}
$nodes{$node}{notified}++;
my $notified = $nodes{$node}{notified};
$nodes{$node}{timedout}++;
my $timedout = $nodes{$node}{timedout};
if ($mode && $state && $timeouts{$mode} &&
$timeouts{$mode}{$state}) {
($timeout, $action) = @{$timeouts{$mode}{$state}};
}
if ($mode eq $TBCOMMAND) {
# It is a command, not a true state
if ($action eq "CMDRETRY") {
if ($action eq $TBTIMEOUTCMDRETRY) {
# Retry the command
notify("$node: Command $state, retry #$notified\n");
notify("$node: Command $state, retry #$timedout\n");
# notify in case we get in a retry loop...
handleCommand($node,$state,$notified);
handleCommand($node,$state,$timedout,1);
} else {
notify("$node: Unknown timeout action for ".
"$mode/$state: '$action'\n");
}
next;
} else {
if ($notified>1) {
notify("$node: Timed out at $now (d=$deadline), ".
"but notified already!\n");
}
}
notify("Node $node has timed out in state $mode/$state".
($action ne "" ? "\n\tRequested action $action." : "").
"\n");
info("Node $node has timed out in state $mode/$state".
($action ne "" ? "\n\tRequested action $action." : "").
"\n");
foreach ($action) {
/^$TBTIMEOUTREBOOT/ && do {
if ($timedout>3) {
# We've tried too many times...
notify("Node $node has timed out too many times!\n".
"Giving up until it boots sucessfully.\n");
$nodes{$node}{noretry} = 1;
} else {
# XXX Temporary! For now notify instead of
# really rebooting, until the timeout/retry
# stuff is gone from os_setup and os_load
notify("Node $node has timed out in state ".
"$mode/$state - REBOOT requested\n");
#handleCommand($node,$TBREBOOT,$timedout,1);
}
last; };
/^$TBTIMEOUTNOTIFY/ && do {
notify("Node $node has timed out in state $mode/$state\n");
last; };
notify("$node: Unknown Timeout Action: $action\n");
}
next;
};
notify("$node: Unknown CtrlEvent: $event\n");
}
}
sub handleCommand($$;$) {
my ($params,$command,$retry) = @_;
sub handleCommand($$;$$) {
my ($params,$command,$retry,$force) = @_;
if (!defined($retry)) { $retry=0; }
if (!defined($force)) { $force=0; }
info("Command: $params, $command (attempt $retry)\n");
......@@ -832,26 +863,53 @@ sub handleCommand($$;$) {
# We may need to do it here (while iterating over the list), or
# make some other fix up in handleEvent.
if ($command eq $TBREBOOT && $retry >=3) {
if ($command eq $TBREBOOT && $retry >=4) {
announce("Node $params has tried rebooting $retry times and has \n".
"still not been successful. Please look into it soon.\n".
"In the meantime, $params will be powered off.\n");
"" );# "In the meantime, $params will be powered off.\n");
# Just return...
return 0;
# change my command to poweroff.
$command = $TBPOWEROFF;
#$command = $TBPOWEROFF;
}
foreach ($command) {
/^$TBREBOOT$/ && do {
# For reboot, the params is a comma-separated list of nodes
my @nodes = split(",",$params);
# Check if any of these are in the middle of things and
# shouldn't be rebooted.
if (!$force) {
foreach $n ( 0 .. $#nodes ) {
$node = $nodes[$n];
debug("Checking rebooting: $node, $nodes{$node}, ".
"$nodes{$node}{state}, $nodes{$node}{noretry}\n");
if (($nodes{$node}{state} ne TBDB_NODESTATE_ISUP) &&
(!$nodes{$node}{noretry}) ) {
# This node shouldn't be rebooted now...
# XXX Send feedback here somehow!
info("$node: Trying to reboot too soon! Skipping.\n");
# Cut it out of the list
debug("Nodelist before ==> ".join(" ",@nodes)."\n");
splice(@nodes,$n,1);
debug("Nodelist after ==> ".join(" ",@nodes)."\n");
}
}
if (@nodes == 0) { next; }
}
my $nodelist=join(" ",@nodes);
info("Rebooting nodes: $nodelist\n");
# Permissions were checked in order to send the message,
# so we don't need to do any fancy stuff here.
my $cmd = "$nodereboot $nodelist &";
debug("$cmd\n") or
#system($cmd) and
my $cmd = "$nodereboot -r $nodelist";
my $redir = " 2>&1 >> /usr/testbed/log/nodereboot.log &";
debug("$cmd $redir\n");
system("date $redir");
system($cmd.$redir) and
notify("$params/$command: ".
"Command '$cmd' failed, error $?: $!\n");
......@@ -874,13 +932,13 @@ sub handleCommand($$;$) {
$TBPOWERON => "on",
$TBPOWEROFF => "off");
my $func = $funcmap{$command};
info("Sending power $func nodes: $nodelist\n");
info("Sending power $func for nodes: $nodelist\n");
# Permissions were checked in order to send the message,
# so we don't need to do any fancy stuff here.
my $cmd = "$power $func $nodelist &";
debug("$cmd\n") or
#system($cmd) and
debug("$cmd\n") and
system($cmd) and
notify("$params/$command: ".
"Command '$cmd' failed, error $?: $!\n");
......
......@@ -18,21 +18,25 @@ use Getopt::Std;
#
sub usage()
{
print STDOUT "Usage: node_reboot [-d] [-f] [-w] node [node ...]\n" .
" node_reboot [-d] [-f] [-w] -e pid,eid\n".
print STDOUT "Usage: node_reboot [-d] [-f] [-n] [-w] node [node ...]\n" .
" node_reboot [-d] [-f] [-n] [-w] -e pid,eid\n".
"Use the -d option to turn on debugging\n" .
"Use the -e option to reboot all the nodes in an experiment\n" .
"Use the -n option to not wait for nodes to go down\n" .
"Use the -w option to to wait for nodes is come back up\n" .
"Use the -f option to power cycle (and not wait for nodes to die)\n";
exit(-1);
}
my $optlist = "dfe:w";
# The hidden -r option runs this in "realmode", ie don't send an event, but
# really do the work instead.
my $optlist = "dfe:nwr";
#
# Configure variables
#
my $TB = "@prefix@";
my $CLIENT_BIN = "@CLIENT_BINDIR@";
my $BOSSNODE = "@BOSSNODE@";
#
# Testbed Support libraries
......@@ -40,6 +44,7 @@ my $CLIENT_BIN = "@CLIENT_BINDIR@";
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use event;
use POSIX qw(strftime);
my $ssh = "$TB/bin/sshtb -n";
......@@ -54,6 +59,8 @@ my @nodes = ();
my $debug = 0;
my $force = 0;
my $waitmode = 0;
my $realmode = 0;
my $nowait = 0;
my $failed = 0;
my $eidmode = 0;
my $pid;
......@@ -90,11 +97,17 @@ if (defined($options{"f"})) {
if (defined($options{"w"})) {
$waitmode = 1;
}
if (defined($options{"r"})) {
$realmode = 1;
}
if (defined($options{"n"}) && !defined($options{"w"})) {
$nowait = 1;
}
if (defined($options{"e"})) {
if (@ARGV) {
usage();
}
$eidmode = $options{"e"};
if ($eidmode =~ /([-\w]*),([-\w]*)/) {
$pid = $1;
......@@ -106,6 +119,12 @@ if (defined($options{"e"})) {
}
}
# XXX Temporary, until we make event sending the default
$realmode=1;
#if ($realmode && $UID && !TBAdmin($UID)) {
# die("*** You cannot use real mode!\n");
#}
#
# If eidmode, then get the node list out of the DB instead of the command
# line. A proper check is made later, so need to be fancy about the query.
......@@ -120,7 +139,7 @@ if ($eidmode) {
! TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_MODIFY)) {
die("*** You not have permission to reboot nodes in $pid/$eid!\n");
}
my $query_result =
DBQueryFatal("select node_id from reserved where ".
"pid='$pid' and eid='$eid'");
......@@ -137,7 +156,7 @@ else {
if (@ARGV == 0) {
usage();
}
# Untaint the nodes.
foreach my $node ( @ARGV ) {
if ($node =~ /^([-\@\w]+)$/) {
......@@ -146,7 +165,7 @@ else {
else {
die("Bad node name: $node.");
}
push(@nodes, $node);
}
......@@ -171,14 +190,14 @@ my %virtnodes = ();
foreach my $node ( @nodes ) {
my $jailed;
if (TBIsNodeVirtual($node, \$jailed)) {
if (! $jailed) {
print "*** Skipping old style (non-jail) virtual node $node ...\n";
next;
}
my $pnode;
if (! TBPhysNodeID($node, \$pnode)) {
die("*** $0:\n".
" No physical node for $node!\n");
......@@ -202,19 +221,42 @@ if (! keys(%realnodes) && ! keys(%virtnodes)) {
exit(0);
}
#
# By here we've done all the preliminaries... send the event, unless we're
# in realmode.
#
my @sortednodes = sort(keys(%realnodes));
if (!$realmode) {
EventSendFatal(host => $BOSSNODE ,
objtype => TBDB_TBEVENT_COMMAND ,
eventtype => TBDB_COMMAND_REBOOT ,
objname => join(",",@sortednodes) );
if (!$nowait) {
# In here we can do some output to tell the user what's going on.
if ($waitmode) {
# Wait for [SHUTDOWN,ISUP]
} else {
# Wait for [SHUTDOWN]
}
}
exit(0);
}
#
# Another shark hack. Well, perhaps not. We really don't want 50 nodes
# all rebooting at the same time, PCs *or* sharks. Lets order them
# so that the shelves are grouped together at least, and issue the reboots
# in batches.
#
my @sortednodes = sort(keys(%realnodes));
while (@sortednodes) {
my @batch = ();
my $i = 0;
my $lastshelf = 0;
while ($i < 8 && @sortednodes > 0) {
my $node = shift(@sortednodes);
my $shelf;
......@@ -230,7 +272,7 @@ while (@sortednodes) {
}
$lastshelf = $shelf;
}
push(@batch, $node);
$i++;
}
......@@ -333,7 +375,7 @@ if ($waitmode) {
# Wait for events to filter through stated! If we do not wait, then we
# could see nodes still in ISUP.
sleep(2);
foreach my $node ( sort(@nodes) ) {
if (!TBNodeStateWait($node, TBDB_NODESTATE_ISUP, $waitstart, (60*6))) {
print STDOUT "$node is alive and well\n";
......@@ -392,7 +434,7 @@ sub RebootNode {
my $oldUID = $UID;
# print STDERR "Saved UID: $oldUID\n" if $debug;
$UID = 0;
#
# Run an ssh command in a child process, protected by an alarm to
# ensure that the ssh is not hung up forever if the machine is in
......@@ -411,7 +453,7 @@ sub RebootNode {
# FreeBSD for example.
#
print STDERR "reboot of $pc returned $?.\n" if $debug;
#
# If either ssh is not running or it timed out,
# send it a ping of death.
......@@ -454,7 +496,7 @@ sub RebootNode {
TBSetNodeEventState($pc,$state);
exit(0);
}
#
# Haven't yet tried an ipod, try that and wait again.
# This further slows down reboot but is probably worth it
......@@ -507,7 +549,7 @@ sub RebootVNode($$) {
# FreeBSD for example.
#
print STDERR "reboot of $vnode returned $exitstatus.\n" if $debug;
#
# Look for setup failure, reported back through ssh.
#
......@@ -525,7 +567,7 @@ sub RebootVNode($$) {
# Must change our real UID to root so that ssh will work.
#
$UID = 0;
exec("$ssh -host $pnode $CLIENT_BIN/vnodesetup -r -j $vnode");
exit(0);
}
......@@ -549,7 +591,7 @@ sub WaitTillDead {
my ($pc) = @_;
print STDERR "Waiting for $pc to die off\n" if $debug;
#
# Sigh, a long ping results in the script waiting until all the
# packets are sent from all the pings, before it will exit. So,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment