Commit 1daaa992 authored by Mac Newbold's avatar Mac Newbold

The new and fully functional rebooting-via-events stuff and the

really-reboot-nodes-that-timeout stuff.

NOTE: Until the timeout/retry stuff is gone from os_load/os_setup, it is
disabled in stated. It will still only send email. But all the stuff is
there and has been tested.

NOTE: Until other things don't depend on the old behavior of node_reboot
(when it returns, all nodes are in SHUTDOWN), the event stuff is disabled.
Real mode is the default, and can be run by anyone.

In short, this commit is new versions of stated and node_reboot that act
almost exactly like the old ones. But I wanted to commit them before I go
on making a bunch more changes, to have a checkpoint that I know works.
parent 243921d5
...@@ -115,8 +115,11 @@ my $TBREBOOT = TBDB_COMMAND_REBOOT; ...@@ -115,8 +115,11 @@ my $TBREBOOT = TBDB_COMMAND_REBOOT;
my $TBPOWEROFF = TBDB_COMMAND_POWEROFF; my $TBPOWEROFF = TBDB_COMMAND_POWEROFF;
my $TBPOWERON = TBDB_COMMAND_POWERON; my $TBPOWERON = TBDB_COMMAND_POWERON;
my $TBPOWERCYCLE = TBDB_COMMAND_POWERCYCLE; my $TBPOWERCYCLE = TBDB_COMMAND_POWERCYCLE;
my $TB_OSID_MBKERNEL = TB_OSID_MBKERNEL;
my $TBISUP = TBDB_NODESTATE_ISUP; my $TBISUP = TBDB_NODESTATE_ISUP;
my $TBTIMEOUTREBOOT = TBDB_STATED_TIMEOUT_REBOOT;
my $TBTIMEOUTNOTIFY = TBDB_STATED_TIMEOUT_NOTIFY;
my $TBTIMEOUTCMDRETRY = TBDB_STATED_TIMEOUT_CMDRETRY;
my $TB_OSID_MBKERNEL = TB_OSID_MBKERNEL;
# This only gets used here, so it isn't in a lib constant. # This only gets used here, so it isn't in a lib constant.
my $TBFREENODE = "FREENODE"; my $TBFREENODE = "FREENODE";
...@@ -362,6 +365,9 @@ sub readStates(;@) { ...@@ -362,6 +365,9 @@ sub readStates(;@) {
$nodes{$node_id}{timestamp} = $timestamp; $nodes{$node_id}{timestamp} = $timestamp;
$nodes{$node_id}{mode} = $mode; $nodes{$node_id}{mode} = $mode;
$nodes{$node_id}{mode_timestamp} = $mode_timestamp; $nodes{$node_id}{mode_timestamp} = $mode_timestamp;
$nodes{$node_id}{notified} = 0;
$nodes{$node_id}{timedout} = 0;
$nodes{$node_id}{noretry} = 0;
# Is there a timeout? If so, set it up! # Is there a timeout? If so, set it up!
setTimeout($mode,$state,$node_id,$timestamp); setTimeout($mode,$state,$node_id,$timestamp);
} }
...@@ -537,20 +543,22 @@ sub stateTransition($$) { ...@@ -537,20 +543,22 @@ sub stateTransition($$) {
# to check if we had a pending command # to check if we had a pending command
if (qfind($node) && if (qfind($node) &&
$timeout_tag{$node} =~ /^$TBCOMMAND:/) { $timeout_tag{$node} =~ /^$TBCOMMAND:/) {
debug("TimeoutTag = '$timeout_tag{$node}'\n");
my ($str,$cmd) = split(":",$timeout_tag{$node}); my ($str,$cmd) = split(":",$timeout_tag{$node});
debug("str=$str\tcmd=$cmd\tTBREBOOT=$TBREBOOT\tstate=$newstate\n");
if ($cmd eq $TBREBOOT) { if ($cmd eq $TBREBOOT) {
if ($state eq TBDB_NODESTATE_SHUTDOWN ) { if ($newstate eq TBDB_NODESTATE_SHUTDOWN ) {
info("$node: $TBREBOOT success\n"); info("$node: $TBREBOOT success\n");
# Timeout will get cleared below by setTimeout call # Timeout will get cleared below by setTimeout call
} else { } else {
notify("$node: $TBREBOOT in progress, but got state $state ". notify("$node: $TBREBOOT in progress, but got state ".
"instead of ". TBDB_NODESTATE_SHUTDOWN ."!\n"); "$newstate instead of ".TBDB_NODESTATE_SHUTDOWN."!\n");
} }
#} elsif ($cmd eq $FOO ) { #} elsif ($cmd eq $FOO ) {
# Add more here... # Add more here...
} else { } else {
notify("$node: Unknown command timeout '$timeout_tag{$node}' ". notify("$node: Unknown command timeout '$timeout_tag{$node}' ".
"found at $mode/$state\n"); "found at $mode/$newstate\n");
} }
} }
...@@ -611,6 +619,9 @@ sub stateTransition($$) { ...@@ -611,6 +619,9 @@ sub stateTransition($$) {
foreach ( @trigs , @nodetrigs) { foreach ( @trigs , @nodetrigs) {
my $trig = $_; my $trig = $_;
/^$TBRESET$/ && do { /^$TBRESET$/ && do {
# We successfully booted, so clear some flags
$nodes{$node}{noretry} = 0;
$nodes{$node}{timedout} = 0;
# Check if we really need to do a reset # Check if we really need to do a reset
my $r = DBQueryWarn("select osid,def_boot_osid from nodes ". my $r = DBQueryWarn("select osid,def_boot_osid from nodes ".
"where node_id='$node'"); "where node_id='$node'");
...@@ -790,40 +801,60 @@ sub handleCtrlEvent($$) { ...@@ -790,40 +801,60 @@ sub handleCtrlEvent($$) {
} }
$nodes{$node}{notified}++; $nodes{$node}{notified}++;
my $notified = $nodes{$node}{notified}; my $notified = $nodes{$node}{notified};
$nodes{$node}{timedout}++;
my $timedout = $nodes{$node}{timedout};
if ($mode && $state && $timeouts{$mode} && if ($mode && $state && $timeouts{$mode} &&
$timeouts{$mode}{$state}) { $timeouts{$mode}{$state}) {
($timeout, $action) = @{$timeouts{$mode}{$state}}; ($timeout, $action) = @{$timeouts{$mode}{$state}};
} }
if ($mode eq $TBCOMMAND) { if ($mode eq $TBCOMMAND) {
# It is a command, not a true state # It is a command, not a true state
if ($action eq "CMDRETRY") { if ($action eq $TBTIMEOUTCMDRETRY) {
# Retry the command # Retry the command
notify("$node: Command $state, retry #$notified\n"); notify("$node: Command $state, retry #$timedout\n");
# notify in case we get in a retry loop... # notify in case we get in a retry loop...
handleCommand($node,$state,$notified); handleCommand($node,$state,$timedout,1);
} else { } else {
notify("$node: Unknown timeout action for ". notify("$node: Unknown timeout action for ".
"$mode/$state: '$action'\n"); "$mode/$state: '$action'\n");
} }
next; next;
} else {
if ($notified>1) {
notify("$node: Timed out at $now (d=$deadline), ".
"but notified already!\n");
}
} }
notify("Node $node has timed out in state $mode/$state". info("Node $node has timed out in state $mode/$state".
($action ne "" ? "\n\tRequested action $action." : ""). ($action ne "" ? "\n\tRequested action $action." : "").
"\n"); "\n");
foreach ($action) {
/^$TBTIMEOUTREBOOT/ && do {
if ($timedout>3) {
# We've tried too many times...
notify("Node $node has timed out too many times!\n".
"Giving up until it boots sucessfully.\n");
$nodes{$node}{noretry} = 1;
} else {
# XXX Temporary! For now notify instead of
# really rebooting, until the timeout/retry
# stuff is gone from os_setup and os_load
notify("Node $node has timed out in state ".
"$mode/$state - REBOOT requested\n");
#handleCommand($node,$TBREBOOT,$timedout,1);
}
last; };
/^$TBTIMEOUTNOTIFY/ && do {
notify("Node $node has timed out in state $mode/$state\n");
last; };
notify("$node: Unknown Timeout Action: $action\n");
}
next; next;
}; };
notify("$node: Unknown CtrlEvent: $event\n"); notify("$node: Unknown CtrlEvent: $event\n");
} }
} }
sub handleCommand($$;$) { sub handleCommand($$;$$) {
my ($params,$command,$retry) = @_; my ($params,$command,$retry,$force) = @_;
if (!defined($retry)) { $retry=0; } if (!defined($retry)) { $retry=0; }
if (!defined($force)) { $force=0; }
info("Command: $params, $command (attempt $retry)\n"); info("Command: $params, $command (attempt $retry)\n");
...@@ -832,26 +863,53 @@ sub handleCommand($$;$) { ...@@ -832,26 +863,53 @@ sub handleCommand($$;$) {
# We may need to do it here (while iterating over the list), or # We may need to do it here (while iterating over the list), or
# make some other fix up in handleEvent. # make some other fix up in handleEvent.
if ($command eq $TBREBOOT && $retry >=3) { if ($command eq $TBREBOOT && $retry >=4) {
announce("Node $params has tried rebooting $retry times and has \n". announce("Node $params has tried rebooting $retry times and has \n".
"still not been successful. Please look into it soon.\n". "still not been successful. Please look into it soon.\n".
"In the meantime, $params will be powered off.\n"); "" );# "In the meantime, $params will be powered off.\n");
# Just return...
return 0;
# change my command to poweroff. # change my command to poweroff.
$command = $TBPOWEROFF; #$command = $TBPOWEROFF;
} }
foreach ($command) { foreach ($command) {
/^$TBREBOOT$/ && do { /^$TBREBOOT$/ && do {
# For reboot, the params is a comma-separated list of nodes # For reboot, the params is a comma-separated list of nodes
my @nodes = split(",",$params); my @nodes = split(",",$params);
# Check if any of these are in the middle of things and
# shouldn't be rebooted.
if (!$force) {
foreach $n ( 0 .. $#nodes ) {
$node = $nodes[$n];
debug("Checking rebooting: $node, $nodes{$node}, ".
"$nodes{$node}{state}, $nodes{$node}{noretry}\n");
if (($nodes{$node}{state} ne TBDB_NODESTATE_ISUP) &&
(!$nodes{$node}{noretry}) ) {
# This node shouldn't be rebooted now...
# XXX Send feedback here somehow!
info("$node: Trying to reboot too soon! Skipping.\n");
# Cut it out of the list
debug("Nodelist before ==> ".join(" ",@nodes)."\n");
splice(@nodes,$n,1);
debug("Nodelist after ==> ".join(" ",@nodes)."\n");
}
}
if (@nodes == 0) { next; }
}
my $nodelist=join(" ",@nodes); my $nodelist=join(" ",@nodes);
info("Rebooting nodes: $nodelist\n"); info("Rebooting nodes: $nodelist\n");
# Permissions were checked in order to send the message, # Permissions were checked in order to send the message,
# so we don't need to do any fancy stuff here. # so we don't need to do any fancy stuff here.
my $cmd = "$nodereboot $nodelist &"; my $cmd = "$nodereboot -r $nodelist";
debug("$cmd\n") or my $redir = " 2>&1 >> /usr/testbed/log/nodereboot.log &";
#system($cmd) and debug("$cmd $redir\n");
system("date $redir");
system($cmd.$redir) and
notify("$params/$command: ". notify("$params/$command: ".
"Command '$cmd' failed, error $?: $!\n"); "Command '$cmd' failed, error $?: $!\n");
...@@ -874,13 +932,13 @@ sub handleCommand($$;$) { ...@@ -874,13 +932,13 @@ sub handleCommand($$;$) {
$TBPOWERON => "on", $TBPOWERON => "on",
$TBPOWEROFF => "off"); $TBPOWEROFF => "off");
my $func = $funcmap{$command}; my $func = $funcmap{$command};
info("Sending power $func nodes: $nodelist\n"); info("Sending power $func for nodes: $nodelist\n");
# Permissions were checked in order to send the message, # Permissions were checked in order to send the message,
# so we don't need to do any fancy stuff here. # so we don't need to do any fancy stuff here.
my $cmd = "$power $func $nodelist &"; my $cmd = "$power $func $nodelist &";
debug("$cmd\n") or debug("$cmd\n") and
#system($cmd) and system($cmd) and
notify("$params/$command: ". notify("$params/$command: ".
"Command '$cmd' failed, error $?: $!\n"); "Command '$cmd' failed, error $?: $!\n");
......
...@@ -18,21 +18,25 @@ use Getopt::Std; ...@@ -18,21 +18,25 @@ use Getopt::Std;
# #
sub usage() sub usage()
{ {
print STDOUT "Usage: node_reboot [-d] [-f] [-w] node [node ...]\n" . print STDOUT "Usage: node_reboot [-d] [-f] [-n] [-w] node [node ...]\n" .
" node_reboot [-d] [-f] [-w] -e pid,eid\n". " node_reboot [-d] [-f] [-n] [-w] -e pid,eid\n".
"Use the -d option to turn on debugging\n" . "Use the -d option to turn on debugging\n" .
"Use the -e option to reboot all the nodes in an experiment\n" . "Use the -e option to reboot all the nodes in an experiment\n" .
"Use the -n option to not wait for nodes to go down\n" .
"Use the -w option to to wait for nodes is come back up\n" . "Use the -w option to to wait for nodes is come back up\n" .
"Use the -f option to power cycle (and not wait for nodes to die)\n"; "Use the -f option to power cycle (and not wait for nodes to die)\n";
exit(-1); exit(-1);
} }
my $optlist = "dfe:w"; # The hidden -r option runs this in "realmode", ie don't send an event, but
# really do the work instead.
my $optlist = "dfe:nwr";
# #
# Configure variables # Configure variables
# #
my $TB = "@prefix@"; my $TB = "@prefix@";
my $CLIENT_BIN = "@CLIENT_BINDIR@"; my $CLIENT_BIN = "@CLIENT_BINDIR@";
my $BOSSNODE = "@BOSSNODE@";
# #
# Testbed Support libraries # Testbed Support libraries
...@@ -40,6 +44,7 @@ my $CLIENT_BIN = "@CLIENT_BINDIR@"; ...@@ -40,6 +44,7 @@ my $CLIENT_BIN = "@CLIENT_BINDIR@";
use lib "@prefix@/lib"; use lib "@prefix@/lib";
use libdb; use libdb;
use libtestbed; use libtestbed;
use event;
use POSIX qw(strftime); use POSIX qw(strftime);
my $ssh = "$TB/bin/sshtb -n"; my $ssh = "$TB/bin/sshtb -n";
...@@ -54,6 +59,8 @@ my @nodes = (); ...@@ -54,6 +59,8 @@ my @nodes = ();
my $debug = 0; my $debug = 0;
my $force = 0; my $force = 0;
my $waitmode = 0; my $waitmode = 0;
my $realmode = 0;
my $nowait = 0;
my $failed = 0; my $failed = 0;
my $eidmode = 0; my $eidmode = 0;
my $pid; my $pid;
...@@ -90,11 +97,17 @@ if (defined($options{"f"})) { ...@@ -90,11 +97,17 @@ if (defined($options{"f"})) {
if (defined($options{"w"})) { if (defined($options{"w"})) {
$waitmode = 1; $waitmode = 1;
} }
if (defined($options{"r"})) {
$realmode = 1;
}
if (defined($options{"n"}) && !defined($options{"w"})) {
$nowait = 1;
}
if (defined($options{"e"})) { if (defined($options{"e"})) {
if (@ARGV) { if (@ARGV) {
usage(); usage();
} }
$eidmode = $options{"e"}; $eidmode = $options{"e"};
if ($eidmode =~ /([-\w]*),([-\w]*)/) { if ($eidmode =~ /([-\w]*),([-\w]*)/) {
$pid = $1; $pid = $1;
...@@ -106,6 +119,12 @@ if (defined($options{"e"})) { ...@@ -106,6 +119,12 @@ if (defined($options{"e"})) {
} }
} }
# XXX Temporary, until we make event sending the default
$realmode=1;
#if ($realmode && $UID && !TBAdmin($UID)) {
# die("*** You cannot use real mode!\n");
#}
# #
# If eidmode, then get the node list out of the DB instead of the command # If eidmode, then get the node list out of the DB instead of the command
# line. A proper check is made later, so need to be fancy about the query. # line. A proper check is made later, so need to be fancy about the query.
...@@ -120,7 +139,7 @@ if ($eidmode) { ...@@ -120,7 +139,7 @@ if ($eidmode) {
! TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_MODIFY)) { ! TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_MODIFY)) {
die("*** You not have permission to reboot nodes in $pid/$eid!\n"); die("*** You not have permission to reboot nodes in $pid/$eid!\n");
} }
my $query_result = my $query_result =
DBQueryFatal("select node_id from reserved where ". DBQueryFatal("select node_id from reserved where ".
"pid='$pid' and eid='$eid'"); "pid='$pid' and eid='$eid'");
...@@ -137,7 +156,7 @@ else { ...@@ -137,7 +156,7 @@ else {
if (@ARGV == 0) { if (@ARGV == 0) {
usage(); usage();
} }
# Untaint the nodes. # Untaint the nodes.
foreach my $node ( @ARGV ) { foreach my $node ( @ARGV ) {
if ($node =~ /^([-\@\w]+)$/) { if ($node =~ /^([-\@\w]+)$/) {
...@@ -146,7 +165,7 @@ else { ...@@ -146,7 +165,7 @@ else {
else { else {
die("Bad node name: $node."); die("Bad node name: $node.");
} }
push(@nodes, $node); push(@nodes, $node);
} }
...@@ -171,14 +190,14 @@ my %virtnodes = (); ...@@ -171,14 +190,14 @@ my %virtnodes = ();
foreach my $node ( @nodes ) { foreach my $node ( @nodes ) {
my $jailed; my $jailed;
if (TBIsNodeVirtual($node, \$jailed)) { if (TBIsNodeVirtual($node, \$jailed)) {
if (! $jailed) { if (! $jailed) {
print "*** Skipping old style (non-jail) virtual node $node ...\n"; print "*** Skipping old style (non-jail) virtual node $node ...\n";
next; next;
} }
my $pnode; my $pnode;
if (! TBPhysNodeID($node, \$pnode)) { if (! TBPhysNodeID($node, \$pnode)) {
die("*** $0:\n". die("*** $0:\n".
" No physical node for $node!\n"); " No physical node for $node!\n");
...@@ -202,19 +221,42 @@ if (! keys(%realnodes) && ! keys(%virtnodes)) { ...@@ -202,19 +221,42 @@ if (! keys(%realnodes) && ! keys(%virtnodes)) {
exit(0); exit(0);
} }
#
# By here we've done all the preliminaries... send the event, unless we're
# in realmode.
#
my @sortednodes = sort(keys(%realnodes));
if (!$realmode) {
EventSendFatal(host => $BOSSNODE ,
objtype => TBDB_TBEVENT_COMMAND ,
eventtype => TBDB_COMMAND_REBOOT ,
objname => join(",",@sortednodes) );
if (!$nowait) {
# In here we can do some output to tell the user what's going on.
if ($waitmode) {
# Wait for [SHUTDOWN,ISUP]
} else {
# Wait for [SHUTDOWN]
}
}
exit(0);
}
# #
# Another shark hack. Well, perhaps not. We really don't want 50 nodes # Another shark hack. Well, perhaps not. We really don't want 50 nodes
# all rebooting at the same time, PCs *or* sharks. Lets order them # all rebooting at the same time, PCs *or* sharks. Lets order them
# so that the shelves are grouped together at least, and issue the reboots # so that the shelves are grouped together at least, and issue the reboots
# in batches. # in batches.
# #
my @sortednodes = sort(keys(%realnodes));
while (@sortednodes) { while (@sortednodes) {
my @batch = (); my @batch = ();
my $i = 0; my $i = 0;
my $lastshelf = 0; my $lastshelf = 0;
while ($i < 8 && @sortednodes > 0) { while ($i < 8 && @sortednodes > 0) {
my $node = shift(@sortednodes); my $node = shift(@sortednodes);
my $shelf; my $shelf;
...@@ -230,7 +272,7 @@ while (@sortednodes) { ...@@ -230,7 +272,7 @@ while (@sortednodes) {
} }
$lastshelf = $shelf; $lastshelf = $shelf;
} }
push(@batch, $node); push(@batch, $node);
$i++; $i++;
} }
...@@ -333,7 +375,7 @@ if ($waitmode) { ...@@ -333,7 +375,7 @@ if ($waitmode) {
# Wait for events to filter through stated! If we do not wait, then we # Wait for events to filter through stated! If we do not wait, then we
# could see nodes still in ISUP. # could see nodes still in ISUP.
sleep(2); sleep(2);
foreach my $node ( sort(@nodes) ) { foreach my $node ( sort(@nodes) ) {
if (!TBNodeStateWait($node, TBDB_NODESTATE_ISUP, $waitstart, (60*6))) { if (!TBNodeStateWait($node, TBDB_NODESTATE_ISUP, $waitstart, (60*6))) {
print STDOUT "$node is alive and well\n"; print STDOUT "$node is alive and well\n";
...@@ -392,7 +434,7 @@ sub RebootNode { ...@@ -392,7 +434,7 @@ sub RebootNode {
my $oldUID = $UID; my $oldUID = $UID;
# print STDERR "Saved UID: $oldUID\n" if $debug; # print STDERR "Saved UID: $oldUID\n" if $debug;
$UID = 0; $UID = 0;
# #
# Run an ssh command in a child process, protected by an alarm to # Run an ssh command in a child process, protected by an alarm to
# ensure that the ssh is not hung up forever if the machine is in # ensure that the ssh is not hung up forever if the machine is in
...@@ -411,7 +453,7 @@ sub RebootNode { ...@@ -411,7 +453,7 @@ sub RebootNode {
# FreeBSD for example. # FreeBSD for example.
# #
print STDERR "reboot of $pc returned $?.\n" if $debug; print STDERR "reboot of $pc returned $?.\n" if $debug;
# #
# If either ssh is not running or it timed out, # If either ssh is not running or it timed out,
# send it a ping of death. # send it a ping of death.
...@@ -454,7 +496,7 @@ sub RebootNode { ...@@ -454,7 +496,7 @@ sub RebootNode {
TBSetNodeEventState($pc,$state); TBSetNodeEventState($pc,$state);
exit(0); exit(0);
} }
# #
# Haven't yet tried an ipod, try that and wait again. # Haven't yet tried an ipod, try that and wait again.
# This further slows down reboot but is probably worth it # This further slows down reboot but is probably worth it
...@@ -507,7 +549,7 @@ sub RebootVNode($$) { ...@@ -507,7 +549,7 @@ sub RebootVNode($$) {
# FreeBSD for example. # FreeBSD for example.
# #
print STDERR "reboot of $vnode returned $exitstatus.\n" if $debug; print STDERR "reboot of $vnode returned $exitstatus.\n" if $debug;
# #
# Look for setup failure, reported back through ssh. # Look for setup failure, reported back through ssh.
# #
...@@ -525,7 +567,7 @@ sub RebootVNode($$) { ...@@ -525,7 +567,7 @@ sub RebootVNode($$) {
# Must change our real UID to root so that ssh will work. # Must change our real UID to root so that ssh will work.
# #
$UID = 0; $UID = 0;
exec("$ssh -host $pnode $CLIENT_BIN/vnodesetup -r -j $vnode"); exec("$ssh -host $pnode $CLIENT_BIN/vnodesetup -r -j $vnode");
exit(0); exit(0);
} }
...@@ -549,7 +591,7 @@ sub WaitTillDead { ...@@ -549,7 +591,7 @@ sub WaitTillDead {
my ($pc) = @_; my ($pc) = @_;
print STDERR "Waiting for $pc to die off\n" if $debug; print STDERR "Waiting for $pc to die off\n" if $debug;
# #
# Sigh, a long ping results in the script waiting until all the # Sigh, a long ping results in the script waiting until all the
# packets are sent from all the pings, before it will exit. So, # packets are sent from all the pings, before it will exit. So,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment