Commit 942a2bc5 authored by Robert Ricci's avatar Robert Ricci

Change the way power cycles are handled - in the child fork()ed to

reboot a node, we never call power directly. Instead, exit with
a status of 2 - the parent process collects a list of nodes that
needed power cycling, and does them all in one call.
parent 85680ced
...@@ -241,7 +241,7 @@ while (@sortednodes) { ...@@ -241,7 +241,7 @@ while (@sortednodes) {
# continue on. We don't wait for them to go down or reboot. # continue on. We don't wait for them to go down or reboot.
# #
info("Force mode: power cycle ".join(" ",@batch)); info("Force mode: power cycle ".join(" ",@batch));
system("$power cycle @batch"); PowerCycle(@batch);
if ($?) { if ($?) {
exit ($? >> 8); exit ($? >> 8);
} }
...@@ -275,12 +275,17 @@ while (@sortednodes) { ...@@ -275,12 +275,17 @@ while (@sortednodes) {
# #
# Wait for all the reboot children to exit before continuing. # Wait for all the reboot children to exit before continuing.
# #
my @needPowercycle = ();
if (! $force) { if (! $force) {
foreach my $node ( sort(keys(%realnodes)) ) { foreach my $node ( sort(keys(%realnodes)) ) {
my $mypid = $pids{$node}; my $mypid = $pids{$node};
waitpid($mypid, 0); waitpid($mypid, 0);
if ($?) { my $status = $? >> 8;
if ($status == 2) {
# Child signaled to us that this node needs a power cycle
push @needPowercycle, $node;
} elsif ($?) {
$failed++; $failed++;
print STDERR "Reboot of node $node failed!\n"; print STDERR "Reboot of node $node failed!\n";
} }
...@@ -290,6 +295,13 @@ if (! $force) { ...@@ -290,6 +295,13 @@ if (! $force) {
} }
} }
#
# Power cycle nodes that couldn't be brought down any other way
#
if (@needPowercycle) {
PowerCycle(@needPowercycle);
}
# #
# Now do vnodes. Do these serially for now (simple). # Now do vnodes. Do these serially for now (simple).
# #
...@@ -363,10 +375,8 @@ sub RebootNode { ...@@ -363,10 +375,8 @@ sub RebootNode {
if (! DoesPing($pc)) { if (! DoesPing($pc)) {
info("$pc appears dead: power cycle"); info("$pc appears dead: power cycle");
print STDERR "$pc appears to be dead. Power cycling ...\n" if $debug; print STDERR "$pc appears to be dead. Power cycling ...\n" if $debug;
if (PowerCycle($pc)) { # Signal to the parent that the node needs to be power cycled
exit(-1); exit(2);
}
exit(0);
} }
# #
...@@ -466,11 +476,7 @@ sub RebootNode { ...@@ -466,11 +476,7 @@ sub RebootNode {
info("$pc: ipod failed ... power cycle"); info("$pc: ipod failed ... power cycle");
print STDERR "$pc is still running. Power cycling ...\n" if $debug; print STDERR "$pc is still running. Power cycling ...\n" if $debug;
if (PowerCycle($pc)) { exit(2);
info("$pc: power cycle failed!");
exit(-1);
}
exit(0);
} }
# #
...@@ -528,9 +534,11 @@ sub RebootVNode($$) { ...@@ -528,9 +534,11 @@ sub RebootVNode($$) {
# Power cycle a PC using the testbed power program. # Power cycle a PC using the testbed power program.
# #
sub PowerCycle { sub PowerCycle {
my ($pc) = @_; my @pcs = @_;
my $pcstring = join(" ",@pcs);
system("$power cycle $pc"); system("$power cycle $pcstring");
return $? >> 8; return $? >> 8;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment