Commit c63b44d5 authored by Mike Hibler's avatar Mike Hibler

Handle an internal "forcecycle" command to make "cycle" of off nodes work.

parent c29cc790
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -477,8 +477,48 @@ foreach my $power_id (keys %outlets) {
print "Control of $nodestr failed.\n"; ++$exitval;
++$errors;
}
} elsif ($type eq 'ipmi15' || $type eq 'ipmi20') {
#
# XXX a "cycle" operation on IPMI will fail if the node is off.
# I can see reasons why you would want that behavior, but it means
# that if someone powers the node off from the OS (i.e., such that
# Emulab doesn't know about it) and then we attempt to power cycle
# it because it appears down, it will fail. We get a lot of nodes
# stuck in reloading or hwdown because of this.
#
# To allow either, we add a node attribute, "cyclewhenoff", that
# will cause us to do an "on" if a "cycle" fails. This is signified
# here by a "forcecycle" op passed to iloctrl.
#
my (@forcenodes, @unforcenodes);
if ($op eq "cycle") {
foreach my $nodeid (@nodes) {
my $node = $nodes{$nodeid};
if ($node->cyclewhenoff()) {
push @forcenodes, $nodeid;
} else {
push @unforcenodes, $nodeid;
}
}
} else {
@unforcenodes = @nodes;
}
if (@forcenodes) {
$nodestr = join(',', @forcenodes);
if (iloctrl($type,"forcecycle",@forcenodes)) {
print "Control of $nodestr failed.\n"; ++$exitval;
++$errors;
}
}
if (@unforcenodes) {
$nodestr = join(',', @unforcenodes);
if (iloctrl($type,$op,@unforcenodes)) {
print "Control of $nodestr failed.\n"; ++$exitval;
++$errors;
}
}
} elsif ($type eq 'ilo3' || $type eq 'ilo2' || $type eq 'ilo' ||
$type eq 'drac' || $type eq 'ipmi15' || $type eq 'ipmi20') {
$type eq 'drac') {
if (iloctrl($type,$op,@nodes)) {
print "Control of $nodestr failed.\n"; ++$exitval;
++$errors;
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2008-2015 University of Utah and the Flux Group.
# Copyright (c) 2008-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -77,12 +77,16 @@ my %portinfo = ();
sub iloctrl($$@) {
my ($type,$cmd,@nodes) = @_;
my $exitval = 0;
my $force = 0;
if ($debug) {
print "iloctrl called with $type,$cmd,(" . join(',',@nodes) . ")\n";
}
if ($cmd ne "cycle" && $cmd ne "on" && $cmd ne "off") {
if ($cmd eq "forcecycle") {
$force = 1;
$cmd = "cycle";
} elsif ($cmd ne "cycle" && $cmd ne "on" && $cmd ne "off") {
warn "invalid power command '$cmd'; \n" .
" valid commands are 'cycle, 'off', and 'on'.\n";
return scalar(@nodes);
......@@ -164,7 +168,7 @@ sub iloctrl($$@) {
my $tret;
eval {
if ($type =~ /^ipmi/) {
$tret = ipmiexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$kgkey,$kprivlvl,$timeout);
$tret = ipmiexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$kgkey,$kprivlvl,$timeout,$force);
} else {
$tret = iloexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$timeout);
}
......@@ -414,8 +418,8 @@ sub iloexec($$$$$$$;$) {
#
# Arguments: $node_id,$type,$cmd,$IP,$key_role,$key_uid,$key[,$kgkey,$privlvl,$timeout]
#
sub ipmiexec($$$$$$$;$$$) {
my ($node_id,$type,$cmd,$IP,$key_role,$key_uid,$key,$kgkey,$privlvl,$timeout) = @_;
sub ipmiexec($$$$$$$;$$$$) {
my ($node_id,$type,$cmd,$IP,$key_role,$key_uid,$key,$kgkey,$privlvl,$timeout,$force) = @_;
if ($debug) {
print "ipmiexec called with (" . join(',',@_) . ")\n";
......@@ -468,6 +472,7 @@ sub ipmiexec($$$$$$$;$$$) {
# XXX IPMI takes about 40 seconds to timeout and doesn't
# have an option to control?!
again:
my $privlvl_args = ($privlvl) ? " -L $privlvl" : '';
my $ipmicmd = "ipmitool -I $iface -H $IP -U $key_uid $privlvl_args -E -K power $cmd";
print "*** Executing '$ipmicmd', output:\n"
......@@ -485,6 +490,23 @@ sub ipmiexec($$$$$$$;$$$) {
delete $ENV{'IPMI_PASSWORD'};
delete $ENV{'IPMI_KGKEY'};
#
# XXX check for failured due to power cycle of a node that is turned off
# and turn it on if desired.
#
if ($stat && $cmd eq "cycle" &&
$output =~ /Command not supported in present state/) {
if ($force) {
print "*** Cycle failed, trying power on instead.\n"
if ($debug > 1);
$cmd = "on";
$force = 0;
goto again;
}
# XXX we should maybe do a POWEROFF state transition here?
$output = "Node is powered off, use 'power on' instead.\n";
}
if ($stat || $debug > 1) {
print "*** '$ipmicmd' failed (stat=$stat):\n"
if ($stat);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment