Handle an internal "forcecycle" command to make "cycle" of off nodes work.

tbsetup/power.in

 #!/usr/bin/perl -wT
 
 #
-# Copyright (c) 2000-2015 University of Utah and the Flux Group.
+# Copyright (c) 2000-2016 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -477,8 +477,48 @@ foreach my $power_id (keys %outlets) {
 	    print "Control of $nodestr failed.\n"; ++$exitval;
 	    ++$errors;
 	}
+    } elsif ($type eq 'ipmi15' || $type eq 'ipmi20') {
+	#
+	# XXX a "cycle" operation on IPMI will fail if the node is off.
+	# I can see reasons why you would want that behavior, but it means
+	# that if someone powers the node off from the OS (i.e., such that
+	# Emulab doesn't know about it) and then we attempt to power cycle
+	# it because it appears down, it will fail. We get a lot of nodes
+	# stuck in reloading or hwdown because of this.
+	#
+	# To allow either, we add a node attribute, "cyclewhenoff", that
+	# will cause us to do an "on" if a "cycle" fails. This is signified
+	# here by a "forcecycle" op passed to iloctrl.
+	# 
+	my (@forcenodes, @unforcenodes);
+	if ($op eq "cycle") {
+	    foreach my $nodeid (@nodes) {
+		my $node = $nodes{$nodeid};
+		if ($node->cyclewhenoff()) {
+		    push @forcenodes, $nodeid;
+		} else {
+		    push @unforcenodes, $nodeid;
+		}
+	    }
+	} else {
+	    @unforcenodes = @nodes;
+	}
+	if (@forcenodes) {
+	    $nodestr = join(',', @forcenodes);
+	    if (iloctrl($type,"forcecycle",@forcenodes)) {
+		print "Control of $nodestr failed.\n"; ++$exitval;
+		++$errors;
+	    }
+	}
+	if (@unforcenodes) {
+	    $nodestr = join(',', @unforcenodes);
+	    if (iloctrl($type,$op,@unforcenodes)) {
+		print "Control of $nodestr failed.\n"; ++$exitval;
+		++$errors;
+	    }
+	}
     } elsif ($type eq 'ilo3' || $type eq 'ilo2' || $type eq 'ilo' ||
-	     $type eq 'drac' || $type eq 'ipmi15' || $type eq 'ipmi20') {
+	     $type eq 'drac') {
 	if (iloctrl($type,$op,@nodes)) {
 	    print "Control of $nodestr failed.\n"; ++$exitval;
             ++$errors;

tbsetup/power_ilo.pm.in

 #!/usr/bin/perl -wT
 
 #
-# Copyright (c) 2008-2015 University of Utah and the Flux Group.
+# Copyright (c) 2008-2016 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -77,12 +77,16 @@ my %portinfo = ();
 sub iloctrl($$@) {
     my ($type,$cmd,@nodes) = @_;
     my $exitval = 0;
+    my $force = 0;
 
     if ($debug) {
 	print "iloctrl called with $type,$cmd,(" . join(',',@nodes) . ")\n";
     }
 
-    if ($cmd ne "cycle" && $cmd ne "on" && $cmd ne "off") {
+    if ($cmd eq "forcecycle") {
+	$force = 1;
+	$cmd = "cycle";
+    } elsif ($cmd ne "cycle" && $cmd ne "on" && $cmd ne "off") {
 	warn "invalid power command '$cmd'; \n" . 
 	    "  valid commands are 'cycle, 'off', and 'on'.\n";
 	return scalar(@nodes);
@@ -164,7 +168,7 @@ sub iloctrl($$@) {
 	    my $tret;
 	    eval {
 		if ($type =~ /^ipmi/) {
-		    $tret = ipmiexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$kgkey,$kprivlvl,$timeout);
+		    $tret = ipmiexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$kgkey,$kprivlvl,$timeout,$force);
 		} else {
 		    $tret = iloexec($n,$type,$cmd,$IP,$krole,$kuid,$kkey,$timeout);
 		}
@@ -414,8 +418,8 @@ sub iloexec($$$$$$$;$) {
 #
 # Arguments: $node_id,$type,$cmd,$IP,$key_role,$key_uid,$key[,$kgkey,$privlvl,$timeout]
 #
-sub ipmiexec($$$$$$$;$$$) {
-    my ($node_id,$type,$cmd,$IP,$key_role,$key_uid,$key,$kgkey,$privlvl,$timeout) = @_;
+sub ipmiexec($$$$$$$;$$$$) {
+    my ($node_id,$type,$cmd,$IP,$key_role,$key_uid,$key,$kgkey,$privlvl,$timeout,$force) = @_;
 
     if ($debug) {
 	print "ipmiexec called with (" . join(',',@_) . ")\n";
@@ -468,6 +472,7 @@ sub ipmiexec($$$$$$$;$$$) {
     # XXX IPMI takes about 40 seconds to timeout and doesn't
     # have an option to control?!
 
+again:
     my $privlvl_args = ($privlvl) ? " -L $privlvl" : '';
     my $ipmicmd = "ipmitool -I $iface -H $IP -U $key_uid $privlvl_args -E -K power $cmd";
     print "*** Executing '$ipmicmd', output:\n"
@@ -485,6 +490,23 @@ sub ipmiexec($$$$$$$;$$$) {
     delete $ENV{'IPMI_PASSWORD'};
     delete $ENV{'IPMI_KGKEY'};
 
+    #
+    # XXX check for failured due to power cycle of a node that is turned off
+    # and turn it on if desired.
+    #
+    if ($stat && $cmd eq "cycle" &&
+	$output =~ /Command not supported in present state/) {
+	if ($force) {
+	    print "*** Cycle failed, trying power on instead.\n"
+		if ($debug > 1);
+	    $cmd = "on";
+	    $force = 0;
+	    goto again;
+	}
+	# XXX we should maybe do a POWEROFF state transition here?
+	$output = "Node is powered off, use 'power on' instead.\n";
+    }
+
     if ($stat || $debug > 1) {
 	print "*** '$ipmicmd' failed (stat=$stat):\n"
 	    if ($stat);