Commit 164da3ba authored by Leigh Stoller's avatar Leigh Stoller

Add support for fully initializing the ilo on geni rack nodes.

The basic operational model is as follows.

* We turn the nodes on.

* Since there is nothing on the disks, they will fall through to
  booting from the PXE and will boot the newnode MFS. They all check
  in.

* We run Jon's script that adds the nodes. They are now in hwdown,
  still nothing on the disks.

* We run my script, which is driven from a datafile we are supposed to
  get from HP. This script has the ilomac, ilopswd, control mac. I
  will add another column initially; the permanent IP to assign to the
  ilo. This script does:

 + Reads the datafile to get all the stuff.
 + Reads the dhcpd.leases file to find the temporary IPs of the ilos.
 + Finds the corresponding nodes in the DB.
 + Sends over an XML file that does the following:
     - Add the elabman user.
     - Add local root's dsa pub key to the new elabman user.
     - Add Utah's root dsa key to the Administrator user
     - Sets the power on mode to auto (so that the node turns on!).
     - Sets the idle timeout to 2 hours.
 + Sets the bootorder so that PXE is first. This has to be done
   with ssh and some expect stuff I culled from power_ilo. Sigh.
 + Calls out to another script that adds the ilo interface to the
   DB (this is the management_iface script I did last month).
 + Sends another XML file that tells the ilo to reset itself, so that
   it picks up its permanent IP address.

* Now we can free the nodes from hwdown.
parent ea9c678d
......@@ -7450,7 +7450,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
install/ops-install install/boss-install install/fs-install \
install/load-descriptors install/dump-descriptors \
install/newnode_sshkeys/GNUmakefile install/smb.conf.head \
install/clrhouse-install \
install/clrhouse-install install/genirack/GNUmakefile \
install/libinstall.pm install/update-install install/update-testbed \
mote/GNUmakefile mote/tbuisp mote/tbsgmotepower mote/newmote \
mote/sgtools/GNUmakefile \
......
......@@ -1143,7 +1143,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
install/ops-install install/boss-install install/fs-install \
install/load-descriptors install/dump-descriptors \
install/newnode_sshkeys/GNUmakefile install/smb.conf.head \
install/clrhouse-install \
install/clrhouse-install install/genirack/GNUmakefile \
install/libinstall.pm install/update-install install/update-testbed \
mote/GNUmakefile mote/tbuisp mote/tbsgmotepower mote/newmote \
mote/sgtools/GNUmakefile \
......
......@@ -165,6 +165,13 @@ sub IsControl($)
return $self->role() eq TBDB_IFACEROLE_CONTROL();
}
sub IsManagement($)
{
my ($self) = @_;
return $self->role() eq TBDB_IFACEROLE_MANAGEMENT();
}
#
# Lookup by card,port
#
......@@ -520,6 +527,51 @@ sub LookupByIface($$$)
return Interface->Lookup($nodeid, $card, $port);
}
#
# Lookup by mac
#
sub LookupByMAC($$)
{
my ($class, $mac) = @_;
my $query_result =
DBQueryWarn("select node_id,card,port from interfaces ".
"where mac='$mac'");
return undef
if (!$query_result);
return undef
if (!$query_result->numrows);
my ($nodeid, $card, $port) = $query_result->fetchrow_array();
return Interface->Lookup($nodeid, $card, $port);
}
#
# Lookup by IP, but only on control or management interfaces.
# Makes no sense for experimental.
#
sub LookupByIP($$)
{
my ($class, $ip) = @_;
my $query_result =
DBQueryWarn("select node_id,card,port from interfaces ".
"where ip='$ip' and ".
" (role='" . TBDB_IFACEROLE_CONTROL() . "' or ".
" role='" . TBDB_IFACEROLE_MANAGEMENT() . "')");
return undef
if (!$query_result);
return undef
if (!$query_result->numrows);
my ($nodeid, $card, $port) = $query_result->fetchrow_array();
return Interface->Lookup($nodeid, $card, $port);
}
#
# Lookup by uuid
#
......
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# All rights reserved.
#
SRCDIR = @srcdir@
TESTBED_SRCDIR = @top_srcdir@
OBJDIR = ../..
SUBDIR = install/genirack
include $(OBJDIR)/Makeconf
TARGETS = initnodes.pl
#
# Force dependencies on the scripts so that they will be rerun through
# configure if the .in file is changed.
#
all: $(TARGETS)
include $(TESTBED_SRCDIR)/GNUmakerules
clean:
rm -f $(TARGETS)
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
use Socket;
use IO::Handle; # thousands of lines just for autoflush :-(
use POSIX ":sys_wait_h";
#
# Setup management interfaces for nodes, given a data file we get
# from HP.
#
sub usage()
{
print STDERR "Usage: initnodes.pl [-d] [-n] <datafile>\n";
exit(-1);
}
my $optlist = "dn";
my $debug = 1;
my $impotent = 0;
my %ilomap = ();
my %ctrlmap = ();
my $tempfile = "/tmp/$$.xml";
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $LEASES = "/var/db/dhcpd.leases";
my $ILOPSWD = "$TB/etc/ilo.pswd";
my $SRCDIR = "@srcdir@";
my $CURL = "/usr/local/bin/curl";
my $ADDMNG = "$TB/sbin/management_iface";
my $DSAKEY = "/root/.ssh/id_dsa";
my $SUDO = "/usr/local/bin/sudo";
my $WAP = "$TB/sbin/withadminprivs";
# Protos
sub Fatal($);
sub ChangeBootOrder($);
sub SendXML($$);
# un-taint path
$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use emdb;
use EmulabConstants;
use emutil;
use User;
use Node;
use Interface;
#
# Turn off line buffering on output
#
$| = 1;
#
# Parse command arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"n"})) {
$impotent = 1;
}
if (@ARGV != 1) {
usage();
}
my $datafile = $ARGV[0];
Fatal("$datafile does not exist")
if (! -e $datafile);
Fatal("$ILOPSWD does not exist")
if (! -e $ILOPSWD);
#
# Must be root if actually doing this.
#
if ($UID && !$impotent) {
Fatal("This script must be run as root! Maybe use sudo?")
}
# This is the iLo password for the elabman user we create.
my $elabmanpswd = `cat $ILOPSWD`;
chomp($elabmanpswd);
# The XML goo.
my $setupgoo = `cat $SRCDIR/setupilo.xml`;
my $resetgoo = `cat $SRCDIR/resetilo.xml`;
# The pubkey.
my $pubkey = `cat ${DSAKEY}.pub`;
if ($?) {
Fatal("Cannot read ${DSAKEY}.pub");
}
chomp($pubkey);
# Need to kill off the comment.
if ($pubkey =~ /^(ssh-dss\s+[^\ ]*)/) {
$pubkey = $1;
}
#
# Read in the datafile.
#
open(DF, $datafile)
or Fatal("Could not open $datafile: $!");
while (<DF>) {
next
if ($_ =~ /^#/ || $_ =~ /^$/);
chomp($_);
my ($ilomac,$passwd,$ctrlmac,$ip) = split(",", $_);
$macmap{$ilomac} = {"ilomac" => lc($ilomac),
"passwd" => $passwd,
"ctrlmac" => lc($ctrlmac),
"iloIP" => $ip,
"tempIP" => undef,
"ctrlIP" => undef};
}
close(DF);
#
# Read the dhcp.leases files.
#
my $IP;
my $mac;
my $state = "free";
open(DF, $LEASES)
or Fatal("Could not open $LEASES: $!");
while (<DF>) {
next
if ($_ =~ /^#/);
# Looking for the start of a lease.
if ($_ =~ /^lease\s*([\d\.]*)/) {
$IP = $1;
}
elsif ($_ =~ /^\s*binding state (\w*)/) {
$state = $1;
}
elsif ($_ =~ /hardware ethernet ([\w:]*)/) {
$mac = $1;
# Convert mac to DB representation.
if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) {
$mac = "$1$2$3$4$5$6";
}
$mac = lc($mac);
}
elsif ($_ =~ /^\}$/) {
if ($state eq "active" && exists($macmap{$mac})) {
print "$IP, $state, $mac\n"
if ($debug);
if ($macmap{$mac}->{"ilomac"} eq $mac) {
$macmap{$mac}->{"tempIP"} = $IP;
}
elsif ($macmap{$mac}->{"ctrlmac"} eq $mac) {
$macmap{$mac}->{"ctrlIP"} = $IP;
}
else {
Fatal("Inconsistent record for $mac");
}
}
}
}
close(DF);
#
# See what iLo interfaces we found. Search the DB for the
# corresponding nodes, using the control mac. We only process
# nodes that have been incorporated into the testbed.
#
foreach my $map (values(%macmap)) {
next
if (!defined($map->{'tempIP'}));
my $ilomac = $map->{'ilomac'};
my $iloIP = $map->{'iloIP'};
my $tempIP = $map->{'tempIP'};
my $ctrlmac = $map->{'ctrlmac'};
my $ilopswd = $map->{'passwd'};
my $control_interface = Interface->LookupByMAC($ctrlmac);
if (!defined($control_interface)) {
print "No matching node for ilo mac: $ilomac/$ctrlmac\n";
next;
}
my $node_id = $control_interface->node_id();
my $node = Node->Lookup($node_id);
Fatal("Cannot lookup node: $node_id")
if (!defined($node));
print "ilo mac $ilomac ($tempIP) corresponds to $node\n";
#
# Make sure the desired IP is not in use.
#
my $ilo_interface = Interface->LookupByIP($iloIP);
if (defined($ilo_interface)) {
Fatal("Interface with $iloIP alraedt exists: $ilo_interface");
}
#
# Before we create the interface, make sure we can install our
# login/password/key info on the ilo.
#
# Replace the appropriate parts of the XML goo.
#
my $xmlgoo = sprintf($setupgoo, $ilopswd, $elabmanpswd, $pubkey);
if (SendXML($tempIP, $xmlgoo)) {
Fatal("Failed to send xmlgoo to $tempIP");
}
#
# The boot order cannot be changed via ribcl. What a pain.
#
if (ChangeBootOrder($tempIP)) {
Fatal("Failed to change the boot order on $ilomac ($iloIP)")
}
#
# Add the management interface
#
my $mcmd =
"$ADDMNG -t ilo3 -a key -s - $node_id $ilomac $iloIP elabman $DSAKEY";
print "$mcmd\n";
if (!$impotent) {
my $output = emutil::ExecQuiet("$SUDO -u elabman $WAP $mcmd");
if ($?) {
print $output;
Fatal("Could not add management iface");
}
}
#
# This resets the ilo so that it will DHCP again and get its new IP.
#
$xmlgoo = sprintf($resetgoo, $ilopswd);
if (SendXML($tempIP, $xmlgoo)) {
Fatal("Failed to send xmlgoo to $tempIP");
}
}
#
# SSH over to change the boot order,
# The "expect" like stuff copied from power_ilo ...
#
sub ChangeBootOrder($)
{
my ($ip) = @_;
my @args = ("ssh", "-tt", "-i", ${DSAKEY}, "elabman\@${ip}");
print "@args\n";
return 0
if ($impotent);
if (! socketpair(CHILD, PARENT, AF_UNIX, SOCK_STREAM, PF_UNSPEC)) {
Fatal("socketpair failed: $!");
}
CHILD->autoflush(1);
PARENT->autoflush(1);
my $childpid = fork();
if (! $childpid) {
close(CHILD);
#
# Dup our descriptors to the parent, and exec the program.
# The parent then talks to it read/write.
#
open(STDIN, "<&PARENT") || die "Can't redirect stdin";
open(STDOUT, ">&PARENT") || die "Can't redirect stdout";
open(STDERR, ">&PARENT") || die "Can't redirect stderr";
exec(@args);
die("ssh exec failed\n");
}
close(PARENT);
my @expect_seq = (['hpiLO-> ',"cd system1/bootconfig1"],
['hpiLO-> ','show bootsource5'],
['hpiLO-> ','set bootsource5 bootorder=1'],
['hpiLO-> ','exit']);
#
# Talk to ssh over the pty: wait for expected output and send responses
#
my @lines = ();
foreach $es (@expect_seq) {
my ($rval,$sval) = @$es;
my $found = 0;
my $line = '';
while (1) {
my $char;
if (read(CHILD,$char,1) != 1) {
warn "Error in read in iLO pseudo expect loop!\n";
print "Had read the following lines:\n";
foreach my $ln (@lines) {
print " $ln\n";
}
last;
}
if ($char eq "\r" || $char eq "\n") {
push @lines,$line;
if ($debug) {
if ($debug > 2) {
print "read '$line' while looking for '$rval'\n";
}
elsif ($line ne '') {
print "$line\n";
}
}
$line = '';
}
else {
$line .= $char;
}
if ($line =~ /$rval$/) {
print CHILD "$sval\r";
print "sent '$sval'\n";
$found = 1;
last;
}
}
if (!$found) {
# some sort of error; try to kill off ssh
kill(15,$childpid);
return -16;
}
}
close(CHILD);
# make sure the local ssh dies:
my $i = 5;
my $dead = 0;
while (--$i) {
my $ret = waitpid($childpid,WNOHANG);
if ($ret == -1 || $ret == $childpid) {
$dead = 1;
last;
}
sleep(1);
}
kill(KILL,$childpid) if (!$dead);
return 0;
}
#
# Send some XML to the ribcl
#
sub SendXML($$)
{
my ($ip, $xmlgoo) = @_;
print $xmlgoo
if ($debug);
# Stick it into a file for curl.
open(XML, ">$tempfile")
or Fatal("Could not create $tempfile");
print XML $xmlgoo;
close(XML);
#
# Ship this off with curl.
#
my $cmd = "$CURL -k --data-binary \@${tempfile} https://$ip/ribcl";
print "$cmd\n";
if (!$impotent) {
my $output = emutil::ExecQuiet($cmd);
if ($?) {
print $output;
Fatal("ribcl failed");
}
my @lines = split('\n', $output);
while (@lines) {
my $line = shift(@lines);
if ($line =~ /^\s*STATUS="(\w*)"/) {
my $status = hex($1);
if ($status != 0) {
my $line = shift(@lines);
$line =~ s/\s*MESSAGE=//;
print "$line\n";
Fatal("ribcl failed");
}
}
}
}
unlink($tempfile)
if (!$debug);
return 0;
}
exit(0);
sub Fatal($)
{
my ($msg) = @_;
die("*** $0:\n".
" $msg\n");
}
<!-- RIBCL Sample Script for HP Lights-Out Products -->
<!--Copyright (c) 2003,2010 Hewlett-Packard Development Company, L.P.-->
<RIBCL VERSION="2.0">
<LOGIN USER_LOGIN="Administrator" PASSWORD="%s">
<RIB_INFO MODE="write">
<RESET_RIB/>
</RIB_INFO>
</LOGIN>
</RIBCL>
<!-- RIBCL Sample Script for HP Lights-Out Products -->
<!--Copyright (c) 2003,2010 Hewlett-Packard Development Company, L.P.-->
<RIBCL VERSION="2.0">
<LOGIN USER_LOGIN="Administrator" PASSWORD="%s">
<USER_INFO MODE="write">
<ADD_USER
USER_NAME="elabman"
USER_LOGIN="elabman"
PASSWORD="%s">
<ADMIN_PRIV value ="Y"/>
<REMOTE_CONS_PRIV value ="Y"/>
<RESET_SERVER_PRIV value ="Y"/>
<VIRTUAL_MEDIA_PRIV value ="Y"/>
<CONFIG_ILO_PRIV value="Yes"/>
</ADD_USER>
</USER_INFO>
<RIB_INFO MODE="write">
<IMPORT_SSH_KEY>
-----BEGIN SSH KEY-----
%s elabman
-----END SSH KEY-----
</IMPORT_SSH_KEY>
</RIB_INFO>
<RIB_INFO MODE="write">
<!-- This allows Utah Emulab to ssh in without a password -->
<IMPORT_SSH_KEY>
-----BEGIN SSH KEY-----
ssh-dss AAAAB3NzaC1kc3MAAACBALrHBPWOanqnIevWTrh1vdU+vrR4+Gc/rwld3InKxgU7XlJGc+hs6AV373Xqfga09guMl+Q9wY8XlH3mHNsYRXSX7/wVZFFM59UEHHT7i9Ny34Ua2z+oqr0dxkuj9YkHWpr1/2+uP6R4148V4UYdYqTKbvHb/pUgYc0m+RqwmotdAAAAFQDjsb8pOXOsBO/O3ENMPxIgZXveUwAAAIBeD/O8SxMga+mPw93yGJuuMQig/pZf8sbZpneuYMCclZpSrL2apUqOdqLs8RtOfRFopMawf3Cpr1Qvqkz9FHJbDQKAg+l9SCiQgbd8tv7khQgR42UdUGKqbS/KvXX2jt2E0Febc3I7lXPf9hL9Nn502S/Q2AzU06MMlRjcvN6QTAAAAIBla9o4rgwNXlJgI6ywi877zecgxt227+28Wpkp6Yiiy+dUtv3rRgxLd1VixEChxhLY0nQe6vBaCOKQ1CTY3iTA0FHaWpm4BtEBlU2JrgF5+10GV0M350wJDzQrCc6Ishz7fTNMBMXHUSzp7EZisH0lrzHe2gdECcdn9cdqsye3QA== Administrator
-----END SSH KEY-----
</IMPORT_SSH_KEY>
</RIB_INFO>
<SERVER_INFO MODE="write">
<!-- Enable automatic power on with a minimum delay. -->
<!-- Note: iLO firmware supports values "Yes" and "No" only. -->
<SERVER_AUTO_PWR VALUE="Yes"/>
</SERVER_INFO>
<RIB_INFO MODE="write">
<!-- Set the autologout timeout to 120 minutes -->
<MOD_GLOBAL_SETTINGS>
<SESSION_TIMEOUT value="120"/>
</MOD_GLOBAL_SETTINGS>
</RIB_INFO>
</LOGIN>
</RIBCL>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment