Commit 9019197f authored by Leigh B Stoller's avatar Leigh B Stoller

A lot of changes for use on privately purchased InstaGeni Racks.

The GPO purchased racks all look the same so we can add the new
nodes statically from the XML file that HP sends us.

Well, for the UEN rack and other privately purchased racks, we do not
have any of that info, and so we have to figure things out ourselves.
We could use the standard newnode route without modification, but that
is a realy pain, say on a 33 node rack where the nodes land in any
order and you have no idea what the corresponding ilo is.

So I whacked this script to run from a data file we generate, which
contains the two bits of info from the top of the node (ilo passwd and
the ilo hostname), and the IPs we want to assign to the node and the
corresponding ilo.

So after turning all the nodes on and they checkin as usual, we run
this script with the -r option, which looks at the dhcpd.lease file to
find out what IP is which ILO, and the it asks each ilo via XMLRPC for
its hosts daya, which includes the ethernet MACs. I use that to find
the corresponding node in the leases file. Now I reorder the new_nodes
table so that pc1 is at the bottom and pcNN is at the top.

Now go back to the web interface and add all of the nodes.

Then run this script again (no -r), which will create all of the
management interfaces in the DB, reset the ilo, and then do the
normal initilo stuff for each node (boot order, password change, ssh
keys, etc).

Presto, done.
parent 354c2ea7
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -22,27 +22,50 @@
# }}}
#
use English;
use strict;
use Getopt::Std;
use Socket;
use IO::Handle; # thousands of lines just for autoflush :-(
use POSIX ":sys_wait_h";
use Data::Dumper;
use XML::Simple;
#
# Setup management interfaces for nodes, given a data file we get
# from HP.
# Set up nodes when we do not get an XML file from HP. We rely on
# a data file the user creates that provides for each node:
#
# Slot: Ilo Password, Serial#, IP, ILOIP
#
# 1: XXXXXX, MXQ327005N, 155.99.144.15, 155.99.144.55
# 2: ....
#
# After the nodes are all turned on and they "check into" the web
# interface, we run this script to reorder (-r) the nodes and find the
# associated ilo interfaces by looking at the dhcp leases file. We ask
# each ilo to tell us its configuration, which includes the MACs of
# all of the ethernet addressed (except the ilo!) and use that to
# match up against the contents of new_nodes and new_interfaces.
#
# Once the new nodes have been reordered, go back to the web interface
# proceed with adding the nodes as normal (do not forget to search
# the switches).
#
# After the nodes are added, we run this script again to actually add
# management interaces and wires to the DB, and initialize the ilo
# with the proper ssh pub key, boot order, etc. This will result in
# the ilo being reset and coming back with the new IP.
#
sub usage()
{
print STDERR "Usage: initnodes.pl [-d] [-n] <datafile>\n";
print STDERR "Usage: initnodes.pl [-d] [-n] -i <ip> <ilopswd>\n";
print STDERR "Usage: initnodes.pl [-d] [-n] [-r] <datafile>\n";
exit(-1);
}
my $optlist = "dni";
my $optlist = "dnr";
my $debug = 0;
my $impotent = 0;
my $ipmode = 0;
my %ilomap = ();
my %ctrlmap = ();
my $reorder = 0;
#
# Configure variables
......@@ -55,9 +78,11 @@ my $ADDMNG = "$TB/sbin/management_iface";
my $DSAKEY = "/root/.ssh/id_dsa";
my $SUDO = "/usr/local/bin/sudo";
my $WAP = "$TB/sbin/withadminprivs";
my $INITILO = "$TB/sbin/initilo.pl";
# Protos
sub Fatal($);
sub getILOcontrolmac($);
# un-taint path
$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/site/bin';
......@@ -87,11 +112,14 @@ if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
$debug++;
}
if (defined($options{"n"})) {
$impotent = 1;
}
if (defined($options{"r"})) {
$reorder = 1;
}
#
# Must be root if actually doing this.
......@@ -117,15 +145,17 @@ while (<DF>) {
if ($_ =~ /^#/ || $_ =~ /^$/);
chomp($_);
my ($ilomac,$passwd,$ctrlmac,$ip) = split(",", $_);
$macmap{$ilomac} = {"ilomac" => lc($ilomac),
"passwd" => $passwd,
"ctrlmac" => lc($ctrlmac),
"iloIP" => $ip,
"tempIP" => undef,
"ctrlIP" => undef};
if ($_ =~ /^(\d*):\s+(\w*),\s+(\w*),\s+([\d\.]*),\s+([\d\.]*)$/) {
$ilomap{"$1"} = {"slot" => $1,
"passwd" => $2,
"serial" => lc($3),
"ctrlIP" => $4,
"iloIP" => $5,
};
}
else {
Fatal("Could not parse input line: '$_'");
}
}
close(DF);
......@@ -134,6 +164,7 @@ close(DF);
#
my $IP;
my $mac;
my $hostname;
my $state = "free";
open(DF, $LEASES)
......@@ -144,7 +175,9 @@ while (<DF>) {
# Looking for the start of a lease.
if ($_ =~ /^lease\s*([\d\.]*)/) {
$IP = $1;
$IP = $1;
$mac = undef;
$hostname = undef;
}
elsif ($_ =~ /^\s*binding state (\w*)/) {
$state = $1;
......@@ -157,57 +190,157 @@ while (<DF>) {
}
$mac = lc($mac);
}
elsif ($_ =~ /client-hostname \"?([\w]*)\"?/) {
$hostname = lc($1);
}
elsif ($_ =~ /^\}$/) {
if ($state eq "active" && exists($macmap{$mac})) {
if ($state eq "active") {
print "$IP, $state, $mac\n"
if ($debug);
if ($macmap{$mac}->{"ilomac"} eq $mac) {
$macmap{$mac}->{"tempIP"} = $IP;
}
elsif ($macmap{$mac}->{"ctrlmac"} eq $mac) {
$macmap{$mac}->{"ctrlIP"} = $IP;
}
else {
Fatal("Inconsistent record for $mac");
next if
(!defined($hostname));
print "--> $hostname\n"
if ($debug);
#
# Search the list looking for a match between the hostname
# and the ilo serial number.
#
foreach my $slot (keys(%ilomap)) {
my $serial = $ilomap{"$slot"}->{"serial"};
if ("ilo${serial}" eq $hostname) {
print "Found match for $hostname in slot $slot\n";
$ilomap{"$slot"}->{"ilomac"} = $mac;
$ilomap{"$slot"}->{"tmp_iloIP"} = $IP;
$ilomap{"$slot"}->{"ctrlmac"} = getILOcontrolmac($IP)
if ($reorder);
print Dumper($ilomap{"$slot"})
if ($debug);
last;
}
}
}
}
}
close(DF);
if ($reorder) {
foreach my $slot (keys(%ilomap)) {
if (!exists($ilomap{"$slot"}->{"ctrlmac"})) {
print "Could not find a DHCP lease for node in slot $slot\n";
}
}
#
# Reorder new_nodes according to data file.
#
my $failures = 0;
foreach my $slot (keys(%ilomap)) {
my $blob = $ilomap{"$slot"};
next
if (!exists($blob->{'ctrlmac'}));
my $ctrlmac = $blob->{'ctrlmac'};
my $query_result =
DBQueryFatal("select i.new_node_id,n.node_id ".
" from new_interfaces as i ".
"left join new_nodes as n on ".
" n.new_node_id=i.new_node_id ".
"where i.mac='$ctrlmac'");
if (!$query_result->numrows) {
print "Could not find new nodes info for $ctrlmac in slot $slot\n";
$failures++;
next;
}
my ($new_node_id, $node_id) = $query_result->fetchrow_array();
$blob->{'new_node_id'} = $new_node_id;
$blob->{'node_id'} = $node_id;
#print Dumper($blob);
}
exit(1)
if ($failures);
#
# Since we could find everything, go ahead and reorder.
#
foreach my $slot (keys(%ilomap)) {
my $blob = $ilomap{"$slot"};
next
if (!exists($blob->{'ctrlmac'}));
my $new_node_id = $blob->{'new_node_id'};
my $node_id = $blob->{'node_id'};
my $new_id = "pc${slot}";
my $new_ip = $blob->{'ctrlIP'};
if ($impotent) {
print "Would change new node $new_node_id from ".
"$node_id to $new_id ($new_ip)\n";
}
else {
print "Changing new node $new_node_id from ".
"$node_id to $new_id ($new_ip)\n";
DBQueryFatal("update new_nodes set ".
" node_id='$new_id',IP='$new_ip' ".
"where new_node_id='$new_node_id'");
}
}
exit(0);
}
#
# See what iLo interfaces we found. Search the DB for the
# corresponding nodes, using the control mac. We only process
# nodes that have been incorporated into the testbed.
#
foreach my $map (values(%macmap)) {
next
if (!defined($map->{'tempIP'}));
my $ilomac = $map->{'ilomac'};
my $iloIP = $map->{'iloIP'};
my $tempIP = $map->{'tempIP'};
my $ctrlmac = $map->{'ctrlmac'};
my $ilopswd = $map->{'passwd'};
foreach my $slot (keys(%ilomap)) {
my $blob = $ilomap{"$slot"};
my $node_id = "pc${slot}";
my $node = Node->Lookup($node_id);
if (!defined($node)) {
print "Cannot lookup node $node_id, skipping ...\n";
next;
}
my $ilomac = $blob->{'ilomac'};
my $iloIP = $blob->{'iloIP'};
my $tempIP = $blob->{'tmp_iloIP'};
my $ctrlIP = $blob->{'ctrlIP'};
my $ilopswd = $blob->{'passwd'};
my $control_interface = Interface->LookupByMAC($ctrlmac);
my $control_interface = Interface->LookupByIP($ctrlIP);
if (!defined($control_interface)) {
print "No matching node for ilo mac: $ilomac/$ctrlmac\n";
print "No matching node: $ilomac/$ctrlIP\n";
next;
}
my $node_id = $control_interface->node_id();
my $node = Node->Lookup($node_id);
Fatal("Cannot lookup node: $node_id")
if (!defined($node));
if ($control_interface->node_id() ne $node_id) {
Fatal("Node mismatch; $node_id,$control_interface");
}
print "ILO mac $ilomac ($tempIP) corresponds to $node ($ctrlIP)\n";
print "ilo mac $ilomac ($tempIP) corresponds to $node\n";
#
# Skip if already created.
#
my $management_interface = Interface->LookupManagement($node_id);
if (defined($management_interface)) {
print "Management interface for $node_id exists; skipping ...\n";
next;
}
#
# Make sure the desired IP is not in use.
#
my $ilo_interface = Interface->LookupByIP($iloIP);
if (defined($ilo_interface)) {
Fatal("Interface with $iloIP alraedt exists: $ilo_interface");
Fatal("Interface with $iloIP already exists: $ilo_interface");
}
#
# Ping the ILO interface to make sure the switch knows its mac.
#
if (!$impotent) {
print "-> pinging $tempIP for a few seconds ...\n";
emutil::ExecQuiet("ping -q -c 3 $tempIP");
}
#
......@@ -216,7 +349,8 @@ foreach my $map (values(%macmap)) {
#
my $mcmd =
"$ADDMNG -t ilo3 -a key -s - $node_id $ilomac $iloIP elabman $DSAKEY";
print "$mcmd\n";
print "$mcmd\n"
if ($debug);
if (!$impotent) {
my $output = emutil::ExecQuiet("$SUDO -u elabman $WAP $mcmd");
if ($?) {
......@@ -225,17 +359,82 @@ foreach my $map (values(%macmap)) {
}
}
#
# Reset the ilo so that it picks up the new IP address. The next
# step requires this. Takes about 30 seconds to reset.
#
print "-> Resetting the ILO ...\n";
$mcmd = "$INITILO -r $tempIP $ilopswd";
print "$mcmd\n"
if ($debug);
if (!$impotent) {
my $output = emutil::ExecQuiet("$SUDO $mcmd");
if ($?) {
print $output;
Fatal("Could not reset ilo");
}
print "-> Waiting 30 seconds for ILO to reboot ...\n";
sleep(30);
}
#
# Initialize iLo interface; when done, it is resetting and will get
# new IP.
#
system("$SUDO -u elabman $WAP perl initilo.pl $tempIP $ilopswd");
fatal("Could not initialize ilo")
if ($?);
print "-> Initializing the ILO ... \n";
$mcmd = "$INITILO $ilopswd $node_id";
print "$mcmd\n"
if ($debug);
if (!$impotent) {
my $output = emutil::ExecQuiet("$SUDO $mcmd");
if ($?) {
print $output;
Fatal("Could not add initialize ilo");
}
}
}
exit(0);
#
# Get the XML data from the ilo at IP and find the control MAC.
#
sub getILOcontrolmac($)
{
my ($ip) = @_;
my $output =
emutil::ExecQuiet("wget -q -O - --no-check-certificate ".
"https://$ip/xmldata?item=all");
if ($?) {
print STDERR "$output";
Fatal("Could not get XML data from $ip");
}
my $xmlparse = eval { XMLin($output,
VarAttr => 'name',
ContentKey => '-content',
SuppressEmpty => undef); };
Fatal($@)
if ($@);
print Dumper($xmlparse)
if ($debug > 1);
if (exists($xmlparse->{"HSI"}->{"NICS"}->{"NIC"})) {
foreach my $nic (@{ $xmlparse->{"HSI"}->{"NICS"}->{"NIC"} }) {
if ($nic->{"PORT"} eq "1") {
my $mac = $nic->{"MACADDR"};
# Convert mac to DB representation.
if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) {
$mac = "$1$2$3$4$5$6";
}
return $mac;
}
}
}
Fatal("Could not find control net mac in ILO xml");
}
sub Fatal($)
{
my ($msg) = @_;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment