initnodes.pl.in 10.7 KB
Newer Older
1 2
#!/usr/bin/perl -w
#
3
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
23 24
#
use English;
25
use strict;
26 27 28 29
use Getopt::Std;
use Socket;
use IO::Handle;     # thousands of lines just for autoflush :-(
use POSIX ":sys_wait_h";
30 31
use Data::Dumper;
use XML::Simple;
32 33

#
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
# Set up nodes when we do not get an XML file from HP. We rely on
# a data file the user creates that provides for each node:
#
# Slot: Ilo Password, Serial#, IP, ILOIP
#
# 1:  XXXXXX, MXQ327005N, 155.99.144.15, 155.99.144.55
# 2:  ....
#
# After the nodes are all turned on and they "check into" the web
# interface, we run this script to reorder (-r) the nodes and find the
# associated ilo interfaces by looking at the dhcp leases file. We ask
# each ilo to tell us its configuration, which includes the MACs of
# all of the ethernet addressed (except the ilo!) and use that to
# match up against the contents of new_nodes and new_interfaces.
#
# Once the new nodes have been reordered, go back to the web interface
# proceed with adding the nodes as normal (do not forget to search
# the switches).
#
# After the nodes are added, we run this script again to actually add
# management interaces and wires to the DB, and initialize the ilo
# with the proper ssh pub key, boot order, etc. This will result in
# the ilo being reset and coming back with the new IP. 
57 58 59
#
sub usage()
{
60
    print STDERR "Usage: initnodes.pl [-d] [-n] [-r] <datafile>\n";
61 62
    exit(-1);
}
63
my $optlist	= "dnr";
64
my $debug	= 0;
65 66 67
my $impotent    = 0;
my %ilomap      = ();
my %ctrlmap     = ();
68
my $reorder     = 0;
69 70 71 72 73 74 75 76 77 78 79 80

#
# Configure variables
#
my $TB	      = "@prefix@";
my $TBOPS     = "@TBOPSEMAIL@";
my $LEASES    = "/var/db/dhcpd.leases";
my $SRCDIR    = "@srcdir@";
my $ADDMNG    = "$TB/sbin/management_iface";
my $DSAKEY    = "/root/.ssh/id_dsa";
my $SUDO      = "/usr/local/bin/sudo";
my $WAP       = "$TB/sbin/withadminprivs";
81
my $INITILO   = "$TB/sbin/initilo.pl";
82 83 84

# Protos
sub Fatal($);
85
sub getILOcontrolmac($);
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114

# un-taint path
$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use emdb;
use EmulabConstants;
use emutil;
use User;
use Node;
use Interface;

#
# Turn off line buffering on output
#
$| = 1;

#
# Parse command arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
115
    $debug++;
116 117 118 119
}
if (defined($options{"n"})) {
    $impotent = 1;
}
120 121 122
if (defined($options{"r"})) {
    $reorder = 1;
}
123 124 125 126 127 128 129 130

#
# Must be root if actually doing this.
#
if ($UID && !$impotent) {
    Fatal("This script must be run as root! Maybe use sudo?")
}

131 132
usage()
    if (@ARGV != 1);
133 134 135 136

#
# Read in the datafile.
#
137 138 139 140
my $datafile = $ARGV[0];
Fatal("$datafile does not exist")
    if (! -e $datafile);

141 142 143 144 145 146 147
open(DF, $datafile)
    or Fatal("Could not open $datafile: $!");
while (<DF>) {
    next
	if ($_ =~ /^#/ || $_ =~ /^$/);
    chomp($_);

148 149 150 151 152 153 154 155 156 157 158
    if ($_ =~ /^(\d*):\s+(\w*),\s+(\w*),\s+([\d\.]*),\s+([\d\.]*)$/) {
	$ilomap{"$1"}  = {"slot"    => $1,
			  "passwd"  => $2,
			  "serial"  => lc($3),
			  "ctrlIP"  => $4,
			  "iloIP"   => $5,
	};
    }
    else {
	Fatal("Could not parse input line: '$_'");
    }
159 160 161 162 163 164 165 166
}
close(DF);

#
# Read the dhcp.leases files.
#
my $IP;
my $mac;
167
my $hostname;
168 169 170 171 172 173 174 175 176 177
my $state = "free";

open(DF, $LEASES)
    or Fatal("Could not open $LEASES: $!");
while (<DF>) {
    next
	if ($_ =~ /^#/);

    # Looking for the start of a lease.
    if ($_ =~ /^lease\s*([\d\.]*)/) {
178 179 180
	$IP  = $1;
	$mac = undef;
	$hostname = undef;
181 182 183 184 185 186 187 188 189 190 191 192
    }
    elsif ($_ =~ /^\s*binding state (\w*)/) {
	$state = $1;
    }
    elsif ($_ =~ /hardware ethernet ([\w:]*)/) {
	$mac = $1;
        # Convert mac to DB representation.
	if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) {
	    $mac = "$1$2$3$4$5$6";
	}
	$mac = lc($mac);
    }
193 194 195
    elsif ($_ =~ /client-hostname \"?([\w]*)\"?/) {
	$hostname = lc($1);
    }
196
    elsif ($_ =~ /^\}$/) {
197
	if ($state eq "active") {
198 199
	    print "$IP, $state, $mac\n"
		if ($debug);
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
	    next if
		(!defined($hostname));
	    print "--> $hostname\n"
		if ($debug);
	    #
	    # Search the list looking for a match between the hostname
	    # and the ilo serial number.
	    #
	    foreach my $slot (keys(%ilomap)) {
		my $serial = $ilomap{"$slot"}->{"serial"};
		if ("ilo${serial}" eq $hostname) {
		    print "Found match for $hostname in slot $slot\n";
		    $ilomap{"$slot"}->{"ilomac"}    = $mac;
		    $ilomap{"$slot"}->{"tmp_iloIP"} = $IP;
		    $ilomap{"$slot"}->{"ctrlmac"}   = getILOcontrolmac($IP)
			if ($reorder);
		    print Dumper($ilomap{"$slot"})
			if ($debug);
		    last;
		}
220 221 222 223 224 225
	    }
	}
    }
}
close(DF);

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
if ($reorder) {
    foreach my $slot (keys(%ilomap)) {
	if (!exists($ilomap{"$slot"}->{"ctrlmac"})) {
	    print "Could not find a DHCP lease for node in slot $slot\n";
	}
    }
    #
    # Reorder new_nodes according to data file.
    #
    my $failures = 0;
    
    foreach my $slot (keys(%ilomap)) {
	my $blob = $ilomap{"$slot"};
	next
	    if (!exists($blob->{'ctrlmac'}));

	my $ctrlmac = $blob->{'ctrlmac'};
	
	my $query_result =
	    DBQueryFatal("select i.new_node_id,n.node_id ".
			 "   from new_interfaces as i ".
			 "left join new_nodes as n on ".
			 "     n.new_node_id=i.new_node_id ".
			 "where i.mac='$ctrlmac'");
	if (!$query_result->numrows) {
	    print "Could not find new nodes info for $ctrlmac in slot $slot\n";
	    $failures++;
	    next;
	}
	my ($new_node_id, $node_id) = $query_result->fetchrow_array();
	$blob->{'new_node_id'} = $new_node_id;
	$blob->{'node_id'}     = $node_id;
	#print Dumper($blob);
    }
    exit(1)
	if ($failures);
    
    #
    # Since we could find everything, go ahead and reorder.
    #
    foreach my $slot (keys(%ilomap)) {
	my $blob = $ilomap{"$slot"};
	next
	    if (!exists($blob->{'ctrlmac'}));

	my $new_node_id = $blob->{'new_node_id'};
	my $node_id     = $blob->{'node_id'};
	my $new_id      = "pc${slot}";
	my $new_ip      = $blob->{'ctrlIP'};

	if ($impotent) {
	    print "Would change new node $new_node_id from ".
		"$node_id to $new_id ($new_ip)\n";
	}
	else {
	    print "Changing new node $new_node_id from ".
		"$node_id to $new_id ($new_ip)\n";

	    DBQueryFatal("update new_nodes set ".
			 "  node_id='$new_id',IP='$new_ip' ".
			 "where new_node_id='$new_node_id'");
	}
    }
    exit(0);
}

292 293 294 295 296
#
# See what iLo interfaces we found. Search the DB for the
# corresponding nodes, using the control mac. We only process
# nodes that have been incorporated into the testbed.
#
297 298 299 300 301 302 303 304 305 306 307 308 309
foreach my $slot (keys(%ilomap)) {
    my $blob    = $ilomap{"$slot"};
    my $node_id = "pc${slot}";
    my $node    = Node->Lookup($node_id);
    if (!defined($node)) {
	print "Cannot lookup node $node_id, skipping ...\n";
	next;
    }
    my $ilomac   = $blob->{'ilomac'};
    my $iloIP    = $blob->{'iloIP'};
    my $tempIP   = $blob->{'tmp_iloIP'};
    my $ctrlIP   = $blob->{'ctrlIP'};
    my $ilopswd  = $blob->{'passwd'};
310

311
    my $control_interface = Interface->LookupByIP($ctrlIP);
312
    if (!defined($control_interface)) {
313
	print "No matching node: $ilomac/$ctrlIP\n";
314 315
	next;
    }
316 317 318 319
    if ($control_interface->node_id() ne $node_id) {
	Fatal("Node mismatch; $node_id,$control_interface");
    }
    print "ILO mac $ilomac ($tempIP) corresponds to $node ($ctrlIP)\n";
320

321 322 323 324 325 326 327 328
    #
    # Skip if already created. 
    #
    my $management_interface = Interface->LookupManagement($node_id);
    if (defined($management_interface)) {
	print "Management interface for $node_id exists; skipping ...\n";
	next;
    }
329 330 331 332 333 334

    #
    # Make sure the desired IP is not in use. 
    #
    my $ilo_interface = Interface->LookupByIP($iloIP);
    if (defined($ilo_interface)) {
335 336 337 338 339 340 341 342 343
	Fatal("Interface with $iloIP already exists: $ilo_interface");
    }

    #
    # Ping the ILO interface to make sure the switch knows its mac.
    #
    if (!$impotent) {
	print "-> pinging $tempIP for a few seconds ...\n";
	emutil::ExecQuiet("ping -q -c 3 $tempIP");
344 345 346
    }

    #
347 348
    # Add the management interface. The interfaces table will now have
    # a proper DHCP entry for it. 
349 350 351
    #
    my $mcmd =
	"$ADDMNG -t ilo3 -a key -s - $node_id $ilomac $iloIP elabman $DSAKEY";
352 353
    print "$mcmd\n"
	if ($debug);
354 355 356 357 358 359 360 361
    if (!$impotent) {
	my $output = emutil::ExecQuiet("$SUDO -u elabman $WAP $mcmd");
	if ($?) {
	    print $output;
	    Fatal("Could not add management iface");
	}
    }

362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
    #
    # Reset the ilo so that it picks up the new IP address. The next
    # step requires this. Takes about 30 seconds to reset.
    #
    print "-> Resetting the ILO ...\n";
    $mcmd = "$INITILO -r $tempIP $ilopswd";
    print "$mcmd\n"
	if ($debug);
    if (!$impotent) {
	my $output = emutil::ExecQuiet("$SUDO $mcmd");
	if ($?) {
	    print $output;
	    Fatal("Could not reset ilo");
	}
	print "-> Waiting 30 seconds for ILO to reboot ...\n";
	sleep(30);
    }
    
380
    #
381 382
    # Initialize iLo interface; when done, it is resetting and will get
    # new IP.
383
    #
384 385 386 387 388 389 390 391 392 393 394
    print "-> Initializing the ILO ... \n";
    $mcmd = "$INITILO $ilopswd $node_id";
    print "$mcmd\n"
	if ($debug);
    if (!$impotent) {
	my $output = emutil::ExecQuiet("$SUDO $mcmd");
	if ($?) {
	    print $output;
	    Fatal("Could not add initialize ilo");
	}
    }
395 396 397 398
}

exit(0);

399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
#
# Get the XML data from the ilo at IP and find the control MAC.
#
sub getILOcontrolmac($)
{
    my ($ip) = @_;

    my $output =
	emutil::ExecQuiet("wget -q -O - --no-check-certificate ".
			  "https://$ip/xmldata?item=all");
    if ($?) {
	print STDERR "$output";
	Fatal("Could not get XML data from $ip");
    }
    my $xmlparse = eval { XMLin($output,
				VarAttr => 'name',
				ContentKey => '-content',
				SuppressEmpty => undef); };
    Fatal($@)
	if ($@);
    
    print Dumper($xmlparse)
	if ($debug > 1);

    if (exists($xmlparse->{"HSI"}->{"NICS"}->{"NIC"})) {
	foreach my $nic (@{ $xmlparse->{"HSI"}->{"NICS"}->{"NIC"} }) {
	    if ($nic->{"PORT"} eq "1") {
		my $mac = $nic->{"MACADDR"};
		# Convert mac to DB representation.
		if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) {
		    $mac = "$1$2$3$4$5$6";
		}
		return $mac;
	    }
	}
    }
    Fatal("Could not find control net mac in ILO xml");
}

438 439 440 441 442 443 444
sub Fatal($)
{
    my ($msg) = @_;

    die("*** $0:\n".
	"    $msg\n");
}