newnode.in 11.6 KB
Newer Older
1 2 3 4
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2003, 2004 University of Utah and the Flux Group.
6 7 8 9 10 11 12
# All rights reserved.
#

#
# newnode - a script for moving nodes from the new_ tables into production.
#

13
use lib '@prefix@/lib';
14 15

use libdb;
Robert Ricci's avatar
Robert Ricci committed
16
use English;
17
use Getopt::Std;
18 19 20

use strict;

21
my $TB = "@prefix@";
22 23

my $switchmac = "$TB/sbin/switchmac";
24 25 26 27
my $os_load = "$TB/bin/os_load";
my $os_select = "$TB/bin/os_select";
my $newnode_reboot = "$TB/sbin/newnode_reboot";
my $named_setup = "$TB/sbin/named_setup";
28 29 30
my $nalloc = "$TB/bin/nalloc";
my $nfree = "$TB/bin/nfree";
my $dhcpd_makeconf = "$TB/sbin/dhcpd_makeconf";
31
my $exports_setup = "$TB/sbin/exports_setup";
32

33
my $dhcpd_conf = "/usr/local/etc/dhcpd.conf";
34 35 36
my $dhcpd_template = "/usr/local/etc/dhcpd.conf.template";
my $dhcpd_rc = "/usr/local/etc/rc.d/2.dhcpd.sh";

37
my $sudo = "/usr/local/bin/sudo -S";
38 39

#
40
# MFS to boot the nodes into initially
41
#
42
my $INITIAL_MFS = TB_OSID_FREEBSD_MFS();
43 44

#
45
# Initial event system state to put the nodes into
46
#
47
my $INITIAL_STATE = TBDB_NODESTATE_SHUTDOWN;
48 49 50 51 52 53

#
# Number of vnodes to create for each physical node
#
my $NUM_VNODES = 50;

54 55 56 57 58
#
# Default number of vnodes that can be hosted on any pnode
#
my $DEFAULT_PCVM_COUNT = 20;

59 60
# Turn off line buffering on output
$| = 1;
61

62 63 64 65
if (!TBAdmin()) {
    die "Sorry, only testbed administrators can run this script!\n";
}

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
#
# Handle command-line options
#
my $force_unconnected_interfaces = 0;
my %options = ();
if (! getopts("f", \%options)) {
    usage();
}

if ($options{f}) {
    $force_unconnected_interfaces = 1;
}

sub usage() {
    die "Usage: $0 [-f] <node_id> ...\n";
}

83 84 85 86 87 88
#
# We seem to have to do this so that ssh gets proper root permissions to read
# the key file. Argh.
#
$UID = $EUID = 0;

89
if (@ARGV < 1) {
90
    usage();
91 92
}

93 94 95
#
# The user has to be able to run sudo, so they can restart dhcpd.
#
96
if (system "$sudo /bin/pwd < /dev/null") {
97 98 99
    die "You must be able to sudo to root to use this script\n";
}

100 101 102 103 104 105 106 107 108 109 110
#
# Make sure that the dhcpd template file exists, and that the real version
# is writable
#
if (!-e $dhcpd_template) {
    die "In order to use this script, $dhcpd_template must exist!\n";
}
if (!-w $dhcpd_conf) {
    die "In order to use this script, you must be able to write $dhcpd_conf\n";
}

111 112 113 114 115 116 117 118 119 120
#
# Find out what op_mode the $INITIAL_MFS runs in
#
my $result = DBQueryFatal("select op_mode from os_info where " .
    " osid='$INITIAL_MFS'");
if ($result->numrows() != 1) {
    die "Unable to find OS information for $INITIAL_MFS\n";
}
my ($INITIAL_OPMODE) = $result->fetchrow();

121 122 123 124 125 126 127 128 129 130
#
# For vnodes - figure out the jail IP base
#
my $IPBASE;
if (TBDB_JAILIPBASE =~ /^(\d+).(\d+).(\d+).(\d+)/) {
    $IPBASE = "$1.$2";
} else {
    die "Problem with JAILIPBASE\n"
}

131 132 133 134 135
my @node_ids = @ARGV;

#
# Now, loop through the nodes given, and add each one
#
136
my (@succeeded_nodes, @succeeded_IPs);
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
NODE: foreach my $node_id (@node_ids) {
    my $query_result;

    #
    # Check to make sure said node does not already exist!
    #
    $query_result = DBQueryFatal("SELECT node_id FROM nodes WHERE " .
	"node_id='$node_id'");
    if ($query_result->num_rows()) {
	warn "Node $node_id failed: a node with that name already exists!\n";
	next NODE;
    }

    #
    # Grab information about the node from the new_nodes table
    #
153 154
    $query_result = DBQueryFatal("SELECT new_node_id, type, IP, temporary_IP, ".
	"floor, building, loc_x, loc_y, contact, phone, room " .
155
	"FROM new_nodes WHERE node_id='$node_id'");
156 157 158 159
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: No pending node with that name exists!\n";
	next NODE;
    }
160 161
    my ($new_node_id, $type, $IP, $tempIP, $floor, $building, $loc_x, $loc_y,
	$contact, $phone, $room) = $query_result->fetchrow();
162

163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
    #
    # Make sure they've given it an IP address that looks valid
    #
    if ($IP !~ /^(\d+).(\d+).(\d+).(\d+)/) {
	warn "Node $node_id has an invalid IP address ($IP) - skipping\n";
	next NODE;
    }

    #
    # Make sure some other node doesn't already have this IP address on its
    # control interface
    #
    $query_result = DBQueryFatal("SELECT node_id FROM interfaces WHERE " .
	"IP='$IP' AND role='" . TBDB_IFACEROLE_CONTROL() . "'");
    if ($query_result->num_rows()) {
	my ($existing_node_id) = $query_result->fetchrow();
	warn "Node $node_id failed: $existing_node_id is already using IP " .
		"address $IP\n";
	next NODE;
    }

184 185 186 187
    #
    # Make sure that the new node is of a valid type, and grab a few other
    # things to fill in as initial values
    #
188
    $query_result = DBQueryFatal("SELECT control_iface FROM node_types " .
189 190 191 192 193
	"WHERE type='$type'");
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Type $type does not exist!\n";
	next NODE;
    }
194
    my ($control_iface) = $query_result->fetchrow();
195 196 197 198

    #
    # Grab the node's MACs from the new_interfaces table
    #
199
    $query_result = DBQueryFatal("SELECT card, MAC, interface_type, " .
200
	"switch_id, switch_card, switch_port, cable, len " .
201
	"FROM new_interfaces WHERE new_node_id='$new_node_id'");
202 203 204 205 206 207
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Must have at least one interface!\n";
	next NODE;
    }

    my %interfaces;
208
    while (my ($card, $MAC, $iface_type, $switch_id, $switch_card,
209
	    $switch_port, $cable, $len) = $query_result->fetchrow()) {
210 211 212
	#
	# Get some more information about this interface type
	#
213 214
	my $iface_query = DBQueryFatal("SELECT max_speed, full_duplex, " .
	    "connector FROM interface_types WHERE type='$iface_type'");
215
	if (!$iface_query->num_rows()) {
216
	    warn "Node $node_id failed: Interface $card is of unknown type " .
217
	    	"$iface_type\n";
218 219 220
	    next NODE;
	}

221 222
	my ($max_speed, $full_duplex, $connector) = $iface_query->fetchrow();

223 224 225 226 227 228
	#
	# Do a sanity check - make sure that we have a switch recorded for all
	# experimental interfaces
	#
	unless ($force_unconnected_interfaces) {
	    my $iface = "eth$card";
229 230
	    if (($iface ne $control_iface) && ($connector !~ /wireless/i)
		&& !$switch_id) {
231 232 233 234 235 236
		warn "Node $node_id failed: Don't know which switch " .
		    "card $iface is connected to\n";
		next NODE;
	    }
	}

237 238 239
	#
	# Stash it away...
	#
240
	$interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex,
241
		$switch_id, $switch_card, $switch_port, $cable, $len];
242 243 244 245 246 247

    }

    #
    # Make up a priority (just used for sorting)
    #
248 249
    $node_id =~ /^(.*\D)(\d+)$/;
    my ($prefix,$nodenum) = ($1, $2);
250
    my $priority;
251 252
    if (defined $nodenum) {
	$priority = $nodenum;
253 254 255 256 257 258 259 260 261
    } else {
	$priority = 1;
    }

    #
    # Okay, time to actually add the node!
    #
    
    DBQueryFatal("INSERT INTO nodes SET node_id='$node_id', type='$type', " .
262
	"phys_nodeid='$node_id', role='testnode', priority=$priority, " .
263 264 265 266
    	"eventstate='$INITIAL_STATE', op_mode='$INITIAL_OPMODE', " .
	"def_boot_osid='$INITIAL_MFS', " .
	"state_timestamp=unix_timestamp(NOW()), " .
	"op_mode_timestamp=unix_timestamp(NOW())");
267

268 269 270
    DBQueryFatal("insert into node_hostkeys (node_id) ".
		 "values ('$node_id')");

271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
    #
    # Copy data into the location_info table, if they provided any
    #
    if ($floor || $building || $loc_x || $loc_y || $contact
	|| $phone || $room) {

	#
	# Handle columns that could have come to us NULL (and thus are undef)
	#
	if (defined $floor) {
	    $floor = "'$floor'";
	} else {
	    $floor = "NULL";
	}
	if (defined $building) {
	    $building = "'$building'";
	} else {
	    $building = "NULL";
	}
	if (defined $room) {
	    $room = "'$room'";
	} else {
	    $room = "NULL";
	}

	DBQueryFatal("INSERT INTO location_info SET node_id='$node_id', " .
	    "floor=$floor, building=$building, loc_x=$loc_x, loc_y=$loc_y, ".
	    "contact='$contact', phone='$phone', room=$room;");
    }

301 302 303 304 305 306 307 308
    #
    # Add some vnodes
    #
    if (defined $nodenum) {
	my $vtype = $type;
	if (!($vtype =~ s/pc/pcvm/)) {
	    $vtype = "$vtype-vm";
	}
309 310 311 312
	DBQueryFatal("INSERT INTO node_auxtypes set node_id='$node_id', " .
	    "type='pcvm', count=$DEFAULT_PCVM_COUNT");
	DBQueryFatal("INSERT INTO node_auxtypes set node_id='$node_id', " .
	    "type='$vtype', count=$DEFAULT_PCVM_COUNT");
313 314 315 316 317 318 319 320 321 322
	for (my $i = 1; $i <= $NUM_VNODES; $i++) {
	    my $vpriority = 10000000 + ($nodenum * 1000) + $i;
	    my $nodename = $node_id;
	    if (!($nodename =~ s/pc/pcvm/)) {
		$nodename = "$nodename-vm";
	    }
	    $nodename .= "-$i";
	    my $jailip = "${IPBASE}.${nodenum}.${i}";

	    DBQueryFatal("INSERT INTO nodes SET node_id='$nodename', " .
323
		"type='$type', phys_nodeid='$node_id', role='virtnode', " .
324 325 326 327 328 329 330
		"priority='$vpriority', op_mode='PCVM', " .
		"eventstate='SHUTDOWN', " .
		"def_boot_osid='emulab-ops-FBSD-JAIL', " .
		"update_accounts=1, jailflag=1, jailip='$jailip'");
	}
    }

331
    while (my ($card, $aref) = each %interfaces) {
332
	my ($MAC, $iface_type, $speed, $duplex, $switch_id, $switch_card,
333
	    $switch_port, $cable, $len) = @$aref;
334
	my $iface = "eth$card";
335 336
	my $iface_IP = "";
	my $wire_type = "Node";
337
	my $iface_role = TBDB_IFACEROLE_EXPERIMENT();
338
	if ($iface eq $control_iface) {
339 340
	    $iface_IP = $IP;
	    $wire_type = "Control";
341
	    $iface_role = TBDB_IFACEROLE_CONTROL();
342 343 344 345
	}
	DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " .
	    "card=$card, port=1, mac='$MAC', IP='$iface_IP', " .
	    "interface_type='$iface_type', iface='$iface', " .
346
	    "current_speed='$speed', duplex=$duplex, role='$iface_role'");
347

348 349 350 351
	if (!$switch_id) {
	    print "No switch found for ${node_id}:$iface - skipping\n";
	    next;
	}
352 353 354 355 356 357 358 359

	my $cable_len = "";
	if ($cable) {
	    $cable_len .= ", cable=$cable";
	}
	if ($len) {
	    $cable_len .= ", len=$len";
	}
360 361 362
	DBQueryFatal("INSERT INTO wires SET type='$wire_type', " .
	    "node_id1='$node_id', card1=$card, port1=1, " .
	    "node_id2='$switch_id', card2='$switch_card', " .
363
	    "port2='$switch_port' $cable_len");
364 365 366
    }

    #
367 368
    # Put it into hwdown for now - I would put them in reloading, but I'm
    # afriad the reload_daemon might do the wrong thing to them
369 370 371 372 373 374
    #
    system "$nalloc emulab-ops hwdown $node_id";

    #
    # Remove the node from the new_ tables
    #
375
    DBQueryFatal("DELETE FROM new_nodes WHERE new_node_id=$new_node_id");
376
    DBQueryFatal("DELETE FROM new_interfaces WHERE new_node_id=$new_node_id");
377

378
    print "$node_id succesfully added!\n";
379 380

    push @succeeded_nodes, $node_id;
381
    push @succeeded_IPs, $tempIP;
382 383 384 385 386 387 388 389 390 391 392 393
}

#
# No point in restarting dhcpd, etc. if there are no nodes that succeeded
#
if (!@succeeded_nodes) {
    die "No nodes succeeded, exiting early\n";
}

#
# Re-generate dhcpd.conf
#
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
print "Re-generating dhcpd.conf\n";
open(CONF,"$dhcpd_makeconf $dhcpd_template|") or die "Unable to fork: $!\n";
my @conf = <CONF>;
close CONF or die "Error reading from dhcpd_makeconf: $!\n";

open(CONF,">$dhcpd_conf") or die "Unable to open $dhcpd_conf for writing\n";
print CONF @conf;
close CONF;

print "Restarting dhcpd: $sudo $dhcpd_rc stop\n";
my $sudo_rv = system "$sudo $dhcpd_rc stop";
if ($sudo_rv) {
    die "Error stopping dhcpd - return value was $sudo_rv\n";
}
sleep 2;
print "Restarting dhcpd: $sudo $dhcpd_rc start\n";
$sudo_rv = system "$sudo $dhcpd_rc start";
if ($sudo_rv) {
    die "Error starting dhcpd - return value was $sudo_rv\n";
}
414

415 416 417 418 419
print "Setting up nameserver\n";
my $named_rv = system "$named_setup";
if ($named_rv) {
    die "Error running named_setup - return value was $named_rv\n";
}
420

421 422 423 424 425 426 427 428 429
#
# Before we boot nodes into the MFS, we have to make sure they can mount
# NFS filesystems
#
print "Running exports_setup\n";
my $exports_rv = system "$exports_setup";
if ($exports_rv) {
    warn "WARNING - exports_setup returned $exports_rv";
}
430

431 432 433 434 435 436 437
#
# Start rebooting nodes
#
print "Rebooting nodes...\n";
foreach my $IP (@succeeded_IPs) {
    print "Rebooting $IP\n";
    system "$newnode_reboot $IP\n";
438 439
}

440 441 442 443 444
print "\n\n";
print "Finished - when you are satisifed that the nodes are working\n";
print "correctly, use nfree on boss to free them from the emulab-ops/hwdown\n";
print "experiment.\n";

445
#
446 447 448
# TODO -
#    disable interfaces
#    console setup
449
#