newnode.in 10.8 KB
Newer Older
1 2 3 4
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2003, 2004 University of Utah and the Flux Group.
6 7 8 9 10 11 12
# All rights reserved.
#

#
# newnode - a script for moving nodes from the new_ tables into production.
#

13
use lib '@prefix@/lib';
14 15

use libdb;
Robert Ricci's avatar
Robert Ricci committed
16
use English;
17
use Getopt::Std;
18 19 20

use strict;

21
my $TB = "@prefix@";
22 23

my $switchmac = "$TB/sbin/switchmac";
24 25 26 27
my $os_load = "$TB/bin/os_load";
my $os_select = "$TB/bin/os_select";
my $newnode_reboot = "$TB/sbin/newnode_reboot";
my $named_setup = "$TB/sbin/named_setup";
28 29 30
my $nalloc = "$TB/bin/nalloc";
my $nfree = "$TB/bin/nfree";
my $dhcpd_makeconf = "$TB/sbin/dhcpd_makeconf";
31
my $exports_setup = "$TB/sbin/exports_setup";
32

33
my $dhcpd_conf = "/usr/local/etc/dhcpd.conf";
34 35 36
my $dhcpd_template = "/usr/local/etc/dhcpd.conf.template";
my $dhcpd_rc = "/usr/local/etc/rc.d/2.dhcpd.sh";

37
my $sudo = "/usr/local/bin/sudo -S";
38 39

#
40
# MFS to boot the nodes into initially
41
#
42
my $INITIAL_MFS = TB_OSID_FREEBSD_MFS();
43 44

#
45
# Initial event system state to put the nodes into
46
#
47
my $INITIAL_STATE = TBDB_NODESTATE_SHUTDOWN;
48 49 50 51 52 53

#
# Number of vnodes to create for each physical node
#
my $NUM_VNODES = 50;

54 55 56 57 58
#
# Default number of vnodes that can be hosted on any pnode
#
my $DEFAULT_PCVM_COUNT = 20;

59 60
# Turn off line buffering on output
$| = 1;
61

62 63 64 65
if (!TBAdmin()) {
    die "Sorry, only testbed administrators can run this script!\n";
}

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
#
# Handle command-line options
#
my $force_unconnected_interfaces = 0;
my %options = ();
if (! getopts("f", \%options)) {
    usage();
}

if ($options{f}) {
    $force_unconnected_interfaces = 1;
}

sub usage() {
    die "Usage: $0 [-f] <node_id> ...\n";
}

83 84 85 86 87 88
#
# We seem to have to do this so that ssh gets proper root permissions to read
# the key file. Argh.
#
$UID = $EUID = 0;

89
if (@ARGV < 1) {
90
    usage();
91 92
}

93 94 95
#
# The user has to be able to run sudo, so they can restart dhcpd.
#
96
if (system "$sudo /bin/pwd < /dev/null") {
97 98 99
    die "You must be able to sudo to root to use this script\n";
}

100 101 102 103 104 105 106 107 108 109 110
#
# Make sure that the dhcpd template file exists, and that the real version
# is writable
#
if (!-e $dhcpd_template) {
    die "In order to use this script, $dhcpd_template must exist!\n";
}
if (!-w $dhcpd_conf) {
    die "In order to use this script, you must be able to write $dhcpd_conf\n";
}

111 112 113 114 115 116 117 118 119 120
#
# Find out what op_mode the $INITIAL_MFS runs in
#
my $result = DBQueryFatal("select op_mode from os_info where " .
    " osid='$INITIAL_MFS'");
if ($result->numrows() != 1) {
    die "Unable to find OS information for $INITIAL_MFS\n";
}
my ($INITIAL_OPMODE) = $result->fetchrow();

121 122 123 124 125 126 127 128 129 130
#
# For vnodes - figure out the jail IP base
#
my $IPBASE;
if (TBDB_JAILIPBASE =~ /^(\d+).(\d+).(\d+).(\d+)/) {
    $IPBASE = "$1.$2";
} else {
    die "Problem with JAILIPBASE\n"
}

131 132 133 134 135
my @node_ids = @ARGV;

#
# Now, loop through the nodes given, and add each one
#
136
my (@succeeded_nodes, @succeeded_IPs);
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
NODE: foreach my $node_id (@node_ids) {
    my $query_result;

    #
    # Check to make sure said node does not already exist!
    #
    $query_result = DBQueryFatal("SELECT node_id FROM nodes WHERE " .
	"node_id='$node_id'");
    if ($query_result->num_rows()) {
	warn "Node $node_id failed: a node with that name already exists!\n";
	next NODE;
    }

    #
    # Grab information about the node from the new_nodes table
    #
153
    $query_result = DBQueryFatal("SELECT new_node_id, type, IP, temporary_IP " .
154
	"FROM new_nodes WHERE node_id='$node_id'");
155 156 157 158
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: No pending node with that name exists!\n";
	next NODE;
    }
159
    my ($new_node_id, $type, $IP, $tempIP) = $query_result->fetchrow();
160

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
    #
    # Make sure they've given it an IP address that looks valid
    #
    if ($IP !~ /^(\d+).(\d+).(\d+).(\d+)/) {
	warn "Node $node_id has an invalid IP address ($IP) - skipping\n";
	next NODE;
    }

    #
    # Make sure some other node doesn't already have this IP address on its
    # control interface
    #
    $query_result = DBQueryFatal("SELECT node_id FROM interfaces WHERE " .
	"IP='$IP' AND role='" . TBDB_IFACEROLE_CONTROL() . "'");
    if ($query_result->num_rows()) {
	my ($existing_node_id) = $query_result->fetchrow();
	warn "Node $node_id failed: $existing_node_id is already using IP " .
		"address $IP\n";
	next NODE;
    }

182 183 184 185
    #
    # Make sure that the new node is of a valid type, and grab a few other
    # things to fill in as initial values
    #
186
    $query_result = DBQueryFatal("SELECT control_iface FROM node_types " .
187 188 189 190 191
	"WHERE type='$type'");
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Type $type does not exist!\n";
	next NODE;
    }
192
    my ($control_iface) = $query_result->fetchrow();
193 194 195 196

    #
    # Grab the node's MACs from the new_interfaces table
    #
197
    $query_result = DBQueryFatal("SELECT card, MAC, interface_type, " .
198
	"switch_id, switch_card, switch_port, cable, len " .
199
	"FROM new_interfaces WHERE new_node_id='$new_node_id'");
200 201 202 203 204 205
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Must have at least one interface!\n";
	next NODE;
    }

    my %interfaces;
206
    while (my ($card, $MAC, $iface_type, $switch_id, $switch_card,
207
	    $switch_port, $cable, $len) = $query_result->fetchrow()) {
208 209 210 211 212 213
	#
	# Get some more information about this interface type
	#
	my $iface_query = DBQueryFatal("SELECT max_speed, full_duplex " .
	    "FROM interface_types WHERE type='$iface_type'");
	if (!$iface_query->num_rows()) {
214
	    warn "Node $node_id failed: Interface $card is of unknown type " .
215
	    	"$iface_type\n";
216 217 218
	    next NODE;
	}

219 220 221 222 223 224 225 226 227 228 229 230 231
	#
	# Do a sanity check - make sure that we have a switch recorded for all
	# experimental interfaces
	#
	unless ($force_unconnected_interfaces) {
	    my $iface = "eth$card";
	    if (($iface ne $control_iface) && !$switch_id) {
		warn "Node $node_id failed: Don't know which switch " .
		    "card $iface is connected to\n";
		next NODE;
	    }
	}

232 233 234 235 236
	my ($max_speed, $full_duplex) = $iface_query->fetchrow();

	#
	# Stash it away...
	#
237
	$interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex,
238
		$switch_id, $switch_card, $switch_port, $cable, $len];
239 240 241 242 243 244

    }

    #
    # Make up a priority (just used for sorting)
    #
245 246
    $node_id =~ /^(.*\D)(\d+)$/;
    my ($prefix,$nodenum) = ($1, $2);
247
    my $priority;
248 249
    if (defined $nodenum) {
	$priority = $nodenum;
250 251 252 253 254 255 256 257 258
    } else {
	$priority = 1;
    }

    #
    # Okay, time to actually add the node!
    #
    
    DBQueryFatal("INSERT INTO nodes SET node_id='$node_id', type='$type', " .
259
	"phys_nodeid='$node_id', role='testnode', priority=$priority, " .
260 261 262 263
    	"eventstate='$INITIAL_STATE', op_mode='$INITIAL_OPMODE', " .
	"def_boot_osid='$INITIAL_MFS', " .
	"state_timestamp=unix_timestamp(NOW()), " .
	"op_mode_timestamp=unix_timestamp(NOW())");
264

265 266 267
    DBQueryFatal("insert into node_hostkeys (node_id) ".
		 "values ('$node_id')");

268 269 270 271 272 273 274 275
    #
    # Add some vnodes
    #
    if (defined $nodenum) {
	my $vtype = $type;
	if (!($vtype =~ s/pc/pcvm/)) {
	    $vtype = "$vtype-vm";
	}
276 277 278 279
	DBQueryFatal("INSERT INTO node_auxtypes set node_id='$node_id', " .
	    "type='pcvm', count=$DEFAULT_PCVM_COUNT");
	DBQueryFatal("INSERT INTO node_auxtypes set node_id='$node_id', " .
	    "type='$vtype', count=$DEFAULT_PCVM_COUNT");
280 281 282 283 284 285 286 287 288 289
	for (my $i = 1; $i <= $NUM_VNODES; $i++) {
	    my $vpriority = 10000000 + ($nodenum * 1000) + $i;
	    my $nodename = $node_id;
	    if (!($nodename =~ s/pc/pcvm/)) {
		$nodename = "$nodename-vm";
	    }
	    $nodename .= "-$i";
	    my $jailip = "${IPBASE}.${nodenum}.${i}";

	    DBQueryFatal("INSERT INTO nodes SET node_id='$nodename', " .
290
		"type='$type', phys_nodeid='$node_id', role='virtnode', " .
291 292 293 294 295 296 297
		"priority='$vpriority', op_mode='PCVM', " .
		"eventstate='SHUTDOWN', " .
		"def_boot_osid='emulab-ops-FBSD-JAIL', " .
		"update_accounts=1, jailflag=1, jailip='$jailip'");
	}
    }

298
    while (my ($card, $aref) = each %interfaces) {
299
	my ($MAC, $iface_type, $speed, $duplex, $switch_id, $switch_card,
300
	    $switch_port, $cable, $len) = @$aref;
301
	my $iface = "eth$card";
302 303
	my $iface_IP = "";
	my $wire_type = "Node";
304
	my $iface_role = TBDB_IFACEROLE_EXPERIMENT();
305
	if ($iface eq $control_iface) {
306 307
	    $iface_IP = $IP;
	    $wire_type = "Control";
308
	    $iface_role = TBDB_IFACEROLE_CONTROL();
309 310 311 312
	}
	DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " .
	    "card=$card, port=1, mac='$MAC', IP='$iface_IP', " .
	    "interface_type='$iface_type', iface='$iface', " .
313
	    "current_speed='$speed', duplex=$duplex, role='$iface_role'");
314

315 316 317 318
	if (!$switch_id) {
	    print "No switch found for ${node_id}:$iface - skipping\n";
	    next;
	}
319 320 321 322 323 324 325 326

	my $cable_len = "";
	if ($cable) {
	    $cable_len .= ", cable=$cable";
	}
	if ($len) {
	    $cable_len .= ", len=$len";
	}
327 328 329
	DBQueryFatal("INSERT INTO wires SET type='$wire_type', " .
	    "node_id1='$node_id', card1=$card, port1=1, " .
	    "node_id2='$switch_id', card2='$switch_card', " .
330
	    "port2='$switch_port' $cable_len");
331 332 333
    }

    #
334 335
    # Put it into hwdown for now - I would put them in reloading, but I'm
    # afriad the reload_daemon might do the wrong thing to them
336 337 338 339 340 341
    #
    system "$nalloc emulab-ops hwdown $node_id";

    #
    # Remove the node from the new_ tables
    #
342
    DBQueryFatal("DELETE FROM new_nodes WHERE new_node_id=$new_node_id");
343
    DBQueryFatal("DELETE FROM new_interfaces WHERE new_node_id=$new_node_id");
344

345
    print "$node_id succesfully added!\n";
346 347

    push @succeeded_nodes, $node_id;
348
    push @succeeded_IPs, $tempIP;
349 350 351 352 353 354 355 356 357 358 359 360
}

#
# No point in restarting dhcpd, etc. if there are no nodes that succeeded
#
if (!@succeeded_nodes) {
    die "No nodes succeeded, exiting early\n";
}

#
# Re-generate dhcpd.conf
#
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
print "Re-generating dhcpd.conf\n";
open(CONF,"$dhcpd_makeconf $dhcpd_template|") or die "Unable to fork: $!\n";
my @conf = <CONF>;
close CONF or die "Error reading from dhcpd_makeconf: $!\n";

open(CONF,">$dhcpd_conf") or die "Unable to open $dhcpd_conf for writing\n";
print CONF @conf;
close CONF;

print "Restarting dhcpd: $sudo $dhcpd_rc stop\n";
my $sudo_rv = system "$sudo $dhcpd_rc stop";
if ($sudo_rv) {
    die "Error stopping dhcpd - return value was $sudo_rv\n";
}
sleep 2;
print "Restarting dhcpd: $sudo $dhcpd_rc start\n";
$sudo_rv = system "$sudo $dhcpd_rc start";
if ($sudo_rv) {
    die "Error starting dhcpd - return value was $sudo_rv\n";
}
381

382 383 384 385 386
print "Setting up nameserver\n";
my $named_rv = system "$named_setup";
if ($named_rv) {
    die "Error running named_setup - return value was $named_rv\n";
}
387

388 389 390 391 392 393 394 395 396
#
# Before we boot nodes into the MFS, we have to make sure they can mount
# NFS filesystems
#
print "Running exports_setup\n";
my $exports_rv = system "$exports_setup";
if ($exports_rv) {
    warn "WARNING - exports_setup returned $exports_rv";
}
397

398 399 400 401 402 403 404
#
# Start rebooting nodes
#
print "Rebooting nodes...\n";
foreach my $IP (@succeeded_IPs) {
    print "Rebooting $IP\n";
    system "$newnode_reboot $IP\n";
405 406
}

407 408 409 410 411
print "\n\n";
print "Finished - when you are satisifed that the nodes are working\n";
print "correctly, use nfree on boss to free them from the emulab-ops/hwdown\n";
print "experiment.\n";

412
#
413 414 415
# TODO -
#    disable interfaces
#    console setup
416
#