newnode.in 10.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
# Copyright (c) 2003 University of Utah and the Flux Group.
# All rights reserved.
#

#
# newnode - a script for moving nodes from the new_ tables into production.
#

13
use lib '@prefix@/lib';
14 15

use libdb;
Robert Ricci's avatar
Robert Ricci committed
16
use English;
17
use Getopt::Std;
18 19 20

use strict;

21
my $TB = "@prefix@";
22 23

my $switchmac = "$TB/sbin/switchmac";
24 25 26 27
my $os_load = "$TB/bin/os_load";
my $os_select = "$TB/bin/os_select";
my $newnode_reboot = "$TB/sbin/newnode_reboot";
my $named_setup = "$TB/sbin/named_setup";
28 29 30
my $nalloc = "$TB/bin/nalloc";
my $nfree = "$TB/bin/nfree";
my $dhcpd_makeconf = "$TB/sbin/dhcpd_makeconf";
31
my $exports_setup = "$TB/sbin/exports_setup";
32

33
my $dhcpd_conf = "/usr/local/etc/dhcpd.conf";
34 35 36
my $dhcpd_template = "/usr/local/etc/dhcpd.conf.template";
my $dhcpd_rc = "/usr/local/etc/rc.d/2.dhcpd.sh";

37
my $sudo = "/usr/local/bin/sudo -S";
38 39

#
40
# MFS to boot the nodes into initially
41
#
42
my $INITIAL_MFS = TB_OSID_FREEBSD_MFS();
43 44

#
45
# Initial event system state to put the nodes into
46
#
47
my $INITIAL_STATE = TBDB_NODESTATE_SHUTDOWN;
48 49 50 51 52 53 54 55

#
# Number of vnodes to create for each physical node
#
my $NUM_VNODES = 50;

# Turn off line buffering on output
$| = 1;
56

57 58 59 60
if (!TBAdmin()) {
    die "Sorry, only testbed administrators can run this script!\n";
}

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
#
# Handle command-line options
#
my $force_unconnected_interfaces = 0;
my %options = ();
if (! getopts("f", \%options)) {
    usage();
}

if ($options{f}) {
    $force_unconnected_interfaces = 1;
}

sub usage() {
    die "Usage: $0 [-f] <node_id> ...\n";
}

78 79 80 81 82 83
#
# We seem to have to do this so that ssh gets proper root permissions to read
# the key file. Argh.
#
$UID = $EUID = 0;

84
if (@ARGV < 1) {
85
    usage();
86 87
}

88 89 90
#
# The user has to be able to run sudo, so they can restart dhcpd.
#
91
if (system "$sudo /bin/pwd < /dev/null") {
92 93 94
    die "You must be able to sudo to root to use this script\n";
}

95 96 97 98 99 100 101 102 103 104 105
#
# Make sure that the dhcpd template file exists, and that the real version
# is writable
#
if (!-e $dhcpd_template) {
    die "In order to use this script, $dhcpd_template must exist!\n";
}
if (!-w $dhcpd_conf) {
    die "In order to use this script, you must be able to write $dhcpd_conf\n";
}

106 107 108 109 110 111 112 113 114 115
#
# Find out what op_mode the $INITIAL_MFS runs in
#
my $result = DBQueryFatal("select op_mode from os_info where " .
    " osid='$INITIAL_MFS'");
if ($result->numrows() != 1) {
    die "Unable to find OS information for $INITIAL_MFS\n";
}
my ($INITIAL_OPMODE) = $result->fetchrow();

116 117 118 119 120 121 122 123 124 125
#
# For vnodes - figure out the jail IP base
#
my $IPBASE;
if (TBDB_JAILIPBASE =~ /^(\d+).(\d+).(\d+).(\d+)/) {
    $IPBASE = "$1.$2";
} else {
    die "Problem with JAILIPBASE\n"
}

126 127 128 129 130
my @node_ids = @ARGV;

#
# Now, loop through the nodes given, and add each one
#
131
my (@succeeded_nodes, @succeeded_IPs);
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
NODE: foreach my $node_id (@node_ids) {
    my $query_result;

    #
    # Check to make sure said node does not already exist!
    #
    $query_result = DBQueryFatal("SELECT node_id FROM nodes WHERE " .
	"node_id='$node_id'");
    if ($query_result->num_rows()) {
	warn "Node $node_id failed: a node with that name already exists!\n";
	next NODE;
    }

    #
    # Grab information about the node from the new_nodes table
    #
148
    $query_result = DBQueryFatal("SELECT new_node_id, type, IP, temporary_IP " .
149
	"FROM new_nodes WHERE node_id='$node_id'");
150 151 152 153
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: No pending node with that name exists!\n";
	next NODE;
    }
154
    my ($new_node_id, $type, $IP, $tempIP) = $query_result->fetchrow();
155

156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
    #
    # Make sure they've given it an IP address that looks valid
    #
    if ($IP !~ /^(\d+).(\d+).(\d+).(\d+)/) {
	warn "Node $node_id has an invalid IP address ($IP) - skipping\n";
	next NODE;
    }

    #
    # Make sure some other node doesn't already have this IP address on its
    # control interface
    #
    $query_result = DBQueryFatal("SELECT node_id FROM interfaces WHERE " .
	"IP='$IP' AND role='" . TBDB_IFACEROLE_CONTROL() . "'");
    if ($query_result->num_rows()) {
	my ($existing_node_id) = $query_result->fetchrow();
	warn "Node $node_id failed: $existing_node_id is already using IP " .
		"address $IP\n";
	next NODE;
    }

177 178 179 180
    #
    # Make sure that the new node is of a valid type, and grab a few other
    # things to fill in as initial values
    #
181
    $query_result = DBQueryFatal("SELECT control_iface FROM node_types " .
182 183 184 185 186
	"WHERE type='$type'");
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Type $type does not exist!\n";
	next NODE;
    }
187
    my ($control_iface) = $query_result->fetchrow();
188 189 190 191

    #
    # Grab the node's MACs from the new_interfaces table
    #
192
    $query_result = DBQueryFatal("SELECT card, MAC, interface_type, " .
193
	"switch_id, switch_card, switch_port, cable, len " .
194
	"FROM new_interfaces WHERE new_node_id='$new_node_id'");
195 196 197 198 199 200
    if (!$query_result->num_rows()) {
	warn "Node $node_id failed: Must have at least one interface!\n";
	next NODE;
    }

    my %interfaces;
201
    while (my ($card, $MAC, $iface_type, $switch_id, $switch_card,
202
	    $switch_port, $cable, $len) = $query_result->fetchrow()) {
203 204 205 206 207 208
	#
	# Get some more information about this interface type
	#
	my $iface_query = DBQueryFatal("SELECT max_speed, full_duplex " .
	    "FROM interface_types WHERE type='$iface_type'");
	if (!$iface_query->num_rows()) {
209
	    warn "Node $node_id failed: Interface $card is of unknown type " .
210
	    	"$iface_type\n";
211 212 213
	    next NODE;
	}

214 215 216 217 218 219 220 221 222 223 224 225 226
	#
	# Do a sanity check - make sure that we have a switch recorded for all
	# experimental interfaces
	#
	unless ($force_unconnected_interfaces) {
	    my $iface = "eth$card";
	    if (($iface ne $control_iface) && !$switch_id) {
		warn "Node $node_id failed: Don't know which switch " .
		    "card $iface is connected to\n";
		next NODE;
	    }
	}

227 228 229 230 231
	my ($max_speed, $full_duplex) = $iface_query->fetchrow();

	#
	# Stash it away...
	#
232
	$interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex,
233
		$switch_id, $switch_card, $switch_port, $cable, $len];
234 235 236 237 238 239

    }

    #
    # Make up a priority (just used for sorting)
    #
240 241
    $node_id =~ /^(.*\D)(\d+)$/;
    my ($prefix,$nodenum) = ($1, $2);
242
    my $priority;
243 244
    if (defined $nodenum) {
	$priority = $nodenum;
245 246 247 248 249 250 251 252 253
    } else {
	$priority = 1;
    }

    #
    # Okay, time to actually add the node!
    #
    
    DBQueryFatal("INSERT INTO nodes SET node_id='$node_id', type='$type', " .
254
	"phys_nodeid='$node_id', role='testnode', priority=$priority, " .
255 256 257 258
    	"eventstate='$INITIAL_STATE', op_mode='$INITIAL_OPMODE', " .
	"def_boot_osid='$INITIAL_MFS', " .
	"state_timestamp=unix_timestamp(NOW()), " .
	"op_mode_timestamp=unix_timestamp(NOW())");
259

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
    #
    # Add some vnodes
    #
    if (defined $nodenum) {
	my $vtype = $type;
	if (!($vtype =~ s/pc/pcvm/)) {
	    $vtype = "$vtype-vm";
	}
	for (my $i = 1; $i <= $NUM_VNODES; $i++) {
	    my $vpriority = 10000000 + ($nodenum * 1000) + $i;
	    my $nodename = $node_id;
	    if (!($nodename =~ s/pc/pcvm/)) {
		$nodename = "$nodename-vm";
	    }
	    $nodename .= "-$i";
	    my $jailip = "${IPBASE}.${nodenum}.${i}";

	    DBQueryFatal("INSERT INTO nodes SET node_id='$nodename', " .
		"type='$vtype', phys_nodeid='$node_id', role='virtnode', " .
		"priority='$vpriority', op_mode='PCVM', " .
		"eventstate='SHUTDOWN', " .
		"def_boot_osid='emulab-ops-FBSD-JAIL', " .
		"update_accounts=1, jailflag=1, jailip='$jailip'");
	}
    }

286
    while (my ($card, $aref) = each %interfaces) {
287
	my ($MAC, $iface_type, $speed, $duplex, $switch_id, $switch_card,
288
	    $switch_port, $cable, $len) = @$aref;
289
	my $iface = "eth$card";
290 291
	my $iface_IP = "";
	my $wire_type = "Node";
292
	my $iface_role = TBDB_IFACEROLE_EXPERIMENT();
293
	if ($iface eq $control_iface) {
294 295
	    $iface_IP = $IP;
	    $wire_type = "Control";
296
	    $iface_role = TBDB_IFACEROLE_CONTROL();
297 298 299 300
	}
	DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " .
	    "card=$card, port=1, mac='$MAC', IP='$iface_IP', " .
	    "interface_type='$iface_type', iface='$iface', " .
301
	    "current_speed='$speed', duplex=$duplex, role='$iface_role'");
302

303 304 305 306
	if (!$switch_id) {
	    print "No switch found for ${node_id}:$iface - skipping\n";
	    next;
	}
307 308 309 310 311 312 313 314

	my $cable_len = "";
	if ($cable) {
	    $cable_len .= ", cable=$cable";
	}
	if ($len) {
	    $cable_len .= ", len=$len";
	}
315 316 317
	DBQueryFatal("INSERT INTO wires SET type='$wire_type', " .
	    "node_id1='$node_id', card1=$card, port1=1, " .
	    "node_id2='$switch_id', card2='$switch_card', " .
318
	    "port2='$switch_port' $cable_len");
319 320 321
    }

    #
322 323
    # Put it into hwdown for now - I would put them in reloading, but I'm
    # afriad the reload_daemon might do the wrong thing to them
324 325 326 327 328 329
    #
    system "$nalloc emulab-ops hwdown $node_id";

    #
    # Remove the node from the new_ tables
    #
330
    DBQueryFatal("DELETE FROM new_nodes WHERE new_node_id=$new_node_id");
331
    DBQueryFatal("DELETE FROM new_interfaces WHERE new_node_id=$new_node_id");
332

333
    print "$node_id succesfully added!\n";
334 335

    push @succeeded_nodes, $node_id;
336
    push @succeeded_IPs, $tempIP;
337 338 339 340 341 342 343 344 345 346 347 348
}

#
# No point in restarting dhcpd, etc. if there are no nodes that succeeded
#
if (!@succeeded_nodes) {
    die "No nodes succeeded, exiting early\n";
}

#
# Re-generate dhcpd.conf
#
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
print "Re-generating dhcpd.conf\n";
open(CONF,"$dhcpd_makeconf $dhcpd_template|") or die "Unable to fork: $!\n";
my @conf = <CONF>;
close CONF or die "Error reading from dhcpd_makeconf: $!\n";

open(CONF,">$dhcpd_conf") or die "Unable to open $dhcpd_conf for writing\n";
print CONF @conf;
close CONF;

print "Restarting dhcpd: $sudo $dhcpd_rc stop\n";
my $sudo_rv = system "$sudo $dhcpd_rc stop";
if ($sudo_rv) {
    die "Error stopping dhcpd - return value was $sudo_rv\n";
}
sleep 2;
print "Restarting dhcpd: $sudo $dhcpd_rc start\n";
$sudo_rv = system "$sudo $dhcpd_rc start";
if ($sudo_rv) {
    die "Error starting dhcpd - return value was $sudo_rv\n";
}
369

370 371 372 373 374
print "Setting up nameserver\n";
my $named_rv = system "$named_setup";
if ($named_rv) {
    die "Error running named_setup - return value was $named_rv\n";
}
375

376 377 378 379 380 381 382 383 384
#
# Before we boot nodes into the MFS, we have to make sure they can mount
# NFS filesystems
#
print "Running exports_setup\n";
my $exports_rv = system "$exports_setup";
if ($exports_rv) {
    warn "WARNING - exports_setup returned $exports_rv";
}
385

386 387 388 389 390 391 392
#
# Start rebooting nodes
#
print "Rebooting nodes...\n";
foreach my $IP (@succeeded_IPs) {
    print "Rebooting $IP\n";
    system "$newnode_reboot $IP\n";
393 394
}

395 396 397 398 399
print "\n\n";
print "Finished - when you are satisifed that the nodes are working\n";
print "correctly, use nfree on boss to free them from the emulab-ops/hwdown\n";
print "experiment.\n";

400
#
401 402 403
# TODO -
#    disable interfaces
#    console setup
404
#