elabinelab.in 50 KB
Newer Older
1
#!/usr/bin/perl -w
2
#
Leigh Stoller's avatar
Leigh Stoller committed
3
# Copyright (c) 2004-2019 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
23 24 25
#
# TODO: ntpinfo table.
#       Current source directory? From where?
Mike Hibler's avatar
Mike Hibler committed
26
#       need to remove management (IPMI) interfaces from interfaces/wires.
27 28 29 30
#
use English;
use Getopt::Std;

31 32 33 34
# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libdb;
use libtestbed;
35
use libtblog;
36
use Experiment;
37
use User;
38
use Lan;
39

40 41 42 43 44
#
# Do things necessary for setting up inner elab experiment. 
#
sub usage()
{
45
    print STDOUT "Usage: elabinelab [-d] [-g] [-u] pid eid\n";
46
    print STDOUT "       elabinelab [-d] [-k | -f] pid eid\n";
47
    print STDOUT "       elabinelab [-d] -r pid eid [node ...]\n";
48 49 50
 
    exit(-1);
}
51
my $optlist  = "dgkfurP";
52
my $debug    = 1;
53
my $verbose  = 0;
54
my $killmode = 0;
55
my $fwboot   = 0;
56
my $dbgooonly= 0;
57 58
my $update   = 0;
my $remove   = 0;
59

60 61 62 63 64 65 66 67
#
# XXX experimental speed hacks.
#     $inparallel    reboots all server in parallel (rather than serially)
#		     after setup
#     $restartnodes  uses a new bootinfo RESTART command to quickly move
#		     inner nodes from control of outer boss to inner boss
#		     avoiding all node reboots
#
68
my $inparallel = 1;
69
my $restartnodes = 0;
70

71 72
sub DumpDBGoo();

73 74 75 76 77 78
#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
my $CONTROL	= "@USERNODE@";
Russ Fish's avatar
Russ Fish committed
79
my $DBNAME      = "@TBDBNAME@";
80 81
my $TBOPSPID    = TBOPSPID();
my $SSH		= "$TB/bin/sshtb";
82
my $SCP		= "/usr/bin/scp";
83
my $nodereboot  = "$TB/bin/node_reboot";
84
my $noderestart	= "$TB/sbin/bootinfosend -R";
85
my $makeconf    = "$TB/sbin/dhcpd_makeconf";
86
my $nodewait    = "$TB/sbin/node_statewait";
87
my $snmpit      = "$TB/bin/snmpit";
88
my $osselect	= "$TB/bin/os_select";
89
my $dumpuser    = "$TB/sbin/dumpuser";
90 91 92

# Protos
sub TearDownEmulab();
93 94
sub RemoveNodes();
sub UpdateEmulab();
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
    die("*** $0:\n".
	"    Must be root! Maybe its a development version?\n");
}

# Locals
my $SAVEUID     = $UID;
my $workdir;
116
my $expdir;
117 118 119
my %noderoles	= ();
my $opsnode;
my $bossnode;
Mike Hibler's avatar
Mike Hibler committed
120 121
my $fsnode;
my $routernode;
122
my @expnodes    = ();
123
my $query_result;
124 125
my $inner_experiment;
my $inner_nsfile;
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"g"})) {
    $dbgooonly = 1;
}
if (defined($options{"d"})) {
    $debug = 1;
}
141 142 143
if (defined($options{"k"})) {
    $killmode = 1;
}
144 145 146
if (defined($options{"f"})) {
    $fwboot = 1;
}
147 148 149 150 151 152
if (defined($options{"u"})) {
    $update = 1;
}
if (defined($options{"r"})) {
    $remove = 1;
}
153 154 155
if (defined($options{"P"})) {
    $inparallel = 1;
}
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
if (! @ARGV) {
    usage();
}
my ($pid,$eid) = @ARGV;

#
# Untaint the arguments.
#
if ($pid =~ /^([-\w]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\w]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}
176 177 178 179 180
my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
    die("*** $0:\n".
	"    Could not map $pid/$eid to its object!\n");
}
181
$workdir = TBExptWorkDir($pid, $eid);
182
$expdir = PROJROOT() . "/$pid/exp/$eid";
183

184 185 186 187 188
# Build Logfile names
my $opslogfile  = "$workdir/opsnode.log";
my $fslogfile   = "$workdir/fsnode.log";
my $bosslogfile = "$workdir/bossnode.log";

189 190 191
#
# Verify user and get his DB uid.
#
192 193 194
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    tbdie("You ($UID) do not exist!");
195
}
196 197 198
my $user_uid   = $this_user->uid();
my $user_name  = $this_user->name();
my $user_email = $this_user->email();
199
my $user_dir   = $this_user->HomeDir();
200

Leigh Stoller's avatar
Leigh Stoller committed
201
TBDebugTimeStampsOn();
202

203 204 205 206
#
# Get elabinelab status to make sure, and to see if we need to fire off
# an experiment inside once its setup.
#
207 208 209 210
my $elabinelab           = $experiment->elabinelab();
my $elabinelab_eid       = $experiment->elabinelab_eid();
my $elabinelab_nosetup   = $experiment->elabinelab_nosetup();
my $elabinelab_singlenet = $experiment->elabinelab_singlenet();
211
my $elabinelab_attributes= $experiment->GetElabInElabAttrs();
212

213 214 215
exit(0)
    if (!$elabinelab);

216 217 218 219
#
# See if the experiment is firewalled
#
my $firewall;
220 221
my $fwtype;
my $firewalled = TBExptFirewall($pid, $eid, \$firewall, undef, undef, \$fwtype);
222

223 224 225 226 227 228 229 230 231
#
# Presetup; turn off firewall.
#
if ($fwboot) {
    exit(0)
	if (!$firewalled);
    
    print "Turning off firewall rules on $firewall\n";
    $UID = 0;
232 233 234 235 236
    if ($fwtype =~ /^iptables/) {
        system("$SSH -host $firewall iptables -I FORWARD 1 -j ACCEPT");
    } else {
        system("$SSH -host $firewall ipfw add 1 allow all from any to any");
    }
237 238 239 240 241 242 243
    if ($?) {
	die("*** $0:\n".
	    "    Error turning off firewall rules ($firewall)!\n");
    }
    exit(0);
}

244 245 246 247 248
#
# If we are going to start an inner experiment, grab the stuff we need
# from the DB and save it. 
#
if (defined($elabinelab_eid)) {
249
    $inner_experiment = Experiment->Lookup($pid, $elabinelab_eid);
250 251
    die("*** $0:\n".
	"    No such experiment in DB for $pid/$elabinelab_eid\n")
252
	if (!defined($inner_experiment));
253

254 255 256
    $inner_experiment->GetNSFile(\$inner_nsfile) == 0 or
	die("*** $0:\n".
	    "    Could not get NS file for $inner_experiment\n");
257 258
    
    die("*** $0:\n".
259 260
	"    No nsfile in DB for $inner_experiment")
	if (!defined($inner_nsfile) || $inner_nsfile eq "");
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
}

#
# Get the role for each node.
#
$query_result =
    DBQueryFatal("select r.node_id,r.inner_elab_role from reserved as r ".
		 "where r.pid='$pid' and r.eid='$eid'");
while (my ($node_id,$role) = $query_result->fetchrow_array()) {
    # Like, the firewall node.
    next
	if (!defined($role));
	
    $noderoles{$node_id} = $role;
    $bossnode = $node_id
276
	if ($role =~ /^boss/);
Mike Hibler's avatar
Mike Hibler committed
277 278
    $routernode = $node_id
	if ($role eq 'router');
279
    $opsnode = $node_id
Mike Hibler's avatar
Mike Hibler committed
280 281 282
	if ($role eq 'ops' || $role eq 'ops+fs');
    $fsnode = $node_id
	if ($role eq 'fs');
283 284 285 286 287 288 289 290 291 292
    push(@expnodes, $node_id)
	if ($role eq 'node');
}

#
# Tear down an inner emulab.
# 
if ($killmode) {
    exit(TearDownEmulab());
}
293 294 295 296 297 298
elsif ($remove) {
    exit(RemoveNodes());
}
elsif ($update) {
    exit(UpdateEmulab());
}
299 300 301 302 303 304

#
# Get elabinelab info. If this is a container for an actual experiment,
# then need to fire off the experiment once the inner emulab is ready to
# go.
# 
Leigh Stoller's avatar
Leigh Stoller committed
305
TBDebugTimeStamp("Dumping DB state");
306 307 308 309 310
DumpDBGoo();
exit(0)
    if ($dbgooonly);

#
311
# For SSH and SCP below
312 313
#
$UID = 0;
Leigh Stoller's avatar
Leigh Stoller committed
314

315
#
316 317
# The firewall should be off at this point; called from os_setup with -f.
# 
318

Leigh Stoller's avatar
Leigh Stoller committed
319 320 321
#
# This is temporary. I think I will switch this over to grabbing the latest
# version from the web server.
322
#
323
# XXX ugh, copy over a newer mkextrafs.pl as well (one that supports -2).
324
# XXX ughII, we only copy over a FreeBSD version, this will break a Linux boss.
325
#
326 327 328 329 330 331 332 333 334 335 336 337 338 339
# XXX ughIII, copy over to /var/run instead of /tmp since we run prepare
# from rc.mkelab and that cleans out /tmp. I thought this would work since
# /tmp/rc.mkelab would be open when /tmp was cleared and it would not affect
# execution of the script, but I actually had it fail once with:
#  ...
#  Building and Installing Software                  
#  | Building (please be patient)                    /tmp/rc.mkelab: Command not found.
#
# right out of the middle of running the boss-install! This was on a VM with
# only 600MB of memory, so it is possible that due to memory pressure perl
# had not read the entire script? The build phase it started never actually
# happened. There was also no "phase stack dump" after that message, so
# everything got blown out of the water suddenly and decisively.
#
340
my $mkelab = "$TB/etc/rc.mkelab";
341 342
if (-e "$expdir/rc.mkelab") {
    $mkelab = "$expdir/rc.mkelab";
343
}
344 345 346
my $mkextrafs = "";
if (-e "$TB/etc/mkextrafs.pl") {
    $mkextrafs = "$TB/etc/mkextrafs.pl";
347 348 349
    if (-e "$expdir/mkextrafs.pl") {
	$mkextrafs = "$expdir/mkextrafs.pl";
    }
350
}
351 352 353
print "Copying $mkelab $mkextrafs to ${bossnode}";
print "/${opsnode}"
    if (defined($opsnode));
354 355 356
print "/${fsnode}"
    if (defined($fsnode));
print "\n";
357 358
system("scp $mkelab $mkextrafs ${bossnode}:/var/run/");
system("scp $mkelab $mkextrafs ${opsnode}:/var/run/")
359
    if (defined($opsnode));
360
system("scp $mkelab $mkextrafs ${fsnode}:/var/run/")
361
    if (defined($fsnode));
362
system("scp $user_dir/.ssl/emulab.pem ${bossnode}:/var/run/");
363

364 365 366
if (defined($fsnode)) {
    TBDebugTimeStamp("Setting up fsnode");
    print "Setting up fsnode on $fsnode\n";
367
    system("$SSH -host $fsnode /var/run/rc.mkelab -s -d > $fslogfile 2>&1");
368 369 370 371 372 373 374
    if ($?) {
	$UID = $SAVEUID;
	SENDMAIL("$user_name <$user_email>",
		 "ElabInElab Failure: $pid/$eid",
		 "Error building the fs node ($fsnode)",
		 $TBOPS,
		 "Cc: $TBOPS",
375
		 ($fslogfile));
376 377 378 379 380
	print STDERR "*** $0:\n".
	    "    Error building the fsnode ($fsnode)!\n";
	exit(($debug ? 0 : -1));
    }
}
381 382 383
if (defined($opsnode)) {
    TBDebugTimeStamp("Setting up opsnode");
    print "Setting up opsnode on $opsnode\n";
384
    system("$SSH -host $opsnode /var/run/rc.mkelab -s -d > $opslogfile 2>&1");
385 386 387 388 389 390 391 392 393 394 395 396
    if ($?) {
	$UID = $SAVEUID;
	SENDMAIL("$user_name <$user_email>",
		 "ElabInElab Failure: $pid/$eid",
		 "Error building the ops node ($opsnode)",
		 $TBOPS,
		 "Cc: $TBOPS",
		 ($opslogfile));
	print STDERR "*** $0:\n".
	             "    Error building the opsnode ($opsnode)!\n";
	exit(($debug ? 0 : -1));
    }
397
}
Leigh Stoller's avatar
Leigh Stoller committed
398
TBDebugTimeStamp("Setting up bossnode");
399
print "Setting up bossnode on $bossnode\n";
400
system("$SSH -host $bossnode /var/run/rc.mkelab -s -d > $bosslogfile 2>&1");
401 402 403 404
if ($?) {
    $UID = $SAVEUID;
    SENDMAIL("$user_name <$user_email>",
	     "ElabInElab Failure: $pid/$eid",
405
	     "Error building the boss node ($bossnode)",
406 407
	     $TBOPS,
	     "Cc: $TBOPS",
408
	     ($bosslogfile));
409 410 411 412 413
    print STDERR "*** $0:\n".
	         "    Error building the bossnode ($bossnode)!\n";
    exit(($debug ? 0 : -1));
}

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
if ($verbose) {
    # Send these log files off now so that we can look at them.
    if (defined($fsnode)) {
	SENDMAIL("$user_name <$user_email>",
		 "ElabInElab Setup Log: $pid/$eid",
		 "Logs for building fs/ops/boss ($fsnode/$opsnode/$bossnode)",
		 $TBOPS,
		 "Cc: $TBOPS",
		 ($fslogfile, $opslogfile, $bosslogfile));
    }
    else {
	SENDMAIL("$user_name <$user_email>",
		 "ElabInElab Setup Log: $pid/$eid",
		 "Logs for building ops/boss ($opsnode/$bossnode)",
		 $TBOPS,
		 "Cc: $TBOPS",
		 ($opslogfile, $bosslogfile));
    }
432
}
433
$UID  = $SAVEUID;
434 435

# Run as real user for the next few scripts, which are setuid.
436
$EUID = $UID;
437

438
goto skipsetup
439
    if ($elabinelab_nosetup);
440

441
#
442 443 444 445 446
# Restart DHCPD, but first mark the nodes as being ready to boot inside
# the inner emulab, so that dhcpd_makeconf knows what nodes to change
# the entries for.
#
DBQueryFatal("update reserved set inner_elab_boot=1 ".
447
	     "where pid='$pid' and eid='$eid'");
448

449 450 451 452 453 454 455
print "Regenerating DHCPD config file and restarting daemon.\n";
system("$makeconf -i -r");
if ($?) {
    die("*** $0:\n".
	"    Failed to reconfig/restart DHCPD.\n");
}

456
if ($inparallel) {
457 458 459
    my $nodes = "$bossnode";
    $nodes .= " $opsnode"
	if (defined($opsnode));
460 461 462 463 464
    $nodes .= " $fsnode"
	if (defined($fsnode));
    print "Rebooting servers ($nodes).\n";
    TBDebugTimeStamp("Rebooting servers");
    system("$nodereboot -w $nodes");
465
    if ($?) {
466 467 468
	print STDERR "*** $0:\n".
	    "    Error rebooting the servers ($nodes)!\n";
	exit(($debug ? 0 : -1));
469 470 471 472 473 474 475 476
    }
} else {
    if (defined($fsnode)) {
	# Reboot fs and wait for it to come back.
	print "Rebooting fsnode ($fsnode).\n";
	TBDebugTimeStamp("Rebooting fsnode");
	system("$nodereboot -w $fsnode");
	if ($?) {
477 478 479
	    print STDERR "*** $0:\n".
		"    Error rebooting the fsnode ($fsnode)!\n";
	    exit(($debug ? 0 : -1));
480 481
	}
    }
482 483 484 485 486 487
    if (defined($opsnode)) {
	# Reboot ops and wait for it to come back.
	print "Rebooting opsnode ($opsnode).\n";
	TBDebugTimeStamp("Rebooting opsnode");
	system("$nodereboot -w $opsnode");
	if ($?) {
488 489 490
	    print STDERR "*** $0:\n".
		"    Error rebooting the opsnode ($opsnode)!\n";
	    exit(($debug ? 0 : -1));
491
	}
492 493 494 495 496 497
    }
    # Reboot boss and wait for it to come back.
    print "Rebooting bossnode ($bossnode).\n";
    TBDebugTimeStamp("Rebooting bossnode");
    system("$nodereboot -w $bossnode");
    if ($?) {
498 499 500
	print STDERR "*** $0:\n".
	    "    Error rebooting the bossnode ($bossnode)!\n";
	exit(($debug ? 0 : -1));
501
    }
502
}
503
$EUID = 0;
Leigh Stoller's avatar
Leigh Stoller committed
504

505 506 507
# Reboot the experimental nodes. They will come up inside the inner elab.
# DO NOT WAIT! They are not going to report ISUP from this point on. 
if (@expnodes) {
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
    #
    # First we try the magic pxeboot restart.
    # The nodes should still be in PXEWAIT, so we send them a restart
    # to make them re-DHCP.  This should get them quickly reparented to
    # the inner boss.
    #
    # If this doesn't work, we fall back on rebooting the nodes.
    #
    if ($restartnodes) {
	TBDebugTimeStamp("Redirecting experimental nodes to inner boss");
	my $stat = 0;
	# Run as real user again.
	$EUID = $UID;
	foreach my $node (@expnodes) {
	    $stat = system("$noderestart $node");
	    last if ($stat);
	}
	$EUID = 0;
	if ($stat) {
	    tbwarn("Node restart failed ($stat), falling back to reboot.");
	    goto rebootnodes;
	}

	#
	# Ssh into inner boss and use a utility script to determine
	# when the nodes have reported in and are in PXEWAIT (part of the
	# inner elab). Note the short timeout, since this operation should
	# be virtually instantaneous.
	#
	print "Waiting for nodes to restart and join the inner emulab.\n";
	TBDebugTimeStamp("Waiting for inner nodes to restart");
	$UID  = 0;
	$stat = system("$SSH -host $bossnode ".
		       "/usr/testbed/sbin/node_statewait -t 15 -a");
	$UID  = $SAVEUID;
	if ($stat) {
	    tbwarn("Error ($stat) waiting for nodes to restart, falling back to reboot.");
	    goto rebootnodes;
	}

	goto restartworked;
    }

rebootnodes:
552
    print "Rebooting inner experimental nodes.\n";
Leigh Stoller's avatar
Leigh Stoller committed
553
    TBDebugTimeStamp("Rebooting experimental nodes");
554 555
    # Run as real user again.
    $EUID = $UID;
556
    system("$nodereboot -b @expnodes");
557 558 559 560
    if ($?) {
	die("*** $0:\n".
	    "    Error rebooting the expnodes (@expnodes)!\n");
    }
561
    $EUID = 0;
562 563 564 565 566

    #
    # Instead, we ssh into the node and use a utility script to determine
    # when the nodes have rebooted and are in PXEWAIT (part of the inner elab).
    #
567
    # Run as real root for ssh.
568 569 570 571 572 573 574 575 576 577 578
    $UID  = 0;

    print "Waiting for nodes to reboot and join the inner emulab.\n";
    TBDebugTimeStamp("Waiting for inner nodes to reboot");
    system("$SSH -host $bossnode /usr/testbed/sbin/node_statewait -t 180 -a");
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Error waiting for inner nodes to join!\n";
	exit(($debug ? 0 : -1));
    }
    $UID  = $SAVEUID;
579

580
restartworked:
581 582 583 584 585 586 587 588
    #
    # To avoid confusion later (with swapmod, which wants them to be ISUP),
    # and so the web interface does not show the nodes as down, set the 
    # state to ISUP.
    #
    foreach my $node (@expnodes) {
	TBSetNodeEventState($node, TBDB_NODESTATE_ISUP());
    }
589
}
590 591

#
592 593 594 595 596
# Fire off inner elab experiment.
# 
if (defined($elabinelab_eid)) {
    # Formatted to make batchexp happy.
    my $nsfilename = "/tmp/$pid-$elabinelab_eid-$$.nsfile";
597
    
598 599 600 601 602 603
    #
    # Write NS file to temp file so we can send it over.
    #
    open(NS, "> /tmp/$$.ns")
	or die("*** $0:\n".
	       "    Could not write ns code to tmp file!\n");
604
    print NS $inner_nsfile;
605 606 607 608 609 610 611
    print NS "\n";
    close(NS);

    #
    # Copy the file over.
    #
    $UID = 0;
612
    print "Sending NS file to inner bossnode ($bossnode).\n";
613
    system("cat /tmp/$$.ns | $SSH -host $bossnode '(cat > $nsfilename)'");
614 615
    if ($?) {
	die("*** $0:\n".
616
	    "    Could not copy ns code to inner boss ($bossnode)!\n");
617
    }
618 619

    #
620 621
    # Now run batchexp on the node as the user. If firewalled, experiment
    # must start async (cause we have to turn the firewall back on). 
622
    #
623 624 625 626
    my $optarg = ($firewalled ? "" : "-w");
	
    print "Starting experiment $pid/$elabinelab_eid on inner emulab.\n";
    TBDebugTimeStamp("Starting inner experiment");
627 628
    system("$SSH -host $bossnode ".
	   " 'sudo -u $user_uid /usr/testbed/bin/batchexp ".
629
	   "  -q -i $optarg -S \"ElabInElab Experiment\" ".
630 631 632 633 634
	   "  -L \"ElabInElab ElabInElab\" -E \"ElabInElab Experiment\" ".
	   "  -p $pid -e $elabinelab_eid $nsfilename'");
    
    $UID = $SAVEUID;
    unlink("/tmp/$$.ns");
635
}
636
skipsetup:
637

638
#
639 640 641 642 643 644
# Turn the firewall back on.
#
# XXX If this fails, we have to do something much stronger! We do not want
# nodes coming up and starting something if the firewall is not active.
# Maybe hit the panic button from here (turning off the control network).
#
645 646
#
if ($firewalled) {
647 648 649 650 651 652 653
    my $cmd;

    if ($fwtype =~ /^iptables/) {
        $cmd = "$SSH -host $firewall iptables -D FORWARD 1";
    } else {
        $cmd = "$SSH -host $firewall ipfw delete 1";
    }
654 655
    print "Turning firewall back on\n";
    $UID = 0;
656
    system($cmd);
657
    if ($?) {
658 659
	print STDERR "*** Error turning back on firewall rules ($firewall)!\n".
		     "    Will retry again.\n";
660
	system($cmd);
661 662 663 664
	if ($?) {
	    die("*** $0:\n".
		"    Error turning back on firewall rules! Retry failed.\n");
	}
665 666 667 668
    }
    $UID = $SAVEUID;
}

669 670 671
TBDebugTimeStamp("ElabInElab setup done");
exit(0);

672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
#
# Dump parts of the DB that are needed for inner elab to run. The idea
# is to create a set of files named by the table name. Note that mysqld
# cannot write to the project tree cause of directory permissions. Put the
# files into the workdir for now, and them copy them over. 
#
sub DumpDBGoo()
{
    my $statedir = "$workdir/elabinelab";

    if (-d $statedir) {
	system("rm -rf $statedir");
    }
    mkdir($statedir, 0777) or
	die("*** $0:\n".
	    "    Could not mkdir $statedir\n");
    
    chmod(0777, $statedir) or
	die("*** $0:\n".
	    "    Could not chmod $statedir\n");

693
    if (exists($elabinelab_attributes->{'CONFIG_ADMINUSERS'}) &&
Leigh Stoller's avatar
Leigh Stoller committed
694
	$elabinelab_attributes->{'CONFIG_ADMINUSERS'} != 0) {
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723

	# XXX Yes, bad. But we have a bunch of users with admin bit set
	# that should not.
	my @adminusers = ("stoller", "mike", "duerig", "ricci", "kwebb",
			  "johnsond", "gary", "amaricq");

	if (! $this_user->IsAdmin()) {
	    die("*** $0:\n".
		"    Must be an admin user to use CONFIG_ADMINUSERS\n");
	}
	foreach my $uid (@adminusers) {
	    system("$dumpuser $uid > $statedir/$uid.xml");
	}

	#
	# Tar up the directory and send it over to ops.
	#
	$UID = 0;
	system("tar cf - -C $statedir . | ".
	       "   gzip | $SSH -F /dev/null -host $CONTROL ".
	       "   '(cat > $expdir/users.tar.gz)'");
	if ($?) {
	    die("*** $0:\n".
		"    Could not create users.tar.gz\n");
	}
	$UID = $SAVEUID;
	return 0;
    }

724 725 726
    #
    # These tables are dumped completely.
    #
727
    my @FULLTABLES = ("node_types", "node_type_attributes", "interface_types",
728 729
		      "interface_capabilities",
		      "switch_paths", "switch_stack_types", "switch_stacks",
Timothy Stack's avatar
Timothy Stack committed
730
		      "node_type_features", "node_types_auxtypes", "osid_map",
731
		      "os_boot_cmd", "emulab_features");
732 733 734

    #
    # These tables are dumped by role (node/ops). For each one dump the table
735 736
    # as is, unless its the fs or ops node. For those we want to change the
    # node_id to "fs" or "ops" and their type to ops.
737
    #
738 739 740
    my @NODETABLES = ("node_auxtypes", "node_status", "nodes", 
		      "node_rusage", "node_hostkeys", "node_activity",
		      "interface_state");
741 742 743 744

    #
    # These tables are dumped by project ID.
    #
745
    my @PROJTABLES = ("projects", "groups", "group_features");
746 747

    #
748
    # These tables are dumped by user ID (for the project members).
749
    #
750
    my @USERTABLES = ("users", "user_pubkeys", "user_features");
751 752 753

    foreach my $table (@FULLTABLES) {
	unlink("$statedir/$table");
754 755 756 757 758
	DBQueryWarn("create temporary table temp_${table} ".
		    "select t.* from $table as t")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");

759 760 761 762 763
	#
	# Reduce the delay capacity by one if we are using one of
	# the experimental interfaces as an inner control network.
	#
	if ($table eq "node_type_attributes" && !$elabinelab_singlenet) {
764
	    my $attributes_result =
765
		DBQueryFatal("select type,attrvalue from temp_${table} ".
766 767 768 769
			     "where attrkey='delay_capacity'");

	    while (my ($ntype,$value) = $attributes_result->fetchrow_array()) {
		my $newvalue = $value - 1;
770 771 772

		next
		    if ($newvalue < 0);
773 774 775
		
		DBQueryFatal("update temp_${table} set ".
			     "   attrvalue='$newvalue' ".
776 777
			     "where type='$ntype' and ".
			     "      attrkey='delay_capacity'");
778
	    }
779 780
	}

781 782 783 784 785
	# filter out community strings
	if ($table eq "switch_stacks" || $table eq "switch_stack_types") {
	    DBQueryFatal("update temp_${table} set snmp_community=NULL");
	}

786
	DBQueryWarn("select * from temp_$table ".
787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }

    foreach my $table (@NODETABLES) {
	unlink("$statedir/$table");
	#
	# Create a temporary table.
	#
	DBQueryWarn("create temporary table temp_${table} ".
		    "select t.* from reserved as r ".
		    "left join $table as t on t.node_id=r.node_id ".
		    "left join virt_nodes as v on v.vname=r.vname and ".
		    "     v.pid=r.pid and v.eid=r.eid ".
		    "where r.pid='$pid' and r.eid='$eid' and ".
		    "      t.node_id is not null and ".
804
		    "      v.inner_elab_role in ('node','fs','ops','ops+fs')")
805 806 807
	    or die("*** $0:\n".
		   "    Could not create temporary table temp_$table\n");
	#
808 809
	# Rename the fs and ops node in each table. For the nodes table,
	# there is a bunch of other stuff to do.
810
	#
811 812 813
	DBQueryFatal("update temp_${table} set node_id='fs' ".
		     "where node_id='$fsnode'")
	    if (defined($fsnode));
814
	DBQueryFatal("update temp_${table} set node_id='ops' ".
815 816
		     "where node_id='$opsnode'")
	    if (defined($opsnode));
817 818

	if ($table eq "nodes") {
819
	    DBQueryFatal("update temp_nodes set ".
820
			 " type='ops', ".
821
			 " phys_nodeid=node_id, ".
822 823
			 " role='ctrlnode', ".
			 " op_mode='OPSNODEBSD' ".
824
			 "where node_id in ('fs','ops')");
825

826 827 828 829 830
	    # Also add the nodes that correspond to infrastructure switches
	    DBQueryFatal("insert into temp_nodes ".
			 "select distinct n.* from switch_stacks as s ".
			 "left join nodes as n on s.node_id=n.node_id ".
			 "where stack_id not like 'ExpStack%'");
831 832

	    # Clear any node reservations on the inside
833
	    DBQueryFatal("update temp_nodes set ".
834
			 " reserved_pid=null where reserved_pid is not null");
835 836 837

	    # Put the inner nodes into "limbo" so they DTRT when restarted
	    if ($restartnodes) {
838
		DBQueryFatal("update temp_nodes set".
839 840 841 842 843 844
			     "  op_mode='PXEKERNEL',next_op_mode='',".
			     "  eventstate='". TBDB_NODESTATE_PXELIMBO . "',".
			     "  temp_boot_osid=NULL,next_boot_osid=NULL,".
			     "  osid=NULL".
			     " where role='testnode'");
	    }
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
	}
    
	DBQueryWarn("select * from temp_$table ".
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }

    foreach my $table (@PROJTABLES) {
	unlink("$statedir/$table");
	DBQueryWarn("select * from $table ".
		    "where pid='$pid' ".
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
    #
    # Cleared versions of the project/group stats tables.
    #
    unlink("$statedir/project_stats");
    DBQueryFatal("create temporary table temp_project_stats ".
		 "like project_stats");
    DBQueryFatal("insert into temp_project_stats (pid,pid_idx) ".
		 "select pid,pid_idx from project_stats ".
		 "where pid='$pid'");
    DBQueryFatal("select * from temp_project_stats ".
		 "into outfile '$statedir/project_stats'");

    unlink("$statedir/group_stats");
    DBQueryFatal("create temporary table temp_group_stats ".
		 "like group_stats");
    DBQueryFatal("insert into temp_group_stats ".
		 "  (pid,pid_idx,gid,gid_idx,gid_uuid) ".
		 "select pid,pid_idx,gid,gid_idx,gid_uuid ".
		 "   from group_stats ".
		 "where pid='$pid'");
    DBQueryFatal("select * from temp_group_stats ".
		 "into outfile '$statedir/group_stats'");
883

884 885 886 887 888 889 890 891 892 893 894 895
    #
    # Special case the group and user policy tables. Not sure what to
    # really do about this; should there be any restrictions inside the
    # inner elab?
    #
    unlink("$statedir/group_policies");
    DBQueryWarn("select * from group_policies ".
		"where pid='$pid' or pid='+' or pid='-' ".
		"into outfile '$statedir/group_policies'")
	or die("*** $0:\n".
	       "    Could not dump table group_policies\n");

896 897
    foreach my $table (@USERTABLES) {
	unlink("$statedir/$table");
898 899

	DBQueryWarn("create temporary table temp_$table ".
900
		    "select distinct t.* from group_membership as gm ".
901 902
		    "left join users as u on u.uid_idx=gm.uid_idx ".
		    "left join $table as t on t.uid_idx=u.uid_idx ".
903 904
		    "where (gm.pid='$pid' or ".
		    "       gm.pid='" . TBOPSPID() . "') and gm.gid=gm.pid ".
905
		    " and t.uid_idx is not NULL and ".
906 907
		    " (u.status='" . USERSTATUS_ACTIVE() . "' or ".
		    "  u.status='" . USERSTATUS_INACTIVE() . "')")
908 909 910
	    or die("*** $0:\n".
		   "    Could not create table temp_$table\n");

911 912 913 914 915
	# Clean up ... these are created in the inner elab.
	DBQueryFatal("delete from temp_${table} ".
		     "where uid='elabman' or uid='elabckup' or ".
		     "      uid='operator'");

916
	if ($table eq "users") {
917
	    my $creator_uid = $experiment->creator();
918 919 920
	    
	    DBQueryFatal("update temp_${table} set ".
			 " admin=1 ".
921
			 "where uid='$creator_uid'");
922 923

	    #
924
	    # Save time; force all other users to start out
925 926 927 928 929
	    # frozen since most users in the project do not ever
	    # actually log in. 
	    #
	    DBQueryFatal("update temp_${table} set ".
			 " status='" . USERSTATUS_FROZEN() . "' ".
930
			 "where uid!='$creator_uid'");
931 932 933
	}

	DBQueryWarn("select * from temp_$table ".
934 935
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
936
		   "    Could not dump table temp_$table\n");
937
    }
938 939 940 941 942 943 944 945 946 947 948 949
    #
    # We want a cleared stats table, so do it here.
    #
    DBQueryFatal("create temporary table temp_user_stats ".
		 "like user_stats");
    DBQueryFatal("insert into temp_user_stats ".
		 "  (uid,uid_idx,uid_uuid) ".
		 "select uid,uid_idx,uid_uuid from temp_users");
    DBQueryWarn("select * from temp_user_stats ".
		"into outfile '$statedir/user_stats'")
	or die("*** $0:\n".
	       "    Could not dump table temp_user_stats\n");
950 951 952

    # The group_membership is also special.
    DBQueryWarn("select gm.* from group_membership as gm ".
953
		"left join users as u on u.uid_idx=gm.uid_idx ".
954
		"where (gm.pid='$pid' or ".
955
		"       (gm.pid='" . TBOPSPID() . "' and gm.pid=gm.gid)) and ".
956 957
		" (u.status='" . USERSTATUS_ACTIVE() . "' or ".
		"  u.status='" . USERSTATUS_INACTIVE() . "') and ".
958 959
		" gm.uid!='elabman' and gm.uid!='elabckup' and ".
		" gm.uid!='operator' ".
960 961 962 963 964 965 966 967 968 969
		"into outfile '$statedir/group_membership'")
	or die("*** $0:\n".
	       "    Could not dump table group_membership\n");

    #
    # interfaces table. Need to tag the interfaces being used as the control
    # network, with the proper tag so they do not say they experimental
    # interfaces in the inner emulab. Use a temp table again.
    #
    DBQueryWarn("create temporary table temp_interfaces ".
970 971 972 973 974 975
		"select t.* ".
		"from reserved as r, interfaces as t, virt_nodes as v where ".
		"  t.node_id=r.node_id and ".
		"  v.vname=r.vname and v.pid=r.pid and v.eid=r.eid and ".
		"  r.pid='$pid' and r.eid='$eid' and ".
		"  v.inner_elab_role in ('node','ops','fs','ops+fs')")
976 977 978
	or die("*** $0:\n".
	       "    Could not create temporary table temp_interfaces\n");

979
    if (! $elabinelab_singlenet) {
980 981 982 983 984 985 986
	# First, mark the real control network as "other" to avoid it being
	# thought of as the control network!.
	DBQueryWarn("update temp_interfaces ".
		    "set role='" . TBDB_IFACEROLE_OUTER_CONTROL() . "' " .
		    "where role='" . TBDB_IFACEROLE_CONTROL() . "'")
	    or die("*** $0:\n".
		   "    Could not delete control ifaces from temp_interfaces\n");
987

988 989 990 991 992 993
	DBQueryWarn("update temp_interfaces set ".
		    " role='" . TBDB_IFACEROLE_CONTROL() . "' " .
		    "where IP!='' and role='" . TBDB_IFACEROLE_EXPERIMENT() . "'")
	    or die("*** $0:\n".
		   "    Could not update roles in temp_interfaces\n");
    }
994

995 996 997 998 999 1000 1001
    # And rename the fs/ops nodes as above.
    if (defined($fsnode)) {
	DBQueryWarn("update temp_interfaces set node_id='fs' ".
		    "where node_id='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not fs node_id in temp_interfaces\n");
    }
1002 1003 1004 1005 1006 1007
    if (defined($opsnode)) {
	DBQueryWarn("update temp_interfaces set node_id='ops' ".
		    "where node_id='$opsnode'")
	    or die("*** $0:\n".
		   "    Could not ops node_id in temp_interfaces\n");
    }
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022

    # Also add the interfaces that correspond to the "trunk" wires.
    DBQueryFatal("insert into temp_interfaces ".
		 "select distinct i.* from wires as w ".
		 "left join interfaces as i on w.node_id1=i.node_id or ".
		 "     w.node_id2=i.node_id ".
		 "where w.type='Trunk'");

    DBQueryWarn("select * from temp_interfaces ".
		"into outfile '$statedir/interfaces'")
	or die("*** $0:\n".
	       "    Could not dump table interfaces\n");

    # And the wires table. Strip out the control wires; not needed.
    DBQueryWarn("create temporary table temp_wires ".
1023 1024 1025 1026 1027 1028 1029
		"select t.* ".
		"from reserved as r, virt_nodes as v, wires as t where ".
		"  v.vname=r.vname and v.pid=r.pid and v.eid=r.eid and ".
		"  t.node_id1=r.node_id and ".
		   ($elabinelab_singlenet ? "" : "t.type='Node' and ") .
		"  r.pid='$pid' and r.eid='$eid' and ".
		"  v.inner_elab_role in ('node','ops','fs','ops+fs')")
1030 1031 1032
	or die("*** $0:\n".
	       "    Could not create temporary table temp_wires\n");

1033 1034 1035 1036 1037 1038 1039
    # And rename the fs/ops node as above.
    if (defined($fsnode)) {
	DBQueryWarn("update temp_wires set node_id1='fs' ".
		    "where node_id1='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not fs node_id in temp_wires\n");
    }
1040 1041 1042 1043 1044 1045
    if (defined($opsnode)) {
	DBQueryWarn("update temp_wires set node_id1='ops' ".
		    "where node_id1='$opsnode'")
	    or die("*** $0:\n".
		   "    Could not ops node_id in temp_wires\n");
    }
1046

1047
    if (! $elabinelab_singlenet) {
1048 1049 1050
	# But we need to take out the wires that are being used as the
	# inner control network, or at least mark them as Control.
	$query_result =
1051
	    DBQueryWarn("select node_id,iface from temp_interfaces ".
1052 1053
			"where role='" . TBDB_IFACEROLE_CONTROL() . "' ");

1054
	while (my ($node_id,$iface) = $query_result->fetchrow_array()) {
1055
	    DBQueryWarn("update temp_wires set type='Control' ".
1056
			"where node_id1='$node_id' iface1='$iface'");
1057
	}
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
    }
    # Okay, now add the "trunk" wires in without any alteration.
    DBQueryWarn("insert into temp_wires ".
		"select * from wires where type='Trunk'") 
	or die("*** $0:\n".
	       "    Could not add trunk lines to temp_wires\n");

    DBQueryWarn("select * from temp_wires ".
		"into outfile '$statedir/wires'")
	or die("*** $0:\n".
	       "    Could not dump table wires\n");

    #
1071 1072
    # Ack, we need to create a reservation for the fs and ops nodes,
    # or else they will look free and it will not be able to check in.
1073 1074 1075 1076 1077 1078
    #
    DBQueryWarn("create temporary table temp_reserved ".
		"select r.* from reserved as r ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"where r.pid='$pid' and r.eid='$eid' ".
1079
		"      and v.inner_elab_role in ('fs','ops','ops+fs')")
1080 1081
	or die("*** $0:\n".
	       "    Could not create temporary table temp_reserved\n");
1082 1083 1084 1085
    if (defined($fsnode)) {
	DBQueryWarn("update temp_reserved set ".
		    "   node_id='fs', ".
		    "   pid='$TBOPSPID', ".
1086 1087
		    "   eid='opsnodes', ".
		    "   exptidx=1 ".
1088 1089 1090 1091
		    "where node_id='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not update temporary table temp_reserved\n");
    }
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
    if (defined($opsnode)) {
	DBQueryWarn("update temp_reserved set ".
		    "   node_id='ops', ".
		    "   pid='$TBOPSPID', ".
		    "   eid='opsnodes', ".
		    "   exptidx=1 ".
		    "where node_id='$opsnode'")
	    or die("*** $0:\n".
		   "    Could not update temporary table temp_reserved\n");
    }
1102 1103 1104 1105 1106
    DBQueryWarn("select * from temp_reserved ".
		"into outfile '$statedir/reserved'")
	or die("*** $0:\n".
	       "    Could not dump table reserved\n");

1107
    # Copy tiplines table for all nodes so web form gives us a console icon!
1108
    DBQueryWarn("select t.tipname,t.node_id,'',t.disabled,0,0,NULL,NULL,0,0 ".
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
		"from reserved as r ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"left join tiplines as t on t.node_id=r.node_id ".
		"where r.pid='$pid' and r.eid='$eid' and ".
		"      v.inner_elab_role='node' ".
		"into outfile '$statedir/tiplines'")
	or die("*** $0:\n".
	       "    Could not dump table tiplines\n");

1119 1120 1121 1122 1123
    #
    # Dump the DB schema too, so we can check in the inner Elab that this data
    # is compatible with the sql/database-create.sql schema file there, *before*
    # loading it into the db.  Added/removed columns would misalign row data.
    #
Russ Fish's avatar
Russ Fish committed
1124 1125 1126
    my $schemafile = "$expdir/outer_db_schema";
    system("rm -f $schemafile")
	if (-f $schemafile);
Leigh Stoller's avatar
Leigh Stoller committed
1127 1128 1129

    my $extraopts  = "";
    my $sqlversion = `/usr/local/bin/mysql_config --version`;
Leigh Stoller's avatar
Leigh Stoller committed
1130 1131 1132
    if ($sqlversion =~ /^(\d+\.\d+)\.\d+$/) {
	$sqlversion = $1;
    }
Leigh Stoller's avatar
Leigh Stoller committed
1133 1134 1135
    if ($sqlversion >= 5) {
	$extraopts .= " --skip-quote-names";
    }
Leigh Stoller's avatar
Leigh Stoller committed
1136
    if ($sqlversion >= 5.7) {
Leigh Stoller's avatar
Leigh Stoller committed
1137 1138
	$extraopts .= " --compact --set-gtid-purged=OFF";
    }
Russ Fish's avatar
Russ Fish committed
1139 1140 1141 1142
    #
    # XXX: Requires that mysqldump be in caller's $PATH - probably an OK
    # assumption, but maybe not always
    #
1143 1144
    my $mysqldump   = "mysqldump -d $extraopts $DBNAME " .
	"@FULLTABLES @NODETABLES @PROJTABLES @USERTABLES";
Russ Fish's avatar
Russ Fish committed
1145
    system("$mysqldump 2> /dev/null > $schemafile");
1146

1147
    #
1148
    # Tar up the directory and send it over to (real) ops.
1149 1150 1151
    #
    $UID = 0;
    system("tar cf - -C $statedir . | ".
Kirk Webb's avatar
Kirk Webb committed
1152
	   "   gzip | $SSH -F /dev/null -host $CONTROL ".
1153
	   "   '(cat > $expdir/dbstate.tar.gz)'");
1154 1155 1156 1157 1158 1159 1160 1161
    if ($?) {
	die("*** $0:\n".
	    "    Could not create dbstate.tar.gz\n");
    }
    $UID = $SAVEUID;
    return 0;
}

1162
#
1163 1164
# Tear down an inner Emulab as cleanly as possible to avoid power cycling
# nodes.
1165 1166 1167 1168 1169 1170
# 
sub TearDownEmulab()
{
    my $tbdir      = "/usr/testbed";
    my $wap        = "$tbdir/sbin/withadminprivs";
    my $nodereboot = "$tbdir/bin/node_reboot";
1171

1172 1173 1174 1175 1176 1177 1178
    #
    # We want to rebuild the DHCPD file so that when we reboot the inner nodes
    # they come back to the outer emulab. We cannot just free the nodes, cause
    # then the reload daemon might beat us to it, and end up power cycling the
    # nodes, and that would be bad. So, munge the DB and clear the "role" slot
    # for inner nodes. 
    #
1179
    DBQueryFatal("update reserved set inner_elab_role=NULL,inner_elab_boot=0 ".
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
		 "where pid='$pid' and eid='$eid'");

    #
    # XXX Failure at this point will leave things in an inconsistent state
    # cause we have just munged the reserved table. Since we were trying
    # to swap out the experiment, I think this will be okay. Wait and see.
    #
    return 0
	if (!defined($bossnode));

    #
    # Now regen the DHCPD file.
    #
1193
    # Run as real user since script is setuid.
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
    $EUID = $UID;
    
    print "Regenerating DHCPD config file and restarting daemon.\n";
    system("$makeconf -i -r");
    if ($?) {
	die("*** $0:\n".
	    "    Failed to reconfig/restart DHCPD.\n");
    }
    $EUID = 0;

1204 1205 1206 1207 1208 1209 1210 1211 1212
    #
    # Kill inner vlans table entries; this is the table that maps
    # inner to outer vlans. We do not care about that anymore since
    # all of the vlans are going to be torn down (using the outer
    # ids).
    #
    DBQueryFatal("delete from elabinelab_vlans ".
		 "where pid='$pid' and eid='$eid'");

1213
    #
1214 1215
    # If firewalled, just return now since all nodes will be powered
    # off anyway.
1216
    #
1217 1218
    if ($firewalled) {
	print "Skipping clean shutdown since experiment is firewalled.\n";
1219 1220 1221
	return 0;
    }

1222 1223 1224 1225 1226
    #
    # When the nodes reboot, we want them to do something reasonable. We
    # have no idea what is loaded on the disk, so they should go into an
    # MFS and wait, but then a bunch of nodes will all try to load the big
    # MFS at once, and that could wreak havoc. So, clear the boot osids
1227 1228
    # so they go into PXEWAIT.
    #
1229 1230 1231 1232 1233 1234 1235
    if (@expnodes) {
	system("$osselect -w @expnodes");
	if ($?) {
	    print STDERR "*** $0:\n".
		         "    Could not clear bootinfo for inner nodes!\n".
			 "    Continuing anyway.\n";
	}
1236
    }
Leigh Stoller's avatar
Leigh Stoller committed
1237
    
1238 1239 1240 1241 1242
    #
    # SSH in and kill the inner DHCPD daemon so that it does not reply
    # to rebooting nodes along the inner control network.
    #
    $UID = 0;
1243

1244 1245 1246
    print "Killing DHCPD on inner boss ($bossnode)\n";
    system("$SSH -host $bossnode /usr/local/etc/rc.d/2.dhcpd.sh stop");
    if ($?) {
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
	#
	# This error is non-fatal. If DHCPD cannot be killed, then the inner
	# boss is scrogged or never set up properly. Just return and let
	# the nodes get power cycled (if need be). At some point we need a
	# state machine to control this setup stuff. 
	# 
	print STDERR "*** $0:\n".
	             "    Could not stop DHCPD on inner bossnode ($bossnode)!\n".
		     "    Continuing anyway; outer boss will use power cycle.\n";
	return 0;
1257
    }
1258

1259 1260 1261
    #
    # Now we ask inner boss to reboot all of the testnodes. Maybe need an
    # option to node_reboot, but for now just pass them on the command line.
1262 1263 1264 1265 1266 1267
    #
    if (! @expnodes) {
	$UID = $SAVEUID;
	return 0;
    }
    
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
    print "Asking inner boss ($bossnode) to reboot inner nodes\n";
    system("$SSH -host $bossnode $wap $nodereboot -b @expnodes");
    if ($?) {
	#
	# This error is non-fatal; Outer boss will just resort to power cycle.
	#
	print STDERR "*** $0:\n".
	             "    Could not reboot some inner nodes!\n".
		     "    Continuing anyway; outer boss will use power cycle.\n";
    }
1278
    $UID = $SAVEUID;
1279

1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
    #
    # Now we wait for them to reach PXEWAIT. Again, use our utility script
    # instead of stated stuff.
    #
    $EUID = $UID;
    print "Waiting for inner nodes to reach PXEWAIT\n";
    system("$nodewait @expnodes");
    if ($?) {
	#
	# This error is non-fatal; Outer boss will just resort to power cycle.
	#
	print STDERR "*** $0:\n".
	             "    Some machines did not reboot properly!\n".
		     "    Continuing anyway; outer boss will use power cycle.\n";
    }
    return 0;
}
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306

#
# Remove nodes from an inner Emulab.
# 
sub RemoveNodes()
{
    my $tbdir      = "/usr/testbed";
    my $wap        = "$tbdir/sbin/withadminprivs";
    my $nodereboot = "$tbdir/bin/node_reboot";
    my $deletenode = "$tbdir/sbin/deletenode";
1307
    my $creator    = $experiment->creator();
1308
    my @nodes	   = ();
1309
    my $paniced    = 0;
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322

    #
    # If firewalled, check to see if paniced. Right now that means the nodes
    # are going to be powered off, so need to do the clean shutdown dance.
    # 
    if ($firewalled) {
	TBExptGetPanicBit($pid, $eid, \$paniced);
    }

    #
    # Actually, this should not even happen; a paniced experiment cannot be
    # modified at all.
    #
1323 1324
    if ($paniced) {
	print "A paniced experiment cannot be modified! What happened?\n";
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
	return -1;
    }

    #
    # Grab the list of nodes. We want to clear the reserved table bits so
    # that we can regen the DHCPD file. 
    #
    shift(@ARGV);	# pid
    shift(@ARGV);	# eid

    foreach my $node (@ARGV) {
	# Untaint the nodes.
	if ($node =~ /^([-\w]+)$/) {
	    $node = $1;
	}
	else {
	    die("*** Tainted node name: $node\n");
	}
	push(@nodes, $node);
    }
    return 0
	if (!@nodes);

    #
    # Grab the vlans table. We need to find any ports used by the nodes
    # getting deleted, and move them back to the default vlan. 
    #
1352 1353
    my @delmembers = ();
    my @todelete   = ();
1354 1355
    
    my $query_result =
1356 1357
	DBQueryWarn("select inner_id,outer_id from elabinelab_vlans ".
		    "where pid='$pid' and eid='$eid'");
1358 1359 1360
    return -1
	if (!$query_result);

1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
    while (my ($inner_id,$outer_id) = $query_result->fetchrow_array()) {
	my $vlan = VLan->Lookup($outer_id);
	if (!defined($vlan)) {
	    print STDERR "*** No such vlan $outer_id ($inner_id)\n";
	    return -1;
	}
	my @members;

	if ($vlan->MemberList(\@members) != 0) {
	    print STDERR "*** Unable to load members for $vlan\n";
	    return -1;
	}
	my $id         = $outer_id;
	my $changed    = 0;
1375

1376 1377 1378
	foreach my $member (@members) {
	    my $node;
	    my $iface;
1379

1380 1381 1382
	    if ($member->GetNodeIface(\$node, \$iface) != 0) {
		print STDERR "Missing attributes for $member in $vlan\n";
		return -1;
1383
	    }
1384 1385 1386 1387 1388 1389
	    my $nodeid = $node->node_id();
	    
	    # See if this node is in the list of nodes to be deleted,
	    if (grep {$_ eq $nodeid} @nodes) {
		push(@todelete, "$nodeid:$iface");
		push(@delmembers, $member);
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
		$changed = 1;
	    }
	}
    }

    # Remove ports from the vlans.
    if (@todelete) {
	print "Removing ports from deleted nodes: @todelete\n";
	system("$snmpit -m default @todelete");
	if ($?) {
	    return -1;
	}
    }
    # Only if the above succeeds, do we update the vlans table.
1404 1405
    foreach my $member (@delmembers) {
	$member->Delete() == 0
1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
	    or return -1;
    }
    
    #
    # We want to rebuild the DHCPD file so that when we reboot the inner nodes
    # they come back to the outer emulab. We cannot just free the nodes, cause
    # then the reload daemon might beat us to it, and end up power cycling the
    # nodes, and that would be bad. So, munge the DB and clear the "role" and
    # boot slots for nodes about to be released (by tbswap).
    #
    DBQueryWarn("update reserved set inner_elab_role=NULL,inner_elab_boot=0 ".
		"where pid='$pid' and eid='$eid' and (".
		join(" or ", map("node_id='$_'", @nodes)) . ")")
	or return -1;

    #
    # Now regen the DHCPD file.
    #
    # Run as real user since script is setuid.
    $EUID = $UID;
    
    print "Regenerating DHCPD config file and restarting daemon.\n";
    system("$makeconf -i -r");
    if ($?) {
	die("*** $0:\n".
	    "    Failed to reconfig/restart DHCPD.\n");
    }
    $EUID = 0;

    #
    # When the nodes reboot, we want them to do something reasonable. We
    # have no idea what is loaded on the disk, so they should go into an
    # MFS and wait, but then a bunch of nodes will all try to load the big
    # MFS at once, and that could wreak havoc. So, clear the boot osids
1440 1441 1442 1443 1444 1445 1446 1447
    # so they go into PXEWAIT.
    #
    system("$osselect -w @nodes");
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Could not clear bootinfo for freed nodes!\n".
		     "    Continuing anyway.\n";
    }
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
    
    #
    # SSH in and kill the inner DHCPD daemon so that it does not reply
    # to rebooting nodes along the inner control network.
    #
    $UID = 0;

    #
    # We are going to do this in a loop, one node at a time. I do not like
    # doing it this way, but its the only reasonable thing to do until we
    # can reboot the inner nodes ourselves (via the outer control network).
    # The reason for doing it one node at a time, is that I cannot delete the
    # node from the inner testbed until its been rebooted. Note that the
    # delete node script regens the dhcpd.conf file, so no need to do that
    # explicitly.
    #
    foreach my $node (@nodes) {
	print "Asking inner boss ($bossnode) to reboot $node\n";
	system("$SSH -host $bossnode $wap $nodereboot -b $node");
	if ($?) {
	    #
	    # This error is non-fatal;
	    # Outer boss will just resort to power cycle.
	    #
	    print STDERR "*** $0:\n".
		         "    Could not reboot $node! Continuing anyway.\n".
			 "    Outer boss will use power cycle.\n";
	}
	print "Asking inner boss ($bossnode) to delete $node\n";
1477
	system("$SSH -host $bossnode sudo -u $creator ".
1478
	       "     $wap $deletenode -b -q -f $node");
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
	if ($?) {
	    #
	    # This error is bad. 
	    #
	    print STDERR "*** $0:\n".
		         "    Could not delete $node! Modify will fail!\n";
	    return -1;
	}
    }
    $UID = $SAVEUID;

    #
    # Now we wait for them to reach PXEWAIT. Again, use our utility script
    # instead of stated stuff.
    #
    $EUID = $UID;
    print "Waiting for inner nodes to reach PXEWAIT\n";
    system("$nodewait @nodes");
    if ($?) {
	#
	# This error is non-fatal; Outer boss will just resort to power cycle.
	#
	print STDERR "*** $0:\n".
	             "    Some machines did not reboot properly!\n".
		     "    Continuing anyway; outer boss will use power cycle.\n";
    }
    return 0;
}

#
# Update an Emulab (add nodes).
# 
sub UpdateEmulab()
{
    my $tbdir      = "/usr/testbed";
1514
    my $statedir   = "$workdir/elabinelab";
1515 1516 1517
    my $wap        = "$tbdir/sbin/withadminprivs";
    my $nodereboot = "$tbdir/bin/node_reboot";
    my $nodewait   = "$tbdir/sbin/node_statewait";
1518
    my $creator    = $experiment->creator();
1519
    my @nodes      = ();
1520
    my $paniced    = 0;
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533

    #
    # If firewalled, check to see if paniced. Right now that means the nodes
    # are going to be powered off, so need to do the clean shutdown dance.
    # 
    if ($firewalled) {
	TBExptGetPanicBit($pid, $eid, \$paniced);
    }

    #
    # Actually, this should not even happen; a paniced experiment cannot be
    # modified at all.
    #
1534 1535
    if ($paniced) {
	print "A paniced experiment cannot be modified! What happened?\n";
1536 1537 1538 1539 1540 1541 1542
	return -1;
    }

    #
    # Grab the list of nodes that have been added to the inner elab.
    #
    my $query_result =
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
	DBQueryFatal("select r.node_id,n.type,i.IP from reserved as r ".
		     "left join nodes as n on n.node_id=r.node_id ".
		     "left join interfaces as i on i.node_id=r.node_id and ".
		     "     i.role='" . TBDB_IFACEROLE_CONTROL() . "' ".
		     "where r.pid='$pid' and r.eid='$eid' and ".
		     "      r.inner_elab_boot=0 and r.inner_elab_role='node'");
    return 0
	if (!$query_result->numrows);

    DBQueryFatal("create temporary table temp_new_nodes like new_nodes");
    DBQueryFatal("create temporary table temp_new_interfaces ".
		 " like new_interfaces");

    my %idmap = ();
    while (my ($node,$type,$ip) = $query_result->fetchrow_array()) {
	my $result =
	    DBQueryFatal("insert into temp_new_nodes set ".
			 " new_node_id=NULL, node_id='$node', ".
			 " type='$type', IP='$ip'");
	
	$idmap{$node} = $result->insertid;
    }
    @nodes = keys(%idmap);

    $query_result =
	DBQueryFatal("select r.node_id,i.ip,i.role,i.card,i.mac, ".
		     "  i.interface_type,w.node_id2,w.card2,w.port2 ".
		     " from reserved as r ".
		     "left join interfaces as i on i.node_id=r.node_id ".
		     "left join wires as w on w.node_id1=i.node_id and ".
		     "     w.card1=i.card and w.port1=i.port ".
		     "where r.pid='$pid' and r.eid='$eid' and ".
		     "      r.inner_elab_boot=0 and r.inner_elab_role='node'");

    while (my ($node,$ip,$role,$card,$mac,$type,$switch,
	       $switch_card,$switch_port) = $query_result->fetchrow_array()) {
	my $nid = $idmap{$node};

	if (! $elabinelab_singlenet) {
	    #
	    # Mark the real control network as "other" to avoid it being
	    # thought of as the control network.
	    #
	    if ($role eq TBDB_IFACEROLE_CONTROL()) {
		$role = TBDB_IFACEROLE_OUTER_CONTROL();
	    }
	    # And mark the inner control network.
	    if ($role eq TBDB_IFACEROLE_EXPERIMENT() &&
		defined($ip) && $ip ne "") {
		$role = TBDB_IFACEROLE_CONTROL();
Mike Hibler's avatar
Mike Hibler committed
1593 1594 1595
		# update the temp_new_nodes table with this IP
		DBQueryFatal("update temp_new_nodes set IP='$ip' ".
			     " where node_id='$node'");
1596 1597 1598 1599 1600 1601 1602
	    }
	}
	DBQueryFatal("insert into temp_new_interfaces set ".
		     " new_node_id='$nid', role='$role', card='$card', ".
		     " mac='$mac', interface_type='$type', ".
		     " switch_id='$switch',switch_card='$switch_card', ".
		     " switch_port='$switch_port'");
1603
    }
1604 1605 1606 1607 1608 1609 1610
    unlink("$statedir/new_nodes");
    DBQueryFatal("select * from temp_new_nodes ".
		 "into outfile '$statedir/new_nodes'");
    unlink("$statedir/new_interfaces");
    DBQueryFatal("select * from temp_new_interfaces ".
		 "into outfile '$statedir/new_interfaces'");

1611
    return 0
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	if ($dbgooonly);
    
    # For SSH and SCP below
    $UID = 0;

    #
    # Send the new_nodes and new_interfaces tables over to the
    # inner elab.
    #
    print "Sending new DB tables to inner boss ($bossnode)\n";
    system("$SCP $statedir/new_nodes $statedir/new_interfaces ".
	   "    ${bossnode}:/tmp");
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Could not scp tables to inner boss\n";
	return -1;
    }
    system("$SSH -host $bossnode mysqlimport -r tbdb ".
	   "     /tmp/new_nodes /tmp/new_interfaces");
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Could not load tables on inner boss\n";
	return -1;
    }
    print "Telling inner boss ($bossnode) to incorporate new nodes: @nodes\n";
    system("$SSH -host $bossnode sudo -u $creator /usr/testbed/sbin/wap ".
Mike Hibler's avatar
Mike Hibler committed
1638
	   "     /usr/testbed/sbin/newnode -f -q -n @nodes");
1639 1640 1641 1642 1643
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Could not incorporate new nodes on inner boss\n";
	return -1;
    }
1644 1645

    # Run as real user for the next few scripts, which are setuid.
1646
    $UID  = $SAVEUID;
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
    $EUID = $UID;

    #
    # Restart DHCPD, but first mark the nodes as being ready to boot inside
    # the inner emulab, so that dhcpd_makeconf knows what nodes to change
    # the entries for.
    #
    DBQueryFatal("update reserved set inner_elab_boot=1 ".
		 "where pid='$pid' and eid='$eid' and ".
		 "      inner_elab_boot=0 and inner_elab_role='node'");

    print "Regenerating DHCPD config file and restarting daemon.\n";
    system("$makeconf -i -r");
    if ($?) {
	die("*** $0:\n".
	    "    Failed to reconfig/restart DHCPD.\n");
    }

    # Reboot the experimental nodes. They will come up inside the inner elab.
    # DO NOT WAIT! They are not going to report ISUP from this point on. 
    print "Rebooting inner new experimental nodes.\n";
    TBDebugTimeStamp("Rebooting experimental nodes");
    system("$nodereboot @nodes");
    if ($?) {
	die("*** $0:\n".
	    "    Error rebooting the nodes (@nodes)!\n");
    }
    $EUID = 0;

    #
    # At this point, not much I can think of do. The nodes will reboot and
    # enter the newnode MFS. I could add a script to wait for that in the
    # inner elab, but not going to bother yet. 
1680 1681 1682 1683 1684 1685 1686 1687
    #
    # To avoid confusion later (with swapmod, which wants them to be ISUP),
    # and so the web interface does not show the nodes as down, set the 
    # state to ISUP.
    #
    foreach my $node (@nodes) {
	TBSetNodeEventState($node, TBDB_NODESTATE_ISUP());
    }
1688 1689
    return 0;
}