tbswap.in 43.8 KB
Newer Older
1 2 3 4
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
6 7 8 9
# All rights reserved.
#
use English;

10 11 12
# Returns 0 on success.
# Returns 1 on non-assign_wrapper failure.
# Returns (1 | assign_wrapper's errorcode) on assign_wrapper failure.
13 14
# Returns |0x40 if update caused a swapout. Icky.
# Returns -1 on uncontrolled error (die called).
15

16 17 18 19 20
# XXX: handle error cases for update? (backup the db?)
# XXX: Shouldn't do idempotent stuff twice for update.
# XXX: repush/calc routing for update??? (tbprerun)
# XXX: previz for update???              (tbprerun)
# XXX: make snmpit faster for update.
Chad Barb's avatar
Chad Barb committed
21 22 23
#
# XXX: for update, expt is swapped out on os_setup fail.
#      (we only recover if assign fails)
24 25 26

sub usage()
{
27
    print STDERR "Usage: $0 -force { in | out | update [-reboot] [-eventsys_restart] } pid eid\n";
28 29 30 31 32 33 34
    exit(-1);
}

#
# Configure variables
#
my $TBROOT         = "@prefix@";
35
my $TBOPS          = "@TBOPSEMAIL@";
Timothy Stack's avatar
Timothy Stack committed
36
my $TBLOGS         = "@TBLOGSEMAIL@";
37 38
my $TESTMODE       = @TESTMODE@;
my $DISABLE_EVENTS = "@DISABLE_EVENT_SCHED@";
39
my $piper          = "$TBROOT/sbin/locpiper";
40 41 42 43 44 45 46 47 48 49 50

# Untaint the path
$ENV{'PATH'} = "/usr/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
    ":$TBROOT/sbin:$TBROOT/bin";

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
51
use libadminctrl;
52
use libadminmfs;
53 54
use libtblog;

55
#require exitonwarn; # exitonwarn isn't really a module, so just require it
56 57 58 59

#
# Actual swap-in and swap-out functions, defined below.
#
60 61 62
sub doSwapout($);
sub doSwapin($);

63 64 65 66
#
# Firewall stuff
# XXX maybe should be elsewhere
#
67 68 69 70 71
sub FWSETUP()     { return 1; }
sub FWADDNODES()  { return 2; }
sub FWDELNODES()  { return 3; }
sub FWTEARDOWN()  { return 4; }
sub doFW($$$$);
72 73

# XXX fixme: should not be hardwired!
74
my $cnetstack = "-S Control";
75 76 77
my $cnetvlanname = "Control";


78 79 80 81
sub REAL()    { return 4; }
sub CLEANUP() { return 3; }
sub RETRY()   { return 2; }
sub UPDATE()  { return 1; }
Chad Barb's avatar
Chad Barb committed
82
sub UPDATE_RECOVER()  { return 0; }
83

84 85 86
#
# Grab global enable of linkdelays.
#
87
my $enablelinkdelays = TBGetSiteVar("general/linux_endnodeshaping");
88

89 90 91 92 93 94
#
# Turn off line buffering on output
#

$| = 1;

95 96
my $updateReboot   = 0;
my $updateReconfig = 1;
97 98
my $update_Eventsys_restart = 0;
my $elabinelab     = 0;
99 100
my $force  = 0;
my $errors = 0;
101
my $updatehosed = 0;
102
my $state;
103
my $canceled;
104 105
my $os_setup_pid;
my $cleanvlans;
106
my $nextState;
107 108

#
109
# First argument is either "in", "out", or "update";
110 111 112 113 114
# this value goes into $swapop.
#

my $swapop = shift;	

115 116 117 118
if (!$swapop || 
    (($swapop ne "in") && 
     ($swapop ne "out") &&
     ($swapop ne "update"))) {
119 120 121 122 123 124 125 126 127 128 129
    usage();
}

#
# Get other arguments.
#

while ($#ARGV > 1) {
    $arg = shift;
    if ($arg eq "-force") {
	$force = 1;
Chad Barb's avatar
Chad Barb committed
130 131
    } elsif ($arg eq "-reboot") {
	$updateReboot = 1;
132
	$updateReconfig = 0;
133 134 135
    } elsif ($arg eq "-noreconfig") {
	$updateReboot   = 0;
	$updateReconfig = 0;
136 137
    } elsif ($arg eq "-eventsys_restart" && $swapop eq "update") {
	$update_Eventsys_restart = 1;
138 139 140 141 142 143 144 145 146
    } else {
	usage();
    }
}
if ($#ARGV < 1) {
    usage();
}
my ($pid,$eid) = @ARGV;

147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\@\w.]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}

163 164 165 166 167 168
#
# Set Error reporting info
# 
tblog_set_info($pid,$eid,$UID);

#
169
# Turn on timestamps
170
#
171 172 173 174 175
TBDebugTimeStampsOn();

#
# Print starting message.
#
176 177
my $exptidx;
TBExptIDX($pid, $eid, \$exptidx);
178

179 180
print "Beginning swap-$swapop for $pid/$eid ($exptidx). " .
    TBTimeStampWithDate() . "\n";
181 182 183 184 185 186
TBDebugTimeStamp("tbswap $swapop started");

#
# Get experiment state; verify that experiment exists.
#
if (! ($state = ExpState($pid, $eid))) {
187
    tbdie "No such experiment $pid/$eid";
188
}
189 190 191
# Sanity check the current state. 
if (!$force) {
    if ($swapop eq "in") {
192
	tbdie("Experiment should be ACTIVATING. Currently $state.")
193
	    if ($state ne EXPTSTATE_ACTIVATING);
194
    }
195
    elsif ($swapop eq "out") {
196
	tbdie("Experiment should be SWAPPING. Currently $state.")
197
	    if ($state ne EXPTSTATE_SWAPPING);
198
    }
199
    elsif ($swapop eq "update") {
200
	tbdie("Experiment should be MODIFY_RESWAP. Currently $state.")
201
	    if ($state ne EXPTSTATE_MODIFY_RESWAP);
202 203
    }
}
204 205
# Get elabinelab status. See below.
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab)) {
206
    tbdie("Could not get elabinelab status for experiment $pid/$eid");
207
}
208

209 210 211 212 213
#
# See if the experiment is firewalled
#
my $firewalled = TBExptFirewall($pid, $eid);

214 215 216
#
# Do actual swapping
#
217 218 219 220 221
if ($swapop eq "out") {
    #
    # Swap out
    #
    $errors = doSwapout(REAL);
222
}
223
elsif ($swapop eq "update") {
224 225 226 227 228 229 230 231 232 233 234 235 236 237
    #
    # Before swapout, do cursory admission control to see if the
    # modified experiment will be swappable. assign_wrapper does a
    # more stringent check using assign.
    #
    print STDERR "Checking with Admission Control ...\n";
    if (! TBAdmissionControlCheck($UID, $pid, $eid, undef)) {
	print STDERR "Admission control failure!\n";
	print "Failingly finished swap-$swapop for $pid/$eid. " .
	    TBTimeStamp() . "\n";
	TBDebugTimeStamp("tbswap $swapop finished (failed)");
	exit(1);
    }
    
238 239 240 241 242 243 244
    #
    # Update.
    #
    # Phase One -- swap experiment partially out.
    #
    print STDERR "Backing up physical state...\n";
    TBExptBackupPhysicalState($pid,$eid);
245

246
    $errors = doSwapout(UPDATE);
Chad Barb's avatar
Chad Barb committed
247

248
    if ($errors) {
249
	#
250 251 252 253 254
	# Clean up the mess, leaving the experiment in the SWAPPED state,
	# 
	print STDERR "Cleaning up after errors.\n";
	doSwapout(CLEANUP);
	$updatehosed = 1;
255
    }
256
    else {
257
	#
258
	# Phase Two -- swap experiment back in.
259
	#
Chad Barb's avatar
Chad Barb committed
260 261
	$errors = doSwapin(UPDATE);

262
	if ($errors) {
Chad Barb's avatar
Chad Barb committed
263 264 265
	    #
	    # There were errors; see if we can recover.
	    #
266
	    my $CanRecover = 1;
Chad Barb's avatar
Chad Barb committed
267 268 269

	    if ($errors != 7) {
		print STDERR "Update failure occurred _after_ assign phase; ";
270
		$CanRecover = 0;
Chad Barb's avatar
Chad Barb committed
271 272
	    }

273 274 275 276 277 278 279 280
	    if ($CanRecover) {
		print STDERR "Recovering virtual and physical state.\n";

		if (TBExptRemoveVirtualState($pid, $eid) ||
		    TBExptRestoreVirtualState($pid, $eid) ||
		    TBExptRestorePhysicalState($pid,$eid)) {
		    print STDERR "Could not restore backed-up state; ";
		    $CanRecover = 0;
Chad Barb's avatar
Chad Barb committed
281
		}
282 283 284 285 286 287 288
		else {
		    print STDERR "Doing a recovery swap-in of old state.\n";

		    if (doSwapin(UPDATE_RECOVER)) {
			print STDERR "Could not swap in old physical state; ";
			$CanRecover = 0;
		    }
Chad Barb's avatar
Chad Barb committed
289 290
		}
	    }
291 292 293 294 295 296 297

	    #
	    # Some part of the recovery failed; must swap it out. swapexp
	    # (caller) will then have to do more clean up, hence the special
	    # exit status indicated by $updatehosed.
	    # 
	    if (! $CanRecover) {
298
		print STDERR "Recovery aborted! Swapping experiment out.\n";
Chad Barb's avatar
Chad Barb committed
299
		doSwapout(CLEANUP);
300 301 302 303
		$updatehosed = 1;
	    }
	    else {
		print STDERR "Update recovery successful.\n";
Chad Barb's avatar
Chad Barb committed
304
	    }
305
	}
Chad Barb's avatar
Chad Barb committed
306 307
    }
}
308 309 310 311 312
elsif ($swapop eq "in") {
    #
    # Swap in
    #
    my $retries = 2;
313 314 315 316 317 318 319 320 321 322 323 324 325

    #
    # Before real swapin, do cursory admission control. assign_wrapper does
    # a more stringent check using assign.
    #
    print STDERR "Checking with Admission Control ...\n";
    if (! TBAdmissionControlCheck($UID, $pid, $eid, undef)) {
	print STDERR "Admission control failure!\n";
	print "Failingly finished swap-$swapop for $pid/$eid. " .
	    TBTimeStamp() . "\n";
	TBDebugTimeStamp("tbswap $swapop finished (failed)");
	exit(1);
    }
326 327
    
    $errors = doSwapin(REAL);
Chad Barb's avatar
Chad Barb committed
328

329 330 331 332 333 334 335 336 337 338
    #
    # Attempt a retry if: 
    #   a) there were errors, 
    #   b) doswapin() indicated (via return code 3) a retry is appropriate,
    #   c) we haven't tried too many times already.
    #   d) The cancelflag has not been set.
    #   e) $TESTMODE == 0.
    #
    while ($errors == 3 && $retries && !$canceled && !$TESTMODE) {
	$retries--;
339

340 341
	print STDERR "Cleaning up after errors; will try again.\n";
	doSwapout(RETRY);
342

343 344
	print STDERR "Trying again...\n";
	$errors = doSwapin(RETRY);
345
    }
346 347
    if ($errors || $canceled) {
	print STDERR "Cleaning up after " .
348
	    ($canceled ? "cancelation" : "errors") . ".\n";
349
	doSwapout(CLEANUP);
350 351 352 353 354 355 356
    }
}

#
# Write appropriate message and exit.
#
if ($errors) {
357
    print "Failingly finished swap-$swapop for $pid/$eid. ".TBTimeStamp()."\n";
358
    TBDebugTimeStamp("tbswap $swapop finished (failed)");
Chad Barb's avatar
Chad Barb committed
359

360 361
    # Pass out magic value to indicate that update failed!
    exit(1 | ($updatehosed ? 0x40 : 0));
362
}
363
print "Successfully finished swap-$swapop for $pid/$eid. " .TBTimeStamp()."\n";
364 365
TBDebugTimeStamp("tbswap $swapop finished (succeeded)");
exit(0);
366 367 368 369

#################################

##
Chad Barb's avatar
Chad Barb committed
370
#
371 372
# doSwapout - Swaps experiment out.
#
Chad Barb's avatar
Chad Barb committed
373
#             If in REAL or CLEANUP,
374 375 376
#             this function will free all nodes for the 
#             experiment.
#
Chad Barb's avatar
Chad Barb committed
377
#             If in RETRY or UDPATE,
378 379 380 381 382 383
#             only nodes not in RES_READY will be freed.
#
#             Returns 0 on success, >0 on failure.
#
##

384 385
sub doSwapout($) {
    my $type = shift; # REAL==4, CLEANUP==3, RETRY==2, UPDATE==1.
386 387 388 389 390 391 392 393 394 395 396
    my $swapout_errors = 0;

    #
    # wait for os_setup;
    # this only applies if called after a failed doswapin.
    #
    if ($os_setup_pid) {
	print "Waiting for os_setup to finish\n";
	waitpid($os_setup_pid, 0);
	undef $os_setup_pid;
    }
Chad Barb's avatar
Chad Barb committed
397

398 399 400 401 402
    print "Getting files accessed via NFS.\n";
    TBDebugTimeStamp("nfstrace started");
    my ($dbuid, $uname, $umail);
    if (UNIX2DBUID($UID, \$dbuid) &&
	UserDBInfo($dbuid, \$uname, \$umail)) {
Timothy Stack's avatar
Timothy Stack committed
403
	SENDMAIL("$TBLOGS",
404 405 406 407 408 409
		 "Files accessed by $pid/$eid via NFS",
		 `nfstrace get $pid $eid`,
		 "$uname <$umail>");
    }
    TBDebugTimeStamp("nfstrace finished");

410
    if (! $TESTMODE) { 
411 412 413
	if (! ($DISABLE_EVENTS || $elabinelab)) {
	    if ($type >= RETRY ||
		($update_Eventsys_restart && $type == UPDATE) ) {
414 415
		print "Stopping the event system\n";
		if (system("eventsys_control stop $pid $eid")) {
416
		    tberror "Failed to stop the event system.";
417 418
		    $swapout_errors = 1;
		}
419 420 421 422 423 424 425

		#
		# Stop the location piper.
		#
		if (-x $piper) {
		    print "Stopping the location piper\n";
		    if (system("$piper -k $pid $eid")) {
426
			tberror "Failed to stop location piper.";
427 428 429
			$swapout_errors = 1;
		    }
		}
430 431
	    }
	}
432 433 434 435 436 437
	
	#
	# Do teardown of inner elab. We must do this before we teardown the
	# vlans since the inner control network is a vlan, and we want that
	# active so inner boss can reboot the inner nodes (avoid power cycle).
	#
438
	if ($elabinelab && $type >= CLEANUP) {
439 440
	    print "Tearing down elabinelab. This could take a while.\n";
	    if (system("elabinelab -k $pid $eid")) {
441
		tberror "Failed to teardown elabinelab!";
442 443 444 445
		$swapout_errors = 1;
	    }
	}

446
	#
Chad Barb's avatar
Chad Barb committed
447
	# Clean up any VLANs in experiment.
448
	#
449 450 451 452 453 454 455
	# When modifying an elabinelab experiment, leave the vlans intact
	# so that the inner networks are not suddenly disconnected!
	#
	if (! ($elabinelab && $type == UPDATE)) {
	    TBDebugTimeStamp("snmpit started");
	    print STDERR "Removing VLANs.\n";
	    if (system("snmpit -r $pid $eid")) {
456
		tberror "Failed to reset VLANs";
457 458 459 460 461
		$swapout_errors = 1;
	    } else {
		$cleanvlans = 0;
	    }
	    TBDebugTimeStamp("snmpit finished");
462
	}
463
    }
Chad Barb's avatar
Chad Barb committed
464

465 466 467
    if ($type >= CLEANUP) {
	#
	# We're not attempting a retry;
468
	#
469
	# Stop all of the vnodes.
470
	#
471
	if (! $TESTMODE) { 	
472 473 474
	    print "Tearing down virtual nodes.\n";
	    TBDebugTimeStamp("vnode_setup -k started");
	    if (system("vnode_setup -d -k $pid $eid")) {
475
		tberror "Failed to tear down vnodes.";
476 477 478
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("vnode_setup finished");
479 480
	}

481 482 483 484 485 486 487
	#
	# Nodes behind a firewall are treated special.
	# See undoFWNodes for details.
	#
	if ($firewalled && undoFWNodes($pid, $eid)) {
	    return 1;
	}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
488

489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
	#
	# Perform swapout time admin actions.  Right now there is at most
	# one of these.  It isn't really a general mechanism, just a hook
	# for state saving or data collection during swapout.
	# A couple of important "fer now" notes:
	#
	#	We don't do this for firewalled experiments.  We need a way
	#	to "tag" the saved disk state to ensure it doesn't get
	#	instantiated outside of a firewall.
	#
	#	We only do this on REAL swapouts, and not on CLEANUPs.
	#	There are some types of CLEANUPs where we may want to
	#	do this, in particular an invocation caused by a failed
	#	modify operation, where the admin action is to save the
	#	experiment state.  So we will need to revisit this.
	#
	my %soaction = ();
	if ($type == REAL && !$firewalled) {
	    TBExptGetSwapoutAction($pid, $eid, \%soaction);
	}
	if ($soaction{'command'}) {
	    my @nodes = ExpNodes($pid, $eid, 1);

512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
	    if (@nodes > 0) {
		print STDERR "Performing swapout admin MFS actions.\n";
		TBDebugTimeStamp("Performing swapout actions");
		my @failed = ();
		my %myargs = ();
		$myargs{'name'} = "tbswap";
		$myargs{'command'} = $soaction{'command'};
		if (defined($soaction{'timeout'})) {
		    $myargs{'timeout'} = $soaction{'timeout'};
		}
		$myargs{'timestamp'} = 1;
		if (TBAdminMfsRunCmd(\%myargs, \@failed, @nodes)) {
		    if ($soaction{'isfatal'}) {
			tberror
			    "Failed to run '" . $soaction{'command'} .
				"' on @failed!";
			return 1;
		    }
		    tbwarn
531 532
			"Failed to run '" . $soaction{'command'} .
			    "' on @failed!";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
533 534
		}
	    }
535 536
	}

537 538
	#
	# remove all nodes from the experiment.
Chad Barb's avatar
Chad Barb committed
539
	# (nfree will send them to RES_FREE_DIRTY)
540 541 542 543
	#
	print STDERR "Freeing nodes.\n";
	TBDebugTimeStamp("nfree started");
	if (system("nfree $pid $eid")) {
544
	    tberror "Could not free nodes.";
545 546 547
	    $swapout_errors = 1;
	}
	TBDebugTimeStamp("nfree finished");
548 549 550 551 552

	#
	# Since this is an actual swapout, 
	# reset our count of swap out nag emails sent.
	#
553
	DBQueryWarn("update experiments set swap_requests='',sim_reswap_count='0' ".
554
		    "where eid='$eid' and pid='$pid'");
555 556
    } else {
	#
557
	# $type == RETRY or $type == UPDATE.
558 559
	# Therefore, don't deallocate nodes which have been successfully
	# incorporated into the experiment (i.e., are RES_READY).
Chad Barb's avatar
Chad Barb committed
560
	# (nfree will send deallocated nodes to RES_FREE_DIRTY)
561
	#
562 563 564 565 566 567 568 569
	my @failedpnodes = ();
	my @failedvnodes = ();
	
	my $db_result =
	    DBQueryFatal("select rv.node_id,n.allocstate,nt.isvirtnode ".
                         "  from reserved as rv ".
			 "left join nodes as n on n.node_id = rv.node_id ".
			 "left join node_types as nt on nt.type=n.type ".
570 571
			 "where rv.pid='$pid' and rv.eid='$eid'");

572
	while (my ($node,$allocstate,$isvirt) = $db_result->fetchrow_array) {
573
	    if ($allocstate ne TBDB_ALLOCSTATE_RES_READY()) {
574 575 576 577 578 579
		if ($isvirt) {
		    push(@failedvnodes, $node);
		}
		else {
		    push(@failedpnodes, $node);
		}
580 581 582
	    }
	}

583 584 585 586 587 588 589
	#
	# Tear down failed vnodes. Perhaps not needed?
	# 
	if (!$TESTMODE && @failedvnodes > 0) {
	    print "Tearing down failed virtual nodes.\n";
	    TBDebugTimeStamp("vnode_setup -k started");
	    if (system("vnode_setup -d -k $pid $eid @failedvnodes")) {
590
		tberror "Failed to tear down vnodes.";
591 592 593 594
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("vnode_setup -k finished");
	}
595

596 597 598 599
	#
	# Release all failed nodes.
	# 
	if (@failedpnodes > 0 || @failedvnodes > 0) {
600 601
	    print STDERR "Freeing failed nodes.\n";
	    
602
	    TBDebugTimeStamp("nfree started");
Chad Barb's avatar
Chad Barb committed
603 604 605 606 607
	    #
	    # Specify -x switch so when a physical node gets freed,
	    # any virtual nodes (owned by this experiment)
	    # sitting on top of it are freed as well.
	    #
608 609
	    if (system("nfree -x $pid $eid " .
		       join(" ", (@failedpnodes, @failedvnodes)))) {
610
		tberror "Could not free nodes.";
611 612 613 614 615 616
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("nfree finished");
	}
    }

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
    if (! $TESTMODE) {
	#
	# If the experiment has no Plab dslice nodes left, but still has
	# a Plab slice, destroy the slice
	#

	# Does the slice exist?
	$db_result =
	    DBQueryFatal("select slicename from plab_slices ".
			 "where pid='$pid' and eid='$eid'");

	if ($db_result->numrows) {
	    # Are there any dslice nodes left?
	    $db_result =
		DBQueryFatal("select n.node_id from nodes as n ".
			     "left join node_types as nt on n.type = nt.type ".
			     "left join reserved as r ".
			     " on r.node_id = n.node_id ".
			     "where r.pid='$pid' and r.eid='$eid' ".
			     " and nt.isplabdslice = 1");

	    if (!$db_result->numrows) {
		print "Destroying Planetlab slice.\n";
		TBDebugTimeStamp("plabslice destroy started");
641
		if (system("plabslice destroy $pid $eid")) {
642
		    tberror "Failed to destroy Plab dslice.";
643 644 645 646 647 648 649
		    $swapout_errors = 1;
		}
		TBDebugTimeStamp("plabslice destroy finished");
	    }
	}
    }

650 651 652 653 654 655 656 657
    if (! $TESTMODE) {
	#
	# All of these errors are non-fatal on swapout. We find out about them
	# via email sent from the individual scripts.
	#

	#
	# Only reset mountpoints if this is an actual swapout, and
658
	# not a failed swapin(cleanup), update, or retry.
659
	#
660
	if ($type == REAL) {
661 662 663
	    print "Resetting mountpoints.\n";
	    TBDebugTimeStamp("exports started");
	    if (system("exports_setup")) {
664
		tberror "Failed to reset mountpoints.";
665 666 667
	    }
	    TBDebugTimeStamp("exports finished");
	}
Chad Barb's avatar
Chad Barb committed
668

669 670 671 672
	#
	# Resetting named maps and email lists is fast and idempotent,
	# so whatever.
	#
673 674 675
	print "Resetting named maps.\n";
	TBDebugTimeStamp("named started");
	if (system("named_setup")) {
676
	    tbwarn "Failed to reset named map.";
677 678
	}
	TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
679

680 681
	print "Resetting email lists.\n";
	TBDebugTimeStamp("genelists started");
682
	if (system("genelists -t")) {
683
	    tbwarn "Failed to reset email lists.";
684 685 686 687
	}
	TBDebugTimeStamp("genelists finished");
    }

688
    #
689 690 691
    # Wipe the DB clean except during UPDATE or RETRY. In those
    #    cases, assign_wrapper will reset the DB after reading
    #    the info.
692
    #
693 694 695 696
    if ( $type >= CLEANUP ) {
	print STDERR "Resetting DB.\n";
	TBExptRemovePhysicalState( $pid, $eid );
    }
697 698 699 700 701

    return $swapout_errors;
}

##
Chad Barb's avatar
Chad Barb committed
702
#
703 704
# doSwapin - Swaps experiment in.
#
Chad Barb's avatar
Chad Barb committed
705 706 707 708 709
#            Returns:
#              0 - successful swapin
#              1 - failed swapin; cleanup required.
#              3 - failed swapin; cleanup required; can retry.
#              7 - failed swapin; assign failed; no cleanup.
710 711
##

712
sub doSwapin($) {
713 714 715
    my $type = shift; # REAL==4, RETRY==2, UPDATE==1, UPDATE_RECOVER=0.
    # Just the physnodes ...
    my @deleted_pnodes = ();
716

717 718 719 720
    #
    # assign_wrapper does all the virtual to physical mapping 
    # and updating the DB state.
    #
721
    
Chad Barb's avatar
Chad Barb committed
722
    if ($type > UPDATE_RECOVER) {
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
        #
        # Hacky test to allow disabling of linkdelays if the node is going
        # to run Linux. See sitevar above.
        #
	if (! $enablelinkdelays) {
	    $db_result =
		DBQueryFatal("select distinct e.pid,e.eid,vl.vnode,vn.osname ".
			 "  from experiments as e ".
			 "left join virt_lans as vl on vl.pid=e.pid and ".
			 "     vl.eid=e.eid ".
			 "left join virt_nodes as vn on vn.pid=e.pid and ".
			 "     vn.eid=e.eid and vn.vname=vl.vnode ".
			 "left join os_info as o on o.osname=vn.osname and ".
			 "  (o.pid=vl.pid or o.pid='" . TBOPSPID() . "') ".
			 "where (vl.uselinkdelay!=0 or e.uselinkdelays!=0 or ".
			 "       e.forcelinkdelays!=0) and ".
			 "     (o.os is NULL or o.os='Linux') and ".
			 "     e.pid='$pid' and e.eid='$eid'");

	    if ($db_result->numrows) {
		print "*** Endnodeshaping is disabled on Linux Images!\n";
		print "*** You must modify your experiment to swap it in.\n";
		return 1;
	    }
	}
	
Chad Barb's avatar
Chad Barb committed
749 750 751 752
	print "Mapping to physical reality ...\n";
	TBDebugTimeStamp("assign_wrapper started");

	#
753 754 755 756
	# Pass the -u (update) switch into assign_wrapper, which turns on
	# update mode. When doing a retry, must also fix the current nodes
	# to avoid stuff jumping around when simply trying to replace a node
	# that did not boot.
Chad Barb's avatar
Chad Barb committed
757 758
	#
	my $exitcode;
759 760 761
	my $wrapper = "assign_wrapper -u";
	$wrapper .= " -f"
	    if ($type == RETRY);
762
	
763
	if (system("$wrapper $pid $eid")) {
764 765
	    $exitcode = $? >> 8;

766
	    tberror "Failed ($exitcode) to map to reality.";
767

768 769
	    # Wrapper sets this bit when recovery is possible.
	    if ($exitcode & 64) {
770
		# We can recover. 
Chad Barb's avatar
Chad Barb committed
771
		return 7;
772 773
	    }
	    else {
774
		# No recovery, no retry.
Chad Barb's avatar
Chad Barb committed
775 776 777 778
		return 1;
	    }
	}
	TBDebugTimeStamp("assign_wrapper finished");
779

Chad Barb's avatar
Chad Barb committed
780 781
	print "Mapped to physical reality!\n";
    }
782

783
    # Check cancel flag before continuing. No retry, 
784
    TBGetCancelFlag($pid, $eid, \$canceled);
785 786 787
    return 1
	if ($canceled);

788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
    #
    # Look for any nodes in RES_TEARDOWN. These need to be released,
    # and if a virtnode, they need to be torn down. We cannot wait for
    # the virtnodes to go down with the physnode they are hosted on,
    # so teardown and release the virtnodes first, and then do the
    # physnodes.
    #
    # Errors are fatal; no recovery or retry.
    #
    if ($type == UPDATE) {
	my $allocstate = TBDB_ALLOCSTATE_RES_TEARDOWN();
	
	$db_result =
	    DBQueryFatal("select r.node_id,nt.isvirtnode,nt.isremotenode ".
			 "  from reserved as r ".
			 "left join nodes as n on n.node_id=r.node_id ".
			 "left join node_types as nt on nt.type=n.type ".
			 "where r.pid='$pid' and r.eid='$eid' and ".
			 "      n.allocstate='$allocstate'");

	if ($db_result->numrows) {
	    my @virtnodes = ();
	    my @physnodes = ();
	    
	    print "Tearing down and releasing unused nodes\n";

	    # First teardown/release virtnodes. 
	    while (my ($node,$isvirt,$isrem) = $db_result->fetchrow_array()) {
		if ($isvirt) {
		    push(@virtnodes, $node);
		}
		elsif (!$isrem) {
		    push(@physnodes, $node);
		}
	    }
823 824 825
	    # See below.
	    @deleted_pnodes = @physnodes;
	    
826 827 828
	    if (@virtnodes) {
		TBDebugTimeStamp("vnode_setup started");
		
829
		if (system("vnode_setup -k $pid $eid @virtnodes")) {
830 831 832 833 834 835 836 837 838 839 840
		    print "Failed to tear down unused virtnodes!\n";
		    return 1;
		}
		TBDebugTimeStamp("vnode_setup finished");
		
		if (system("nfree $pid $eid @virtnodes")) {
		    print "Failed to nfree unused virtnodes!\n";
		    return 1;
		}
	    }
	    if (@physnodes) {
841 842 843
		if ($elabinelab) {
		    print "Removing nodes from inner elab.\n";
		    if (system("elabinelab -r $pid $eid @physnodes")) {
844
			tberror "Failed to remove inner nodes!";
845 846 847 848 849
			return 1;
		    }
		}

		#
850 851
		# If the experiment is firewalled, cleanup the nodes
		# we are releasing.
852
		# 
853
		if ($firewalled && undoFWNodes($pid, $eid, @deleted_pnodes)) {
854 855 856
		    return 1;
		}
		
857 858 859 860 861 862 863 864
		if (system("nfree $pid $eid @physnodes")) {
		    print "Failed to nfree unused physnodes!\n";
		    return 1;
		}
	    }
	}
    }

865 866 867 868 869 870
    # Exit here if we are testing.
    if ($TESTMODE) {
	print "Testing run - Stopping here.\n";
	return 0;
    }

871 872 873 874 875 876 877 878 879 880 881
    #
    # Handle tarballs - we might need to fetch some from URLs if the user
    # asked for that.
    #
    print "Fetching tarballs and RPMs (if any) ...\n";
    TBDebugTimeStamp("tarfiles_setup started");

    if (system("tarfiles_setup $pid $eid")) {
	#
	# No recovery for now - what would we do?
	#
882
	tberror "Failed to set up tarballs.";
883 884 885 886
	return 1;
    }
    TBDebugTimeStamp("tarfiles_setup finished");

887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
    #
    # If there are any Plab dslice nodes in the experiment, create the
    # dslice now
    #
    if ($type > UPDATE_RECOVER) {
	# Are there any Plab nodes?
	$db_result =
	    DBQueryFatal("select n.node_id from nodes as n ".
			 "left join node_types as nt on n.type = nt.type ".
			 "left join reserved as r on r.node_id = n.node_id ".
			 "where r.pid='$pid' and r.eid='$eid' ".
			 " and nt.isplabdslice = 1");

	if ($db_result->numrows) {
	    # Does slice already exist?
	    $db_result =
		DBQueryFatal("select slicename from plab_slices ".
			     "where pid='$pid' and eid='$eid'");

	    if (! $db_result->numrows) {
		my @plabnodes = ();
		
		while (my ($node) = $db_result->fetchrow_array()) {
		    push(@plabnodes, $node);
		}
		
		print "Creating Planetlab slice.\n";
		TBDebugTimeStamp("plabslice create started");
915
		if (system("plabslice create $pid $eid")) {
916
		    tberror "Failed to create Plab dslice";
917 918 919 920 921 922 923
		    return 3;
		}
		TBDebugTimeStamp("plabslice alloc finished");
	    }
	}
    }

924
    # Check cancel flag before continuing. No retry, 
925
    TBGetCancelFlag($pid, $eid, \$canceled);
926 927 928
    return 1
	if ($canceled);

929 930 931 932 933 934 935 936 937
    #
    # These things need to get started before the nodes come up, so we'll
    # do them before the os_setup. Everything else can done in parallel with
    # os_setup. (Actually, these probably can too, since they should finish
    # long before the nodes reboot, but better safe than sorry)
    #
    print "Setting up mountpoints.\n";
    TBDebugTimeStamp("mountpoints started");
    if (system("exports_setup")) {
938
	tberror "Failed to setup mountpoints.";
939 940 941
	return 1;
    }
    TBDebugTimeStamp("mountpoints finished");
Chad Barb's avatar
Chad Barb committed
942

943 944 945
    TBDebugTimeStamp("named started");
    print "Setting up named maps.\n";
    if (system("named_setup")) {
946
	tbwarn "Failed to add node names to named map.";
947 948
	#
	# This is a non-fatal error.
Chad Barb's avatar
Chad Barb committed
949
	#
950 951
    }
    TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
952

953 954 955 956 957 958 959 960
    print "Cleaning NFS traces.\n";
    TBDebugTimeStamp("nfstrace gc started");
    if (system("nfstrace gc $pid $eid")) {
	tberror "Failed to setup nfstrace.";
	return 1;
    }
    TBDebugTimeStamp("nfstrace gc finished");

961
    # Check cancel flag before continuing. No retry, 
962
    TBGetCancelFlag($pid, $eid, \$canceled);
963 964
    return 1
	if ($canceled);
Chad Barb's avatar
Chad Barb committed
965

966 967 968 969
    #
    # Setup any control-net firewall.
    # This must be done before reloading and rebooting nodes.
    #
970 971
    if ($firewalled && ($type == REAL || $type == UPDATE) &&
	doFW($pid, $eid, (($type == UPDATE) ? FWADDNODES : FWSETUP), undef)) {
972 973 974
	return 1;
    }

Chad Barb's avatar
Chad Barb committed
975 976
    #
    # If user specified -reboot to update,
977
    # and we are successfully performing the update,
978
    # then mark all nodes in experiment so os_setup will reboot them.
Chad Barb's avatar
Chad Barb committed
979
    #
980 981 982
    if (($type == UPDATE) &&
	($updateReboot || $updateReconfig)) {
	print STDERR "Marking nodes for reboot/reconfig.\n";
Chad Barb's avatar
Chad Barb committed
983
	$db_result =
984 985 986
	    DBQueryFatal("select r.node_id,n.allocstate from reserved as r ".
			 "left join nodes as n on n.node_id=r.node_id ".
			 "where r.pid='$pid' and r.eid='$eid'");
Chad Barb's avatar
Chad Barb committed
987

988 989 990 991 992 993 994 995
	while (my ($node,$allocstate) = $db_result->fetchrow_array) {
	    #
	    # If the node is INIT_CLEAN, leave it alone. It will still get
	    # rebooted, but will not falsely be tagged as dirty. This is
	    # important for vnodes too, where INIT_CLEAN indicated the vnode
	    # does not even exist yet (plab nodes).
	    #
	    if ($allocstate ne TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
996 997 998 999
		TBSetNodeAllocState($node,
				    ($updateReboot ?
				     TBDB_ALLOCSTATE_RES_INIT_DIRTY() :
				     TBDB_ALLOCSTATE_RES_RECONFIG()));
1000
	    }
Chad Barb's avatar
Chad Barb committed
1001 1002 1003
	}
    }

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
    #
    # Lets run gentopofile again, so we get ltmap right. This will come out
    # later, most likely.
    #
    TBDebugTimeStamp("gentopofile started");
    print "Generating ltmap (again) ...\n";

    if (system("gentopofile $pid $eid")) {
        tberror("gentopofile failed!");
	return 1;
    }
    TBDebugTimeStamp("gentopofile finished");

1017 1018 1019 1020 1021
    #
    # Since it'll take a while for the nodes to reboot, we'll start now, and
    # wait for the os_setup to finish, down below
    #
    print "Resetting OS and rebooting.\n";
1022
    TBDebugTimeStamp("launching os_setup");
1023 1024 1025
    if (!($os_setup_pid = fork())) { 
	exec("os_setup $pid $eid") or return 1;
    } elsif ($os_setup_pid == -1) {
1026
	tberror "Fork failed.";
1027 1028
	return 1;
    }
Chad Barb's avatar
Chad Barb committed
1029

1030
    #
1031 1032 1033 1034 1035
    # XXX
    # Don't add any steps between here and the waitpid() call below
    # without verifying that 1) It's OK for nodes to come up before
    # the step has completed and 2) It's OK for the command to run in
    # parallel with os_setup (no DB dependencies, etc.)
1036 1037 1038 1039 1040
    #

    print "Setting up VLANs.\n";
    TBDebugTimeStamp("snmpit started");
    if (system("snmpit -t $pid $eid")) {
1041
	tberror "Failed to set up VLANs.";
1042 1043 1044
	return 1;
    }
    TBDebugTimeStamp("snmpit finished");
Chad Barb's avatar
Chad Barb committed
1045

1046 1047 1048 1049
    #
    # An error now means that the VLANS need to be cleaned up.
    #
    $cleanvlans = 1;
Chad Barb's avatar
Chad Barb committed
1050

1051 1052
    print "Setting up email lists.\n";
    TBDebugTimeStamp("genelists started");
1053
    if (system("genelists -t")) {
1054
	tbwarn "Failed to update email lists.";
1055 1056 1057 1058 1059
	#
	# This is a non-fatal error.
	# 
    }
    TBDebugTimeStamp("genelists finished");
Chad Barb's avatar
Chad Barb committed
1060

1061 1062 1063 1064 1065 1066 1067 1068
    #
    # Don't clear port counters on UPDATE.
    # (XXX should clear new nodes' port counters.)

    if ($type >= RETRY) {
	print "Clearing port counters.\n";
	TBDebugTimeStamp("portstats started");
	if (system("portstats -z -a -q $pid $eid")) {
1069
	    tbwarn "Failed to clear port counters.";
1070 1071 1072 1073 1074
	    #
	    # This is a non-fatal error.
	    # 
	}
	TBDebugTimeStamp("portstats finished");
1075
    }
Chad Barb's avatar
Chad Barb committed
1076

1077 1078 1079 1080 1081 1082 1083
    #
    # OK, let's see how that os_setup did
    #
    $kid = waitpid($os_setup_pid,0);
    if ($kid == $os_setup_pid) {
	undef $os_setup_pid; # Make sure doswapout() doesn't wait for it.
	if ($CHILD_ERROR) {
1084
	    tberror "Failed to reset OS and reboot nodes.";
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
	    #
	    # If there is a firewall involved, it could be that the
	    # firewall rules are preventing essential communication,
	    # so don't retry.
	    #
	    # XXX should only do this if the user has specified additional
	    # rules.  But right now, I may screw up too!
	    #
	    if ($firewalled) {
		print STDERR "Not retrying, ".
		    "firewall may be preventing setup.\n";
		return 1;
	    }

1099
	    #
1100 1101
	    # Use returncode from os_setup process to
	    # set global $retry flag, indicating to caller
1102 1103 1104
	    # that it may be beneficial to attempt
	    # a doSwapin() again.
	    #
1105
            if (($CHILD_ERROR >> 8) == 1) {
Chad Barb's avatar
Chad Barb committed
1106
		return 3;
1107 1108
	    } else {
		print STDERR "Not retrying due to error type.\n";
Chad Barb's avatar
Chad Barb committed
1109
		return 1;
1110
	    }
1111 1112 1113
	}
    } else {
	undef $os_setup_pid;
1114
	tberror "Error waiting for os_setup to finish.";
1115 1116 1117
	return 1;
    }
    TBDebugTimeStamp("os_setup finished");
Chad Barb's avatar
Chad Barb committed
1118

1119 1120 1121 1122
    #
    # Okay, start the event system now that we know all the nodes have
    # rebooted (os_setup is done). This only takes a moment (puts itself
    # in the background), so its not enough of a delay to worry about.
1123 1124
    # Don't do this during an update, since we didn't kill the 
    # event system previously, so starting it again will fail!
1125
    # 
1126
    if (! ($DISABLE_EVENTS || $elabinelab)) {
1127 1128 1129 1130 1131 1132 1133
	#
	# For the robot testbed, start the location piper *before* the event
	# system.
	#
	if (-x $piper && ($type != UPDATE && $type != UPDATE_RECOVER)) {
	    print "Starting the location piper.\n";
	    if (system("$piper $pid $eid")) {
1134
		tberror "Failed to start the location piper.";
1135 1136 1137 1138
		return 1;
	    }
	}
	
1139 1140
	if ( $update_Eventsys_restart || 
	    ($type != UPDATE && $type != UPDATE_RECOVER) ) {
1141 1142 1143
	    print "Starting the event system.\n";
	    TBDebugTimeStamp("eventsys_control started");
	    if (system("eventsys_control start $pid $eid")) {
1144
		tberror "Failed to start the event system.";
1145 1146 1147
		return 1;
	    }
	    TBDebugTimeStamp("eventsys_control finished");
1148 1149
	}
    }
Chad Barb's avatar
Chad Barb committed
1150

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
    #
    # Do linktest if user requested it at swapin.
    #
    my $query_result =
	DBQueryFatal("select linktest_level,linktest_pid from experiments ".
		     "where pid='$pid' and eid='$eid'");
    my ($linktest_level,$linktest_pid) = $query_result->fetchrow_array();
 
    if ($linktest_level && ($type == REAL || $type == UPDATE)) {
	if ($linktest_pid) {
	    print STDERR "*** Linktest is already running! $linktest_pid\n";
	}
	else {
	    #
	    # Run it. No worries about failures.
	    #
1167
	    my $optarg = "-l $linktest_level -t 600 -m";
1168 1169 1170 1171 1172 1173 1174 1175
    
	    print "Starting linktest ... this could take a while!'\n";
	    if (system("linktest_control $optarg $pid $eid") != 0) {
		print STDERR "*** Linktest run returned non-zero status!\n";
	    }
	}
    }

1176 1177 1178
    #
    # ElabinElab setup. This might not be the right place for this!
    #
1179 1180 1181
    if ($elabinelab && !$TESTMODE && ($type == REAL || $type == UPDATE)) {
	my $optarg = ($type == UPDATE ? "-u" : "");
	
1182 1183
	print "Setting up elabinelab. This could take a while!\n";
	TBDebugTimeStamp("elabinelab setup started");
1184
	if (system("elabinelab $optarg $pid $eid")) {
1185
	    tberror "Failed to setup elabinelab!";
1186 1187
	    return 1;
	}
1188
	TBDebugTimeStamp("ElabInElab setup finished");
1189 1190
    }

1191 1192 1193
    #
    # Tell the event scheduler to START.
    # 
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204
    if (! ($DISABLE_EVENTS || $elabinelab)) {
	if ( $update_Eventsys_restart || 
	    ($type != UPDATE && $type != UPDATE_RECOVER) ) {
	    TBDebugTimeStamp("Starting event time");
	    if (system("tevc -e $pid/$eid now __ns_sequence start")) {
		print STDERR "*** Failed to start event time.\n";
		return 1;
	    }
	}
    }

1205 1206
    return 0;
}
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218

#
# Setup and teardown experiment firewall.
#
# XXX note that right now, we just setup the switch infrastructure
# first, and then just let everything else go.  Firewalled nodes will
# not boot until the firewall is up since the VLAN is isolated til then.
# The firewall will boot ok since it still talks to the real control net.
#
# XXX for tearing down firewalls, we assume that nodes have been "cleansed"
# and it is safe to put ports back into the default control net VLAN.
#
1219 1220
sub doFW($$$$) {
    my ($pid, $eid, $action, $nodelist) = @_;
1221
    my ($fwnode, $fwvlanname, $fwvlan, $fwport, $fwvid);
1222
    my %nodenames;
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235

    #
    # See if there is a firewall, fetching node/VLAN info if so.
    # If not, we are all done.
    #
    if (!TBExptFirewall($pid, $eid, \$fwnode, \$fwvid, \$fwvlan)) {
	return 0;
    }

    if ($action == FWSETUP) {
	$fwvid = TBGetUniqueIndex("cnet_vlanid");

	print "Setting up control net firewall.\n";
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
    }
    else {
	if ($action == FWADDNODES) {
	    print "Adding nodes to control net firewall.\n";
	}
	elsif ($action == FWDELNODES) {
	    print "Removing nodes from control net firewall.\n";
	}
	else {
	    print "Tearing down control net firewall.\n";
	}
1247 1248 1249 1250 1251 1252 1253

	# Prior setup didn't succeed, nothing to do
	if (!defined($fwvid)) {
	    return 0;
	}
    }

1254 1255 1256 1257 1258 1259 1260
    # See below.
    if (defined($nodelist)) {
	foreach my $node (@$nodelist) {
	    $nodenames{$node} = $node;
	}
    }

1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
    # XXX vlanid in the DB is currently an int, we need a more unique name
    $fwvlanname = "fw$fwvid";

    #
    # Find all the experiment nodes and their control interface switch ports
    #
    # XXX this may be replaced by a call to SNMPIT that just specifies
    # the pid/eid.  In that case someone else will first have to poplulate
    # the vlans table with this same info. 
    #
    my $db_result =
1272 1273 1274 1275 1276 1277
	DBQueryWarn("SELECT r.node_id,w.card1 ".
		    "  FROM wires AS w, reserved AS r ".
		    "WHERE r.node_id=w.node_id1 AND r.pid='$pid' ".
		    "  AND r.eid='$eid' AND w.type='Control'");
    return 1
	if (!$db_result);
1278 1279 1280

    my $portlist = "";
    while (my ($node,$cif) = $db_result->fetchrow_array()) {
1281
	print "$node $cif\n";
1282 1283
	if ($node eq $fwnode) {
	    $fwport = "$node:$cif";
1284 1285 1286 1287 1288 1289 1290 1291
	}
	elsif (defined($nodelist)) {
 	    print "foo @$nodelist\n";
	    # Only nodes we are moving in/out of the experiment.
	    $portlist .= " $node:$cif"
		if (exists($nodenames{$node}));
	}
	else {
1292 1293 1294 1295
	    $portlist .= " $node:$cif";
	}
    }
    if (!defined($fwport)) {
1296
	tberror "Firewall node '$fwnode' not found in $pid/${eid}!";
1297 1298 1299
	return 0;
    }
    if ($portlist eq "") {
1300 1301 1302 1303 1304
	#
	# We catch this up in swapexp; admin users can specify just a firewall,
	# but mere users must have at least one firewalled node. Just print
	# the warning though. 
	# 
1305
	tberror "No firewalled nodes in $pid/${eid}!";
1306 1307 1308 1309 1310 1311
    }

    #
    # XXX hack commands til we nail down the API
    #
    my $fwsetupstr1 = "snmpit $cnetstack -m $fwvlanname $portlist";
1312 1313
    my $fwsetupstr2 = "snmpit $cnetstack -N $fwvlanname";
    my $fwsetupstr3 = "snmpit $cnetstack -T $fwport $cnetvlanname $fwvlanname";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
1314
    my $fwtakedownstr0 = "snmpit $cnetstack -e $fwport";
1315 1316
    my $fwtakedownstr1 = ($portlist eq "" ? "true" :
			  "snmpit $cnetstack -m $cnetvlanname $portlist");
1317 1318 1319 1320 1321 1322
    my $fwtakedownstr2 = "snmpit $cnetstack -o $fwvlanname";
    my $fwtakedownstr3 = "snmpit $cnetstack -U $fwport";