tbswap.in 23.3 KB
Newer Older
Chad Barb's avatar
 
Chad Barb committed
1 2 3 4
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
Chad Barb's avatar
 
Chad Barb committed
6 7 8 9
# All rights reserved.
#
use English;

Chad Barb's avatar
 
Chad Barb committed
10 11 12
# Returns 0 on success.
# Returns 1 on non-assign_wrapper failure.
# Returns (1 | assign_wrapper's errorcode) on assign_wrapper failure.
13 14
# Returns |0x40 if update caused a swapout. Icky.
# Returns -1 on uncontrolled error (die called).
Chad Barb's avatar
 
Chad Barb committed
15

Chad Barb's avatar
 
Chad Barb committed
16 17 18 19 20
# XXX: handle error cases for update? (backup the db?)
# XXX: Shouldn't do idempotent stuff twice for update.
# XXX: repush/calc routing for update??? (tbprerun)
# XXX: previz for update???              (tbprerun)
# XXX: make snmpit faster for update.
Chad Barb's avatar
Chad Barb committed
21 22 23
#
# XXX: for update, expt is swapped out on os_setup fail.
#      (we only recover if assign fails)
Chad Barb's avatar
 
Chad Barb committed
24 25 26

sub usage()
{
27
    print STDERR "Usage: $0 -force { in | out | update [-reboot] [-eventsys_restart] } pid eid\n";
Chad Barb's avatar
 
Chad Barb committed
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
    exit(-1);
}

#
# Configure variables
#
my $TBROOT         = "@prefix@";
my $TESTMODE       = @TESTMODE@;
my $DISABLE_EVENTS = "@DISABLE_EVENT_SCHED@";

# Untaint the path
$ENV{'PATH'} = "/usr/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
    ":$TBROOT/sbin:$TBROOT/bin";

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
require exitonwarn; # exitonwarn isn't really a module, so just require it

#
# Actual swap-in and swap-out functions, defined below.
#
Chad Barb's avatar
 
Chad Barb committed
53 54 55 56 57 58 59
sub doSwapout($);
sub doSwapin($);

sub REAL()    { return 4; }
sub CLEANUP() { return 3; }
sub RETRY()   { return 2; }
sub UPDATE()  { return 1; }
Chad Barb's avatar
Chad Barb committed
60
sub UPDATE_RECOVER()  { return 0; }
Chad Barb's avatar
 
Chad Barb committed
61 62 63 64 65 66 67

#
# Turn off line buffering on output
#

$| = 1;

68 69
my $updateReboot   = 0;
my $updateReconfig = 1;
70 71
my $update_Eventsys_restart = 0;
my $elabinelab     = 0;
Chad Barb's avatar
 
Chad Barb committed
72 73
my $force  = 0;
my $errors = 0;
74
my $updatehosed = 0;
Chad Barb's avatar
 
Chad Barb committed
75
my $state;
76
my $canceled;
Chad Barb's avatar
 
Chad Barb committed
77 78
my $os_setup_pid;
my $cleanvlans;
79
my $nextState;
Chad Barb's avatar
 
Chad Barb committed
80 81

#
Chad Barb's avatar
 
Chad Barb committed
82
# First argument is either "in", "out", or "update";
Chad Barb's avatar
 
Chad Barb committed
83 84 85 86 87
# this value goes into $swapop.
#

my $swapop = shift;	

Chad Barb's avatar
 
Chad Barb committed
88 89 90 91
if (!$swapop || 
    (($swapop ne "in") && 
     ($swapop ne "out") &&
     ($swapop ne "update"))) {
Chad Barb's avatar
 
Chad Barb committed
92 93 94 95 96 97 98 99 100 101 102
    usage();
}

#
# Get other arguments.
#

while ($#ARGV > 1) {
    $arg = shift;
    if ($arg eq "-force") {
	$force = 1;
Chad Barb's avatar
Chad Barb committed
103 104
    } elsif ($arg eq "-reboot") {
	$updateReboot = 1;
105
	$updateReconfig = 0;
106 107 108
    } elsif ($arg eq "-noreconfig") {
	$updateReboot   = 0;
	$updateReconfig = 0;
109 110
    } elsif ($arg eq "-eventsys_restart" && $swapop eq "update") {
	$update_Eventsys_restart = 1;
Chad Barb's avatar
 
Chad Barb committed
111 112 113 114 115 116 117 118 119
    } else {
	usage();
    }
}
if ($#ARGV < 1) {
    usage();
}
my ($pid,$eid) = @ARGV;

120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\@\w.]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}

Chad Barb's avatar
 
Chad Barb committed
136 137 138 139 140 141 142 143 144 145 146 147 148
TBDebugTimeStampsOn();

#
# Print starting message.
#

print "Beginning swap-$swapop for $pid/$eid. " . TBTimeStamp() . "\n";
TBDebugTimeStamp("tbswap $swapop started");

#
# Get experiment state; verify that experiment exists.
#
if (! ($state = ExpState($pid, $eid))) {
149 150
    die("*** $0:\n".
	"    No such experiment $pid/$eid\n");
Chad Barb's avatar
 
Chad Barb committed
151
}
152 153 154 155 156 157
# Sanity check the current state. 
if (!$force) {
    if ($swapop eq "in") {
	die("*** $0:\n".
	    "    Experiment should be ACTIVATING. Currently $state.\n")
	    if ($state ne EXPTSTATE_ACTIVATING);
Chad Barb's avatar
 
Chad Barb committed
158
    }
159 160 161 162
    elsif ($swapop eq "out") {
	die("*** $0:\n".
	    "    Experiment should be SWAPPING. Currently $state.\n")
	    if ($state ne EXPTSTATE_SWAPPING);
Chad Barb's avatar
 
Chad Barb committed
163
    }
164 165 166 167
    elsif ($swapop eq "update") {
	die("*** $0:\n".
	    "    Experiment should be MODIFY_RESWAP. Currently $state.\n")
	    if ($state ne EXPTSTATE_MODIFY_RESWAP);
Chad Barb's avatar
 
Chad Barb committed
168 169
    }
}
170 171 172 173 174
# Get elabinelab status. See below.
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab)) {
    die("*** $0:\n".
	"    Could not get elabinelab status for experiment $pid/$eid\n");
}
Chad Barb's avatar
 
Chad Barb committed
175 176 177 178

#
# Do actual swapping
#
179 180 181 182 183
if ($swapop eq "out") {
    #
    # Swap out
    #
    $errors = doSwapout(REAL);
Chad Barb's avatar
 
Chad Barb committed
184
}
185 186 187 188 189 190 191 192
elsif ($swapop eq "update") {
    #
    # Update.
    #
    # Phase One -- swap experiment partially out.
    #
    print STDERR "Backing up physical state...\n";
    TBExptBackupPhysicalState($pid,$eid);
Chad Barb's avatar
 
Chad Barb committed
193

194
    $errors = doSwapout(UPDATE);
Chad Barb's avatar
Chad Barb committed
195

196
    if ($errors) {
Chad Barb's avatar
 
Chad Barb committed
197
	#
198 199 200 201 202
	# Clean up the mess, leaving the experiment in the SWAPPED state,
	# 
	print STDERR "Cleaning up after errors.\n";
	doSwapout(CLEANUP);
	$updatehosed = 1;
Chad Barb's avatar
 
Chad Barb committed
203
    }
204
    else {
Chad Barb's avatar
 
Chad Barb committed
205
	#
206
	# Phase Two -- swap experiment back in.
Chad Barb's avatar
 
Chad Barb committed
207
	#
Chad Barb's avatar
Chad Barb committed
208 209
	$errors = doSwapin(UPDATE);

210
	if ($errors) {
Chad Barb's avatar
Chad Barb committed
211 212 213
	    #
	    # There were errors; see if we can recover.
	    #
214
	    my $CanRecover = 1;
Chad Barb's avatar
Chad Barb committed
215 216 217

	    if ($errors != 7) {
		print STDERR "Update failure occurred _after_ assign phase; ";
218
		$CanRecover = 0;
Chad Barb's avatar
Chad Barb committed
219 220
	    }

221 222 223 224 225 226 227 228
	    if ($CanRecover) {
		print STDERR "Recovering virtual and physical state.\n";

		if (TBExptRemoveVirtualState($pid, $eid) ||
		    TBExptRestoreVirtualState($pid, $eid) ||
		    TBExptRestorePhysicalState($pid,$eid)) {
		    print STDERR "Could not restore backed-up state; ";
		    $CanRecover = 0;
Chad Barb's avatar
Chad Barb committed
229
		}
230 231 232 233 234 235 236
		else {
		    print STDERR "Doing a recovery swap-in of old state.\n";

		    if (doSwapin(UPDATE_RECOVER)) {
			print STDERR "Could not swap in old physical state; ";
			$CanRecover = 0;
		    }
Chad Barb's avatar
Chad Barb committed
237 238
		}
	    }
239 240 241 242 243 244 245

	    #
	    # Some part of the recovery failed; must swap it out. swapexp
	    # (caller) will then have to do more clean up, hence the special
	    # exit status indicated by $updatehosed.
	    # 
	    if (! $CanRecover) {
246
		print STDERR "Recovery aborted! Swapping experiment out.\n";
Chad Barb's avatar
Chad Barb committed
247
		doSwapout(CLEANUP);
248 249 250 251
		$updatehosed = 1;
	    }
	    else {
		print STDERR "Update recovery successful.\n";
Chad Barb's avatar
Chad Barb committed
252
	    }
Chad Barb's avatar
 
Chad Barb committed
253
	}
Chad Barb's avatar
Chad Barb committed
254 255
    }
}
256 257 258 259 260 261 262
elsif ($swapop eq "in") {
    #
    # Swap in
    #
    my $retries = 2;
    
    $errors = doSwapin(REAL);
Chad Barb's avatar
Chad Barb committed
263

264 265 266 267 268 269 270 271 272 273
    #
    # Attempt a retry if: 
    #   a) there were errors, 
    #   b) doswapin() indicated (via return code 3) a retry is appropriate,
    #   c) we haven't tried too many times already.
    #   d) The cancelflag has not been set.
    #   e) $TESTMODE == 0.
    #
    while ($errors == 3 && $retries && !$canceled && !$TESTMODE) {
	$retries--;
Chad Barb's avatar
 
Chad Barb committed
274

275 276
	print STDERR "Cleaning up after errors; will try again.\n";
	doSwapout(RETRY);
Chad Barb's avatar
 
Chad Barb committed
277

278 279
	print STDERR "Trying again...\n";
	$errors = doSwapin(RETRY);
Chad Barb's avatar
 
Chad Barb committed
280
    }
281 282
    if ($errors || $canceled) {
	print STDERR "Cleaning up after " .
283
	    ($canceled ? "cancelation" : "errors") . ".\n";
284
	doSwapout(CLEANUP);
Chad Barb's avatar
 
Chad Barb committed
285 286 287 288 289 290 291
    }
}

#
# Write appropriate message and exit.
#
if ($errors) {
292
    print "Failingly finished swap-$swapop for $pid/$eid. ".TBTimeStamp()."\n";
Chad Barb's avatar
 
Chad Barb committed
293
    TBDebugTimeStamp("tbswap $swapop finished (failed)");
Chad Barb's avatar
Chad Barb committed
294

295 296
    # Pass out magic value to indicate that update failed!
    exit(1 | ($updatehosed ? 0x40 : 0));
Chad Barb's avatar
 
Chad Barb committed
297
}
298
print "Successfully finished swap-$swapop for $pid/$eid. " .TBTimeStamp()."\n";
299 300
TBDebugTimeStamp("tbswap $swapop finished (succeeded)");
exit(0);
Chad Barb's avatar
 
Chad Barb committed
301 302 303 304

#################################

##
Chad Barb's avatar
Chad Barb committed
305
#
Chad Barb's avatar
 
Chad Barb committed
306 307
# doSwapout - Swaps experiment out.
#
Chad Barb's avatar
Chad Barb committed
308
#             If in REAL or CLEANUP,
Chad Barb's avatar
 
Chad Barb committed
309 310 311
#             this function will free all nodes for the 
#             experiment.
#
Chad Barb's avatar
Chad Barb committed
312
#             If in RETRY or UDPATE,
Chad Barb's avatar
 
Chad Barb committed
313 314 315 316 317 318
#             only nodes not in RES_READY will be freed.
#
#             Returns 0 on success, >0 on failure.
#
##

Chad Barb's avatar
 
Chad Barb committed
319 320
sub doSwapout($) {
    my $type = shift; # REAL==4, CLEANUP==3, RETRY==2, UPDATE==1.
Chad Barb's avatar
 
Chad Barb committed
321 322 323 324 325 326 327 328 329 330 331
    my $swapout_errors = 0;

    #
    # wait for os_setup;
    # this only applies if called after a failed doswapin.
    #
    if ($os_setup_pid) {
	print "Waiting for os_setup to finish\n";
	waitpid($os_setup_pid, 0);
	undef $os_setup_pid;
    }
Chad Barb's avatar
Chad Barb committed
332

Chad Barb's avatar
 
Chad Barb committed
333
    if (! $TESTMODE) { 
334 335 336
	if (! ($DISABLE_EVENTS || $elabinelab)) {
	    if ($type >= RETRY ||
		($update_Eventsys_restart && $type == UPDATE) ) {
Chad Barb's avatar
 
Chad Barb committed
337 338 339 340 341
		print "Stopping the event system\n";
		if (system("eventsys_control stop $pid $eid")) {
		    print STDERR "*** Failed to stop the event system.\n";
		    $swapout_errors = 1;
		}
Chad Barb's avatar
 
Chad Barb committed
342 343 344
	    }
	}
	#
Chad Barb's avatar
Chad Barb committed
345
	# Clean up any VLANs in experiment.
Chad Barb's avatar
 
Chad Barb committed
346
	#
Chad Barb's avatar
Chad Barb committed
347 348 349 350 351 352 353
	TBDebugTimeStamp("snmpit started");
	print STDERR "Removing VLANs.\n";
	if (system("snmpit -r $pid $eid")) {
	    print STDERR "*** Failed to reset VLANs\n";
	    $swapout_errors = 1;
	} else {
	    $cleanvlans = 0;
Chad Barb's avatar
 
Chad Barb committed
354
	}
Chad Barb's avatar
Chad Barb committed
355
	TBDebugTimeStamp("snmpit finished");
356
    }
Chad Barb's avatar
Chad Barb committed
357

358 359 360
    if ($type >= CLEANUP) {
	#
	# We're not attempting a retry;
Chad Barb's avatar
 
Chad Barb committed
361
	#
362
	# Stop all of the vnodes.
Chad Barb's avatar
 
Chad Barb committed
363
	#
364
	if (! $TESTMODE) { 	
Chad Barb's avatar
 
Chad Barb committed
365 366 367 368 369 370 371
	    print "Tearing down virtual nodes.\n";
	    TBDebugTimeStamp("vnode_setup -k started");
	    if (system("vnode_setup -d -k $pid $eid")) {
		print STDERR "*** Failed to tear down vnodes.\n";
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("vnode_setup finished");
Chad Barb's avatar
 
Chad Barb committed
372 373 374 375
	}

	#
	# remove all nodes from the experiment.
Chad Barb's avatar
 
Chad Barb committed
376
	# (nfree will send them to RES_FREE_DIRTY)
Chad Barb's avatar
 
Chad Barb committed
377 378 379 380 381 382 383 384
	#
	print STDERR "Freeing nodes.\n";
	TBDebugTimeStamp("nfree started");
	if (system("nfree $pid $eid")) {
	    print STDERR "*** Could not free nodes.\n";
	    $swapout_errors = 1;
	}
	TBDebugTimeStamp("nfree finished");
Chad Barb's avatar
 
Chad Barb committed
385 386 387 388 389

	#
	# Since this is an actual swapout, 
	# reset our count of swap out nag emails sent.
	#
390
	DBQueryWarn("update experiments set swap_requests='',sim_reswap_count='0' ".
391
		    "where eid='$eid' and pid='$pid'");
Chad Barb's avatar
 
Chad Barb committed
392 393
    } else {
	#
Chad Barb's avatar
 
Chad Barb committed
394
	# $type == RETRY or $type == UPDATE.
Chad Barb's avatar
 
Chad Barb committed
395 396
	# Therefore, don't deallocate nodes which have been successfully
	# incorporated into the experiment (i.e., are RES_READY).
Chad Barb's avatar
 
Chad Barb committed
397
	# (nfree will send deallocated nodes to RES_FREE_DIRTY)
Chad Barb's avatar
 
Chad Barb committed
398
	#
399 400 401 402 403 404 405 406
	my @failedpnodes = ();
	my @failedvnodes = ();
	
	my $db_result =
	    DBQueryFatal("select rv.node_id,n.allocstate,nt.isvirtnode ".
                         "  from reserved as rv ".
			 "left join nodes as n on n.node_id = rv.node_id ".
			 "left join node_types as nt on nt.type=n.type ".
Chad Barb's avatar
 
Chad Barb committed
407 408
			 "where rv.pid='$pid' and rv.eid='$eid'");

409
	while (my ($node,$allocstate,$isvirt) = $db_result->fetchrow_array) {
Chad Barb's avatar
 
Chad Barb committed
410
	    if ($allocstate ne TBDB_ALLOCSTATE_RES_READY()) {
411 412 413 414 415 416
		if ($isvirt) {
		    push(@failedvnodes, $node);
		}
		else {
		    push(@failedpnodes, $node);
		}
Chad Barb's avatar
 
Chad Barb committed
417 418 419
	    }
	}

420 421 422 423 424 425 426 427 428 429 430 431
	#
	# Tear down failed vnodes. Perhaps not needed?
	# 
	if (!$TESTMODE && @failedvnodes > 0) {
	    print "Tearing down failed virtual nodes.\n";
	    TBDebugTimeStamp("vnode_setup -k started");
	    if (system("vnode_setup -d -k $pid $eid @failedvnodes")) {
		print STDERR "*** Failed to tear down vnodes.\n";
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("vnode_setup -k finished");
	}
Chad Barb's avatar
 
Chad Barb committed
432

433 434 435 436
	#
	# Release all failed nodes.
	# 
	if (@failedpnodes > 0 || @failedvnodes > 0) {
437 438
	    print STDERR "Freeing failed nodes.\n";
	    
Chad Barb's avatar
 
Chad Barb committed
439
	    TBDebugTimeStamp("nfree started");
Chad Barb's avatar
 
Chad Barb committed
440 441 442 443 444
	    #
	    # Specify -x switch so when a physical node gets freed,
	    # any virtual nodes (owned by this experiment)
	    # sitting on top of it are freed as well.
	    #
445 446
	    if (system("nfree -x $pid $eid " .
		       join(" ", (@failedpnodes, @failedvnodes)))) {
Chad Barb's avatar
 
Chad Barb committed
447 448 449 450 451 452 453
		print STDERR "*** Could not free nodes.\n";
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("nfree finished");
	}
    }

454
    if (! $TESTMODE) {
455 456 457 458 459 460 461 462 463 464 465 466
	#
	# Restart DHCPD as soon as possible after freeing the nodes; they
	# are going into a reload, so DHCPD better be ready for them!
	# 
	if ($elabinelab) {
	    print "Regenerating DHCPD config file and restarting daemon.\n";
	    if (system("dhcpd_makeconf -i -r")) {
		print STDERR "*** Failed to reconfig/restart DHCPD.\n";
		$swapout_errors = 1;
	    }
	}

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
	#
	# If the experiment has no Plab dslice nodes left, but still has
	# a Plab slice, destroy the slice
	#

	# Does the slice exist?
	$db_result =
	    DBQueryFatal("select slicename from plab_slices ".
			 "where pid='$pid' and eid='$eid'");

	if ($db_result->numrows) {
	    # Are there any dslice nodes left?
	    $db_result =
		DBQueryFatal("select n.node_id from nodes as n ".
			     "left join node_types as nt on n.type = nt.type ".
			     "left join reserved as r ".
			     " on r.node_id = n.node_id ".
			     "where r.pid='$pid' and r.eid='$eid' ".
			     " and nt.isplabdslice = 1");

	    if (!$db_result->numrows) {
		print "Destroying Planetlab slice.\n";
		TBDebugTimeStamp("plabslice destroy started");
490
		if (system("plabslice destroy $pid $eid")) {
491 492 493 494 495 496 497 498
		    print STDERR "*** Failed to destroy Plab dslice\n";
		    $swapout_errors = 1;
		}
		TBDebugTimeStamp("plabslice destroy finished");
	    }
	}
    }

Chad Barb's avatar
 
Chad Barb committed
499 500 501 502 503 504 505 506
    if (! $TESTMODE) {
	#
	# All of these errors are non-fatal on swapout. We find out about them
	# via email sent from the individual scripts.
	#

	#
	# Only reset mountpoints if this is an actual swapout, and
Chad Barb's avatar
 
Chad Barb committed
507
	# not a failed swapin(cleanup), update, or retry.
Chad Barb's avatar
 
Chad Barb committed
508
	#
Chad Barb's avatar
 
Chad Barb committed
509
	if ($type == REAL) {
Chad Barb's avatar
 
Chad Barb committed
510 511 512 513 514 515 516
	    print "Resetting mountpoints.\n";
	    TBDebugTimeStamp("exports started");
	    if (system("exports_setup")) {
		print STDERR "*** Failed to reset mountpoints.\n";
	    }
	    TBDebugTimeStamp("exports finished");
	}
Chad Barb's avatar
Chad Barb committed
517

Chad Barb's avatar
 
Chad Barb committed
518 519 520 521
	#
	# Resetting named maps and email lists is fast and idempotent,
	# so whatever.
	#
Chad Barb's avatar
 
Chad Barb committed
522 523 524 525 526 527
	print "Resetting named maps.\n";
	TBDebugTimeStamp("named started");
	if (system("named_setup")) {
	    print "*** WARNING: Failed to reset named map.\n";
	}
	TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
528

Chad Barb's avatar
 
Chad Barb committed
529 530
	print "Resetting email lists.\n";
	TBDebugTimeStamp("genelists started");
531
	if (system("genelists -t")) {
Chad Barb's avatar
 
Chad Barb committed
532 533 534 535 536
	    print "*** WARNING: Failed to reset email lists.\n";
	}
	TBDebugTimeStamp("genelists finished");
    }

Chad Barb's avatar
 
Chad Barb committed
537
    #
538 539 540
    # Wipe the DB clean except during UPDATE or RETRY. In those
    #    cases, assign_wrapper will reset the DB after reading
    #    the info.
Chad Barb's avatar
 
Chad Barb committed
541
    #
542 543 544 545
    if ( $type >= CLEANUP ) {
	print STDERR "Resetting DB.\n";
	TBExptRemovePhysicalState( $pid, $eid );
    }
Chad Barb's avatar
 
Chad Barb committed
546 547 548 549 550

    return $swapout_errors;
}

##
Chad Barb's avatar
Chad Barb committed
551
#
Chad Barb's avatar
 
Chad Barb committed
552 553
# doSwapin - Swaps experiment in.
#
Chad Barb's avatar
Chad Barb committed
554 555 556 557 558
#            Returns:
#              0 - successful swapin
#              1 - failed swapin; cleanup required.
#              3 - failed swapin; cleanup required; can retry.
#              7 - failed swapin; assign failed; no cleanup.
Chad Barb's avatar
 
Chad Barb committed
559 560
##

Chad Barb's avatar
 
Chad Barb committed
561
sub doSwapin($) {
Chad Barb's avatar
Chad Barb committed
562
    my $type = shift; # REAL==4, RETRY==2, UPDATE==1, UPDATE_RECOVER=0. 
Chad Barb's avatar
 
Chad Barb committed
563

Chad Barb's avatar
 
Chad Barb committed
564 565 566 567 568
    #
    # assign_wrapper does all the virtual to physical mapping 
    # and updating the DB state.
    #

Chad Barb's avatar
Chad Barb committed
569 570 571 572 573
    if ($type > UPDATE_RECOVER) {
	print "Mapping to physical reality ...\n";
	TBDebugTimeStamp("assign_wrapper started");

	#
574 575 576 577
	# Pass the -u (update) switch into assign_wrapper, which turns on
	# update mode. When doing a retry, must also fix the current nodes
	# to avoid stuff jumping around when simply trying to replace a node
	# that did not boot.
Chad Barb's avatar
Chad Barb committed
578 579
	#
	my $exitcode;
580 581 582
	my $wrapper = "assign_wrapper -u";
	$wrapper .= " -f"
	    if ($type == RETRY);
583
	
584
	if (system("$wrapper $pid $eid")) {
585 586 587 588
	    $exitcode = $? >> 8;

	    print STDERR "*** Failed ($exitcode) to map to reality.\n";

589 590
	    # Wrapper sets this bit when recovery is possible.
	    if ($exitcode & 64) {
591
		# We can recover. 
Chad Barb's avatar
Chad Barb committed
592
		return 7;
593 594
	    }
	    else {
595
		# No recovery, no retry.
Chad Barb's avatar
Chad Barb committed
596 597 598 599
		return 1;
	    }
	}
	TBDebugTimeStamp("assign_wrapper finished");
Chad Barb's avatar
 
Chad Barb committed
600

Chad Barb's avatar
Chad Barb committed
601 602
	print "Mapped to physical reality!\n";
    }
Chad Barb's avatar
 
Chad Barb committed
603

604
    # Check cancel flag before continuing. No retry, 
605
    TBGetCancelFlag($pid, $eid, \$canceled);
606 607 608
    return 1
	if ($canceled);

609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
    #
    # Look for any nodes in RES_TEARDOWN. These need to be released,
    # and if a virtnode, they need to be torn down. We cannot wait for
    # the virtnodes to go down with the physnode they are hosted on,
    # so teardown and release the virtnodes first, and then do the
    # physnodes.
    #
    # Errors are fatal; no recovery or retry.
    #
    if ($type == UPDATE) {
	my $allocstate = TBDB_ALLOCSTATE_RES_TEARDOWN();
	
	$db_result =
	    DBQueryFatal("select r.node_id,nt.isvirtnode,nt.isremotenode ".
			 "  from reserved as r ".
			 "left join nodes as n on n.node_id=r.node_id ".
			 "left join node_types as nt on nt.type=n.type ".
			 "where r.pid='$pid' and r.eid='$eid' and ".
			 "      n.allocstate='$allocstate'");

	if ($db_result->numrows) {
	    my @virtnodes = ();
	    my @physnodes = ();
	    
	    print "Tearing down and releasing unused nodes\n";

	    # First teardown/release virtnodes. 
	    while (my ($node,$isvirt,$isrem) = $db_result->fetchrow_array()) {
		if ($isvirt) {
		    push(@virtnodes, $node);
		}
		elsif (!$isrem) {
		    push(@physnodes, $node);
		}
	    }
	    if (@virtnodes) {
		TBDebugTimeStamp("vnode_setup started");
		
647
		if (system("vnode_setup -k $pid $eid @virtnodes")) {
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
		    print "Failed to tear down unused virtnodes!\n";
		    return 1;
		}
		TBDebugTimeStamp("vnode_setup finished");
		
		if (system("nfree $pid $eid @virtnodes")) {
		    print "Failed to nfree unused virtnodes!\n";
		    return 1;
		}
	    }
	    if (@physnodes) {
		if (system("nfree $pid $eid @physnodes")) {
		    print "Failed to nfree unused physnodes!\n";
		    return 1;
		}
	    }
	}
    }

Chad Barb's avatar
 
Chad Barb committed
667 668 669 670 671 672
    # Exit here if we are testing.
    if ($TESTMODE) {
	print "Testing run - Stopping here.\n";
	return 0;
    }

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
    #
    # Handle tarballs - we might need to fetch some from URLs if the user
    # asked for that.
    #
    print "Fetching tarballs and RPMs (if any) ...\n";
    TBDebugTimeStamp("tarfiles_setup started");

    if (system("tarfiles_setup $pid $eid")) {
	#
	# No recovery for now - what would we do?
	#
	print STDERR "*** Failed to set up tarballs.\n";
	return 1;
    }
    TBDebugTimeStamp("tarfiles_setup finished");

689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
    #
    # If there are any Plab dslice nodes in the experiment, create the
    # dslice now
    #
    if ($type > UPDATE_RECOVER) {
	# Are there any Plab nodes?
	$db_result =
	    DBQueryFatal("select n.node_id from nodes as n ".
			 "left join node_types as nt on n.type = nt.type ".
			 "left join reserved as r on r.node_id = n.node_id ".
			 "where r.pid='$pid' and r.eid='$eid' ".
			 " and nt.isplabdslice = 1");

	if ($db_result->numrows) {
	    # Does slice already exist?
	    $db_result =
		DBQueryFatal("select slicename from plab_slices ".
			     "where pid='$pid' and eid='$eid'");

	    if (! $db_result->numrows) {
		my @plabnodes = ();
		
		while (my ($node) = $db_result->fetchrow_array()) {
		    push(@plabnodes, $node);
		}
		
		print "Creating Planetlab slice.\n";
		TBDebugTimeStamp("plabslice create started");
717
		if (system("plabslice create $pid $eid")) {
718 719 720 721 722 723 724 725
		    print STDERR "*** Failed to create Plab dslice\n";
		    return 3;
		}
		TBDebugTimeStamp("plabslice alloc finished");
	    }
	}
    }

726
    # Check cancel flag before continuing. No retry, 
727
    TBGetCancelFlag($pid, $eid, \$canceled);
728 729 730
    return 1
	if ($canceled);

Chad Barb's avatar
 
Chad Barb committed
731 732 733 734 735 736 737 738 739 740 741 742 743
    #
    # These things need to get started before the nodes come up, so we'll
    # do them before the os_setup. Everything else can done in parallel with
    # os_setup. (Actually, these probably can too, since they should finish
    # long before the nodes reboot, but better safe than sorry)
    #
    print "Setting up mountpoints.\n";
    TBDebugTimeStamp("mountpoints started");
    if (system("exports_setup")) {
	print STDERR "*** Failed to setup mountpoints.\n";
	return 1;
    }
    TBDebugTimeStamp("mountpoints finished");
Chad Barb's avatar
Chad Barb committed
744

Chad Barb's avatar
 
Chad Barb committed
745 746 747 748 749 750
    TBDebugTimeStamp("named started");
    print "Setting up named maps.\n";
    if (system("named_setup")) {
	print STDERR "*** WARNING: Failed to add node names to named map.\n";
	#
	# This is a non-fatal error.
Chad Barb's avatar
Chad Barb committed
751
	#
Chad Barb's avatar
 
Chad Barb committed
752 753
    }
    TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
754

755
    # Check cancel flag before continuing. No retry, 
756
    TBGetCancelFlag($pid, $eid, \$canceled);
757 758
    return 1
	if ($canceled);
Chad Barb's avatar
Chad Barb committed
759 760 761

    #
    # If user specified -reboot to update,
Chad Barb's avatar
 
Chad Barb committed
762
    # and we are successfully performing the update,
763
    # then mark all nodes in experiment so os_setup will reboot them.
Chad Barb's avatar
Chad Barb committed
764
    #
765 766 767
    if (($type == UPDATE) &&
	($updateReboot || $updateReconfig)) {
	print STDERR "Marking nodes for reboot/reconfig.\n";
Chad Barb's avatar
Chad Barb committed
768
	$db_result =
769 770 771
	    DBQueryFatal("select r.node_id,n.allocstate from reserved as r ".
			 "left join nodes as n on n.node_id=r.node_id ".
			 "where r.pid='$pid' and r.eid='$eid'");
Chad Barb's avatar
Chad Barb committed
772

773 774 775 776 777 778 779 780
	while (my ($node,$allocstate) = $db_result->fetchrow_array) {
	    #
	    # If the node is INIT_CLEAN, leave it alone. It will still get
	    # rebooted, but will not falsely be tagged as dirty. This is
	    # important for vnodes too, where INIT_CLEAN indicated the vnode
	    # does not even exist yet (plab nodes).
	    #
	    if ($allocstate ne TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
781 782 783 784
		TBSetNodeAllocState($node,
				    ($updateReboot ?
				     TBDB_ALLOCSTATE_RES_INIT_DIRTY() :
				     TBDB_ALLOCSTATE_RES_RECONFIG()));
785
	    }
Chad Barb's avatar
Chad Barb committed
786 787 788
	}
    }

Chad Barb's avatar
 
Chad Barb committed
789 790 791 792 793
    #
    # Since it'll take a while for the nodes to reboot, we'll start now, and
    # wait for the os_setup to finish, down below
    #
    print "Resetting OS and rebooting.\n";
794
    TBDebugTimeStamp("launching os_setup");
Chad Barb's avatar
 
Chad Barb committed
795 796 797 798 799 800
    if (!($os_setup_pid = fork())) { 
	exec("os_setup $pid $eid") or return 1;
    } elsif ($os_setup_pid == -1) {
	print STDERR "*** Fork failed.\n";
	return 1;
    }
Chad Barb's avatar
Chad Barb committed
801

Chad Barb's avatar
 
Chad Barb committed
802
    #
803 804 805 806 807
    # XXX
    # Don't add any steps between here and the waitpid() call below
    # without verifying that 1) It's OK for nodes to come up before
    # the step has completed and 2) It's OK for the command to run in
    # parallel with os_setup (no DB dependencies, etc.)
Chad Barb's avatar
 
Chad Barb committed
808 809 810 811 812 813 814 815 816
    #

    print "Setting up VLANs.\n";
    TBDebugTimeStamp("snmpit started");
    if (system("snmpit -t $pid $eid")) {
	print STDERR "*** Failed to set up VLANs.\n";
	return 1;
    }
    TBDebugTimeStamp("snmpit finished");
Chad Barb's avatar
Chad Barb committed
817

Chad Barb's avatar
 
Chad Barb committed
818 819 820 821
    #
    # An error now means that the VLANS need to be cleaned up.
    #
    $cleanvlans = 1;
Chad Barb's avatar
Chad Barb committed
822

Chad Barb's avatar
 
Chad Barb committed
823 824
    print "Setting up email lists.\n";
    TBDebugTimeStamp("genelists started");
825
    if (system("genelists -t")) {
Chad Barb's avatar
 
Chad Barb committed
826 827 828 829 830 831
	print STDERR "*** WARNING: Failed to update email lists.\n";
	#
	# This is a non-fatal error.
	# 
    }
    TBDebugTimeStamp("genelists finished");
Chad Barb's avatar
Chad Barb committed
832

Chad Barb's avatar
 
Chad Barb committed
833 834 835 836 837 838 839 840 841 842 843 844 845 846
    #
    # Don't clear port counters on UPDATE.
    # (XXX should clear new nodes' port counters.)

    if ($type >= RETRY) {
	print "Clearing port counters.\n";
	TBDebugTimeStamp("portstats started");
	if (system("portstats -z -a -q $pid $eid")) {
	    print STDERR "*** WARNING: Failed to clear port counters.\n";
	    #
	    # This is a non-fatal error.
	    # 
	}
	TBDebugTimeStamp("portstats finished");
Chad Barb's avatar
 
Chad Barb committed
847
    }
Chad Barb's avatar
Chad Barb committed
848

Chad Barb's avatar
 
Chad Barb committed
849 850 851 852 853 854 855 856 857
    #
    # OK, let's see how that os_setup did
    #
    $kid = waitpid($os_setup_pid,0);
    if ($kid == $os_setup_pid) {
	undef $os_setup_pid; # Make sure doswapout() doesn't wait for it.
	if ($CHILD_ERROR) {
	    print STDERR "*** Failed to reset OS and reboot nodes.\n";
	    #
Chad Barb's avatar
 
Chad Barb committed
858 859
	    # Use returncode from os_setup process to
	    # set global $retry flag, indicating to caller
Chad Barb's avatar
 
Chad Barb committed
860 861 862
	    # that it may be beneficial to attempt
	    # a doSwapin() again.
	    #
Chad Barb's avatar
 
Chad Barb committed
863
            if (($CHILD_ERROR >> 8) == 1) {
Chad Barb's avatar
Chad Barb committed
864
		return 3;
Chad Barb's avatar
 
Chad Barb committed
865 866
	    } else {
		print STDERR "Not retrying due to error type.\n";
Chad Barb's avatar
Chad Barb committed
867
		return 1;
Chad Barb's avatar
 
Chad Barb committed
868
	    }
Chad Barb's avatar
 
Chad Barb committed
869 870 871 872 873 874 875
	}
    } else {
	undef $os_setup_pid;
	print STDERR "*** Error waiting for os_setup to finish.\n";
	return 1;
    }
    TBDebugTimeStamp("os_setup finished");
Chad Barb's avatar
Chad Barb committed
876

Chad Barb's avatar
 
Chad Barb committed
877 878 879 880
    #
    # Okay, start the event system now that we know all the nodes have
    # rebooted (os_setup is done). This only takes a moment (puts itself
    # in the background), so its not enough of a delay to worry about.
Chad Barb's avatar
 
Chad Barb committed
881 882
    # Don't do this during an update, since we didn't kill the 
    # event system previously, so starting it again will fail!
Chad Barb's avatar
 
Chad Barb committed
883
    # 
884
    if (! ($DISABLE_EVENTS || $elabinelab)) {
885 886
	if ( $update_Eventsys_restart || 
	    ($type != UPDATE && $type != UPDATE_RECOVER) ) {
Chad Barb's avatar
 
Chad Barb committed
887 888 889 890 891 892 893
	    print "Starting the event system.\n";
	    TBDebugTimeStamp("eventsys_control started");
	    if (system("eventsys_control start $pid $eid")) {
		print STDERR "*** Failed to start the event system.\n";
		return 1;
	    }
	    TBDebugTimeStamp("eventsys_control finished");
Chad Barb's avatar
 
Chad Barb committed
894 895
	}
    }
Chad Barb's avatar
Chad Barb committed
896

897 898 899 900 901 902 903 904 905 906 907 908 909
    #
    # ElabinElab setup. This might not be the right place for this!
    #
    if ($elabinelab && !$TESTMODE &&  $type == REAL) {
	print "Setting up elabinelab. This could take a while!\n";
	TBDebugTimeStamp("elabinelab setup started");
	if (system("elabinelab $pid $eid")) {
	    print STDERR "*** Failed to setup elabinelab!\n";
	    return 1;
	}
	TBDebugTimeStamp("elabinelab setup finished");
    }

Chad Barb's avatar
 
Chad Barb committed
910 911
    return 0;
}