os_setup.in 17.6 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
#
3
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
23
#
24
use English;
25
use Getopt::Std;
26
use POSIX ":sys_wait_h";
27
use Data::Dumper;
28

29
#
30 31 32 33
# Reboot the nodes in an experiment. The nodes table will already contain
# all the information. This script deals with possible disk reloading,
# rebooting, and waiting for nodes to come back alive before allowing
# experiment creation to continue.
34
#
35
# usage: os_setup <pid> <eid>
36
#
37 38 39 40 41
# errorcode:  0 - all reboots succeeded.
#             1 - some/all reboots failed; retry may help.
#            -1 - failure; retry is inappropriate.
#

42 43
sub usage()
{
44
    print STDERR "Usage: os_setup [-d] <pid> <eid>\n";
45 46
    exit(-1);
}
47 48 49
my $optlist  = "id";
my $debug    = 1;
my $impotent = 0;
50 51 52 53 54

#
# Configure variables
#
my $TB		= "@prefix@";
55
my $TBOPS       = "@TBOPSEMAIL@";
56 57 58 59 60 61 62

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

# Turn off line buffering on output
$| = 1; 
63

64 65 66 67 68
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
69
use libossetup;
70 71
use libreboot;
use libosload;
72
use libtestbed;
Kevin Atkinson's avatar
Kevin Atkinson committed
73
use libtblog;
74
use NodeType;
75
use Experiment;
76
use OSImage;
77
use User;
78
use Node;
79
use EmulabFeatures;
80

81 82 83
# Simmer down EmulabFeatures!
$EmulabFeatures::verbose = 0;

84 85
# Is this needed any longer?
my $dolastload  = 0;
86

87
TBDebugTimeStampsOn();
88

89 90 91 92 93 94 95 96 97 98 99
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 2) {
    usage();
}
100
if (defined($options{"d"})) {
101
    $debug = 1;
102
}
103 104
if (defined($options{"i"})) {
    $impotent = 1;
105 106
}

107
#
108
# Verify user and get his DB uid and other info for later.
109
#
110 111 112
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    die_noretry("You ($UID) do not exist!");
113
}
114 115 116
my $user_uid      = $this_user->uid();
my $user_name     = $this_user->name();
my $user_email    = $this_user->email();
117 118
my $user_email_to = "$user_name <$user_email>";

119 120 121
#
# Check permission.
#
122
my $experiment = Experiment->Lookup($ARGV[0], $ARGV[1]);
123
if (!defined($experiment)) {
124
    die_noretry("Could not find experiment object");
125 126 127 128
}
if (!$experiment->AccessCheck($this_user, TB_EXPT_MODIFY)) {
    die_noretry("You do not have permission to swap this experiment!");
}
129 130 131
my $pid = $experiment->pid();
my $eid = $experiment->eid();

132 133
TBDebugTimeStamp("os_setup started");

134
#
135 136 137 138 139 140 141 142 143 144
# List of all nodes in the experiment.
#
my @nodelist = $experiment->NodeList(0, 1);
if (! @nodelist) {
    tbinfo("No nodes in experiment. Exiting ...\n");
    exit(0);
}

#
# Create this "structure" to pass around to the type specific modules.
145
#
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
my $MyStruct = libossetup->New($this_user, $experiment, @nodelist);
if (!defined($MyStruct)) {
    die_noretry("Could not create local data structure!");
}
$MyStruct->debug($debug);
$MyStruct->impotent($impotent);
$MyStruct->noretry(0);
$MyStruct->dolastload($dolastload);

#
# See if the experiment is firewalled and stash for later.
#
$MyStruct->firewalled($experiment->IsFirewalled());

if ($MyStruct->firewalled()) {
    my $firewall;
    my $firewalled = $experiment->IsFirewalled(\$firewall);
    $MyStruct->firewall($firewall);
}
165 166 167 168

#
# Ditto ElabinElab.
#
169
$MyStruct->elabinelab($experiment->elabinelab());
170

171 172 173 174 175
#
# Ditto PlabinElab.
#
my $plcnode;
if (TBExptPlabInElabPLC($pid, $eid, \$plcnode)) {
176 177 178 179 180
    $MyStruct->plabinelab(1);
    $MyStruct->plcnode($plcnode);
}
else {
    $MyStruct->plabinelab(0);
181 182
}

183 184 185 186 187 188
#
# External node management means that someone else is going to be
# answering DHCP after nodes reboot. For nodes in PXEWAIT, we need
# to make sure they are really rebooted and not just told to check
# in with bootinfo again.
#
189 190
if (EmulabFeatures->FeatureEnabled("ExternalNodeManagement", 
				   undef, undef, $experiment)) {
191 192 193
    $MyStruct->realreboot(1);
}

194
#
195 196
# First pass to check that all local files exist. This should probably
# happen earlier in the swap path.
197
#
198 199 200
foreach my $node (@nodelist) {
    if (defined($node->def_boot_path())) {
	my $path = $node->def_boot_path();
201

202 203 204 205 206 207 208 209 210 211 212
	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
213
		die_noretry("File $path for node $node must reside in $TFTP");
214 215
	    }

216
	    if (! -f $path) {
217
		die_noretry("File $path for node $node does not exist!");
218 219
	    }
	}
220
    }
221 222
    if (defined($node->next_boot_path())) {
	my $path = $node->next_boot_path();
223

224 225 226 227 228 229 230 231 232 233 234
	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
235
		die_noretry("File $path for node $node must reside in $TFTP");
236 237
	    }

238
	    if (! -f $path) {
239
		die_noretry("File $path for node $node does not exist!");
240 241
	    }
	}
242 243
    }

244 245 246
    #
    # XXX - Ditto for RPMs.
    #
247
    foreach my $rpm (split(":", $node->rpms())) {
248
	if (! -f $rpm) {
249 250 251
	    die_noretry({type => 'primary', severity => SEV_ERROR,
			 error => ['file_not_found', 'rpm', $rpm, $node]},
			"RPM $rpm for node $node does not exist!");
252 253
	}
    }
254

255 256 257
    #
    # XXX - Ditto for tarfiles.
    #
258
    foreach my $tarspec (split(":", $node->tarballs())) {
259
	my ($dir, $tar) = split(" ", $tarspec);
260

261
	if (! -f $tar) {
262 263 264
	    die_noretry({type => 'primary', severity => SEV_ERROR,
			 error => ['file_not_found', 'tar', $tar, $node]},
			"Tarfile $tar for node $node does not exist!");
265 266
	}
    }
267
}
268

269
#
270 271
# First pass through to let type/class specific modules see what is
# going on and mark nodes as needed.
272
#
273 274 275 276 277
foreach my $node (@nodelist) {
    my $node_id   = $node->node_id();
    my $type      = $node->type();
    my $class     = $node->class();
    my $imageable = $node->imageable();
278

279 280 281
    # Not sure where to put this.
    $node->_issharednode(defined($node->sharing_mode()) &&
			$node->sharing_mode() eq 'using_shared_local');
282

283 284 285 286
    # Not sure where to put this.
    $node->_iseinenode($MyStruct->elabinelab() &&
		      defined($node->inner_elab_role()) &&
		      $node->inner_elab_role() eq 'node');
287 288

    #
289 290 291
    # Look for type specific module first. Eventually this should be more
    # dynamic in how the modules are loaded/defined, perhaps specified on
    # a per-type basis in the DB.
292
    #
293 294 295 296 297 298 299 300 301 302 303
    my $object = $MyStruct->TypeLookup($node);
    if (!defined($object)) {
	$object = $MyStruct->NewType($type);
	if (!defined($object)) {
	    #
	    # Otherwise use the class.
	    #
	    $object = $MyStruct->NewType($class);
	    if ($@) {
		die_noretry("No type/class specific setup module for $node");
	    }
304 305
	}
    }
306 307 308
    print STDERR "Adding $node_id to type object " . $object->type() . "\n"
	if ($debug);
    $object->AddNode($node);
309 310
}

311 312 313 314
while (1) {
    my $objects    = $MyStruct->OperationList();
    my @volunteers = ();
    my @nodes      = ();
315 316

    #
317
    # Do not bother if we got canceled.
318
    #
319 320
    if (! $MyStruct->canceled()) {
	my $canceled = $experiment->canceled();
321
	if ($canceled) {
322 323
	    $MyStruct->canceled($canceled);
	    
324 325 326
	    tbnotice({cause => 'canceled', severity => SEV_IMMEDIATE,
		      error => ['cancel_flag']},
		     "Swap canceled; will terminate os_setup early!");
327
	    last;
Chad Barb's avatar
Chad Barb committed
328
	}
329
    }
330
    
331
    #
332 333
    # Clear the inform lists, since we want to send email in batches
    # as things fail.
334
    #
335
    $MyStruct->ClearInformLists();
336

337
    #
338
    # Go through and ask each one for volunteers. 
339
    #
340 341 342 343 344 345
    foreach my $object (@{ $objects }) {
	print "Asking $object for volunteers\n"
	    if ($debug);
	my @list = $object->Volunteers();
	print "$object returns volunteers: @list\n"
	    if ($debug && @list);
346
	next
347 348 349
	    if (! @list);
	@nodes = (@nodes, @list);
	push(@volunteers, [$object, \@list]);
350
    }
351 352
    last
	if (!@nodes);
353

354
    #
355
    # Light up the nodes in parallel.
356
    #
357 358 359 360 361
    my @results   = ();
    my $coderef   = sub {
	my ($ref) = @_;
	my ($object, $noderef) = @{ $ref };
	my @nodelist = @{ $noderef };
362

363 364 365 366
	print STDERR "Lighting up nodes: @nodelist\n"
	    if ($debug);
	if ($object->LightUpNodes(@nodelist)) {
	    return -1;
367
	}
368 369 370 371 372 373 374
	return 0;
    };
    print STDERR "Lighting up nodes in parallel ...\n";
    
    if (ParRun({"maxwaittime" => 99999},
	       \@results, $coderef, @volunteers)) {
	$MyStruct->die_noretry("*** LightUpNodes: Internal error\n");
375
    }
376

377
    #
378 379
    # Check the exit codes. An error at this phase is unusual, and
    # we want to turn off retry.
380
    #
381 382 383 384 385
    my $errors = 0;
    my $count  = 0;
    foreach my $result (@results) {
	my ($object, $noderef) = @{ $volunteers[$count] };
	my @nodelist = @{ $noderef };
Kirk Webb's avatar
Kirk Webb committed
386

387 388 389 390
	if ($result != 0) {
	    print STDERR "*** Error lighting up nodes: @nodelist\n"
		if ($debug);
	    $MyStruct->noretry(1);
391

392
	    #
393 394
	    # Make sure all the nodes are marked as down so that
	    # we do not wait for them.
395
	    #
396 397
	    foreach my $node (@nodelist) {
		$node->SetAllocState(TBDB_ALLOCSTATE_DOWN());
398
	    }
399
	}
400
	$count++;
401
    }
402

403 404 405 406 407 408 409 410 411
    # XXX: Poke at stated to move along nodes that are going to be
    # externally managed.  This feels kind of kludgy.
    if (EmulabFeatures->FeatureEnabled("ExternalNodeManagement", 
				       undef, undef, $experiment)) {
	foreach my $node (@nodes) {
	    $node->SetEventState(TBDB_NODESTATE_BOOTING());
	}
    }

412 413 414
    # And wait. 
    print STDERR "Waiting for nodes ...\n";
    $MyStruct->WaitForNodes(@nodes);
415 416

    #
417
    # Fire off email for this batch.
418
    #
419 420 421 422 423 424 425
    $MyStruct->InformTBopsFatal();
    $MyStruct->InformTBopsWarn();
    $MyStruct->InformUser();

    if ($MyStruct->aborted()) {
	print STDERR "Aborting os_setup cause of fatal errors.\n";
	last;
426
    }
427 428
}

429 430 431
########################################################################
# All of this stuff is for summary reporting. I did not touch it, as
# the code is simply awful. 
432
#
433
# Global variables need for the summary
434
#
435 436 437 438
my $users_fault;
my %tally;
my %total;
my $summary = '';
439

440 441 442 443
sub add_defaults($) {
    my ($d) = (@_);
    $d->{failed_fatal}    = 0 unless defined $d->{failed_fatal};
    $d->{failed_nonfatal} = 0 unless defined $d->{failed_nonfatal};
444
}
445 446 447 448 449 450 451 452 453 454

sub add_non_fatal($%) {
    my ($line, %d) = @_;
    if ($d{failed_nonfatal} > 0) {
	my $count = ($d{failed_nonfatal} == $d{failed}
		     ? "all"
		     : "$d{failed_nonfatal}/$d{failed}");
	$line .= " ($count non-fatal)";
    }
    return $line;
455 456
}

Kevin Atkinson's avatar
Kevin Atkinson committed
457
sub list_failed_nodes ($%) {
458
    local $^W = 0;
Kevin Atkinson's avatar
Kevin Atkinson committed
459 460 461 462 463 464 465
    my ($max_length,%d) = @_;
    my $byvname = sub { $vname{$a} cmp $vname{$b} };
    my @nodes = (sort $byvname @{$d{failed_fatal_list}}, 
		 sort $byvname @{$d{failed_nonfatal_list}});
    @nodes = map {"$vname{$_}($_)"} @nodes;
    my $line = join ' ', @nodes;
    if (length($line) > $max_length) {
466
	$line = '';
Kevin Atkinson's avatar
Kevin Atkinson committed
467 468 469
	$max_length -= 4;
	my $length = 0;
	foreach (@nodes) {
470
	    $length += length($_) + 1;
Kevin Atkinson's avatar
Kevin Atkinson committed
471 472 473
	    last if $length > $max_length;
	    $line .= "$_ ";
	}
474
	$line .= "..." if $length > $max_length;
Kevin Atkinson's avatar
Kevin Atkinson committed
475
    }
476
    return $line;
Kevin Atkinson's avatar
Kevin Atkinson committed
477 478 479 480 481 482 483 484 485 486 487 488
}

sub add_failed_nodes ($$%) {
    my ($line, $indent, %d) = @_;
    my $nodes_line = list_failed_nodes(78 - $indent, %d);
    if (length($line) + 2 + length($nodes_line) > 78) {
	return "$line:\n".(' 'x$indent)."$nodes_line\n";
    } else {
	return "$line: $nodes_line\n";
    }
}

489 490 491
#
# First gather stats
#
492 493 494
foreach (keys(%{ $MyStruct->failedlist() })) {
    my $node   = $MyStruct->node($_);
    my $osinfo = $node->_bootosinfo();
495
    my $osid   = $osinfo->osid();
496 497
    my $type   = $node->type();
    my ($what,$fatal)  = @{ $MyStruct->failedlist()->{$_} };
498

499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
    my ($error_type, $severity);

    if ($what eq 'boot') {
	$error_type = 'node_boot_failed';
    } elsif ($what eq 'reload') {
	$error_type = 'node_load_failed';
    }

    if ($fatal eq 'fatal') {
	$severity = SEV_ERROR;
    } elsif ($fatal eq 'nonfatal') {
	$severity = SEV_WARNING;
    }

    if (defined($error_type) && defined($severity)) {
514
	tbreport($severity, $error_type, $node, $type, $osinfo);
515 516
    }

517 518
    $tally{$what}{$osid} = {} unless defined $tally{$what}{$osid};
    my $t = $tally{$what}{$osid};
519

520 521
    $t->{any_type}{failed}++;
    $t->{any_type}{"failed_${fatal}"}++;
522

523 524
    $t->{by_type}{$type}{failed}++;
    $t->{by_type}{$type}{"failed_${fatal}"}++;
525

526 527
    push @{$t->{any_type}{"failed_${fatal}_list"}}, $_;
    push @{$t->{by_type}{$type}{"failed_${fatal}_list"}}, $_;
528 529 530 531 532 533 534 535 536 537

    #
    # Image success/failure tracking but only on first load since we do
    # not know what the user has done after that.
    #
    if ($node->_setupoperation() eq $RELOAD) {
	my $image = $node->_loadimage();
	$image->RecordImageStatus($experiment, $node,
				  ($what eq 'boot' ? "bootfail" : "reloadfail"));
    }
538
}
539 540 541 542 543 544 545 546
foreach (keys(%{ $MyStruct->nodelist() })) {
    my $node   = $MyStruct->node($_);
    my $osinfo = $node->_bootosinfo();

    # Was not setup to do anything, so ignore.
    next
	if (!defined($osinfo));
    
547
    my $osid   = $osinfo->osid();
548
    my $type   = $node->type();
549 550 551 552 553 554 555 556 557 558 559 560
    $total{$osid}{any_type}++;
    $total{$osid}{by_type}{$type}++;
}

#
# Now report any failed nodes in a concise summary
#
if (defined $tally{reload}) {

    $users_fault = 0;

    foreach my $osid (sort keys %{$tally{reload}}) {
561 562
	my $osimage = OSImage->Lookup($osid);
	my $osname  = $osimage->imagename();
563 564 565 566 567
	
	my %d     = %{$tally{reload}{$osid}{any_type}};
	my $total = $total{$osid}{any_type};
	
	my $line;
568 569
	$line = sprintf("%d/%d nodes failed to load the os \"%s\"",
			$d{failed}, $total, $osname);
570 571 572 573 574 575 576 577 578 579
	$line = add_failed_nodes($line, 2, %d);
	
	$summary .= $line;
    }

} elsif (defined $tally{boot}) {

    $users_fault = 1;

    foreach my $osid (sort keys %{$tally{boot}}) {
580 581 582
	my $osimage    = OSImage->Lookup($osid);
	my $osname     = $osimage->imagename();
	my $user_image = ($osimage->pid() eq TBOPSPID() ? 0 : 1);
583 584 585 586 587 588 589
	
	add_defaults($tally{boot}{$osid}{any_type});
	my %d   = %{$tally{boot}{$osid}{any_type}};
	my %d_t = %{$tally{boot}{$osid}{by_type}};
	my $total   = $total{$osid}{any_type};
	my %total_t = %{$total{$osid}{by_type}};
	
Kevin Atkinson's avatar
Kevin Atkinson committed
590
	my $byfailure = sub {
591
	    my $cmp = $d_t{$b}{failed} <=> $d_t{$a}{failed};
Kevin Atkinson's avatar
Kevin Atkinson committed
592 593 594
	    return $cmp if $cmp != 0;
	    return $a cmp $b;
	};
595 596
	my @node_types = sort $byfailure keys %d_t;
	
597
	$users_fault = 0 if !$user_image;
Kevin Atkinson's avatar
Kevin Atkinson committed
598
	foreach my $type (@node_types) {
599
	    $users_fault = 0 if $d_t{$type}{failed} < $total_t{$type};
Kevin Atkinson's avatar
Kevin Atkinson committed
600 601
	}
	
602 603 604 605 606 607 608
	my $line = sprintf("%d/%d %s with a %s osid of \"%s\" failed to boot",
			   $d{failed}, $total,
			   @node_types == 1 ? "$node_types[0]'s" : "nodes",
			   $user_image ? "user" : "system", 
			   $osname);
	$line = add_non_fatal($line, %d);
	
Kevin Atkinson's avatar
Kevin Atkinson committed
609
	if (@node_types == 1) {
610
	    
Kevin Atkinson's avatar
Kevin Atkinson committed
611
	    my $type = $node_types[0];
612 613
	    
	    $summary .= add_failed_nodes($line, 2, %{$d_t{$type}});
614

Kevin Atkinson's avatar
Kevin Atkinson committed
615
	} else {
616
	    
Kevin Atkinson's avatar
Kevin Atkinson committed
617
	    $summary .= "$line:\n";
618
	    
Kevin Atkinson's avatar
Kevin Atkinson committed
619
	    foreach my $type (@node_types) {
620

621
		add_defaults($d_t{$type});
Kevin Atkinson's avatar
Kevin Atkinson committed
622
		my %d = %{$d_t{$type}};
623 624
		my $total = $total_t{$type};
		
Kevin Atkinson's avatar
Kevin Atkinson committed
625
		if ($d{failed} > 0) {
626 627 628
		    $line = sprintf("  %d/%d %s with this os failed to boot",
				    $d{failed}, $total,
				    "${type}'s");
629
		    $line = add_non_fatal($line, %d);
630
		    $line = add_failed_nodes($line, 4, %d);
Kevin Atkinson's avatar
Kevin Atkinson committed
631
		} else {
632
		    $line = sprintf("  %d %s with this os successfully booted.\n",
633 634
				    $total,
				    $total_t{$type} == 1 ? "$type" : "${type}'s");
635
		}
636
		$summary .= $line;
637 638 639 640
	    }
	}
    }
}
641
if (my $count = $MyStruct->failed()) {
642
    tberror ({type=>'summary', cause=>($users_fault ? 'user' : 'unknown')}, 
643 644 645 646
	     "There were $count failed nodes.\n\n", $summary);
}
elsif ($summary) {
    tbwarn($summary);
647
}
648

649 650 651
# Look to see if anyone set the no retry flag.
my $exit_code = (($experiment->canceled() || $MyStruct->noretry()) ? -1 :
		 $MyStruct->failed() ? 1 : 0);
652 653 654 655 656

#
# If not failing for any reason, record some stats
#
if ($exit_code == 0) {
657 658 659
    # Record some stats on the OS requested and the images loaded to
    # the image_history table. Put in in an eval loop to catch any
    # errors so they are non-fatal.
660 661 662 663 664 665 666 667 668
    my %todo;

    # Collect the list of nodes and store the osid requested
    # and the imageid loaded (if any).
    foreach my $object (@{$MyStruct->OperationList()}) {
	foreach my $node_id (keys %{$object->{NODES}}) {
	    my $node = $object->{NODES}{$node_id};
	    my $osinfo = $node->_bootosinfo();
	    my $op = $node->_setupoperation();
669
	    my $image;
670 671
	    # Only set the imageid if the disk needed a reload
	    if ($op == $RELOAD) {
672
		$image = $node->_loadimage();
673
	    }
674
	    $todo{$node_id} = [$osinfo, $image];
675 676 677 678 679 680 681 682

	    #
	    # Image success/failure tracking but only on first load since we do
	    # not know what the user has done after that.
	    #
	    if (defined($image)) {
		$image->RecordImageStatus($experiment, $node, "success");
	    }
683 684
	}
    }
685 686
    OSImage->RecordImageHistory($experiment, 'os_setup',
				tblog_session(), $impotent, \%todo);
687 688
}

689 690 691 692 693 694 695 696
#
# If not failing for any reason, save off swap state.
#
# For all nodes in the experiment that are booting from the disk,
# figure out the image from which they are booting and stash away the
# appropriate info to enable disk state saving at swapout.
#
my $swapstate;
697
if ($exit_code == 0 &&
698 699 700 701 702
    TBExptGetSwapState($pid, $eid, \$swapstate) && $swapstate) {
    TBDebugTimeStamp("Stashing image signatures");
    osload_setupswapinfo($pid, $eid);
    TBDebugTimeStamp("Finished stashing image signatures");
}
703
TBDebugTimeStamp("os_setup finished");
704

705
exit($exit_code);