tbswap.in 20.6 KB
Newer Older
Chad Barb's avatar
   
Chad Barb committed
1
2
3
4
5
6
7
8
9
#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
use English;

Chad Barb's avatar
   
Chad Barb committed
10
11
12
13
# Returns 0 on success.
# Returns 1 on non-assign_wrapper failure.
# Returns (1 | assign_wrapper's errorcode) on assign_wrapper failure.

Chad Barb's avatar
   
Chad Barb committed
14
15
16
17
18
# XXX: handle error cases for update? (backup the db?)
# XXX: Shouldn't do idempotent stuff twice for update.
# XXX: repush/calc routing for update??? (tbprerun)
# XXX: previz for update???              (tbprerun)
# XXX: make snmpit faster for update.
Chad Barb's avatar
Chad Barb committed
19
20
21
#
# XXX: for update, expt is swapped out on os_setup fail.
#      (we only recover if assign fails)
Chad Barb's avatar
   
Chad Barb committed
22
23
24

sub usage()
{
Chad Barb's avatar
Chad Barb committed
25
    print STDERR "Usage: $0 { in | out [-force] | update [-reboot] } pid eid\n";
Chad Barb's avatar
   
Chad Barb committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    exit(-1);
}

#
# Configure variables
#
my $TBROOT         = "@prefix@";
my $TESTMODE       = @TESTMODE@;
my $DISABLE_EVENTS = "@DISABLE_EVENT_SCHED@";

# Untaint the path
$ENV{'PATH'} = "/usr/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
    ":$TBROOT/sbin:$TBROOT/bin";

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
require exitonwarn; # exitonwarn isn't really a module, so just require it

#
# Actual swap-in and swap-out functions, defined below.
#
Chad Barb's avatar
   
Chad Barb committed
51
52
53
54
55
56
57
sub doSwapout($);
sub doSwapin($);

sub REAL()    { return 4; }
sub CLEANUP() { return 3; }
sub RETRY()   { return 2; }
sub UPDATE()  { return 1; }
Chad Barb's avatar
Chad Barb committed
58
sub UPDATE_RECOVER()  { return 0; }
Chad Barb's avatar
   
Chad Barb committed
59
60
61
62
63
64
65

#
# Turn off line buffering on output
#

$| = 1;

Chad Barb's avatar
Chad Barb committed
66
my $updateReboot = 0;
Chad Barb's avatar
   
Chad Barb committed
67
68
my $force  = 0;
my $errors = 0;
Chad Barb's avatar
Chad Barb committed
69
my $assignWrapperErrorCode = 0;
Chad Barb's avatar
   
Chad Barb committed
70

Chad Barb's avatar
   
Chad Barb committed
71
72
73
74
75
76
my $state;

my $os_setup_pid;
my $cleanvlans;

#
Chad Barb's avatar
   
Chad Barb committed
77
# First argument is either "in", "out", or "update";
Chad Barb's avatar
   
Chad Barb committed
78
79
80
81
82
# this value goes into $swapop.
#

my $swapop = shift;	

Chad Barb's avatar
   
Chad Barb committed
83
84
85
86
if (!$swapop || 
    (($swapop ne "in") && 
     ($swapop ne "out") &&
     ($swapop ne "update"))) {
Chad Barb's avatar
   
Chad Barb committed
87
88
89
90
91
92
93
94
95
96
97
    usage();
}

#
# Get other arguments.
#

while ($#ARGV > 1) {
    $arg = shift;
    if ($arg eq "-force") {
	$force = 1;
Chad Barb's avatar
Chad Barb committed
98
99
    } elsif ($arg eq "-reboot") {
	$updateReboot = 1;
Chad Barb's avatar
   
Chad Barb committed
100
101
102
103
104
105
106
107
108
    } else {
	usage();
    }
}
if ($#ARGV < 1) {
    usage();
}
my ($pid,$eid) = @ARGV;

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\@\w.]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}

Chad Barb's avatar
   
Chad Barb committed
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
TBDebugTimeStampsOn();

#
# Print starting message.
#

print "Beginning swap-$swapop for $pid/$eid. " . TBTimeStamp() . "\n";
TBDebugTimeStamp("tbswap $swapop started");

#
# Get experiment state; verify that experiment exists.
#

if (! ($state = ExpState($pid, $eid))) {
    print STDERR "*** No such experiment $pid/$eid\n";
    $errors = 1;
}

#
# Figure out which state we're looking for 
# and which state we're going to set according to the operation.
#

my $desiredState;
my $nextState;
my $suggestion;
my $suggestUseOfForce;

if ($swapop eq "in") {
    $desiredState = EXPTSTATE_SWAPPED;
    $nextState    = EXPTSTATE_ACTIVATING;
    $suggestion   = "Must be swapped out.";
    $suggestUseOfForce = 0;
Chad Barb's avatar
   
Chad Barb committed
158
} elsif ($swapop eq "out") {
Chad Barb's avatar
   
Chad Barb committed
159
160
161
162
163
164
165
166
167
    if (! $TESTMODE) {
	$desiredState = EXPTSTATE_ACTIVE;
	$suggestion   = "Must be running.";
    } else {
	$desiredState = EXPTSTATE_TESTING;
	$suggestion   = "Must be in testing state.";
    }
    $nextState    = EXPTSTATE_SWAPPING;
    $suggestUseOfForce = 1;
Chad Barb's avatar
   
Chad Barb committed
168
169
170
171
172
} elsif ($swapop eq "update") {
    $desiredState = EXPTSTATE_ACTIVE;
    $nextState    = EXPTSTATE_ACTIVATING; # XXX add an updating state?
    $suggestion   = "Must be running.";
    $suggestUseOfForce = 0;
Chad Barb's avatar
   
Chad Barb committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
}

#
# Check that experiment is in correct state.
# If it is not, print appropriate error message.
#

if (! $errors) {
    if ($state ne $desiredState) {
	if (! $force) {
	    if ($state eq EXPTSTATE_SWAPPED) {
		print STDERR "*** Experiment is already swapped out. $suggestion\n";
	    } elsif ($state eq EXPTSTATE_ACTIVE) {
		print STDERR "*** Experiment is already running. $suggestion\n";
	    } else {
		print STDERR "*** Experiment is in the wrong state: $state. $suggestion\n";
	    }
	    if ($suggestUseOfForce) {
		print STDERR "    Try using -force to ignore improper state.\n";
	    }
	    $errors = 1;
	} else {
	    print STDERR "*** WARNING: Ignoring improper state: $state.\n";
	}
    }
}

#
# Set intermediate experiment state.
#

if (! $errors) {
    if (! SetExpState($pid, $eid, $nextState)) {
	print STDERR "*** Failed to set intermediate experiment state.\n";
	$errors = 1;
    }
}

#
# Do actual swapping
#

Chad Barb's avatar
   
Chad Barb committed
215
undef $nextState;
Chad Barb's avatar
   
Chad Barb committed
216
217

if (! $errors) {
Chad Barb's avatar
   
Chad Barb committed
218
    if ($swapop eq "out") {
Chad Barb's avatar
   
Chad Barb committed
219
	#
Chad Barb's avatar
   
Chad Barb committed
220
	# Swap out
Chad Barb's avatar
   
Chad Barb committed
221
	#
Chad Barb's avatar
   
Chad Barb committed
222
223
224
225
226
227
228
	$errors = doSwapout(REAL);
	if (! $errors) {
	    $nextState = EXPTSTATE_SWAPPED;
	} else {
	    # leave $nextState undefined
	    # experiment will stay in SWAPPING.
	}
Chad Barb's avatar
   
Chad Barb committed
229
230

	#
Chad Barb's avatar
   
Chad Barb committed
231
232
233
234
235
	# Update Accounting Information
	#
	TBSetExpSwapTime($pid, $eid);
    }
}
Chad Barb's avatar
   
Chad Barb committed
236

Chad Barb's avatar
Chad Barb committed
237
238
239
#
# Update, Phase One -- swap experiment partially out.
#
Chad Barb's avatar
   
Chad Barb committed
240
241
if (! $errors) {
    if ($swapop eq "update") {
Chad Barb's avatar
Chad Barb committed
242
	print STDERR "Backing up physical state...\n";
243
	TBExptBackupPhysicalState($pid,$eid);
Chad Barb's avatar
Chad Barb committed
244

Chad Barb's avatar
   
Chad Barb committed
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
	$errors = doSwapout(UPDATE);
	if (! $errors) {
	    # leave $nextState undefined. 
	    # (this will get defined below.)
	} else {
	    # Error; Clean up and put experiment in SWAPPED.
	    print STDERR "Cleaning up after errors.\n";
	    doSwapout(CLEANUP);
	    $nextState = EXPTSTATE_SWAPPED;
	}
	
	#
	# Update Accounting Information
	#
	TBSetExpSwapTime($pid, $eid);
    }
}

Chad Barb's avatar
Chad Barb committed
263
264
265
266
267
#
# Update, Phase Two -- swap experiment back in.
#
if (! $errors) { 
    if ($swapop eq "update") {
Chad Barb's avatar
   
Chad Barb committed
268
	#
Chad Barb's avatar
Chad Barb committed
269
	# Update
Chad Barb's avatar
   
Chad Barb committed
270
	#
Chad Barb's avatar
Chad Barb committed
271
272
273
274
275
276
277
	$errors = doSwapin(UPDATE);

	if (! $errors) {
	    #
	    # Update Accounting Information
	    #
	    TBSetExpSwapTime($pid, $eid);
Chad Barb's avatar
   
Chad Barb committed
278

Chad Barb's avatar
Chad Barb committed
279
280
281
282
283
284
285
286
287
	    #
	    # Swapin worked; exp is now ACTIVE.
	    # (or TESTING) 
	    #
	    if ($TESTMODE) {
		$nextState = EXPTSTATE_TESTING;
	    } else {
		$nextState = EXPTSTATE_ACTIVE;
	    }
Chad Barb's avatar
   
Chad Barb committed
288
	} else {
Chad Barb's avatar
Chad Barb committed
289
290
291
292
293
294
295
296
297
298
299
300
	    #
	    # There were errors; see if we can recover.
	    #
	    my $canStillRecover = 1;

	    if ($errors != 7) {
		print STDERR "Update failure occurred _after_ assign phase; ";
		$canStillRecover = 0;
	    }

	    if ($canStillRecover) {
		print STDERR "Recovering physical state...\n";
301
		my $recoverErrors = TBExptRestorePhysicalState($pid,$eid);
Chad Barb's avatar
Chad Barb committed
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
		if ($recoverErrors) {
		    print STDERR "Could not restore backed-up physical state; ";
		    $canStillRecover = 0;
		}
	    }

	    if ($canStillRecover) {
		print STDERR "Doing a recovery swap-in of old state...\n";
		$recoverErrors = doSwapin(UPDATE_RECOVER);
		if ($recoverErrors) {
		    print STDERR "Could not swap in old physical state; ";
		    $canStillRecover = 0;
		}
	    }

	    if ($canStillRecover) {
		print STDERR "Update recovery successful.\n";
		$nextState = EXPTSTATE_ACTIVE;
	    } else {
		print STDERR "update recovery aborted!\n".
		             "Swapping experiment out...\n";
		doSwapout(CLEANUP);

		$nextState = EXPTSTATE_SWAPPED;
	    }
Chad Barb's avatar
   
Chad Barb committed
327
	}
Chad Barb's avatar
Chad Barb committed
328
329
330
331
332
333
334
335
336
    }
}

if (! $errors) {
    if ($swapop eq "in") {
	#
	# Swap in
	#
	$errors = doSwapin(REAL);
Chad Barb's avatar
   
Chad Barb committed
337

Chad Barb's avatar
   
Chad Barb committed
338
	my $retries = 2;
Chad Barb's avatar
   
Chad Barb committed
339
340
341
342

	#
	# Attempt a retry if: 
	#   a) there were errors, 
Chad Barb's avatar
Chad Barb committed
343
	#   b) doswapin() indicated (via return code 3) a retry is appropriate,
Chad Barb's avatar
   
Chad Barb committed
344
345
	#   c) we haven't tried too many times already.
	#
Chad Barb's avatar
   
Chad Barb committed
346
347
348
	# NOTE that if $TESTMODE == 1, retries are impossible, since
        # $retry will never get set to 1.
	#
Chad Barb's avatar
Chad Barb committed
349
	while ($errors == 3 && $retries ) {
Chad Barb's avatar
   
Chad Barb committed
350
351
352
	    $retries--;

	    print STDERR "Cleaning up after errors; will try again.\n";
Chad Barb's avatar
   
Chad Barb committed
353
	    doSwapout(RETRY);
Chad Barb's avatar
   
Chad Barb committed
354
355

	    print STDERR "Trying again...\n";
Chad Barb's avatar
   
Chad Barb committed
356
	    $errors = doSwapin(RETRY);
Chad Barb's avatar
   
Chad Barb committed
357
	}
Chad Barb's avatar
Chad Barb committed
358

Chad Barb's avatar
   
Chad Barb committed
359
360
361
362
363
364
365
366
	if (! $errors) {
	    #
	    # Update Accounting Information
	    #
	    TBSetExpSwapTime($pid, $eid);

	    #
	    # Swapin worked; exp is now ACTIVE.
Chad Barb's avatar
Chad Barb committed
367
	    # (or TESTING)
Chad Barb's avatar
   
Chad Barb committed
368
	    #
Chad Barb's avatar
   
Chad Barb committed
369
370
371
372
373
	    if ($TESTMODE) {
		$nextState = EXPTSTATE_TESTING;
	    } else {
		$nextState = EXPTSTATE_ACTIVE;
	    }
Chad Barb's avatar
Chad Barb committed
374
	} else {
Chad Barb's avatar
   
Chad Barb committed
375
	    print STDERR "Cleaning up after errors.\n";
Chad Barb's avatar
   
Chad Barb committed
376
	    doSwapout(CLEANUP);
Chad Barb's avatar
   
Chad Barb committed
377
378

	    #
Chad Barb's avatar
Chad Barb committed
379
	    # Regardless of how well cleanup swapout worked,
Chad Barb's avatar
   
Chad Barb committed
380
381
382
383
384
385
386
387
388
389
	    # send exp to SWAPPED.
	    #
	    $nextState = EXPTSTATE_SWAPPED; 
	}
    }
}

if (defined $nextState) {
    if (! SetExpState($pid, $eid, $nextState)) {
	print STDERR "*** Failed to set experiment state.\n";
Chad Barb's avatar
   
Chad Barb committed
390
	$errors = 1;
Chad Barb's avatar
   
Chad Barb committed
391
392
393
394
395
396
397
398
399
400
    }
}

#
# Write appropriate message and exit.
#

if ($errors) {
    print "Failingly finished swap-$swapop for $pid/$eid. " . TBTimeStamp() . "\n";
    TBDebugTimeStamp("tbswap $swapop finished (failed)");
Chad Barb's avatar
Chad Barb committed
401

Chad Barb's avatar
   
Chad Barb committed
402
403
404
405
    # pass assign wrapper info along.
    # other codes in 'errors' (3 or 7) are meaningless and
    # should just be reported as 1's.
    exit( 1 | $assignWrapperErrorCode );
Chad Barb's avatar
   
Chad Barb committed
406
407
408
} else {
    print "Successfully finished swap-$swapop for $pid/$eid. " . TBTimeStamp() . "\n";
    TBDebugTimeStamp("tbswap $swapop finished (succeeded)");
Chad Barb's avatar
   
Chad Barb committed
409
    exit( 0 );
Chad Barb's avatar
   
Chad Barb committed
410
411
412
413
414
}

#################################

##
Chad Barb's avatar
Chad Barb committed
415
#
Chad Barb's avatar
   
Chad Barb committed
416
417
# doSwapout - Swaps experiment out.
#
Chad Barb's avatar
Chad Barb committed
418
#             If in REAL or CLEANUP,
Chad Barb's avatar
   
Chad Barb committed
419
420
421
#             this function will free all nodes for the 
#             experiment.
#
Chad Barb's avatar
Chad Barb committed
422
#             If in RETRY or UDPATE,
Chad Barb's avatar
   
Chad Barb committed
423
424
425
426
427
428
#             only nodes not in RES_READY will be freed.
#
#             Returns 0 on success, >0 on failure.
#
##

Chad Barb's avatar
   
Chad Barb committed
429
430
sub doSwapout($) {
    my $type = shift; # REAL==4, CLEANUP==3, RETRY==2, UPDATE==1.
Chad Barb's avatar
   
Chad Barb committed
431
432
433
434
435
436
437
438
439
440
441
    my $swapout_errors = 0;

    #
    # wait for os_setup;
    # this only applies if called after a failed doswapin.
    #
    if ($os_setup_pid) {
	print "Waiting for os_setup to finish\n";
	waitpid($os_setup_pid, 0);
	undef $os_setup_pid;
    }
Chad Barb's avatar
Chad Barb committed
442

Chad Barb's avatar
   
Chad Barb committed
443
444
    if (! $TESTMODE) { 
	if (! $DISABLE_EVENTS) {
Chad Barb's avatar
   
Chad Barb committed
445
446
447
448
449
450
	    if ($type >= RETRY) {
		print "Stopping the event system\n";
		if (system("eventsys_control stop $pid $eid")) {
		    print STDERR "*** Failed to stop the event system.\n";
		    $swapout_errors = 1;
		}
Chad Barb's avatar
   
Chad Barb committed
451
452
	    }
	}
Chad Barb's avatar
Chad Barb committed
453

Chad Barb's avatar
   
Chad Barb committed
454
	#
Chad Barb's avatar
Chad Barb committed
455
	# Clean up any VLANs in experiment.
Chad Barb's avatar
   
Chad Barb committed
456
	#
Chad Barb's avatar
Chad Barb committed
457
458
459
460
461
462
463
	TBDebugTimeStamp("snmpit started");
	print STDERR "Removing VLANs.\n";
	if (system("snmpit -r $pid $eid")) {
	    print STDERR "*** Failed to reset VLANs\n";
	    $swapout_errors = 1;
	} else {
	    $cleanvlans = 0;
Chad Barb's avatar
   
Chad Barb committed
464
	}
Chad Barb's avatar
Chad Barb committed
465
466
	TBDebugTimeStamp("snmpit finished");

Chad Barb's avatar
   
Chad Barb committed
467
468
469
	#
	# This is a hack. We need a more general os_teardown, but for now
	# we just kill off the vnode stuff. 
Chad Barb's avatar
   
Chad Barb committed
470
	# (don't kill off vnodes when UPDATEing.)
Chad Barb's avatar
   
Chad Barb committed
471
	#
Chad Barb's avatar
   
Chad Barb committed
472
473
474
475
476
477
478
479
	if ($type >= RETRY) {
	    print "Tearing down virtual nodes.\n";
	    TBDebugTimeStamp("vnode_setup -k started");
	    if (system("vnode_setup -d -k $pid $eid")) {
		print STDERR "*** Failed to tear down vnodes.\n";
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("vnode_setup finished");
Chad Barb's avatar
   
Chad Barb committed
480
481
482
	}
    }

Chad Barb's avatar
   
Chad Barb committed
483
    if ($type >= CLEANUP) {
Chad Barb's avatar
   
Chad Barb committed
484
485
486
	#
	# We're not attempting a retry; 
	# remove all nodes from the experiment.
Chad Barb's avatar
   
Chad Barb committed
487
	# (nfree will send them to RES_FREE_DIRTY)
Chad Barb's avatar
   
Chad Barb committed
488
489
490
491
492
493
494
495
	#
	print STDERR "Freeing nodes.\n";
	TBDebugTimeStamp("nfree started");
	if (system("nfree $pid $eid")) {
	    print STDERR "*** Could not free nodes.\n";
	    $swapout_errors = 1;
	}
	TBDebugTimeStamp("nfree finished");
Chad Barb's avatar
   
Chad Barb committed
496
497
498
499
500

	#
	# Since this is an actual swapout, 
	# reset our count of swap out nag emails sent.
	#
501
502
	DBQueryWarn("update experiments set swap_requests='' ".
		    "where eid='$eid' and pid='$pid'");
Chad Barb's avatar
   
Chad Barb committed
503
504
    } else {
	#
Chad Barb's avatar
   
Chad Barb committed
505
	# $type == RETRY or $type == UPDATE.
Chad Barb's avatar
   
Chad Barb committed
506
507
	# Therefore, don't deallocate nodes which have been successfully
	# incorporated into the experiment (i.e., are RES_READY).
Chad Barb's avatar
   
Chad Barb committed
508
	# (nfree will send deallocated nodes to RES_FREE_DIRTY)
Chad Barb's avatar
   
Chad Barb committed
509
510
511
512
513
514
515
516
517
518
	#

	my @failedNodes = ();
	$db_result =
	    DBQueryFatal("select rv.node_id, n.allocstate ".
                         "from reserved as rv ".
			 "left join nodes as n on ".
			 "n.node_id = rv.node_id ".
			 "where rv.pid='$pid' and rv.eid='$eid'");

519
	while (my ($node,$allocstate) = $db_result->fetchrow_array) {
Chad Barb's avatar
   
Chad Barb committed
520
521
522
523
524
	    if ($allocstate ne TBDB_ALLOCSTATE_RES_READY()) {
		push(@failedNodes, $node);
	    }
	}

Chad Barb's avatar
   
Chad Barb committed
525
526
527
	# XXX reboot nodes [e.g., put them in RES_DIRTY()] 
	# if doing an update -restart.

Chad Barb's avatar
   
Chad Barb committed
528
	if (@failedNodes > 0) {
529
530
	    print STDERR "Freeing failed nodes.\n";
	    
Chad Barb's avatar
   
Chad Barb committed
531
	    TBDebugTimeStamp("nfree started");
Chad Barb's avatar
   
Chad Barb committed
532
533
534
535
536
537
	    #
	    # Specify -x switch so when a physical node gets freed,
	    # any virtual nodes (owned by this experiment)
	    # sitting on top of it are freed as well.
	    #
	    if (system("nfree -x $pid $eid " . join(" ", @failedNodes))) {
Chad Barb's avatar
   
Chad Barb committed
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
		print STDERR "*** Could not free nodes.\n";
		$swapout_errors = 1;
	    }
	    TBDebugTimeStamp("nfree finished");
	}
    }

    if (! $TESTMODE) {
	#
	# All of these errors are non-fatal on swapout. We find out about them
	# via email sent from the individual scripts.
	#

	#
	# Only reset mountpoints if this is an actual swapout, and
Chad Barb's avatar
   
Chad Barb committed
553
	# not a failed swapin(cleanup), update, or retry.
Chad Barb's avatar
   
Chad Barb committed
554
	#
Chad Barb's avatar
   
Chad Barb committed
555
	if ($type == REAL) {
Chad Barb's avatar
   
Chad Barb committed
556
557
558
559
560
561
562
	    print "Resetting mountpoints.\n";
	    TBDebugTimeStamp("exports started");
	    if (system("exports_setup")) {
		print STDERR "*** Failed to reset mountpoints.\n";
	    }
	    TBDebugTimeStamp("exports finished");
	}
Chad Barb's avatar
Chad Barb committed
563

Chad Barb's avatar
   
Chad Barb committed
564
565
566
567
	#
	# Resetting named maps and email lists is fast and idempotent,
	# so whatever.
	#
Chad Barb's avatar
   
Chad Barb committed
568
569
570
571
572
573
	print "Resetting named maps.\n";
	TBDebugTimeStamp("named started");
	if (system("named_setup")) {
	    print "*** WARNING: Failed to reset named map.\n";
	}
	TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
574

Chad Barb's avatar
   
Chad Barb committed
575
576
577
578
579
580
581
582
	print "Resetting email lists.\n";
	TBDebugTimeStamp("genelists started");
	if (system("genelists")) {
	    print "*** WARNING: Failed to reset email lists.\n";
	}
	TBDebugTimeStamp("genelists finished");
    }

Chad Barb's avatar
   
Chad Barb committed
583
584
585
    #
    # Wipe the DB clean.
    #
Chad Barb's avatar
Chad Barb committed
586

Chad Barb's avatar
   
Chad Barb committed
587
    print STDERR "Resetting DB.\n";
Chad Barb's avatar
   
Chad Barb committed
588
    TBExptRemovePhysicalState( $pid, $eid );
Chad Barb's avatar
   
Chad Barb committed
589
590
591
592
593

    return $swapout_errors;
}

##
Chad Barb's avatar
Chad Barb committed
594
#
Chad Barb's avatar
   
Chad Barb committed
595
596
# doSwapin - Swaps experiment in.
#
Chad Barb's avatar
Chad Barb committed
597
598
599
600
601
#            Returns:
#              0 - successful swapin
#              1 - failed swapin; cleanup required.
#              3 - failed swapin; cleanup required; can retry.
#              7 - failed swapin; assign failed; no cleanup.
Chad Barb's avatar
   
Chad Barb committed
602
603
604
605
606
#
#            Will set $retry = 1 if os_setup failed.
#
##

Chad Barb's avatar
   
Chad Barb committed
607
sub doSwapin($) {
Chad Barb's avatar
Chad Barb committed
608
    my $type = shift; # REAL==4, RETRY==2, UPDATE==1, UPDATE_RECOVER=0. 
Chad Barb's avatar
   
Chad Barb committed
609

Chad Barb's avatar
   
Chad Barb committed
610
611
612
613
614
    #
    # assign_wrapper does all the virtual to physical mapping 
    # and updating the DB state.
    #

Chad Barb's avatar
Chad Barb committed
615
616
617
618
619
620
621
622
623
624
    if ($type > UPDATE_RECOVER) {
	print "Mapping to physical reality ...\n";
	TBDebugTimeStamp("assign_wrapper started");

	#
	# Pass the -u (update) switch into assign_wrapper,
	# So any nodes already in the experiment are
	# fixed.
	#
	my $exitcode;
625
	my $wrapper = "assign_wrapper";
626
	
627
	if (system("$wrapper -u $pid $eid")) {
Chad Barb's avatar
Chad Barb committed
628
629
	    #
	    # save this off so it will get passed back later.
630
	    # Note that -1 is an uncontrolled error. No recovery.
Chad Barb's avatar
Chad Barb committed
631
	    #
632
633
634
635
636
637
	    $assignWrapperErrorCode = $?;
	    $exitcode = $? >> 8;

	    print STDERR "*** Failed ($exitcode) to map to reality.\n";

	    if (($exitcode & 64) && ($exitcode != 255)) {
Chad Barb's avatar
   
Chad Barb committed
638
		# so batchexp doesn't choke.
639
640
641
		$assignWrapperErrorCode -= 64
		    if ($exitcode != 255);
		# We can recover. 
Chad Barb's avatar
Chad Barb committed
642
		return 7;
643
644
645
	    }
	    else {
		# No recovery.
Chad Barb's avatar
Chad Barb committed
646
647
648
649
		return 1;
	    }
	}
	TBDebugTimeStamp("assign_wrapper finished");
Chad Barb's avatar
   
Chad Barb committed
650

Chad Barb's avatar
Chad Barb committed
651
652
	print "Mapped to physical reality!\n";
    }
Chad Barb's avatar
   
Chad Barb committed
653

654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
    #
    # Look for any nodes in RES_TEARDOWN. These need to be released,
    # and if a virtnode, they need to be torn down. We cannot wait for
    # the virtnodes to go down with the physnode they are hosted on,
    # so teardown and release the virtnodes first, and then do the
    # physnodes.
    #
    # Errors are fatal; no recovery or retry.
    #
    if ($type == UPDATE) {
	my $allocstate = TBDB_ALLOCSTATE_RES_TEARDOWN();
	
	$db_result =
	    DBQueryFatal("select r.node_id,nt.isvirtnode,nt.isremotenode ".
			 "  from reserved as r ".
			 "left join nodes as n on n.node_id=r.node_id ".
			 "left join node_types as nt on nt.type=n.type ".
			 "where r.pid='$pid' and r.eid='$eid' and ".
			 "      n.allocstate='$allocstate'");

	if ($db_result->numrows) {
	    my @virtnodes = ();
	    my @physnodes = ();
	    
	    print "Tearing down and releasing unused nodes\n";

	    # First teardown/release virtnodes. 
	    while (my ($node,$isvirt,$isrem) = $db_result->fetchrow_array()) {
		if ($isvirt) {
		    push(@virtnodes, $node);
		}
		elsif (!$isrem) {
		    push(@physnodes, $node);
		}
	    }
	    if (@virtnodes) {
		TBDebugTimeStamp("vnode_setup started");
		
692
		if (system("vnode_setup -k $pid $eid @virtnodes")) {
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
		    print "Failed to tear down unused virtnodes!\n";
		    return 1;
		}
		TBDebugTimeStamp("vnode_setup finished");
		
		if (system("nfree $pid $eid @virtnodes")) {
		    print "Failed to nfree unused virtnodes!\n";
		    return 1;
		}
	    }
	    if (@physnodes) {
		if (system("nfree $pid $eid @physnodes")) {
		    print "Failed to nfree unused physnodes!\n";
		    return 1;
		}
	    }
	}
    }

Chad Barb's avatar
   
Chad Barb committed
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
    # Exit here if we are testing.
    if ($TESTMODE) {
	print "Testing run - Stopping here.\n";
	return 0;
    }

    #
    # These things need to get started before the nodes come up, so we'll
    # do them before the os_setup. Everything else can done in parallel with
    # os_setup. (Actually, these probably can too, since they should finish
    # long before the nodes reboot, but better safe than sorry)
    #
    print "Setting up mountpoints.\n";
    TBDebugTimeStamp("mountpoints started");
    if (system("exports_setup")) {
	print STDERR "*** Failed to setup mountpoints.\n";
	return 1;
    }
    TBDebugTimeStamp("mountpoints finished");
Chad Barb's avatar
Chad Barb committed
731

Chad Barb's avatar
   
Chad Barb committed
732
733
734
735
736
737
    TBDebugTimeStamp("named started");
    print "Setting up named maps.\n";
    if (system("named_setup")) {
	print STDERR "*** WARNING: Failed to add node names to named map.\n";
	#
	# This is a non-fatal error.
Chad Barb's avatar
Chad Barb committed
738
	#
Chad Barb's avatar
   
Chad Barb committed
739
740
    }
    TBDebugTimeStamp("named finished");
Chad Barb's avatar
Chad Barb committed
741
742
743
744


    #
    # If user specified -reboot to update,
Chad Barb's avatar
   
Chad Barb committed
745
    # and we are successfully performing the update,
Chad Barb's avatar
Chad Barb committed
746
747
748
    # then mark all nodes in experiment
    # so os_setup will reboot them.
    #
Chad Barb's avatar
   
Chad Barb committed
749
    if (($type == UPDATE) && $updateReboot) {
Chad Barb's avatar
Chad Barb committed
750
751
752
753
754
755
756
757
	print STDERR "Marking nodes for reboot.\n";
	$db_result =
	    DBQueryFatal("select rv.node_id ".
                         "from reserved as rv ".
			 "left join nodes as n on ".
			 "n.node_id = rv.node_id ".
			 "where rv.pid='$pid' and rv.eid='$eid'");

758
	while (my ($node) = $db_result->fetchrow_array) {
Chad Barb's avatar
Chad Barb committed
759
760
761
762
	    TBSetNodeAllocState( $node, TBDB_ALLOCSTATE_RES_INIT_DIRTY() );
	}
    }

Chad Barb's avatar
   
Chad Barb committed
763
764
765
766
767
768
769
770
771
772
773
774
    #
    # Since it'll take a while for the nodes to reboot, we'll start now, and
    # wait for the os_setup to finish, down below
    #
    print "Resetting OS and rebooting.\n";
    TBDebugTimeStamp("os_setup started");
    if (!($os_setup_pid = fork())) { 
	exec("os_setup $pid $eid") or return 1;
    } elsif ($os_setup_pid == -1) {
	print STDERR "*** Fork failed.\n";
	return 1;
    }
Chad Barb's avatar
Chad Barb committed
775

Chad Barb's avatar
   
Chad Barb committed
776
    #
777
778
779
780
781
    # XXX
    # Don't add any steps between here and the waitpid() call below
    # without verifying that 1) It's OK for nodes to come up before
    # the step has completed and 2) It's OK for the command to run in
    # parallel with os_setup (no DB dependencies, etc.)
Chad Barb's avatar
   
Chad Barb committed
782
783
784
785
786
787
788
789
790
    #

    print "Setting up VLANs.\n";
    TBDebugTimeStamp("snmpit started");
    if (system("snmpit -t $pid $eid")) {
	print STDERR "*** Failed to set up VLANs.\n";
	return 1;
    }
    TBDebugTimeStamp("snmpit finished");
Chad Barb's avatar
Chad Barb committed
791

Chad Barb's avatar
   
Chad Barb committed
792
793
794
795
    #
    # An error now means that the VLANS need to be cleaned up.
    #
    $cleanvlans = 1;
Chad Barb's avatar
Chad Barb committed
796

Chad Barb's avatar
   
Chad Barb committed
797
798
799
800
801
802
803
804
805
    print "Setting up email lists.\n";
    TBDebugTimeStamp("genelists started");
    if (system("genelists")) {
	print STDERR "*** WARNING: Failed to update email lists.\n";
	#
	# This is a non-fatal error.
	# 
    }
    TBDebugTimeStamp("genelists finished");
Chad Barb's avatar
Chad Barb committed
806

Chad Barb's avatar
   
Chad Barb committed
807
808
809
810
811
812
813
814
815
816
817
818
819
820
    #
    # Don't clear port counters on UPDATE.
    # (XXX should clear new nodes' port counters.)

    if ($type >= RETRY) {
	print "Clearing port counters.\n";
	TBDebugTimeStamp("portstats started");
	if (system("portstats -z -a -q $pid $eid")) {
	    print STDERR "*** WARNING: Failed to clear port counters.\n";
	    #
	    # This is a non-fatal error.
	    # 
	}
	TBDebugTimeStamp("portstats finished");
Chad Barb's avatar
   
Chad Barb committed
821
    }
Chad Barb's avatar
Chad Barb committed
822

Chad Barb's avatar
   
Chad Barb committed
823
824
825
826
827
828
829
830
831
    #
    # OK, let's see how that os_setup did
    #
    $kid = waitpid($os_setup_pid,0);
    if ($kid == $os_setup_pid) {
	undef $os_setup_pid; # Make sure doswapout() doesn't wait for it.
	if ($CHILD_ERROR) {
	    print STDERR "*** Failed to reset OS and reboot nodes.\n";
	    #
Chad Barb's avatar
   
Chad Barb committed
832
833
	    # Use returncode from os_setup process to
	    # set global $retry flag, indicating to caller
Chad Barb's avatar
   
Chad Barb committed
834
835
836
	    # that it may be beneficial to attempt
	    # a doSwapin() again.
	    #
Chad Barb's avatar
   
Chad Barb committed
837
            if (($CHILD_ERROR >> 8) == 1) {
Chad Barb's avatar
Chad Barb committed
838
		return 3;
Chad Barb's avatar
   
Chad Barb committed
839
840
	    } else {
		print STDERR "Not retrying due to error type.\n";
Chad Barb's avatar
Chad Barb committed
841
		return 1;
Chad Barb's avatar
   
Chad Barb committed
842
	    }
Chad Barb's avatar
   
Chad Barb committed
843
844
845
846
847
848
849
	}
    } else {
	undef $os_setup_pid;
	print STDERR "*** Error waiting for os_setup to finish.\n";
	return 1;
    }
    TBDebugTimeStamp("os_setup finished");
Chad Barb's avatar
Chad Barb committed
850

Chad Barb's avatar
   
Chad Barb committed
851
852
853
854
    #
    # Okay, start the event system now that we know all the nodes have
    # rebooted (os_setup is done). This only takes a moment (puts itself
    # in the background), so its not enough of a delay to worry about.
Chad Barb's avatar
   
Chad Barb committed
855
856
    # Don't do this during an update, since we didn't kill the 
    # event system previously, so starting it again will fail!
Chad Barb's avatar
   
Chad Barb committed
857
858
    # 
    if (! $DISABLE_EVENTS) {
Chad Barb's avatar
Chad Barb committed
859
	if ($type != UPDATE && $type != UPDATE_RECOVER) {
Chad Barb's avatar
   
Chad Barb committed
860
861
862
863
864
865
866
	    print "Starting the event system.\n";
	    TBDebugTimeStamp("eventsys_control started");
	    if (system("eventsys_control start $pid $eid")) {
		print STDERR "*** Failed to start the event system.\n";
		return 1;
	    }
	    TBDebugTimeStamp("eventsys_control finished");
Chad Barb's avatar
   
Chad Barb committed
867
868
	}
    }
Chad Barb's avatar
Chad Barb committed
869

Chad Barb's avatar
   
Chad Barb committed
870
871
    return 0;
}