swapexp.in 23.8 KB
Newer Older
1
#!/usr/bin/perl -wT
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
3
4

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
6
7
8
# All rights reserved.
#

9
10
11
12
use English;
use Getopt::Std;

#
Chad Barb's avatar
Chad Barb committed
13
# This gets invoked from the Web interface.
Chad Barb's avatar
   
Chad Barb committed
14
# Swap an experiment in, swap it out, restart or modify.
15
#
Chad Barb's avatar
Chad Barb committed
16

17
18
sub usage()
{
19
    print STDOUT "Usage: swapexp [-b] [-i | -a | -f] [-r] ".
20
	"<-s in | out | restart | modify | pause> <pid> <eid> [<nsfile>]\n";
21
22
    exit(-1);
}
23
my  $optlist = "biafrs:";
24

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#
# Exit codes are important; they tell the web page what has happened so
# it can say something useful to the user. Fatal errors are mostly done
# with die(), but expected errors use this routine. At some point we will
# use the DB to communicate the actual error.
#
# $status < 0 - Fatal error. Something went wrong we did not expect.
# $status = 0 - Termination is proceeding in the background. Notified later.
# $status > 0 - Expected error. User not allowed for some reason. 
# 
sub ExitWithStatus($$)
{
    my ($status, $message) = @_;
    
    if ($status < 0) {
	die("*** $0:\n".
	    "    $message\n");
    }
    else {
	print STDERR "$message\n";
    }
    exit($status);
}

49
50
51
52
53
54
#
# Configure variables
#
my $TB     = "@prefix@";
my $TBOPS  = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
55
my $TBINFO = "$TB/expinfo";
56
my $TBDOCBASE = "@TBDOCBASE@";
57
58
59
60
61
62
63
64
65

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;

my $tbdir    = "$TB/bin/";
66
my $tbdata   = "tbdata";
67
my $batch    = 0;
68
my $idleswap = 0;
69
70
my $autoswap = 0;
my $force    = 0;
Chad Barb's avatar
Chad Barb committed
71
my $reboot   = 0;
72
my $errorstat= -1;
73
my $modifyHosed = 0;
Chad Barb's avatar
   
Chad Barb committed
74

75
76
77
78
79
my $inout;
my $logname;
my $dbuid;
my $user_name;
my $user_email;
80
my @allnodes;
81
my @row;
82
my $action;
83
my $nextswapstate;
84
my $termswapstate;
Chad Barb's avatar
   
Chad Barb committed
85

86
87
88
#
# Untaint the path
# 
89
$ENV{'PATH'} = "/bin:/usr/bin:$TB/libexec/vis";
90
91
92
93
94
95
96
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

97
98
99
100
101
102
103
#
# Set umask for start/swap. We want other members in the project to be
# able to swap/end experiments, so the log and intermediate files need
# to be 664 since some are opened for append.
#
umask(0002);

104
105
106
107
108
109
110
111
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
112
113
114
if (defined($options{"i"})) {
    $idleswap = 1;
}
115
116
117
118
119
120
if (defined($options{"a"})) {
    $autoswap = 1;
}
if (defined($options{"f"})) {
    $force = 1;
}
121
122
123
if (defined($options{"b"})) {
    $batch = 1;
}
Chad Barb's avatar
   
Chad Barb committed
124
125
126
if (defined($options{"r"})) {
    $reboot = 1;
}
127
128
129
if (defined($options{"s"})) {
    $inout = $options{"s"};

Chad Barb's avatar
Chad Barb committed
130
131
132
    if ($inout ne "out"     &&
	$inout ne "in"      &&
	$inout ne "restart" &&
133
	$inout ne "pause"   &&
Chad Barb's avatar
   
Chad Barb committed
134
	$inout ne "modify") {
135
136
137
138
139
140
141
	usage();
    }
}
else {
    usage();
}

Chad Barb's avatar
   
Chad Barb committed
142
143
144
145
146
147
if (@ARGV != (($inout eq "modify") ? 3 : 2)) {
    usage();
}
my $pid   = $ARGV[0];
my $eid   = $ARGV[1];

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\@\w.]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}
163
my $repfile = "$eid.report";
164
165
my $workdir = TBExptWorkDir($pid, $eid);
my $userdir = TBExptUserDir($pid, $eid);
166
167
168
169
170
171
172
173
174
175
176
my $tempnsfile;
my $modnsfile;

if ($inout eq "modify") {
    $tempnsfile = $ARGV[2];

    #
    # Untaint nsfile argument; Allow slash.
    #
    if ($tempnsfile =~ /^([-\w.\/]+)$/) {
	$tempnsfile = $1;
177
178
    }
    else {
179
180
181
182
	die("Tainted nsfile name: $tempnsfile");
    }
    $modnsfile = "$eid-modify.ns";
}
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200

#
# Verify user and get his DB uid.
#
if (! UNIX2DBUID($UID, \$dbuid)) {
    die("*** $0:\n".
	"    You do not exist in the Emulab Database.\n");
}

#
# Get email info for user.
#
if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
    die("*** $0:\n".
	"    Cannot determine your name and email address.\n");
}

#
Chad Barb's avatar
   
Chad Barb committed
201
# Verify that this person can muck with the experiment.
202
203
204
205
206
# Note that any script down the line has to do an admin check also. 
#
if ($UID && !TBAdmin($UID) &&
    !TBExptAccessCheck($dbuid, $pid, $eid, TB_EXPT_DESTROY)) {
    die("*** $0:\n".
Chad Barb's avatar
   
Chad Barb committed
207
	"    You do not have permission to swap or modify this experiment!\n");
208
209
}

210
211
212
213
# Must do this before lock tables!
# idleswap is in minutes, threshold is in hours
$idleswap_time = 60 * TBGetSiteVar("idle/threshold");

214
215
216
217
218
#
# We have to protect against trying to end an experiment that is currently
# in the process of being terminated. We use a "wrapper" state (actually
# a timestamp so we can say when termination was requested) since
# terminating consists of a couple of different experiment states down inside
Chad Barb's avatar
Chad Barb committed
219
# the tb scripts.
220
221
222
223
224
225
226
227
228
229
230
231
232
#
DBQueryFatal("lock tables experiments write");

$query_result =
    DBQueryFatal("SELECT * FROM experiments WHERE eid='$eid' and pid='$pid'");

if (! $query_result->numrows) {
    die("*** $0:\n".
	"    No such experiment $pid/$eid exists!\n");
}
my %hashrow = $query_result->fetchhash();
my $expt_head_login = $hashrow{'expt_head_uid'};
my $estate          = $hashrow{'state'};
233
my $batchstate      = $hashrow{'batchstate'};
234
my $expt_path       = $hashrow{'path'};
235
my $expt_locked     = $hashrow{'expt_locked'};
236
my $isbatchexpt     = $hashrow{'batchmode'};
237
my $canceled        = $hashrow{'canceled'};
238
239
240
241
242
243
244
245
246
247
my $swappablebit= $hashrow{'swappable'};
my $idleswapbit = $hashrow{'idleswap'};
my $autoswapbit = $hashrow{'autoswap'};
my $swappablestr= ( $swappablebit ? "Yes" : "No" );
my $idleswapstr = ( $idleswapbit ? "Yes" : "No" );
my $autoswapstr = ( $autoswapbit ? "Yes" : "No" );
my $noswap      = $hashrow{'noswap_reason'};
my $noidleswap  = $hashrow{'noidleswap_reason'};
my $idleswaptime= $hashrow{'idleswap_timeout'} / 60.0;
my $autoswaptime= $hashrow{'autoswap_timeout'} / 60.0;
248

249
250
if ($inout ne "out") {
    # I'm going to update this below, so fix the value before I use it.
251
    $idleswap_time = min($idleswaptime * 60, $idleswap_time);
252
253
254
    $idleswaptime = $idleswap_time / 60.0;
}

255
256
my $swapsettings = 
  "Idle-Swap:   $idleswapstr".
257
  ($idleswapbit ? ", at $idleswaptime hours\n" : " (Reason: $noidleswap)\n").
258
259
  "Auto-Swap:   $autoswapstr".
  ($autoswapbit ? ", at $autoswaptime hours\n" : "\n");
260

261
if (! chdir($workdir)) {
262
    die("*** $0:\n".
263
	"    Could not chdir to $workdir: $!\n");
264
265
}

266
#
267
268
269
# This script is called from the batch daemon.
# 
if ($batch) {
270
    #
271
272
273
    # Sanity Check. If called from the daemon, must already be locked,
    # must be a batch experiment, and must be in proper state for the
    # operation requested. 
274
    #
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
    die("*** $0:\n".
	"    Experiment $pid/$eid is supposed to be a batch experiment!\n")
	if (!$isbatchexpt);
    
    die("*** $0:\n".
	"    Batch experiment $pid/$eid should be locked!\n")
	if (!defined($expt_locked) ||
	    $batchstate ne BATCHSTATE_LOCKED());

    if ($inout eq "in") {
	die("*** $0:\n".
	    "    Batch experiment $pid/$eid is not in the proper state!\n".
	    "    Currently $estate, but should be QUEUED.\n")
	    if ($estate ne EXPTSTATE_QUEUED);
	
	die("*** $0:\n".
	    "    Batch experiment $pid/$eid has been canceled! Aborting.\n")
	    if ($canceled);
    }
    elsif ($inout eq "out") {
	die("*** $0:\n".
	    "    Batch experiment $pid/$eid is not in the proper state!\n".
	    "    Currently $estate, but should be ACTIVE.\n")
	    if ($estate ne EXPTSTATE_ACTIVE);
299
300
    }
    else {
301
302
303
304
305
306
	die("*** $0:\n".
	    "    Improper request from batch daemon for $pid/$eid!\n");
    }
}
else {
    if ($isbatchexpt) {
307
308
309
310
	#
	# User is requesting that a batch either be injected or paused.
	# Sanity check the state, but otherwise let the batch daemon
	# handle it.
311
312
	#
	ExitWithStatus(1, "Batch experiment $pid/$eid is still canceling!")
313
	    if ($canceled);
314

315
	if ($inout eq "in") {
316
	    ExitWithStatus(1,
317
318
319
320
			   "Batch experiment $pid/$eid must be SWAPPED to\n".
			   "QUEUE. Currently $estate.")
		if ($estate ne EXPTSTATE_SWAPPED);
	    SetExpState($pid, $eid, EXPTSTATE_QUEUED);
321
322
	}
	elsif ($inout eq "out") {
323
	    ExitWithStatus(1,
324
325
326
327
			   "Batch experiment $pid/$eid must be ACTIVE or\n".
			   "ACTIVATING to swap out. Currently $estate.")
		if ($estate ne EXPTSTATE_ACTIVE &&
		    $estate ne EXPTSTATE_ACTIVATING);
328
329
330
331
332

	    #
	    # Since the batch daemon has control, all we can do is set
	    # the cancel bit.
	    # 
333
	    TBSetCancelFlag($pid, $eid, EXPTCANCEL_SWAP);
334
335
	}
	elsif ($inout eq "pause") {
336
	    ExitWithStatus(1,
337
338
339
			   "Batch experiment $pid/$eid must be QUEUED to\n".
			   "DEQUEUE. Currently $estate.")
		if ($estate ne EXPTSTATE_QUEUED);
340
341

	    #
342
343
344
345
	    # XXX. The batch daemon might already have the experiment, but
	    # not have shipped it off to startexp. Change the state
	    # anyway. The error will be noticed later when startexp dies,
	    # and the batch daemon gets the error back. This sucks.
346
	    #
347
	    SetExpState($pid, $eid, EXPTSTATE_SWAPPED);
348
	}
349
	elsif ($inout eq "modify") {
350
	    ExitWithStatus(1,
351
352
353
354
355
356
			   "Batch experiment $pid/$eid must be SWAPPED or\n".
			   "ACTIVE to modify. Currently $estate.")
		if (($estate ne EXPTSTATE_SWAPPED &&
		     $estate ne EXPTSTATE_ACTIVATING) ||
		    $batchstate != BATCHSTATE_UNLOCKED());

357
	    #
358
	    # Otherwise, proceed with the modify. The experiment will be
359
360
	    # locked below, and so it cannot be injected or otherwise messed
	    # with since its state is going to be changed before we unlock
361
362
363
364
	    # the experiments table. The batch daemon will leave it alone
	    # until the modify is done. If the modify fails and cannot recover
	    # it is going to get swapped out; that is okay since the batch
	    # daemon does not keep state internally. 
365
	    #
366
367
	    goto doit;
	}
368
369
	else {
	    die("*** $0:\n",
370
		"    Operation $inout not allowed on a batch experiment!\n");
371
	}
372
373
	ExitWithStatus(0, 
		       "Batch experiment $pid/$eid state has been changed.\n");
374
      doit:
375
    }
376
377
378
379
380
381
382
383
384
385
    else {
	#
	# If the cancel flag is set, then user must wait for that to
	# clear before we can do anything else.
	#
	ExitWithStatus(1,
		       "Experiment $pid/$eid has its cancel flag set!.\n".
		       "You must wait for that to clear before you can swap\n".
		       "or modify the experiment.\n")
	    if ($canceled);
386

387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
	#
	# Check the state for the various operations.
	#
	if (!$force) {
	  SWITCH: for ($inout) {
	      /^in$/i && do {
		  if ($estate ne EXPTSTATE_SWAPPED()) {
		      ExitWithStatus(1,
				     "Experiment $pid/$eid is not swapped out!");
		  }
		  last SWITCH;
	      };
	      /^out$/i && do {
		  if ($estate ne EXPTSTATE_ACTIVE() &&
		      $estate ne EXPTSTATE_ACTIVATING()) {
		      ExitWithStatus(1,
				     "Experiment $pid/$eid is not swapped in ".
				     "or activating!\n");
		  }
		  
		  if ($estate eq EXPTSTATE_ACTIVATING()) {
		      #
		      # All we can do is set the cancel flag and hope that
		      # it gets noticed. We do not wait. 
		      # 
		      TBSetCancelFlag($pid, $eid, EXPTCANCEL_SWAP);
		      
		      ExitWithStatus(0,
				     "Experiment $pid/$eid swapin has been  ".
				     "marked for cancelation.\n".
				     "You will receive email when the original ".
				     "swap request has finished.");
		  }
		  last SWITCH;
	      };
	      /^restart$/i && do {
		  if ($estate ne EXPTSTATE_ACTIVE()) {
		      ExitWithStatus(1,
				     "Experiment $pid/$eid is not swapped in!");
		  }
		  last SWITCH;
	      };
	      /^modify$/i && do {
		  if ($estate ne EXPTSTATE_ACTIVE() &&
		      $estate ne EXPTSTATE_SWAPPED()) {
		      ExitWithStatus(1,
				     "Experiment $pid/$eid must be ACTIVE or\n".
				     "SWAPPED to modify!\n");
		  }
		  last SWITCH;
	      };
	      die("*** $0:\n".
		  "    Missing state check for action: $action\n");
	  }
441
442
	}
    }
443
444
}

445
446
447
448
449
450
451
#
# Determine the temporary and next state for experiment. If the experiment
# is a batch experiment, then the next state is actually handled by the
# batch daemon, but we still have to deal with the temporary state. 
#
SWITCH: for ($inout) {
    /^in$/i && do {
452
	$nextswapstate = EXPTSTATE_ACTIVATING();
453
454
455
	last SWITCH;
    };
    /^out$/i && do {
456
	$nextswapstate = EXPTSTATE_SWAPPING();
457
458
459
	last SWITCH;
    };
    /^restart$/i && do {
460
	$nextswapstate = EXPTSTATE_RESTARTING();
461
462
463
	last SWITCH;
    };
    /^modify$/i && do {
464
465
	$nextswapstate = (($estate eq EXPTSTATE_SWAPPED()) ?
			  EXPTSTATE_MODIFY_PARSE() : EXPTSTATE_MODIFY_REPARSE());
466
467
	last SWITCH;
    };
468
    die("*** $0:\n".
469
	"    Missing state check for action: $action\n");
470
}
471
472
 
# Update idleswap_timeout to whatever the current value is.
473
if ($inout ne "out") {
474
475
476
    DBQueryFatal("update experiments set idleswap_timeout='$idleswap_time' ".
		 "where eid='$eid' and pid='$pid'");
}
477

478
479
480
481
482
483
484
#
# On a failure, we go back to this swapstate. Might be modified below.
# 
$termswapstate = $estate;

# Lock the record, set the nextstate, and unlock the table.
TBLockExp($pid, $eid, $nextswapstate);
485
486
487
488
DBQueryFatal("unlock tables");

#
# XXX - At this point a failure is going to leave things in an
489
490
491
492
# inconsistent state. Be sure to call fatal() only since we are
# going into the background, and we have to send email since no
# one is going to see printed error messages (output goes into the
# log file, which will be sent along in the email). 
493
494
#

495
496
497
498
499
500
501
502
503
if ($inout eq "in") {
    $action = "swapped in";
}
if ($inout eq "out") {
    $action = "swapped out";
}
if ($inout eq "restart") {
    $action = "restarted";
}
Chad Barb's avatar
   
Chad Barb committed
504
505
506
if ($inout eq "modify") {
    $action = "modified";
}
507

508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
#
# Get email address of the experiment head, which may be different than
# the person who is actually terminating the experiment, since its polite
# to let the original creator know whats going on. 
#
my $expt_head_name;
my $expt_head_email;

if (! UserDBInfo($expt_head_login, \$expt_head_name, \$expt_head_email)) {
    print STDERR "*** WARNING: ".
	         "Could not determine name/email for $expt_head_login.\n";
    $expt_head_name  = "TBOPS";
    $expt_head_email = $TBOPS;
}

523
524
525
526
527
528
529
530
531
532
533
534
#
# Before going to background, we have to copy out the NS file!
#
if ($inout eq "modify") {
    unlink($modnsfile);
    if (system("/bin/cp", "$tempnsfile", "$modnsfile")) {
	die("*** $0:\n".
	    "    Could not copy $tempnsfile to $modnsfile");
    }
    chmod(0664, "$modnsfile");
}

535
536
537
538
#
# If not in batch mode, go into the background. Parent exits.
#
if (! $batch) {
539
    $logname = TBExptCreateLogFile($pid, $eid, "swapexp");
540
    TBExptSetLogFile($pid, $eid, $logname);
541
    TBExptOpenLogFile($pid, $eid);
Chad Barb's avatar
Chad Barb committed
542

543
544
545
546
    if (TBBackGround($logname)) {
	#
	# Parent exits normally
	#
547
548
	print "Experiment $pid/$eid is now being $action.\n".
	    "You will be notified via email when the this is done.\n";
549
550
551
552
	exit(0);
    }
}

553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
#
# Gather stats; start clock ticking
#
if ($inout eq "in") {
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPIN, 0,
		    TBDB_STATS_FLAGS_START);
}
elsif ($inout eq "out") {
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT, 0,
		    TBDB_STATS_FLAGS_START);
}
elsif ($inout eq "modify") {
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPMODIFY, 0,
		    TBDB_STATS_FLAGS_START);
}

569
570
571
#
# Remove old report file since its contents are going to be invalid.
#
572
if ($inout ne "restart" && -e $repfile) {
573
574
575
    unlink("$repfile");
}

576
577
578
579
#
# Sanity check states in case someone changes something.
#
if ($inout eq "out") {
580
581
582
583
    my $optarg = (($force || $idleswap) ? "-force" : "");
    
    print STDOUT "Running 'tbswap out $optarg $pid $eid'\n";
    if (system("$tbdir/tbswap out $optarg $pid $eid") != 0) {
584
	$errorstat = $? >> 8;
585
	fatal("tbswap out failed!");
586
    }
587
    SetExpState($pid, $eid, EXPTSTATE_SWAPPED);
588
}
589
elsif ($inout eq "in") {
590
    print STDOUT "Running 'tbswap in $pid $eid'\n";
Chad Barb's avatar
   
Chad Barb committed
591
    if (system("$tbdir/tbswap in $pid $eid") != 0) {
592
	$errorstat = $? >> 8;
593
	fatal("tbswap in failed!");
594
    }
595
    SetExpState($pid, $eid, EXPTSTATE_ACTIVE);
596

597
    system("$tbdir/tbreport -b $pid $eid 2>&1 > $repfile");
Chad Barb's avatar
Chad Barb committed
598
}
Chad Barb's avatar
   
Chad Barb committed
599
elsif ($inout eq "modify") {
Chad Barb's avatar
Chad Barb committed
600
    my $modifyError = "";
601
    my $oldstate    = $estate;
Chad Barb's avatar
Chad Barb committed
602

603
604
605
    GatherSwapStats($pid, $eid, $dbuid,
		    TBDB_STATS_SWAPMODIFY, 0, TBDB_STATS_FLAGS_PREMODIFY);

Chad Barb's avatar
Chad Barb committed
606
    print "Backing up old experiment state ... " . TBTimeStamp() . "\n";
607
    if (TBExptBackupVirtualState($pid, $eid)) {
608
	fatal("Could not backup experiment state; cannot safely continue!");
Chad Barb's avatar
Chad Barb committed
609
610
611
612
613
    }

    #
    # Rerun tbprerun if modifying.
    #
614
615
    print STDOUT "Running 'tbprerun $pid $eid $modnsfile'\n";
    if (system("$tbdir/tbprerun $pid $eid $modnsfile") != 0) {
Chad Barb's avatar
Chad Barb committed
616
617
618
	$modifyError = "tbprerun failed!";
    }

Chad Barb's avatar
   
Chad Barb committed
619
    #
620
    # Our next state depends on whether the experiment was active or swapped.
Chad Barb's avatar
   
Chad Barb committed
621
    #
622
623
624
    if (! $modifyError) {
	if ($estate eq EXPTSTATE_SWAPPED) {
	    SetExpState($pid, $eid, EXPTSTATE_SWAPPED);
Chad Barb's avatar
   
Chad Barb committed
625
	}
626
627
628
629
630
631
632
633
634
635
	else {
	    SetExpState($pid, $eid, EXPTSTATE_MODIFY_RESWAP);
	    
	    my $optarg = ($reboot ? "-reboot" : "");

	    print STDOUT "Running 'tbswap update $optarg $pid $eid'\n";
	    if (system("$tbdir/tbswap update $optarg $pid $eid") != 0) {
		$errorstat = $? >> 8;
		$modifyError = "tbswap update failed!";
	    }
Chad Barb's avatar
   
Chad Barb committed
636

637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
	    #
	    # See what tbswap did. It might have swapped it out if there
	    # was an error. 
	    # 
	    if (! $modifyError) {
		SetExpState($pid, $eid, EXPTSTATE_ACTIVE);
		$estate = EXPTSTATE_ACTIVE;
	    }
	    elsif ($errorstat & 0x40) {
		#
		# Icky. Magic return code that says tbswap swapped it out.
		# We do not want tbswap to muck with states anymore, so
		# need to know what it did. At some point we should clean
		# up the exit reporting! Anyway, fatal() needs to know the
		# the right state to go back to (no longer ACTIVE).
		#
		$estate = EXPTSTATE_SWAPPED;
		$termswapstate = EXPTSTATE_SWAPPED;
                # Old accounting info.
		TBSetExpSwapTime($pid, $eid);
Chad Barb's avatar
   
Chad Barb committed
657
	    }
658
	}
Chad Barb's avatar
Chad Barb committed
659
660
661
    }

    if ($modifyError) {
662
	print STDOUT "Modify Error: $modifyError\n";
Chad Barb's avatar
Chad Barb committed
663
	print STDOUT "Recovering experiment state...\n";
664
	
665
666
	# Must deal with the prerender explicitly since it runs background.
	system("prerender -r $pid $eid");
667
	TBExptRemoveVirtualState($pid, $eid);
668
	
669
	if (TBExptRestoreVirtualState($pid, $eid) == 0) {
670
671
	    # Must deal with the prerender explicitly since it runs background.
	    system("prerender -t $pid $eid");
672
673
674
675
676
	    fatal("Update aborted; old state restored.");
	}
	else {
	    $modifyHosed = 1;
	    fatal("Experiment state could not be restored!");
Chad Barb's avatar
Chad Barb committed
677
	}
Chad Barb's avatar
   
Chad Barb committed
678
    }
679
    
680
    TBExptClearBackupState($pid, $eid);
681
    system("$tbdir/tbreport -b $pid $eid 2>&1 > $repfile");
682
}
Chad Barb's avatar
   
Chad Barb committed
683
else { # $inout eq "restart" assumed.
684
    print STDOUT "Running 'tbrestart $pid $eid'\n";
685
    if (system("$tbdir/tbrestart $pid $eid") != 0) {
686
	fatal("tbrestart failed!");
687
    }
688
    SetExpState($pid, $eid, EXPTSTATE_ACTIVE);
689
}
690

691
692
693
694
695
696
697
698
699
700
#
# Try to copy off the files for testbed information gathering.
#
TBSaveExpLogFiles($pid, $eid);

#
# Make a copy of the work dir in the user visible space so the user
# can see the log files. This overwrites existing files of course,
# but thats okay.
#
701
system("cp -Rfp $workdir/ $userdir/tbdata/");
702

703
704
705
706
#
# Gather stats. 
#
if ($inout eq "in") {
707
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPIN, 0);
708
709
}
elsif ($inout eq "out") {
710
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT, 0,
711
		    ($idleswap ? TBDB_STATS_FLAGS_IDLESWAP() : 0));
712
713
}
elsif ($inout eq "modify") {
714
    GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPMODIFY, 0);
715
}
716
717
# Old accounting info.
TBSetExpSwapTime($pid, $eid);
718

719
720
721
722
723
724
#
# Set the swapper uid on success only, and *after* gathering swap stats!
#
TBExptSetSwapUID($pid, $eid, $dbuid);

#
725
# In batch mode, just exit without sending email or unlocking. The
726
# batch daemon will take care of that.
727
728
729
730
731
#
if ($batch) {
    exit(0);
}

Chad Barb's avatar
   
Chad Barb committed
732
733
734
735
736
#
# HACK! if successful, put new NS file in DB.
#

if ($inout eq "modify") {
737
    $nsdata_string = `cat $modnsfile`;
Chad Barb's avatar
   
Chad Barb committed
738
739
740
741
742
743
    if (defined($nsdata_string)) {
	$nsdata_string = DBQuoteSpecial($nsdata_string);

	DBQueryWarn("delete from nsfiles WHERE eid='$eid' and pid='$pid'");
	DBQueryWarn("insert into nsfiles (pid, eid, nsfile) ".
		    "VALUES('$pid', '$eid', $nsdata_string)");
744
745
746
    }
    else {
	print "Warning!! Could not read nsfile '$modnsfile'!\n";
Chad Barb's avatar
   
Chad Barb committed
747
748
749
    }
}

750
751
752
753
754
755
756
#
# Clear the log file so the web page stops spewing. 
#
if (defined($logname)) {
    TBExptCloseLogFile($pid, $eid);
}

757
758
759
#
# Must unlock before exit.
#
760
TBUnLockExp($pid, $eid);
761
762
763
764
765

#
# Since the swap completed, clear the cancel flag. This must be done
# after we change the experiment state (above). 
#
766
TBSetCancelFlag($pid, $eid, EXPTCANCEL_CLEAR);
767
768
769

print "Swap Success!\n";

770
771
772
773
#
# Send email notification to user.
#
my $message =
774
775
    "Experiment $eid in project $pid has been ";

776
if ($inout eq "out" && ($idleswap || $autoswap || $force) ) {
777
    $message .= "forcibly swapped out by\nEmulab";
778
779
780
781
782
    if ($idleswap) {
	$message .= " because it was idle for too long (Idle-Swap).\n".
	  "(See also the Idle-Swap info in \n".
	  "$TBDOCBASE/docwrapper.php3?docname=swapping.html )\n";
    } elsif ($autoswap) {
783
784
	$message .= " because it exceeded its Maximum Duration.\n".
	  "(See also the Max. Duration info in \n".
785
786
787
788
789
	  "$TBDOCBASE/docwrapper.php3?docname=swapping.html )\n";
    } elsif ($force) {
	$message .= ". (See also our Node Usage Policies in \n".
	  "$TBDOCBASE/docwrapper.php3?docname=swapping.html )\n";
    }
790
791
792
793
794
}
else {
    $message .= "$action.\n";
}

795
796
797
798
799
if ($inout eq "in") {
    # Add the swap settings...
    $message .="\nCurrent swap settings:\n$swapsettings";
}

800
801
$message .=
    "\n".
802
803
    "Appended below is the output. If you have any questions or comments,\n" .
    "please include the output in your message to $TBOPS\n";
804
805

SENDMAIL("$user_name <$user_email>",
806
	 "Experiment $pid/$eid \u$action",
807
	 $message,
808
	 ($idleswap ? $TBOPS : "$user_name <$user_email>"),
809
810
	 "Cc:  $expt_head_name <$expt_head_email>\n".
	 "Bcc: $TBLOGS",
811
812
	 (($inout eq "restart") ? ($logname) :
	  (($repfile, $logname), (defined($modnsfile) ? ($modnsfile) : ()))));
813
814
815
816
817
818

exit 0;

sub fatal($)
{
    my($mesg) = $_[0];
Chad Barb's avatar
Chad Barb committed
819

820
821
    print STDOUT "*** $0:\n".
	         "    $mesg\n";
822

823
824
825
826
827
828
829
830
831
832
833
834
835
    #
    # Gather stats. 
    #
    if ($inout eq "in") {
	GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPIN, $errorstat);
    }
    elsif ($inout eq "out") {
	GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT, $errorstat);
    }
    elsif ($inout eq "modify") {
	GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPMODIFY, $errorstat);
    }

836
837
838
839
840
841
842
    #
    # Clear backup state since not needed anymore; experiment is toast. 
    # 
    if ($inout eq "modify") {
	TBExptClearBackupState($pid, $eid);
    }

Chad Barb's avatar
   
Chad Barb committed
843
    #
844
    # If hosed, we entirely terminate the experiment.
Chad Barb's avatar
   
Chad Barb committed
845
    #
846
    if ($modifyHosed) {
Chad Barb's avatar
   
Chad Barb committed
847
	#
848
	# Note: $estate is indeed still set appropriately!
Chad Barb's avatar
   
Chad Barb committed
849
850
	#
	if ($estate eq EXPTSTATE_ACTIVE) {
851
	    print "Running 'tbswap out -force $pid $eid'\n";
Chad Barb's avatar
   
Chad Barb committed
852
853
854
855
	    if (system("$tbdir/tbswap out -force $pid $eid") != 0) {
		print "tbswap out failed!\n";
	    }
	}
Chad Barb's avatar
Chad Barb committed
856

857
	print "Running 'tbend -force $pid $eid'\n";
Chad Barb's avatar
   
Chad Barb committed
858
859
860
	if (system("$tbdir/tbend -force $pid $eid") != 0) {
	    print "tbend failed!\n";
	}
861
	# Must override since we are so badly hosed. 
862
	$termswapstate = EXPTSTATE_TERMINATED;
Chad Barb's avatar
   
Chad Barb committed
863
864
    }

865
866
867
    # Copy over the log files so the user can see them.
    system("/bin/cp -Rfp $workdir/ $userdir/tbdata");

868
869
870
    # Set proper state, which is typically the way we came in.
    SetExpState($pid, $eid, $termswapstate);

871
    #
872
    # In batch mode, exit without sending the email or unlocking. The
873
    # batch daemon will take care of that.
874
875
    #
    if ($batch) {
876
	exit($errorstat);
877
878
    }

879
    #
Chad Barb's avatar
Chad Barb committed
880
    # Clear the log file so the web page stops spewing.
881
882
883
884
885
    #
    if (defined($logname)) {
	TBExptCloseLogFile($pid, $eid);
    }

886
887
    # Unlock and reset state to its terminal value.
    TBUnLockExp($pid, $eid);
888
889
890
891
892

    #
    # Clear the cancel flag now that the operation is complete. Must be done
    # after we change the experiment state (above).
    #
893
    TBSetCancelFlag($pid, $eid, EXPTCANCEL_CLEAR);
894

895
896
897
898
    #
    # Send a message to the testbed list. Append the logfile.
    #
    SENDMAIL("$user_name <$user_email>",
899
	     "Swap ${inout} Failure: $pid/$eid",
900
	     $mesg,
901
	     ($idleswap ? $TBOPS : "$user_name <$user_email>"),
902
	     "Cc:  $expt_head_name <$expt_head_email>\n".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
903
	     "Cc:  $TBOPS",
904
	     (($logname), (defined($modnsfile) ? ($modnsfile) : ())));
905

Leigh B. Stoller's avatar
Leigh B. Stoller committed
906
    if ($modifyHosed) {
Chad Barb's avatar
   
Chad Barb committed
907
908
909
910
911
912
913
914
915
	#
	# Copy off the workdir to the user directory, Then back up both of
	# them for post-mortem debugging.
	#
	system("/bin/cp -Rfp $workdir/ $userdir/tbdata");
	system("/bin/rm -rf  ${workdir}-failed");
	system("/bin/mv -f   $workdir ${workdir}-failed");
	system("/bin/rm -rf  ${userdir}-failed");
	system("/bin/mv -f   $userdir ${userdir}-failed");
Chad Barb's avatar
Chad Barb committed
916
	TBExptDestroy($pid, $eid);
Chad Barb's avatar
   
Chad Barb committed
917
918
    }

919
    exit($errorstat);
920
}