tbtest 18.1 KB
Newer Older
1
2
#!/usr/bin/perl -w

Leigh B. Stoller's avatar
Leigh B. Stoller committed
3
4
5
6
7
8
9
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# All rights reserved.
#


10
11
12
13
14
15
16
17
18
19
# tbtest

# This is the toplevel of the testbed testing setup.  This command
# should be invokved by a user and script.  It well then create a
# separate object tree, configure it appropriately, setup a testing
# database, and then proceed to run tests.

# IMPORTANT: This command should be come from a tree configured 
# for the real DB.

20
# tbtest [options] <mode stuff>
21
22
# Options:
#   -path <path> - Path to test tree, otherwise uses cwd.
23
24
25
#   -full - Do full testing.
#   -frontend - Do frontend only testing.#
#   -leavedb - Avoids removing the DB on exit.
26
#   -exitonfail - Exit on the first failure.
27
#   -session <sessionid> - Specify an alternate session id.
28
#   -flest - Generate flest.log file for flest runs.
29
30
#
# <mode> is one of:
31
#    run <db> <testlist> [<pid> <eid> <num>] - Do everything.
32
#    init <db> [<pid> <eid> <num>] - Initiatlize.  
33
34
#    test <testlist> - Run tests
#    single <tests> - Run only <tests>
35
#    finish - Finish run
36
# <num> is "X <type> ...", for example "10 pc 16 shark"
37
38
39
40

use DBI;
use POSIX;

41
42
#die("*** $0:\n".
#    "    Testsuite Disabled. Come edit me if need be.\n");
43

44
45
$| = 1;

46
47
48
49
50
51
#
# Set our WITH_TB_ADMIN_PRIVS environment variable, so that we have the
# privs we need to clear experiments out of the test database, etc.
#
$ENV{WITH_TB_ADMIN_PRIVS} = 1;

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
$pwd = `/bin/pwd`;
chop $pwd;

# Figure out where the script is
if ($0 =~ m|/|) {
    # Not pwd
    @a = split("/",$0);
    $dir = join("/",@a[0..$#a-1]);
    if ($0 =~ m|^/|) {
	# absolute path
	$basedir = $dir;
    } else {
	# relative path
	chdir $dir;
	$basedir = `/bin/pwd`;
	chop $basedir;
	chdir $pwd;
    }
} else {
    $basedir = `/bin/pwd`;
    chop $basedir;
}

$path = $pwd;

77
$ENV{'PATH'} = "/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/site/bin:$basedir:.";
78
79
80
$basepath = $ENV{'PATH'};

$type = "frontend";
81
82

# Parse argv
83
84
85
86
87
88
sub show_help {
    print STDERR "Syntax: $0 [-leavedb] [-path <path>] [-frontend] [-full] <mode>\n";
    print STDERR "Options:\n";
    print STDERR "  -leavedb - Do not drop the test database on exit.\n";
    print STDERR "  -path <path> - Path to directory to store test files.\n";
    print STDERR "  -frontend - Run in frontend mode.\n";
89
    print STDERR "  -full - Run in full mode.\n";
90
    print STDERR "  -exitonfail - Exit on any failure.\n";
91
    print STDERR "  -session <sessionid> - Specify an alternate session id.\n";
92
    print STDERR "  -flest - Generate flest.log for flest runs.\n";
93
    print STDERR "Mode:\n";
94
    print STDERR "  run <db> <testlist> [<pid> <eid> <num>]\n";
95
    print STDERR "  init <db> [<pid> <eid> <num>]\n";
96
97
    print STDERR "  test <testlist>\n";
    print STDERR "  single <tests>\n";
98
    print STDERR "  finish\n";
99
100
101
102
103
    print STDERR "<num>: Must be even lengthed list of <number> <type>.\n";
    print STDERR "   Ex: \"4 pc 16 shark\"\n";
    print STDERR "Examples:\n";
    print STDERR "   $0 run tbdb frontend\n";
    print STDERR "   $0 -full run tbdb frontend testbed mytest \"10 pc 16 shark\"\n";
104
    print STDERR "   $0 init tbdb;$0 single ping basic;$0 finish\n";
105
106
107
108
109
    exit(1);
};

$mode = "";
$leavedb = 0;
110
$exitonfail = 0;
111
$flestlog = 0;
112
113
$user=`whoami`;
chop $user;
114
115

while (($#ARGV > -1) && ($mode eq "")) {
116
117
    $arg = shift;
    if ($arg eq "-path") {
118
	if ($#ARGV == -1) {show_help;}
119
	$path = shift;
120
121
122
123
124
125
    } elsif ($arg eq "-frontend") {
	$type = "frontend";
    } elsif ($arg eq "-full") {
	$type = "full";
    } elsif ($arg eq "-leavedb") {
	$leavedb = 1;
126
127
    } elsif ($arg eq "-exitonfail") {
	$exitonfail = 1;
128
129
130
    } elsif ($arg eq "-session") {
	if ($#ARGV == -1) {show_help;}
	$user = shift;
131
132
    } elsif ($arg eq "-flest") {
	$flestlog = 1;
133
    } else {
134
	$mode = $arg;
135
136
137
138
    }
}

chdir $path;
139
140
$logfile = "test.log";
$statefile = "state";
141

142
143
144
if ($mode eq "run") {
    if ($type eq "frontend") {
	if ($#ARGV != 1) {show_help;}
145
	($TBDB,$testlist) = @ARGV;
146
147
    } else {
	if ($#ARGV != 4) {show_help;}
148
	($TBDB,$testlist,$pid,$eid,$num) = @ARGV;
149
150
151
152
153
154
155
    }
} elsif ($mode eq "init") {
    if ($type eq "frontend") {
	if ($#ARGV != 0) {show_help;}
	($TBDB) = @ARGV;
    } else {
	if ($#ARGV != 3) {show_help;}
156
	($TBDB,$pid,$eid,$num) = @ARGV;
157
158
159
    }
} elsif ($mode eq "test") {
    if ($#ARGV != 0) {show_help;}
160
    ($testlist) = @ARGV;
161
} elsif ($mode eq "single") {
162
163
    if ($#ARGV == -1) {show_help;}
    (@tests) = @ARGV;
164
    $testlist = "N/A";
165
} elsif ($mode eq "finish") {
166
    if ($#ARGV != -1) {show_help;}
167
168
169
} else {
    print STDERR "Unknown mode: $mode\n";
    show_help;
170
171
}

172
173
174
175
176
177
178
179
180
181
182
if (($mode ne "run") && ($mode ne "init")) {
    # Read state
    open(STATE,$statefile) || do {
	print STDERR "Could not find state file ($statefile).\n";
	print STDERR "  Run in init mode first.\n";
	exit(1);
    };
    ($type,$TBDB,$user,$pid,$eid) = split(" ",<STATE>);
    close(STATE);
}

183
184
185
186
187
188
189
if (($type eq "full") && (($mode eq "init") || ($mode eq "run"))) {
    @t = split(" ",$num);
    if (($#t % 2) == 0) {
	print STDERR "<num> must be list of even length.\n";
	show_help;
    }
}
190
191

$testdb = "tbdb_$user";
192
193
194
195
196

# Test counts
$pass = 0;
$fail = 0;
$unknown = 0;
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216

# prints sends the string to stdout and the logfile.
sub prints {
    print $_[0];
    print LOG $_[0];
};

# tbexec <cmd> <file>
# Executes <cmd> storing results to <file> and displaying appropriate
# message and exiting on failure.
sub tbexec {
    my ($cmd,$file) = @_;
    open(EXEC,"$cmd 2>&1 |");
    open(FILE,"> $file");
    while (<EXEC>) {
	print FILE $_;
    }
    close(FILE);
    close(EXEC);
    if (($? >> 8) == 0) {
Christopher Alfeld's avatar
Christopher Alfeld committed
217
	prints "PASS\n";
218
219
	$pass++;
    } else {
Christopher Alfeld's avatar
Christopher Alfeld committed
220
	prints "FAIL - See $file\n";
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
	$fail++;
	doexit(1);
    }
};

# tbexect <cmd> <logfp>
# This is similar to tbexec except is used for running commands specific
# to tests.  It runs the command appending the output to the file handle
# <logfp> and returns the exit code.
sub tbexect {
    my ($cmd,$logfp) = @_;
    print $logfp "$cmd\n";
    open(EXEC,"$cmd 2>&1 |");
    while (<EXEC>) {
	print $logfp $_;
    }
    close(EXEC);
    return ($? >> 8);
};

# doexit
# This displays exit counts and then exits with the specified exit code.
sub doexit {
    printf LOG "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
    printf "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
246
247
    if ($_[0] != 0) {
	prints "Run in finish mode to clean up any residual state.\n";
248
    }
249
    close(LOG);
250
251
252
    exit($_[0]);
};

253
254
255
256
257
258
259
260
261
262
263
# These are flest routines
sub pass {
    if ($flestlog) {
	print FLESTLOG "PASS $_[0]\n";
    }
};
sub fail {
    if ($flestlog) {
	print FLESTLOG "FAIL $_[0]\n";
    }
};
264
265
266
267
268
sub fail2 {
    if ($flestlog) {
	print FLESTLOG "FAIL: $_[0]\n";
    }
}
269

270
271
sub step_init {
    open(LOG,"> $logfile");
272

273
    prints "START INIT\n";
274
    prints "Session:  $user\n";
275
276
    prints "Logfile:  $logfile\n";
    prints "Database: $TBDB\n";
277
    prints "Type:     $type\n";
278
    prints "\n";
279
    prints "Beginning test run. " . ctime(time);
280

281
282
283
284
285
286
    # Initialize DB
    $dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || 
	die "Could not connect to DB.\n";
    # Having made sure we can, we now disconnect so as not to harm DB.
    $dbh->disconnect;

287
288
289
290
291
292
293
294
295
    # Write state
    open(STATE,"> $statefile");
    if ($type eq "full") {
	print STATE "$type $TBDB $user $pid $eid\n";
    } else {
	print STATE "$type $TBDB $user\n";
    }
    close(STATE);

296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
    # Reserve nodes
    if ($type eq "full") {
	prints "Reserving nodes.\n";
	
	my (@nums) = split(" ",$num);
	my $num;
	my $type;
	my %want;
	my %have;
	my $node;

	while ($#nums > -1) {
	    $num = shift @nums;
	    $type = shift @nums;
	    $want{$type} = $num;
	    $have{$type} = [];
	}
	
314
	open(AVAIL,"/usr/testbed/sbin/avail includevirt |");
315
316
317
	while (<AVAIL>) {
	    if (! /^\|/) {next};
	    if (/node_id/) {next;}
318
	    ($node,$type,$class) = /^\|([-a-zA-Z0-9]+)\s*\|(\w+)\s*\|(\w+)\s*\|$/;
319
320
321
	    if (defined($want{$type}) && $want{$type} > 0) {
		push(@{$have{$type}},$node);
		$want{$type}--;
322
323
324
	    } elsif (defined($want{$class}) && $want{$class} > 0) {
		push(@{$have{$class}},$node);
		$want{$class}--;
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
	    }
	}
	close(AVAIL);

	my $good = 1;
	foreach $type (keys(%want)) {
	    if ($want{$type} > 0) {
		print STDERR "Could not get enough $type - Need $want{$type} more.\n";
		$good = 0;
	    }
	}
	if (! $good) {exit(1);}

	my $raw = "";
	foreach $type (keys(%have)) {
340
	    $raw .= " " . join(" ",@{$have{$type}});
341
342
343
	}
	prints "  Reserving ";
	tbexec("/usr/testbed/bin/nalloc $pid $eid $raw","$path/reserve.log");
344
345
346
347

	prints "  Setting up tmcc redirect ";
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || 
	    die "Could not connect to DB.\n";
348
349
350
351
352
353
354
	$dbh->do("UPDATE experiments set testdb=\"$testdb\" where" .
		 " pid=\"$pid\" and eid=\"$eid\"") || do {
		     prints "FAIL\n";
		     $fail++;
		     doexit(1);
		 };
	$pass++;
355
356
	prints "PASS\n";
	$dbh->disconnect;
357
358
    }

359
360
361
362
    # Set up object tree
    prints "Setting up object tree.\n";
    if (! -d $path) {
	mkdir $path,0755;
363
    }
364
365
366
367
    mkdir "tbobj",0755;
    mkdir "install",0755;
    chdir "tbobj";
    system("cp -f $basedir/test-defs $path/defs");
368
369
370
371
    if (($? >> 8) != 0) {
	print STDERR "ERROR: Could not copy defs file.\n";
	doexit(1);
    }
372
373
374
375
376
377
378
    open(DEFS,">>$path/defs");
    print DEFS "TBDBNAME=$testdb\n";
    print DEFS "TBOPSEMAIL=$user\@emulab.net\n";
    print DEFS "TBLOGSEMAIL=$user\@emulab.net\n";
    close(DEFS);
    
    prints "  Configuring ";
379
380
381
382
383
384
385
    my $testmode;
    if ($type eq "full") {
	$testmode = "";
    } else {
	$testmode = "--enable-testmode";
    }
    tbexec("$basedir/../configure --prefix=$path/install $testmode ".
386
	   "--disable-events --with-TBDEFS=$path/defs",
387
388
389
390
391
392
393
394
395
	   "$path/configure.log");
    
    prints "  Building ";
    tbexec("gmake","$path/build.log");
    
    prints "  Installing ";
    tbexec("gmake boss-install","$path/install.log");
    
    prints "  Post-Installing ";
Christopher Alfeld's avatar
Christopher Alfeld committed
396
    tbexec("sudo gmake post-install","$path/postinstall.log");
397
398
399
400
401
402
        
    chdir $path;
    
    # Setup test database
    prints "Setting up test database.\n";
    prints "  Dumping Current ";
403
    tbexec("mysqldump -e -a $TBDB","$path/dbdump.txt");
404
    # Now add a line to the dumped database that sets flest as an admin
Christopher Alfeld's avatar
Christopher Alfeld committed
405
    prints "  Setting up flest admin account ";
406
    tbexec("echo \"update users set admin=1 where uid='flest';\"".
407
	   " >> $path/dbdump.txt", "dbdump.log");
408
409
    
    prints "  Creating DB ";
410
    system("echo \"drop database if exists $testdb\" | mysql");
411
412
413
    tbexec("echo \"create database $testdb\" | mysql","createdb.log");
    
    prints "  Filling DB ";
414
    tbexec("cat $path/dbdump.txt | mysql -q -s $testdb","fill.log");
415
416


417
418
    $dbh = DBI->connect("DBI:mysql:database=$testdb;host=localhost") || do {
	prints "ERROR: Could not connect to new DB.\n";
419
	doexit(1);
420
    };
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
    $ENV{'PATH'} = "$basepath:$path/install/bin:$path/install/sbin";

    #
    # Clear our extraneous stuff.
    # 
    $sth = $dbh->prepare("SELECT * from testsuite_preentables");
    $sth->execute;
    while (($table,$action) = $sth->fetchrow_array) {
	$_ = $action;
        SWITCH: {
	    /^drop$/ && do {
		prints "    Dropping table $table\n";
		$dbh->do("drop table $table");
		last SWITCH;
	    };
	    /^clean$/ && do {
		prints "    Cleaning table $table\n";
		$dbh->do("delete from $table");
		last SWITCH;
	    };
	    /^prune$/ && do {
		prints "    Pruning table $table\n";
		$dbh->do("delete from $table ".
			 "where pid!='emulab-ops' and pid!='testbed'");
		last SWITCH;
	    };
	}
    }
449
450
451
452
453

    if ($type eq "frontend") {
	# Clear out experiments and set up test experiment.     
	prints "  Clearing out existing experiments ";
	open(CLEARLOG,"> clear.log");
454
455
	$sth = $dbh->prepare("SELECT pid,eid from experiments ".
			     "where !(pid='emulab-ops' and eid='hwdown')");
456
457
458
	$sth->execute;
	while (($pid,$eid) = $sth->fetchrow_array) {
	    print CLEARLOG "$pid $eid\n";
459
460
	    print CLEARLOG "tbswapout -force $pid $eid\n";
	    open(EXEC,"tbswapout -force $pid $eid 2>&1 |");
461
462
463
464
465
466
467
468
469
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints "FAIL - For $pid $eid - On tbswapout - See clear.log\n";
		$fail++;
		doexit(1);
	    } 
470
471
	    print CLEARLOG "tbend $pid $eid\n";
	    open(EXEC,"tbend $pid $eid 2>&1 |");
472
473
474
475
476
477
478
479
480
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints " FAIL - For $pid $eid - On tbend - See clear.log\n";
		$fail++;
		doexit(1);
	    }   
481
	}
482
	# now we clear 
483
484
	$dbh->do("delete from reserved ".
		 "where pid='emulab-ops' and eid!='hwdown'") || do {
485
486
487
488
489
	    prints "FAIL\n";
	    $fail++;
	    doexit(1);
	};

490
491
492
493
494
495
496
	prints "PASS\n";
	$pass++;

	close(CLEARLOG);
    } else {
	# Free nodes we allocated for the experiment and allocate all
	# others.
497
	prints "  Creating testbed/unavailable in test DB ";
498
499
500
501
502
503
504
	open(AVAIL,"avail |");
	$raw = "";
	while (<AVAIL>) {
	    if (! /^\|/) {next};
	    if (/node_id/) {next;}
	    ($node) = /^\|([-a-zA-Z0-9]+)/;
	    $raw .= " $node";
505
	}
506
507
508
509
510
511
512
513
514
515
516
	close(AVAIL);
	
	$dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"unavailable\"");
	$dbh->do("INSERT into experiments (pid,eid) "
		 . "values (\"testbed\",\"unavailable\")") || do {
		     prints "FAIL\n";
		     $fail++;
		     doexit(1);
		 };
	print "PASS\n";
	$pass++;
517

518
519
	prints "  Marking nodes as unavailable ";
	tbexec("nalloc testbed unavailable $raw","unavailable.log");
520

521
522
523
524
	prints "  Freeing test nodes ";
	tbexec("nfree $pid $eid","free.log");
    }
    
525
526
527
528
529
530
531
532
533
534
535
536
    prints "  Setting up test experiment ";
    $dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"test\"");
    $dbh->do("INSERT into experiments (pid,eid) "
	     . "values (\"testbed\",\"test\")") || do {
		 prints "FAIL\n";
		 $fail++;
		 doexit(1);
	     };
    print "PASS\n";
    $pass++;
    
    prints "  Snapshotting clean state ";
537
    tbexec("mysqldump -e -a $testdb","clean.txt");
538
    prints "END INIT\n";
539
540
}

541
542
543
544
545
sub step_test {
    open(LOG,">> $logfile");
    
    prints "START TEST\n";
    
546
547
548
549
550
551
    if ($flestlog) {
	open(FLESTLOG,"> flest.log");
    }
    
    pass("Pre-Test Setup");

552
553
    # Time to run tests!
    prints "Running Tests\n";
554
    prints "Testlist: $testlist\n";
555

556
557
558
559
560
561
562
563
    # Let's add some stuff to our path
    $ENV{'PATH'} = "$basepath:$path/install/bin:$path/install/sbin";
    
    my @tests;
    my $testmode;
    my $dir;

    if ($#_ == -1) {
564
565
566
567
568
569
570
	open(TESTLIST,"$basedir/testlists/$testlist");
	@tests = ();
	while (<TESTLIST>) {
	    chop;
	    push(@tests,$_);
	}
	close(TESTLIST);
571
572
    } else {
	@tests = @_;
573
574
    }

575
576
577
    mkdir "tests",0755;
    foreach $test (@tests) {
	if (($test eq ".") || ($test eq "..") || ($test eq "CVS")) {
578
579
	    next;
	}
580
581
	prints "  $test ";
	
582
583
	mkdir "tests/$test",0755;
	chdir "tests/$test";
584
585
586
587
588
589
	open(TESTLOG,"> test.log");
	
	# Set up DB state
	if (tbexect("echo \"drop database if exists $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not drop test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
590
	    fail2("$test - Drop test DB");
591
592
593
594
595
596
	    $fail++;
	    doexit(1);
	}
	if (tbexect("echo \"create database $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not recreate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
597
	    fail2("$test - Recreate test DB");
598
	    $fail++;
599
600
	    doexit(1);
	}
601
	if (tbexect("cat $path/clean.txt | mysql -q -s $testdb",TESTLOG)) {
602
603
	    prints "FAIL - Could not populate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
604
	    fail2("$test - Populate test DB");
605
606
607
608
	    $fail++;
	    doexit(1);
	}
	
609
610
	if (-f "$basedir/tests/$test/dbstate") {
	    if (tbexect("cat $basedir/tests/$test/dbstate | mysql $testdb",TESTLOG)) {
611
		prints "FAIL - Could not evaluate dbstate.\n";
612
		fail2("$test - Evaluate dbstate");
613
		$fail++;
614
		if ($exitonfail) {doexit(1);}
615
		next;
Christopher Alfeld's avatar
Christopher Alfeld committed
616
	    }
617
618
619
620
621
622
623
	}
	
	# We're now ready to run the test.
	
	# The last line of this command will be either PASS
	# or FAIL <msg>, all output before that should be sent 
	# to the log file.
624
	open(TESTFP,"tbstub $testdb testbed test $test $basedir/tests/$test |");
625
626
627
628
629
630
631
632
	while (<TESTFP>) {
	    print TESTLOG $_;
	    $status = $_;
	}
	chop $status;
	close(TESTFP);
	if (($? >> 8) != 0) {
	    prints "UNKNOWN - Test exited with non-0 code.\n";
633
	    $unknown++;
634
635
636
637
638
639
	} else {
	    if ($status =~ /^PASS/) {
		$pass++;
	    } elsif ($status =~ /^FAIL/) {
		$fail++;
		# grab copy of testbed
640
		system("mysqldump -e -a $testdb > db.txt");
641
642
643
644
		if (($? >> 8) != 0) {
		    prints "ERROR: Could not grab copy of database.\n";
		    doexit(1);
		}
645
		if ($exitonfail) {prints "FAIL\n"; doexit(1);}
646
647
	    } else {
		$unknown++;
648
		$status = "UKNOWN";
649
650
	    }
	    prints "$status\n";
651
652
653
654
	    if ($flestlog) {
		@t = split(" ",$status);
		$result = $t[0];
		$msg = join(" ",@t[1..$#t]);
655
		print FLESTLOG "$result: $test $msg\n";
656
	    }
657
	}
658
659
660
661
	
	close(TESTLOG);
	
	chdir $path;
662
    }
663
664
665
    if ($flestlog) {
	close FLESTLOG;
    }
666
    prints "END TEST\n";
667
}
668
   
669
670
sub step_cleanup {
    open(LOG,">> $logfile");
671
672
    
    prints "START cleanup\n";
673

674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
    # Try to do the necessary cleanup
    if ($leavedb == 0) {
	prints "  Dropping test DB ";
	system("echo \"drop database if exists $testdb\" | mysql");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
	    prints "PASS\n";
	    $pass++;
	}
    }
    if (($type eq "full")) { 	
	prints "  Clearing redirects ";
	$ok = 1;
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || do {
	    $ok = 0;
	};
	if (! $ok) {
	    prints "FAIL - Could not connect to db\n";
	    $fail++;
	} else {
	    $ok = 1;
697
698
699
700
	    $dbh->do("UPDATE experiments set testdb=NULL" .
		     " where pid=\"$pid\" and eid=\"$eid\"") || do {
			 $ok = 0;
		     };
701
	    if ($ok) {
702
		$pass++;
703
		prints "PASS\n";
704
705
706
	    } else {
		$fail++;
		prints "FAIL\n";
707
708
709
	    }
	    $dbh->disconnect;
	}
710

711
712
713
714
715
716
	prints "  Freeing nodes ";
	system("/usr/testbed/bin/nfree $pid $eid > /dev/null");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
717
	    prints("PASS\n");
718
719
	    $pass++;
	}
720
    }
721
722
723
724
725
726
727
728
729
    prints "  Clearing state ";
    system("rm -f $statefile");
    if (($? >> 8) != 0) {
	prints "FAIL\n";
	$fail++;
    } else {
	prints "PASS\n";
	$pass++;
    }
730
    prints "End test run. " . ctime(time);
731
732
    prints "END CLEANUP\n";
}
733
734

	     
735
736
737
738
739
740
741
742
743
744
745
746
747
if ($mode eq "run") {
    step_init;
    step_test;
    step_cleanup;
} elsif ($mode eq "init") {
    step_init;
} elsif ($mode eq "test") {
    step_test;
} elsif ($mode eq "single") {
    step_test @tests;
} elsif ($mode eq "finish") {
    step_cleanup;
}
748

749
750
open(LOG,">> $logfile");

751
752
# Do exit does all cleanup.
doexit(0);