tbtest 20.8 KB
Newer Older
1 2
#!/usr/bin/perl -w

Leigh B. Stoller's avatar
Leigh B. Stoller committed
3 4
#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
6 7 8 9
# All rights reserved.
#


10 11 12 13 14 15 16
# tbtest

# This is the toplevel of the testbed testing setup.  This command
# should be invokved by a user and script.  It well then create a
# separate object tree, configure it appropriately, setup a testing
# database, and then proceed to run tests.

17
# IMPORTANT: This command should be come from a tree configured
18 19
# for the real DB.

20
# tbtest [options] <mode stuff>
21 22
# Options:
#   -path <path> - Path to test tree, otherwise uses cwd.
23 24 25
#   -full - Do full testing.
#   -frontend - Do frontend only testing.#
#   -leavedb - Avoids removing the DB on exit.
26
#   -exitonfail - Exit on the first failure.
27
#   -session <sessionid> - Specify an alternate session id.
28
#   -flest - Generate flest.log file for flest runs.
29
#   -daikon - 
30 31
#
# <mode> is one of:
32
#    run <db> <testlist> [<pid> <eid> <num>] - Do everything.
33
#    init <db> [<pid> <eid> <num>] - Initiatlize.
34 35
#    test <testlist> - Run tests
#    single <tests> - Run only <tests>
36
#    finish - Finish run
37
# <num> is "X <type> ...", for example "10 pc 16 shark"
38 39 40

use DBI;
use POSIX;
41
use BSD::Resource;
42

43 44
#die("*** $0:\n".
#    "    Testsuite Disabled. Come edit me if need be.\n");
45

46 47
$| = 1;

48 49 50
# Lower priority since this whole thing consumes CPU like nuts
setpriority(PRIO_PROCESS, 0, 15);

51 52 53 54 55 56
#
# Set our WITH_TB_ADMIN_PRIVS environment variable, so that we have the
# privs we need to clear experiments out of the test database, etc.
#
$ENV{WITH_TB_ADMIN_PRIVS} = 1;

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
$pwd = `/bin/pwd`;
chop $pwd;

# Figure out where the script is
if ($0 =~ m|/|) {
    # Not pwd
    @a = split("/",$0);
    $dir = join("/",@a[0..$#a-1]);
    if ($0 =~ m|^/|) {
	# absolute path
	$basedir = $dir;
    } else {
	# relative path
	chdir $dir;
	$basedir = `/bin/pwd`;
	chop $basedir;
	chdir $pwd;
    }
} else {
    $basedir = `/bin/pwd`;
    chop $basedir;
}

$path = $pwd;
81
$TB = "$path/install";
82

83
$ENV{'PATH'} = "/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/site/bin:$basedir:.";
84 85 86
$basepath = $ENV{'PATH'};

$type = "frontend";
87 88

# Parse argv
89 90 91 92 93 94
sub show_help {
    print STDERR "Syntax: $0 [-leavedb] [-path <path>] [-frontend] [-full] <mode>\n";
    print STDERR "Options:\n";
    print STDERR "  -leavedb - Do not drop the test database on exit.\n";
    print STDERR "  -path <path> - Path to directory to store test files.\n";
    print STDERR "  -frontend - Run in frontend mode.\n";
95
    print STDERR "  -full - Run in full mode.\n";
96
    print STDERR "  -exitonfail - Exit on any failure.\n";
97
    print STDERR "  -session <sessionid> - Specify an alternate session id.\n";
98
    print STDERR "  -flest - Generate flest.log for flest runs.\n";
99
    print STDERR "  -daikon\n";
100
    print STDERR "Mode:\n";
101
    print STDERR "  run <db> <testlist> [<pid> <eid> <num>]\n";
102
    print STDERR "  init <db> [<pid> <eid> <num>]\n";
103 104
    print STDERR "  test <testlist>\n";
    print STDERR "  single <tests>\n";
105
    print STDERR "  finish\n";
106 107 108 109 110
    print STDERR "<num>: Must be even lengthed list of <number> <type>.\n";
    print STDERR "   Ex: \"4 pc 16 shark\"\n";
    print STDERR "Examples:\n";
    print STDERR "   $0 run tbdb frontend\n";
    print STDERR "   $0 -full run tbdb frontend testbed mytest \"10 pc 16 shark\"\n";
111
    print STDERR "   $0 init tbdb;$0 single ping basic;$0 finish\n";
112 113 114 115 116
    exit(1);
};

$mode = "";
$leavedb = 0;
117
$exitonfail = 0;
118
$flestlog = 0;
119
$daikon = 0;
120 121
$user=`whoami`;
chop $user;
122 123

while (($#ARGV > -1) && ($mode eq "")) {
124 125
    $arg = shift;
    if ($arg eq "-path") {
126
	if ($#ARGV == -1) {show_help;}
127
	$path = shift;
128 129 130 131 132 133
    } elsif ($arg eq "-frontend") {
	$type = "frontend";
    } elsif ($arg eq "-full") {
	$type = "full";
    } elsif ($arg eq "-leavedb") {
	$leavedb = 1;
134 135
    } elsif ($arg eq "-exitonfail") {
	$exitonfail = 1;
136 137 138
    } elsif ($arg eq "-session") {
	if ($#ARGV == -1) {show_help;}
	$user = shift;
139 140
    } elsif ($arg eq "-flest") {
	$flestlog = 1;
141 142
    } elsif ($arg eq "-daikon") {
	$daikon = 1;
143
    } else {
144
	$mode = $arg;
145 146 147 148
    }
}

chdir $path;
149 150
$logfile = "test.log";
$statefile = "state";
151

152 153 154
if ($mode eq "run") {
    if ($type eq "frontend") {
	if ($#ARGV != 1) {show_help;}
155
	($TBDB,$testlist) = @ARGV;
156 157
    } else {
	if ($#ARGV != 4) {show_help;}
158
	($TBDB,$testlist,$pid,$eid,$num) = @ARGV;
159 160 161 162 163 164 165
    }
} elsif ($mode eq "init") {
    if ($type eq "frontend") {
	if ($#ARGV != 0) {show_help;}
	($TBDB) = @ARGV;
    } else {
	if ($#ARGV != 3) {show_help;}
166
	($TBDB,$pid,$eid,$num) = @ARGV;
167 168 169
    }
} elsif ($mode eq "test") {
    if ($#ARGV != 0) {show_help;}
170
    ($testlist) = @ARGV;
171
} elsif ($mode eq "single") {
172 173
    if ($#ARGV == -1) {show_help;}
    (@tests) = @ARGV;
174
    $testlist = "N/A";
175
} elsif ($mode eq "finish") {
176
    if ($#ARGV != -1) {show_help;}
177 178 179
} else {
    print STDERR "Unknown mode: $mode\n";
    show_help;
180 181
}

182 183 184 185 186 187 188 189 190 191 192
if (($mode ne "run") && ($mode ne "init")) {
    # Read state
    open(STATE,$statefile) || do {
	print STDERR "Could not find state file ($statefile).\n";
	print STDERR "  Run in init mode first.\n";
	exit(1);
    };
    ($type,$TBDB,$user,$pid,$eid) = split(" ",<STATE>);
    close(STATE);
}

193 194 195 196 197 198 199
if (($type eq "full") && (($mode eq "init") || ($mode eq "run"))) {
    @t = split(" ",$num);
    if (($#t % 2) == 0) {
	print STDERR "<num> must be list of even length.\n";
	show_help;
    }
}
200 201

$testdb = "tbdb_$user";
202
$bossnode = `hostname`;
203 204 205 206 207

# Test counts
$pass = 0;
$fail = 0;
$unknown = 0;
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227

# prints sends the string to stdout and the logfile.
sub prints {
    print $_[0];
    print LOG $_[0];
};

# tbexec <cmd> <file>
# Executes <cmd> storing results to <file> and displaying appropriate
# message and exiting on failure.
sub tbexec {
    my ($cmd,$file) = @_;
    open(EXEC,"$cmd 2>&1 |");
    open(FILE,"> $file");
    while (<EXEC>) {
	print FILE $_;
    }
    close(FILE);
    close(EXEC);
    if (($? >> 8) == 0) {
228
	prints "PASS\n";
229 230
	$pass++;
    } else {
231
	prints "FAIL - See $file\n";
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
	$fail++;
	doexit(1);
    }
};

# tbexect <cmd> <logfp>
# This is similar to tbexec except is used for running commands specific
# to tests.  It runs the command appending the output to the file handle
# <logfp> and returns the exit code.
sub tbexect {
    my ($cmd,$logfp) = @_;
    print $logfp "$cmd\n";
    open(EXEC,"$cmd 2>&1 |");
    while (<EXEC>) {
	print $logfp $_;
    }
    close(EXEC);
    return ($? >> 8);
};

# doexit
# This displays exit counts and then exits with the specified exit code.
sub doexit {
    printf LOG "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
    printf "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
257 258
    if ($_[0] != 0) {
	prints "Run in finish mode to clean up any residual state.\n";
259
    }
260
    close(LOG);
261 262 263
    exit($_[0]);
};

264 265 266 267 268 269 270 271 272 273 274
# These are flest routines
sub pass {
    if ($flestlog) {
	print FLESTLOG "PASS $_[0]\n";
    }
};
sub fail {
    if ($flestlog) {
	print FLESTLOG "FAIL $_[0]\n";
    }
};
275 276 277 278 279
sub fail2 {
    if ($flestlog) {
	print FLESTLOG "FAIL: $_[0]\n";
    }
}
280

281 282
sub step_init {
    open(LOG,"> $logfile");
283

284
    prints "START INIT\n";
285
    prints "Session:  $user\n";
286 287
    prints "Logfile:  $logfile\n";
    prints "Database: $TBDB\n";
288
    prints "Type:     $type\n";
289
    prints "\n";
290
    prints "Beginning test run. " . ctime(time);
291

292
    # Initialize DB
293
    $dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") ||
294 295 296 297
	die "Could not connect to DB.\n";
    # Having made sure we can, we now disconnect so as not to harm DB.
    $dbh->disconnect;

298 299 300 301 302 303 304 305 306
    # Write state
    open(STATE,"> $statefile");
    if ($type eq "full") {
	print STATE "$type $TBDB $user $pid $eid\n";
    } else {
	print STATE "$type $TBDB $user\n";
    }
    close(STATE);

307 308 309
    # Reserve nodes
    if ($type eq "full") {
	prints "Reserving nodes.\n";
310

311 312 313 314 315 316 317 318 319 320 321 322 323
	my (@nums) = split(" ",$num);
	my $num;
	my $type;
	my %want;
	my %have;
	my $node;

	while ($#nums > -1) {
	    $num = shift @nums;
	    $type = shift @nums;
	    $want{$type} = $num;
	    $have{$type} = [];
	}
324

325
	open(AVAIL,"/usr/testbed/sbin/avail includevirt rand |");
326
	while (<AVAIL>) {
327 328 329 330 331 332 333 334 335 336 337 338 339
	    chomp;
	    if (! /^\|/) { next; }
	    if (/^\|node_id\s*\|/) { next; }
	    if ( ($node,$type,$class) =
		 /^\|([-a-z0-9]+)\s*\|([-a-z0-9]+)\s*\|(\w+)\s*\|$/i ) {
		if (defined($want{$type}) && $want{$type} > 0) {
		    push(@{$have{$type}},$node);
		    $want{$type}--;
		} elsif (defined($want{$class}) && $want{$class} > 0) {
		    push(@{$have{$class}},$node);
		    $want{$class}--;
		}
	    } else { print "Couldn't parse avail line '$_'\n"; }
340 341 342 343 344 345 346 347 348 349 350 351 352 353
	}
	close(AVAIL);

	my $good = 1;
	foreach $type (keys(%want)) {
	    if ($want{$type} > 0) {
		print STDERR "Could not get enough $type - Need $want{$type} more.\n";
		$good = 0;
	    }
	}
	if (! $good) {exit(1);}

	my $raw = "";
	foreach $type (keys(%have)) {
354
	    $raw .= " " . join(" ",@{$have{$type}});
355 356 357
	}
	prints "  Reserving ";
	tbexec("/usr/testbed/bin/nalloc $pid $eid $raw","$path/reserve.log");
358 359 360
	# Have to run exports_setup, or the nodes won't come up properly
	prints "  Setting up exports ";
	tbexec("/usr/testbed/sbin/exports_setup","$path/exports_setup.log");
361 362

	prints "  Setting up tmcc redirect ";
363
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") ||
364
	    die "Could not connect to DB.\n";
365 366 367 368 369 370 371
	$dbh->do("UPDATE experiments set testdb=\"$testdb\" where" .
		 " pid=\"$pid\" and eid=\"$eid\"") || do {
		     prints "FAIL\n";
		     $fail++;
		     doexit(1);
		 };
	$pass++;
372 373
	prints "PASS\n";
	$dbh->disconnect;
374 375
    }

376 377 378 379
    # Set up object tree
    prints "Setting up object tree.\n";
    if (! -d $path) {
	mkdir $path,0755;
380
    }
381 382 383 384
    mkdir "tbobj",0755;
    mkdir "install",0755;
    chdir "tbobj";
    system("cp -f $basedir/test-defs $path/defs");
385 386 387 388
    if (($? >> 8) != 0) {
	print STDERR "ERROR: Could not copy defs file.\n";
	doexit(1);
    }
389
    open(DEFS,">>$path/defs");
390 391 392
    print DEFS "TBDBNAME=$testdb\n";
    print DEFS "TBOPSEMAIL=$user\@emulab.net\n";
    print DEFS "TBLOGSEMAIL=$user\@emulab.net\n";
393 394
    print DEFS "TBSTATEDEMAIL=$user\@emulab.net\n";
    print DEFS "OURDOMAIN=test.$user.emulab.net\n";
395
    print DEFS "BOSSNODE=$bossnode\n";
396
    close(DEFS);
397

398
    prints "  Configuring ";
399 400 401 402
    my $testmode;
    if ($type eq "full") {
	$testmode = "";
    } else {
403
	$testmode = "--enable-testmode --disable-events";
404
    }
405 406
    open(CONF,">$path/config");
    print CONF "$basedir/../configure \\\n".
407
      "--prefix=$TB \\\n$testmode \\\n--with-TBDEFS=$path/defs\n";
408
    close(CONF);
409
    tbexec("$basedir/../configure --prefix=$TB $testmode ".
410
	   "--with-TBDEFS=$path/defs",
411
	   "$path/configure.log");
412

413 414
    prints "  Building ";
    tbexec("gmake","$path/build.log");
415

416 417
    prints "  Installing ";
    tbexec("gmake boss-install","$path/install.log");
418

419
    prints "  Post-Installing ";
420
    tbexec("sudo gmake post-install","$path/postinstall.log");
421

422
    chdir $path;
423

424 425 426
    # Setup test database
    prints "Setting up test database.\n";
    prints "  Dumping Current ";
427
    tbexec("mysqldump -e -a $TBDB","$path/dbdump.txt");
428
    # Now add a line to the dumped database that sets flest as an admin
429 430
    prints "  Setting up admin account for $user in test database ";
    tbexec("echo \"update users set admin=1 where uid='$user';\"".
431
	   " >> $path/dbdump.txt", "dbdump.log");
432

433
    prints "  Creating DB ";
434
    system("echo \"drop database if exists $testdb\" | mysql");
435
    tbexec("echo \"create database $testdb\" | mysql","createdb.log");
436

437
    prints "  Filling DB ";
438
    tbexec("cat $path/dbdump.txt | mysql -q -s $testdb","fill.log");
439 440


441 442
    $dbh = DBI->connect("DBI:mysql:database=$testdb;host=localhost") || do {
	prints "ERROR: Could not connect to new DB.\n";
443
	doexit(1);
444
    };
445
    $ENV{'PATH'} = "$basepath:$TB/bin:$TB/sbin";
446 447 448

    #
    # Clear our extraneous stuff.
449
    #
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
    $sth = $dbh->prepare("SELECT * from testsuite_preentables");
    $sth->execute;
    while (($table,$action) = $sth->fetchrow_array) {
	$_ = $action;
        SWITCH: {
	    /^drop$/ && do {
		prints "    Dropping table $table\n";
		$dbh->do("drop table $table");
		last SWITCH;
	    };
	    /^clean$/ && do {
		prints "    Cleaning table $table\n";
		$dbh->do("delete from $table");
		last SWITCH;
	    };
	    /^prune$/ && do {
		prints "    Pruning table $table\n";
		$dbh->do("delete from $table ".
			 "where pid!='emulab-ops' and pid!='testbed'");
		last SWITCH;
	    };
	}
    }
473 474

    if ($type eq "frontend") {
475
	# Clear out experiments and set up test experiment.
476 477
	prints "  Clearing out existing experiments ";
	open(CLEARLOG,"> clear.log");
478
	$sth = $dbh->prepare("SELECT pid,eid from experiments ".
479 480
			     "where !(pid='emulab-ops' and eid='hwdown') " .
			     "and state != 'swapped'");
481 482 483
	$sth->execute;
	while (($pid,$eid) = $sth->fetchrow_array) {
	    print CLEARLOG "$pid $eid\n";
484 485
	    print CLEARLOG "tbswap out -force $pid $eid\n";
	    open(EXEC,"tbswap out -force $pid $eid 2>&1 |");
486 487 488 489 490 491 492 493
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints "FAIL - For $pid $eid - On tbswapout - See clear.log\n";
		$fail++;
		doexit(1);
494
	    }
495 496
	    print CLEARLOG "tbend -force $pid $eid\n";
	    open(EXEC,"tbend -force $pid $eid 2>&1 |");
497 498 499 500 501 502 503 504
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints " FAIL - For $pid $eid - On tbend - See clear.log\n";
		$fail++;
		doexit(1);
505 506 507
	    }
	    #
	    # tbend does not clear the experiments or nsfiles table.
508
	    #
509 510 511 512
	    $dbh->do("delete from experiments ".
		     "where pid='$pid' and eid='$eid'");
	    $dbh->do("delete from nsfiles ".
		     "where pid='$pid' and eid='$eid'");
513
	}
514
	# Clear reserved just to make sure.
515 516
	$dbh->do("delete from reserved ".
		 "where pid='emulab-ops' and eid!='hwdown'") || do {
517 518 519 520 521
	    prints "FAIL\n";
	    $fail++;
	    doexit(1);
	};

522 523 524
	# Clear all the users except the person running the test.
	$dbh->do("delete from users where uid!='$user'");

525 526 527 528 529 530 531
	prints "PASS\n";
	$pass++;

	close(CLEARLOG);
    } else {
	# Free nodes we allocated for the experiment and allocate all
	# others.
532
	prints "  Creating testbed/unavailable in test DB ";
533
	open(AVAIL,"avail includevirt |");
534 535 536 537 538 539
	$raw = "";
	while (<AVAIL>) {
	    if (! /^\|/) {next};
	    if (/node_id/) {next;}
	    ($node) = /^\|([-a-zA-Z0-9]+)/;
	    $raw .= " $node";
540
	}
541
	close(AVAIL);
542

543 544 545 546 547 548 549 550 551
	$dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"unavailable\"");
	$dbh->do("INSERT into experiments (pid,eid) "
		 . "values (\"testbed\",\"unavailable\")") || do {
		     prints "FAIL\n";
		     $fail++;
		     doexit(1);
		 };
	print "PASS\n";
	$pass++;
552

553 554
	prints "  Marking nodes as unavailable ";
	tbexec("nalloc testbed unavailable $raw","unavailable.log");
555

556 557 558
	prints "  Freeing test nodes ";
	tbexec("nfree $pid $eid","free.log");
    }
559

560 561
    prints "  Setting up test experiment ";
    $dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"test\"");
562 563
    $dbh->do("INSERT into experiments (pid,gid,eid,testdb) "
	     . "values (\"testbed\",\"testbed\",\"test\",\"$testdb\")") || do {
564 565 566 567 568 569
		 prints "FAIL\n";
		 $fail++;
		 doexit(1);
	     };
    print "PASS\n";
    $pass++;
570

571
    prints "  Snapshotting clean state ";
572
    tbexec("mysqldump -e -a $testdb","clean.txt");
573
    prints "END INIT\n";
574 575
}

576
sub test_pass {
577
    prints "START TEST\n";
578

579 580 581
    if ($flestlog) {
	open(FLESTLOG,"> flest.log");
    }
582

583 584
    pass("Pre-Test Setup");

585 586
    # Time to run tests!
    prints "Running Tests\n";
587
    prints "Testlist: $testlist\n";
588

589
    # Let's add some stuff to our path
590
    $ENV{'PATH'} = "$basepath:$TB/bin:$TB/sbin";
591

592 593 594 595 596
    my @tests;
    my $testmode;
    my $dir;

    if ($#_ == -1) {
597 598 599 600 601 602 603
	open(TESTLIST,"$basedir/testlists/$testlist");
	@tests = ();
	while (<TESTLIST>) {
	    chop;
	    push(@tests,$_);
	}
	close(TESTLIST);
604 605
    } else {
	@tests = @_;
606 607
    }

608 609 610
    mkdir "tests",0755;
    foreach $test (@tests) {
	if (($test eq ".") || ($test eq "..") || ($test eq "CVS")) {
611 612
	    next;
	}
613
	prints "  $test ";
614

615 616
	mkdir "tests/$test",0755;
	chdir "tests/$test";
617
	open(TESTLOG,"> test.log");
618

619 620 621 622
	# Set up DB state
	if (tbexect("echo \"drop database if exists $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not drop test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
623
	    fail2("$test - Drop test DB");
624 625 626 627 628 629
	    $fail++;
	    doexit(1);
	}
	if (tbexect("echo \"create database $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not recreate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
630
	    fail2("$test - Recreate test DB");
631
	    $fail++;
632 633
	    doexit(1);
	}
634
	if (tbexect("cat $path/clean.txt | mysql -q -s $testdb",TESTLOG)) {
635 636
	    prints "FAIL - Could not populate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
637
	    fail2("$test - Populate test DB");
638 639 640
	    $fail++;
	    doexit(1);
	}
641

642 643
	if (-f "$basedir/tests/$test/dbstate") {
	    if (tbexect("cat $basedir/tests/$test/dbstate | mysql $testdb",TESTLOG)) {
644
		prints "FAIL - Could not evaluate dbstate.\n";
645
		fail2("$test - Evaluate dbstate");
646
		$fail++;
647
		if ($exitonfail) {doexit(1);}
648
		next;
649
	    }
650
	}
651 652 653

	# Start stated
	if ($type eq "full") {
654
	  # Returns non-zero for failures
655 656
	  system("$TB/sbin/stated") and
	    die "Error running '$TB/sbin/stated': $!\n";
657
	}
658

659
	# We're now ready to run the test.
660

661
	# The last line of this command will be either PASS
662
	# or FAIL <msg>, all output before that should be sent
663
	# to the log file.
664
	open(TESTFP,"tbstub $testdb testbed test $test $basedir/tests/$test |");
665 666 667 668 669 670 671 672
	while (<TESTFP>) {
	    print TESTLOG $_;
	    $status = $_;
	}
	chop $status;
	close(TESTFP);
	if (($? >> 8) != 0) {
	    prints "UNKNOWN - Test exited with non-0 code.\n";
673
	    $unknown++;
674 675 676 677 678 679
	} else {
	    if ($status =~ /^PASS/) {
		$pass++;
	    } elsif ($status =~ /^FAIL/) {
		$fail++;
		# grab copy of testbed
680
		system("mysqldump -e -a $testdb > db.txt");
681 682 683 684
		if (($? >> 8) != 0) {
		    prints "ERROR: Could not grab copy of database.\n";
		    doexit(1);
		}
685
		if ($exitonfail) {prints "FAIL\n"; doexit(1);}
686 687
	    } else {
		$unknown++;
688
		$status = "UKNOWN";
689 690
	    }
	    prints "$status\n";
691 692 693 694
	    if ($flestlog) {
		@t = split(" ",$status);
		$result = $t[0];
		$msg = join(" ",@t[1..$#t]);
695
		print FLESTLOG "$result: $test $msg\n";
696
	    }
697
	}
698

699
	if ($type eq "full") {
700
	    my $statedpidfile = "$TB/locks/stated.pid";
701 702
	    my $statedpid = `cat $statedpidfile` ||
	      prints "WARNING: Unable to find stated pid: $!\n";
703 704 705
	    if (! kill(15, $statedpid) ) {
		prints "WARNING: Unable to kill stated process ($statedpid)\n";
	    }
706
	}
707

708
	close(TESTLOG);
709

710
	chdir $path;
711
    }
712 713 714
    if ($flestlog) {
	close FLESTLOG;
    }
715
    prints "END TEST\n";
716
}
717

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
sub step_test {
    open(LOG,">> $logfile");

    if (!$daikon) {

	test_pass;

    } else {

	prints "DAIKONIZING: STAGE 1 ... ";
	system("$TB/bin/daikonize clean > daikon-clean.log 2>&1");
	if (($? >> 8) != 0 && ($? >> 8) != 1) {
	    prints "FAIL - see \"daikon-clean.log\" for why.\n";
	    prints "EXITING.\n";
	    return;
	}
	system("$TB/bin/daikonize stage1 > daikon-stage1.log 2>&1");
	if (($? >> 8) != 0 && ($? >> 8) != 1) {
	    prints "FAIL - see \"daikon-stage1.log\" for why.\n";
	    prints "EXITING.\n";
	    return;
	} else {
	    prints "PASS\n";
	}

	test_pass;

	prints "DAIKONIZING: STAGE 2 ... ";
	system("$TB/bin/daikonize stage2 > daikon-stage2.log 2>&1");
	if (($? >> 8) != 0 && ($? >> 8) != 1) {
	    prints "FAIL - see \"daikon-stage2.log\" for why.\n";
	    prints "EXITING.\n";
	    return;
	} else {
	    prints "PASS\n";
	}

	test_pass;

	prints "DAIKONIZING: ANALYZE ... ";
	system("$TB/bin/daikonize analyze > daikon-analyze.log 2>&1");
	if (($? >> 8) != 0 && ($? >> 8) != 1) {
	    prints "FAIL - see \"daikon-analyze.log\" for why.\n";
	    prints "EXITING.\n";
	    return;
	} else {
	    prints "PASS\n";
	}
    }
}

769 770
sub step_cleanup {
    open(LOG,">> $logfile");
771

772
    prints "START cleanup\n";
773

774 775 776 777 778 779 780 781 782 783 784 785
    # Try to do the necessary cleanup
    if ($leavedb == 0) {
	prints "  Dropping test DB ";
	system("echo \"drop database if exists $testdb\" | mysql");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
	    prints "PASS\n";
	    $pass++;
	}
    }
786
    if (($type eq "full")) {
787 788 789 790 791 792 793 794 795 796
	prints "  Clearing redirects ";
	$ok = 1;
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || do {
	    $ok = 0;
	};
	if (! $ok) {
	    prints "FAIL - Could not connect to db\n";
	    $fail++;
	} else {
	    $ok = 1;
797 798 799 800
	    $dbh->do("UPDATE experiments set testdb=NULL" .
		     " where pid=\"$pid\" and eid=\"$eid\"") || do {
			 $ok = 0;
		     };
801
	    if ($ok) {
802
		$pass++;
803
		prints "PASS\n";
804 805 806
	    } else {
		$fail++;
		prints "FAIL\n";
807 808 809
	    }
	    $dbh->disconnect;
	}
810

811 812 813 814 815 816
	prints "  Freeing nodes ";
	system("/usr/testbed/bin/nfree $pid $eid > /dev/null");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
817
	    prints("PASS\n");
818 819
	    $pass++;
	}
820
    }
821 822 823 824 825 826 827 828 829
    prints "  Clearing state ";
    system("rm -f $statefile");
    if (($? >> 8) != 0) {
	prints "FAIL\n";
	$fail++;
    } else {
	prints "PASS\n";
	$pass++;
    }
830
    prints "End test run. " . ctime(time);
831 832
    prints "END CLEANUP\n";
}
833

834

835 836 837 838 839 840 841 842 843 844 845 846 847
if ($mode eq "run") {
    step_init;
    step_test;
    step_cleanup;
} elsif ($mode eq "init") {
    step_init;
} elsif ($mode eq "test") {
    step_test;
} elsif ($mode eq "single") {
    step_test @tests;
} elsif ($mode eq "finish") {
    step_cleanup;
}
848

849 850
open(LOG,">> $logfile");

851 852
# Do exit does all cleanup.
doexit(0);