tbtest 16 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
#!/usr/bin/perl -w

# tbtest

# This is the toplevel of the testbed testing setup.  This command
# should be invokved by a user and script.  It well then create a
# separate object tree, configure it appropriately, setup a testing
# database, and then proceed to run tests.

# IMPORTANT: This command should be come from a tree configured 
# for the real DB.

13
# tbtest [options] <mode stuff>
14
15
# Options:
#   -path <path> - Path to test tree, otherwise uses cwd.
16
17
18
#   -full - Do full testing.
#   -frontend - Do frontend only testing.#
#   -leavedb - Avoids removing the DB on exit.
19
#   -exitonfail - Exit on the first failure.
20
21
22
#
# <mode> is one of:
#    run <db> <testdir> [<pid> <eid> <num>] - Do everything.
23
#    init <db> [<pid> <eid> <num>] - Initiatlize.  
24
25
#    test <testdir> - Run tests
#    single <testdir> <tests> - Run only <tests>
26
#    finish - Finish run
27
# <num> is "X <type> ...", for example "10 pc 16 shark"
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

use DBI;
use POSIX;

$| = 1;

$pwd = `/bin/pwd`;
chop $pwd;

# Figure out where the script is
if ($0 =~ m|/|) {
    # Not pwd
    @a = split("/",$0);
    $dir = join("/",@a[0..$#a-1]);
    if ($0 =~ m|^/|) {
	# absolute path
	$basedir = $dir;
    } else {
	# relative path
	chdir $dir;
	$basedir = `/bin/pwd`;
	chop $basedir;
	chdir $pwd;
    }
} else {
    $basedir = `/bin/pwd`;
    chop $basedir;
}

$path = $pwd;

59
$ENV{'PATH'} = "/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/site/bin:$basedir:.";
60
61
62
$basepath = $ENV{'PATH'};

$type = "frontend";
63
64

# Parse argv
65
66
67
68
69
70
sub show_help {
    print STDERR "Syntax: $0 [-leavedb] [-path <path>] [-frontend] [-full] <mode>\n";
    print STDERR "Options:\n";
    print STDERR "  -leavedb - Do not drop the test database on exit.\n";
    print STDERR "  -path <path> - Path to directory to store test files.\n";
    print STDERR "  -frontend - Run in frontend mode.\n";
71
    print STDERR "  -full - Run in full mode.\n";
72
    print STDERR "  -exitonfail - Exit on any failure.\n";
73
74
75
76
77
    print STDERR "Mode:\n";
    print STDERR "  run <db> <testdir> [<pid> <eid> <num>]\n";
    print STDERR "  init <db> [<pid> <eid> <num>]\n";
    print STDERR "  test <testdir>\n";
    print STDERR "  single <testdir> <tests>\n";
78
    print STDERR "  finish\n";
79
80
81
82
83
84
    print STDERR "<num>: Must be even lengthed list of <number> <type>.\n";
    print STDERR "   Ex: \"4 pc 16 shark\"\n";
    print STDERR "Examples:\n";
    print STDERR "   $0 run tbdb frontend\n";
    print STDERR "   $0 -full run tbdb frontend testbed mytest \"10 pc 16 shark\"\n";
    print STDERR "   $0 init tbdb;$0 single frontend ping basic;$0 finish\n";
85
86
87
88
89
    exit(1);
};

$mode = "";
$leavedb = 0;
90
$exitonfail = 0;
91
92

while (($#ARGV > -1) && ($mode eq "")) {
93
94
    $arg = shift;
    if ($arg eq "-path") {
95
	if ($#ARGV == -1) {show_help;}
96
	$path = shift;
97
98
99
100
101
102
    } elsif ($arg eq "-frontend") {
	$type = "frontend";
    } elsif ($arg eq "-full") {
	$type = "full";
    } elsif ($arg eq "-leavedb") {
	$leavedb = 1;
103
104
    } elsif ($arg eq "-exitonfail") {
	$exitonfail = 1;
105
    } else {
106
	$mode = $arg;
107
108
109
110
    }
}

chdir $path;
111
112
113
114
$user=`whoami`;
chop $user;
$logfile = "test.log";
$statefile = "state";
115

116
117
118
119
120
121
if ($mode eq "run") {
    if ($type eq "frontend") {
	if ($#ARGV != 1) {show_help;}
	($TBDB,$testdir) = @ARGV;
    } else {
	if ($#ARGV != 4) {show_help;}
122
	($TBDB,$testdir,$pid,$eid,$num) = @ARGV;
123
124
125
126
127
128
129
    }
} elsif ($mode eq "init") {
    if ($type eq "frontend") {
	if ($#ARGV != 0) {show_help;}
	($TBDB) = @ARGV;
    } else {
	if ($#ARGV != 3) {show_help;}
130
	($TBDB,$pid,$eid,$num) = @ARGV;
131
132
133
134
135
136
137
138
    }
} elsif ($mode eq "test") {
    if ($#ARGV != 0) {show_help;}
    ($testdir) = @ARGV;
} elsif ($mode eq "single") {
    if ($#ARGV <= 0) {show_help;}
    ($testdir,@tests) = @ARGV;
} elsif ($mode eq "finish") {
139
    if ($#ARGV != -1) {show_help;}
140
141
142
} else {
    print STDERR "Unknown mode: $mode\n";
    show_help;
143
144
}

145
146
147
148
149
150
151
152
153
154
155
156
if (($mode ne "run") && ($mode ne "init")) {
    # Read state
    open(STATE,$statefile) || do {
	print STDERR "Could not find state file ($statefile).\n";
	print STDERR "  Run in init mode first.\n";
	exit(1);
    };
    ($type,$TBDB,$user,$pid,$eid) = split(" ",<STATE>);
    close(STATE);
}


157
158
159
160
if (($mode eq "run") || ($mode eq "test") || ($mode eq "single")) {
    if (! ($testdir =~ m|^/|)) {
	$testdir = "$basedir/tests/$testdir";
    }
161
}
162
163
164
165
166
167
168
if (($type eq "full") && (($mode eq "init") || ($mode eq "run"))) {
    @t = split(" ",$num);
    if (($#t % 2) == 0) {
	print STDERR "<num> must be list of even length.\n";
	show_help;
    }
}
169
170

$testdb = "tbdb_$user";
171
172
173
174
175

# Test counts
$pass = 0;
$fail = 0;
$unknown = 0;
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195

# prints sends the string to stdout and the logfile.
sub prints {
    print $_[0];
    print LOG $_[0];
};

# tbexec <cmd> <file>
# Executes <cmd> storing results to <file> and displaying appropriate
# message and exiting on failure.
sub tbexec {
    my ($cmd,$file) = @_;
    open(EXEC,"$cmd 2>&1 |");
    open(FILE,"> $file");
    while (<EXEC>) {
	print FILE $_;
    }
    close(FILE);
    close(EXEC);
    if (($? >> 8) == 0) {
Christopher Alfeld's avatar
Christopher Alfeld committed
196
	prints "PASS\n";
197
198
	$pass++;
    } else {
Christopher Alfeld's avatar
Christopher Alfeld committed
199
	prints "FAIL - See $file\n";
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
	$fail++;
	doexit(1);
    }
};

# tbexect <cmd> <logfp>
# This is similar to tbexec except is used for running commands specific
# to tests.  It runs the command appending the output to the file handle
# <logfp> and returns the exit code.
sub tbexect {
    my ($cmd,$logfp) = @_;
    print $logfp "$cmd\n";
    open(EXEC,"$cmd 2>&1 |");
    while (<EXEC>) {
	print $logfp $_;
    }
    close(EXEC);
    return ($? >> 8);
};

# doexit
# This displays exit counts and then exits with the specified exit code.
sub doexit {
    printf LOG "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
    printf "Total Tests: %-5d  Pass: %-5d  Fail: %-5d  Unknown: %-5d\n", $pass+$fail+$unknown, $pass, $fail, $unknown;
225
226
    if ($_[0] != 0) {
	prints "Run in finish mode to clean up any residual state.\n";
227
    }
228
    close(LOG);
229
230
231
    exit($_[0]);
};

232
233
sub step_init {
    open(LOG,"> $logfile");
234

235
236
237
238
    prints "START INIT\n";
    prints "User:     $user\n";
    prints "Logfile:  $logfile\n";
    prints "Database: $TBDB\n";
239
    prints "Type:     $type\n";
240
    prints "\n";
241
    prints "Beginning test run. " . ctime(time);
242

243
244
245
246
247
248
    # Initialize DB
    $dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || 
	die "Could not connect to DB.\n";
    # Having made sure we can, we now disconnect so as not to harm DB.
    $dbh->disconnect;

249
250
251
252
253
254
255
256
257
    # Write state
    open(STATE,"> $statefile");
    if ($type eq "full") {
	print STATE "$type $TBDB $user $pid $eid\n";
    } else {
	print STATE "$type $TBDB $user\n";
    }
    close(STATE);

258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
    # Reserve nodes
    if ($type eq "full") {
	prints "Reserving nodes.\n";
	
	my (@nums) = split(" ",$num);
	my $num;
	my $type;
	my %want;
	my %have;
	my $node;

	while ($#nums > -1) {
	    $num = shift @nums;
	    $type = shift @nums;
	    $want{$type} = $num;
	    $have{$type} = [];
	}
	
	open(AVAIL,"/usr/testbed/sbin/avail |");
	while (<AVAIL>) {
	    if (! /^\|/) {next};
	    if (/node_id/) {next;}
	    ($node,$type) = /^\|([-a-zA-Z0-9]+)\s*\|(\w+)\s*\|$/;
	    if (defined($want{$type}) && $want{$type} > 0) {
		push(@{$have{$type}},$node);
		$want{$type}--;
	    }
	}
	close(AVAIL);

	my $good = 1;
	foreach $type (keys(%want)) {
	    if ($want{$type} > 0) {
		print STDERR "Could not get enough $type - Need $want{$type} more.\n";
		$good = 0;
	    }
	}
	if (! $good) {exit(1);}

	my $raw = "";
	foreach $type (keys(%have)) {
	    $raw .= join(" ",@{$have{$type}});
	}
	prints "  Reserving ";
	tbexec("/usr/testbed/bin/nalloc $pid $eid $raw","$path/reserve.log");
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324

	prints "  Setting up tmcc redirect ";
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || 
	    die "Could not connect to DB.\n";
	foreach $type (keys(%have)) {
	    foreach $node (@{$have{$type}}) {
		$dbh->do("DELETE from tmcd_redirect" .
			 " where node_id=\"$node\"") || do {
			     $fail++;
			     prints "FAIL - $node\n";
			     doexit(1);
			 };
		$dbh->do("INSERT into tmcd_redirect (node_id,dbname) values" .
			 " (\"$node\",\"$testdb\")") || do {
			     $fail++;
			     prints "FAIL - $node\n";
			     doexit(1);
			 };
	    }
	}
	prints "PASS\n";
	$dbh->disconnect;
325
326
    }

327
328
329
330
    # Set up object tree
    prints "Setting up object tree.\n";
    if (! -d $path) {
	mkdir $path,0755;
331
    }
332
333
334
335
    mkdir "tbobj",0755;
    mkdir "install",0755;
    chdir "tbobj";
    system("cp -f $basedir/test-defs $path/defs");
336
337
338
339
    if (($? >> 8) != 0) {
	print STDERR "ERROR: Could not copy defs file.\n";
	doexit(1);
    }
340
341
342
343
344
345
346
    open(DEFS,">>$path/defs");
    print DEFS "TBDBNAME=$testdb\n";
    print DEFS "TBOPSEMAIL=$user\@emulab.net\n";
    print DEFS "TBLOGSEMAIL=$user\@emulab.net\n";
    close(DEFS);
    
    prints "  Configuring ";
347
348
349
350
351
352
353
    my $testmode;
    if ($type eq "full") {
	$testmode = "";
    } else {
	$testmode = "--enable-testmode";
    }
    tbexec("$basedir/../configure --prefix=$path/install $testmode ".
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
	   "--with-TBDEFS=$path/defs --with-LEDA=/usr/testbed/LEDA/LEDA-4.0",
	   "$path/configure.log");
    
    prints "  Building ";
    tbexec("gmake","$path/build.log");
    
    prints "  Installing ";
    tbexec("gmake boss-install","$path/install.log");
    
    prints "  Post-Installing ";
    tbexec("su1 gmake post-install","$path/postinstall.log");
        
    chdir $path;
    
    # Setup test database
    prints "Setting up test database.\n";
    prints "  Dumping Current ";
    tbexec("mysqldump $TBDB","$path/dbdump.txt");
    
    prints "  Creating DB ";
374
    system("echo \"drop database if exists $testdb\" | mysql");
375
376
377
378
    tbexec("echo \"create database $testdb\" | mysql","createdb.log");
    
    prints "  Filling DB ";
    tbexec("cat $path/dbdump.txt | mysql $testdb","fill.log");
379
380


381
382
    $dbh = DBI->connect("DBI:mysql:database=$testdb;host=localhost") || do {
	prints "ERROR: Could not connect to new DB.\n";
383
	doexit(1);
384
385
    };
    $ENV{'PATH'} = "$basepath:$path/install/bin:$path/install/sbin";    
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416

    if ($type eq "frontend") {
	# Clear out experiments and set up test experiment.     
	prints "  Clearing out existing experiments ";
	open(CLEARLOG,"> clear.log");
	$sth = $dbh->prepare("SELECT pid,eid from experiments");
	$sth->execute;
	while (($pid,$eid) = $sth->fetchrow_array) {
	    print CLEARLOG "$pid $eid\n";
	    print CLEARLOG "tbswapout -nologfile -force $pid $eid\n";
	    open(EXEC,"tbswapout -nologfile -force $pid $eid 2>&1 |");
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints "FAIL - For $pid $eid - On tbswapout - See clear.log\n";
		$fail++;
		doexit(1);
	    } 
	    print CLEARLOG "tbend -nologfile $pid $eid\n";
	    open(EXEC,"tbend -nologfile $pid $eid 2>&1 |");
	    while (<EXEC>) {
		print CLEARLOG $_;
	    }
	    close(EXEC);
	    if (($? >> 8) != 0) {
		prints " FAIL - For $pid $eid - On tbend - See clear.log\n";
		$fail++;
		doexit(1);
	    }   
417
	}
418
419
420
421
422
423
424
	prints "PASS\n";
	$pass++;

	close(CLEARLOG);
    } else {
	# Free nodes we allocated for the experiment and allocate all
	# others.
425
	prints "  Creating testbed/unavailable in test DB ";
426
427
428
429
430
431
432
	open(AVAIL,"avail |");
	$raw = "";
	while (<AVAIL>) {
	    if (! /^\|/) {next};
	    if (/node_id/) {next;}
	    ($node) = /^\|([-a-zA-Z0-9]+)/;
	    $raw .= " $node";
433
	}
434
435
436
437
438
439
440
441
442
443
444
	close(AVAIL);
	
	$dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"unavailable\"");
	$dbh->do("INSERT into experiments (pid,eid) "
		 . "values (\"testbed\",\"unavailable\")") || do {
		     prints "FAIL\n";
		     $fail++;
		     doexit(1);
		 };
	print "PASS\n";
	$pass++;
445

446
447
	prints "  Marking nodes as unavailable ";
	tbexec("nalloc testbed unavailable $raw","unavailable.log");
448

449
450
451
452
	prints "  Freeing test nodes ";
	tbexec("nfree $pid $eid","free.log");
    }
    
453
454
455
456
457
458
459
460
461
462
463
464
465
466
    prints "  Setting up test experiment ";
    $dbh->do("DELETE from experiments where pid=\"testbed\" and eid=\"test\"");
    $dbh->do("INSERT into experiments (pid,eid) "
	     . "values (\"testbed\",\"test\")") || do {
		 prints "FAIL\n";
		 $fail++;
		 doexit(1);
	     };
    print "PASS\n";
    $pass++;
    
    prints "  Snapshotting clean state ";
    tbexec("mysqldump $testdb","clean.txt");
    prints "END INIT\n";
467
468
}

469
470
471
472
473
474
475
476
sub step_test {
    open(LOG,">> $logfile");
    
    prints "START TEST\n";
    
    # Time to run tests!
    prints "Running Tests\n";
    prints "Testdir: $testdir\n";
477

478
479
480
481
482
483
484
485
486
487
488
489
490
    # Let's add some stuff to our path
    $ENV{'PATH'} = "$basepath:$path/install/bin:$path/install/sbin";
    
    my @tests;
    my $testmode;
    my $dir;

    if ($#_ == -1) {
	opendir(TESTDIR,$testdir);
	@tests = readdir(TESTDIR);
	closedir(TESTDIR);
    } else {
	@tests = @_;
491
492
    }

493
494
495
    mkdir "tests",0755;
    foreach $test (@tests) {
	if (($test eq ".") || ($test eq "..") || ($test eq "CVS")) {
496
497
	    next;
	}
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
	prints "  $test ";
	
	mkdir "tests/$test",0755;
	chdir "tests/$test";
	open(TESTLOG,"> test.log");
	
	# Set up DB state
	if (tbexect("echo \"drop database if exists $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not drop test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
	    $fail++;
	    doexit(1);
	}
	if (tbexect("echo \"create database $testdb\" | mysql",TESTLOG)) {
	    prints "FAIL - Could not recreate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
514
	    $fail++;
515
516
517
518
519
520
521
522
523
524
525
526
527
	    doexit(1);
	}
	if (tbexect("cat $path/clean.txt | mysql $testdb",TESTLOG)) {
	    prints "FAIL - Could not populate test DB.\n";
	    prints "Failure was FATAL - Exiting.\n";
	    $fail++;
	    doexit(1);
	}
	
	if (-f "$testdir/$test/dbstate") {
	    if (tbexect("cat $testdir/$test/dbstate | mysql $testdb",TESTLOG)) {
		prints "FAIL - Could not evaluate dbstate.\n";
		$fail++;
528
		if ($exitonfail) {doexit(1);}
529
		next;
Christopher Alfeld's avatar
Christopher Alfeld committed
530
	    }
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
	}
	
	# We're now ready to run the test.
	
	# The last line of this command will be either PASS
	# or FAIL <msg>, all output before that should be sent 
	# to the log file.
	open(TESTFP,"tbstub $testdb testbed test $test $testdir/$test |");
	while (<TESTFP>) {
	    print TESTLOG $_;
	    $status = $_;
	}
	chop $status;
	close(TESTFP);
	if (($? >> 8) != 0) {
	    prints "UNKNOWN - Test exited with non-0 code.\n";
547
	    $unknown++;
548
549
550
551
552
553
554
555
556
557
558
	} else {
	    if ($status =~ /^PASS/) {
		$pass++;
	    } elsif ($status =~ /^FAIL/) {
		$fail++;
		# grab copy of testbed
		system("mysqldump $testdb > db.txt");
		if (($? >> 8) != 0) {
		    prints "ERROR: Could not grab copy of database.\n";
		    doexit(1);
		}
559
		if ($exitonfail) {doexit(1);}
560
561
562
563
564
	    } else {
		$unknown++;
		$status = "UKNONW";
	    }
	    prints "$status\n";
565
	}
566
567
568
569
	
	close(TESTLOG);
	
	chdir $path;
570
    }
571
    prints "END TEST\n";
572
}
573
   
574
575
sub step_cleanup {
    open(LOG,">> $logfile");
576
577
    
    prints "START cleanup\n";
578

579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    # Try to do the necessary cleanup
    if ($leavedb == 0) {
	prints "  Dropping test DB ";
	system("echo \"drop database if exists $testdb\" | mysql");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
	    prints "PASS\n";
	    $pass++;
	}
    }
    if (($type eq "full")) { 	
	prints "  Clearing redirects ";
	$ok = 1;
	$dbh = DBI->connect("DBI:mysql:database=$TBDB;host=localhost") || do {
	    $ok = 0;
	};
	if (! $ok) {
	    prints "FAIL - Could not connect to db\n";
	    $fail++;
	} else {
	    $sth=$dbh->prepare("SELECT node_id from reserved" .
			       " where pid=\"$pid\" and eid=\"$eid\"");
	    $sth->execute;
	    $ok = 1;
	    while (($node) = $sth->fetchrow_array) {
		$dbh->do("DELETE from tmcd_redirect" .
			 " where node_id=\"$node\"") || do {
			     prints "FAIL - $node\n";
			     $fail++;
			     $ok = 0;
			     last;
			 }
	    }
	    if ($ok) {
		prints "PASS\n";
	    }
	    $dbh->disconnect;
	}
619

620
621
622
623
624
625
626
627
628
	prints "  Freeing nodes ";
	system("/usr/testbed/bin/nfree $pid $eid > /dev/null");
	if (($? >> 8) != 0) {
	    prints "FAIL\n";
	    $fail++;
	} else {
	    prints "PASS\n";
	    $pass++;
	}
629
    }
630
631
632
633
634
635
636
637
638
    prints "  Clearing state ";
    system("rm -f $statefile");
    if (($? >> 8) != 0) {
	prints "FAIL\n";
	$fail++;
    } else {
	prints "PASS\n";
	$pass++;
    }
639
    prints "End test run. " . ctime(time);
640
641
    prints "END CLEANUP\n";
}
642
643

	     
644
645
646
647
648
649
650
651
652
653
654
655
656
if ($mode eq "run") {
    step_init;
    step_test;
    step_cleanup;
} elsif ($mode eq "init") {
    step_init;
} elsif ($mode eq "test") {
    step_test;
} elsif ($mode eq "single") {
    step_test @tests;
} elsif ($mode eq "finish") {
    step_cleanup;
}
657

658
659
open(LOG,">> $logfile");

660
661
# Do exit does all cleanup.
doexit(0);