Commit 177702c3 authored by Kevin Atkinson's avatar Kevin Atkinson

Added image testing framework.  Incomplete (ie doesn't test everything
it should), but functional.
parent fa66a269
package ImageTest;
use Exporter;
@ISA = "Exporter";
@EXPORT = qw (test test_cmd test_ssh test_rcmd test_experiment
ERR_NONE ERR_FAILED ERR_SWAPIN ERR_FATAL ERR_CLEANUP);
use IO::File;
use strict;
use vars qw(%parms %dependencies %tally);
use vars qw($eid $pid $datadir $resultsdir);
use vars qw(@mapping @nodes @pnodes %to_physical %from_physical);
use vars qw($FAILED);
# exit values
sub ERR_NONE {0};
sub ERR_FAILED {1}; # tests failed
sub ERR_SWAPIN {2}; # swapin failed
sub ERR_FATAL {3}; # fatal error
sub ERR_CLEANUP {4}; # fatal error - cleanup needed
#
# Performs a test on a swapped in experient. Returns true if the test
# passed.
#
# Each test can depend and on any number of other tests. The test
# will be skipped unless all the dependencies are satisfied.
#
# Examples:
# test 'test 1', [], sub {...};
# test 'test 2', ['test 1'], sub {...};
# test 'login prompt', [], sub {
# local $_ = cat "/var/log/tiplogs/$pnode.run";
# /login\: /;
# }
#
sub test ($$&) {
my ($name,$requires,$test) = @_;
$tally{total}++;
my $deps_sat = 1;
foreach (@$requires) {$deps_sat = 0 unless $dependencies{$_}}
unless ($deps_sat) {
print "Skipping test \"$name\" due to unsatisfied dependies.\n";
return 0;
}
my $res;
eval {$res = &$test(%parms)};
if ($res) {
$tally{passed}++;
$dependencies{$name} = 1;
print "\"$name\" succeeded\n";
return 1;
} elsif ($@) {
$tally{failed}++;
print $FAILED "$name\n";
print "*** \"$name\" died: $@";
return 0;
} else {
$tally{failed}++;
print $FAILED "$name\n";
print "*** \"$name\" failed\n";
return 0;
}
}
#
# Performs a test by executing a command and optionally checking the output
#
# Example :
# test_cmd "ssh $node", [], "ssh $node.$eid.$pid true";
#
sub test_cmd ($$$;&) {
my ($name,$requires,$cmd,$output_test) = @_;
test $name, $requires, sub {
print "Executing: $cmd\n";
if (not defined $output_test) {
system $cmd;
return ($? >> 8 == 0);
} else {
local $/ = undef;
my $F = new IO::File;
open $F, "$cmd |" or return 0;
local $_ = <$F>;
close $F;
return 0 unless ($? >> 8 == 0);
open $F, ">$resultsdir/$name.out";
print $F $_;
close $F;
my $res = &$output_test;
print "*** output of \"$cmd\" did not match expected output\n" unless $res;
return $res;
}
};
}
#
# Test that ssh is working on a node
#
# Example:
# test_ssh 'node';
#
sub test_ssh ($) {
my ($node) = @_;
test_cmd "ssh-$node", [], "ssh -o BatchMode=yes -o StrictHostKeyChecking=no $node.$parms{eid}.$parms{pid} true";
}
#
# Performs a test by executing a remote command and optionally
# checking the output. test_ssh must be executed on the node before
# any remote commands for the node.
#
# Examples:
# test_rcmd 'sudo', [], 'node', 'sudo touch /afile.txt';
# test_rcmd 'hostname' , [], 'node', 'hostname', sub {/^node\.$eid\.$pid/};
#
sub test_rcmd ($$$$;&) {
my ($name,$requires,$node,$cmd,$output_test) = @_;
&test_cmd($name, ["ssh-$node", @$requires],
"ssh -o BatchMode=yes $node.$parms{eid}.$parms{pid} $cmd", $output_test);
}
#
# Scans a log file for any errors or serious warnings. The log file
# may also be a pipe as the string is passed directly to open.
#
# Example:
# test_scanlog 'error free log', [], 'log';
#
sub test_scanlog ($$$) {
my ($name,$requires,$log) = @_;
test $name, $requires, sub {
my $F = new IO::File;
open $F, $log or die "Unable to open \"$log\" for reading.";
my $errors = 0;
while (<$F>) {
next unless /^\*\*\* /;
print;
$errors = 1;
}
close $F;
return $errors == 0;
};
}
#
#
#
sub cat ($) {
open F, $_[0];
local $/ = undef;
local $_ = <F>;
close F;
return $_;
};
#
#
#
sub single_node_tests ($) {
my ($node) = @_;
my ($pnode) = $to_physical{$node};
test_ssh $node;
test_rcmd "sudo-$node", [], $node, 'sudo touch /afile.txt';
test_rcmd "hostname-$node" , [], $node, 'hostname', sub {
/^$node\.$eid\.$pid/i || /^$pnode/i;
};
test "login_prompt-$node", [], sub {
local $_ = cat "/var/log/tiplogs/$pnode.run";
/login\: /;
};
}
#
#
#
sub multi_node_tests () {
#test_cmd 'linktest', [], "run_linktest.pl -v -e $pid/$eid";
sleep 10;
test_cmd 'linktest1', [], "run_linktest.pl -v -L 1 -l 1 -e $pid/$eid";
sleep 2;
test_cmd 'linktest2', [], "run_linktest.pl -v -L 2 -l 2 -e $pid/$eid";
sleep 2;
test_cmd 'linktest3', [], "run_linktest.pl -v -L 3 -l 3 -e $pid/$eid";
sleep 2;
test_cmd 'linktest4', [], "run_linktest.pl -v -L 4 -l 4 -e $pid/$eid";
}
#
# Creats and swaps in a test experment in and performs tests on the
# images after it is done. When all the tests are done swap the experment out,
# copy the experment data dir to EXP-YYYYMMDDHHMM and terminate the experiment.
#
# Will fork a child to do the actual work and return the pid of the child
#
# Expects a hash of parmaters as input for example:
# test_experiment
# pid => 'tbres',
# eid => 'it-single',
# os => 'RHL73-STD',
# hardware => 'pc850',
# datadir => '...',
# resultsdir => '...',
#
# The .ns file for the experiment is expected to be named "<datadir>/nsfile.ns".
# The file will be scanned and any instances of @PARM@ will be substituted
# for the value of PARM. Any image specific tests should be located in
# "<datadir>/tests.ns".
#
sub test_experiment (%) {
{
my $pid = fork();
die unless defined $pid;
return $pid if $pid != 0; # child
}
%parms = @_;
%dependencies = ();
%tally = (total => 0, passed => 0, failed => 0);
$eid = $parms{eid};
$pid = $parms{pid};
$datadir = $parms{datadir};
$resultsdir = $parms{resultsdir};
my $exit = 0;
$SIG{__DIE__} = sub {
return unless defined $^S && !$^S;
$! = ERR_FATAL;
die $_[0];
};
mkdir $resultsdir, 0777;
open STDOUT, ">$resultsdir/log" or die;
open STDERR, ">&STDOUT" or die;
$FAILED = new IO::File ">$resultsdir/failed-tests" or die;
my ($F,$O);
$F = new IO::File ">$resultsdir/parms" or die;
foreach (sort keys %parms) {
print $F "$_: $parms{$_}\n";
}
if ($parms{stages} =~ /c/) {
$F = new IO::File "$datadir/nsfile.ns" or die;
$O = new IO::File ">$resultsdir/nsfile.ns" or die;
while (<$F>) {
s/\@([^@]+)\@/$parms{lc $1}/g;
print $O $_;
}
close $O;
close $F;
system("/usr/testbed/bin/startexp -w -i -f".
" -E \"Experiment For Testing Images\"".
" -p $pid -e $eid $resultsdir/nsfile.ns");
if ($? >> 8 != 0) {
print "*** Could not create experment\n";
exit ERR_FATAL;
}
}
my $swapin_success = 1;
if ($parms{stages} =~ /s/) {
$swapin_success = test_cmd 'swapin', [],
"/usr/testbed/bin/swapexp -w -e $pid,$eid in";
}
if ($swapin_success) {
if ($parms{stages} =~ /s/) {
# FIXME: need proper way to get the log file
test_scanlog 'error_free_swapin', [],
`ls -t /proj/$pid/exp/$eid/tbdata/swapexp.* | head -1`;
}
local (@mapping, @nodes, @pnodes, %to_physical, %from_physical);
open F, "/usr/testbed/bin/expinfo -m $pid $eid | ";
while (<F>) {
last if /^---/;
}
while (<F>) {
next unless /\w/;
local @_ = split /\s+/;
push @mapping, [@_];
push @nodes, $_[0];
push @pnodes, $_[3];
$to_physical{$_[0]} = $_[3];
$from_physical{$_[3]} = $_[0];
}
if ($parms{stages} =~ /t/) {
foreach my $node (@nodes) {
single_node_tests($node);
}
if (@nodes > 1) {
multi_node_tests();
}
if (-e "$datadir/tests.pl") {
do "$datadir/tests.pl";
if ($@) {
print "*** Unable to complete tests: $@";
print "*** Results may not be accurate.\n";
$exit = ERR_FATAL
}
}
}
if ($parms{stages} =~ /[oe]/) {
test_cmd 'loghole', [], "loghole -e $pid/$eid sync";
foreach my $node (@nodes) {
my $pnode = $to_physical{$node};
system "cp -pr /var/log/tiplogs/$pnode.run $resultsdir/tiplog-$node";
if ($? >> 8 != 0) {
print "*** WARNING: Unable to copy tiplog for node $node.\n";
}
}
test_cmd 'swapout', [],
"/usr/testbed/bin/swapexp -w -e $pid,$eid out";
# FIXME: need proper way to get the log file
test_scanlog 'error_free_swapout', ['swapout'],
`ls -t /proj/$pid/exp/$eid/tbdata/swapexp.* | head -1`;
}
if ($parms{stages} =~ /t/) {
print "\n";
print "Num Tests: $tally{total}\n";
print "Passed: $tally{passed}\n";
print "Failed: $tally{failed}\n";
my $unex = $tally{total} - $tally{passed} - $tally{failed};
print "Unable to Execute: $unex\n";
}
} else {
$exit = ERR_SWAPIN;
}
if ($parms{stages} =~ /e/) {
system("cp -pr /proj/$pid/exp/$eid $resultsdir/exp-data");
if ($? >> 8 != 0) {
print "*** Unable to copy exp data. Not terminating exp\n";
exit ERR_CLEANUP;
}
system("/usr/testbed/bin/endexp -w -e $pid,$eid");
if ($? >> 8 != 0) {
print "*** Could not terminate experiment. Must do manually\n";
exit ERR_CLEANUP;
}
}
$exit = ERR_FAILED if $exit == 0 && $tally{failed} > 0;
exit $exit;
}
SETUP
Copy the entire directory tree into a /proj/PID/ directory. You
might want to also change the variable $eid_prefix at the start of
"image-test" to something different, your username might be a good
choice.
BASIC USAGE
From the base directory run:
./image-test OS
where <OS> is an exiting image name such as RHL90-STD.
This will run each of the experiments in series. The project to run
them in will automatically be determined based on the current working
directory.
The results will be in results-DATE. It will print the directory
where the results will can be found to stdout. The results of each
experiment can than be found in RESULTSDIR/EXP
Some of the files in this directory include:
log: all output to stdout and stderr is redirected here
nsfile.ns: the nsfile used
parms: the value of the different parmaters used in the experment.
This is diffrent from the one found in the test directory described
latter.
failed-tests: a list of tests that failed
exp-data/: a copy of the experiment data as found in /proj/PID/exp/EID
tiplog-NODE: a copy of tiplog for NODE
Running each of the experiments in series can take a long time. For this
reason it is possible to run more than one tests at once. To do this use
./image-test -p NUM OS
where NUM is the maxim number of nodes to use at once. If a
particular experiment uses more than NUM nodes it will be run in series
after all the experiments that can be run in parallel have finished.
Thus setting NUM to 0 will force all the experiments to run in series.
To only run particular experiment use:
./image-test OS EXPS
which will only run the experiments listed in EXPS. For example:
./image-test RHL90-STD single-pc850 single-pc3000
will only run the experiments single-pc850 and single-pc3000.
Certain experiments can be run with different parameters. These
parameters will be appended to the base experiment name. For example
the experiment "single-pc850" is really the experiment "single" with
the first parameter being "pc850". To run a particular experiment in
all the possible combinations just specify the base name, for example:
./image-test RHL90-STD single
Each experiment consists of 5 stages:
(c) Create the experiment
(s) Swap it in
(t) Perform a series of tests of the experiment
(o) Swap it out
(e) End the experiment
To only run particular stages of an experiment use:
./image-test -s STAGES ...
where STAGES consists of the letters above. Note that 'e' implies 'o'.
For example to avoid swapping an experiment out use:
./image-test -s cst ...
Or to just run the tests on a already swapped in experiment use:
./image-test -s t ...
TESTING FRAMEWORK
Each individual experiment is expected to be the directory "tests/EXP".
This directory must contain the following files
num-nodes
nsfile.ns
and may also contain:
parms
tests.pl
and any other experiment specific files.
The file "num-nodes" consists of a single line which is the number of
nodes the experiment will use.
The file "nsfile.ns" in the template ns file for the experiment. Any
strings of the form "@PARM@" will be substituted for the value of the
parameter. The following parameters are available to all experiments:
OS: OS to use as given in the command line
DATADIR: BASE/tests/EXP
The file "parms" can be used to specify additional parameters for the
experiment. all possible combinations of the parameters specified will
be tried with each combination being run with the experiment name
"EXP-PARM1-PARM2-...".
The file "tests.pl" can be used to specify additional experiment
specific tests. This code will be run inside the "ImageTest"
package. See the file "ImageTest.pm" and existing tests for more
info.
For every experiment a number of standard tests will be run.
For each node in the experiment the following tests will be run:
ssh-NODE: try to ssh into the node
sudo-NODE: make sure sudo is working correctly
hostname-NODE: make sure host name is what it is expected to be
login_prompt-NODE: make sure that the login prompt appears
in the console
For experiments with more than one node:
linktest: run linktest
Additional standard tests will be added over time.
#!/usr/bin/perl -w
########################################################################
#
# This value is prefixed to all experiments to avoid conflicts with
# existing experient names
#
my $eid_prefix = 'it-';
########################################################################
#
#
#
autoflush STDOUT, 1;
autoflush STDERR, 1;
use ImageTest;
use Cwd;
use Getopt::Long;
use Data::Dumper;
use strict;
sub true() {1}
sub false() {0}
sub usage() {
die "usage: $0 -h | [-l] [-s CSTOE] [-p NUM] IMAGE [TESTS ...]\n";
}
sub help() {
print
("usage: $0 -h | [<OPTIONS>] <OS> [TESTS ...]\n".
" IMAGE image to use\n".
" TESTS if present only run these particular tests\n".
"OPTIONS:\n".
" -s STAGES only execute particular stages of the test where\n".
" STAGES is any one of:\n".
" c: create, s: swapin, t: test o: swapout, e: end experment\n".
" (note 'e' implies 'o')\n".
" -p NUM max number of nodes to alloacate when running experiments\n".
" in parallel. Experments larger than NUM will be run in series\n".
" after all the parallel experiments have run.\n".
" Default: 0, which means all experiments will run in series\n".
" -l Just list the available experements.\n");
}
die usage unless @ARGV > 0;
my $cwd = cwd();
$cwd =~ s~^/q/~/~;
die "Working Directory needs to be in /proj\n" unless $cwd =~ m~^/proj~;
my $basedir = $cwd;
my $srcdir = $cwd;
my $destdir = $cwd;
my $date=`date +%Y%m%d%H%M`;
chop $date;
$destdir="$destdir/results-$date";
my $testsdir = "$srcdir/tests";
my ($pid) = $cwd =~ m~^/proj/(.+?)/~;
my @argv;
my %exp_opts;
my $stages = 'cstoe';
my $max_parallel = 0;
my $just_list;
GetOptions
"h|help" => sub {help(); exit 0;},
"s|stages=s" => \$stages,
"p=i" => \$max_parallel,
"l|list" => \$just_list
or usage();
my $os = shift @ARGV;
my @exps_torun = map {lc} @ARGV;
if ($stages =~ /([^cstoe])/) {
print STDERR "Unknown stage, \"$1\", in stage string.\n" if $stages =~ /([^cstoe])/;
usage();
}
my @exps;
my %exps;
my @exps_inseries;
my $num_exps_inparallel = 0;
my @exps_inparallel_bysize;
########################################################################
#
#
#
sub get_parms ($);
sub get_numnodes ($);
sub mktests ($$$@);
my @p;
opendir D, "$testsdir";
while (my $test = readdir D) {
next if $test =~ /^\./;
@p = get_parms $test;
mktests $test, get_numnodes $test, [], @p;
}
sub get_parms ($) {
my ($test) = @_;
my @p;
open F, "$testsdir/$test/parms" or return;
while (<F>) {
chop;
next if /^\s*$/;
my ($key, $value) = /^\s*(.+?)\s*:\s*(.+?)\s*$/ or die "?$_?";
my @values = split /\s+/, $value;
push @p, [$key, [@values]];
}
return @p;
}
sub get_numnodes ($) {
# FIXME: needs better error detection
my ($test) = @_;
open F, "$testsdir/$test/num-nodes";
local $_ = <F>;
chop;
return $_;
}
sub use_exp ($);
sub mktests ($$$@) {
my ($test, $numnodes, $what, @parms) = @_;
if (@parms == 0) {
my $t = $test;
for (my $i = 1; $i < @$what; $i += 2) {$t .= "-".lc($what->[$i])}
return unless use_exp $t;
$exps{$t} = {pid => $pid,
eid => "$eid_prefix$t",
numnodes => $numnodes,
os => $os,
datadir => "$testsdir/$test",
resultsdir => "$destdir/$t",
@$what,
stages => $stages,
%exp_opts};
push @exps, $t;
} else {
my $p = shift @parms;
foreach (@{$p->[1]}) {
mktests $test, $numnodes, [@$what, $p->[0], $_], @parms;
}
}
}
sub use_exp ($) {
my ($exp) = @_;
return true if (@exps_torun == 0);
foreach my $e (@exps_torun) {
return true if ($e eq $exp);
return true if $exp =~ /^$e-/;
}
return false;
}
########################################################################
#
#
#
if ($just_list) {
foreach (@exps) {
print "$_\n";
}
exit 0;
}
########################################################################
#
#
#
sub get_exps ($@); # parms: available nodes, size list
# returns list of experments, see below