Commit 27b6505a authored by Kevin Tew's avatar Kevin Tew

testsuite/testswap cleanups

parent 0d70b240
ImageTest - Traffic Generation Bug Report sync
Bug Report TempResourceError
Fix Exector dies
OldTests OldTests
Cleanup SSH Cleanup SSH
Retry Prep Retry Prep
Painpoints Painpoints
================================= =================================
Teststeps - don't run certian parts create, swapin, run, swapout, end, implemented for parallel tests
Testlayout - Suites and Tests Testlayout - Suites and Tests
Testoutput - Testoutput -
...@@ -13,7 +15,6 @@ DOCS TODO ...@@ -13,7 +15,6 @@ DOCS TODO
backoff example backoff example
TODO TODO
TIMEOUT of XMLRPC Calls
TESTLWP TESTLWP
scp cleanup scp cleanup
VERBOSENESS VERBOSENESS
......
...@@ -445,7 +445,9 @@ sub return_and_report { ...@@ -445,7 +445,9 @@ sub return_and_report {
sub handleItemResult { sub handleItemResult {
my ($s, $result) = @_; my ($s, $result) = @_;
my $executor = $s->tasks->[$result->itemid]->item; my $executor = $s->tasks->[$result->itemid]->item;
$result->name($executor->e->eid); if ($executor->can('e') and $executor->e) {
$result->name($executor->e->eid);
}
if ($executor->can('handleResult')) { if ($executor->can('handleResult')) {
my $rc = $executor->handleResult($s, $result); my $rc = $executor->handleResult($s, $result);
if ($rc == RETURN_AND_REPORT) { $s->return_and_report($result) } if ($rc == RETURN_AND_REPORT) { $s->return_and_report($result) }
......
package TAP::Parser::Iterator::StdOutErr;
use strict;
use warnings;
use vars qw($VERSION @ISA);
use TAP::Parser::Iterator::Process ();
use IO::Select;
@ISA = 'TAP::Parser::Iterator::Process';
sub _initialize {
my $self = shift;
$self->{out} = shift || die "Need out";
$self->{err} = shift || die "Need err";
$self->{sel} = IO::Select->new( $self->{out}, $self->{err} );
$self->{pid} = shift || die "Need pid";
$self->{exit} = undef;
$self->{chunk_size} = 65536;
return $self;
}
package TestBed::Harness;
use SemiModern::Perl; use SemiModern::Perl;
use TAP::Harness; use TAP::Harness;
require Exporter; require Exporter;
our @ISA = qw(Exporter TAP::Harness); our @ISA = qw(Exporter TAP::Harness);
our @EXPORT = qw(runharness); our @EXPORT = qw(runharness);
use TestBed::TestSuite; use TestBed::TestSuite;
use TestBed::ForkFramework;
use TestBed::ParallelRunner;
sub build_TAP_stream {
use TestBed::TestSuite;
my ($in, $out, $err, $pid) = TestBed::ForkFramework::fork_child_redir(sub { TestBed::ParallelRunner::GlobalRunner->runtests; });
return TAP::Parser::Iterator::StdOutErr->new($out, $err, $pid);
}
sub parser_args_callback { sub parser_args_callback {
my $args = shift; my $args = shift;
...@@ -11,7 +41,7 @@ sub parser_args_callback { ...@@ -11,7 +41,7 @@ sub parser_args_callback {
if (ref $ref and $ref->isa('TestBed::ParallelRunner')) { if (ref $ref and $ref->isa('TestBed::ParallelRunner')) {
delete $args->{source}; delete $args->{source};
$args->{'stream'} = $ref->build_TAP_stream; $args->{'stream'} = build_TAP_stream;
} }
$args; $args;
} }
...@@ -44,12 +74,17 @@ sub runharness { ...@@ -44,12 +74,17 @@ sub runharness {
$harness->runtests(@$ts); $harness->runtests(@$ts);
} }
=head1 NAME =head1 NAME
TestBed::Harness TestBed::Harness
=over 4 =over 4
=item C<< build_TAP_stream >>
given a TestBed::ParallelRunner returns a TAP stream
=item C<< runharness(@test_file_names) >> =item C<< runharness(@test_file_names) >>
ex. runharness( 't/lib/*.t', 't/xmlrpc/*.t' 'test/BasicTopologies.pm' ) ex. runharness( 't/lib/*.t', 't/xmlrpc/*.t' 'test/BasicTopologies.pm' )
......
#!/usr/bin/perl #!/usr/bin/perl
package TAP::Parser::Iterator::StdOutErr;
use strict;
use warnings;
use vars qw($VERSION @ISA);
use TAP::Parser::Iterator::Process ();
use Config;
use IO::Select;
@ISA = 'TAP::Parser::Iterator::Process';
sub _initialize {
my ( $self, $args ) = @_;
shift;
$self->{out} = shift || die "Need out";
$self->{err} = shift || die "Need err";
$self->{sel} = IO::Select->new( $self->{out}, $self->{err} );
$self->{pid} = shift || die "Need pid";
$self->{exit} = undef;
$self->{chunk_size} = 65536;
return $self;
}
package TestBed::ParallelRunner; package TestBed::ParallelRunner;
use SemiModern::Perl; use SemiModern::Perl;
use TestBed::ParallelRunner::Executor; use TestBed::ParallelRunner::Executor;
use TestBed::ForkFramework; use TestBed::ForkFramework;
use TestBed::TestBuilderWrapper;
use Data::Dumper; use Data::Dumper;
use Mouse; use Mouse;
use TBConfig; use TBConfig;
our $Executors = []; has executors => ( is => 'rw', default => sub { [] } );
our $GlobalRunner = TestBed::ParallelRunner->new;
sub executor {
my ($s, $itemId) = @_;
$s->executors->[$itemId];
}
my $teste_desc = <<'END';
Not enough arguments to teste
rege(eid, $ns, $sub, $test_count, $desc);
rege($pid, $eid, $ns, $sub, $test_count, $desc);
rege($pid, $gid, $eid, $ns, $sub, $test_count, $desc);
END
sub add_executor { sub add_executor {
my $executor = shift; my ($s, $executor) = @_;
push @$Executors, $executor; push @{$s->executors}, $executor;
$executor; $executor;
} }
sub build_executor { sub build_executor {
my $executor = TestBed::ParallelRunner::Executor::build(@_); my $s = shift;
add_executor($executor); $s->add_executor(TestBed::ParallelRunner::Executor::build(@_));
} }
sub runtests { sub runtests {
my ($concurrent_pre_runs, $concurrent_node_count_usage ) = @_; my ($s, $concurrent_pre_runs, $concurrent_node_count_usage) = @_;
$concurrent_pre_runs ||= $TBConfig::concurrent_prerun_jobs; $concurrent_pre_runs ||= $TBConfig::concurrent_prerun_jobs;
$concurrent_node_count_usage ||= $TBConfig::concurrent_node_usage; $concurrent_node_count_usage ||= $TBConfig::concurrent_node_usage;
if ( $TBConfig::single ) { if ( $TBConfig::single ) {
$Executors = [ (grep { $_->e->eid eq $TBConfig::single } @$Executors) ]; $s->Executors = [ (grep { $_->e->eid eq $TBConfig::single } @{$s->executors}) ];
} }
#prerun step #prerun step
my $result = TestBed::ForkFramework::ForEach::max_work($concurrent_pre_runs, sub { shift->prep }, $Executors); my $result = TestBed::ForkFramework::ForEach::max_work($concurrent_pre_runs, sub { shift->prerun }, $s->executors);
if ($result->has_errors) { if ($result->has_errors) {
sayd($result->errors); sayd($result->errors);
warn 'TestBed::ParallelRunner::runtests died during test prep'; warn 'TestBed::ParallelRunner::runtests died during test prep';
} }
my $workscheduler = TestBed::ForkFramework::WeightedScheduler->new( my $workscheduler = TestBed::ForkFramework::WeightedScheduler->new(
items => $Executors, items => $s->executors,
proc => \&tap_wrapper, proc => \&tap_wrapper,
maxnodes => $concurrent_node_count_usage, maxnodes => $concurrent_node_count_usage,
); );
...@@ -79,8 +53,7 @@ sub runtests { ...@@ -79,8 +53,7 @@ sub runtests {
#add taskss to scheduler step #add taskss to scheduler step
my $total_test_count = 0; my $total_test_count = 0;
for (@{$result->successes}) { for (@{$result->successes}) {
my $itemId = $_->itemid; my $executor = $s->executor($_->itemId);
my $executor = $Executors->[$itemId];
my $maximum_nodes = $_->result->{'maximum_nodes'}; my $maximum_nodes = $_->result->{'maximum_nodes'};
my $eid = $executor->e->eid; my $eid = $executor->e->eid;
...@@ -94,7 +67,7 @@ sub runtests { ...@@ -94,7 +67,7 @@ sub runtests {
} }
USE_TESTBULDER_PREAMBLE: { USE_TESTBULDER_PREAMBLE: {
reset_test_builder($total_test_count, no_numbers => 1); TestBed::TestBuilderWrapper::reset_test_builder($total_test_count, no_numbers => 1);
} }
#run tests #run tests
...@@ -103,11 +76,10 @@ sub runtests { ...@@ -103,11 +76,10 @@ sub runtests {
USE_TESTBULDER_POSTAMBLE: { USE_TESTBULDER_POSTAMBLE: {
$total_test_count = 0; $total_test_count = 0;
for (@{$result->successes}) { for (@{$result->successes}) {
my $item_id = $_->itemid; my $executor = $s->executor($_->itemId);
my $executor = $Executors->[$item_id];
$total_test_count += $executor->test_count; $total_test_count += $executor->test_count;
} }
set_test_builder_to_end_state($total_test_count); TestBed::TestBuilderWrapper::set_test_builder_to_end_state($total_test_count);
} }
if ($result->has_errors) { if ($result->has_errors) {
...@@ -117,35 +89,6 @@ sub runtests { ...@@ -117,35 +89,6 @@ sub runtests {
return; return;
} }
sub set_test_builder_to_end_state {
my ($test_count, %options) = @_;
use Test::Builder;
my $b = Test::Builder->new;
$b->current_test($test_count);
}
sub reset_test_builder {
my ($test_count, %options) = @_;
use Test::Builder;
my $b = Test::Builder->new;
$b->reset;
$b->use_numbers(0) if $options{no_numbers};
if ($test_count) { $b->expected_tests($test_count); }
else { $b->no_plan; }
}
sub setup_test_builder_ouputs {
my ($out, $err) = @_;
use Test::Builder;
my $b = Test::Builder->new;
$b->output($out);
$b->fail_output($out);
$b->todo_output($out);
}
#use Carp;
#$SIG{ __DIE__ } = sub { Carp::confess( @_ ) };
our $ENABLE_SUBTESTS_FEATURE = 0; our $ENABLE_SUBTESTS_FEATURE = 0;
sub tap_wrapper { sub tap_wrapper {
...@@ -163,8 +106,8 @@ sub tap_wrapper { ...@@ -163,8 +106,8 @@ sub tap_wrapper {
ok(1, $te->desc) if $ENABLE_SUBTESTS_FEATURE && $tapp; ok(1, $te->desc) if $ENABLE_SUBTESTS_FEATURE && $tapp;
}, },
sub { sub {
reset_test_builder($te->test_count) if $ENABLE_SUBTESTS_FEATURE; TestBed::TestBuilderWrapper::reset_test_builder($te->test_count) if $ENABLE_SUBTESTS_FEATURE;
setup_test_builder_ouputs(*STDOUT, *STDERR); TestBed::TestBuilderWrapper::setup_test_builder_ouputs(*STDOUT, *STDERR);
$te->execute; $te->execute;
}); });
} }
...@@ -174,30 +117,31 @@ sub tap_wrapper { ...@@ -174,30 +117,31 @@ sub tap_wrapper {
return 0; return 0;
} }
sub build_TAP_stream {
use TestBed::TestSuite;
my ($in, $out, $err, $pid) = TestBed::ForkFramework::fork_child_redir(sub { runtests; });
return TAP::Parser::Iterator::StdOutErr->new($out, $err, $pid);
}
=head1 NAME =head1 NAME
TestBed::ParallelRunner TestBed::ParallelRunner
=over 4 =over 4
=item C<< build_executor >> =item C<< $pr->executor($itemid) >>
return the $itemid th executor
=item C<< $pr->build_executor >>
helper function called by rege. helper function called by rege.
creates a TestBed::ParallelRunner::Executor job creates a TestBed::ParallelRunner::Executor job
=item C<< add_executor($executor) >> =item C<< $pr->add_executor($executor) >>
pushes $executor onto $s->executors list
pushes $executor onto @$Executors =item C<< $pr->runtests($concurrent_pre_runs, $concurrent_node_count_usage) >>
=item C<< runtests >> allows a maximum of $concurrent_pre_runs during parallel execution
allows a maximum of $concurrent_nodes during parallel execution
kicks off execution of parallel tests. start the execution of parallel tests
=item C<< set_test_builder_to_end_state >> =item C<< set_test_builder_to_end_state >>
=item C<< reset_test_builder >> =item C<< reset_test_builder >>
...@@ -209,10 +153,6 @@ B<INTERNAL> functions to get Test::Builder to behave correctly with parallel tes ...@@ -209,10 +153,6 @@ B<INTERNAL> functions to get Test::Builder to behave correctly with parallel tes
wraps two different ways of executing parallel tests and wrapping their TAP output stream wraps two different ways of executing parallel tests and wrapping their TAP output stream
=item C<< build_TAP_stream >>
given a TestBed::ParallelRunner returns a TAP stream
=back =back
=cut =cut
......
...@@ -181,6 +181,8 @@ incrementes attempted retries ...@@ -181,6 +181,8 @@ incrementes attempted retries
schedules task for exectution after backoff time schedules task for exectution after backoff time
=back
=cut =cut
1; 1;
...@@ -4,7 +4,7 @@ use Mouse; ...@@ -4,7 +4,7 @@ use Mouse;
has original => ( is => 'rw'); has original => ( is => 'rw');
no Mouse; no Mouse;
package TestBed::ParallelRunner::Executor::PrepError; package TestBed::ParallelRunner::Executor::PrerunError;
use Mouse; use Mouse;
extends('TestBed::ParallelRunner::Executor::Exception'); extends('TestBed::ParallelRunner::Executor::Exception');
no Mouse; no Mouse;
...@@ -93,17 +93,17 @@ sub handleResult { ...@@ -93,17 +93,17 @@ sub handleResult {
$s->error_strategy->handleResult( @_); $s->error_strategy->handleResult( @_);
} }
sub prep { sub prerun{
my $s = shift; my $s = shift;
if (checkno('create')) { if (checkexclude('create')) {
return +{'maximum_nodes' => 0}; return +{'maximum_nodes' => 0};
} }
my $r = eval { $s->e->create_and_get_metadata($s->ns_text); }; my $r = eval { $s->e->create_and_get_metadata($s->ns_text); };
die TestBed::ParallelRunner::Executor::PrepError->new( original => $@ ) if $@; die TestBed::ParallelRunner::Executor::PrerunError->new( original => $@ ) if $@;
return $r; return $r;
} }
sub checkno { sub checkexclude {
my $stage = shift; my $stage = shift;
return grep { $_ eq $stage } @{ $TBConfig::exclude_steps }; return grep { $_ eq $stage } @{ $TBConfig::exclude_steps };
} }
...@@ -115,18 +115,18 @@ sub execute { ...@@ -115,18 +115,18 @@ sub execute {
my $run_exception; my $run_exception;
my $swapout_exception; my $swapout_exception;
eval { $e->swapin_wait; } unless checkno('swapin'); eval { $e->swapin_wait; } unless checkexclude('swapin');
my $swapin_exception = $@; my $swapin_exception = $@;
unless ($swapin_exception) { unless ($swapin_exception) {
eval { $s->proc->($e); } unless checkno('run'); eval { $s->proc->($e); } unless checkexclude('run');
$run_exception = $@; $run_exception = $@;
eval { $e->swapout_wait; } unless checkno('swapout'); eval { $e->swapout_wait; } unless checkexclude('swapout');
$swapout_exception = $@; $swapout_exception = $@;
} }
eval { $e->end_wait; } unless checkno('end'); eval { $e->end_wait; } unless checkexclude('end');
my $end_exception = $@; my $end_exception = $@;
die TestBed::ParallelRunner::Executor::SwapinError->new( original => $@ ) if $swapin_exception; die TestBed::ParallelRunner::Executor::SwapinError->new( original => $@ ) if $swapin_exception;
...@@ -149,7 +149,15 @@ Represents a ParallelRunner Job ...@@ -149,7 +149,15 @@ Represents a ParallelRunner Job
constructs a TestBed::ParallelRunner::Test job constructs a TestBed::ParallelRunner::Test job
=item C<< $prt->prep >> =item C<< checkexclude($stage_name) >>
checks if $stage_name is in $TBConfig::exclude_steps
=item C<< $prt->ns_text >>
checks if ns_text is a CODE reference, is so execute it otherwise return ns_text
=item C<< $prt->prerun >>
executes the pre_running phase of experiment and determines min and max node counts. executes the pre_running phase of experiment and determines min and max node counts.
......
package TestBed::TestBuilderWrapper;
use Test::Builder;
sub set_test_builder_to_end_state {
my ($test_count, %options) = @_;
my $b = Test::Builder->new;
$b->current_test($test_count);
}
sub reset_test_builder {
my ($test_count, %options) = @_;
my $b = Test::Builder->new;
$b->reset;
$b->use_numbers(0) if $options{no_numbers};
if ($test_count) { $b->expected_tests($test_count); }
else { $b->no_plan; }
}
sub setup_test_builder_ouputs {
my ($out, $err) = @_;
my $b = Test::Builder->new;
$b->output($out);
$b->fail_output($out);
$b->todo_output($out);
}
1;
...@@ -19,23 +19,11 @@ my $error_sub = sub { ...@@ -19,23 +19,11 @@ my $error_sub = sub {
require Exporter; require Exporter;
our @ISA = qw(Exporter); our @ISA = qw(Exporter);
our @EXPORT = qw(e CartProd CartProdRunner concretize defaults override rege runtests prun prunout get_free_node_names); our @EXPORT = qw(e CartProd CartProdRunner concretize defaults override rege prun prunout get_free_node_names);
sub e { TestBed::TestSuite::Experiment->new(_build_e_from_positionals(@_)); } sub e { TestBed::TestSuite::Experiment::build_e(@_); }
sub rege { sub rege { TestBed::ParallelRunner::GlobalRunner->build_executor(@_); }
return TestBed::ParallelRunner::build_executor(@_);
}
sub runtests { TestBed::ParallelRunner::runtests(@_); }
sub _build_e_from_positionals {
if (@_ == 0) { return {}; }
if (@_ == 1) { return { 'eid' => shift }; }
if (@_ == 2) { return { 'pid' => shift, 'eid' => shift }; }
if (@_ == 3) { return { 'pid' => shift, 'gid' => shift, 'eid' => shift }; }
if (@_ > 3) { die 'Too many args to e'; }
}
sub CartProd { sub CartProd {
my $config = shift; my $config = shift;
...@@ -148,13 +136,6 @@ creates a new experiment with pid, gid, and eid ...@@ -148,13 +136,6 @@ creates a new experiment with pid, gid, and eid
registers experiement with parallel test running engine registers experiement with parallel test running engine
=item C<runtests($concurrent_pre_runs, $concurrent_node_count_usage) >
allows a maximum of $concurrent_pre_runs during parallel execution
allows a maximum of $concurrent_nodes during parallel execution
start the execution of parallel tests, not needed
=item C<CartProd($hashref)> Cartesian Product Runner =item C<CartProd($hashref)> Cartesian Product Runner