Commit 7ee4b28a authored by Jonathon Duerig's avatar Jonathon Duerig

Set of scripts for stress testing virtualization technologies and gathering...

Set of scripts for stress testing virtualization technologies and gathering results into pretty pictures.
parent 5dad6440
This is a set of scripts to stress test virtual containers under many
different conditions. There are two sets of scripts here, in two
directories:
testswap/ -- Run many experiments using the testswap infrastructure to
swapmod into using different numbers of pairs. These
scripts need to be run on a machine with testswap set up
properly.
ops/ -- Perform a run on an experiment that is already set up
properly. These scripts need to be installed on ops.
===============
Important files
===============
-------------------------------------------------------------------------------
testswap/master-virt.pl <proj> <exp> <"type" | "name"> <typeName | nodeName>
<runPath> <resultPath> <pairCount [...]>
master-virt.pl will use the testswap framework to setup and run a
series of experimental runs. <proj> and <exp> must describe an
experiment which exists but is swapped out. The node selected for test
can be chosen by type (i.e. 'pc3000') or by name
(i.e. 'pc24'). runPath is the path on ops of the ops
scripts. resultPath is the path on ops specifying where results will
be stored.
master-virt will run experiments using different numbers of pairs of
nodes. The final arguments specify which particular pair groupings to
run. '1 2 5' means run the experiment first with just one pair, then
with 2 pairs, then with 5 pairs.
master-virt will only gather data. You must run process.pl separately
to generate graphs.
-------------------------------------------------------------------------------
ops/process.pl <resultPath>
The <resultPath> given to process.pl should be the same as that given
to master-virt. For each bandwidth, it will create a set of graphs in
that directory. Note that while you can gather any set of
bandwidths/pairs using other scripts, process.pl will try (if
available) to make graphs among the following bandwidths:
my @bwList = (500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, "unlimited");
and the following pairs:
my @pairList = (1, 2, 5, 10, 15, 20, 25, 30, 35);
-------------------------------------------------------------------------------
ops/parallel-run.pl <run-path> <result-path> <proj> <exp> <pairCount>
<"unlimited" | "limited">
If you already have an experiment swapped in and it is set up
correctly, you can use parallel-run to make an experimental run and
then gather results on it. <pairCount> should reflect the number of
pairs in the experiment and <"unlimited" | "limited"> should reflect
whether or not traffic shaping is turned on in the experiment.
-------------------------------------------------------------------------------
#!/usr/bin/perl
if (scalar(@ARGV) < 5)
{
print STDERR "Usage: bw-run.pl <run-path> <result-path> <proj> <exp> <pair-count> <bandwidth [...]>\n";
print STDERR "The special string 'unlimited' removes bandwidth constraints\n";
exit(1);
}
$runPrefix = shift(@ARGV);
$resultPrefix = shift(@ARGV);
$proj = shift(@ARGV);
$exp = shift(@ARGV);
$numPairs = shift(@ARGV);
@bwList = @ARGV;
system("mkdir -p $resultPrefix/output");
sub runtest
{
my $bw = shift(@_);
my $delay = shift(@_);
if ($bw ne "unlimited")
{
my $command = "perl $runPrefix/network.pl $proj $exp " . $numPairs . " "
. $bw . " " . $delay;
print $command."\n";
system($command);
}
$command = "perl $runPrefix/run.pl $proj $exp $resultPrefix/output/$numPairs-$bw-$delay $runPrefix $numPairs";
print $command."\n";
system($command);
}
my @delayList = (0);
my $j = 0;
my $k = 0;
for ($j = 0; $j < scalar(@bwList); ++$j)
{
for ($k = 0; $k < scalar(@delayList); ++$k)
{
print "\n\n\n======BANDWIDTH: " . $bwList[$j] . ", DELAY: "
. $delayList[$k]."\n";
runtest($bwList[$j], $delayList[$k]);
}
}
#!/usr/bin/perl
sub printLog
{
my $line = shift(@_);
my $time = `date +%F_%H:%M:%S`;
chomp($time);
print STDERR $time . " " . $line . "\n";
}
if (scalar(@ARGV) != 4)
{
print STDERR "Usage: client.pl <barrier-name> <pair-number> <duration (s)> <run-path>\n";
exit(1);
}
$DELAY = 1;
$name = $ARGV[0];
$num = $ARGV[1];
$duration = $ARGV[2];
$runPrefix = $ARGV[3];
printLog("$num Client Begin");
printLog("$num Client Before server barrier");
$status = (system("$runPrefix/run-sync.pl 30 -n $name-server") >> 8);
if ($status == 240)
{
exit(1);
}
printLog("$num Client Before iperf");
system("/usr/local/etc/emulab/emulab-iperf -c server-$num -w 256k -i $DELAY -p 4000 -t $duration");
printLog("$num Client Before client barrier");
system("$runPrefix/run-sync.pl $duration -n $name-client");
printLog("$num Client End");
#!/usr/bin/perl
# Usage: command.pl <command> <args>
# Begin executing command
$commandPid = fork();
if ($commandPid == 0)
{
# Command Child
exec(@ARGV);
}
else
{
# Parent
$readPid = fork();
if ($readPid == 0)
{
# Read Child
$readResult = undef;
$readBuffer = 0;
while (! defined($readResult) || $readResult != 0)
{
$readResult = sysread(STDIN, $readBuffer, 1);
}
}
else
{
# Parent
$deadPid = wait();
if ($deadPid != $commandPid)
{
kill('KILL', $commandPid);
}
if ($deadPid != $readPid)
{
kill('KILL', $readPid);
}
}
}
my @pairList = (1, 2, 5, 10, 15, 20, 25, 30, 35);
my $i = 0;
for ($i = 0; $i < scalar(@pairList); ++$i)
{
my $pairs = $pairList[$i];
system("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
. "vhost-0.virt$pairs.tbres.emulab.net "
. "'perl /proj/tbres/duerig/virt/copy-single-delay.pl $pairs'");
}
my $pairs = $ARGV[0];
my $count = $pairs * 2;
system("sudo killall delay-agent");
my $i = 0;
for ($i = 1; $i <= $count; ++$i)
{
system("sudo cp /proj/tbres/duerig/delay-agent/src/testbed/event/delay-agent/new/delay-agent /vz/private/$i/usr/local/etc/emulab");
print "Copying $pairs-$i\n";
}
if (scalar(@ARGV) != 3)
{
print STDERR "Usage: kill.pl <proj> <exp> <pairCount>\n";
exit(1);
}
$proj = $ARGV[0];
$exp = $ARGV[1];
$pairCount = $ARGV[2];
sub killProgram
{
my $agentName = shift(@_);
my $string = "/usr/testbed/bin/tevc ";
my $string = $string."-e $proj/$exp now $agentName stop";
system($string);
}
sub killAll
{
my $i = 1;
killProgram("vhost-0_program");
for ($i = 1; $i <= $pairCount; ++$i)
{
killProgram("client-$i-agent");
killProgram("server-$i-agent");
}
}
killAll();
#!/usr/bin/perl
sub printLog
{
my $line = shift(@_);
my $time = `date +%F_%H:%M:%S`;
chomp($time);
print STDERR $time . " " . $line . "\n";
}
if (scalar(@ARGV) != 3)
{
print STDERR "Usage: monitor.pl <barrier-name> <duration> <run-path>\n";
exit(1);
}
$DELAY = 1;
$COUNT = 3000;
my $name = $ARGV[0];
my $duration = $ARGV[1];
my $runPrefix = $ARGV[2];
$timeout = $duration * 2;
printLog("Monitor Begin");
system("vmstat -n $DELAY $COUNT >& /local/logs/vmstat-cpu.out &");
system("vmstat -d -n $DELAY $COUNT >& /local/logs/vmstat-disk.out &");
system("vmstat -m -n $DELAY $COUNT >& /local/logs/vmstat-slab.out &");
printLog("Monitor Before server barrier");
system("$runPrefix/run-sync.pl 30 -n $name-server");
printLog("Monitor Before client barrier");
system("$runPrefix/run-sync.pl $timeout -n $name-client");
system("killall -9 vmstat");
printLog("Monitor End");
#!/usr/bin/perl
if (scalar(@ARGV) != 5)
{
print STDERR "Usage: network.pl <proj> <exp> <pair-count> <bw> <delay>\n";
}
$proj = $ARGV[0];
$exp = $ARGV[1];
$count = $ARGV[2];
$bw = $ARGV[3];
$delay = $ARGV[4];
for ($i = 1; $i <= $count; ++$i)
{
# $command = "/usr/testbed/bin/tevc -e $proj/$exp now link-$i modify delay=$delay";
# print $command."\n";
# system($command);
$command = "/usr/testbed/bin/tevc -e $proj/$exp now link-$i modify bandwidth=$bw";
print $command."\n";
system($command);
}
#!/usr/bin/perl
if (scalar(@ARGV) != 5)
{
print STDERR "Usage: parallel-run.pl <run-path> <result-path> <proj>\n"
." <exp> <pairCount> <unlimited | limited>\n";
exit(1);
}
my $runPath = shift(@ARGV);
my $resultPrefix = shift(@ARGV);
my $proj = shift(@ARGV);
my $exp = shift(@ARGV);
my $pairCount = shift(@ARGV);
my $unlimited = shift(@ARGV);
system("mkdir -p $resultPrefix/log");
my @bwList = (500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000);
#my @bwList = (500, 1000);
my $bwString = join(" ", @bwList);
if ($unlimited eq "unlimited")
{
$bwString = "unlimited";
}
#my $i = 0;
#for ($i = 0; $i < scalar(@pairList); ++$i)
#{
# my $pairCount = $pairList[$i];
my $command = "perl bw-run.pl $runPath $resultPrefix $proj $exp "
. "$pairCount $bwString | tee $resultPrefix/log/$pairCount.log";
system($command);
#}
#!/usr/bin/perl
if (scalar(@ARGV) != 1)
{
print STDERR "Usage: process.pl <result-path>\n";
exit(1);
}
$resultPrefix = $ARGV[0];
$IDLE_FIELD = 15;
$SYSTEM_FIELD = 14;
$FREE_FIELD = 4;
$INTERRUPT_FIELD = 11;
sub getBandwidthList
{
my $prefix = shift(@_);
my $pairCount = shift(@_);
my @result = ();
my $i = 0;
for ($i = 1; $i <= $pairCount; ++$i)
{
my $file = "$prefix/client-$i/local/logs/client-$i-agent.out";
my $line = `tail -n 1 $file`;
my $current = 0;
if ($line =~ /([0-9.]+) Gbits\/sec/)
{
$current = $1 * 1024;
}
elsif ($line =~ /([0-9.]+) Mbits\/sec/)
{
$current = $1;
}
elsif ($line =~ /([0-9.]+) Kbits\/sec/)
{
$current = $1 / 1024;
}
elsif ($line =~ /([0-9.]+) bits\/sec/)
{
$current = $1 / 1024 / 1024;
}
push(@result, $current);
}
return @result;
}
sub getSlabList
{
my $prefix = shift(@_);
my $systemString = `grep skbuff_fclone_cache $prefix/vhost-0/local/logs/vmstat-slab.out | sed 's/ / /g' | sed 's/ / /g' | sed 's/ / /g' | sed 's/ / /g' | cut -d ' ' -f 3 | tail -n +10`;
my @lines = split(/\n/, $systemString);
my @result = ();
my $i = 0;
for ($i = 0; $i < scalar(@lines); ++$i)
{
push(@result, $lines[$i] * 384);
}
return @result;
}
sub getCpuList
{
my $prefix = shift(@_);
my $field = shift(@_);
my $systemString = `tail -n +7 $prefix/vhost-0/local/logs/vmstat-cpu.out | sed 's/ / /g' | sed 's/ / /g' | sed 's/ / /g' | sed 's/^ //' | cut -d ' ' -f $field`;
my @systems = split(/\n/, $systemString);
my @result = ();
my $i = 0;
for ($i = 0; $i < scalar(@systems); ++$i)
{
push(@result, $systems[$i]);
}
return @result;
}
sub scaleList
{
my $factor = shift(@_);
my $i = 0;
my @result = ();
for ($i = 0; $i < scalar(@_); ++$i)
{
push(@result, $factor * @_[$i]);
}
return @result;
}
sub invertBoundList
{
my $bound = shift(@_);
my $i = 0;
my @result = ();
for ($i = 0; $i < scalar(@_); ++$i)
{
push(@result, $bound - @_[$i]);
}
return @result;
}
sub totalList
{
my $i = 0;
my $result = 0;
for ($i = 0; $i < scalar(@_); ++$i)
{
$result += @_[$i];
}
return $result;
}
sub meanList
{
my $result = -999;
my $total = totalList(@_);
if (scalar(@_) > 0)
{
$result = $total / scalar(@_);
}
return $result;
}
sub stderrList
{
my @data = @_;
my $i = 0;
my $n = scalar(@data);
if ($n < 2)
{
return 0;
}
my $mean = meanList(@data);
my $total = 0;
for ($i = 0; $i < $n; ++$i)
{
my $diff = $data[$i] - $mean;
$total += $diff * $diff;
}
my $stddev = sqrt($total / ($n-1));
my $stderr = $stddev / sqrt($n);
return $stderr;
}
sub lowerBound
{
my $mean = shift(@_);
my $stderr = shift(@_);
my $lower = $mean - $stderr * 1.96;
return $lower;
}
sub upperBound
{
my $mean = shift(@_);
my $stderr = shift(@_);
my $upper = $mean + $stderr * 1.96;
return $upper;
}
sub errorBarLine
{
my $count = shift(@_);
my @data = @_;
my $mean = meanList(@data);
my $stderr = stderrList(@data);
my $lower = lowerBound($mean, $stderr);
my $upper = upperBound($mean, $stderr);
return "$count $mean $lower $upper\n";
}
system("mkdir -p $resultPrefix/graph");
system("mkdir -p $resultPrefix/plot");
my @pairList = (1, 2, 5, 10, 15, 20, 25, 30, 35);
my @bwList = (500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, "unlimited");
my $delay = 0;
sub printPlot
{
my $nick = shift(@_);
my $prefix = shift(@_);
my $output = shift(@_);
my @bwList = @_;
open PLOT_FILE, ">$resultPrefix/plot/$nick-plot";
print PLOT_FILE "$prefix\n";
print PLOT_FILE "set term postscript enhanced\n";
print PLOT_FILE "set output\"$resultPrefix/$output.ps\"\n";
print PLOT_FILE "plot ";
my $i = 0;
for ($i = 0; $i < scalar(@bwList); ++$i)
{
if ($i != 0)
{
print PLOT_FILE ", ";
}
my $bw = $bwList[$i];
if ($bw ne "unlimited")
{
$bw = $bw / 1000;
}
print PLOT_FILE "\"$resultPrefix/graph/$nick-" . $bwList[$i]
. "-0.graph\" with linespoints title \"$bw Mbps, 0 ms\"";
}
print PLOT_FILE ";\n";
close PLOT_FILE;
}
sub runPlot
{
my $nick = shift(@_);
system("gnuplot $resultPrefix/plot/$nick-plot");
}
my $totalPlot = <<'PLOTEND';
set title "Aggregate Bandwidth";
set xlabel "# of Pairs";
set ylabel "Aggregate Bandwidth (Mbps)";
set yrange [0:];
PLOTEND
printPlot("total", $totalPlot, "openvz-total-bandwidth", @bwList);
my $meanPlot = <<'PLOTEND';
set title "Mean Bandwidth";
set xlabel "# of Pairs";
set ylabel "Mean Bandwidth (Mbps)";
set yrange [0:];
PLOTEND
printPlot("mean", $meanPlot, "openvz-mean-bandwidth", @bwList);
my $cpuPlot = <<'PLOTEND';
set title "Mean CPU Usage (total)";
set xlabel "# of Pairs";
set ylabel "Mean CPU usage (total) (%)";
set yrange [0:100];
PLOTEND
printPlot("cpu", $cpuPlot, "openvz-cpu", @bwList);
my $systemPlot = <<'PLOTEND';
set title "Mean CPU Usage (system)";
set xlabel "# of Pairs";
set ylabel "Mean CPU usage (system) (%)";
set yrange [0:100];
PLOTEND
printPlot("system", $systemPlot, "openvz-system", @bwList);
my $memoryPlot = <<'PLOTEND';
set title "Mean Free Page Count";
set xlabel "# of Pairs";
set ylabel "Mean Free Page Count (thousands)";
set yrange [0:];
PLOTEND
printPlot("memory", $memoryPlot, "openvz-memory", @bwList);
my $interruptPlot = <<'PLOTEND';
set title "Mean Interrupt Rate";
set xlabel "# of Pairs";
set ylabel "Mean Interrupt Rate (per second)";
set yrange [:];
set key left top;
PLOTEND
printPlot("interrupt", $interruptPlot, "openvz-interrupt", @bwList);
my $slabPlot = <<'PLOTEND';
set title "Mean skbuff-fclone-cache Slab Size"
set xlabel "# of Pairs"
set ylabel "Mean Slab Size (KB)"
set yrange [:];
set key left top;
PLOTEND
printPlot("slab", $slabPlot, "openvz-slab", @bwList);
my $i = 0;
my $j = 0;
for ($i = 0; $i < scalar(@bwList); ++$i)
{
open TOTAL_OUT, ">$resultPrefix/graph/total-".$bwList[$i]."-0.graph";
open MEAN_OUT, ">$resultPrefix/graph/mean-".$bwList[$i]."-0.graph";
open CPU_OUT, ">$resultPrefix/graph/cpu-".$bwList[$i]."-0.graph";
open SYSTEM_OUT, ">$resultPrefix/graph/system-".$bwList[$i]."-0.graph";
open MEMORY_OUT, ">$resultPrefix/graph/memory-".$bwList[$i]."-0.graph";
open INTERRUPT_OUT, ">$resultPrefix/graph/interrupt-".$bwList[$i]."-0.graph";
open SLAB_OUT, ">$resultPrefix/graph/slab-".$bwList[$i]."-0.graph";
for ($j = 0; $j < scalar(@pairList); ++$j)
{
my $count = $pairList[$j];
my $prefix = "$resultPrefix/output/" . $pairList[$j] . "-"
. $bwList[$i] . "-$delay/logs";
if (-e $prefix)
{
my @bwResults = getBandwidthList($prefix, $count);
my $total = totalList(@bwResults);
print TOTAL_OUT "$count $total\n";
print MEAN_OUT errorBarLine($count, @bwResults);
my @idleResults = getCpuList($prefix, $IDLE_FIELD);
my @cpuResults = invertBoundList(100, @idleResults);
print CPU_OUT errorBarLine($count, @cpuResults);
my @systemResults = getCpuList($prefix, $SYSTEM_FIELD);
print SYSTEM_OUT errorBarLine($count, @systemResults);
my @memoryResults = getCpuList($prefix, $FREE_FIELD);
my @memoryThousands = scaleList(1/1000.0, @memoryResults);
print MEMORY_OUT errorBarLine($count, @memoryThousands);
my @interruptResults = getCpuList($prefix, $INTERRUPT_FIELD);
print INTERRUPT_OUT errorBarLine($count, @interruptResults);
my @slabResults = getSlabList($prefix);
my @slabK = scaleList(1/1024.0, @slabResults);
print SLAB_OUT errorBarLine($count, @slabK);
}
}
close TOTAL_OUT;
close MEAN_OUT;
close CPU_OUT;
close SYSTEM_OUT;
close MEMORY_OUT;
close INTERRUPT_OUT;
close SLAB_OUT;
}
runPlot("total");
runPlot("mean");
runPlot("cpu");
runPlot("system");
runPlot("memory");
runPlot("interrupt");
runPlot("slab");
#!/usr/bin/perl
$pid = `cat /var/run/syncd.pid`;
$command = "sudo kill -s HUP $pid";
print "$command\n";
system($command);
#!/usr/bin/perl
use POSIX;
#use POSIX ":sys_wait_h";
my $timeout = shift(@ARGV);
sub runProgram
{
my $command = shift(@_);
my $group = getpgrp();
my $parentPid = POSIX::getpid();
my $pid = fork();
my $result = 0;
if ($pid == 0)
{
setpgid(0, $parentPid);
# Child
exec($command);
}
else
{
setpgid($pid, $parentPid);
# Parent
waitpid($pid, 0);
$result = ($? >> 8);
}
return $result;
}
sub runSync
{
my $tries = 5;