Commit 2a8299ee authored by Pramod R Sanaga's avatar Pramod R Sanaga

SanityCheck.py is the script for checking cross-correlation of Flexlab

runs. It works for both TCP and UDP transfers. I also changed
auto-pelab.ns and added a "-t" option to start-experiment
and stop-experiment to enable these sanity checks.

Note: I haven't figured out how to kill sudo tcpdump sessions
on PlanetLab with program agents - so, for now they have to be
killed manually after a stop-experiment. Will change this after
talking to Mike.

The cross-correlation code works correctly - I may need to tweak it
a bit in the near future for averaging times.
Also, it puts too much load on Ops and (i think) takes too much time.
I will change it so that all the correlation calculations are done on
the experimental nodes.
parent 9812a826
This diff is collapsed.
......@@ -73,6 +73,7 @@ set plabsites {}
set plabnodes {}
# Example node list
#set plabnodes {plab518 plab541 plab628 plab736 plab360}
set plabnodes {plab540 plab18}
#
# This provides you with an easy way to skip certain hosts in the above lists
......@@ -94,6 +95,7 @@ set pelab_tar "/proj/tbres/CHANGEME.tar.gz"
# value, like "pc3000", to get specific types of PCs
#
set hwtype "pc"
set do_bsd 0
#
# Server and client to use for automated testing. If set, will automatically
......@@ -119,7 +121,6 @@ set allnodesprog "/bin/true"
#
# Run FreeBSD 6.1 on elab_ nodes. Forces use of dbmonitor.
#
set do_bsd 0
#
# If non-zero, uses the DB-based "monitor" to control the cloud shaping. This
......@@ -136,7 +137,9 @@ if {$do_bsd} {
# same node. Only supported when use_magent is set
#
set stub_port 3149
set udp_stub_peer_port 3492
######
# Options for the dbmonitor (simple models)
######
......@@ -207,7 +210,7 @@ set no_stub 0
# If set to 1, uses the new stub as a receiver
# If set to 2, uses a separate iperfd to receive traffic
#
set use_magent 2
set use_magent 1
# End of user-settable options
##########
......@@ -221,8 +224,8 @@ if {$do_bsd} {
set node_os FBSD61-STD
} else {
# XXX -UPDATE for now. Contains bug fixes (progagent command lines).
set node_os PLAB-DEVBOX-UPDATE
#set node_os PLAB-DEVBOX
#set node_os PLAB-DEVBOX-UPDATE
set node_os PLAB-DEVBOX
}
tb-set-delay-os $delay_os
......@@ -255,7 +258,7 @@ for {set i 0} {$i < [llength $skiplist]} {incr i} {
}
set stub_peer_port $stub_port
set stub_command_port [expr $stub_port + 1]
set stub_command_port 3150
#
# Tarballs and RPMs we install on all nodes
......@@ -311,6 +314,8 @@ set clientlist {}
set tflist {}
set anlist {}
set sanitychecklist {}
#
# Create all of the nodes
#
......@@ -326,6 +331,10 @@ for {set i 1} {$i <= $num_pcs} {incr i} {
lappend stublist $planetstub($i)
lappend planetstublist $planetstub($i)
set plabsanitycheck($i) [$planet($i) program-agent -expected-exit-code $ecode -command "sudo /usr/sbin/tcpdump -w /local/logs/SanityCheck.log -i vnet \\( \\(tcp and dst host $planet($i) and dst port $stub_peer_port \\) or \\(udp and dst host $planet($i) and dst port $udp_stub_peer_port \\) \\) "]
lappend sanitychecklist $plabsanitycheck($i)
tb-set-node-tarfiles $planet($i) $tarfiles
tb-set-node-rpms $planet($i) $plab_rpms
......@@ -376,6 +385,10 @@ for {set i 1} {$i <= $num_pcs} {incr i} {
lappend serverlist $server($i)
lappend clientlist $client($i)
set elabsanitycheck($i) [$elab($i) program-agent -expected-exit-code $ecode -command "sudo tcpdump -w /local/logs/SanityCheck.log -i any \\( \\(tcp and dst host $elab($i) \\) or \\(udp and dst host $elab($i) \\) \\) "]
lappend sanitychecklist $elabsanitycheck($i)
tb-set-node-tarfiles $elab($i) $tarfiles
if {$elab_rpms != ""} {
tb-set-node-rpms $elab($i) $elab_rpms
......@@ -429,6 +442,8 @@ set allclients [$ns event-group $clientlist]
set tfhosts [$ns event-group $tflist]
set anhosts [$ns event-group $anlist]
set sanityCheckDumps [$ns event-group $sanitychecklist]
set tracelist {}
#
......@@ -550,6 +565,8 @@ if {$no_plab || $real_plab} {
$start append "$link trace stop"
}
$start append "$sanityCheckDumps stop"
# clean out log files
# XXX original script passed --root, cleanlogs does not--may be a problem.
$start append "$ns log \"##### Cleaning logs...\""
......@@ -620,6 +637,8 @@ if {$no_plab || $real_plab} {
$stop append "$allservers stop"
}
$stop append "$sanityCheckDumps stop"
# stop link logging and save logs
$stop append "$ns log \"##### Stop link tracing...\""
foreach link $tracelist {
......
......@@ -19,6 +19,7 @@ my $INITCOND = "init-elabnodes.pl";
my $realplab = 0;
my $initelab = 0;
my $sanityChecks = 0;
my $UNKNOWN = "<unknown>";
my $stub_cmd = "/bin/sh /local/pelab/stub/auto-stub.sh";
......@@ -39,10 +40,11 @@ sub usage()
" -s command-args append args to stub command line\n".
" -M command-line replace the current monitor command arguments\n".
" -m command-args append args to monitor command line\n".
" -t start tcpdumps for cross-correlation checks\n".
" -C show current command line(s)\n";
exit(1);
}
my $optlist = "CS:s:M:m:piN:O:";
my $optlist = "CS:s:M:m:piN:O:t";
my $pelabdir;
if ($0 =~ /(.*)\/[^\/]+$/) {
......@@ -151,6 +153,9 @@ if (defined($options{"i"})) {
# die "Unknown -O type '", $options{"O"}, "'\n";
# }
#}
if (defined($options{"t"})) {
$sanityChecks = 1;
}
my $use_magent = ($stub_cmd =~ /magent/) ? 1 : 0;
my $use_simplemodel = ($mon_cmd =~ /dbmonitor/) ? 1 : 0;
......@@ -333,6 +338,11 @@ if (system "$TEVC -e $pid/$eid now monitorgroup start") {
# die "Error running tevc\n";
#}
print "##### Starting tcpdumps for cross-correlation sanity checks\n";
if (system "$TEVC -e $pid/$eid now sanityCheckDumps start") {
die "Error running tevc\n";
}
print "##### Done\n";
exit(0);
......
......@@ -11,8 +11,9 @@ use Getopt::Std;
my $TEVC = "/usr/testbed/bin/tevc";
my $LOGHOLE = "/usr/testbed/bin/loghole";
my $optlist = "n";
my $optlist = "n:t";
my $nologs = 0;
my $sanityChecks = 0;
#
# Parse command arguments.
......@@ -34,6 +35,10 @@ if (defined($options{"n"})) {
$nologs = 1;
}
if (defined($options{"t"})) {
$sanityChecks = 1;
}
#
# Stop the stubs and monitors
#
......@@ -79,6 +84,11 @@ if (system "$TEVC -e $pid/$eid now tdhosts stop") {
die "Error running tevc\n";
}
print "##### Stopping sanity check tcpdumps\n";
if (system "$TEVC -e $pid/$eid now sanityCheckDumps stop") {
die "Error running tevc\n";
}
#
# Stop running servers
#
......@@ -96,16 +106,25 @@ if (!$nologs) {
die "Error running loghole\n";
}
# Run cross-correlation checks on the collected Tcpdump files.
if ($sanityChecks) {
print "##### Running cross-correlation sanity checks\n";
if (system "python SanityCheck.py -t -u /proj/$pid/exp/$eid/") {
die "Error running cross-correlation sanity check\n";
}
}
#
# Create an archive containing these logfiles
#
print "##### Creating archive\n";
if (system "$LOGHOLE -e $pid/$eid archive") {
die "Error running loghole\n";
}
## if (system "$LOGHOLE -e $pid/$eid archive") {
## die "Error running loghole\n";
## }
} else {
print "##### Ignoring logfiles\n";
}
print "##### Done\n";
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment