Commit 09b8130b authored by Pramod R Sanaga's avatar Pramod R Sanaga

Changes to integrate Rubenstein's and wavelet processing into a common set

of scripts - and a lot of CVS cleanup.

Also, added a check for variance in the case of Rubenstein congestion detection
code - the results are identical to those of wavelets in my Emulab experiment.
I guess we'll carry on working with both..
parent 82f1b29c
......@@ -4,51 +4,104 @@
# Copyright (c) 2006 University of Utah and the Flux Group.
# All rights reserved.
#
use Getopt::Long;
local $RubensteinScriptPath = "/proj/tbres/pramod/bottleneck-emulab/filter/genjitco.i386";
local $MinSikKimScriptPath = "/proj/tbres/pramod/bottleneck-emulab/devel/MinSikKim/MinSikKimProgram";
local $ClusterProgramPath = "/proj/tbres/pramod/bottleneck-emulab/devel/Clustering/c++-samples/wvcluster";
local $elabInitScript = "/proj/tbres/duerig/testbed/pelab/init-elabnodes.pl";
my $expName;
my $projName;
my $logsDir;
local $newExpName = "";
local $newProjName = "";
local $logsDir = "";
local $nodeListFilename = "";
local $Algorithm = "";
local $initialConditionsFilename = "";
my $newExpName;
my $newProjName;
local $scratchSpaceDir = "/tmp/bw-bottleneck/rubenstein";
%bottleNecks = {};
my %nodeClasses;
my %nodeNameMap = {};
# Print usage and die.
die "Usage: perl sharedBottle.pl proj_name exp_name newProj_name newExp_name initial_conditions.txt(Optional)"
if($#ARGV < 3);
sub PrintUsage()
{
print "Usage: perl createClusters.pl -NewProj=newProjName -NewExp=newExpName -LogDir=/path/to/logs -nodeFile=/path/to/nodeListFile -Algorithm=rubestein/wavelet (rest optional) -RubensteinPath=/path/to/genjitco.arch -MinSikKimPath=/path/to/MinSikKimProgram -ClusterProgPath=/path/to/wvcluster -InitConditions=/path/to/initial_conditions.txt\n";
exit(1);
}
$projName = $ARGV[0];
$expName = $ARGV[1];
$newProjName = $ARGV[2];
$newExpName = $ARGV[3];
$initialConditionsFilename = $ARGV[4];
# Check for the validity of command line arguments.
$logsDir = "/tmp/testLogs";
sub processOptions()
{
&GetOptions("LogDir=s"=>\$logsDir,
"NewProj=s"=>\$newProjName,
"NewExp=s"=>\$newExpName,
"InitConditions=s"=>\$initialConditionsFilename,
"NodeFile=s"=>\$nodeListFilename,
"Algorithm=s"=>\$Algorithm,
"RubensteinPath=s"=>\$RubensteinScriptPath,
"MinSikKimPath=s"=>\$MinSikKimScriptPath,
"ClusterProgPath=s"=>\$ClusterProgramPath);
if($newExpName eq "" or $newProjName eq "" or $logsDir eq "" or $nodeListFilename eq "" or $Algorithm eq "")
{
print "ERROR: One of the mandatory arguments is empty.\n";
&PrintUsage();
}
if(not(-e $logsDir))
{
print "ERROR: Path provided to the logs directory is invalid.\n";
&PrintUsage();
}
if(not(-e $nodeListFilename))
{
print "ERROR: Path provided to the file with node names is invalid.\n";
&PrintUsage();
}
if( not( ($Algorithm =~ /rubenstein/i) || ($Algorithm =~ /wavelet/i) ) )
{
print "ERROR: Please choose either 'rubenstein' or 'wavelet' ( without quotes, case insensitive ) as the algorithm\n";
&PrintUsage();
}
if($initialConditionsFilename ne "")
{
if(not(-e $initialConditionsFilename))
{
print "WARNING: The initial conditions file path provided at command line is invalid.\n";
print "Reverting back to using init-elabnodes.pl\n";
}
}
}
#############################################################################
############################ MAIN #####################################
#############################################################################
# Process the command line options.
&processOptions();
# Get the initial conditions.
$elabInitScript = "/proj/tbres/duerig/testbed/pelab/init-elabnodes.pl";
# Get initial conditions for the paths of interest
# from the database, using init-elabnodes.pl
$initConditionsCommand = $elabInitScript . " -o /tmp/initial-conditions.txt " . $newProjName . " " . $newExpName;
if($#ARGV == 3)
if( ($initialConditionsFilename eq "") or (not(-e $initialConditionsFilename)) )
{
system($initConditionsCommand);
$initialConditionsFilename = "/tmp/initial-conditions.txt";
}
open(CONDITIONS, $initialConditionsFilename);
my @initialConditions = ();
while(<CONDITIONS>)
{
chomp( $_ );
push(@initialConditions, $_);
}
my @initialConditions = <CONDITIONS>;
chomp(@initialConditions);
close(CONDITIONS);
my @addressList = ();
......@@ -57,16 +110,8 @@ my $addrFlag = 0;
my %bwMap = {};
my %delayMap = {};
my %elabMap = {};
# Create & send events.
# Get initial conditions for the paths of interest
# from the database, using init-elabnodes.pl
my $tevc = "/usr/testbed/bin/tevc -e $newProjName/$newExpName now";
#@@@`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
#@@@`$tevc elabc create start`;
# Create a list of the IP addresses.
my $numNodes = 0;
# Create a list of the IP addresses listed in the initial-conditions file.
foreach $conditionLine (@initialConditions)
{
if($conditionLine =~ /(\d*?\.\d*?\.\d*?\.(\d*?))\s(\d*?\.\d*?\.\d*?\.\d*?)\s(\d*?)\s(\d*?)\s[\d\w\-\.]*/)
......@@ -85,9 +130,9 @@ foreach $conditionLine (@initialConditions)
if($addrFlag == 0)
{
push(@addressList, $srcAddress);
$numNodes += 1;
$elabMap{$srcAddress} = "elabc-elab-" . $2;
print "Mapping $2 TO $elabMap{$srcAddress}\n";
}
# Create a mapping of the initial conditions.
......@@ -96,6 +141,109 @@ foreach $conditionLine (@initialConditions)
}
}
# Read the file listing the hostnames of the nodes involved in this
# particular measurement.
open(INFILE, "<$nodeListFilename");
my @nodeNameList = <INFILE>;
chomp(@nodeNameList);
close(INFILE);
# A little checking to ensure that the number of nodes in the NodeList file
# matches the values in the initial conditions file.
if($numNodes != ($#nodeNameList+1) )
{
print "ERROR: Mismatch between the number of nodes in InitialConditions.txt and in the NodeList file.\n";
print "Both the files should have the same number of nodes for a 1-to-1 mapping to be possible.\n";
exit(1);
}
# Convert the delay data produced by the wavelet code
# into something palatable to the Rubenstein program.
if($Algorithm =~ /rubenstein/i)
{
if(-e $scratchSpaceDir)
{
`rm -rf $scratchSpaceDir`;
}
`mkdir $scratchSpaceDir`;
opendir(logsDirHandle, $logsDir);
foreach $sourceName (readdir(logsDirHandle))
{
if( (-d $logsDir . "/" . $sourceName ) and $sourceName ne "." and $sourceName ne ".." )
{
if( grep(/^$sourceName$/, @nodeNameList) )
{
@dirNameList = ();
opendir(sourceDirHandle, $logsDir . "/" . $sourceName);
foreach $destOne (readdir(sourceDirHandle))
{
if( (-d $logsDir . "/" . $sourceName . "/" . $destOne) && $destOne ne "." && $destOne ne ".." )
{
if( grep(/^$destOne$/, @nodeNameList) )
{
if(-e "$logsDir/$sourceName/$destOne/delay.log")
{
push(@dirNameList, $destOne);
}
else
{
print "WARNING: Missing log file $logsDir/$sourceName/$destOne/delay.log\n";
}
}
}
}
closedir(sourceDirHandle);
my $firstLevelIndex = 0, $secondLevelIndex = 0;
foreach $destOne (@dirNameList)
{
$secondLevelIndex = 0;
foreach $destTwo (@dirNameList)
{
if($secondLevelIndex > $firstLevelIndex)
{
`mkdir -p $scratchSpaceDir/$sourceName/$destOne/$destTwo`;
`paste -d ' ' $logsDir/$sourceName/$destOne/delay.log $logsDir/$sourceName/$destTwo/delay.log > $scratchSpaceDir/$sourceName/$destOne/$destTwo/delay.log `;
}
$secondLevelIndex++;
}
$firstLevelIndex++;
}
}
}
}
closedir(sourceDirHandle);
# Call our sharedBottle.pl to feed these .filter files we created to
# sharedBottle.pl - It will then create equivalence classes for the
# paths.
system("perl sharedBottle.pl $scratchSpaceDir $newProjName $newExpName $initialConditionsFilename $RubensteinScriptPath");
# Our work here is done, sign off.
exit(0);
}
#############################################################################
######################### Wavelet processing ################################
#############################################################################
%bottleNecks = {};
my %nodeClasses;
my %nodeNameMap = {};
# Create & send events.
my $tevc = "/usr/testbed/bin/tevc -e $newProjName/$newExpName now";
`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
`$tevc elabc create start`;
opendir(logsDirHandle, $logsDir);
my $addrIndex = 0;
......@@ -133,8 +281,6 @@ foreach $sourceName (readdir(logsDirHandle))
my @denoisedDelays = ();
# print "Into directory $sourceName\n";
foreach $destOne (readdir(sourceDirHandle))
{
if( (-d $logsDir . "/" . $sourceName . "/" . $destOne) && $destOne ne "." && $destOne ne ".." )
......@@ -142,10 +288,8 @@ foreach $sourceName (readdir(logsDirHandle))
# Inside each destination directory, look for
# delay.log file with delay values.
# print "Into directory $sourceName/$destOne\n";
$fullPath = "$logsDir/$sourceName/$destOne/delay.log";
$waveletScript = "./MinSikKim/MinSikKimProgram $fullPath";
$waveletScript = "$MinSikKimScriptPath $fullPath";
my @scriptOutput;
@scriptOutput = `$waveletScript`;
......@@ -174,7 +318,7 @@ foreach $sourceName (readdir(logsDirHandle))
# Put this destination in a seperate cluster - we
# have zero samples/delay values.
if( ($#scriptOutput == 0) or ($#scriptOutput < 120) )
if( ($#scriptOutput == 0) or ($#scriptOutput < 128) )
{
my @newEquivClass = ();
push(@newEquivClass, $i);
......@@ -234,7 +378,7 @@ foreach $sourceName (readdir(logsDirHandle))
}
close(RECORDFILE);
$clusteringProgram = "/tmp/clustering/c++-samples/wvcluster $tmpTreeRecordFile /tmp/tmp.idx";
$clusteringProgram = "$ClusterProgramPath $tmpTreeRecordFile /tmp/tmp.idx";
my @clusteringOutput ;
......@@ -298,18 +442,17 @@ foreach $sourceName (readdir(logsDirHandle))
}
# my $delayEventCommand = "$tevc $elabMap{$addrNodeMapping{$sourceName}} modify DEST=" . $addrNodeMapping{$destSeen[$tmpName2]};
my $delayEventCommand = "$tevc ".$elabMap{$addrNodeMapping{$nodeNumberMapping{$tmpName2}}}." modify DEST=" . $addrNodeMapping{$nodeNumberMapping{$tmpName2}}." SRC=".$addrNodeMapping{$sourceName};
$delayEventCommand = $delayEventCommand . " " . "DELAY=" . ($delayMap{$addrNodeMapping{$sourceName}}{$addrNodeMapping{$nodeNumberMapping{$tmpName2}}});
# Execute the delay event command.
print "EXECUTE $delayEventCommand\n";
#@@@ `$delayEventCommand`;
`$delayEventCommand`;
}
$bwEventCommand = $bwEventCommand . " " . "BANDWIDTH=" . $maxBw;
# Execute the event to set the bandwidth for this equivalence class.
print "EXECUTE $bwEventCommand\n";
#@@@ `$bwEventCommand`;
`$bwEventCommand`;
}
print "\n\n";
......
# delay2dump.pl
if ($ARGV != 2)
if ($#ARGV != 1)
{
die ("Usage: delay2dump.pl <in-file> <out-path>");
}
......@@ -16,17 +16,35 @@ $sequence = 0;
while ($line = <IN>)
{
$line =~ /^([0-9.]+) ([0-9.]+) ([0-9.]+) ([0-9.]+)$/;
$send1 = $1;
$send2 = $2;
$delay1 = $3;
$delay2 = $4;
$send1 = $2;
$send2 = $4;
$delay1 = $1;
$delay2 = $3;
$send1_delay = ($send1 + $delay1)*1000;
$send2_delay = ($send2 + $delay2)*1000;
$send1 = $send1*1000;
$send2 = $send2*1000;
$send1_sec = $send1/1000000;
$send1_usec = $send1 % 1000000;
$send2_sec = $send2/1000000;
$send2_usec = $send2 % 1000000;
$send1_delay_sec = $send1_delay/1000000;
$send1_delay_usec = $send1_delay % 1000000;
$send2_delay_sec = $send2_delay/1000000;
$send2_delay_usec = $send2_delay % 1000000;
if ($send1 ne -9999 && $send2 ne -9999)
{
print (OUT_SOURCE, $send1." 10.0.0.2 ".$sequence);
print (OUT_SOURCE, $send2." 10.0.0.3 ".$sequence);
print (OUT_DEST1, ($send1 + $delay1)." 10.0.0.1 ".$sequence);
print (OUT_DEST2, ($send2 + $delay2)." 10.0.0.1 ".$sequence);
printf (OUT_SOURCE "%d.%d 10.0.0.2 %d\n", $send1_sec,$send1_usec,$sequence);
printf (OUT_SOURCE "%d.%d 10.0.0.3 %d\n", $send2_sec,$send2_usec,$sequence);
printf (OUT_DEST1 "%d.%d 10.0.0.2 %d\n",$send1_delay_sec,$send1_delay_usec,$sequence);
printf (OUT_DEST2 "%d.%d 10.0.0.3 %d\n",$send2_delay_sec,$send2_delay_usec,$sequence);
}
$sequence++;
......
......@@ -16,27 +16,27 @@ sub setupDest
my %hash;
my $file;
open($file, "<".$fileName)
or die("Couldn't open file $fileName for reading");
or die("Couldn't open file $fileName for reading");
my @array = <$file>;
for ($i = 0; $i < @array; ++$i)
{
$array[$i] =~ /$dumpPattern/;
$hash{$3} = $1;
$array[$i] =~ /$dumpPattern/;
$hash{$3} = $1;
}
$array[0] =~ /$dumpPattern/;
if ($_[0] == 1)
{
$dest1Address = $2;
$dest1Address = $2;
}
else
{
$dest2Address = $2;
$dest2Address = $2;
}
return %hash;
}
open(SOURCE_IN, "<$basePath/source.dump")
or die("Couldn't open file $basePath/source.dump for reading");
or die("Couldn't open file $basePath/source.dump for reading");
@source = <SOURCE_IN>;
close(SOURCE_IN);
......@@ -60,32 +60,33 @@ for ($i = 0; $i < @source; ++$i)
$flowAddress = $2;
$flowSequence = $3;
if ($flowAddress == $dest1Address)
if ($flowAddress eq $dest1Address)
{
$destNum = 1;
$destSequence = $sequence1;
++$sequence1;
$destTime = $dest1{$3};
$destNum = 1;
$destSequence = $sequence1;
++$sequence1;
$destTime = $dest1{$3};
}
else
{
$destNum = 2;
$destSequence = $sequence2;
++$sequence2;
$destTime = $dest2{$3};
$destNum = 2;
$destSequence = $sequence2;
++$sequence2;
$destTime = $dest2{$3};
}
if (defined($destTime))
{
if ($flowAddress == $dest1Address)
{
print DEST1_OUT 'Rcvr: (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime.' arr '.$destTime."\n";
}
else
{
print DEST2_OUT 'Rcvr: (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime.' arr '.$destTime."\n";
}
print SOURCE_OUT 'Src1: send (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime."\n";
++$merged;
if ($flowAddress eq $dest1Address)
{
print DEST1_OUT 'Rcvr: (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime.' arr '.$destTime."\n";
}
else
{
print DEST2_OUT 'Rcvr: (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime.' arr '.$destTime."\n";
}
print SOURCE_OUT 'Src1: send (f'.$destNum.':'.$destSequence.':M'.$merged.')@'.$sourceTime." \n";
++$merged;
}
}
......@@ -16,27 +16,27 @@ my $newProjName;
%bottleNecks = {};
my %nodeClasses;
die "Usage: perl sharedBottle.pl proj_name exp_name newProj_name newExp_name initial_conditions.txt(Optional)"
if($#ARGV < 3);
die "Usage: perl sharedBottle.pl logsDir newProj_name newExp_name initial_conditions.txt DansScript"
if($#ARGV < 4);
$projName = $ARGV[0];
$expName = $ARGV[1];
$newProjName = $ARGV[2];
$newExpName = $ARGV[3];
$initialConditionsFilename = $ARGV[4];
$logsDir = $ARGV[0];
$newProjName = $ARGV[1];
$newExpName = $ARGV[2];
$initialConditionsFilename = $ARGV[3];
$DansScriptPath = $ARGV[4];
$logsDir = "/proj/$projName/exp/$expName/logs/dump";
#$logsDir = "/proj/$projName/exp/$expName/logs/dump";
# Get the initial conditions.
$elabInitScript = "/proj/tbres/duerig/testbed/pelab/init-elabnodes.pl";
$initConditionsCommand = $elabInitScript . " -o /tmp/initial-conditions.txt " . $newProjName . " " . $newExpName;
#$elabInitScript = "/proj/tbres/duerig/testbed/pelab/init-elabnodes.pl";
#$initConditionsCommand = $elabInitScript . " -o /tmp/initial-conditions.txt " . $newProjName . " " . $newExpName;
if($#ARGV == 3)
{
system($initConditionsCommand);
$initialConditionsFilename = "/tmp/initial-conditions.txt";
}
#if($#ARGV == 2)
#{
# system($initConditionsCommand);
# $initialConditionsFilename = "/tmp/initial-conditions.txt";
#}
open(CONDITIONS, $initialConditionsFilename);
......@@ -111,37 +111,60 @@ foreach $sourceName (readdir(logsDirHandle))
}
}
rewinddir(logsDirHandle);
# Descend into all the source directories
foreach $sourceName (readdir(logsDirHandle))
{
if( (-d $logsDir . "/" . $sourceName ) && $sourceName ne "." && $sourceName ne ".." )
if( (-d "$logsDir/$sourceName" ) && $sourceName ne "." && $sourceName ne ".." )
{
my @destLists;
# Then search for all possible destinations for
# a particular source.
opendir(sourceDirHandle, $logsDir . "/" . $sourceName);
opendir(sourceDirHandle, "$logsDir/$sourceName");
foreach $destOne (readdir(sourceDirHandle))
{
if( (-d $logsDir . "/" . $sourceName . "/" . $destOne) && $destOne ne "." && $destOne ne ".." )
if( (-d "$logsDir/$sourceName/$destOne") && $destOne ne "." && $destOne ne ".." )
{
# Inside each destination directory, look for
# all possible second destinations.
opendir(destDirHandle, $logsDir . "/" . $sourceName . "/" . $destOne);
opendir(destDirHandle, "$logsDir/$sourceName/$destOne");
foreach $destTwo (readdir(destDirHandle))
{
if( (-d $logsDir . "/" . $sourceName . "/" . $destOne . "/" . $destTwo) && $destTwo ne "." && $destTwo ne ".." )
if( (-d "$logsDir/$sourceName/$destOne/$destTwo") && $destTwo ne "." && $destTwo ne ".." )
{
$fullPath = "$logsDir/$sourceName/$destOne/$destTwo";
# Run Rubenstein's code on the ".filter" files
# inside the second destination directory.
`perl /proj/tbres/duerig/testbed/pelab/bw-bottleneck/dump2filter.pl $fullPath`;
$DansScript = "/proj/tbres/duerig/filter/genjitco.FreeBSD";
if(-r "$fullPath/delay.log")
{
($path1Variance, $path2Variance) = &CheckVariance("$fullPath/delay.log");
# One of the two paths had low variance - declare this pair of paths as being
# uncorrelated.
if(not( $path1Variance >= 25 and $path2Variance >= 25) )
{
push(@destLists,$destOne);
push(@destLists,$destTwo);
printf("WARNING: One or more of paths($sourceName -> $destOne, $sourceName -> $destTwo) have low variance(%.3f,%.3f)\n.Considering these paths as uncorrelated\n", $path1Variance, $path2Variance);
next;
}
}
else
{
print "Missing file(delay.log). Cannot process $fullPath\n";
next;
}
system("perl delay2dump.pl $fullPath/delay.log $fullPath");
system("perl dump2filter.pl $fullPath");
$filterFile1 = $fullPath . "/" . "source.filter";
$filterFile2 = $fullPath . "/" . "dest1.filter";
$filterFile3 = $fullPath . "/" . "dest2.filter";
......@@ -149,14 +172,14 @@ foreach $sourceName (readdir(logsDirHandle))
if (!(-r $filterFile1) || !(-r $filterFile2) || !(-r $filterFile3))
{
print "Missing file. Cannot process $fullPath\n";
continue;
next;
}
$sharedBottleneckCheck = $DansScript ." ". $filterFile1
." ". $filterFile2 ." ". $filterFile3;
$sharedBottleneckCheck = $DansScriptPath ." ". $filterFile1." ". $filterFile2 ." ". $filterFile3;
print "EXECUTE: $sharedBottleneckCheck\n";
my @scriptOutput = ();
@scriptOutput = `$sharedBottleneckCheck | tail -n 2`;
#system("$sharedBottleneckCheck");
# $scriptOutput[0] = "last CHANGE was CORRELATED, corr case: 30203 pkts, test case: 30203 pkts";
# $scriptOutput[1] = "testingabcdef";
......@@ -167,9 +190,9 @@ foreach $sourceName (readdir(logsDirHandle))
if($scriptOutput[$#scriptOutput - 1] =~ /last CHANGE\s(\w*)\s(\w*)\,[\w\d\W\:\,]*/)
{
print "For source $sourceName: Comparing $destOne $destTwo: $scriptOutput[$#scriptOutput]";
$corrResult = $2;
$corrResult = $2;
if($corrResult eq "CORRELATED"
&& !($scriptOutput[$#scriptOutput] =~ /nan/))
&& !($scriptOutput[$#scriptOutput] =~ /nan/))
{
push(@{ $bottleNecks{$sourceName} },$destOne . ":" . $destTwo);
push(@destLists,$destOne);
......@@ -178,7 +201,7 @@ foreach $sourceName (readdir(logsDirHandle))
}
elsif($corrResult eq "UNCORRELATED"
&& !($scriptOutput[$#scriptOutput] =~ /nan/))
&& !($scriptOutput[$#scriptOutput] =~ /nan/))
{
push(@destLists,$destOne);
push(@destLists,$destTwo);
......@@ -357,10 +380,10 @@ foreach $sourceName (readdir(logsDirHandle))
{
if( ($deletionFlagArray[$counter1] != 1) and ($deletionFlagArray[$counter2] != 1) and ($counter1 != $counter2) )
{
$intersectionFlag = &checkIntersection($counter1, $counter2);
$intersectionFlag = &CheckIntersection($counter1, $counter2);
if(