Skip to content

GitLab

Commit 29e2a29e authored by Pramod R Sanaga's avatar Pramod R Sanaga
Browse files

No major changes - syncing my dev version.

parent 71a66f24
Hide whitespace changes
Inline Side-by-side
pelab/bw-bottleneck/createClusters.pl
View file @ 29e2a29e
......@@ -11,6 +11,7 @@ local $MinSikKimScriptPath = "/proj/tbres/pramod/bottleneck-emulab/devel/MinSikK
local $ClusterProgramPath = "/proj/tbres/pramod/bottleneck-emulab/devel/Clustering/c++-samples/wvcluster";
local $elabInitScript = "/proj/tbres/duerig/testbed/pelab/init-elabnodes.pl";
local $newExpName = "";
local $newProjName = "";
local $logsDir = "";
......@@ -133,6 +134,7 @@ foreach $conditionLine (@initialConditions)
$numNodes += 1;
$elabMap{$srcAddress} = "elabc-elab-" . $2;
print "Mapping $srcAddress to $elabMap{$srcAddress}\n";
}
# Create a mapping of the initial conditions.
......@@ -158,6 +160,22 @@ if($numNodes != ($#nodeNameList+1) )
exit(1);
}
foreach $tmpDirName (@nodeNameList)
{
# Create empty directories if some of the log files were missing.
if(not (-d "$logsDir/$tmpDirName"))
{
print "@ WARNING: Creating dummy directory $logsDir/$tmpDirName and its sub-directories.\n";
foreach $tmpDirName2 (@nodeNameList)
{
if($tmpDirName2 ne $tmpDirName)
{
system("mkdir -p $logsDir/$tmpDirName/$tmpDirName2");
}
}
}
}
# Convert the delay data produced by the wavelet code
# into something palatable to the Rubenstein program.
if($Algorithm =~ /rubenstein/i)
......@@ -167,7 +185,7 @@ if($Algorithm =~ /rubenstein/i)
`rm -rf $scratchSpaceDir`;
}
`mkdir $scratchSpaceDir`;
`mkdir -p $scratchSpaceDir`;
opendir(logsDirHandle, $logsDir);
foreach $sourceName (readdir(logsDirHandle))
......@@ -191,6 +209,7 @@ if($Algorithm =~ /rubenstein/i)
}
else
{
push(@dirNameList, $destOne);
print "WARNING: Missing log file $logsDir/$sourceName/$destOne/delay.log\n";
}
}
......@@ -208,7 +227,10 @@ if($Algorithm =~ /rubenstein/i)
if($secondLevelIndex > $firstLevelIndex)
{
`mkdir -p $scratchSpaceDir/$sourceName/$destOne/$destTwo`;
`paste -d ' ' $logsDir/$sourceName/$destOne/delay.log $logsDir/$sourceName/$destTwo/delay.log > $scratchSpaceDir/$sourceName/$destOne/$destTwo/delay.log `;
if((-e "$logsDir/$sourceName/$destOne/delay.log") and (-e "$logsDir/$sourceName/$destTwo/delay.log"))
{
`paste -d ' ' $logsDir/$sourceName/$destOne/delay.log $logsDir/$sourceName/$destTwo/delay.log > $scratchSpaceDir/$sourceName/$destOne/$destTwo/delay.log `;
}
}
$secondLevelIndex++;
......@@ -241,8 +263,8 @@ my %nodeNameMap = {};
# Create & send events.
my $tevc = "/usr/testbed/bin/tevc -e $newProjName/$newExpName now";
`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
`$tevc elabc create start`;
#`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
#`$tevc elabc create start`;
opendir(logsDirHandle, $logsDir);
......@@ -261,6 +283,7 @@ foreach $sourceName (readdir(logsDirHandle))
$addrNodeMapping{$sourceName} = $addressList[$addrIndex];
$nodeNameMapping{$sourceName} = $addrIndex + 1;
$nodeNumberMapping{$addrIndex + 1} = $sourceName;
print "Mapping $sourceName to $addressList[$addrIndex]\n";
$addrIndex++;
$numDests++;
}
......@@ -271,7 +294,7 @@ rewinddir(logsDirHandle);
# Descend into all the source directories
foreach $sourceName (readdir(logsDirHandle))
{
if( (-d $logsDir . "/" . $sourceName ) && $sourceName ne "." && $sourceName ne ".." )
if( (-d "$logsDir/$sourceName" ) && $sourceName ne "." && $sourceName ne ".." )
{
my @destLists;
......@@ -281,30 +304,49 @@ foreach $sourceName (readdir(logsDirHandle))
my @denoisedDelays = ();
for($i = 0; $i < $numDests; $i++)
{
push(@denoisedDelays, 0);
}
foreach $destOne (readdir(sourceDirHandle))
{
if( (-d $logsDir . "/" . $sourceName . "/" . $destOne) && $destOne ne "." && $destOne ne ".." )
if( (-d "$logsDir/$sourceName/$destOne") && $destOne ne "." && $destOne ne ".." )
{
# Inside each destination directory, look for
# delay.log file with delay values.
$fullPath = "$logsDir/$sourceName/$destOne/delay.log";
my @scriptOutput = ();
if(not(-r $fullPath))
{
$denoisedDelays[$nodeNameMapping{$destOne}] = [ @scriptOutput ];
next;
}
$waveletScript = "$MinSikKimScriptPath $fullPath";
my @scriptOutput;
@scriptOutput = `$waveletScript`;
$denoisedDelays[$nodeNameMapping{$destOne}] = [ @scriptOutput ];
# Indicates low variance of the delay samples.
if((&CheckVariance($fullPath) == 1))
{
$denoisedDelays[$nodeNameMapping{$destOne}] = [ @scriptOutput ];
}
else
{
@scriptOutput = `$waveletScript`;
$denoisedDelays[$nodeNameMapping{$destOne}] = [ @scriptOutput ];
}
}
}
closedir(sourceDirHandle);
my $numOfSamples = 128;
local @equivClasses = ();
$tmpTreeRecordFile = "/tmp/bw-wavelet-clustering.rcd";
open(RECORDFILE, ">$tmpTreeRecordFile");
print RECORDFILE "128\n";
print RECORDFILE "$numOfSamples\n";
for($i = 1; $i <= $numDests; $i++)
{
if($i == $nodeNameMapping{$sourceName})
......@@ -318,13 +360,13 @@ foreach $sourceName (readdir(logsDirHandle))
# Put this destination in a seperate cluster - we
# have zero samples/delay values.
if( ($#scriptOutput == 0) or ($#scriptOutput < 128) )
if( ($#scriptOutput == 0) or ($#scriptOutput < $numOfSamples) )
{
my @newEquivClass = ();
push(@newEquivClass, $i);
push(@equivClasses, [@newEquivClass]);
print "$sourceName: WARNING: Creating a new equiv class/cluster for $nodeNumberMapping{$i} due to lack of samples($#scriptOutput)\n";
print "@@ $sourceName: WARNING: Creating a new equiv class/cluster for $nodeNumberMapping{$i} due to lack of samples($#scriptOutput) or low variance\n";
}
else
{
......@@ -335,7 +377,7 @@ foreach $sourceName (readdir(logsDirHandle))
{
chomp($delayValue);
if($counter < 128)
if($counter < $numOfSamples)
{
$avgValue += $delayValue;
push(@delayValueArray, $delayValue);
......@@ -346,30 +388,33 @@ foreach $sourceName (readdir(logsDirHandle))
last;
}
}
$avgValue = $avgValue/128.0;
#############################################
#$avgValue = $avgValue/$numOfSamples;
my $denominator = 0;
foreach $delayValue (@delayValueArray)
{
$denominator += ($delayValue - $avgValue)*($delayValue - $avgValue);
}
$denominator = sqrt($denominator);
#my $denominator = 0;
#foreach $delayValue (@delayValueArray)
#{
# $denominator += ($delayValue - $avgValue)*($delayValue - $avgValue);
#}
#$denominator = sqrt($denominator);
# Exclude paths with low-variance.
if($denominator < 25)
{
my @newEquivClass = ();
push(@newEquivClass, $i);
push(@equivClasses, [@newEquivClass]);
print "$sourceName: WARNING: Creating a new equiv class/cluster for $nodeNumberMapping{$i} due to low variance of samples($denominator)\n";
next;
}
#if($denominator < 25)
#{
# my @newEquivClass = ();
# push(@newEquivClass, $i);
#
# push(@equivClasses, [@newEquivClass]);
# print "$sourceName: WARNING: Creating a new equiv class/cluster for $nodeNumberMapping{$i} due to low variance of samples($denominator)\n";
# next;
# }
#############################################
foreach $delayValue (@delayValueArray)
{
$delayValue = ($delayValue - $avgValue)/$denominator;
print RECORDFILE "$delayValue:";
#$delayValue = ($delayValue - $avgValue)/$denominator;
#print RECORDFILE "$delayValue:";
printf (RECORDFILE "%.4f:",$delayValue);
}
print RECORDFILE "$i\n";
......@@ -378,9 +423,10 @@ foreach $sourceName (readdir(logsDirHandle))
}
close(RECORDFILE);
$clusteringProgram = "$ClusterProgramPath $tmpTreeRecordFile /tmp/tmp.idx";
my @clusteringOutput ;
my @clusteringOutput = ();
@clusteringOutput = `$clusteringProgram`;
......@@ -400,17 +446,18 @@ foreach $sourceName (readdir(logsDirHandle))
push(@equivClasses, [@newEquivClass]);
}
print "Clusters for $sourceName:\n";
print "@@@ Clusters for $sourceName:\n";
foreach $tmpName (@equivClasses)
{
print "@@@ ";
foreach $tmpName2 (@$tmpName)
{
print "$nodeNumberMapping{$tmpName2} ";
}
print "\n";
print " \n";
}
print "\n";
print "@@@ \n";
# Send the events to all the nodes which form an equivalent class.
foreach $tmpName (@equivClasses)
......@@ -447,12 +494,12 @@ foreach $sourceName (readdir(logsDirHandle))
$delayEventCommand = $delayEventCommand . " " . "DELAY=" . ($delayMap{$addrNodeMapping{$sourceName}}{$addrNodeMapping{$nodeNumberMapping{$tmpName2}}});
# Execute the delay event command.
print "EXECUTE $delayEventCommand\n";
`$delayEventCommand`;
#`$delayEventCommand`;
}
$bwEventCommand = $bwEventCommand . " " . "BANDWIDTH=" . $maxBw;
# Execute the event to set the bandwidth for this equivalence class.
print "EXECUTE $bwEventCommand\n";
`$bwEventCommand`;
#`$bwEventCommand`;
}
print "\n\n";
......@@ -460,3 +507,51 @@ foreach $sourceName (readdir(logsDirHandle))
}
closedir(logsDirHandle);
sub CheckVariance()
{
open(FILEHANDLE, $_[0]);
my @delayValueArray = ();
while(<FILEHANDLE>)
{
if(/(\-?[0-9]*) ([0-9]*)/)
{
push(@delayValueArray, $1);
}
}
close(FILEHANDLE);
chomp($delayValueArray);
my $avgValue = 0;
if($#delayValueArray == -1)
{
return 1;
}
foreach $delayValue (@delayValueArray)
{
$avgValue += $delayValue;
}
$avgValue = $avgValue/($#delayValueArray+1);
my $variance = 0;
foreach $delayValue (@delayValueArray)
{
$variance += ($delayValue - $avgValue)*($delayValue - $avgValue);
}
$variance = sqrt($variance);
# Exclude paths with low-variance.
if($variance < 25)
{
print "@@ WARNING: Low Variance($variance) for $fullPath\n";
return 1;
}
else
{
return 0;
}
}
pelab/bw-bottleneck/sharedBottle.pl
View file @ 29e2a29e
......@@ -62,8 +62,8 @@ my %elabMap = {};
# from the database, using init-elabnodes.pl
my $tevc = "/usr/testbed/bin/tevc -e $newProjName/$newExpName now";
`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
`$tevc elabc create start`;
#`/usr/testbed/bin/tevc -w -e $newProjName/$newExpName now elabc reset`;
#`$tevc elabc create start`;
# Create a list of the IP addresses.
foreach $conditionLine (@initialConditions)
......@@ -153,13 +153,13 @@ foreach $sourceName (readdir(logsDirHandle))
push(@destLists,$destOne);
push(@destLists,$destTwo);
printf("WARNING: One or more of paths($sourceName -> $destOne, $sourceName -> $destTwo) have low variance(%.3f,%.3f)\n.Considering these paths as uncorrelated\n", $path1Variance, $path2Variance);
printf("@@ WARNING: One or more of paths($sourceName -> $destOne, $sourceName -> $destTwo) have low variance(%.3f,%.3f)\n.Considering these paths as uncorrelated\n", $path1Variance, $path2Variance);
next;
}
}
else
{
print "Missing file(delay.log). Cannot process $fullPath\n";
print "@@ Missing file(delay.log). Cannot process $fullPath\n";
next;
}
system("perl delay2dump.pl $fullPath/delay.log $fullPath");
......@@ -171,12 +171,12 @@ foreach $sourceName (readdir(logsDirHandle))
if (!(-r $filterFile1) || !(-r $filterFile2) || !(-r $filterFile3))
{
print "Missing file. Cannot process $fullPath\n";
print "@@ WARNING: Missing file. Cannot process $fullPath\n";
next;
}
$sharedBottleneckCheck = $DansScriptPath ." ". $filterFile1." ". $filterFile2 ." ". $filterFile3;
print "EXECUTE: $sharedBottleneckCheck\n";
print "@ EXECUTE: $sharedBottleneckCheck\n";
my @scriptOutput = ();
@scriptOutput = `$sharedBottleneckCheck | tail -n 2`;
#system("$sharedBottleneckCheck");
......@@ -189,7 +189,7 @@ foreach $sourceName (readdir(logsDirHandle))
if($scriptOutput[$#scriptOutput - 1] =~ /last CHANGE\s(\w*)\s(\w*)\,[\w\d\W\:\,]*/)
{
print "For source $sourceName: Comparing $destOne $destTwo: $scriptOutput[$#scriptOutput]";
print "@ For source $sourceName: Comparing $destOne $destTwo: $scriptOutput[$#scriptOutput]";
$corrResult = $2;
if($corrResult eq "CORRELATED"
&& !($scriptOutput[$#scriptOutput] =~ /nan/))
......@@ -197,7 +197,7 @@ foreach $sourceName (readdir(logsDirHandle))
push(@{ $bottleNecks{$sourceName} },$destOne . ":" . $destTwo);
push(@destLists,$destOne);
push(@destLists,$destTwo);
print "CORRELATED\n\n";
print "@ CORRELATED\n\n";
}
elsif($corrResult eq "UNCORRELATED"
......@@ -205,19 +205,19 @@ foreach $sourceName (readdir(logsDirHandle))
{
push(@destLists,$destOne);
push(@destLists,$destTwo);
print "UNCORRELATED\n\n";
print "@ UNCORRELATED\n\n";
}
else
{
print "ERROR:($sourceName) Something went wrong in pattern matching phase.\n";
print "Skipping this source node\n";
print "ERROR: Skipping this path from the source node\n";
next;
}
}
else
{
print "ERROR:($sourceName) Output line from Rubenstein's code did not match regular expression.\n";
print "Skipping this source node\n";
print "ERROR: Skipping this path from the source node\n";
next;
}
}
......@@ -331,7 +331,7 @@ foreach $sourceName (readdir(logsDirHandle))
if($firstDestClassNum != $secondDestClassNum)
{
print "ERROR($sourceName): Two destinations sharing a bottleneck link are in different equiv. classes $i $j\n";
print "Skipping this source node\n";
print "ERROR: Skipping this source node\n";
}
}
......@@ -415,16 +415,18 @@ foreach $sourceName (readdir(logsDirHandle))
if($retVal != 0)
{
print "WARNING:($sourceName) Transitive property has been violated.\n";
print "@@ WARNING:($sourceName) Transitive property has been violated.\n";
}
print "(Debugging) Equivalence classes - $sourceName\n";
print "@@@ (Debugging) Equivalence classes - $sourceName\n";
#######################
foreach $tmpName (@equivClasses)
{
print "@@@ ";
foreach $tmpName2 (@$tmpName)
{
print $destSeen[$tmpName2] . " ";
}
print "\n";
print " \n";
}
......@@ -442,9 +444,11 @@ for($i = 0; $i < $numDests; $i++)
if($lonerFlag == 0)
{
print $destSeen[$i] . "\n";
print "@@@ $destSeen[$i] \n";
}
}
print "@@@ \n";
#######################
$nodeClasses{$sourceName} = [ @equivClasses ];
......@@ -485,12 +489,12 @@ foreach $tmpName (@equivClasses)
$delayEventCommand = $delayEventCommand . " " . "DELAY=" . ($delayMap{$addrNodeMapping{$sourceName}}{$addrNodeMapping{$destSeen[$tmpName2]}});
# Execute the delay event command.
print "EXECUTE $delayEventCommand\n";
`$delayEventCommand`;
#`$delayEventCommand`;
}
$bwEventCommand = $bwEventCommand . " " . "BANDWIDTH=" . $maxBw;
# Execute the event to set the bandwidth for this equivalence class.
print "EXECUTE $bwEventCommand\n";
`$bwEventCommand`;
#`$bwEventCommand`;
}
# Create and send events for all the loner dest nodes reachable from this source node.
......@@ -514,7 +518,7 @@ foreach $tmpName (@equivClasses)
# Execute the event to set the bandwidth for this path.
print "EXECUTE: $bwEventCommand\n";
`$bwEventCommand`;
#`$bwEventCommand`;
my $delayEventCommand = "$tevc ".$elabMap{$addrNodeMapping{$destSeen[$i]}}." modify DEST=" . $addrNodeMapping{$destSeen[$i]}." SRC=".$addrNodeMapping{$sourceName};
......@@ -522,7 +526,7 @@ foreach $tmpName (@equivClasses)
# Execute the delay event command.
print "EXECUTE: $delayEventCommand\n";
`$delayEventCommand`;
#`$delayEventCommand`;
}
}
......@@ -549,7 +553,10 @@ sub CheckSanity()
{
if($adjMatrix[$classElement][$secondIter] != 1)
{
print "WARNING: $destSeen[$classElement] $destSeen[$secondIter] violates transitive property\n";
if($classElement < $secondIter)
{
print "@@ WARNING: $destSeen[$classElement] $destSeen[$secondIter] violates transitive property\n";
}
$adjMatrix[$classElement][$secondIter] = 1;
$retVal = -1;
}
......@@ -612,7 +619,7 @@ sub CheckVariance()
while($line = <HANDLE>)
{
if($line =~ /^(\d*)\s(\d*)\s(\d*)\s(\d*)/)
if($line =~ /^(\-?\d*)\s(\d*)\s(\-?\d*)\s(\d*)/)
{
push(@delayArray1, $1);
push(@delayArray2, $3);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment