Commit 11b0f767 authored by Leigh Stoller's avatar Leigh Stoller

Add BW shaping outside the container to ensure that container does not

use more then it's allotted amount. We do this for both ingress and
egress. To handle ingress, I had to add another veth pair and bridge
between the physical device, but in retrospect, I could have done this
with an IFB on the existing root context side of the existing pair.
I will probably redo this, but wanted to get this working code into
the repo before I destroy it.
parent 09b67305
......@@ -156,6 +156,8 @@ my $USE_NETEM = 0;
my $USE_MACVLAN = 0;
# Use control network bridging for all containers.
my $USE_CTRLBR = 1;
# Use extra bridging for BW limiting. Always on shared hosts.
my $USE_TCBRS = 1;
# Switch to openvswitch
my $USE_OPENVSWITCH = 0;
......@@ -177,6 +179,9 @@ if ($kmaj >= 2 && $kmin >= 6 && $kpatch >= 32) {
$USE_MACVLAN = 1;
}
}
if (SHAREDHOST()) {
$USE_TCBRS = 1;
}
#
# Helpers.
......@@ -196,7 +201,6 @@ sub vmstatus($);
sub vmrunning($);
sub vmstopped($);
sub GClvm($);
sub GCbridge($);
sub SetupCaps($$$$);
#
......@@ -212,9 +216,13 @@ sub addbr($)
sub delbr($)
{
my $br = $_[0];
my $cmd = ($USE_OPENVSWITCH ? "$OVSCTL del-br" : "$BRCTL delbr") . " $br";
system($cmd);
if ($USE_OPENVSWITCH) {
mysystem2("$OVSCTL del-br $br");
}
else {
mysystem2("$IFCONFIG $br down");
mysystem2("$BRCTL delbr $br");
}
}
sub addbrif($$)
{
......@@ -518,7 +526,7 @@ sub vz_rootPreConfig {
}
for (my $i = 0; $i < $MAXIMQ; $i++) {
$MDB{"$i"} = ""
if (!exists($MDB{"$i"}));
if (!defined($MDB{"$i"}));
}
dbmclose(%MDB);
......@@ -623,7 +631,7 @@ sub vz_rootPreConfigNetwork {
mysystem2("$ETHTOOL -K $vdev tso off gso off");
$physdev = $vdev;
$brname = "${prefix}$vdev";
$brname = "${prefix}${vdev}";
$brs{$brname}{ENCAP} = 1;
$brs{$brname}{SHORT} = 0;
$brs{$brname}{PHYSDEV} = $vdev;
......@@ -660,27 +668,25 @@ sub vz_rootPreConfigNetwork {
# underlying physdev to "host" the macvlan.
#
foreach my $k (keys(%brs)) {
# postpass to setup SHORT if only two members and no PMAC
# NOTE: We have never used this code. What is it?
if (exists($brs{$k}{MEMBERS})) {
if ($brs{$k}{MEMBERS} == 2) {
$brs{$k}{SHORT} = 1;
}
else {
$brs{$k}{SHORT} = 0;
}
$brs{$k}{MEMBERS} = undef;
}
if (!$USE_MACVLAN) {
# building bridges is an important activity
#
# This bridge might be shared with other containers, so difficult
# to delete. This really only matters on shared nodes though, where
# bridges and vlans could stack up forever (or just a long time).
# The good news is that because we create a traffic shaping bridge
# (see below) on shared nodes, all modifications to the bridge
# are under lock, so each time a container exits, we can check to
# see if there anyone else is using it. So, just record that we
# are using this bridge. See additional note below.
#
if (! -d "/sys/class/net/$k/bridge") {
addbr($k);
goto bad
if ($?);
# record bridge created.
$private->{'bridges'}->{$k} = $k;
}
# record bridge used
$private->{'physbridges'}->{$k} = $k;
# repetitions of this should not hurt anything
mysystem2("$IFCONFIG $k 0 up");
}
......@@ -689,25 +695,33 @@ sub vz_rootPreConfigNetwork {
if (!$USE_MACVLAN) {
my $physdev = $brs{$k}{PHYSDEV};
my $ifc = $brs{$k}{IFC};
# make sure this iface isn't already part of another bridge;
# if it it is, remove it from there first and add to
# this bridge.
#
# This interface should not be a member of another bridge.
# If it is, it is an error.
#
# Continuing the comment above, this bridge and this interface
# might be shared with other containers, so we cannot remove it
# unless it is the only one left.
#
my $obr = findBridge($physdev);
if (defined($obr)) {
if (defined($obr) && $obr ne $k) {
# Avoid removing the device from the bridge if it
# is in the correct bridge.
delbrif($obr, $physdev);
goto bad
if ($?);
$obr = undef;
}
if (!defined($obr)) {
addbrif($k, $physdev);
goto bad
if ($?);
# rebuild hashes
makeBridgeMaps();
}
addbrif($k, $physdev);
goto bad
if ($?);
# record iface added to bridge
$private->{'bridgeifaces'}->{$k}->{$physdev} = $k;
if (SHAREDHOST()) {
if ($USE_TCBRS) {
#
# We do not put the virtual interface into the same
# bridge as the physical device. We create a second
......@@ -834,7 +848,7 @@ sub vz_rootPreConfigNetwork {
bad:
#
# Unwind anything we did.
# Unwind anything we did.
#
KillCaps($private);
......@@ -854,8 +868,6 @@ sub vz_rootPreConfigNetwork {
if (exists($private->{'bridges'})) {
foreach my $brname (keys(%{ $private->{'bridges'} })) {
mysystem2("$IFCONFIG $brname down");
# We can delete this cause we still have the lock and
# no one else got a chance to add to it.
delbr($brname);
delete($private->{'bridges'}->{$brname})
if (! $?);
......@@ -896,6 +908,18 @@ sub vz_rootPreConfigNetwork {
}
dbmclose(%MDB);
}
if ($USE_TCBRS && exists($private->{'physbridges'})) {
makeBridgeMaps();
foreach my $brname (keys(%{ $private->{'physbridges'} })) {
my @ifaces = findBridgeIfaces($brname);
if (@ifaces <= 1) {
delbr($brname);
delete($private->{'physbridges'}->{$brname})
if (! $?);
}
}
}
badbad:
TBScriptUnlock();
return -1;
......@@ -1237,23 +1261,14 @@ sub vz_vnodeTearDown {
if ($?);
delete($ref->{$brname}->{$iface});
}
# Delete bridge when no more members.
# Another container might have created the bridge, but
# it exited first. So we need to clean it up.
if (my $foo = GCbridge($brname)) {
goto badbad
if ($foo < 0);
delete($private->{'bridges'}->{$brname})
if (exists($private->{'bridges'}->{$brname}));
}
delete($private->{'bridgeifaces'}->{$brname});
}
}
# Delete bridges we created which have no current members.
if (exists($private->{'bridges'})) {
foreach my $brname (keys(%{ $private->{'bridges'} })) {
goto badbad
if (GCbridge($brname) < 0);
mysystem2("$IFCONFIG $brname down");
mysystem2("$BRCTL delbr $brname");
delete($private->{'bridges'}->{$brname});
}
}
......@@ -1293,6 +1308,18 @@ sub vz_vnodeTearDown {
mysystem2("$OVSCTL -- --if-exists del-port vzbr0 $cnet_veth");
delete($private->{'controlveth'});
}
if ($USE_TCBRS && exists($private->{'physbridges'})) {
makeBridgeMaps();
foreach my $brname (keys(%{ $private->{'physbridges'} })) {
my @ifaces = findBridgeIfaces($brname);
if (@ifaces <= 1) {
delbr($brname);
delete($private->{'physbridges'}->{$brname})
if (! $?);
}
}
}
#
# A word about these two. David sez that it is impossible to garbage
......@@ -1979,7 +2006,7 @@ sub vz_vnodePreConfigExpNetwork {
# gres and route tables are a global resource.
#
if (TBScriptLock($GLOBAL_CONF_LOCK, 0, 900) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the tunne lock after a long time!\n";
print STDERR "Could not get the global lock after a long time!\n";
return -1;
}
my %key2gre = ();
......@@ -2186,7 +2213,6 @@ sub vz_vnodePreConfigExpNetwork {
}
$private->{'iprules'}->{$veth} = $veth;
my $net = inet_ntoa(inet_aton($inetip) & inet_aton($mask));
mysystem2("/sbin/ip route replace ".
" default dev $gre table $routetable");
if ($?) {
......@@ -2561,54 +2587,6 @@ sub GClvm($)
return 0;
}
#
# See if we can delete a bridge.
#
sub GCbridge($)
{
my ($brname) = @_;
#
# See if we need to delete the bridge; once all the members
# are removed, we want to get rid of the bridge. But the
# bridge might be shared with other containers, to have to
# explicitly check (we have the lock).
#
if ($USE_OPENVSWITCH) {
my $foo = `$OVSCTL list-ports $brname | wc -l`;
if ($?) {
print STDERR "'$OVSCTL list-ports $brname' failed\n";
return -1;
}
else {
chomp($foo);
print STDERR "$foo ports in $brname\n";
if ("$foo" eq "0") {
mysystem2("$IFCONFIG $brname down");
mysystem2("$OVSCTL del-br $brname");
return 1;
}
}
}
else {
my $foo = `$BRCTL showmacs $brname | wc -l`;
if ($?) {
print STDERR "'brctl showmacs $brname' failed\n";
return -1;
}
else {
chomp($foo);
# Just a header line. Ick.
if ($foo <= 1) {
mysystem2("$IFCONFIG $brname down");
mysystem2("$BRCTL delbr $brname");
return 1;
}
}
}
return 0;
}
#
# Create a logical volume for the image if it doesn't already exist.
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment