Commit 3b2da725 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

More absurd changes to prevent iptables racing.

parent 9f30226c
......@@ -30,7 +30,7 @@ use Exporter;
findControlNet existsIface findIface findMac
existsBridge findBridge findBridgeIfaces
downloadImage getKernelVersion createExtraFS
forwardPort removePortForward lvSize
forwardPort removePortForward lvSize DoIPtables
);
use Data::Dumper;
......@@ -96,26 +96,45 @@ sub forwardPort($;$) {
# Are we removing or adding the rule?
my $op = (defined($remove) && $remove) ? "D" : "A";
#
# Oh jeez, iptables is about the dumbest POS I've ever seen;
# it fails if you run two at the same time. So we have to
# serialize the calls.
#
return -1
if (DoIPtables("-v -t nat -$op PREROUTING -p $protocol -d $ext_ip ".
"--dport $ext_port -j DNAT ".
"--to-destination $int_ip:$int_port"));
return 0;
}
#
# Oh jeez, iptables is about the dumbest POS I've ever seen; it fails
# if you run two at the same time. So we have to serialize the calls.
# The problem is that XEN also manipulates things, and so it is hard
# to get a perfect lock. So, we do our best and if it fails sleep for
# a couple of seconds and try again.
#
sub DoIPtables($)
{
my ($cmd) = @_;
if (TBScriptLock("iptables", 0, 900) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the iptables lock after a long time!\n";
return -1;
}
mysystem2("$IPTABLES -v -t nat -$op PREROUTING -p $protocol -d $ext_ip ".
"--dport $ext_port -j DNAT ".
"--to-destination $int_ip:$int_port");
TBScriptUnlock();
if ($? == 0) {
return 0;
my $retries = 5;
my $status = 0;
while ($retries > 0) {
mysystem2("$IPTABLES $cmd");
$status = $?;
last
if (!$status || $status >> 8 != 4);
print STDERR "will retry in a couple of seconds ...\n";
sleep(2);
$retries--;
}
# Operation failed after multiple retries - return error
print STDERR "WARNING: forwardPort: Failed to manipulate NAT!\n";
return -1;
TBScriptUnlock();
# Operation failed - return error
return -1
if (!$retries || $status);
return 0;
}
sub removePortForward($) {
......
......@@ -57,6 +57,7 @@ use libtmcc;
use libutil;
use libtestbed;
use libgenvnode;
use libvnode;
#
# Configure.
......@@ -130,21 +131,18 @@ chomp($outer_controlif);
sub Online()
{
# Prevent dhcp requests from leaving the physical host.
mysystem2("$IPTABLES -A FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
return -1
if ($?);
DoIPtables("-A FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP")
== 0 or return -1;
#
# We ask vif-bridge to turn on antispoofing; this rule would negate that.
#
if (0) {
mysystem2("$IPTABLES -A FORWARD -m physdev ".
"--physdev-in $vif -j ACCEPT");
return -1
if ($?);
DoIPtables("-A FORWARD -m physdev --physdev-in $vif -j ACCEPT")
== 0 or return -1;
}
# Start a tmcc proxy (handles both TCP and UDP)
......@@ -164,24 +162,21 @@ sub Online()
}
# Reroute tmcd calls to the proxy on the physical host
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
return -1
if ($?);
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
return -1
if ($?);
DoIPtables("-t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port")
== 0 or return -1;
DoIPtables("-t nat -A PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port")
== 0 or return -1;
# Reroute evproxy to use the local daemon.
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
return -1
if ($?);
DoIPtables("-t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT")
== 0 or return -1;
#
# GROSS! source-nat all traffic destined the fs node, to come from the
......@@ -196,11 +191,10 @@ sub Online()
# filesystems to the guest IPs if the guest is on a shared host.
#
if (!SHAREDHOST()) {
mysystem2("$IPTABLES -t nat -A POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge");
return -1
if ($?);
DoIPtables("-t nat -A POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge")
== 0 or return -1;
}
#
......@@ -212,19 +206,17 @@ sub Online()
# rely on the SNAT rule below.
#
if (!REMOTEDED()) {
mysystem2("$IPTABLES -t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask");
return -1
if ($?);
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask")
== 0 or return -1;
#
# Boss/ops/fs specific rules in case the control network is
# segmented like it is in Utah.
#
mysystem2("$IPTABLES -t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip");
return -1
if ($?);
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip")
== 0 or return -1;
}
#
......@@ -232,10 +224,9 @@ sub Online()
# jail network in on our node, and all of them are bridged
# togther anyway.
#
mysystem2("$IPTABLES -t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask");
return -1
if ($?);
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask")
== 0 or return -1;
#
# Otherwise, setup NAT so that traffic leaving the vnode on its
......@@ -243,8 +234,10 @@ sub Online()
# control net iface, is NAT'd to the phys host's control
# net IP, using SNAT.
#
mysystem2("$IPTABLES -t nat -A POSTROUTING ".
" -s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
DoIPtables("-t nat -A POSTROUTING ".
"-s $vnode_ip -o $outer_controlif ".
"-j SNAT --to-source $host_ip")
== 0 or return -1;
return 0;
}
......@@ -252,25 +245,25 @@ sub Online()
sub Offline()
{
# dhcp
mysystem2("$IPTABLES -D FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
DoIPtables("-D FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
# See above.
if (0) {
mysystem2("$IPTABLES -D FORWARD -m physdev ".
"--physdev-in $vif -j ACCEPT");
DoIPtables("-D FORWARD -m physdev ".
"--physdev-in $vif -j ACCEPT");
}
# tmcc
# Reroute tmcd calls to the proxy on the physical host
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
DoIPtables("-t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
DoIPtables("-t nat -D PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
if (-e "/var/run/tmccproxy-$vnode_id.pid") {
my $pid = `cat /var/run/tmccproxy-$vnode_id.pid`;
......@@ -279,29 +272,29 @@ sub Offline()
}
if (!SHAREDHOST()) {
mysystem2("$IPTABLES -t nat -D POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge");
DoIPtables("-t nat -D POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge");
}
mysystem2("$IPTABLES -t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask");
if (!REMOTEDED()) {
mysystem2("$IPTABLES -t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask");
mysystem2("$IPTABLES -t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip");
}
mysystem2("$IPTABLES -t nat -D POSTROUTING ".
" -s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
DoIPtables("-t nat -D POSTROUTING ".
"-s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
# evproxy
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
DoIPtables("-t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
return 0;
}
......@@ -327,6 +320,8 @@ if (@ARGV) {
TBScriptUnlock();
exit(1);
}
TBScriptUnlock();
my $rval = 0;
my $op = shift(@ARGV);
if ($op eq "online") {
......@@ -335,7 +330,6 @@ if (@ARGV) {
elsif ($op eq "offline") {
$rval = Offline();
}
TBScriptUnlock();
exit($rval);
}
exit(0);
......@@ -56,19 +56,7 @@ print STDERR "@ARGV\n";
print STDERR $ENV{"vif"} . "\n";
my $script = shift(@ARGV);
#
# Oh jeez, iptables is about the dumbest POS I've ever seen;
# it fails if you run two at the same time. So we have to
# serialize the calls. Rather then worry about each call, just
# take a big lock here since there are going to be iptables
# calls out of vif-bridge.
#
if (TBScriptLock("iptables", 0, 300) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the iptables lock after a long time!\n";
exit(-1);
}
system("/bin/sh $script @ARGV");
my $ecode = $? >> 8;
TBScriptUnlock();
exit($ecode);
......@@ -1567,16 +1567,16 @@ sub vnodeBoot($$$$)
libutil::setState("BOOTING");
# and finally, create the VM
mysystem2("$XM create $config");
my $status = RunWithLock("xmtool", "$XM create $config");
# We have a problem with intermittent failures.
if ($?) {
if ($status) {
print "Guest failure: retrying after a short wait.\n";
sleep(20);
mysystem2("$XM create $config");
$status = RunWithLock("xmtool", "$XM create $config");
}
if ($?) {
print STDERR "$XM create failed: $?\n";
if ($status) {
print STDERR "$XM create failed: $status\n";
return -1;
}
print "Created virtual machine $vnode_id\n";
......@@ -1595,7 +1595,8 @@ sub vnodeReboot($$$$)
if ($vmid =~ m/(.*)/){
$vmid = $1;
}
mysystem("$XM reboot $vmid");
my $status = RunWithLock("xmtool", "$XM reboot $vmid");
return $status >> 8;
}
sub vnodeTearDown($$$$)
......@@ -1654,7 +1655,7 @@ sub vnodeDestroy($$$$)
$vnode_id = $1;
}
if (domainExists($vnode_id)) {
mysystem("$XM destroy $vnode_id");
RunWithLock("xmtool", "$XM destroy $vnode_id");
# XXX hang out awhile waiting for domain to disappear
domainGone($vnode_id, 15);
}
......@@ -1735,7 +1736,9 @@ sub vnodeHalt($$$$)
#
if ($stat) {
print STDERR "$XM shutdown returned $stat. Doing a destroy!\n";
mysystem("$XM destroy $vnode_id");
my $status = RunWithLock("xmtool", "$XM destroy $vnode_id");
fatal("Could not destroy $vnode_id")
if ($status);
}
}
else {
......@@ -1744,8 +1747,8 @@ sub vnodeHalt($$$$)
# Temporarily unblock and set to default so we die.
#
local $SIG{TERM} = 'DEFAULT';
exec("$XM shutdown -w $vnode_id");
exit(1);
my $status = RunWithLock("xmtool", "$XM shutdown -w $vnode_id");
exit($status >> 8);
}
return 0;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment