Commit 296dd003 authored by Mike Hibler's avatar Mike Hibler
Browse files

Revert "Major overhaul to support thin snapshot volumes and also fixup locking."

This reverts commit a9e75f33.
parent e494cda5
#!/usr/bin/perl -wT
#
# Copyright (c) 2008-2015 University of Utah and the Flux Group.
# Copyright (c) 2008-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -252,12 +252,11 @@ sub fatal($)
#
# Run a command string, redirecting output to a logfile.
#
sub mysystem($;$)
sub mysystem($)
{
my ($command,$doecho) = @_;
$doecho = 1 if (!defined($doecho));
my ($command) = @_;
if ($doecho) {
if (1) {
print STDERR "mysystem: '$command'\n";
}
......@@ -266,12 +265,11 @@ sub mysystem($;$)
fatal("Command failed: $? - $command");
}
}
sub mysystem2($;$)
sub mysystem2($)
{
my ($command,$doecho) = @_;
$doecho = 1 if (!defined($doecho));
my ($command) = @_;
if ($doecho) {
if (1) {
print STDERR "mysystem: '$command'\n";
}
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2008-2015 University of Utah and the Flux Group.
# Copyright (c) 2008-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -30,8 +30,8 @@ use Exporter;
findControlNet existsIface findIface findMac
existsBridge findBridge findBridgeIfaces
downloadImage getKernelVersion createExtraFS
forwardPort removePortForward lvSize DoIPtables DoIPtablesNoFail
restartDHCP computeStripeSize
forwardPort removePortForward lvSize DoIPtables restartDHCP
computeStripeSize
);
use Data::Dumper;
......@@ -51,7 +51,6 @@ my $PCNET_GW_FILE = "/var/emulab/boot/routerip";
my $IPTABLES = "/sbin/iptables";
my $debug = 0;
my $lockdebug = 0;
sub setDebug($) {
$debug = shift;
......@@ -112,58 +111,34 @@ sub forwardPort($;$) {
# to get a perfect lock. So, we do our best and if it fails sleep for
# a couple of seconds and try again.
#
sub _DoIPtables($@)
sub DoIPtables(@)
{
my ($nofail, @rules) = @_;
my $rv = 0;
my (@rules) = @_;
TBDebugTimeStamp("DoIPtables: grabbing iptables lock")
if ($lockdebug);
if (TBScriptLock("iptables", 0, 900) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the iptables lock after a long time!\n";
return -1;
}
TBDebugTimeStamp(" got iptables lock")
if ($lockdebug);
foreach my $rule (@rules) {
my $retries = 10;
my $retries = 5;
my $status = 0;
while ($retries > 0) {
TBDebugTimeStamp(" doing 'iptables $rule'");
mysystem2("$IPTABLES $rule", 0);
mysystem2("$IPTABLES $rule");
$status = $?;
last
if (!$status || $status >> 8 != 4);
print STDERR "will retry in one second ...\n";
sleep(1);
print STDERR "will retry in a couple of seconds ...\n";
sleep(2);
$retries--;
}
# Operation failed - either return error or do the rest
# Operation failed - return error
if (!$retries || $status) {
if (!$nofail) {
TBDebugTimeStamp(" releasing iptables lock on error")
if ($lockdebug);
TBScriptUnlock();
return -1;
}
# we still return an error code
$rv = -1;
TBScriptUnlock();
return -1;
}
}
TBDebugTimeStamp(" releasing iptables lock")
if ($lockdebug);
TBScriptUnlock();
return $rv;
}
sub DoIPtables(@)
{
return _DoIPtables(0, @_);
}
sub DoIPtablesNoFail(@)
{
return _DoIPtables(1, @_);
return 0;
}
sub removePortForward($) {
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -26,7 +26,6 @@ use Getopt::Std;
use English;
use Data::Dumper;
use POSIX qw(setsid);
use POSIX ":sys_wait_h";
use Socket;
#
......@@ -60,8 +59,6 @@ use libtestbed;
use libgenvnode;
use libvnode;
my $lockdebug = 0;
#
# Configure.
#
......@@ -168,7 +165,6 @@ chomp($outer_controlif);
#
sub Online()
{
my @rules;
mysystem2("ifconfig $vif txqueuelen 256");
if ($VIFROUTING) {
......@@ -190,14 +186,12 @@ sub Online()
mysystem("$ARPING -c 4 -A -I $bridge $vnode_ip");
}
@rules = ();
# Prevent dhcp requests from leaving the physical host.
push(@rules,
"-A FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast ".
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
DoIPtables("-A FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP")
== 0 or return -1;
#
# We turn on antispoofing. In bridge mode, vif-bridge adds a rule
......@@ -211,11 +205,11 @@ sub Online()
# these chains just accept everything.
#
if ($VIFROUTING) {
push(@rules,
"-A FORWARD -i $vif -s $vnode_ip ".
"-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
push(@rules,
"-A FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}");
DoIPtables("-A FORWARD -i $vif -s $vnode_ip ".
" -m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}")
== 0 or return -1;
DoIPtables("-A FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}")
== 0 or return -1;
#
# Another wrinkle. We have to think about packets coming from
......@@ -224,9 +218,9 @@ sub Online()
# another chain. We make sure there are appropriate rules in
# the OUTGOING chain to protect the host.
#
push(@rules,
"-A INPUT -i $vif -s $vnode_ip ".
"-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
DoIPtables("-A INPUT -i $vif -s $vnode_ip ".
" -m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}")
== 0 or return -1;
#
# This rule effectively says that if the packet was not filtered
......@@ -234,8 +228,8 @@ sub Online()
# output to the container; we do not want it to go through the
# dom0 rules.
#
push(@rules,
"-A OUTPUT -o $vif -j ACCEPT");
DoIPtables("-A OUTPUT -o $vif -j ACCEPT")
== 0 or return -1;
}
else {
#
......@@ -244,13 +238,13 @@ sub Online()
# packets we want to filter. But we still have to allow the
# DHCP request packets through.
#
push(@rules,
"-I FORWARD -m physdev --physdev-is-bridged ".
"--physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}");
DoIPtables("-I FORWARD -m physdev --physdev-is-bridged ".
" --physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}")
== 0 or return -1;
push(@rules,
"-I FORWARD -m physdev --physdev-is-bridged ".
"--physdev-out $vif -j INCOMING_${vnode_id}");
DoIPtables("-I FORWARD -m physdev --physdev-is-bridged ".
" --physdev-out $vif -j INCOMING_${vnode_id}")
== 0 or return -1;
#
# Another wrinkle. We have to think about packets coming from
......@@ -263,33 +257,19 @@ sub Online()
# if the vnode_ip is unroutable, the packet appears to come from
# eth0, according to iptables logging. WTF!
#
push(@rules,
"-A INPUT -s $vnode_ip -j OUTGOING_${vnode_id}");
DoIPtables("-A INPUT -s $vnode_ip ".
" -j OUTGOING_${vnode_id}")
== 0 or return -1;
push(@rules,
"-A OUTPUT -d $vnode_ip -j ACCEPT");
DoIPtables("-A OUTPUT -d $vnode_ip -j ACCEPT")
== 0 or return -1;
}
# Apply the rules
DoIPtables(@rules) == 0 or
return -1;
# Start a tmcc proxy (handles both TCP and UDP)
my $tmccpid = fork();
if ($tmccpid) {
# Give child a chance to react.
sleep(1);
# Make sure it is alive.
if (waitpid($tmccpid, &WNOHANG) == $tmccpid) {
print STDERR "$vnode_id: tmcc proxy failed to start\n";
return -1;
}
if (open(FD, ">/var/run/tmccproxy-$vnode_id.pid")) {
print FD "$tmccpid\n";
close(FD);
}
mysystem2("echo $tmccpid > /var/run/tmccproxy-$vnode_id.pid");
}
else {
POSIX::setsid();
......@@ -338,24 +318,22 @@ sub Online()
}
}
@rules = ();
# Reroute tmcd calls to the proxy on the physical host
push(@rules,
"-t nat -A PREROUTING -j DNAT -p tcp ".
"--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
"--to-destination $host_ip:$local_tmcd_port");
DoIPtables("-t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port")
== 0 or return -1;
push(@rules,
"-t nat -A PREROUTING -j DNAT -p udp ".
"--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
"--to-destination $host_ip:$local_tmcd_port");
DoIPtables("-t nat -A PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port")
== 0 or return -1;
# Reroute evproxy to use the local daemon.
push(@rules,
"-t nat -A PREROUTING -j DNAT -p tcp ".
"--dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
"--to-destination $host_ip:$EVPROXY_PORT");
DoIPtables("-t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT")
== 0 or return -1;
#
# GROSS! source-nat all traffic destined the fs node, to come from the
......@@ -370,10 +348,10 @@ sub Online()
# filesystems to the guest IPs if the guest is on a shared host.
#
if (!SHAREDHOST()) {
push(@rules,
"-t nat -A POSTROUTING -j SNAT ".
"--to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
"-o $bridge");
DoIPtables("-t nat -A POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge")
== 0 or return -1;
}
#
......@@ -385,31 +363,31 @@ sub Online()
# rely on the SNAT rule below.
#
if (!REMOTEDED()) {
push(@rules,
"-t nat -A POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $network/$cnet_mask");
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask")
== 0 or return -1;
#
# Do not rewrite multicast (frisbee) traffic. Client throws up.
#
push(@rules,
"-t nat -A POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d 224.0.0.0/4");
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d 224.0.0.0/4")
== 0 or return -1;
#
# Ditto the apod packet.
#
push(@rules,
"-t nat -A POSTROUTING -j ACCEPT ".
"-s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6");
DoIPtables("-t nat -A POSTROUTING -j ACCEPT ".
" -s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6")
== 0 or return -1;
#
# Boss/ops/fs specific rules in case the control network is
# segmented like it is in Utah.
#
push(@rules,
"-t nat -A POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $boss_ip,$ops_ip");
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip")
== 0 or return -1;
}
#
......@@ -417,9 +395,9 @@ sub Online()
# jail network in on our node, and all of them are bridged
# togther anyway.
#
push(@rules,
"-t nat -A POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $jail_network/$jail_netmask");
DoIPtables("-t nat -A POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask")
== 0 or return -1;
#
# Otherwise, setup NAT so that traffic leaving the vnode on its
......@@ -427,73 +405,51 @@ sub Online()
# control net iface, is NAT'd to the phys host's control
# net IP, using SNAT.
#
push(@rules,
"-t nat -A POSTROUTING ".
"-s $vnode_ip -o $outer_controlif ".
"-j SNAT --to-source $host_ip");
DoIPtables("-t nat -A POSTROUTING ".
"-s $vnode_ip -o $outer_controlif ".
"-j SNAT --to-source $host_ip")
== 0 or return -1;
# Apply the rules
DoIPtables(@rules) == 0 or
return -1;
return 0;
}
sub Offline()
{
my @rules;
@rules = ();
# dhcp
push(@rules,
"-D FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast ".
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
DoIPtables("-D FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
# See above.
if ($VIFROUTING) {
push(@rules,
"-D FORWARD -i $vif -s $vnode_ip ".
"-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
push(@rules,
"-D FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}");
push(@rules,
"-D INPUT -i $vif -s $vnode_ip ".
"-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
push(@rules,
"-D OUTPUT -o $vif -j ACCEPT");
DoIPtables("-D FORWARD -i $vif -s $vnode_ip ".
" -m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
DoIPtables("-D FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}");
DoIPtables("-D INPUT -i $vif -s $vnode_ip ".
" -m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
DoIPtables("-D OUTPUT -o $vif -j ACCEPT");
}
else {
push(@rules,
"-D FORWARD -m physdev --physdev-is-bridged ".
"--physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}");
push(@rules,
"-D FORWARD -m physdev --physdev-is-bridged ".
"--physdev-out $vif -j INCOMING_${vnode_id}");
push(@rules,
"-D INPUT -s $vnode_ip -j OUTGOING_${vnode_id}");
push(@rules,
"-D OUTPUT -d $vnode_ip -j ACCEPT");
DoIPtables("-D FORWARD -m physdev --physdev-is-bridged ".
" --physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}");
DoIPtables("-D FORWARD -m physdev --physdev-is-bridged ".
" --physdev-out $vif -j INCOMING_${vnode_id}");
DoIPtables("-D INPUT -s $vnode_ip ".
" -j OUTGOING_${vnode_id}");
DoIPtables("-D OUTPUT -d $vnode_ip -j ACCEPT");
}
# tmcc
# Reroute tmcd calls to the proxy on the physical host
push(@rules,
"-t nat -D PREROUTING -j DNAT -p tcp ".
"--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
"--to-destination $host_ip:$local_tmcd_port");
push(@rules,
"-t nat -D PREROUTING -j DNAT -p udp ".
"--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
"--to-destination $host_ip:$local_tmcd_port");
# Apply the rules
if (DoIPtables(@rules) != 0) {
print STDERR "WARNING: could not remove iptables rules\n";
}
DoIPtables("-t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
DoIPtables("-t nat -D PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
if (-e "/var/run/tmccproxy-$vnode_id.pid") {
my $pid = `cat /var/run/tmccproxy-$vnode_id.pid`;
......@@ -507,81 +463,55 @@ sub Offline()
mysystem2("/bin/kill $pid");
}
@rules = ();
if (!SHAREDHOST()) {
push(@rules,
"-t nat -D POSTROUTING -j SNAT ".
"--to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
"-o $bridge");
DoIPtables("-t nat -D POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
" -o $bridge");
}
push(@rules,
"-t nat -D POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $jail_network/$jail_netmask");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $jail_network/$jail_netmask");
if (!REMOTEDED()) {
push(@rules,
"-t nat -D POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $network/$cnet_mask");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $network/$cnet_mask");
push(@rules,
"-t nat -D POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d $boss_ip,$ops_ip");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d $boss_ip,$ops_ip");
push(@rules,
"-t nat -D POSTROUTING -j ACCEPT ".
"-s $vnode_ip -d 224.0.0.0/4");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT " .
" -s $vnode_ip -d 224.0.0.0/4");
push(@rules,
"-t nat -D POSTROUTING -j ACCEPT ".
"-s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6");
DoIPtables("-t nat -D POSTROUTING -j ACCEPT ".
" -s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6");
}
push(@rules,
"-t nat -D POSTROUTING ".
"-s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
DoIPtables("-t nat -D POSTROUTING ".
"-s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
# evproxy
push(@rules,
"-t nat -D PREROUTING -j DNAT -p tcp ".
"--dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
"--to-destination $host_ip:$EVPROXY_PORT");
# Apply the rules
if (DoIPtablesNoFail(@rules) != 0) {
print STDERR "WARNING: could not remove iptables rules\n";
}
DoIPtables("-t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
return 0;
}
#
# Run the Xen vif-* script under our iptables lock.
#
sub Runscript($@)
{
my ($vnode_ip, @args) = @_;
my $rv = 0;
if (@ARGV) {
#
# Oh jeez, iptables is about the dumbest POS I've ever seen;
# it fails if you run two at the same time. So we have to
# serialize the calls. Rather then worry about each call, just
# take a big lock here.
#
TBDebugTimeStamp("$vnode_id emulab-cnet: grabbing iptables lock")
if ($lockdebug);
if (TBScriptLock("iptables", 0, 300) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the iptables lock after a long time!\n";
return -1;
exit(-1);
}
TBDebugTimeStamp(" got iptables lock")
if ($lockdebug);
#
# First run the xen script to do the bridge interface. We do this
# inside the lock since vif-bridge/vif-route do some iptables stuff.
# inside the lock since vif-bridge does some iptables stuff.
#
# vif-bridge/vif-route has bugs that cause it to leave iptables
# rules behind. If we put this stuff into the environment, they
......@@ -591,41 +521,25 @@ sub Runscript($@)
if ($VIFROUTING) {
$ENV{"netdev"} = "xenbr0";
$ENV{"gatewaydev"} = "xenbr0";
mysystem2("/etc/xen/scripts/vif-route-emulab @args");
mysystem2("/etc/xen/scripts/vif-route-emulab @ARGV");
}