Commit 5cd9a6ae authored by Leigh Stoller's avatar Leigh Stoller

Redo shell script as perl since its so much easier. Remove

the slothd, and remove the NFS rule that causes all traffic
leaving the VM to look like it came from the physical host.
Cleanup the tmcc proxy startup and teardown.
parent 53a870e8
#!/bin/sh
#
# Invoked by xmcreate script to configure the control network for a vnode.
# Usage:
# emulab-cnet vmid host_ip vnode_name vnode_ip (online | offline)
#
# NOTE: vmid should be an integer ID.
#
#boss=@BOSSNODE_IP@
#ops=@USERNODE_IP@
#fs=@FSNODE_IP@
boss=155.98.32.70
ops=155.98.33.74
fs=155.98.33.74
# XXX should be configed
tmcd_port=7777
slothd_port=8509
evproxy_port=16505
LOGDIR=/var/emulab/logs
if [ $# < 3 ]; then
echo "usage: $0 vmid host_ip vnode_name vnode_ip [ other_args ]"
exit 1
fi
# $vif comes from the environment
vmid=$1; shift
host_ip=$1; shift
vhost_id=$1; shift
vif_ip=$1; shift
local_tmcd_port=`expr $tmcd_port \+ $vmid`
#echo "${vhost_id}: op=$1, host_ip=$host_ip, vmid=$vmid, [vif=$vif], vnode_ip=$vif_ip, other=$*" >> /var/emulab/logs/xen-cnet.log
# xen's configuration for a vif
sh /etc/xen/scripts/vif-bridge $*
iptables=/sbin/iptables
do_offline() {
# dhcp
$iptables -D OUTPUT -j DROP -o $vif -m pkttype --pkt-type broadcast -m physdev --physdev-out $vif
# tmcc
$iptables -t nat -D PREROUTING -j DNAT -p tcp --dport $tmcd_port -d $boss -s $vif_ip --to-destination $host_ip:$local_tmcd_port
$iptables -t nat -D PREROUTING -j DNAT -p udp --dport $tmcd_port -d $boss -s $vif_ip --to-destination $host_ip:$local_tmcd_port
if [ -f /var/run/tmcc-$vhost_id.pid ]; then
kill `cat /var/run/tmcc-$vhost_id.pid`
fi
# slothd
$iptables -t nat -D POSTROUTING -j SNAT -p udp --dport $slothd_port --to-source $host_ip -s $vif_ip --destination $boss -o eth0
$iptables -t nat -D POSTROUTING -j SNAT --to-source $host_ip -s $vif_ip --destination $fs -o eth0
# evproxy
$iptables -t nat -D PREROUTING -j DNAT -p tcp --dport $evproxy_port -d $ops -s $vif_ip --to-destination $host_ip:$evproxy_port
}
do_online() {
# prevent dhcp requests from reaching eth0
$iptables -A OUTPUT -j DROP -o $vif -m pkttype --pkt-type broadcast -m physdev --physdev-out $vif
# reroute tmcd calls to the proxy on the physical host
$iptables -t nat -A PREROUTING -j DNAT -p tcp --dport $tmcd_port -d $boss -s $vif_ip --to-destination $host_ip:$local_tmcd_port
$iptables -t nat -A PREROUTING -j DNAT -p udp --dport $tmcd_port -d $boss -s $vif_ip --to-destination $host_ip:$local_tmcd_port
# start a tmcc proxy (handles both TCP and UDP)
/usr/local/etc/emulab/tmcc.bin -d -t 15 -n $vhost_id -X $host_ip:$local_tmcd_port -s $boss -p $tmcd_port -o $LOGDIR/tmccproxy.$vhost_id.log &
echo "$!" > /var/run/tmcc-$vhost_id.pid
# slothd
$iptables -t nat -A POSTROUTING -j SNAT -p udp --dport $slothd_port --to-source $host_ip -s $vif_ip --destination $boss -o eth0
# source-nat all traffic to fs node to come from vnode host
# XXX "fs" is usually "ops", so make sure other ops redirects happen first
#
# TODO: only forward ports the mount server needs (use rpcinfo on fs node)
# TODO: also only route back to the vif ip
$iptables -t nat -A POSTROUTING -j SNAT --to-source $host_ip -s $vif_ip --destination $fs -o eth0
# reroute evproxy packets
$iptables -t nat -A PREROUTING -j DNAT -p tcp --dport $evproxy_port -d $ops -s $vif_ip --to-destination $host_ip:$evproxy_port
}
case "$1" in
'online')
do_online
;;
'offline')
do_offline
;;
esac
exit 0
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use strict;
use Getopt::Std;
use English;
use Data::Dumper;
use POSIX qw(setsid);
#
# Invoked by xmcreate script to configure the control network for a vnode.
#
# NOTE: vmid should be an integer ID.
#
sub usage()
{
print "Usage: emulab-cnet ".
"vmid host_ip vnode_name vnode_ip (online|offline)\n";
exit(1);
}
#
# Turn off line buffering on output
#
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
use libtmcc;
use libvnode;
#
# Configure.
#
my $TMCD_PORT = 7777;
my $SLOTHD_PORT = 8509;
my $EVPROXY_PORT= 16505;
my $IPTABLES = "/sbin/iptables";
usage()
if (@ARGV < 4);
my $vmid = shift(@ARGV);
my $host_ip = shift(@ARGV);
my $vnode_id = shift(@ARGV);
my $vnode_ip = shift(@ARGV);
# The caller (xmcreate) puts this into the environment.
my $vif = $ENV{'vif'};
my $XENBUS_PATH = $ENV{'XENBUS_PATH'};
my $bridge = `xenstore-read "$XENBUS_PATH/bridge"`;
chomp($bridge);
# We need these IP addresses.
my $boss_ip = `host boss | grep 'has address'`;
if ($boss_ip =~ /has address ([0-9\.]*)$/) {
$boss_ip = $1;
}
my $ops_ip = `host ops | grep 'has address'`;
if ($ops_ip =~ /has address ([0-9\.]*)$/) {
$ops_ip = $1;
}
my $fs_ip = `host fs | grep 'has address'`;
if ($fs_ip =~ /has address ([0-9\.]*)$/) {
$fs_ip = $1;
}
# Each container gets a tmcc proxy running on another port.
my $local_tmcd_port = $TMCD_PORT + $vmid;
# Need this too.
my $outer_controlif = `cat $BOOTDIR/controlif`;
chomp($outer_controlif);
#
# First run the xen script to setup the bridge interface.
#
mysystem2("/etc/xen/scripts/vif-bridge @ARGV");
exit(1)
if ($?);
#
# We setup a bunch of iptables rules when a container goes online, and
# then clear them when it goes offline.
#
sub Online()
{
# Prevent dhcp requests from leaving the physical host.
mysystem2("$IPTABLES -A FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
return -1
if ($?);
mysystem2("$IPTABLES -A FORWARD -m physdev --physdev-in $vif -j ACCEPT");
return -1
if ($?);
# Start a tmcc proxy (handles both TCP and UDP)
my $tmccpid = fork();
if ($tmccpid) {
# Give child a chance to react.
sleep(1);
mysystem2("echo $tmccpid > /var/run/tmccproxy-$vnode_id.pid");
}
else {
POSIX::setsid();
exec("$BINDIR/tmcc.bin -d -t 15 -n $vnode_id ".
" -X $host_ip:$local_tmcd_port -s $boss_ip -p $TMCD_PORT ".
" -o $LOGDIR/tmccproxy.$vnode_id.log");
die("Failed to exec tmcc proxy");
}
# Reroute tmcd calls to the proxy on the physical host
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
return -1
if ($?);
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
return -1
if ($?);
# Reroute evproxy to use the local daemon.
mysystem2("$IPTABLES -t nat -A PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
return -1
if ($?);
#
# GROSS! source-nat all traffic destined the fs node, to come from the
# vnode host, so that NFS mounts work. We do this for non-shared nodes.
# Shared nodes do the mounts normally from inside the guest. Maybe this
# distinction is pointless, but it lowers the number of exported mounts
# on the file server.
#
if (!SHAREDHOST()) {
mysystem2("$IPTABLES -t nat -A POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip --destination $fs_ip ".
" -o $bridge");
return -1
if ($?);
}
return 0;
}
sub Offline()
{
# dhcp
mysystem2("$IPTABLES -D FORWARD -o $bridge -m pkttype ".
"--pkt-type broadcast " .
"-m physdev --physdev-in $vif --physdev-is-bridged ".
"--physdev-out $outer_controlif -j DROP");
mysystem2("$IPTABLES -D FORWARD -m physdev --physdev-in $vif -j ACCEPT");
# tmcc
# Reroute tmcd calls to the proxy on the physical host
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p udp ".
" --dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
" --to-destination $host_ip:$local_tmcd_port");
if (-e "/var/run/tmccproxy-$vnode_id.pid") {
my $pid = `cat /var/run/tmccproxy-$vnode_id.pid`;
chomp($pid);
mysystem2("/bin/kill $pid");
}
if (!SHAREDHOST()) {
mysystem2("$IPTABLES -t nat -D POSTROUTING -j SNAT ".
" --to-source $host_ip -s $vnode_ip --destination $fs_ip ".
" -o $bridge");
}
# evproxy
mysystem2("$IPTABLES -t nat -D PREROUTING -j DNAT -p tcp ".
" --dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
" --to-destination $host_ip:$EVPROXY_PORT");
return 0;
}
if (@ARGV) {
my $op = shift(@ARGV);
if ($op eq "online") {
exit(Online());
}
elsif ($op eq "offline") {
exit(Offline());
}
}
exit(0);
......@@ -7595,7 +7595,6 @@ outfiles="$outfiles clientside/GNUmakefile \
clientside/tmcc/freebsd/supfile clientside/tmcc/freebsd/sethostname \
clientside/tmcc/linux/GNUmakefile clientside/tmcc/linux/supfile \
clientside/tmcc/linux/sethostname.dhclient \
clientside/tmcc/linux/xen/emulab-cnet \
clientside/tmcc/linux9/GNUmakefile clientside/tmcc/linux9/supfile \
clientside/tmcc/fedora/GNUmakefile clientside/tmcc/fedora/supfile \
clientside/tmcc/fedora15/GNUmakefile \
......
......@@ -1262,7 +1262,6 @@ outfiles="$outfiles clientside/GNUmakefile \
clientside/tmcc/freebsd/supfile clientside/tmcc/freebsd/sethostname \
clientside/tmcc/linux/GNUmakefile clientside/tmcc/linux/supfile \
clientside/tmcc/linux/sethostname.dhclient \
clientside/tmcc/linux/xen/emulab-cnet \
clientside/tmcc/linux9/GNUmakefile clientside/tmcc/linux9/supfile \
clientside/tmcc/fedora/GNUmakefile clientside/tmcc/fedora/supfile \
clientside/tmcc/fedora15/GNUmakefile \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment