Commit 7ad42ea2 authored by Mike Hibler's avatar Mike Hibler

(Most of) the rest of the gated-in-jail changes.

Add a -V option to mkjail.pl telling it to use the virtual control net
IP (172.16.0.1) for the default route instead of the real control net
IP (155.101.132.1).  In this case, it also assigns a virtual control net
IP alias (172.17.<pnode>.0) to the physical interface so that the
aforementioned routes can actually be installed.

Add the same -V option (only valid with -j) in vnodesetup so that we
can get it through from bootvnodes.  (Also cleaned up the usage message
some)

The virtual control net address/mask are currently constants in mkjail.pl.
At some point they can/should be changed to config time options so that we
can easily use this on minibed too.

In theory, this (-V) shouldn't be an option and we should just use
virtual control net routing all the time.  But I want to get some more
testing and we have to resolve the minibed conflicts first.  So for now,
this is an option and it is off by default.

Haven't tied this option in with gated yet, so creating a vnode experiment
with Session routing still won't work.  If you want to do that, create the
experiment, watch gated blow up right and left, login to all the physical
nodes and change bootvnodes to add -V to the invocation of vnodesetup,
and then reboot everything.  Now it should be working!
parent 509227ed
......@@ -1106,7 +1106,11 @@ sub dorouterconfig (;$)
# though more work may be needed on the gated config files to make
# this work (i.e., to import existing kernel routes).
#
if ($usegated) {
# XXX if rtabid is set, we are setting up routing from outside a
# jail on behalf of a jail. We don't want to enable gated in this
# case, it will be run inside the jail.
#
if ($usegated && !defined($rtabid)) {
$rcline = gatedsetup();
print RC " $rcline\n";
}
......
......@@ -18,11 +18,17 @@ BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
sub usage()
{
print "Usage: vnodesetup [-j [-s] | -p] [-b | -k | -r | -h] [-d] <vnodeid>\n".
"Use the -k option to kill the virtual node.\n";
print "Usage: vnodesetup -j [-sV] [-b | -k | -r | -h] [-d] <vnodeid>\n".
" vnodesetup -p [-b | -k | -r | -h] [-d] <vnodeid>\n".
" -j creates a BSD jail-based virtual node.\n".
" -p creates a Planetlab virtual node.\n".
"\n".
"Use -b when starting the virtual node at boot time.\n".
"Use -h when halting the virtual node.\n".
"Use -k when killing the virtual node (removes filesystems).\n";
exit(1);
}
my $optlist = "kbdjsrhp";
my $optlist = "kbdjsVrhp";
# Locals
my $killit = 0;
......@@ -31,8 +37,9 @@ my $haltit = 0;
my $debug = 0;
my $fromboot = 0;
my $dojail = 0;
my $doplab = 0;
my $interactive = 0;
my $usevcnetroutes = 0;
my $doplab = 0;
my $cleaning = 0;
my $rebooting = 0;
my $leavejail = 0;
......@@ -110,6 +117,9 @@ if (defined($options{"j"})) {
if (defined($options{"s"})) {
$interactive = 1;
}
if (defined($options{"V"})) {
$usevcnetroutes = 1;
}
}
if (defined($options{"p"})) {
$doplab = 1;
......@@ -386,10 +396,16 @@ if ($dojail) {
fatal("Jail exited unexpectedly!");
}
else {
my $option = ($interactive ? "-s" : "");
my $options = "";
my $jailhostname = "$vname.$eid.$pid.$DOMAINNAME";
exec("mkjail.pl $option -p $pid -h $jailhostname $vnodeid");
if ($interactive) {
$options .= "-s ";
}
if ($usevcnetroutes) {
$options .= "-V ";
}
exec("mkjail.pl $options -p $pid -h $jailhostname $vnodeid");
die("*** $0:\n".
" Could not start the jail!\n");
}
......
......@@ -31,11 +31,11 @@ use libsetup qw(JailedMounts REMOTE LOCALROOTFS TMPASSDB TMGROUPDB);
#
sub usage()
{
print("Usage: mkjail.pl [-s] [-i <ipaddr>] [-p <pid>] ".
print("Usage: mkjail.pl [-V] [-s] [-i <ipaddr>] [-p <pid>] ".
"[-h <hostname>] <vnodeid>\n");
exit(-1);
}
my $optlist = "i:p:e:sh:";
my $optlist = "Vi:p:e:sh:";
#
# Only real root can run this script.
......@@ -73,6 +73,10 @@ $SIG{TERM} = 'IGNORE';
STDOUT->autoflush(1);
STDERR->autoflush(1);
# XXX
my $JAILCNET = "172.16.0.0";
my $JAILCNETMASK = "255.240.0.0";
#
# Locals
#
......@@ -102,6 +106,7 @@ my $nfsmounts = 0;
my $jailpid;
my $tmccpid;
my $interactive = 0;
my $USEVCNETROUTES = 0;
# This stuff is passed from tmcd, which we parse into a config string
# and an option set.
......@@ -142,6 +147,10 @@ if (defined($options{'s'})) {
$interactive = 1;
}
if (defined($options{'V'})) {
$USEVCNETROUTES = 1;
}
#
# Get the parent IP.
#
......@@ -209,6 +218,9 @@ else {
getjailconfig("$JAILPATH/$vnodeid");
}
my $phys_cnet_if = `control_interface`;
chomp($phys_cnet_if);
#
# See if special options supported, and if so setup args as directed.
#
......@@ -246,8 +258,7 @@ else {
# virtual nodes from the same subnet on this node.
#
if (defined($IPALIAS)) {
mysystem("ifconfig `control_interface` alias $IPALIAS ".
"netmask 255.255.255.255");
setcnethostalias($IPALIAS);
}
#
......@@ -713,7 +724,7 @@ sub cleanup()
# If the jail has its own IP, clean the alias.
if (defined($IPALIAS)) {
system("ifconfig `control_interface` -alias $IPALIAS");
clearcnethostalias($IPALIAS);
}
while (@mntpoints) {
......@@ -943,8 +954,7 @@ sub addroutestorc($rc)
open(RC, ">$rc") or
fatal("Could not open $rc to append static routes");
my $routerip = `cat $BOOTDIR/routerip`;
chomp($routerip);
my $routerip = getcnetrouter();
my $hostip = `cat $BOOTDIR/myip`;
chomp($hostip);
......@@ -960,25 +970,31 @@ sub addroutestorc($rc)
print RC "static_routes=\"\$static_routes jailnet\"\n";
print RC "route_jailnet=\"-net $IP -interface lo0 255.255.255.0\"\n";
# Need a route for the private network.
my $ctrliface = `control_interface`;
chomp($ctrliface);
print RC "static_routes=\"\$static_routes privnet\"\n";
print RC "route_privnet=\"-net $IP -interface $ctrliface $IPMASK\"\n";
print RC "route_privnet=\"-net $IP -interface $phys_cnet_if $IPMASK\"\n";
}
#
# Now a list of routes for each of the IPs the jail has access
# to. The idea here is to override the interface route such that
# traffic to the local interface goes through lo0 instead. This
# avoids going through traffic shaping when, say, pinging your own
# interface!
#
foreach my $ip (@jailips) {
print RC "static_routes=\"ip${count} \$static_routes\"\n";
print RC "route_ip${count}=\"$ip -interface lo0\"\n";
$count++;
# XXX I don't think this is really a virtual control net issue, but
# rather a gated issue. However, this is the only hook I have right now.
#
# This just in! It looks like whatever the gated problem was, it went
# away after fixing numerous other bugs. But I'll leave the conditional
# here for a little while just in case...
#
if (1 || !$USEVCNETROUTES) {
#
# Now a list of routes for each of the IPs the jail has access
# to. The idea here is to override the interface route such that
# traffic to the local interface goes through lo0 instead. This
# avoids going through traffic shaping when, say, pinging your own
# interface!
#
foreach my $ip (@jailips) {
print RC "static_routes=\"ip${count} \$static_routes\"\n";
print RC "route_ip${count}=\"$ip -interface lo0\"\n";
$count++;
}
}
close(RC);
return 0;
......@@ -1039,3 +1055,86 @@ sub getnextrtabid()
return $nextrtabid;
}
#
# Return the number of alias on the jailhost control interface
#
sub jailcnetaliases()
{
my $count = 0;
if (open(IFC, "ifconfig $phys_cnet_if |")) {
while (<IFC>) {
if ($_ =~ /inet ([0-9\.]*) netmask 0xffffffff/) {
my $host = $1;
if (inet_ntoa(inet_aton($host) & inet_aton($JAILCNETMASK)) eq
$JAILCNET) {
$count++;
}
}
}
close(IFC);
}
return $count;
}
sub setcnethostalias($)
{
my ($vnodeip) = @_;
mysystem("ifconfig $phys_cnet_if alias $vnodeip netmask 255.255.255.255");
if (!$USEVCNETROUTES) {
return;
}
#
# If the jail's IP is part of the local virtual control net,
# make sure the physical host has an alias on it as well.
#
# The convention is that .0 is the physical host alias.
#
my $cnalias = inet_aton($vnodeip);
if ((inet_ntoa($cnalias & inet_aton($JAILCNETMASK)) eq $JAILCNET) &&
jailcnetaliases() == 1) {
my $palias = inet_ntoa($cnalias & inet_aton("255.255.255.0"));
mysystem("ifconfig $phys_cnet_if alias $palias netmask $JAILCNETMASK");
}
}
sub clearcnethostalias($)
{
my ($vnodeip) = @_;
system("ifconfig $phys_cnet_if -alias $vnodeip");
if (!$USEVCNETROUTES) {
return;
}
#
# If the jail's IP is part of the local virtual control net,
# and we were the last jail, remove the physical host's alias.
#
my $cnalias = inet_aton($vnodeip);
if ((inet_ntoa($cnalias & inet_aton($JAILCNETMASK)) eq $JAILCNET) &&
jailcnetaliases() == 0) {
my $palias = inet_ntoa($cnalias & inet_aton("255.255.255.0"));
mysystem("ifconfig $phys_cnet_if -alias $palias");
}
}
sub getcnetrouter()
{
my $routerip;
if (!$USEVCNETROUTES) {
$routerip = `cat $BOOTDIR/routerip`;
chomp($routerip);
} else {
$routerip = inet_ntoa(inet_aton($JAILCNET) |
inet_aton("0.0.0.1"));
}
return $routerip;
}
......@@ -1106,7 +1106,11 @@ sub dorouterconfig (;$)
# though more work may be needed on the gated config files to make
# this work (i.e., to import existing kernel routes).
#
if ($usegated) {
# XXX if rtabid is set, we are setting up routing from outside a
# jail on behalf of a jail. We don't want to enable gated in this
# case, it will be run inside the jail.
#
if ($usegated && !defined($rtabid)) {
$rcline = gatedsetup();
print RC " $rcline\n";
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment