Commit 25a29bbb authored by Mike Hibler's avatar Mike Hibler

Another step along the Linux vserver path. It is now to the point that

vservers can be configured with experimental interfaces.  Think duct tape
and baling wire here...

 * commmon/bootvnodes: did some code refactoring in anticipation of
   something that was never needed.  Oh well, it looks purdy anyway!

 * common/libsetup.pm: LINUXJAILED() predicate to indentify local Linux
   vserver setups.  getlocalevserver() to return the IP/hostname of the
   "local" event server.

 * common/rc.{linktest,linkagent,progagent,trace,trafgen}: use the
   getlocalevserver() function for use with -s options (Linux vserver
   based vnodes cannot bind to localhost to talk to the pnode pubsubd)

 * common/config/rc.ifconfig: run this for Linux vservers, put out iface
   map for veths as well as physical interfaces

 * common/delaysetup: add -j vnodeid option, will need this at some point

 * linux/liblocsetup.pm: veth (actually etun) setup for Linux vserver vnodes

 * linux/mkvserver.pl: first cut at getting interfaces configured in
   vservers, do all the necessary etun/br plumbing (NOT a pretty sight...)

 * linux/vserver/rc.invserver (moved here from linux/rc.invserver):
   run linktest in local vservers (though it won't run yet due to NFS
   problems in vservers)

 * linux/vserver/vserver-cnet.sh: statically configure the control net
   in a vserver, no DHCP here!

 * linux/vserver/vserver-{init,rc}.sh: two parts of the Funky Interface
   Setup Dance that run inside the vserver
parent fb16500e
......@@ -46,6 +46,11 @@ my $daemon = 1;
my $reconfig= 0;
my $action;
# Prototypes
sub prebootvnodes($$);
sub postbootvnodes($$);
sub bootvnode($$$);
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
......@@ -103,38 +108,6 @@ if ($daemon && TBBackGround($logname)) {
exit(0);
}
#
# Helper function to boot/kill/halt/reboot a specific vnode.
#
sub bootvnode($$$)
{
my ($vnode, $action, $jailed) = @_;
my $opt;
my $act;
if ($action eq "halt") {
$opt = "-h";
$act = "Halting";
}
elsif ($action eq "reboot") {
$opt = "-r";
$act = "Rebooting";
}
elsif ($action eq "kill") {
$opt = "-k";
$act = "Killing";
}
else {
$opt = "-b";
$act = "Booting";
}
$opt .= ($jailed ? " -jVt" : " -i");
print "$act vnode $vnode ...\n";
system("vnodesetup $opt $vnode");
return($?);
}
#
# This applies to whatever vnodes are running. Do it and exit.
#
......@@ -221,40 +194,54 @@ exit(0)
exit(0)
if (! scalar(keys(%newvnodelist)));
#
# XXX for fake jails, we don't do most of the following,
# we just start em up.
#
if ($fakejails) {
foreach my $vnode (sort(keys(%newvnodelist))) {
# Blocks until mostly setup.
bootvnode($vnode, "boot", $newvnodelist{$vnode});
}
exit(0);
}
prebootvnodes(\%curvnodelist, \%newvnodelist)
if (!$fakejails);
#
# If booting with vnodes, then see about creating the extra FS.
# This will fail if it already exists. Keep going on any failure though.
#
if (!REMOTE()) {
system("mkextrafs.pl " . ($islinux ? "/vservers" : $vndir));
foreach my $vnode (sort(keys(%newvnodelist))) {
# Blocks until mostly setup.
bootvnode($vnode, "boot", $newvnodelist{$vnode});
}
postbootvnodes(\%curvnodelist, \%newvnodelist)
if (!$fakejails);
exit(0);
#
# Symlink /local into it.
# Prepare the vnode host prior to individual vnode setup
# Takes references to the current and new vnode hashes.
#
if (! -e "/local") {
system("ln -s " . LOCALROOTFS() . " /local");
}
sub prebootvnodes($$)
{
my ($cvn, $nvn) = @_;
#
# For local vnodes, create the extra FS for filesystems and other state.
# This will fail if it already exists, so just keep going.
#
if (!REMOTE()) {
my $mntdir = $islinux ? "/vservers" : $vndir;
system("mkextrafs.pl $mntdir");
}
#
# Symlink /local into it.
#
if (! -e "/local") {
system("ln -s " . LOCALROOTFS() . " /local");
}
#
# XXX OS dependent stuff. Should move elsewhere
#
if ($islinux) {
return;
}
if (!$islinux) {
#
# Make sure enough vn devices exist
#
for (my $i = 0;
$i < scalar(keys(%newvnodelist)) + scalar(keys(%curvnodelist));
$i++) {
for (my $i = 0; $i < scalar(keys(%$nvn)) + scalar(keys(%$cvn)); $i++) {
my $dev = "vn${i}";
if (! -e "/dev/${dev}c") {
system("(cd /dev; ./MAKEDEV $dev)");
......@@ -268,32 +255,71 @@ if (!$islinux) {
#
system("sysctl net.inet.ip.intr_queue_maxlen=128 >/dev/null 2>&1");
system("sysctl vfs.nfs.eacces_retry_enable=1 >/dev/null 2>&1");
#
# XXX grossed out yet? Try this one: the mount command will HUP mountd
# after every successful mount. Thus if mounts in one jail overlap NFS
# activity in another, the latter may be randomly killed off.
#
# Elegant solution: rename the mountd.pid file so mount cannot find it!
# XXX hmm...mountd_sux hack above should take care of this, and we need
# to hup mountd when we modify the exports file.
#
rename("/var/run/mountd.pid", "/var/run/mountd.thisreallysux")
if (-e "/var/run/mountd.pid");
}
#
# XXX grossed out yet? Try this one: the mount command will HUP mountd
# after every successful mount. Thus if mounts in one jail overlap NFS
# activity in another, the latter may be randomly killed off.
#
# Elegant solution: rename the mountd.pid file so mount cannot find it!
# XXX hmm...mountd_sux hack above should take care of this, and we need
# to hup mountd when we modify the exports file.
# Final setup of vnode host after individual vnode setup
# Takes references to the current and new vnode hashes.
#
rename("/var/run/mountd.pid", "/var/run/mountd.thisreallysux")
if (-e "/var/run/mountd.pid");
sub postbootvnodes($$)
{
my ($cvn, $nvn) = @_;
foreach my $vnode (sort(keys(%newvnodelist))) {
# Blocks until mostly setup.
bootvnode($vnode, "boot", $newvnodelist{$vnode});
}
#
# XXX OS dependent stuff. Should move elsewhere
#
if ($islinux) {
return;
}
if (!$islinux) {
my $PIDFILE = "/var/run/progagent.pid";
foreach my $vnode (keys(%newvnodelist)) {
foreach my $vnode (keys(%$nvn)) {
system("rtprio 15 -`cat $vndir/$vnode/root/$PIDFILE`")
if (-e "$vndir/$vnode/root/$PIDFILE");
}
}
exit(0);
#
# Helper function to boot/kill/halt/reboot a specific vnode.
#
sub bootvnode($$$)
{
my ($vnode, $action, $jailed) = @_;
my $opt;
my $act;
if ($action eq "halt") {
$opt = "-h";
$act = "Halting";
}
elsif ($action eq "reboot") {
$opt = "-r";
$act = "Rebooting";
}
elsif ($action eq "kill") {
$opt = "-k";
$act = "Killing";
}
else {
$opt = "-b";
$act = "Booting";
}
$opt .= ($jailed ? " -jVt" : " -i");
print "$act vnode $vnode ...\n";
system("vnodesetup $opt $vnode");
return($?);
}
......@@ -46,7 +46,7 @@ use librc;
# Not all clients support this.
#
exit(0)
if (MFS() || JAILED() || REMOTE() || PLAB());
if (MFS() || (JAILED() && !LINUXJAILED()) || REMOTE() || PLAB());
# Protos.
sub doboot();
......@@ -189,6 +189,7 @@ sub doboot()
my $vmac = $ifconfig->{VMAC};
my $pmac = $ifconfig->{PMAC};
my $iface = $ifconfig->{IFACE};
my $viface = $ifconfig->{VIFACE};
my $ifrtabid = undef;
my $encap = $ifconfig->{ENCAP};
my $vtag = $ifconfig->{VTAG};
......@@ -226,6 +227,10 @@ sub doboot()
$downcmds .= TMROUTECONFIG . " $inet down\n ";
$downcmds .= "$downline\n ";
}
# Trivially parsable map for users, which associate an IP
# with a local interface.
push(@ifacemap, "$viface $inet $vmac");
}
}
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004 University of Utah and the Flux Group.
# Copyright (c) 2004, 2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -132,7 +132,8 @@ sub doboot()
print "Starting Link Agent ...\n";
system("$LAGENT -v -e $pid/$eid -s localhost -l $LOGFILE ".
my $server = getlocalevserver();
system("$LAGENT -v -e $pid/$eid -s $server -l $LOGFILE ".
"-i $PIDFILE -k " . TMEVENTKEY() . "$args\n");
if ($?) {
fatal("Could not start program agent!");
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004-2007 University of Utah and the Flux Group.
# Copyright (c) 2004-2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -187,6 +187,8 @@ sub doboot()
print "Starting Program Agent ...\n";
my $server = getlocalevserver();
#
# PlanetLab nodes don't have a fixed elvind port, so we need to
# look that up and pass it in. Also for plab, we always restart
......@@ -209,12 +211,12 @@ sub doboot()
# this failure should not happen
fatal("invalid format for plabconfig info");
}
system("$PAGENT -P -e $pid/$eid -s localhost -l $LOGFILE ".
system("$PAGENT -P -e $pid/$eid -s $server -l $LOGFILE ".
"-d -i $PIDFILE -k " . TMEVENTKEY() . " -c $CONFIG ".
"-v $vname -r -p $elvind_port -t $TOKEN");
}
else {
system("$PAGENT -e $pid/$eid -s localhost -l $LOGFILE ".
system("$PAGENT -e $pid/$eid -s $server -l $LOGFILE ".
"-v $vname -t $TOKEN ".
"-d -i $PIDFILE -k " . TMEVENTKEY() . " -c $CONFIG");
exit(0)
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2005, 2006 University of Utah and the Flux Group.
# Copyright (c) 2005, 2006, 2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -203,7 +203,7 @@ sub doboot()
my $trace_snaplen = $trace->{"TRACE_SNAPLEN"};
my $log = "$LOGDIR/trace_${linkname}-${vnode}.debug";
my $optargs = "";
my $eventargs = "-e $pid/$eid -s localhost";
my $eventargs = "-e $pid/$eid -s " . getlocalevserver();
my $expr = "";
#
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004 University of Utah and the Flux Group.
# Copyright (c) 2004, 2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -162,7 +162,7 @@ sub doboot()
$SIG{HUP} = \&Pcleanup;
# We connect to the local elvind and talk to the master via the proxy.
my $cmdline = "$BINDIR/trafgen -s localhost ";
my $cmdline = "$BINDIR/trafgen -s " . getlocalevserver();
if ($pid) {
$cmdline .= " -E $pid/$eid -k " . TMEVENTKEY();
}
......
......@@ -21,12 +21,13 @@ use Exporter;
jailsetup dojailconfig findiface libsetup_getvnodeid
ixpsetup libsetup_refresh gettopomap getfwconfig gettiptunnelconfig
gettraceconfig genhostsfile getmotelogconfig calcroutes fakejailsetup
getlocalevserver
TBDebugTimeStamp TBDebugTimeStampsOn
MFS REMOTE CONTROL WINDOWS JAILED PLAB LOCALROOTFS IXP USESFS
SIMTRAFGEN SIMHOST ISDELAYNODEPATH JAILHOST DELAYHOST STARGATE
ISFW FAKEJAILED
ISFW FAKEJAILED LINUXJAILED
CONFDIR LOGDIR TMDELAY TMJAILNAME TMSIMRC TMCC
TMNICKNAME TMSTARTUPCMD FINDIF
......@@ -86,6 +87,12 @@ sub libsetup_getvnodeid()
#
my $injail;
#
# True if $injail == TRUE and running on Linux.
# Right now this means vserves on RHL.
#
my $inlinuxjail;
#
# True if running as a fake jail (no jail, just processes).
#
......@@ -135,6 +142,9 @@ BEGIN
libsetup_setvnodeid($vid);
$injail = 1;
if ($^O eq "linux") {
$inlinuxjail = 1;
}
}
elsif (exists($ENV{'FAKEJAIL'})) {
# Fake jail.
......@@ -292,6 +302,7 @@ sub STARGATE() { if (-e "$ETCDIR/isstargate") { return 1; } else { return 0; }
#
sub JAILED() { if ($injail) { return $vnodeid; } else { return 0; } }
sub FAKEJAILED(){ if ($nojail) { return $vnodeid; } else { return 0; } }
sub LINUXJAILED(){ if ($injail && $inlinuxjail) { return $vnodeid; } else { return 0; } }
#
# Are we on plab?
......@@ -1828,6 +1839,23 @@ sub whatsmynickname()
return "$vname.$eid.$pid";
}
#
# Return the hostname or IP to use for a local event server.
# Normally this is "localhost", but for virtual nodes which share an
# event server via the physical host, it may be the IP of the physical host.
#
sub getlocalevserver()
{
my $evserver = "localhost";
if (-e "$BOOTDIR/localevserver") {
$evserver = `cat $BOOTDIR/localevserver`;
chomp($evserver);
}
return $evserver;
}
#
# Put ourselves into the background, directing output to the log file.
# The caller provides the logfile name, which should have been created
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2005, 2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -82,6 +82,7 @@ my ($pid, $eid, $vname) = check_nickname();
#
print("Starting linktest daemon\n");
my $server = getlocalevserver();
system("linktest -e $pid/$eid ".
"-s localhost -l $logfile -i $pidfile -k $keyfile -u $swapper");
"-s $server -l $logfile -i $pidfile -k $keyfile -u $swapper");
exit($? >> 0);
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# Copyright (c) 2000-2008 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -16,12 +16,13 @@ use Getopt::Std;
#
sub usage()
{
print "Usage: delaysetup [-i | -u]\n";
print "Usage: delaysetup [-i | -u] [-j <vnodeid>]\n";
exit(1);
}
my $optlist = "iu";
my $optlist = "iuj:";
my $install = 0;
my $uninstall = 0;
my $vnodeid;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
......@@ -33,8 +34,8 @@ $| = 1;
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
# library and initialize itself.
#
use libsetup;
use libtmcc;
......@@ -55,10 +56,21 @@ if (defined($options{"i"})) {
if (defined($options{"u"})) {
$uninstall = 1;
}
if (defined($options{"j"})) {
$vnodeid = $options{"j"};
libsetup_setvnodeid($vnodeid);
# Tell tmcc library, although thats already been done with previous call.
configtmcc("subnode", $vnodeid);
}
if (@ARGV) {
usage();
}
# XXX not yet
if (defined($vnodeid)) {
exit(0);
}
#
# Delay node configuration goop.
#
......
......@@ -45,6 +45,7 @@ BEGIN
# Convenience.
sub REMOTE() { return libsetup::REMOTE(); }
sub PLAB() { return libsetup::PLAB(); }
sub LINUXJAILED(){ return libsetup::LINUXJAILED(); }
#
# Various programs and things specific to Linux and that we want to export.
......@@ -86,11 +87,12 @@ my $GATED = "/usr/sbin/gated";
my $ROUTE = "/sbin/route";
my $SHELLS = "/etc/shells";
my $DEFSHELL = "/bin/tcsh";
my $IWCONFIG = '/usr/local/sbin/iwconfig';
my $WLANCONFIG = '/usr/local/bin/wlanconfig';
my $RMMOD = '/sbin/rmmod';
my $MODPROBE = '/sbin/modprobe';
my $IWPRIV = '/usr/local/sbin/iwpriv';
my $IWCONFIG = '/usr/local/sbin/iwconfig';
my $WLANCONFIG = '/usr/local/bin/wlanconfig';
my $RMMOD = '/sbin/rmmod';
my $MODPROBE = '/sbin/modprobe';
my $IWPRIV = '/usr/local/sbin/iwpriv';
my $BRCTL = "/usr/sbin/brctl";
#
# OS dependent part of cleanup node state.
......@@ -421,7 +423,7 @@ sub os_ifconfig_line($$$$$$$$;$$$)
#
# Specialized function for configing virtual ethernet devices:
#
# 'veth' locally hacked veth devices (not on Linux)
# 'veth' one end of an etun device embedded in a vserver
# 'vlan' 802.1q tagged vlan devices
# 'alias' IP aliases on physical interfaces
#
......@@ -431,15 +433,98 @@ sub os_ifconfig_veth($$$$$;$$$$%)
$rtabid, $encap, $vtag, $itype, $cookie) = @_;
my ($uplines, $downlines);
if ($itype !~ /^(alias|vlan)$/) {
if ($itype eq "veth") {
warn("veth${id}: 'veth' not supported on Linux\n");
} else {
warn("Unknown virtual interface type $itype\n");
}
if ($itype !~ /^(alias|vlan|veth)$/) {
warn("Unknown virtual interface type $itype\n");
return "";
}
#
# Veth.
#
# Veths for Linux vservers mean vtun devices. One end is outside
# the vserver and is bridged with other veths and peths as appropriate
# to form the topology. The other end goes in the vserver and is
# configured with an IP address. This final step is not done here
# as the vserver must be running first.
#
# In the current configuration, there is configuration that takes
# place both inside and outside the vserver.
#
# Inside:
# The inside case (LINUXJAILED() == 1) just configures the IP info on
# the interface.
#
# Outside:
# The outside actions are much more involved as described above.
# The VTAG identifies a bridge device "ebrN" to be used.
# The RTABID identifies the namespace, but we don't care here.
#
# To create a etun pair you do:
# echo etun0,etun1 > /sys/module/etun/parameters/newif
# To destroy do:
# echo etun0 > /sys/module/etun/parameters/delif
#
if ($itype eq "veth") {
#
# We are inside a Linux jail.
# We configure the interface pretty much like normal.
#
if (LINUXJAILED()) {
if ($inet eq "") {
$uplines .= "$IFCONFIGBIN $iface up";
}
else {
$uplines .= sprintf($IFCONFIG, $iface, $inet, $mask);
$downlines = "$IFCONFIGBIN $iface down";
}
return ($uplines, $downlines);
}
#
# Outside jail.
# Create tunnels and bridge and plumb them all together.
#
my $brdev = "ebr$vtag";
my $iniface = "veth$id";
my $outiface = "peth$id";
my $devdir = "/sys/module/etun/parameters";
# UP
$uplines = "";
# modprobe (should be done already for cnet setup, but who cares)
$uplines .= "modprobe etun\n";
# make sure bridge device exists and is up
$uplines .= " $IFCONFIGBIN $brdev >/dev/null 2>&1 || {";
$uplines .= " $BRCTL addbr $brdev\n";
$uplines .= " $IFCONFIGBIN $brdev up\n";
$uplines .= " }\n";
# create the tunnel device
$uplines .= " echo $outiface,$iniface > $devdir/newif || exit 1\n";
# bring up outside IF, insert into bridge device
$uplines .= " $IFCONFIGBIN $outiface up || exit 2\n";
$uplines .= " $BRCTL addif $brdev $outiface || exit 3\n";
# configure the MAC address for the inside device
$uplines .= " $IFCONFIGBIN $iniface hw ether $vmac || exit 4\n";
# DOWN
$downlines = "";
# remove IF from bridge device, down it (remove bridge if empty?)
$downlines .= "$BRCTL delif $brdev $outiface || exit 13\n";
$downlines .= " $IFCONFIGBIN $outiface down || exit 12\n";
# destroy tunnel devices (this will fail if inside IF in vserver still)
$downlines .= " echo $iniface > $devdir/delif || exit 11\n";
return ($uplines, $downlines);
}
#
# IP aliases
#
......@@ -510,12 +595,15 @@ sub os_viface_name($)
# alias: There is an alias device, but not sure what it is good for
# so for now we just return the phys device.
# vlan: vlan<VTAG>
# veth: veth<ID>
#
my $itype = $ifconfig->{"ITYPE"};
if ($itype eq "alias") {
return $piface;
} elsif ($itype eq "vlan") {
return $itype . $ifconfig->{"VTAG"};
} elsif ($itype eq "veth") {
return $itype . $ifconfig->{"ID"};
}
warn("Linux does not support virtual interface type '$itype'\n");
......@@ -1429,7 +1517,7 @@ sub os_config_gre($$$$$$$)
if (system("ip tunnel add $dev mode gre remote $dsthost local $srchost") ||
system("ip link set $dev up") ||
system("ip addr add $inetip dev $dev") ||
system("ifconfig $dev netmask $mask")) {
system("$IFCONFIGBIN $dev netmask $mask")) {
warn("Could not start tunnel!\n");
return -1;
}
......
This diff is collapsed.
......@@ -8,6 +8,8 @@ use strict;
use English;
use Getopt::Std;
my $DOSETUP = 1;
sub usage()
{
print "Usage: [-d] start|stop\n";
......@@ -60,7 +62,7 @@ if ($action eq "stop") {
$SIG{TERM} = 'IGNORE';
system("kill -TERM -1");
sleep(1);
system("kill -KILL -1");
#system("kill -KILL -1");
exit(0);
}
# So that rc will know to invoke us to stop.
......@@ -87,8 +89,14 @@ exit(0);
#
sub DoBoot()
{
if (!$DOSETUP) {
TBDebugTimeStamp("rc.invserver WARNING not starting up");
system("touch $BOOTDIR/vrunning");
return;
}
TBDebugTimeStamp("rc.invserver starting up");
print("Checking Testbed reservation status\n");
my ($pid, $eid, $vname) = jailsetup();
......@@ -127,9 +135,8 @@ sub DoBoot()
}
TBDebugTimeStamp("rc.invserver done running config scripts");
# Linktest daemon now runs inside jails.
if (-x "$RCDIR/rc.linktest") {
print("Starting linktest daemon\n");
# Linktest daemon now runs inside (local) jails.
if (-x "$RCDIR/rc.linktest" && !REMOTE()) {
system("$RCDIR/rc.linktest start");
if ($?) {
BootFatal("Error running $RCDIR/rc.linktest");
......
#!/bin/sh
#
# (De)configure vserver interfaces: loopback and control net.
# This is installed as: /etc/rc3.d/S00cnet on RHL based vservers.
#
action=$1
. /etc/init.d/functions
. /etc/emulab/paths.sh