Commit 169bd788 authored by David Johnson's avatar David Johnson

openvz support. Main thing to note is that I ditched the

bootvnodes/vnodesetup/mkX.pl train in favor of my own strawman design for
a more generic form of virt node support.  The strawman is incomplete and
probably wrong in places, but I had to abandon the quest for anything
better for now.  For now, uses same server side stuff as jails.
parent e0e7a65f
......@@ -135,7 +135,8 @@ sub doboot()
$gname = lc($gname)
if (PLAB());
if (REMOTE() && !REMOTEDED() && !JAILED() && !PLAB()) {
if (REMOTE() && !REMOTEDED() && !JAILED() && !GENVNODE()
&& !PLAB()) {
$gname = "emu-${gname}";
}
$newgroups{"$gname"} = $2
......
......@@ -182,6 +182,46 @@ sub doboot()
# with a local interface.
push(@ifacemap, "$iface $inet $mac");
}
elsif ($ifconfig->{ISVIRT}
&& (GENVNODE() && GENVNODETYPE() eq 'openvz')) {
#
# Yes, these are veths, but we config them from the inside just
# like for a phys node!
#
my $inet = $ifconfig->{IPADDR};
my $type = $ifconfig->{ITYPE};
my $mask = $ifconfig->{IPMASK};
my $mac = $ifconfig->{VMAC};
my $speed = $ifconfig->{SPEED};
my $duplex = $ifconfig->{DUPLEX};
my $aliases = $ifconfig->{ALIASES};
my $iface = $ifconfig->{IFACE};
my $settings = $ifconfig->{SETTINGS};
my $lan = $ifconfig->{LAN};
my $ifrtabid = undef;
my ($upline, $downline) =
os_ifconfig_line($iface, $inet, $mask,
$speed, $duplex, $aliases,
$type, $lan, $settings,
$ifrtabid, \%oscookie);
if (defined($upline) && $upline) {
$upcmds .= "$upline\n ";
$upcmds .= TMROUTECONFIG . " $inet up\n "
if ($inet ne "");
}
if (defined($downline)) {
$downcmds .= TMROUTECONFIG . " $inet down\n "
if ($inet ne "");
$downcmds .= "$downline\n ";
}
# Trivially parsable map for users, which associate an IP
# with a local interface.
push(@ifacemap, "$iface $inet $mac");
}
else {
my $itype = $ifconfig->{ITYPE};
my $inet = $ifconfig->{IPADDR};
......
......@@ -58,7 +58,7 @@ use librc;
# Not all clients support this.
#
exit(0)
if (REMOTE() || JAILED() || DELAYHOST());
if (REMOTE() || JAILED() || GENVNODE() || DELAYHOST());
# Protos.
sub doboot();
......
......@@ -51,7 +51,7 @@ $LOGGER = "/usr/local/etc/emulab/motelogd";
# Not all clients support this.
#
exit(0)
if (MFS() || PLAB() || JAILED() || WINDOWS() || REMOTE());
if (MFS() || PLAB() || JAILED() || GENVNODE() || WINDOWS() || REMOTE());
# Protos.
sub doboot();
......
......@@ -50,7 +50,7 @@ use librc;
# Not all clients support this.
#
exit(0)
if (REMOTE() || JAILED() || PLAB());
if (REMOTE() || JAILED() || GENVNODE() || PLAB());
# Protos.
sub doboot();
......
......@@ -40,7 +40,7 @@ use librc;
# Not all clients support this.
#
exit(0)
if (MFS() || PLAB() || JAILED());
if (MFS() || PLAB() || JAILED() || GENVNODE());
# Protos.
sub doboot();
......
......@@ -141,7 +141,7 @@ sub doboot()
# Create the /local partition to hold the logs, and mkdir the log
# directory. This works on Linux too!
#
if (! JAILED()) {
if (! JAILED() && ! GENVNODE()) {
if (! -d "/local") {
system("mkdir /local") == 0
or fatal("Could not mkdir /local");
......@@ -234,7 +234,7 @@ sub doboot()
$optargs .= " -P " . (4442 + $idx);
}
if (! JAILED() && ! PLAB() ) {
if (! JAILED() && ! GENVNODE() && ! PLAB() ) {
#
# Need the ethernet address for the vnode so we can create an ether
# rule for tcpdump. We want to make sure that we get the correct
......
......@@ -21,19 +21,20 @@ use Exporter;
jailsetup dojailconfig findiface libsetup_getvnodeid
ixpsetup libsetup_refresh gettopomap getfwconfig gettiptunnelconfig
gettraceconfig genhostsfile getmotelogconfig calcroutes fakejailsetup
getlocalevserver
getlocalevserver genvnodesetup getgenvnodeconfig stashgenvnodeconfig
TBDebugTimeStamp TBDebugTimeStampsOn
MFS REMOTE REMOTEDED CONTROL WINDOWS JAILED PLAB LOCALROOTFS IXP USESFS
SIMTRAFGEN SIMHOST ISDELAYNODEPATH JAILHOST DELAYHOST STARGATE
ISFW FAKEJAILED LINUXJAILED
ISFW FAKEJAILED LINUXJAILED GENVNODE GENVNODETYPE GENVNODEHOST
CONFDIR LOGDIR TMDELAY TMJAILNAME TMSIMRC TMCC TMCCBIN
TMNICKNAME TMSTARTUPCMD FINDIF
TMROUTECONFIG TMLINKDELAY TMDELMAP TMTOPOMAP TMLTMAP TMLTPMAP
TMGATEDCONFIG TMSYNCSERVER TMKEYHASH TMNODEID TMEVENTKEY
TMCREATOR TMSWAPPER TMFWCONFIG
TMCREATOR TMSWAPPER TMFWCONFIG TMGENVNODECONFIG
);
# Must come after package declaration!
......@@ -93,6 +94,11 @@ my $injail;
#
my $inlinuxjail;
#
# True if running inside a vm.
#
my $ingenvnode;
#
# True if running as a fake jail (no jail, just processes).
#
......@@ -127,6 +133,7 @@ BEGIN
# Make sure these exist! They will not exist on a PLAB vserver initially.
mkdir("$VARDIR", 0775);
mkdir("$VARDIR/jails", 0775);
mkdir("$VARDIR/vms", 0775);
mkdir("$VARDIR/db", 0755);
mkdir("$VARDIR/logs", 0775);
mkdir("$VARDIR/boot", 0775);
......@@ -160,6 +167,15 @@ BEGIN
libsetup_setvnodeid($vid);
$inplab = 1;
}
elsif (-e "$BOOTDIR/vmname") {
open(VN, "$BOOTDIR/vmname");
my $vid = <VN>;
close(VN);
libsetup_setvnodeid($vid);
$ingenvnode = 1;
}
$role = "";
# Get our role.
......@@ -195,10 +211,37 @@ sub TMLTPMAP() { "$BOOTDIR/ltpmap";}
#
sub JAILDIR() { "$VARDIR/jails/$vnodeid"; }
#
# This path is valid only *outside* the vm. Sucks, but this is the best we
# can do. Probably the only way to make this vm-specific if necessary is to
# have them create their dir and symlink.
#
sub GENVNODEDIR() { "$VARDIR/vms/$vnodeid"; }
#
# XXX: eventually this needs to come from tmcd, but that's not ready yet.
#
sub GENVNODETYPE() {
if (-e "$ETCDIR/genvmtype") {
my $vmtype = `cat $ETCDIR/genvmtype`;
chomp($vmtype);
return $vmtype;
}
return undef;
}
#
# Also valid outside the jail, this is where we put local project storage.
#
sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
sub LOCALROOTFS() {
return "/users/local"
if (REMOTE());
return "$VARDIR/jails/local"
if (JAILED());
return "$VARDIR/vms/local"
if (GENVNODE());
}
#
# Okay, here is the path mess. There are three environments.
......@@ -211,7 +254,9 @@ sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
#
sub CONFDIR() {
return $BOOTDIR
if ($injail || $inplab);
if ($injail || $inplab || $ingenvnode);
return GENVNODEDIR()
if ($vnodeid && GENVNODETYPE());
return JAILDIR()
if ($vnodeid);
return $BOOTDIR;
......@@ -219,7 +264,9 @@ sub CONFDIR() {
# Cause of fakejails, we want log files in the right place.
sub LOGDIR() {
return $LOGDIR
if ($injail || $inplab);
if ($injail || $inplab || $ingenvnode);
return GENVNODEDIR()
if ($vnodeid && GENVNODETYPE());
return JAILDIR()
if ($vnodeid);
return $LOGDIR;
......@@ -232,6 +279,7 @@ sub TMNICKNAME() { CONFDIR() . "/nickname";}
sub TMJAILNAME() { CONFDIR() . "/jailname";}
sub TMFAKEJAILNAME() { CONFDIR() . "/fakejail";}
sub TMJAILCONFIG() { CONFDIR() . "/jailconfig";}
sub TMGENVNODECONFIG() { CONFDIR() . "/genvnodeconfig";}
sub TMSTARTUPCMD() { CONFDIR() . "/startupcmd";}
sub TMROUTECONFIG() { CONFDIR() . "/rc.route";}
sub TMGATEDCONFIG() { CONFDIR() . "/gated.conf";}
......@@ -310,6 +358,11 @@ sub JAILED() { if ($injail) { return $vnodeid; } else { return 0; } }
sub FAKEJAILED(){ if ($nojail) { return $vnodeid; } else { return 0; } }
sub LINUXJAILED(){ if ($injail && $inlinuxjail) { return $vnodeid; } else { return 0; } }
#
# Are we using the generic vm abstraction for this vnode? See above.
#
sub GENVNODE() { if ($ingenvnode) { return $vnodeid; } else { return 0; } }
#
# Are we on plab?
#
......@@ -333,6 +386,7 @@ sub SIMTRAFGEN(){ if (-e ISSIMTRAFGENPATH()) { return 1; } else { return 0; } }
# A jail host?
sub JAILHOST() { if ($role eq "virthost") { return 1; } else { return 0; } }
sub GENVNODEHOST() { if ($role eq "virthost") { return 1; } else { return 0; } }
# A delay host? Either a delay node or a node using linkdelays
sub DELAYHOST() { if (-e ISDELAYNODEPATH()) { return 1; } else { return 0; } }
......@@ -632,7 +686,7 @@ sub getifconfig($;$)
# The pmac refers to the underlying physical interface the veth
# is attached to, which we do not see from inside the jail.
#
if (JAILED()) {
if (JAILED() || GENVNODE()) {
if (! ($iface = findiface($vmac))) {
warn("*** WARNING: Could not map $vmac to a veth!\n");
next;
......@@ -1711,7 +1765,153 @@ sub vnodejailsetup($)
dojailconfig();
return ($pid, $eid, $vname);
}
}
#
# All we do is store it away in the file. This makes it avail later.
#
sub stashgenvnodeconfig()
{
my @tmccresults;
# XXX: use GENVNODECONFIG once it's written!
if (tmcc(TMCCCMD_JAILCONFIG, undef, \@tmccresults) < 0) {
warn("*** WARNING: Could not get genvnodeconfig from server!\n");
return -1;
}
return 0
if (! @tmccresults);
if (!open(RC, ">" . TMGENVNODECONFIG)) {
warn "*** WARNING: Could not write " . TMGENVNODECONFIG . "\n";
return -1;
}
foreach my $str (@tmccresults) {
print RC $str;
}
close(RC);
chmod(0755, TMGENVNODECONFIG);
return 0;
}
#
# Return the generic vnode config info in a hash. XXX: For now uses jailconfig.
#
sub getgenvnodeconfig($;$)
{
my ($rptr,$nocache) = @_;
my @tmccresults = ();
my %vconfig = ();
my %tmccopts = ();
if ($nocache) {
$tmccopts{"nocache"} = 1;
}
# XXX uses jailconfig instead of genvmconfig
if (tmcc(TMCCCMD_JAILCONFIG, undef, \@tmccresults, %tmccopts) < 0) {
warn("*** WARNING: Could not get genvmconfig from server!\n");
%$rptr = ();
return -1;
}
foreach my $line (@tmccresults) {
if ($line =~ /^(.*)="(.*)"$/ ||
$line =~ /^(.*)=(.+)$/) {
if ($1 eq 'JAILIP'
&& $2 =~ /^(\d+\.\d+\.\d+\.\d+),(\d+\.\d+\.\d+\.\d+)$/) {
$vconfig{"CTRLIP"} = $1;
$vconfig{"CTRLMASK"} = $2;
}
}
}
%$rptr = %vconfig;
return 0;
}
#
# Virtual node vm setup. This happens outside in the root context.
#
sub genvnodesetup($;$$)
{
my ($vid) = @_;
#
# Set global vnodeid for tmcc commands.
#
libsetup_setvnodeid($vid);
#
# This is the directory where the rc files go.
#
if (! -e GENVNODEDIR()) {
os_mkdir(GENVNODEDIR(),"0755");
#die("*** $0:\n".
# " No such directory: " . GENVNODEDIR() . "\n");
}
#
# Always remove the old nickname file. No need to worry about a project
# change at this level (see bootsetup) but we do need to make sure we
# pick up on a vnode/jail being reassigned to a different virtual node.
#
unlink TMNICKNAME;
# Do not bother if somehow got released.
if (! check_status()) {
print "Node is free!\n";
return undef;
}
#
# Create /local directories for users.
#
if (! -e LOCALROOTFS()) {
os_mkdir(LOCALROOTFS(), "0755");
}
if (-e LOCALROOTFS()) {
my $piddir = LOCALROOTFS() . "/$pid";
my $eiddir = LOCALROOTFS() . "/$pid/$eid";
my $viddir = LOCALROOTFS() . "/$pid/$vid";
if (! -e $piddir) {
mkdir($piddir, 0777) or
die("*** $0:\n".
" mkdir filed - $piddir: $!\n");
}
if (! -e $eiddir) {
mkdir($eiddir, 0777) or
die("*** $0:\n".
" mkdir filed - $eiddir: $!\n");
}
if (! -e $viddir) {
mkdir($viddir, 0775) or
die("*** $0:\n".
" mkdir filed - $viddir: $!\n");
}
chmod(0777, $piddir);
chmod(0777, $eiddir);
chmod(0775, $viddir);
}
#
# Tell libtmcc to get the full config for the jail. At the moment
# we do not use SFS inside jails, so okay to do this now (usually
# have to call initsfs() first). The full config will be copied
# to the proper location inside the jail by mkjail.
#
tmccgetconfig();
#
# XXX: Get jail config. For now we just snoop on this, but should do our
# own later.
#
print STDOUT "Checking Testbed generic VM configuration ...\n";
stashgenvnodeconfig();
return ($pid, $eid, $vname);
}
#
# This happens inside a Plab vserver.
......
......@@ -40,7 +40,7 @@ my $logtemplate;
# Get the config parameters so we can open up a log file in the proper
# location. See below.
#
if (JAILED() || PLAB()) {
if (JAILED() || GENVNODE() || PLAB()) {
$logtemplate = "$VARDIR/logs/runlog.XXXXXX";
}
else {
......
......@@ -81,7 +81,8 @@ my %faketimes;
#
my %iv = (
check => 0,
isalive => ((REMOTE() == 1) ? (PLAB() ? 600 : 60) : (JAILED() ? 600 : 180)),
isalive => ((REMOTE() == 1) ? (PLAB() ? 600 : 60) : ((JAILED()
|| GENVNODE()) ? 600 : 180)),
drift => (60 * 60 * 12),
cvsup => (60 * 60 * 12),
rusage => 0,
......@@ -213,7 +214,7 @@ $SIG{INT} = \&handler;
#
# If jailed, get our jailname.
#
if (JAILED() || PLAB()) {
if (JAILED() || GENVNODE() || PLAB()) {
my $vnodeid = libsetup_getvnodeid();
# Tell the tmcc library. Note that its actually been done via libsetup
......
......@@ -71,13 +71,18 @@ if (@ARGV) {
# switch to it.
#
my $KERNEL100 = "emulab";
my $KERNELJAIL = "jail";
# XXX: this has the same problems calling GENVNODETYPE() does -- need to
# make vnode type explicit from tmcd!
my $KERNELJAIL = GENVNODETYPE();
my $KERNELLDELAY= "linkdelay";
# just use the linkdelay kernel; it's all the same...
my $KERNELDELAY = $KERNELLDELAY;
my $TMDELMAP = TMDELMAP; # Really comes from libloc.
my $TC = "/usr/local/sbin/tc"; # This is the working version!
my $TMDELMAP = TMDELMAP; # Really comes from libloc.
my $TC = "/usr/local/sbin/tc"; # This is the working version!
if (! -e $TC && -e "/sbin/tc") { # If we hacked iproute rpm
$TC = "/sbin/tc";
}
my $IPTABLES = "/usr/local/sbin/iptables"; # This is the working version!
my $IFCONFIG = "/sbin/ifconfig";
my $MODPROBE = "/sbin/modprobe";
......
......@@ -252,3 +252,16 @@ vserver-install: dir-install
$(INSTALL) -m 755 $(SRCDIR)/vserver/vserver-rc.sh $(VSDIR)/
$(INSTALL) -m 755 $(SRCDIR)/mkvserver.pl $(BINDIR)/mkvserver.pl
-ln -sf $(BINDIR)/mkvserver.pl $(BINDIR)/mkjail.pl
openvz-install: dir-install
$(INSTALL) -m 755 $(SRCDIR)/libvnode.pm $(BINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/openvz/libvnode_openvz.pm $(BINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/openvz/vznetinit-elab.sh $(BINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/vnodectl $(BINDIR)/
echo "openvz" > $(ETCDIR)/genvmtype
-ln -sf $(BINDIR)/vnodectl $(BINDIR)/vnodesetup
-ln -sf $(BINDIR)/vnodectl $(BINDIR)/bootvnodes
$(INSTALL) -m 755 $(SRCDIR)/openvz/vzmount-elab.sh $(BINDIR)/
-ln -sf $(BINDIR)/vzmount-elab.sh /etc/vz/conf/vps.mount
$(INSTALL) -m 755 $(SRCDIR)/openvz/vzumount-elab.sh $(BINDIR)/
-ln -sf $(BINDIR)/vzumount-elab.sh /etc/vz/conf/vps.umount
......@@ -347,69 +347,76 @@ sub os_ifconfig_line($$$$$$$$;$$$)
}
#
# Need to check units on the speed. Just in case.
# Only do this stuff if we have a physical interface, otherwise it doesn't
# mean anything. We need this for virtnodes whose networks must be
# config'd from inside the container, vm, whatever.
#
if ($speed =~ /(\d*)([A-Za-z]*)/) {
if ($2 eq "Mbps") {
$speed = $1;
}
elsif ($2 eq "Kbps") {
$speed = $1 / 1000;
}
else {
warn("*** Bad speed units $2 in ifconfig, default to 100Mbps\n");
$speed = 100;
}
if ($speed == 1000) {
$media = $IFC_1000MBS;
if ($iface_type ne 'veth') {
#
# Need to check units on the speed. Just in case.
#
if ($speed =~ /(\d*)([A-Za-z]*)/) {
if ($2 eq "Mbps") {
$speed = $1;
}
elsif ($2 eq "Kbps") {
$speed = $1 / 1000;
}
else {
warn("*** Bad speed units $2 in ifconfig, default to 100Mbps\n");
$speed = 100;
}
if ($speed == 1000) {
$media = $IFC_1000MBS;
}
elsif ($speed == 100) {
$media = $IFC_100MBS;
}
elsif ($speed == 10) {
$media = $IFC_10MBS;
}
else {
warn("*** Bad Speed $speed in ifconfig, default to 100Mbps\n");
$speed = 100;
$media = $IFC_100MBS;
}
}
elsif ($speed == 100) {
$media = $IFC_100MBS;
if ($duplex eq "full") {
$media = "$media-$IFC_FDUPLEX";
}
elsif ($speed == 10) {
$media = $IFC_10MBS;
elsif ($duplex eq "half") {
$media = "$media-$IFC_HDUPLEX";
}
else {
warn("*** Bad Speed $speed in ifconfig, default to 100Mbps\n");
$speed = 100;
$media = $IFC_100MBS;
warn("*** Bad duplex $duplex in ifconfig, default to full\n");
$duplex = "full";
$media = "$media-$IFC_FDUPLEX";
}
}
if ($duplex eq "full") {
$media = "$media-$IFC_FDUPLEX";
}
elsif ($duplex eq "half") {
$media = "$media-$IFC_HDUPLEX";
}
else {
warn("*** Bad duplex $duplex in ifconfig, default to full\n");
$duplex = "full";
$media = "$media-$IFC_FDUPLEX";
}
#
# Linux is apparently changing from mii-tool to ethtool but some drivers
# don't support the new interface (3c59x), some don't support the old
# interface (e1000), and some (eepro100) support the new interface just
# enough that they can report success but not actually do anything. Sweet!
#
my $ethtool;
if (-e "/sbin/ethtool") {
$ethtool = "/sbin/ethtool";
} elsif (-e "/usr/sbin/ethtool") {
$ethtool = "/usr/sbin/ethtool";
}
if (defined($ethtool)) {
# this seems to work for returning an error on eepro100
$uplines =
"if $ethtool $iface >/dev/null 2>&1; then\n " .
" $ethtool -s $iface autoneg off speed $speed duplex $duplex\n " .
" sleep 2 # needed due to likely bug in e100 driver on pc850s\n".
"else\n " .
" /sbin/mii-tool --force=$media $iface\n " .
"fi\n ";
} else {
$uplines = "/sbin/mii-tool --force=$media $iface\n ";
#
# Linux is apparently changing from mii-tool to ethtool but some drivers
# don't support the new interface (3c59x), some don't support the old
# interface (e1000), and some (eepro100) support the new interface just
# enough that they can report success but not actually do anything. Sweet!
#
my $ethtool;
if (-e "/sbin/ethtool") {
$ethtool = "/sbin/ethtool";
} elsif (-e "/usr/sbin/ethtool") {
$ethtool = "/usr/sbin/ethtool";
}
if (defined($ethtool)) {
# this seems to work for returning an error on eepro100
$uplines =
"if $ethtool $iface >/dev/null 2>&1; then\n " .
" $ethtool -s $iface autoneg off speed $speed duplex $duplex\n " .
" sleep 2 # needed due to likely bug in e100 driver on pc850s\n".
"else\n " .
" /sbin/mii-tool --force=$media $iface\n " .
"fi\n ";
} else {
$uplines = "/sbin/mii-tool --force=$media $iface\n ";
}
}
if ($inet eq "") {
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2008-2009 University of Utah and the Flux Group.
# All rights reserved.
#
# Implements the libvnode API for OpenVZ support in Emulab.
#
package libvnode;
use Exporter;
@ISA = "Exporter";