Commit 5b16105a authored by Leigh B. Stoller's avatar Leigh B. Stoller

Add Mike's NFS/NULL mount changes to mkjail.pl

Also a couple perf hacks:

* Local vnodes can start with the password/group file from the
  physnode, since locally they will be the same anyway. This avoids a
  blizzard of accounts requests at startup, which is by far the
  biggest chunk of data returned (well, except for host tables).

* To help serialize boot startup, vnodesetup now waits for the jail to
  finish starting up before it exits. It does this via a "goofy"
  mechanism I will not bother to describe.
parent 1c28424e
......@@ -174,7 +174,8 @@ foreach my $vnode (keys(%curvnodelist)) {
$jailflag = "-j";
}
print "Setting up vnode $vnode ...\n";
# This will not return until the vnode is fully running.
system("vnodesetup -b $jailflag $vnode");
sleep(10);
sleep(1);
}
exit(0);
......@@ -22,12 +22,12 @@ use Exporter;
jailsetup dojailconfig JailedMounts findiface
tmccdie tmcctimeout libsetup_getvnodeid dotrafficconfig
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED LOCALROOTFS
TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
TMNICKNAME HOSTSFILE TMSTARTUPCMD FINDIF TMTUNNELCONFIG
TMTRAFFICCONFIG TMROUTECONFIG TMLINKDELAY TMDELMAP TMMOUNTDB
TMPROGAGENTS
TMPROGAGENTS TMPASSDB TMGROUPDB
TMCCCMD_REBOOT TMCCCMD_STATUS TMCCCMD_IFC TMCCCMD_ACCT TMCCCMD_DELAY
TMCCCMD_HOSTS TMCCCMD_RPM TMCCCMD_TARBALL TMCCCMD_STARTUP
......@@ -122,13 +122,17 @@ use liblocsetup;
sub TMCC() { "$BINDIR/tmcc"; }
sub TMHOSTS() { "$ETCDIR/hosts"; }
sub FINDIF() { "$BINDIR/findif"; }
sub LOCALROOTFS() { "/users/local"; }
sub HOSTSFILE() { "/etc/hosts"; }
#
# This path is valid only *outside* the jail when its setup.
#
sub JAILDIR() { "$VARDIR/jails/$vnodeid"; }
#
# Also valid outside the jail, this is where we put local project storage.
#
sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
#
# Okay, here is the path mess. There are three environments.
# 1. A local node where everything goes in one place ($VARDIR/boot).
......@@ -2165,8 +2169,12 @@ sub jailsetup()
print STDOUT "Checking Testbed hostnames configuration ... \n";
dohostnames();
print STDOUT "Checking Testbed group/user configuration ... \n";
doaccounts();
if (REMOTE()) {
# Locally, the password/group files initially comes from
# outside the jail.
print STDOUT "Checking Testbed group/user configuration ... \n";
doaccounts();
}
print STDOUT "Checking Testbed RPM configuration ... \n";
dorpms();
......@@ -2236,6 +2244,8 @@ sub vnodesetup($)
die("*** $0:\n".
" mkdir filed - $eiddir: $!\n");
}
chmod(0777, $piddir);
chmod(0777, $eiddir);
}
#
......
......@@ -179,8 +179,26 @@ if (-e $pidfile) {
#
if (!$debug && !$interactive && TBBackGround($logname)) {
#
# Parent exits normally
# Parent exits normally, but we wait until we think the jail is
# setup first. This whole approach is wildly hacky.
#
if ($dojail) {
my $now = time();
my $goofy = "$vnodedir/root/var/run/emulab-watchdog.pid";
my $count = 30;
while ($count--) {
sleep(1);
if (-e $goofy) {
my (undef,undef,undef,undef,undef,undef,undef,undef,
undef,$mtime,$ctime,undef,undef)
= stat($goofy);
exit(0)
if ($mtime >= $now || $ctime >= $now);
}
}
exit(1);
}
exit(0);
}
......@@ -447,8 +465,8 @@ sub cleanup()
$SIG{TERM} = 'IGNORE';
kill('TERM', -$pgrp);
print "Waiting 10 seconds for process group to die off ...\n";
sleep(10);
print "Waiting 5 seconds for process group to die off ...\n";
sleep(5);
if (! $leavejail) {
removevnodedir($vnodedir);
}
......@@ -476,8 +494,8 @@ sub reboot()
$SIG{TERM} = 'IGNORE';
kill('TERM', -$pgrp);
print "Waiting 10 seconds for process group to die off ...\n";
sleep(10);
print "Waiting 5 seconds for process group to die off ...\n";
sleep(5);
$SIG{TERM} = \&handler;
}
......
......@@ -16,7 +16,7 @@ use Fcntl ':flock';
# Drag in path stuff so we can find emulab stuff. Also untaints path.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
use libsetup qw(JailedNFSMounts REMOTE);
use libsetup qw(JailedMounts REMOTE LOCALROOTFS TMPASSDB TMGROUPDB);
#
# Questions:
......@@ -78,7 +78,7 @@ STDERR->autoflush(1);
#
my $JAILPATH = "/var/emulab/jails";
my $ETCJAIL = "/etc/jail";
my $LOCALFS = "/users/local";
my $LOCALFS = LOCALROOTFS();
my $LOCALMNTPNT = "/local";
my $TMCC = "$BINDIR/tmcc";
my $JAILCONFIG = "jailconfig";
......@@ -88,7 +88,7 @@ my @ROOTMKDIRS = ("dev", "tmp", "var", "usr", "proc", "users", "opt",
my @ROOTMNTDIRS = ("bin", "sbin", "usr");
my @EMUVARDIRS = ("logs", "db", "jails", "boot", "lock");
my $VNFILESECT = 64 * ((1024 * 1024) / 512); # 64MB in 512b sectors.
my $MAXVNDEVS = 10;
my $MAXVNDEVS = 100;
my $IP;
my $IPALIAS;
my $IPMASK;
......@@ -98,6 +98,7 @@ my $debug = 1;
my $cleaning = 0;
my $vndevice;
my @mntpoints = ();
my $nfsmounts = 0;
my $jailpid;
my $tmccpid;
my $interactive = 0;
......@@ -357,7 +358,11 @@ sub mkrootfs($)
# Okay, mount some other directories to save space.
#
foreach my $dir (@ROOTMNTDIRS) {
mysystem("mount -r localhost:/$dir $path/root/$dir");
if ($nfsmounts) {
mysystem("mount -r localhost:/$dir $path/root/$dir");
} else {
mysystem("mount -r -t null /$dir $path/root/$dir");
}
push(@mntpoints, "$path/root/$dir");
}
......@@ -384,6 +389,10 @@ sub mkrootfs($)
$makedevs .= " jail";
}
mysystem("cd $path/root/dev; cp -p /dev/MAKEDEV .; ./MAKEDEV $makedevs");
# Remove some extraneous devices.
mysystem("cd $path/root/dev; rm -f pci io klog console; ".
"ln -sf null console");
#
# Create stub /var and create the necessary log files.
......@@ -428,10 +437,7 @@ sub mkrootfs($)
mysystem("rm -f $path/root/etc/rc.conf.local");
mysystem("cp -p $ETCJAIL/rc.local $path/root/etc");
mysystem("cp -p $ETCJAIL/crontab $path/root/etc");
mysystem("cp -p $ETCJAIL/group $path/root/etc");
mysystem("cp -p $ETCJAIL/master.passwd $path/root/etc");
mysystem("cp /dev/null $path/root/etc/fstab");
mysystem("pwd_mkdb -p -d $path/root/etc $path/root/etc/master.passwd");
# No X11 forwarding.
mysystem("cat $path/root/etc/ssh/sshd_config | ".
......@@ -455,12 +461,31 @@ sub mkrootfs($)
"sed -e 's/127\.0\.0\.1/$hostip/' > ".
"$path/root/etc/resolv.conf");
#
# Password/group file stuff. If remote the jail picks them up. Locally
# we start with current set of accounts on the physnode, since this is
# more efficient (less tmcd work), and besides, the physnode starts with
# exactly the same accounts.
#
if (REMOTE()) {
mysystem("cp -p $ETCJAIL/group $path/root/etc");
mysystem("cp -p $ETCJAIL/master.passwd $path/root/etc");
mysystem("pwd_mkdb -p -d $path/root/etc $path/root/etc/master.passwd");
}
else {
mysystem("cp -p /etc/group $path/root/etc");
mysystem("cp -p /etc/master.passwd $path/root/etc");
mysystem("pwd_mkdb -p -d $path/root/etc $path/root/etc/master.passwd");
mysystem("cp -p " . TMPASSDB() . ".db $path/root/$DBDIR");
mysystem("cp -p " . TMGROUPDB() . ".db $path/root/$DBDIR");
}
#
# If the jail gets its own routing table, must arrange for it to
# be populated with some extras when it boots up.
#
if ($jailflags & $JAIL_ROUTING) {
addroutestorc("$path/root/etc/rc.conf");
addroutestorc("$path/root/etc/rc.conf.routes");
}
#
......@@ -469,7 +494,13 @@ sub mkrootfs($)
#
if (defined($PID) && -e $LOCALFS && -e "$LOCALFS/$PID") {
mysystem("mkdir -p $path/root/$LOCALMNTPNT/$PID");
mysystem("mount localhost:$LOCALFS/$PID $path/root/$LOCALMNTPNT/$PID");
if ($nfsmounts) {
mysystem("mount localhost:$LOCALFS/$PID ".
" $path/root/$LOCALMNTPNT/$PID");
} else {
mysystem("mount -t null $LOCALFS/$PID ".
" $path/root/$LOCALMNTPNT/$PID");
}
push(@mntpoints, "$path/root/$LOCALMNTPNT/$PID");
}
......@@ -479,7 +510,7 @@ sub mkrootfs($)
# but not sure what to do about that.
#
if (! REMOTE()) {
foreach my $dir ( JailedNFSMounts($vnodeid, "$path/root") ) {
foreach my $dir ( JailedMounts($vnodeid, "$path/root", $nfsmounts) ) {
push(@mntpoints, "$path/root/$dir");
}
}
......@@ -525,7 +556,11 @@ sub restorerootfs($)
# Okay, mount some other directories to save space.
#
foreach my $dir (@ROOTMNTDIRS) {
mysystem("mount -r localhost:/$dir $path/root/$dir");
if ($nfsmounts) {
mysystem("mount -r localhost:/$dir $path/root/$dir");
} else {
mysystem("mount -r -t null /$dir $path/root/$dir");
}
push(@mntpoints, "$path/root/$dir");
}
......@@ -535,13 +570,26 @@ sub restorerootfs($)
mysystem("mount -t procfs proc $path/root/proc");
push(@mntpoints, "$path/root/proc");
#
# Must rebuild the routes list cause of swapmodify.
#
if ($jailflags & $JAIL_ROUTING) {
addroutestorc("$path/root/etc/rc.conf.routes");
}
#
# Give the jail an NFS mount of the local project directory. This one
# is read-write.
#
if (defined($PID) && -e $LOCALFS && -e "$LOCALFS/$PID") {
mysystem("mkdir -p $path/root/$LOCALMNTPNT/$PID");
mysystem("mount localhost:$LOCALFS/$PID $path/root/$LOCALMNTPNT/$PID");
if ($nfsmounts) {
mysystem("mount localhost:$LOCALFS/$PID ".
" $path/root/$LOCALMNTPNT/$PID");
} else {
mysystem("mount -t null $LOCALFS/$PID ".
" $path/root/$LOCALMNTPNT/$PID");
}
push(@mntpoints, "$path/root/$LOCALMNTPNT/$PID");
}
......@@ -551,7 +599,7 @@ sub restorerootfs($)
# but not sure what to do about that.
#
if (! REMOTE()) {
foreach my $dir ( JailedNFSMounts($vnodeid, "$path/root") ) {
foreach my $dir ( JailedMounts($vnodeid, "$path/root", $nfsmounts) ) {
push(@mntpoints, "$path/root/$dir");
}
}
......@@ -572,6 +620,7 @@ sub cleanmess($) {
mysystem("rm -f $path/root/etc/emulab.cdkey");
mysystem("rm -f $path/root/etc/emulab.pkey");
# This is /etc/emulab inside the jail.
mysystem("rm -f $path/root/$ETCDIR/*.pem");
mysystem("rm -f $path/root/$ETCDIR/cvsup.auth");
mysystem("rm -rf $path/root/$ETCDIR/.cvsup");
......@@ -878,7 +927,7 @@ sub addroutestorc($rc)
my ($rc) = @_;
my $count = 0;
open(RC, ">>$rc") or
open(RC, ">$rc") or
fatal("Could not open $rc to append static routes");
my $routerip = `cat $BOOTDIR/routerip`;
......
......@@ -22,12 +22,12 @@ use Exporter;
jailsetup dojailconfig JailedMounts findiface
tmccdie tmcctimeout libsetup_getvnodeid dotrafficconfig
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED LOCALROOTFS
TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
TMNICKNAME HOSTSFILE TMSTARTUPCMD FINDIF TMTUNNELCONFIG
TMTRAFFICCONFIG TMROUTECONFIG TMLINKDELAY TMDELMAP TMMOUNTDB
TMPROGAGENTS
TMPROGAGENTS TMPASSDB TMGROUPDB
TMCCCMD_REBOOT TMCCCMD_STATUS TMCCCMD_IFC TMCCCMD_ACCT TMCCCMD_DELAY
TMCCCMD_HOSTS TMCCCMD_RPM TMCCCMD_TARBALL TMCCCMD_STARTUP
......@@ -122,13 +122,17 @@ use liblocsetup;
sub TMCC() { "$BINDIR/tmcc"; }
sub TMHOSTS() { "$ETCDIR/hosts"; }
sub FINDIF() { "$BINDIR/findif"; }
sub LOCALROOTFS() { "/users/local"; }
sub HOSTSFILE() { "/etc/hosts"; }
#
# This path is valid only *outside* the jail when its setup.
#
sub JAILDIR() { "$VARDIR/jails/$vnodeid"; }
#
# Also valid outside the jail, this is where we put local project storage.
#
sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
#
# Okay, here is the path mess. There are three environments.
# 1. A local node where everything goes in one place ($VARDIR/boot).
......@@ -2165,8 +2169,12 @@ sub jailsetup()
print STDOUT "Checking Testbed hostnames configuration ... \n";
dohostnames();
print STDOUT "Checking Testbed group/user configuration ... \n";
doaccounts();
if (REMOTE()) {
# Locally, the password/group files initially comes from
# outside the jail.
print STDOUT "Checking Testbed group/user configuration ... \n";
doaccounts();
}
print STDOUT "Checking Testbed RPM configuration ... \n";
dorpms();
......@@ -2236,6 +2244,8 @@ sub vnodesetup($)
die("*** $0:\n".
" mkdir filed - $eiddir: $!\n");
}
chmod(0777, $piddir);
chmod(0777, $eiddir);
}
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment