Commit f3beec6e authored by Austin Clements's avatar Austin Clements

Plab node setup now basically works. There are just a few bugs to

iron out.  TMCD/libsetup now has a plabconfig commands that parallels
the jailconfig command.  The Plab boot process has been added to
libsetup and a -p option has been added to vnodesetup to parallel the
-j option.  Parts of the code that were Jail-specific, but labeled
just as vnode stuff have been renamed.  $vnodedir in vnodesetup has
been removed, since it was redunant with libsetup's CONFDIR, and
CONFDIR is much more intelligent.
parent 49f9aed2
......@@ -18,13 +18,13 @@ use Exporter;
doifconfig dohostnames domounts dotunnels check_nickname
doaccounts dorpms dotarballs dostartupcmd install_deltas
bootsetup nodeupdate startcmdstatus whatsmynickname dosyncserver
TBBackGround TBForkCmd vnodesetup dorouterconfig
jailsetup dojailconfig JailedMounts findiface
TBBackGround TBForkCmd vnodejailsetup plabsetup vnodeplabsetup
dorouterconfig jailsetup dojailconfig JailedMounts findiface
tmccdie tmcctimeout libsetup_getvnodeid dotrafficconfig
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED LOCALROOTFS
TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
CONFDIR TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
TMNICKNAME HOSTSFILE TMSTARTUPCMD FINDIF TMTUNNELCONFIG
TMTRAFFICCONFIG TMROUTECONFIG TMLINKDELAY TMDELMAP TMMOUNTDB
TMPROGAGENTS TMPASSDB TMGROUPDB
......@@ -58,6 +58,11 @@ sub libsetup_getvnodeid()
#
my $injail;
#
# True if running in a Plab vserver.
#
my $inplab;
# Load up the paths. Its conditionalized to be compatabile with older images.
# Note this file has probably already been loaded by the caller.
BEGIN
......@@ -85,6 +90,21 @@ BEGIN
$injail = 1;
}
# Determine if running inside a Plab vserver.
if (-e "$BOOTDIR/plabname") {
open(VN, "$BOOTDIR/plabname");
$vnodeid = <VN>;
close(VN);
if ($vnodeid =~ /^([-\w]+)$/) {
$vnodeid = $1;
}
else {
die("Bad data in vnodeid: $vnodeid");
}
$inplab = 1;
}
# Make sure these exist!
if (! -e "$VARDIR/logs") {
mkdir("$VARDIR", 0775);
......@@ -136,14 +156,14 @@ sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
#
# Okay, here is the path mess. There are three environments.
# 1. A local node where everything goes in one place ($VARDIR/boot).
# 2. A virtual node inside a jail ($VARDIR/boot).
# 2. A virtual node inside a jail or a Plab vserver ($VARDIR/boot).
# 3. A virtual node outside a jail (JAILDIR()).
#
# As for #3, whether setting up a old-style virtual node or a new style
# jailed node, the code that sets it up needs a different per-vnode path.
#
sub CONFDIR() {
if ($injail) {
if ($injail || $inplab) {
return $BOOTDIR;
}
if ($vnodeid) {
......@@ -165,6 +185,7 @@ sub TMGROUPDB() { $VARDIR . "/db/groupdb"; }
sub TMNICKNAME() { CONFDIR() . "/nickname";}
sub TMJAILNAME() { CONFDIR() . "/jailname";}
sub TMJAILCONFIG() { CONFDIR() . "/jailconfig";}
sub TMPLABCONFIG() { CONFDIR() . "/rc.plab";}
sub TMSTARTUPCMD() { CONFDIR() . "/startupcmd";}
sub TMPROGAGENTS() { CONFDIR() . "/progagents";}
sub TMIFC() { CONFDIR() . "/rc.ifc"; }
......@@ -222,6 +243,7 @@ sub TMCCCMD_ISALIVE() { "isalive"; }
sub TMCCCMD_SFSHOSTID() { "sfshostid"; }
sub TMCCCMD_SFSMOUNTS() { "sfsmounts"; }
sub TMCCCMD_JAILCONFIG(){ "jailconfig"; }
sub TMCCCMD_PLABCONFIG(){ "plabconfig"; }
sub TMCCCMD_LINKDELAYS(){ "linkdelays"; }
sub TMCCCMD_PROGRAMS() { "programs"; }
sub TMCCCMD_SYNCSERVER(){ "syncserver"; }
......@@ -269,6 +291,11 @@ sub CONTROL() { if (-e "$ETCDIR/isctrl") { return 1; } else { return 0; } }
#
sub JAILED() { if ($injail) { return $vnodeid; } else { return 0; } }
#
# Are we on plab?
#
sub PLAB() { if ($inplab) { return $vnodeid; } else { return 0; } }
#
# Do not try this on the MFS since it has such a wimpy perl installation.
#
......@@ -310,6 +337,10 @@ sub OPENTMCC($;$$)
die("\n") if $tmccdie;
return undef;
}
# XXX For debugging
print STDERR "$foo\n";
return (*TM);
}
......@@ -1149,7 +1180,7 @@ sub doaccounts()
#
my $gname = "$1";
if (REMOTE() && !JAILED()) {
if (REMOTE() && !JAILED() && !PLAB()) {
$gname = "emu-${gname}";
}
$newgroups{"$gname"} = $2
......@@ -1397,6 +1428,15 @@ sub doaccounts()
warn "*** WARNING: Error adding new user $login\n";
next;
}
if (PLAB() && ! -e $hdir) {
if (! os_mkdir($hdir, "0755")) {
warn "*** WARNING: Error creating user homedir\n";
next;
}
chown($login, $login, $hdir);
}
# Add to DB only if successful.
$PWDDB{$login} = "$uid:$serial";
}
......@@ -1567,7 +1607,8 @@ sub dotarballs ()
{
my @tarballs = ();
my $jailoption = (JAILED() ? "-j" : "");
# XXX Plab option?
my $TM = OPENTMCC(TMCCCMD_TARBALL);
while (<$TM>) {
push(@tarballs, $_);
......@@ -2073,6 +2114,72 @@ sub dosyncserver()
return 0;
}
#
# Plab configuration. Currently sets up sshd and the DNS resolver
#
sub doplabconfig()
{
my $plabconfig;
my $TM = OPENTMCC(TMCCCMD_PLABCONFIG);
$_ = <$TM>;
if (defined($_)) {
$plabconfig = $_;
}
CLOSETMCC($TM);
if (! $plabconfig) {
return 0;
}
open(RC, ">" . TMPLABCONFIG)
or die("Could not open " . TMPLABCONFIG . ": $!");
if ($plabconfig =~ /SSHDPORT=(\d+)/) {
my $sshdport = $1;
print RC "#!/bin/sh\n";
# Note that it's important to never directly modify the config
# file unless it's already been recreated due to vserver's
# immutable-except-delete flag
print(RC
"function setconfigopt()\n".
"{\n".
" file=\$1\n".
" opt=\$2\n".
" value=\$3\n".
" if ( ! grep -q \"^\$opt[ \t]*\$value\\\$\" \$file ); then\n".
" sed -e \"s/^\\(\$opt[ \t]*.*\\)/#\\1/\" < \$file".
" > \$file.tmp\n".
" mv -f \$file.tmp \$file\n".
" echo \$opt \$value >> \$file;\n".
" fi\n".
"}\n\n");
# Make it look like it's in Emulab domain
print RC "setconfigopt /etc/resolv.conf domain emulab.net\n";
print RC "setconfigopt /etc/resolv.conf search emulab.net\n\n";
# No SSH X11 Forwarding
print RC "setconfigopt /etc/ssh/sshd_config X11Forwarding no\n";
# Set SSH port
print RC "setconfigopt /etc/ssh/sshd_config Port $sshdport\n";
# Start sshd
print RC "/etc/init.d/sshd restart\n";
}
else {
warn "*** WARNING: Bad plab line: $_";
}
close(RC);
chmod(0755, TMPLABCONFIG);
return 0;
}
#
# Boot Startup code. This is invoked from the setup OS dependent script,
# and this fires up all the stuff above.
......@@ -2269,9 +2376,10 @@ sub jailsetup()
}
#
# Remote Node virtual node setup. This happens outside the jailed env.
# Remote Node virtual node jail setup. This happens outside the jailed
# env.
#
sub vnodesetup($)
sub vnodejailsetup($)
{
my ($vid) = @_;
......@@ -2327,6 +2435,100 @@ sub vnodesetup($)
return ($pid, $eid, $vname);
}
#
# This happens inside a Plab vserver.
#
sub plabsetup()
{
#
# vnodeid will either be found in BEGIN block or will be passed to
# vnodeplabsetup, so it doesn't need to be found here
#
#
# Do account stuff.
#
{
print STDOUT "Checking Testbed reservation status ... \n";
if (! check_status()) {
print STDOUT " Free!\n";
return 0;
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
#
# Setup SFS hostid.
#
if ($USESFS) {
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
# print STDOUT "Mounting project and home directories ... \n";
# domounts();
print STDOUT "Checking Testbed plab configuration ...\n";
doplabconfig();
print STDOUT "Checking Testbed hostnames configuration ... \n";
dohostnames();
print STDOUT "Checking Testbed group/user configuration ... \n";
doaccounts();
print STDOUT "Checking Testbed RPM configuration ... \n";
dorpms();
print STDOUT "Checking Testbed Tarball configuration ... \n";
dotarballs();
# print STDOUT "Checking Testbed routing configuration ... \n";
# dorouterconfig();
# print STDOUT "Checking Testbed traffic generation configuration ...\n";
# dotrafficconfig();
# print STDOUT "Checking Testbed program agent configuration ... \n";
# doprogagent();
print STDOUT "Checking Testbed Experiment Startup Command ... \n";
dostartupcmd();
}
return $vnodeid;
}
#
# Remote node virtual node Plab setup. This happens inside the vserver
# environment (because on Plab you can't escape)
#
sub vnodeplabsetup($)
{
my ($vid) = @_;
#
# Set global vnodeid for tmcc commands.
#
$vnodeid = $vid;
$inplab = 1;
# Do not bother if somehow got released.
if (! check_status()) {
print "Node is free!\n";
return undef;
}
#
# Create a file so that libsetup knows it's inside Plab and what
# its ID is.
#
system("echo '$vnodeid' > $BOOTDIR/plabname");
# XXX Anything else to do?
return ($pid, $eid, $vname);
}
#
# Report startupcmd status back to the TMCC. Called by the runstartup
# script.
......
......@@ -23,11 +23,11 @@ BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
sub usage()
{
print "Usage: vnodesetup [-j [-s]] [-b | -k | -r | -h] [-d] <vnodeid>\n".
print "Usage: vnodesetup [-j [-s] | -p] [-b | -k | -r | -h] [-d] <vnodeid>\n".
"Use the -k option to kill the virtual node.\n";
exit(1);
}
my $optlist = "kbdjsrh";
my $optlist = "kbdjsrhp";
# Locals
my $killit = 0;
......@@ -36,6 +36,7 @@ my $haltit = 0;
my $debug = 0;
my $fromboot = 0;
my $dojail = 0;
my $doplab = 0;
my $interactive = 0;
my $cleaning = 0;
my $rebooting = 0;
......@@ -59,6 +60,9 @@ use libsetup;
sub killvnode();
sub rebootvnode();
sub cleanup();
sub killvserver();
sub fatal($);
sub removeconfdir($);
#
# Must be root.
......@@ -111,6 +115,9 @@ if (defined($options{"j"})) {
$interactive = 1;
}
}
if (defined($options{"p"})) {
$doplab = 1;
}
if (@ARGV != 1) {
usage();
}
......@@ -123,11 +130,14 @@ else {
die("Bad data in vnodeid: $vnodeid.");
}
if ($dojail && $doplab) {
die("Can't do both jail and Plab");
}
#
# Hacky. All this path stuff is hacky.
#
my $pidfile = "/var/run/tbvnode-${vnodeid}.pid";
my $vnodedir = "/var/emulab/jails/$vnodeid";
my $logname = "$LOGDIR/tbvnode-${vnodeid}.log";
#
......@@ -184,7 +194,7 @@ if (!$debug && !$interactive && TBBackGround($logname)) {
#
if ($dojail) {
my $now = time();
my $goofy = "$vnodedir/root/var/run/emulab-watchdog.pid";
my $goofy = CONFDIR() . "/root/var/run/emulab-watchdog.pid";
my $count = 30;
while ($count--) {
......@@ -260,13 +270,13 @@ $SIG{USR2} = \&handler;
#
# Kill existing directory in case its still there.
#
if (!$fromboot && -e $vnodedir) {
removevnodedir($vnodedir);
if (!$fromboot && -e CONFDIR()) {
removeconfdir(CONFDIR());
}
if (! -e $vnodedir) {
mkdir($vnodedir, 0755) or
if (! -e CONFDIR()) {
mkdir(CONFDIR(), 0755) or
die("*** $0:\n".
" Could not mkdir $vnodedir: $!\n");
" Could not mkdir ".CONFDIR().": $!\n");
}
#
......@@ -276,16 +286,23 @@ REBOOT:
system("tmcc -n $vnodeid state BOOTING");
#
# Invoke remotevnodesetup routine in the setup library. This will talk
# to tmcd and create the rc files.
# Invoke vnode setup routine in the setup library. This will talk to
# tmcd and create the rc files.
#
my ($pid, $eid, $vname) = vnodesetup($vnodeid);
my ($pid, $eid, $vname);
if ($doplab) {
($pid, $eid, $vname) = vnodeplabsetup($vnodeid);
}
else {
($pid, $eid, $vname) = vnodejailsetup($vnodeid);
}
if (!defined($pid)) {
#
# Hmm, suddenly got free.
#
#
system("rm -f $pidfile");
removevnodedir($vnodedir);
removeconfdir(CONFDIR());
exit(0);
}
......@@ -295,10 +312,10 @@ if (!defined($pid)) {
$UID = 0;
#
# This stuff is done only when the node is not in a jail. The jail setup
# code does all this, so we avoid duplication of effort.
# This stuff is done only when the node is not in a jail or on Plab. The
# jail/Plab setup code does all this, so we avoid duplication of effort.
#
if (!$dojail) {
if (!$dojail && !$doplab) {
print STDOUT "Checking Testbed tunnel configuration ... \n";
dotunnels();
......@@ -363,6 +380,34 @@ if ($dojail) {
" Could not start the jail!\n");
}
}
elsif ($doplab) {
# Already running inside vserver, so there is no need to create it,
# just "boot" the running vserver
$vserverpid = fork();
if ($vserverpid) {
#
# Parent waits for reboot/halt/kill signal
#
while (1) {
sleep 5;
if ($rebooting) {
# Kill all processes except me
killvserver();
$rebooting = 0;
goto REBOOT;
}
}
# kill or halt (equivalent in vserver)
cleanup();
}
else {
exec("$BINDIR/rc.inplab");
die("*** $0:\n".
" Could not start vserver setup!\n");
}
}
else {
#
# Inform the TMCD we are ready.
......@@ -463,12 +508,19 @@ sub cleanup()
undef($jailpid);
}
#
# Kill off vserver
#
if (defined($vserverpid)) {
killvserver();
}
$SIG{TERM} = 'IGNORE';
kill('TERM', -$pgrp);
print "Waiting 5 seconds for process group to die off ...\n";
sleep(5);
if (! $leavejail) {
removevnodedir($vnodedir);
removeconfdir(CONFDIR());
}
system("rm -f $pidfile");
}
......@@ -499,6 +551,38 @@ sub reboot()
$SIG{TERM} = \&handler;
}
#
# Simulate shutting down a vserver by killing all processes in it (except
# for me and other processes of a similar persuation, that would be silly)
#
sub killvserver()
{
print "Killing all processes in vserver ...\n";
foreach my $signal ('TERM', 'TERM', 'TERM', 'KILL') {
# Works on Linux ps, but isn't portable
my @processes = `ps -eo pid,comm --no-headers`;
foreach my $process (@processes) {
if ($process =~ /\s*([0-9]*)\s*(.*)/) {
$procpid = $1;
$procname = $2;
if ($procpid == 1 || $procpid == $PID) { next; }
if ($procname eq 'vnodesetup') { next; }
print "Sending $signal to $procpid ($procname)\n";
kill($signal, $procpid);
}
else {
warn("*** Bad ps line: $process\n");
}
}
sleep 1;
}
}
#
# Print error and exit.
#
......@@ -514,7 +598,7 @@ sub fatal($)
#
# Ug, with NFS mounts inside the jail, we need to be really careful.
#
sub removevnodedir($)
sub removeconfdir($)
{
my ($dir) = @_;
......
......@@ -18,13 +18,13 @@ use Exporter;
doifconfig dohostnames domounts dotunnels check_nickname
doaccounts dorpms dotarballs dostartupcmd install_deltas
bootsetup nodeupdate startcmdstatus whatsmynickname dosyncserver
TBBackGround TBForkCmd vnodesetup dorouterconfig
jailsetup dojailconfig JailedMounts findiface
TBBackGround TBForkCmd vnodejailsetup plabsetup vnodeplabsetup
dorouterconfig jailsetup dojailconfig JailedMounts findiface
tmccdie tmcctimeout libsetup_getvnodeid dotrafficconfig
OPENTMCC CLOSETMCC RUNTMCC MFS REMOTE JAILED LOCALROOTFS
TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
CONFDIR TMCC TMIFC TMDELAY TMRPM TMTARBALLS TMHOSTS TMJAILNAME
TMNICKNAME HOSTSFILE TMSTARTUPCMD FINDIF TMTUNNELCONFIG
TMTRAFFICCONFIG TMROUTECONFIG TMLINKDELAY TMDELMAP TMMOUNTDB
TMPROGAGENTS TMPASSDB TMGROUPDB
......@@ -58,6 +58,11 @@ sub libsetup_getvnodeid()
#
my $injail;
#
# True if running in a Plab vserver.
#
my $inplab;
# Load up the paths. Its conditionalized to be compatabile with older images.
# Note this file has probably already been loaded by the caller.
BEGIN
......@@ -85,6 +90,21 @@ BEGIN
$injail = 1;
}
# Determine if running inside a Plab vserver.
if (-e "$BOOTDIR/plabname") {
open(VN, "$BOOTDIR/plabname");
$vnodeid = <VN>;
close(VN);
if ($vnodeid =~ /^([-\w]+)$/) {
$vnodeid = $1;
}
else {
die("Bad data in vnodeid: $vnodeid");
}
$inplab = 1;
}
# Make sure these exist!
if (! -e "$VARDIR/logs") {
mkdir("$VARDIR", 0775);
......@@ -136,14 +156,14 @@ sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
#
# Okay, here is the path mess. There are three environments.
# 1. A local node where everything goes in one place ($VARDIR/boot).
# 2. A virtual node inside a jail ($VARDIR/boot).
# 2. A virtual node inside a jail or a Plab vserver ($VARDIR/boot).
# 3. A virtual node outside a jail (JAILDIR()).
#
# As for #3, whether setting up a old-style virtual node or a new style
# jailed node, the code that sets it up needs a different per-vnode path.
#
sub CONFDIR() {
if ($injail) {
if ($injail || $inplab) {
return $BOOTDIR;
}
if ($vnodeid) {
......@@ -165,6 +185,7 @@ sub TMGROUPDB() { $VARDIR . "/db/groupdb"; }
sub TMNICKNAME() { CONFDIR() . "/nickname";}
sub TMJAILNAME() { CONFDIR() . "/jailname";}
sub TMJAILCONFIG() { CONFDIR() . "/jailconfig";}
sub TMPLABCONFIG() { CONFDIR() . "/rc.plab";}
sub TMSTARTUPCMD() { CONFDIR() . "/startupcmd";}
sub TMPROGAGENTS() { CONFDIR() . "/progagents";}
sub TMIFC() { CONFDIR() . "/rc.ifc"; }
......@@ -222,6 +243,7 @@ sub TMCCCMD_ISALIVE() { "isalive"; }
sub TMCCCMD_SFSHOSTID() { "sfshostid"; }
sub TMCCCMD_SFSMOUNTS() { "sfsmounts"; }
sub TMCCCMD_JAILCONFIG(){ "jailconfig"; }
sub TMCCCMD_PLABCONFIG(){ "plabconfig"; }
sub TMCCCMD_LINKDELAYS(){ "linkdelays"; }
sub TMCCCMD_PROGRAMS() { "programs"; }
sub TMCCCMD_SYNCSERVER(){ "syncserver"; }
......@@ -269,6 +291,11 @@ sub CONTROL() { if (-e "$ETCDIR/isctrl") { return 1; } else { return 0; } }
#
sub JAILED() { if ($injail) { return $vnodeid; } else { return 0; } }
#
# Are we on plab?
#
sub PLAB() { if ($inplab) { return $vnodeid; } else { return 0; } }
#
# Do not try this on the MFS since it has such a wimpy perl installation.
#
......@@ -310,6 +337,10 @@ sub OPENTMCC($;$$)
die("\n") if $tmccdie;
return undef;
}
# XXX For debugging
print STDERR "$foo\n";
return (*TM);
}
......@@ -1149,7 +1180,7 @@ sub doaccounts()
#
my $gname = "$1";
if (REMOTE() && !JAILED()) {
if (REMOTE() && !JAILED() && !PLAB()) {
$gname = "emu-${gname}";
}
$newgroups{"$gname"} = $2
......@@ -1397,6 +1428,15 @@ sub doaccounts()
warn "*** WARNING: Error adding new user $login\n";
next;
}
if (PLAB() && ! -e $hdir) {
if (! os_mkdir($hdir, "0755")) {
warn "*** WARNING: Error creating user homedir\n";
next;
}
chown($login, $login, $hdir);
}
# Add to DB only if successful.
$PWDDB{$login} = "$uid:$serial";
}
......@@ -1567,7 +1607,8 @@ sub dotarballs ()
{
my @tarballs = ();
my $jailoption = (JAILED() ? "-j" : "");
# XXX Plab option?
my $TM = OPENTMCC(TMCCCMD_TARBALL);
while (<$TM>) {
push(@tarballs, $_);
......@@ -2073,6 +2114,72 @@ sub dosyncserver()
return 0;
}
#
# Plab configuration. Currently sets up sshd and the DNS resolver
#
sub doplabconfig()
{
my $plabconfig;
my $TM = OPENTMCC(TMCCCMD_PLABCONFIG);
$_ = <$TM>;
if (defined($_)) {
$plabconfig = $_;
}
CLOSETMCC($TM);
if (! $plabconfig) {
return 0;
}
open(RC, ">" . TMPLABCONFIG)
or die("Could not open " . TMPLABCONFIG . ": $!");
if ($plabconfig =~ /SSHDPORT=(\d+)/) {
my $sshdport = $1;
print RC "#!/bin/sh\n";
# Note that it's important to never directly modify the config
# file unless it's already been recreated due to vserver's
# immutable-except-delete flag
print(RC
"function setconfigopt()\n".
"{\n".
" file=\$1\n".
" opt=\$2\n".
" value=\$3\n".
" if ( ! grep -q \"^\$opt[ \t]*\$value\\\$\" \$file ); then\n".
" sed -e \"s/^\\(\$opt[ \t]*.*\\)/#\\1/\" < \$file".
" > \$file.tmp\n".
" mv -f \$file.tmp \$file\n".
" echo \$opt \$value >> \$file;\n".
" fi\n".
"}\n\n");
# Make it look like it's in Emulab domain
print RC "setconfigopt /etc/resolv.conf domain emulab.net\n";
print RC "setconfigopt /etc/resolv.conf search emulab.net\n\n";
# No SSH X11 Forwarding
print RC "setconfigopt /etc/ssh/sshd_config X11Forwarding no\n";
# Set SSH port
print RC "setconfigopt /etc/ssh/sshd_config Port $sshdport\n";
# Start sshd
print RC "/etc/init.d/sshd restart\n";
}
else {
warn "*** WARNING: Bad plab line: $_";
}
close(RC);
chmod(0755, TMPLABCONFIG);
return 0;
}