Commit 6569a40b authored by Leigh B. Stoller's avatar Leigh B. Stoller
Browse files

A set of changes for CMU widearea nodes. Basically, this cleans up and

integrates some of the early RON support, adding a fake jail setup that
does a lot of what happens inside a jail, but without the actual jail.
Remains to be seen how well this is going to work out.
parent 42ceca6e
......@@ -133,8 +133,7 @@ sub bootvnode($$$)
$opt = "-b";
$act = "Booting";
}
$opt .= " -jVt"
if ($jailed);
$opt .= ($jailed ? " -jVt" : " -i");
print "$act vnode $vnode ...\n";
system("vnodesetup $opt $vnode");
......@@ -152,8 +151,7 @@ if (defined($action) && !$reconfig) {
foreach my $file (@files) {
if ($file =~ /^(pcvm[-\w]*)$/) {
# Assume jailed for now.
bootvnode($1, $action, 1);
bootvnode($1, $action, (-e "$vndir/$file/fakejail" ? 0 : 1));
}
}
exit(0);
......@@ -192,7 +190,7 @@ if ($reconfig) {
foreach my $file (@files) {
if ($file =~ /^(pcvm[-\w]*)$/) {
$curvnodelist{$1} = 1; # XXX Assume jailed.
$curvnodelist{$1} = (-e "$vndir/$file/fakejail" ? 0 : 1);
}
}
......
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -23,7 +23,7 @@ SCRIPTS = $(addprefix $(SRCDIR)/, \
rc.syncserver rc.linkagent rc.mkelab rc.localize \
rc.keys rc.trafgen rc.tarfiles rc.rpms rc.progagent \
rc.startcmd rc.simulator rc.topomap rc.firewall \
rc.tiptunnels rc.trace rc.motelog)
rc.tiptunnels rc.trace rc.motelog rc.fakejail)
include $(OBJDIR)/Makeconf
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004 University of Utah and the Flux Group.
# Copyright (c) 2004, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -44,12 +44,6 @@ use liblocsetup;
use libtmcc;
use librc;
#
# Not all clients support this.
#
exit(0)
if (REMOTE() && !(PLAB() || JAILED()));
# Protos.
sub doboot();
sub doshutdown();
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004, 2005 University of Utah and the Flux Group.
# Copyright (c) 2004-2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -82,6 +82,9 @@ if (MFS()) {
@bootscripts = ("rc.misc", "rc.localize", "rc.mounts", "rc.accounts",
"rc.hostnames", "rc.keys", "rc.tarfiles", "rc.rpms");
}
elsif (FAKEJAILED()) {
@bootscripts = ("rc.misc", "rc.keys", "rc.progagent");
}
elsif (WINDOWS()) {
@bootscripts = ("rc.misc", "rc.localize",
"rc.keys", "rc.mounts",
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
#
# This script is the jail equiv of ../common/rc.bootsetup. It runs
# inside the jail and does a limited set of bootstrapping tasks.
#
sub usage()
{
print "Usage: " . scriptname() . " -j vnodeid\n";
exit(1);
}
my $optlist = "j:";
my $vnodeid;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
# Only root.
if ($EUID != 0) {
die("*** $0:\n".
" Must be root to run this script!\n");
}
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
use libtmcc;
use librc;
# Protos.
sub doboot();
sub doshutdown();
sub BootFatal($);
# Parse command line.
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{'j'})) {
$vnodeid = $options{'j'};
libsetup_setvnodeid($vnodeid);
}
else {
usage();
}
# Script specific goo. Needs to go after we set the vnode above.
my $RCDIR = "$BINDIR/rc";
my $EVPIDFILE = CONFDIR() . "/evproxy.pid";
my $done = 0;
# The caller (vnodesetup) is writing this file. We want to send it back.
my $BOOTLOG = "$LOGDIR/tbvnode-${vnodeid}.log";
sub handler ($) {
my ($signame) = @_;
$SIG{INT} = 'IGNORE';
$SIG{TERM} = 'IGNORE';
$SIG{USR1} = 'IGNORE';
$SIG{HUP} = 'IGNORE';
$done = 1;
}
$SIG{INT} = \&handler;
$SIG{HUP} = \&handler;
$SIG{USR1} = \&handler;
# Do not want to be shutdown except by our parent, when machine reboots.
$SIG{TERM} = 'IGNORE';
# Do the "boot"
doboot();
# And just spin waiting to be killed off
print("Waiting to be killed off\n");
while (!$done) {
sleep(1);
}
# Kill it all off.
doshutdown();
exit(0);
#
# This version of fatal sends boot status to tmcd, and then generates
# a TBFAILED state transition.
#
sub BootFatal($)
{
my ($msg) = @_;
doshutdown();
#
# Send the console log to the server.
#
if (-e $BOOTLOG && -s $BOOTLOG) {
tmcc(TMCCCMD_BOOTLOG, "", undef,
("datafile" => $BOOTLOG, "timeout" => 5));
}
if (tmcc(TMCCCMD_BOOTERRNO, "-1") < 0) {
print "Error sending boot errno to Emulab Control!\n";
}
if (tmcc(TMCCCMD_STATE, "TBFAILED") < 0) {
print "Error sending TBFAILED to Emulab Control!\n";
}
exit(-1);
}
#
# Boot Action.
#
sub doboot()
{
TBDebugTimeStamp("rc.nojail starting up");
my ($pid, $eid, $vname) = fakejailsetup();
#
# At this point, if we are a free node just return. Something went wacky.
#
if (!defined($pid)) {
return;
}
# Get the boss info for below.
my ($bossname, $bossip) = tmccbossinfo();
if (!defined($bossname)) {
BootFatal("Could not determine the name of the boss server!");
}
my ($domain) = ($bossname =~ /^[^\.]+\.(.*)$/);
#
# Start the elvin proxy.
#
if (-x "$BINDIR/evproxy") {
print("Starting elvin proxy daemon\n");
system("$BINDIR/evproxy -i $EVPIDFILE ".
" -s event-server.${domain} -e $pid/$eid");
if ($?) {
BootFatal("Error running $BINDIR/evproxy");
}
}
#
# This is where we run all of the config scripts. These talk to the
# DB and setup the node the way it is supposed to be.
#
TBDebugTimeStamp("rc.injail running config scripts");
print("Running config scripts\n");
system("$RCDIR/rc.config");
if ($?) {
BootFatal("Could not start program agent for $vnodeid");
}
TBDebugTimeStamp("rc.nojail done running config scripts");
print("Informing the testbed that we are up and running\n");
if (tmcc(TMCCCMD_STATE, "ISUP") < 0) {
BootFatal("Error sending ISUP to Emulab Control!");
}
#
# Send the console log to the server.
#
if (-e $BOOTLOG && -s $BOOTLOG) {
tmcc(TMCCCMD_BOOTLOG, "", undef,
("datafile" => $BOOTLOG, "timeout" => 5));
}
}
#
# Shutdown Action.
#
sub doshutdown()
{
print("Running termination scripts\n");
system("$RCDIR/rc.config shutdown");
if (-e $EVPIDFILE) {
my $mpid = `cat $EVPIDFILE`;
chomp($mpid);
# untaint
if ($mpid =~ /^([\d]+)$/) {
$mpid = $1;
print("Killing the event proxy\n");
if (kill('TERM', $mpid) == 0) {
print "*** Could not kill(TERM) process $mpid: $!\n";
return -1;
}
#
# Wait for the pidfile to be removed. Do not wait too long though.
#
my $i = 15;
while (-e $EVPIDFILE && $i > 0) {
sleep(1);
$i -= 1;
}
if (-e $EVPIDFILE) {
print "*** Not able to kill evproxy process $mpid!\n";
}
}
}
}
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004, 2005, 2006 University of Utah and the Flux Group.
# Copyright (c) 2004-2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -24,14 +24,6 @@ $| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
# Script specific goo
my $LOGFILE = "$LOGDIR/progagent.debug";
my $WRAPLOG = "$LOGDIR/progwrap.debug";
my $PIDFILE = "/var/run/progagent.pid";
my $CONFIG = "$BOOTDIR/progagents";
my $PAGENT = "$BINDIR/program-agent";
my $TOKEN = "/var/tmp/progagent-token";
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
......@@ -53,7 +45,16 @@ if ($EUID != 0) {
# Not all clients support this.
#
exit(0)
if (MFS() || REMOTE() && !PLAB());
if (MFS() || (REMOTE() && !(PLAB() || FAKEJAILED() || JAILED())));
# Script specific goo.
my $LOGFILE = LOGDIR() . "/progagent.debug";
my $WRAPLOG = LOGDIR() . "/progwrap.debug";
my $PIDFILE = (FAKEJAILED() ?
CONFDIR() . "/progagent.pid" : "/var/run/progagent.pid");
my $CONFIG = CONFDIR() . "/progagents";
my $PAGENT = "$BINDIR/program-agent";
my $TOKEN = "/var/tmp/progagent-token";
# Protos.
sub doboot();
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004, 2005, 2006 University of Utah and the Flux Group.
# Copyright (c) 2004, 2005, 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -41,7 +41,7 @@ if ($EUID != 0 && !WINDOWS()) {
# Not all clients support this.
#
exit(0)
if (MFS());
if (MFS() || (REMOTE() && !(PLAB() || JAILED())));
# Protos.
sub doboot($);
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
# TODO: Signal handlers for protecting db files.
......@@ -20,15 +20,15 @@ use Exporter;
TBBackGround TBForkCmd vnodejailsetup plabsetup vnodeplabsetup
jailsetup dojailconfig findiface libsetup_getvnodeid
ixpsetup libsetup_refresh gettopomap getfwconfig gettiptunnelconfig
gettraceconfig genhostsfile getmotelogconfig calcroutes
gettraceconfig genhostsfile getmotelogconfig calcroutes fakejailsetup
TBDebugTimeStamp TBDebugTimeStampsOn
MFS REMOTE CONTROL WINDOWS JAILED PLAB LOCALROOTFS IXP USESFS
SIMTRAFGEN SIMHOST ISDELAYNODEPATH JAILHOST DELAYHOST STARGATE
ISFW
ISFW FAKEJAILED
CONFDIR TMDELAY TMJAILNAME TMSIMRC TMCC
CONFDIR LOGDIR TMDELAY TMJAILNAME TMSIMRC TMCC
TMNICKNAME TMSTARTUPCMD FINDIF
TMROUTECONFIG TMLINKDELAY TMDELMAP TMTOPOMAP TMLTMAP TMLTPMAP
TMGATEDCONFIG TMSYNCSERVER TMKEYHASH TMNODEID TMEVENTKEY
......@@ -86,6 +86,11 @@ sub libsetup_getvnodeid()
#
my $injail;
#
# True if running as a fake jail (no jail, just processes).
#
my $nojail;
#
# True if running in a Plab vserver.
#
......@@ -131,9 +136,13 @@ BEGIN
libsetup_setvnodeid($vid);
$injail = 1;
}
# Determine if running inside a Plab vserver.
if (-e "$BOOTDIR/plabname") {
elsif (exists($ENV{'FAKEJAIL'})) {
# Fake jail.
libsetup_setvnodeid($ENV{'FAKEJAIL'});
$nojail = 1;
}
elsif (-e "$BOOTDIR/plabname") {
# Running inside a Plab vserver.
open(VN, "$BOOTDIR/plabname");
my $vid = <VN>;
close(VN);
......@@ -186,24 +195,31 @@ sub LOCALROOTFS() { (REMOTE() ? "/users/local" : "$VARDIR/jails/local");}
# 2. A virtual node inside a jail or a Plab vserver ($VARDIR/boot).
# 3. A virtual (or sub) node, from the outside.
#
# As for #3, whether setting up a old-style virtual node or a new style
# As for #3, whether setting up a old-style (fake) virtual node or a new style
# jailed node, the code that sets it up needs a different per-vnode path.
#
sub CONFDIR() {
if ($injail || $inplab) {
return $BOOTDIR;
}
if ($vnodeid) {
return JAILDIR();
}
return $BOOTDIR
if ($injail || $inplab);
return JAILDIR()
if ($vnodeid);
return $BOOTDIR;
}
# Cause of fakejails, we want log files in the right place.
sub LOGDIR() {
return $LOGDIR
if ($injail || $inplab);
return JAILDIR()
if ($vnodeid);
return $LOGDIR;
}
#
# The rest of these depend on the environment running in (inside/outside jail).
#
sub TMNICKNAME() { CONFDIR() . "/nickname";}
sub TMJAILNAME() { CONFDIR() . "/jailname";}
sub TMFAKEJAILNAME() { CONFDIR() . "/fakejail";}
sub TMJAILCONFIG() { CONFDIR() . "/jailconfig";}
sub TMSTARTUPCMD() { CONFDIR() . "/startupcmd";}
sub TMROUTECONFIG() { CONFDIR() . "/rc.route";}
......@@ -275,6 +291,7 @@ sub STARGATE() { if (-e "$ETCDIR/isstargate") { return 1; } else { return 0; }
# Are we jailed? See above.
#
sub JAILED() { if ($injail) { return $vnodeid; } else { return 0; } }
sub FAKEJAILED(){ if ($nojail) { return $vnodeid; } else { return 0; } }
#
# Are we on plab?
......@@ -1574,6 +1591,41 @@ sub jailsetup()
return ($pid, $eid, $vname);
}
#
# Bogus emulation of jails without a jail,
#
sub fakejailsetup()
{
$nojail = 1;
# Stick this into the environment so that sub processes know.
$ENV{'FAKEJAIL'} = $vnodeid;
#
# Create a file inside so that libsetup inside the jail knows its
# inside a jail and what its ID is.
#
system("echo '$vnodeid' > " . TMFAKEJAILNAME());
# Need to unify this with jailname.
system("echo '$vnodeid' > " . TMNODEID());
#
# Always remove the old nickname file. No need to worry about a project
# change at this level (see bootsetup) but we do need to make sure we
# pick up on a vnode/jail being reassigned to a different virtual node.
#
unlink TMNICKNAME;
print STDOUT "Checking Testbed reservation status ... \n";
if (! check_status()) {
print STDOUT " Free!\n";
return 0;
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
return ($pid, $eid, $vname);
}
#
# Remote Node virtual node jail setup. This happens outside the jailed
# env.
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -32,7 +32,7 @@ use Exporter;
TMCCCMD_BOOTERRNO TMCCCMD_BOOTLOG TMCCCMD_BATTERY TMCCCMD_USERENV
TMCCCMD_TIPTUNNELS TMCCCMD_TRACEINFO TMCCCMD_ELVINDPORT
TMCCCMD_PLABEVENTKEYS TMCCCMD_PORTREGISTER
TMCCCMD_MOTELOG
TMCCCMD_MOTELOG TMCCCMD_BOOTWHAT
);
# Must come after package declaration!
......@@ -176,6 +176,7 @@ my %commandset =
"plabeventkeys" => {TAG => "plabeventkeys"},
"motelog" => {TAG => "motelog"},
"portregister" => {TAG => "portregister"},
"bootwhat" => {TAG => "bootwhat"},
);
#
......@@ -233,7 +234,8 @@ sub TMCCCMD_TRACEINFO (){ $commandset{"traceinfo"}->{TAG}; }
sub TMCCCMD_ELVINDPORT (){ $commandset{"elvindport"}->{TAG}; }
sub TMCCCMD_PLABEVENTKEYS(){ $commandset{"plabeventkeys"}->{TAG}; }
sub TMCCCMD_MOTELOG() { $commandset{"motelog"}->{TAG}; }
sub TMCCCMD_PORTREGISTER() { $commandset{"portregister"}->{TAG}; }
sub TMCCCMD_PORTREGISTER(){ $commandset{"portregister"}->{TAG}; }
sub TMCCCMD_BOOTWHAT() { $commandset{"bootwhat"}->{TAG}; }
#
# Caller uses this routine to set configuration of this library
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004, 2005, 2006 University of Utah and the Flux Group.
# Copyright (c) 2004, 2005, 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -33,10 +33,6 @@ if ($EUID != 0) {
" Must be root to run this script!\n");
}
# Script specific goo.
my $RCDIR = "$BINDIR/rc";
my $LOGFILE = "$LOGDIR/bootsetup.debug";
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
......@@ -54,6 +50,12 @@ if (MFS()) {
" Could not exec $RCDIR/rc.mfs!\n");
}
# Script specific goo.
my $RCDIR = "$BINDIR/rc";
my $LOGFILE = "$LOGDIR/bootsetup.debug";
my $BOOTLOG = "/var/log/emulab.bootlog";
my %TMCCTIMEOUT = (REMOTE() ? ("timeout" => 5) : ());
# Protos.
sub doboot();
sub doshutdown();
......@@ -73,10 +75,29 @@ if (@ARGV) {
$action = $ARGV[0];
}
#
# Booting up. If this is a widearea node, we do not want to hang
# on boss if its down, but instead let the machine boot and do this
# stuff later.
#
if (REMOTE() && $action eq "boot") {
require POSIX;
import POSIX;
if (TBBackGround($BOOTLOG)) {
sleep(1);
exit(0);
}
# Fully disconnect from bootup.
POSIX::setsid();
# Let the boot continue on for a bit.
sleep(5);
}
#
# We want to save all of the output off, but also dup it to the console.
#
if ($action eq "boot" || $action eq "reconfig") {
#
# We want to save all of the output off, but also dup it to the console.
#
open(LOG, "> $LOGFILE") or
BootFatal(-1, "Could not open $LOGFILE!");
my $ofh = select(LOG);
......@@ -107,9 +128,14 @@ if ($action eq "boot" || $action eq "reconfig") {
exit(0);
}
# Need this or ssh will hang, even though it was called with -n option.
open(STDIN, "</dev/null");
open(STDERR, ">&STDOUT");
if ($action eq "reconfig") {
# Need this or ssh will hang, even though it was called with -n option.
open(STDIN, "</dev/null");
}
if (REMOTE()) {
# Maybe just do this all the time.
open(STDERR, ">&STDOUT");
}
}
# Execute the action.
......@@ -146,15 +172,16 @@ sub BootFatal($$)
# Send the console log to the server.
#
if (-e $LOGFILE && -s $LOGFILE &&
tmcc(TMCCCMD_BOOTLOG, "", undef, ("datafile" => $LOGFILE)) < 0) {
tmcc(TMCCCMD_BOOTLOG, "", undef,
("datafile" => $LOGFILE, "timeout" => 5)) < 0) {
print "Error sending TBFAILED to Emulab Control!\n";
}
if (tmcc(TMCCCMD_BOOTERRNO, $code) < 0) {
if (tmcc(TMCCCMD_BOOTERRNO, $code, undef, %TMCCTIMEOUT) < 0) {
print "Error sending boot errno to Emulab Control!\n";