Commit 2b72f2c9 authored by Leigh B. Stoller's avatar Leigh B. Stoller
Browse files

Brave new world of tmcc client side caching. The goal is to reduce the

number of connections to tmcd, and the resulting number of DB queries.
Currently thats about 24 per node when it boots. Each vnode adds
another 24 or so. The new approach is to use the "fullconfig" command,
which dumps the entire config in one shot, saving about 20 of those
connections. We still need to do the status/state commands for real of
course. When a node boots, it requests the fullconfig; the client side
takes this fullconfig, and dumps the individual sections to
/var/emulab/boot/tmcc/section_name. Subsequent requests first look for
it locally in the above named files, falling back to real tmcc if none
exists. The update command also refreshes the cache.

Tested for jails and plab node vservers as well.
parent 82d87bdb
......@@ -21,7 +21,7 @@ use Exporter;
TBBackGround TBForkCmd vnodejailsetup plabsetup vnodeplabsetup
dorouterconfig jailsetup dojailconfig JailedMounts findiface
tmcctimeout libsetup_getvnodeid dotrafficconfig
ixpsetup dokeyhash donodeid
ixpsetup dokeyhash donodeid libsetup_refresh
MFS REMOTE JAILED PLAB LOCALROOTFS IXP
......@@ -50,6 +50,9 @@ libtmcc::configtmcc("version", TMCD_VERSION());
# Control tmcc timeout.
sub libsetup_settimeout($) { libtmcc::configtmcc("timeout", $_[0]); };
# Redresh tmcc cache.
sub libsetup_refresh() { libtmcc::tmccgetconfig(); };
#
# For virtual (multiplexed nodes). If defined, tack onto tmcc command.
# and use in pathnames. Used in conjunction with jailed virtual nodes.
......@@ -430,25 +433,7 @@ sub domounts()
my %sfsdeletes;
my @tmccresults;
#
# Update our SFS hostid first. If this fails, dosfshostid will
# unset USESFS.
#
if ($USESFS) {
if (! MFS()) {
#
# Setup SFS hostid.
#
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
else {
# No SFS on the MFS.
$USESFS = 0;
}
}
if (tmcc(TMCCCMD_MOUNTS, "USESFS=$USESFS", \@tmccresults) < 0) {
if (tmcc(TMCCCMD_MOUNTS, undef , \@tmccresults) < 0) {
warn("*** WARNING: Could not get mount list from server!\n");
return -1;
}
......@@ -701,6 +686,7 @@ sub JailedMounts($$$)
dbmclose(%MDB);
return @mountlist;
}
#
# Do SFS hostid setup.
# Creates an SFS host key for this node, if it doesn't already exist,
......@@ -2350,6 +2336,9 @@ sub doplabconfig()
sub bootsetup()
{
my $oldpid;
# Tell libtmcc to forget anything it knows.
tmccclrconfig();
print STDOUT "Checking Testbed reservation status ... \n";
......@@ -2387,6 +2376,21 @@ sub bootsetup()
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
#
# Setup SFS hostid.
#
if ($USESFS && !MFS()) {
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
#
# Tell libtmcc to get the full config. Note that this must happen
# AFTER dosfshostid() right above, since that changes what tmcd
# is going to tell us.
#
tmccgetconfig();
#
# Mount the project and user directories and symlink SFS "mounted"
# directories
......@@ -2502,32 +2506,21 @@ sub jailsetup()
system("echo '$vnodeid' > " . TMJAILNAME());
# Need to unify this with jailname.
system("echo '$vnodeid' > " . TMNODEID());
#
# Do account stuff.
#
{
print STDOUT "Checking Testbed reservation status ... \n";
if (! check_status()) {
print STDOUT " Free!\n";
return 0;
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
print STDOUT "Checking Testbed reservation status ... \n";
if (! check_status()) {
print STDOUT " Free!\n";
return 0;
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
{
#
# XXX just generates interface list for routing config
#
print STDOUT "Checking Testbed interface configuration ... \n";
doifconfig();
#
# Setup SFS hostid.
#
if ($USESFS) {
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
# print STDOUT "Mounting project and home directories ... \n";
# domounts();
......@@ -2630,6 +2623,14 @@ sub vnodejailsetup($)
chmod(0775, $viddir);
}
#
# Tell libtmcc to get the full config for the jail. At the moment
# we do not use SFS inside jails, so okay to do this now (usually
# have to call dosfshostid() first). The full config will be copied
# to the proper location inside the jail by mkjail.
#
tmccgetconfig();
#
# Get jail config.
#
......@@ -2644,6 +2645,9 @@ sub vnodejailsetup($)
#
sub plabsetup()
{
# Tell libtmcc to forget anything it knows.
tmccclrconfig();
#
# vnodeid will either be found in BEGIN block or will be passed to
# vnodeplabsetup, so it doesn't need to be found here
......@@ -2668,6 +2672,13 @@ sub plabsetup()
dosfshostid();
}
#
# Tell libtmcc to get the full config. Note that this must happen
# AFTER dosfshostid() right above, since that changes what tmcd
# is going to tell us.
#
tmccgetconfig();
# print STDOUT "Mounting project and home directories ... \n";
# domounts();
......
......@@ -14,14 +14,15 @@
package libtmcc;
use Exporter;
@ISA = "Exporter";
@EXPORT = qw(configtmcc tmcc tmccbossname
@EXPORT = qw(configtmcc tmcc tmccbossname tmccgetconfig tmccclrconfig
tmcccopycache
TMCCCMD_REBOOT TMCCCMD_STATUS TMCCCMD_STATE TMCCCMD_IFC
TMCCCMD_ACCT TMCCCMD_DELAY TMCCCMD_HOSTS TMCCCMD_RPM
TMCCCMD_TARBALL TMCCCMD_STARTUP TMCCCMD_DELTA TMCCCMD_STARTSTAT
TMCCCMD_READY TMCCCMD_MOUNTS TMCCCMD_ROUTING TMCCCMD_TRAFFIC
TMCCCMD_BOSSINFO TMCCCMD_TUNNEL TMCCCMD_NSECONFIGS
TMCCCMD_VNODELIST TMCCCMD_SUBNODELIST TMCCCMD_ISALIVE
TMCCCMD_SFSHOSTID TMCCCMD_SFSMOUNTS TMCCCMD_JAILCONFIG
TMCCCMD_SFSHOSTID TMCCCMD_JAILCONFIG
TMCCCMD_PLABCONFIG TMCCCMD_SUBCONFIG TMCCCMD_LINKDELAYS
TMCCCMD_PROGRAMS TMCCCMD_SYNCSERVER TMCCCMD_KEYHASH TMCCCMD_NODEID
TMCCCMD_NTPINFO TMCCCMD_NTPDRIFT
......@@ -30,6 +31,11 @@ use Exporter;
# Must come after package declaration!
use English;
#
# Turn off line buffering on output
#
$| = 1;
# Load up the paths. Done like this in case init code is needed.
BEGIN
{
......@@ -43,6 +49,8 @@ BEGIN
# The actual TMCC binary.
my $TMCCBIN = "$BINDIR/tmcc.bin";
my $PROXYDEF = "$BOOTDIR/proxypath";
my $CACHEDIR = "$BOOTDIR";
my $CACHENAME = "tmcc";
my $debug = 0;
my $beproxy = 0;
......@@ -64,45 +72,73 @@ my $beproxy = 0;
"logfile" => undef,
);
# The cache directory is named by the vnodeid. This avoids some confusion.
sub CacheDir()
{
return "$CACHEDIR/$CACHENAME"
if (!defined($config{"subnode"}));
return "$CACHEDIR/$CACHENAME" . "." . $config{"subnode"};
}
# Copy a cachedir. This is for mkjail, which needs to copy the cache
# dir into the jail to avoid another download of the full config.
sub tmcccopycache($$)
{
my ($vnodeid, $root) = @_;
my $fromdir = "$CACHEDIR/$CACHENAME" . "." . $vnodeid;
my $todir = "$root/$fromdir";
if (! -d $fromdir) {
warn("*** WARNING: No such directory $fromdir!\n");
return -1;
}
if (! -d $root) {
warn("*** WARNING: No such directory $root!\n");
return -1;
}
return system("cp -rp $fromdir $todir");
}
#
# List of TMCD commands. Some of these have to be passed through to
# tmcd, others can come from a local config file if it exists.
#
my %commandset =
( "reboot" => {TAG => "reboot", LOCAL => 0},
"status" => {TAG => "status", LOCAL => 0},
"state" => {TAG => "state", LOCAL => 0},
"ifconfig" => {TAG => "ifconfig", LOCAL => 1},
"accounts" => {TAG => "accounts", LOCAL => 1},
"delay" => {TAG => "delay", LOCAL => 1},
"hostnames" => {TAG => "hostnames", LOCAL => 1},
"rpms" => {TAG => "rpms", LOCAL => 1},
"tarballs" => {TAG => "tarballs", LOCAL => 1},
"startupcmd" => {TAG => "startupcmd", LOCAL => 1},
"deltas" => {TAG => "deltas", LOCAL => 1},
"startstatus" => {TAG => "startstatus", LOCAL => 0},
"ready" => {TAG => "ready", LOCAL => 0},
"mounts" => {TAG => "mounts", LOCAL => 1},
"routing" => {TAG => "routing", LOCAL => 1},
"trafgens" => {TAG => "trafgens", LOCAL => 1},
"bossinfo" => {TAG => "bossinfo", LOCAL => 0},
"tunnels" => {TAG => "tunnels", LOCAL => 1},
"nseconfigs" => {TAG => "nseconfigs", LOCAL => 1},
"vnodelist" => {TAG => "vnodelist", LOCAL => 1},
"subnodelist" => {TAG => "subnodelist", LOCAL => 1},
"isalive" => {TAG => "isalive", LOCAL => 0},
"sfshostid" => {TAG => "sfshostid", LOCAL => 0},
"sfsmounts" => {TAG => "sfsmounts", LOCAL => 1},
"jailconfig" => {TAG => "jailconfig", LOCAL => 1},
"plabconfig" => {TAG => "plabconfig", LOCAL => 1},
"subconfig" => {TAG => "subconfig", LOCAL => 1},
"linkdelays" => {TAG => "linkdelays", LOCAL => 1},
"programs" => {TAG => "programs", LOCAL => 1},
"syncserver" => {TAG => "syncserver", LOCAL => 1},
"keyhash" => {TAG => "keyhash", LOCAL => 1},
"nodeid" => {TAG => "nodeid", LOCAL => 1},
"ntpinfo" => {TAG => "ntpinfo", LOCAL => 1},
"ntprift" => {TAG => "ntpdrift.", LOCAL => 0},
( "reboot" => {TAG => "reboot"},
"status" => {TAG => "status"},
"state" => {TAG => "state"},
"ifconfig" => {TAG => "ifconfig"},
"accounts" => {TAG => "accounts"},
"delay" => {TAG => "delay"},
"hostnames" => {TAG => "hostnames"},
"rpms" => {TAG => "rpms"},
"tarballs" => {TAG => "tarballs"},
"startupcmd" => {TAG => "startupcmd"},
"deltas" => {TAG => "deltas"},
"startstatus" => {TAG => "startstatus"},
"ready" => {TAG => "ready"},
"mounts" => {TAG => "mounts"},
"routing" => {TAG => "routing"},
"trafgens" => {TAG => "trafgens"},
"bossinfo" => {TAG => "bossinfo"},
"tunnels" => {TAG => "tunnels"},
"nseconfigs" => {TAG => "nseconfigs"},
"vnodelist" => {TAG => "vnodelist"},
"subnodelist" => {TAG => "subnodelist"},
"isalive" => {TAG => "isalive"},
"sfshostid" => {TAG => "sfshostid"},
"jailconfig" => {TAG => "jailconfig"},
"plabconfig" => {TAG => "plabconfig"},
"subconfig" => {TAG => "subconfig"},
"linkdelay" => {TAG => "linkdelay"},
"programs" => {TAG => "programs"},
"syncserver" => {TAG => "syncserver"},
"keyhash" => {TAG => "keyhash"},
"nodeid" => {TAG => "nodeid"},
"ntpinfo" => {TAG => "ntpinfo"},
"ntprift" => {TAG => "ntpdrift"},
"sdparams" => {TAG => "sdparams"},
);
#
......@@ -131,11 +167,10 @@ sub TMCCCMD_VNODELIST() { $commandset{"vnodelist"}->{TAG}; }
sub TMCCCMD_SUBNODELIST(){ $commandset{"subnodelist"}->{TAG}; }
sub TMCCCMD_ISALIVE() { $commandset{"isalive"}->{TAG}; }
sub TMCCCMD_SFSHOSTID() { $commandset{"sfshostid"}->{TAG}; }
sub TMCCCMD_SFSMOUNTS() { $commandset{"sfsmounts"}->{TAG}; }
sub TMCCCMD_JAILCONFIG(){ $commandset{"jailconfig"}->{TAG}; }
sub TMCCCMD_PLABCONFIG(){ $commandset{"plabconfig"}->{TAG}; }
sub TMCCCMD_SUBCONFIG() { $commandset{"subconfig"}->{TAG}; }
sub TMCCCMD_LINKDELAYS(){ $commandset{"linkdelays"}->{TAG}; }
sub TMCCCMD_LINKDELAYS(){ $commandset{"linkdelay"}->{TAG}; }
sub TMCCCMD_PROGRAMS() { $commandset{"programs"}->{TAG}; }
sub TMCCCMD_SYNCSERVER(){ $commandset{"syncserver"}->{TAG}; }
sub TMCCCMD_KEYHASH() { $commandset{"keyhash"}->{TAG}; }
......@@ -158,7 +193,6 @@ sub configtmcc($$)
$config{$opt} = $val;
}
#
# Convert the config hash to an option string. This is separate so that
# the user can provide their own option hash on a single tmcc call, which
......@@ -272,6 +306,42 @@ sub tmcc ($;$$%)
{
my ($cmd, $args, $results, %opthash) = @_;
#
# See if this is a cmd we can get from the local config stash.
#
if (!defined($args) || $args eq "") {
foreach my $key (keys(%commandset)) {
my $tag = $commandset{$key}->{TAG};
if ($cmd eq $tag) {
#
# If we can get it, great! Otherwise go to tmcd.
#
my $filename = CacheDir() . "/$tag";
my @strings = ();
if (-e $filename && open(TD, $filename)) {
#
# Read file contents and return
#
print STDERR "Fetching locally from $filename\n"
if ($debug);
while (<TD>) {
next
if ($_ =~ /^\*\*\* $tag$/);
push(@strings, $_);
}
@$results = @strings
if (defined($results));
return 0;
}
last;
}
}
}
#
# Check for proxypath file, but do not override config option.
#
......@@ -321,5 +391,81 @@ sub tmccbossname()
return $bossname;
}
#
# Special entrypoint to clear the current config cache (say, at reboot).
#
sub tmccclrconfig()
{
my $dir = CacheDir();
if (-d $dir) {
system("rm -rf $dir");
}
}
#
# Special entrypoint to download the entire configuration and cache for
# subsequent calls.
#
sub tmccgetconfig()
{
my @tmccresults;
my $cdir = CacheDir();
#
# Check for proxypath file, but do not override config option.
#
if (!$config{"dounix"} && -e $PROXYDEF) {
#
# Suck out the path and untaint.
#
open(PP, "$BOOTDIR/proxypath");
my $ppath = <PP>;
close(PP);
if ($ppath =~ /^([-\w\.\/]+)$/) {
$config{"dounix"} = $1;
}
else {
die("Bad data in tmccproxy path: $ppath");
}
}
tmccclrconfig();
if (!mkdir("$cdir", 0775)) {
warn("*** WARNING: Could not mkdir $cdir: $!\n");
return -1;
}
if (runtmcc("fullconfig", undef, \@tmccresults) < 0 ||
!scalar(@tmccresults)) {
warn("*** WARNING: Could not get fullconfig from tmcd!\n");
return -1;
}
my $str;
while ($str = shift(@tmccresults)) {
if ($str =~ /^\*\*\* ([-\w]*)$/) {
my $param = $1;
if (open(TD, "> $cdir/$param")) {
while (@tmccresults) {
$str = shift(@tmccresults);
last
if ($str =~ /^\*\*\* $param$/);
print TD $str;
}
close(TD);
}
else {
warn("*** WARNING: Could not create $cdir/$str: $!\n");
return -1;
}
}
}
return 0;
}
1;
......@@ -60,7 +60,9 @@ if (my $rval = tmcc($CMD, $ARGS, \@results) != 0) {
exit($rval);
}
if (@results) {
print STDOUT "@results";
foreach my $str (@results) {
print STDOUT "$str";
}
}
exit(0);
......
......@@ -169,6 +169,7 @@ while (flock(LOCK, LOCK_EX|LOCK_NB) == 0) {
# to avoid blocking at boot time.
#
libsetup_settimeout(30);
libsetup_refresh();
#
# Order matters!
......
......@@ -398,14 +398,16 @@ sub LinkDelaySetup()
# We need to know if any jailed nodes. That changes which kernel
# we want to boot. Temporary until the jail stuff is better tested.
#
if (tmcc(TMCCCMD_VNODELIST, undef, \@jails) < 0) {
warn("*** WARNING: Could not get jails from server!\n");
return -1;
}
foreach my $str (@jails) {
if ($str =~ /^VNODEID=([-\w]+) JAILED=(\d)$/) {
if ($2) {
$gotjails++;
if (!$gotjails) {
if (tmcc(TMCCCMD_VNODELIST, undef, \@jails) < 0) {
warn("*** WARNING: Could not get jails from server!\n");
return -1;
}
foreach my $str (@jails) {
if ($str =~ /^VNODEID=([-\w]+) JAILED=(\d)$/) {
if ($2) {
$gotjails++;
}
}
}
}
......
......@@ -17,6 +17,7 @@ use Fcntl ':flock';
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
use libsetup qw(JailedMounts REMOTE LOCALROOTFS TMPASSDB TMGROUPDB);
use libtmcc;
#
# Questions:
......@@ -430,6 +431,7 @@ sub mkrootfs($)
fatal("Could not mkdir 'dir' in $path/root/var/emulab: $!");
}
}
tmcccopycache($vnodeid, "$path/root");
#
# Stash the control net IP if not the same as the host IP
......@@ -591,6 +593,7 @@ sub restorerootfs($)
}
push(@mntpoints, "$path/root/$dir");
}
tmcccopycache($vnodeid, "$path/root");
#
# The proc FS in the jail is per-jail of course.
......
......@@ -309,7 +309,7 @@ sub os_useradd($$$$$$$$$)
# Locally, if directory exists and is populated, skip -m
# cause FreeBSD copies files in anyway!
$args .= "-m "
if (! -e "$homedir/.cshrc");
if (! -d $homedir || ! -e "$homedir/.cshrc");
}
else {
# populate on remote nodes. At some point will tar files over.
......@@ -324,6 +324,7 @@ sub os_useradd($$$$$$$$$)
warn "*** WARNING: $USERADD $login error.\n";
return -1;
}
chown($uid, $gid, $homedir);
if (system("$CHPASS '$pswd' $login") != 0) {
warn "*** WARNING: $CHPASS $login error.\n";
......
......@@ -21,7 +21,7 @@ use Exporter;
TBBackGround TBForkCmd vnodejailsetup plabsetup vnodeplabsetup
dorouterconfig jailsetup dojailconfig JailedMounts findiface
tmcctimeout libsetup_getvnodeid dotrafficconfig
ixpsetup dokeyhash donodeid
ixpsetup dokeyhash donodeid libsetup_refresh
MFS REMOTE JAILED PLAB LOCALROOTFS IXP
......@@ -50,6 +50,9 @@ libtmcc::configtmcc("version", TMCD_VERSION());
# Control tmcc timeout.
sub libsetup_settimeout($) { libtmcc::configtmcc("timeout", $_[0]); };
# Redresh tmcc cache.
sub libsetup_refresh() { libtmcc::tmccgetconfig(); };
#
# For virtual (multiplexed nodes). If defined, tack onto tmcc command.
# and use in pathnames. Used in conjunction with jailed virtual nodes.
......@@ -430,25 +433,7 @@ sub domounts()
my %sfsdeletes;
my @tmccresults;
#
# Update our SFS hostid first. If this fails, dosfshostid will
# unset USESFS.
#
if ($USESFS) {
if (! MFS()) {
#
# Setup SFS hostid.
#
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
else {
# No SFS on the MFS.
$USESFS = 0;
}
}
if (tmcc(TMCCCMD_MOUNTS, "USESFS=$USESFS", \@tmccresults) < 0) {
if (tmcc(TMCCCMD_MOUNTS, undef , \@tmccresults) < 0) {
warn("*** WARNING: Could not get mount list from server!\n");
return -1;
}
......@@ -701,6 +686,7 @@ sub JailedMounts($$$)
dbmclose(%MDB);
return @mountlist;
}
#
# Do SFS hostid setup.
# Creates an SFS host key for this node, if it doesn't already exist,
......@@ -2350,6 +2336,9 @@ sub doplabconfig()
sub bootsetup()
{
my $oldpid;
# Tell libtmcc to forget anything it knows.
tmccclrconfig();
print STDOUT "Checking Testbed reservation status ... \n";
......@@ -2387,6 +2376,21 @@ sub bootsetup()
}
print STDOUT " Allocated! $pid/$eid/$vname\n";
#
# Setup SFS hostid.
#
if ($USESFS && !MFS()) {
print STDOUT "Setting up for SFS ... \n";
dosfshostid();
}
#
# Tell libtmcc to get the full config. Note that this must happen
# AFTER dosfshostid() right above, since that changes what tmcd
# is going to tell us.
#
tmccgetconfig();