Commit 87eed168 authored by Mike Hibler's avatar Mike Hibler

Performance improvements to the vnode startup path.

The bigest improvement happened on day one when I took out the 20 second sleep
between vnode starts in bootvnodes. That appears to have been an artifact of
an older time and an older Xen. Or, someone smarter than me saw the potential
of getting bogged down for, oh say three weeks, trying to micro-optimize the
process and instead just went for the conservative fix!

Following day one, the ensuing couple of weeks was a long strange trip to
find the maximum number of simultaneous vnode creations that could be done
without failure. In that time I tried a lot of things, generated a lot of
graphs, produced and tweaked a lot of new constants, and in the end, wound
up with the same two magic numbers (3 and 5) that were in the original code!
To distinguish myself, I added a third magic number (1, the loneliest of
them all).

All I can say is that now, the choice of 3 or 5 (or 1), is based on more
solid evidence than before. Previously it was 5 if you had a thin-provisioning
LVM, 3 otherwise. Now it is based more directly on host resources, as
described in a long comment in the code, the important part of which is:

 #
 # if (dom0 physical RAM < 1GB) MAX = 1;
 # if (any swap activity) MAX = 1;
 #
 #    This captures pc3000s/other old machines and overloaded (RAM) machines.
 #
 # if (# physical CPUs <= 2) MAX = 3;
 # if (# physical spindles == 1) MAX = 3;
 # if (dom0 physical RAM <= 2GB) MAX = 3;
 #
 #    This captures d710s, Apt r320, and Cloudlab m510s. We may need to
 #    reconsider the latter since its single drive is an NVMe device.
 #    But first we have to get Xen working with them (UEFI issues)...
 #
 # else MAX = 5;

In my defense, I did fix some bugs and stuff too (and did I mention
the cool graphs?) See comments in the code and gitlab emulab/emulab-devel
issue #148.
parent ee854767
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -35,10 +35,10 @@ use POSIX qw(strftime);
#
sub usage()
{
print "Usage: bootvnodes [-d] [-f] [-k | -h | -r | -b | -c]\n";
print STDERR "Usage: bootvnodes [-w sec] [-d] [-f] [-k|-h|-r|-b|-c]\n";
exit(1);
}
my $optlist = "kdfhrcb";
my $optlist = "kdfhrcbw:";
#
# Turn off line buffering on output
......@@ -62,6 +62,7 @@ my $vndir = "$VARDIR/jails";
my $debug = 0;
my $daemon = 1;
my $reconfig= 0;
my $waittime = 0;
my $action;
# Prototypes
......@@ -105,6 +106,13 @@ if (defined($options{"b"})) {
if (defined($options{"c"})) {
$reconfig = 1;
}
if (defined($options{"w"})) {
if ($options{"w"} !~ /^(\d+)$/) {
print STDERR "Invalid wait time for -w\n";
usage();
}
$waittime = $1;
}
if (@ARGV) {
usage();
}
......@@ -153,6 +161,20 @@ if (GENVNODE() &&
exit($? >> 8);
}
#
# Sort by vnode number if it follows are convention. We need this because
# we don't use a fixed field width for the vnode number.
# XXX I am sure there are more perl-ish ways to do this.
#
sub byvnode {
if ($a =~ /^(.*)-(\d+)$/ && ($apre = $1) && ($anum = $2) &&
$b =~ /^(.*)-(\d+)$/ && ($bpre = $1) && ($bnum = $2) &&
$apre eq $bpre) {
return $anum <=> $bnum;
}
return $a cmp $b;
}
#
# This applies to whatever vnodes are running. Do it and exit.
#
......@@ -162,11 +184,23 @@ if (defined($action) && !$reconfig) {
my @files = readdir(DIR);
closedir(DIR);
#
# XXX make a feeble effort to weed out random files.
#
# We used to do this by making sure the name was of the form
# pcvm<foo>-<num>, but not all clusters use the "pc" naming scheme.
# So now, the best we can do is ensure that the name is like
# <foo>vm<bar>-<num> and is a directory.
#
my @vnfiles = ();
foreach my $file (@files) {
if ($file =~ /^((?:pc|homenet)vm[-\w]*)$/) {
bootvnode($1, $action, (-e "$vndir/$file/fakejail" ? 0 : 1));
if ($file =~ /^([-\w]+vm[-\w]*-\d+)$/ && -d "$vndir/$file") {
push(@vnfiles, $1);
}
}
foreach my $file (sort byvnode @vnfiles) {
bootvnode($file, $action, (-e "$vndir/$file/fakejail" ? 0 : 1));
}
exit(0);
}
......@@ -190,22 +224,8 @@ foreach my $str (@tmccresults) {
if ($2 eq "0");
}
else {
warn("*** WARNING: Skipping bad subnodeid: '$str'\n");
}
}
#
# Sort by vnode number if it follows are convention. We need this because
# we don't use a fixed field width for the vnode number.
# XXX I am sure there are more perl-ish ways to do this.
#
sub byvnode {
if ($a =~ /^(.*)-(\d+)$/ && ($apre = $1) && ($anum = $2) &&
$b =~ /^(.*)-(\d+)$/ && ($bpre = $1) && ($bnum = $2) &&
$apre eq $bpre) {
return $anum <=> $bnum;
warn("*** WARNING: Skipping bad VNODEID: '$str'\n");
}
return $a cmp $b;
}
#
......@@ -218,14 +238,19 @@ if ($reconfig) {
my @files = readdir(DIR);
closedir(DIR);
# XXX make a feeble effort to weed out random files.
my @vnfiles = ();
foreach my $file (@files) {
if ($file =~ /^((?:pc|homenet)vm[-\w]*)$/) {
if (-e "$vndir/$file/fakejail") {
$fakejails++;
$curvnodelist{$1} = 0;
} else {
$curvnodelist{$1} = 1;
}
if ($file =~ /^([-\w]+vm[-\w]*-\d+)$/ && -d "$vndir/$file") {
push(@vnfiles, $1);
}
}
foreach my $file (sort byvnode @vnfiles) {
if (-e "$vndir/$file/fakejail") {
$fakejails++;
$curvnodelist{$file} = 0;
} else {
$curvnodelist{$file} = 1;
}
}
......@@ -360,7 +385,7 @@ sub bootvnode($$$)
my ($vnode, $action, $jailed) = @_;
my $opt;
my $act;
my $extrawait;
my $extrawait = $waittime;
if ($action eq "halt") {
$opt = "-h";
......@@ -369,8 +394,8 @@ sub bootvnode($$$)
elsif ($action eq "reboot") {
$opt = "-r";
$act = "Rebooting";
$extrawait = 20
if (GENVNODETYPE() eq "xen");
## XXX should no longer be needed
#$extrawait = 20 if (GENVNODETYPE() eq "xen");
}
elsif ($action eq "kill") {
$opt = "-k";
......@@ -379,8 +404,8 @@ sub bootvnode($$$)
else {
$opt = "-b";
$act = "Booting";
$extrawait = 20
if (GENVNODETYPE() eq "xen");
## XXX should no longer be needed
#$extrawait = 20 if (GENVNODETYPE() eq "xen");
}
$opt .= ($jailed ? " -jVt" : " -i");
......@@ -390,6 +415,6 @@ sub bootvnode($$$)
return($?)
if ($?);
sleep($extrawait)
if (defined($extrawait));
if ($extrawait > 0);
return 0;
}
......@@ -628,7 +628,7 @@ if (! -e "$VNDIR/vnode.info") {
($ret,$err) = safeLibOp('vnodeCreate',0,0);
if ($err) {
MyFatal("vnodeCreate failed");
MyFatal("vnodeCreate failed: $err");
}
$vmid = $ret;
......
......@@ -356,6 +356,7 @@ sub handler ($) {
my ($signame) = @_;
print STDERR "vnodesetup ($PID) caught a SIG${signame}!\n";
TBDebugTimeStampWithDate("vnodesetup shutting down ...");
$SIG{USR1} = 'IGNORE';
$SIG{USR2} = 'IGNORE';
......
......@@ -795,9 +795,14 @@ sub restartDHCP()
if (mysystem2("/sbin/initctl restart $dhcpd_service") != 0) {
mysystem2("/sbin/initctl start $dhcpd_service");
}
} else {
#sysvinit
} elsif (-x '/bin/systemctl') {
# systemd
mysystem2("/bin/systemctl restart $dhcpd_service.service");
} elsif (-x '/etc/init.d/$dhcpd_service') {
# sysvinit
mysystem2("/etc/init.d/$dhcpd_service restart");
} else {
print STDERR "restartDHCP: could not restart dhcpd!\n";
}
}
......
......@@ -27,6 +27,7 @@ use English;
use Data::Dumper;
use POSIX qw(setsid);
use POSIX ":sys_wait_h";
use POSIX ":signal_h";
use Socket;
#
......@@ -302,6 +303,9 @@ sub Online()
else {
POSIX::setsid();
# XXX make sure we can kill the proxy when done
local $SIG{TERM} = 'DEFAULT';
exec("$BINDIR/tmcc.bin -d -t 15 -n $vnode_id ".
" -X $host_ip:$local_tmcd_port -s $boss_ip -p $TMCD_PORT ".
" -o $LOGDIR/tmccproxy.$vnode_id.log");
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment