Commit 069dc7d3 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Medium size cleanup of the client side code. The main impetus behind

this was to add soft reconfig support so that nodes could be
reconfigured without having to reboot them. This appears to work, and
has been tested with jails getting moved around. I've also tested the
new code on the MFS, but still no testing has been done on PLAB nodes.

The main change is that most of the code moved out of libsetup.pm, and
was split into constituent rc scripts, each of which does its own
thing, including cleaning up and preparing for making an image. Most
of that central knowledge has been moved out into the scripts. Still more
to do but this was a good start.
parent bba0b1b3
......@@ -27,7 +27,7 @@ DESTDIR =
ETCDIR = $(DESTDIR)$(CLIENT_ETCDIR)
BINDIR = $(DESTDIR)$(CLIENT_BINDIR)
VARDIR = $(DESTDIR)$(CLIENT_VARDIR)
RCDIR = $(DESTDIR)/usr/local/etc/rc.d
RCDIR = $(BINDIR)/rc
TBBINDIR = $(DESTDIR)/usr/testbed/bin
TBLIBDIR = $(DESTDIR)/usr/testbed/lib
INSTALL = /usr/bin/install -c
......@@ -73,10 +73,12 @@ common-script-install: dir-install
$(INSTALL) -m 755 $(SRCDIR)/runstartup $(BINDIR)/runstartup
$(INSTALL) -m 755 $(SRCDIR)/runcvsup.sh $(BINDIR)/runcvsup.sh
$(INSTALL) -m 755 $(SRCDIR)/update $(BINDIR)/update
$(INSTALL) -m 755 $(SRCDIR)/ifsetup $(BINDIR)/ifsetup
$(INSTALL) -m 755 $(SRCDIR)/vnodesetup $(BINDIR)/vnodesetup
$(INSTALL) -m 755 $(SRCDIR)/bootsubnodes $(BINDIR)/bootsubnodes
$(INSTALL) -m 755 $(SRCDIR)/bootvnodes $(BINDIR)/bootvnodes
$(INSTALL) -m 755 $(SRCDIR)/startcmddone $(BINDIR)/startcmddone
(cd config; $(MAKE) script-install)
symlinks: dir-install
rm -f $(TBBINDIR)/tevc
......@@ -87,14 +89,11 @@ symlinks: dir-install
ln -s $(BINDIR) $(TBLIBDIR)
local-script-install: common-script-install
$(INSTALL) -m 755 $(SRCDIR)/bootsetup $(BINDIR)/bootsetup
$(INSTALL) -m 755 $(SRCDIR)/sendevent $(BINDIR)/sendevent
$(INSTALL) -m 755 $(SRCDIR)/rc.testbed $(BINDIR)/rc.testbed
$(INSTALL) -m 755 $(SRCDIR)/rc.agents $(BINDIR)/rc.agents
$(INSTALL) -m 755 $(SRCDIR)/rc.progagent $(BINDIR)/rc.progagent
$(INSTALL) -m 755 $(SRCDIR)/rc.linktest $(BINDIR)/rc.linktest
$(INSTALL) -m 755 $(SRCDIR)/rc.setup $(BINDIR)/rc.setup
$(INSTALL) -m 755 $(SRCDIR)/rc.slothd $(BINDIR)/rc.slothd
$(INSTALL) -m 755 $(SRCDIR)/rc.testbed $(RCDIR)/rc.testbed
$(INSTALL) -m 755 $(SRCDIR)/rc.bootsetup $(RCDIR)/rc.bootsetup
$(INSTALL) -m 755 $(SRCDIR)/rc.slothd $(RCDIR)/rc.slothd
$(INSTALL) -m 755 $(SRCDIR)/rc.linktest $(RCDIR)/rc.linktest
# Symlink this cause we invoke it from boss, and its too much
# of a hassle to worry about right now.
rm -f $(ETCDIR)/update
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
#
# Then invoke the bootsetup routine in the library. All the work happens
# in there.
#
bootsetup();
exit 0;
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#
use Getopt::Std;
......@@ -18,10 +18,10 @@ use POSIX qw(strftime);
#
sub usage()
{
print "Usage: bootvnodes [-d] [-f] [-k | -h]\n";
print "Usage: bootvnodes [-d] [-f] [-k | -h | -r | -b | -c]\n";
exit(1);
}
my $optlist = "kdfh";
my $optlist = "kdfhrcb";
#
# Turn off line buffering on output
......@@ -40,16 +40,11 @@ use libtmcc;
# Locals
my $logname = "$LOGDIR/bootvnodes.debug";
my $vndir = "/var/emulab/jails";
my $vndir = "$VARDIR/jails";
my $debug = 0;
my $daemon = 1;
my $killit = 0;
my $haltit = 0;
#
# Forward declarations for prototype checking
#
sub bootvnodes();
my $reconfig= 0;
my $action;
#
# Parse command arguments. Once we return from getopts, all that should be
......@@ -65,11 +60,20 @@ if (defined($options{"d"})) {
if (defined($options{"f"})) {
$daemon = 0;
}
if (defined($options{"r"})) {
$action = "reboot";
}
if (defined($options{"k"})) {
$killit = 1;
$action = "kill";
}
if (defined($options{"h"})) {
$haltit = 1;
$action = "halt";
}
if (defined($options{"b"})) {
$action = "boot";
}
if (defined($options{"c"})) {
$reconfig = 1;
}
if (@ARGV) {
usage();
......@@ -104,14 +108,45 @@ if ($daemon && TBBackGround($logname)) {
exit(0);
}
if ($killit | $haltit) {
my $opt = ($haltit ? "-h" : "-k");
my $act = ($haltit ? "Halting" : "Killing");
#
# Helper function to boot/kill/halt/reboot a specific vnode.
#
sub bootvnode($$$)
{
my ($vnode, $action, $jailed) = @_;
my $opt;
my $act;
if ($action eq "halt") {
$opt = "-h";
$act = "Halting";
}
elsif ($action eq "reboot") {
$opt = "-r";
$act = "Rebooting";
}
elsif ($action eq "kill") {
$opt = "-k";
$act = "Killing";
}
else {
$opt = "-b";
$act = "Booting";
}
$opt .= " -jV"
if ($jailed);
print "$act vnodes ... please be patient ...\n";
print "$act vnode $vnode ...\n";
system("vnodesetup $opt $vnode");
return($?);
}
opendir(DIR, "/var/emulab/jails") or
die("Cannot opendir /var/emulab/jails: $!\n");
#
# This applies to whatever vnodes are running. Do it and exit.
#
if (defined($action) && !$reconfig) {
opendir(DIR, "$vndir") or
die("Cannot opendir $vndir: $!\n");
my @files = readdir(DIR);
closedir(DIR);
......@@ -119,15 +154,15 @@ if ($killit | $haltit) {
if ($file ne "." && $file ne ".." && $file ne "local" &&
$file =~ /^([-\w]*)$/) {
print "$act vnode $1 ...\n";
system("vnodesetup $opt $1");
# Assume jailed for now.
bootvnode($1, $action, 1);
}
}
exit(0);
}
my %curvnodelist;
my @vnodes;
my %newvnodelist = ();
my %curvnodelist = ();
#
# Get the current set of vnodes that are supposed to be running on
......@@ -138,21 +173,55 @@ my @tmccresults;
if (tmcc(TMCCCMD_VNODELIST, undef, \@tmccresults) < 0) {
die("*** WARNING: Could not get vnode list from server!\n");
}
if (! @tmccresults) {
print "No vnodes. Exiting gracefully ...\n";
exit(0);
}
foreach my $str (@tmccresults) {
if ($str =~ /^VNODEID=([-\w]+) JAILED=(\d)$/) {
$curvnodelist{$1} = $2;
$newvnodelist{$1} = $2;
}
else {
warn("*** WARNING: Skipping bad subnodeid: '$str'\n");
}
}
@vnodes = keys(%curvnodelist);
print "Vnodelist is @vnodes.\n";
#
# If doing a reconfig, get the current set of vnodes that are running.
# We want to compare that list against the list we just got from tmcd.
#
if ($reconfig) {
opendir(DIR, "$vndir") or
die("Cannot opendir $vndir: $!\n");
my @files = readdir(DIR);
closedir(DIR);
foreach my $file (@files) {
if ($file ne "." && $file ne ".." && $file ne "local" &&
$file =~ /^([-\w]*)$/) {
$curvnodelist{$1} = 1; # XXX Assume jailed.
}
}
#
# Reboot nodes that are running and in the current list from tmcd.
# Kill nodes that are running but not in the current list from tmcd.
#
foreach my $vnode (keys(%curvnodelist)) {
if (exists($newvnodelist{$vnode})) {
bootvnode($vnode,
(defined($action) ? $action : "reboot"),
$newvnodelist{$vnode});
delete($newvnodelist{$vnode});
}
else {
bootvnode($vnode, "kill", $curvnodelist{$vnode});
}
}
}
# XXX Exit if specific action requested, and it was not "boot"
exit(0)
if (defined($action) && $action ne "boot");
# Exit if nothing left to do (no new vnodes).
exit(0)
if (! scalar(keys(%newvnodelist)));
#
# If booting with vnodes, then see about creating the extra FS.
......@@ -165,21 +234,16 @@ if (!REMOTE()) {
#
# Make sure enough vn devices exist
#
for (my $i = 0; $i < scalar(@vnodes); $i++) {
for (my $i = 0;
$i < scalar(keys(%newvnodelist)) + scalar(keys(%curvnodelist)); $i++) {
my $dev = "vn${i}";
if (! -e "/dev/${dev}c") {
system("(cd /dev; ./MAKEDEV $dev)");
}
}
foreach my $vnode (keys(%curvnodelist)) {
my $jailflag = "";
if ($curvnodelist{$vnode}) {
$jailflag = "-jV";
}
print "Setting up vnode $vnode ...\n";
# This will not return until the vnode is fully running.
system("vnodesetup -b $jailflag $vnode");
sleep(1);
foreach my $vnode (keys(%newvnodelist)) {
# Blocks until mostly setup.
bootvnode($vnode, "boot", $newvnodelist{$vnode});
}
exit(0);
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#
#
# Tuny helper library for Emulab rc scripts.
#
package librc;
use Exporter;
@ISA = "Exporter";
@EXPORT = qw(fatal warning scriptname);
# Must come after package declaration!
use English;
#
# Turn off line buffering on output
#
$| = 1;
# Load up the paths. Done like this in case init code is needed.
BEGIN
{
if (! -e "/etc/emulab/paths.pm") {
die("Yikes! Could not require /etc/emulab/paths.pm!\n");
}
require "/etc/emulab/paths.pm";
import emulabpaths;
}
#
# Fatal error. Display and die.
#
sub fatal($)
{
my($mesg) = $_[0];
die("*** $0:\n".
" $mesg\n");
}
#
# Display warning.
#
sub warning($)
{
my($mesg) = $_[0];
print("*** $0:\n".
" WARNING: $mesg\n");
}
#
# Return scriptname with no path.
#
sub scriptname()
{
my ($dirpath,$base) = ($PROGRAM_NAME =~ m#^(.*/)?(.*)#);
return $base;
}
1;
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
sub usage()
{
print "Usage: " .
scriptname() . " [-j vnodeid] boot|shutdown|reconfig|reset\n";
exit(1);
}
my $optlist = "j:";
my $action = "boot";
# Turn off line buffering on output
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
# Only root.
if ($EUID != 0) {
die("*** $0:\n".
" Must be root to run this script!\n");
}
# Script specific goo.
#
# These go in /var/emulab. Good for all environments!
#
my $PASSDB = "$VARDIR/db/passdb";
my $GROUPDB = "$VARDIR/db/groupdb";
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
use liblocsetup;
use libtmcc;
use librc;
#
# Not all clients support this.
#
exit(0)
if (REMOTE() && !(PLAB() || JAILED()));
# Protos.
sub doboot();
sub doshutdown();
sub doreconfig();
sub docleanup();
# Parse command line.
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{'j'})) {
my $vnodeid = $options{'j'};
libsetup_setvnodeid($vnodeid);
}
# Allow default above.
if (@ARGV) {
$action = $ARGV[0];
}
# Execute the action.
SWITCH: for ($action) {
/^boot$/i && do {
doboot();
last SWITCH;
};
/^shutdown$/i && do {
doshutdown();
last SWITCH;
};
/^reconfig$/i && do {
doreconfig();
last SWITCH;
};
/^reset$/i && do {
docleanup();
last SWITCH;
};
fatal("Invalid action: $action\n");
}
exit(0);
# More protos.
sub TBNewsshKeyfile($$$$$);
#
# Boot Action.
#
sub doboot()
{
my @tmccresults;
my %newaccounts = ();
my %newgroups = ();
my %pubkeys1 = ();
my %pubkeys2 = ();
my @sfskeys = ();
my %deletes = ();
my %lastmod = ();
my %PWDDB;
my %GRPDB;
print STDOUT "Checking Testbed user accounts configuration ... \n";
if (tmcc(TMCCCMD_ACCT, undef, \@tmccresults) < 0) {
fatal("Could not get account info from server!");
}
# Important; if no results then do nothing. We do not want to remove
# accounts cause the server failed to give us anything!
return 0
if (! @tmccresults);
#
# The strategy is to keep a record of all the groups and accounts
# added by the testbed system so that we know what to remove. We
# use a vanilla perl dbm for that, one for the groups and one for
# accounts.
#
# First just get the current set of groups/accounts from tmcd.
#
foreach my $str (@tmccresults) {
if ($str =~ /^ADDGROUP NAME=([-\@\w.]+) GID=([0-9]+)/) {
#
# Group info goes in the hash table.
#
my $gname = "$1";
# New plab images are stupid.
$gname = lc($gname)
if (PLAB());
if (REMOTE() && !JAILED() && !PLAB()) {
$gname = "emu-${gname}";
}
$newgroups{"$gname"} = $2
}
elsif ($str =~ /^ADDUSER LOGIN=([0-9A-Za-z]+)/) {
#
# Account info goes in the hash table.
#
$newaccounts{$1} = $str;
next;
}
elsif ($str =~ /^PUBKEY LOGIN=([0-9A-Za-z]+) KEY="(.*)"/) {
#
# Keys go into hash as a list of keys.
#
my $login = $1;
my $key = $2;
#
# P1 or P2 key. Must be treated differently below.
#
if ($key =~ /^\d+\s+.*$/) {
if (! defined($pubkeys1{$login})) {
$pubkeys1{$login} = [];
}
push(@{$pubkeys1{$login}}, $key);
}
else {
if (! defined($pubkeys2{$login})) {
$pubkeys2{$login} = [];
}
push(@{$pubkeys2{$login}}, $key);
}
next;
}
elsif ($str =~ /^SFSKEY KEY="(.*)"/) {
#
# SFS key goes into the array.
#
push(@sfskeys, $1);
next;
}
else {
warning("Bad accounts line: $str");
}
}
if (! MFS()) {
#
# On the MFS, these will just start out as empty hashes.
#
dbmopen(%PWDDB, $PASSDB, 0660) or
fatal("Cannot open $PASSDB: $!");
dbmopen(%GRPDB, $GROUPDB, 0660) or
fatal("Cannot open $GROUPDB: $!");
}
#
# Create any groups that do not currently exist. Add each to the
# DB as we create it.
#
while (($group, $gid) = each %newgroups) {
my ($exists,undef,$curgid) = getgrnam($group);
if ($exists) {
if ($gid != $curgid) {
warning("$group/$gid mismatch with existing group");
}
next;
}
print "Adding group: $group/$gid\n";
if (os_groupadd($group, $gid)) {
warning("Error adding new group $group/$gid");
next;
}
# Add to DB only if successful.
$GRPDB{$group} = $gid;
}
#
# Now remove the ones that we created previously, but are now no longer
# in the group set (as told to us by the TMCD). Note, we cannot delete
# them directly from the hash since that would mess up the foreach loop,
# so just stick them in temp and postpass it.
#
while (($group, $gid) = each %GRPDB) {
if (defined($newgroups{$group})) {
next;
}
print "Removing group: $group/$gid\n";
if (os_groupdel($group)) {
warning("Error removing group $group/$gid");
next;
}
# Delete from DB only if successful.
$deletes{$group} = $gid;
}
while (($group, $gid) = each %deletes) {
delete($GRPDB{$group});
}
%deletes = ();