Commit 82e1d812 authored by Leigh Stoller's avatar Leigh Stoller

BIG reorganization of the install code.

* Split up boss/ops/fs install into indvidual modules; generally, what
  was a toplevel phase in the original files is not a file. This
  allowed for better code/variable reuse. No longer monolithic, which
  makes it easy to test and rerun parts.

* Incorporate "update" into the install process. Certain phase file
  can be used in update mode, as when the IP/subnet/domain changes.

* Moved the MFS setup from rc.mkelab into the normal install process.
  Users no longer have to do this themselves. Good thing.

* installvars.pm is a new library that has the merged set of the
  zillion variables that were at the top of boss/fs/ops install.
parent 21e7e909
This diff is collapsed.
......@@ -7260,7 +7260,7 @@ fi
outfiles="$outfiles Makeconf GNUmakefile \
assign/GNUmakefile \
named/GNUmakefile firewall/GNUmakefile \
ssl/GNUmakefile ssl/mksig ssl/usercert.cnf \
ssl/GNUmakefile ssl/mksig ssl/usercert.cnf ssl/mkserial \
capture/GNUmakefile \
db/GNUmakefile \
db/EmulabConstants.pm db/EmulabFeatures.pm db/Experiment.pm \
......@@ -7409,7 +7409,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
flash/GNUmakefile \
dhcpd/dhcpd.conf.template dhcpd/GNUmakefile \
dhcpd/dhcpd.conf.subboss.template \
install/GNUmakefile \
install/GNUmakefile install/installvars.pm install/emulab-install \
install/ops-install install/boss-install install/fs-install \
install/load-descriptors install/dump-descriptors \
install/newnode_sshkeys/GNUmakefile install/smb.conf.head \
......
......@@ -964,7 +964,7 @@ AC_SUBST(MERGE_BUILD_SANDBOX)
outfiles="$outfiles Makeconf GNUmakefile \
assign/GNUmakefile \
named/GNUmakefile firewall/GNUmakefile \
ssl/GNUmakefile ssl/mksig ssl/usercert.cnf \
ssl/GNUmakefile ssl/mksig ssl/usercert.cnf ssl/mkserial \
capture/GNUmakefile \
db/GNUmakefile \
db/EmulabConstants.pm db/EmulabFeatures.pm db/Experiment.pm \
......@@ -1113,7 +1113,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
flash/GNUmakefile \
dhcpd/dhcpd.conf.template dhcpd/GNUmakefile \
dhcpd/dhcpd.conf.subboss.template \
install/GNUmakefile \
install/GNUmakefile install/installvars.pm install/emulab-install \
install/ops-install install/boss-install install/fs-install \
install/load-descriptors install/dump-descriptors \
install/newnode_sshkeys/GNUmakefile install/smb.conf.head \
......
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2011 University of Utah and the Flux Group.
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -13,7 +13,8 @@ include $(OBJDIR)/Makeconf
ifeq ($(STANDALONE_CLEARINGHOUSE),0)
TARGETS = libinstall.pm boss-install ops-install fs-install dump-descriptors \
load-descriptors update-install update-mfs update-testbed testbed-version
load-descriptors update-install update-mfs update-testbed testbed-version \
update-ipdomain installvars.pm emulab-install
else
TARGETS = clrhouse-install
endif
......@@ -27,10 +28,10 @@ all: $(TARGETS)
include $(TESTBED_SRCDIR)/GNUmakerules
install: $(INSTALL_LIBDIR)/libinstall.pm \
$(INSTALL_LIBDIR)/installvars.pm \
$(INSTALL_SBINDIR)/update-install \
$(INSTALL_SBINDIR)/update-testbed \
$(INSTALL_SBINDIR)/testbed-version
clean:
rm -f boss-install ops-install fs-install update-install
rm -f update-testbed
rm -f $(TARGETS)
This diff is collapsed.
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010-2012 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
use English;
use Getopt::Std;
use Data::Dumper;
#
# Install Emulab.
#
sub usage()
{
print STDERR "Usage: emulab-install [-c] [-i script] boss|ops|fs\n";
print STDERR
"-c - Syntax check install scripts by loading them only\n" .
"-i <name> - Run (or check) just the one install script\n" .
"-b - Batch mode, do not ask for confirmation\n" .
"-s - Turn on makes. Huh?\n" .
"-P|-F - Set the name of the boss/ops and fs port\n" .
"-w <pswd> - Provide password instead of being asked for it\n" .
"-p <dir> - Set the package directory\n";
exit(-1);
}
my $optlist = "dsi:p:qcbP:F:w:nu";
my $debug = 0;
my $single = 0;
my $quiet = 0;
my $check = 0;
my $impotent = 0;
my $updatemode = 0;
my $batchmode = 0;
my $phasepath = "@srcdir@/phases";
my $phase;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $logfile = "/var/tmp/emulab-install.log";
my $logfp;
# Protos
sub Fatal($);
# un-taint path
$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
use libinstall;
use installvars;
#
# Parse command arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"b"})) {
$batchmode = 1;
}
if (defined($options{"c"})) {
$check = 1;
}
if (defined($options{"n"})) {
$impotent = 1;
}
if (defined($options{"q"})) {
$quiet = 1;
}
if (defined($options{"u"})) {
$updatemode = 1;
}
if (defined($options{w})) {
$password = $options{w};
}
if (defined($options{"p"})) {
$packagedir = $options{"p"};
}
if (defined($options{"i"})) {
$single = 1;
$phase = $options{"i"};
}
if (defined($options{"s"})) {
$domakes = 1;
}
if (defined($options{"F"})) {
$FS_PORT = $options{"F"};
}
if (@ARGV != 1) {
usage();
}
# Don't just charge into making ports from source by default.
if (!($check || $single || $impotent || $updatemode) &&
$packagedir eq "" && $domakes eq 0) {
print "At least one of -p and -s must be given.\n";
usage();
}
if (!($check || $single || $impotent || $updatemode) &&
$packagedir ne "" && $domakes eq 1) {
print "Only one of -p and -s can be given.\n";
usage();
}
my $server = $ARGV[0];
usage()
if (! ($server eq $BOSS_SERVERNAME ||
$server eq $OPS_SERVERNAME ||
$server eq $FS_SERVERNAME));
# Do this after we know the server name.
if (defined($options{"P"})) {
if ($server eq "boss") {
$BOSS_PORT = $options{"P"};
}
else {
$OPS_PORT = $options{"P"};
}
}
#
# Must be root if actually doing this.
#
if ($UID && !($check || $impotent)) {
Fatal("This script must be run as root! Maybe use sudo?")
}
#
# Make sure they know what they're getting into...
#
if (! ($batchmode || $check || $impotent)) {
if ($updatemode) {
print STDERR
"WARNING: This script is ONLY intended to be run your $server node,\n".
"and only if you are updating the IP addresses/subnet and/or the\n",
"domain name of your installation. Continue? [y/N] ";
}
else {
print STDERR
"WARNING: This script is ONLY intended to be run on a machine\n".
"that is being set up as a dedicated $server node. Continue? [y/N] ";
}
my $response = <STDIN>;
die "Aborted!\n" unless ($response =~ /^y/i);
}
if ($impotent) {
open(LOGFP, "> $logfile")
or Fatal("Could not open $logfile");
$logfp = *LOGFP;
SET_IMPOTENT_MODE($logfp);
}
#
# Very simple list of files. Be fancy later.
#
my @files = ();
if ($single) {
# Just the one file.
@files = ($phase);
}
elsif ($server eq "boss") {
@files = ('sperl', 'usersgroups', 'dirs', 'tftp',
'boss/ports', 'boss/portfix', 'boss/patches', 'cracklib',
'apache', 'boss/rcfiles', 'boss/rcconf', 'boss/syslog',
'boss/database', 'etchosts', 'resolvetest',
'exports', 'nfsmounts', 'boss/mibs', 'boss/crontab', 'sudoers',
'boss/ssh', 'boss/rndc', 'boss/loaderconf', 'boss/sysctlconf',
'boss/sslcerts', 'boss/mailman', 'boss/pubsub',
'boss/software',
#
# The next few items must be after the software install since
# they use testbed libraries and such.
#
'boss/dhcpd', 'boss/named', 'boss/flyspray',
'boss/firstuser', 'boss/checkupuser', 'boss/wikidocs',
'boss/experiments',
);
}
elsif ($server eq "fs") {
@files = ('sperl', 'dirs',
'fs/ports', 'fs/portfix', 'ops/rcconf',
'etchosts', 'resolvetest', 'exports', 'quotas', 'sudoers',
'samba', 'ops/ssh');
}
elsif ($server eq "ops") {
@files = ('sperl', 'usersgroups', 'dirs', 'etchosts', 'resolvetest',
'ops/ports', 'ops/portfix', 'ops/patches', 'ops/rcconf',
'ops/sendmail', 'exports', 'nfsmounts', 'ops/syslog',
'ops/crontab', 'sudoers', 'samba', 'ops/ssh', 'capture',
'ops/rcfiles', 'apache', 'ops/database', 'ops/mailman',
'ops/cvsd', 'ops/flyspray', 'ops/twiki');
}
#
# In update mode, we want to run etchosts at the very end.
#
if ($updatemode) {
push(@files, "boss/update-ipdomain")
if ($server eq $BOSS_SERVERNAME);
@files = grep(!/^etchosts$/, @files);
push(@files, "etchosts");
# We need other libraries for updating.
unshift(@INC, "@prefix@/lib");
}
#
# Now process each file starting at the start version.
#
sub RunInstall()
{
foreach my $file (@files) {
my $fullpath = "$phasepath/$file";
Fatal("No such file: $fullpath")
if (! -e "$fullpath");
if ($check) {
print "Syntax checking update $fullpath\n";
}
# Undefine this to make sure we get a new version each file.
undef &Install;
# This just loads the file.
my $return = do $fullpath;
if (!defined($return)) {
Fatal(" could not parse $fullpath: $@") if $@;
Fatal(" could not do $fullpath: $!") if $!;
}
next
if ($check);
# Then we run it. Turn off update and impotent flags.
my $result;
eval { $result = Install($server, $updatemode, $impotent); };
if ($result || $@) {
print STDERR "*** script failure: $fullpath\n";
Fatal("aborting install ...\n");
}
}
}
if ($check) {
print "Syntax checking install scripts ...\n";
RunInstall();
exit(0);
}
if ($impotent) {
print "Running (impotent mode) install scripts ...\n";
print "More detailed info saved to $logfile\n";
RunInstall();
exit(0);
}
RunInstall();
#
# This stuff might go elsewhere ...
#
if ($updatemode) {
print "-----------------------------------------------------------------\n";
print "Update completed succesfully!\n\n";
if (!PhaseWasSkipped("sslcerts")) {
print
"Since your SSL certificates were regenerated, you will need to\n".
"regenerate the user SSL certificates as well.\n";
if ($PGENISUPPORT) {
print
"\n".
"You also have ProtoGENI enabled, so those certificates were\n".
"regenerated as well. Please be sure to send the new version\n".
"of your root CA ($PREFIX/etc/emulab.pem) to $PROTOGENI_EMAIL\n".
"Then you can reregister with the clearinghouse.\n";
}
}
exit(0);
}
#
# Stuff to do at the end.
#
if ($server eq $OPS_SERVERNAME) {
exit(0)
if ($ELABINELAB);
print "-----------------------------------------------------------------\n";
print "Installation completed succesfully!\n";
print "Please reboot this machine before proceeding with boss setup\n";
print "Local mailing lists have been created, with no members, in\n";
print "$LIST_DIR . Please add members to the following lists:\n";
print map "$_\n", @LOCAL_MAILING_LISTS;
}
elsif ($server eq $FS_SERVERNAME) {
print "-----------------------------------------------------------------\n";
print "Installation completed succesfully!\n";
print "Please reboot this machine before proceeding with ops and boss setup\n";
}
elsif ($server eq $BOSS_SERVERNAME) {
print "-----------------------------------------------------------------\n";
print "Installation completed succesfully!\n";
print "Please reboot this machine before proceeding with boss setup\n";
}
exit(0);
sub Fatal($)
{
my ($msg) = @_;
die("*** $0:\n".
" $msg\n");
}
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003-2011 University of Utah and the Flux Group.
# Copyright (c) 2003-2012 University of Utah and the Flux Group.
# All rights reserved.
#
#
......@@ -10,57 +10,22 @@
#
use POSIX qw(strftime);
use Exporter;
use vars qw(@EXPORT $TOP_OBJDIR $TOP_SRCDIR $ELABINELAB
$TBROOT $LOGDIR $MAINSITE $PGENISUPPORT $GMAKE $PKG_INFO
$PORTSDIR $VARRUN $RCDIR $MYSQL $DBNAME
$PROJROOT $PROTOUSER $WAP
$SQL_UPDATE_MAJOR_REVISION $INSTALL_UPDATE_MAJOR_REVISION
$HAVE_XERCES $PROTOGENI_RPCNAME $PROTOGENI_RPCPORT
$OUTERBOSS_XMLRPCPORT
$INSTALL_APACHE_CONFIG $APACHE_VERSION $APACHE_START_COMMAND);
@EXPORT = qw($TOP_OBJDIR $TOP_SRCDIR $ELABINELAB
$TBROOT $LOGDIR $MAINSITE $PGENISUPPORT $GMAKE $PKG_INFO
$PORTSDIR $VARRUN $RCDIR $MYSQL $DBNAME
$PROJROOT $PROTOUSER $WAP
use vars qw(@EXPORT $TOP_SRCDIR $TOP_OBJDIR
$SQL_UPDATE_MAJOR_REVISION $INSTALL_UPDATE_MAJOR_REVISION
$MAGIC_TESTBED_VERSION $MAGIC_TESTBED_START $MAGIC_TESTBED_END);
@EXPORT = qw($TOP_SRCDIR $TOP_OBJDIR
$SQL_UPDATE_MAJOR_REVISION $INSTALL_UPDATE_MAJOR_REVISION
$HAVE_XERCES $PROTOGENI_RPCNAME $PROTOGENI_RPCPORT
$OUTERBOSS_XMLRPCPORT
$INSTALL_APACHE_CONFIG $APACHE_VERSION $APACHE_START_COMMAND);
# Configure variables
$TBROOT = "@prefix@";
$TOP_SRCDIR = "@top_srcdir@";
$LOGDIR = "$TBROOT/log";
$MAINSITE = @TBMAINSITE@;
$PGENISUPPORT = @PROTOGENI_SUPPORT@;
$ELABINELAB = @ELABINELAB@;
$GMAKE = "/usr/local/bin/gmake";
$PKG_INFO = "/usr/sbin/pkg_info";
$WAP = "$TBROOT/sbin/withadminprivs";
$PORTSDIR = "/usr/ports";
$VARRUN = "/var/run";
$RCDIR = "/usr/local/etc/rc.d";
$PROJROOT = "@PROJROOT_DIR@";
$PROTOUSER = "elabman";
$DBNAME = "@TBDBNAME@";
$MYSQL = "/usr/local/bin/mysql";
$HAVE_XERCES = "@HAVE_XERCES@";
$INSTALL_APACHE_CONFIG = "@INSTALL_APACHE_CONFIG@";
$APACHE_VERSION = "@APACHE_VERSION@";
$APACHE_START_COMMAND = "@APACHE_START_COMMAND@";
$PROTOGENI_RPCNAME = "@PROTOGENI_RPCNAME@";
$PROTOGENI_RPCPORT = "@PROTOGENI_RPCPORT@";
$OUTERBOSS_XMLRPCPORT = "@OUTERBOSS_XMLRPCPORT@";
$MAGIC_TESTBED_VERSION $MAGIC_TESTBED_START $MAGIC_TESTBED_END);
# Change these if the major numbers in sql/updates or install/updates
# are changed.
$SQL_UPDATE_MAJOR_REVISION = 4;
$INSTALL_UPDATE_MAJOR_REVISION = 5;
# Configure vars
$TOP_SRCDIR = "@top_srcdir@";
#
# Make sure that output gets printed right away
#
......@@ -69,9 +34,9 @@ $| = 1;
#
# Magic string that shows up in files already edited
#
my $MAGIC_TESTBED_VERSION = $INSTALL_UPDATE_MAJOR_REVISION + ".0";
my $MAGIC_TESTBED_START = "Added by Emulab - Version: ";
my $MAGIC_TESTBED_END = "End of Emulab added section";
$MAGIC_TESTBED_VERSION = $INSTALL_UPDATE_MAJOR_REVISION + ".0";
$MAGIC_TESTBED_START = "Added by Emulab - Version: ";
$MAGIC_TESTBED_END = "End of Emulab added section";
sub MAGIC_TESTBED_START { $MAGIC_TESTBED_START . $MAGIC_TESTBED_VERSION; }
sub MAGIC_TESTBED_END { $MAGIC_TESTBED_END; }
......@@ -82,6 +47,7 @@ my $updatemode = 0;
# Used by update-install to bump the version number.
sub SET_TESTBED_VERSION($) { $updatemode = $MAGIC_TESTBED_VERSION = $_[0]; }
# Set by update-install.
my $impotent = 0;
my $logfp;
......@@ -196,7 +162,7 @@ sub Phase($$$) {
$$parentNonSkipped++;
$libinstall::phaseResults{$name} = $_;
$die = 1;
$message = "Installation failed: $libinstall::reason";
$message = "$libinstall::reason";
if ($isSubPhase) {
#
# Propagate failure up the tree
......@@ -293,7 +259,7 @@ sub Phase($$$) {
# If we decided that we need to die, do that now
#
if ($die) {
print "\n##### Installation failed in phase $name. The error was:\n";
print "\n##### Installation failed in phase $name:\n";
print "$message\n";
PrintPhaseTrace();
PrintLastOutput();
......@@ -438,7 +404,7 @@ sub DoneIfIdentical($$) {
}
system("cmp -s $filename1 $filename2");
if (! $?) {
PhaseSkip("Files $filename1 and $filename2 are identical");
PhaseSkip("Files has not changed");
}
}
......@@ -474,6 +440,52 @@ sub BackUpFileFatal($)
ExecQuietFatal("/bin/cp -p $filename $backup")
if (!$impotent);
# Update the comment.
$libinstall::reason = "Backed up to $backup";
}
#
# Delete a file or fail.
#
sub DeleteFileFatal($)
{
my ($filename) = @_;
PhaseSucceed("Already deleted")
if (! -e $filename);
ExecQuietFatal("/bin/rm -f $filename")
if (!$impotent);
# Update the comment.
$libinstall::reason = "Deleted $filename";
}
#
# Rename a directory or fail.
#
sub BackupDirectoryFatal($)
{
my ($path) = @_;
my $suffix = time();
my $backup = $path . "-" . $suffix;
PhaseFail("$path does not exist")
if (! -e $path);
if (-e $backup) {
sleep(1);
$backup = $path . "-" . time();
PhaseFail("$backup already exists")
if (-e $backup);
}
ExecQuietFatal("rsync -a $path/ $backup")
if (!$impotent);
# Update the comment.
$libinstall::reason = "Backed up to $backup";
}
#
......@@ -749,7 +761,7 @@ sub GetPackage($$) {
# Print out the phase stack that got us here
#
sub PrintPhaseTrace() {
print "-------------------------------------------------- Phase Stack\n";
print "-------------------- Phase Stack ----------------------------\n";
my @tmpphase = @libinstall::phasestack;
my @tmpdescr = @libinstall::descrstack;
my ($phase, $descr);
......@@ -783,4 +795,148 @@ sub GenSecretKey()
return $key;
}
sub EscapeShellArg($)
{
my ($str) = @_;
my @chars = split('', $str);
my $result = "";
foreach my $ch (@chars) {
if ($ch eq '\'') {
$result = $result . "\'\\\'";
}
$result = $result . "$ch";
}
return "'$result'";
}
#
# Update a file. The idea here is to comment out particular lines, as
# described by the patterns, and then add the new lines.
#
# Returns undef if it succeeds, or an error string if it fails.
#
sub UpdateFileFatal($$@)
{
my ($filename, $patterns, @lines) = @_;
my $old = "";
my $new = "";
if (! -e $filename) {
PhaseFail("Bad filename passed to UpdateFile");
}
open(FH,$filename) or
PhaseFail("Cannot open $filename for reading");
while (<FH>) {
$old .= $_;
foreach my $pat (@$patterns) {
if ($_ =~ $pat) {
# XXX Deal with other file types (different comment char)
$new .= '# ';
last;
}
}
$new .= $_;
pass:
}
close(FH);
PhaseSkip("No changes made")
if ($old eq $new);
# Stash old copy.
BackUpFileFatal($filename);
DeleteFileFatal($filename);
# and create new version.
CreateFileFatal($filename, $new . join("\n", @lines));
return undef;
}
#
# Check to see if the phase is already done, as evidenced by the existence of
# the *exact* text block in the file.
#
sub DoneIfUpdated($@)
{
my ($filename, @lines) = @_;
my $block = join("\n", @lines);
if (! -e $filename) {
PhaseFail("No such file $filename");
}
open(FH, $filename) or
PhaseFail("Could not open $filename");
local $/;
my $content = <FH>;
close(FH);
PhaseSkip("Already updated")
if ($content =~ /^($block)$/m);
}
#
# Update a file by doing the equivalent of query/replace.
#
# Returns undef if it succeeds, or an error string if it fails.
#
sub QueryReplaceFileFatal($$)
{
my ($filename, $replacements) = @_;
my $old = "";
my $new = "";
if (! -e $filename) {
PhaseFail("Bad filename passed to UpdateFile");
}
open(FH,$filename) or
PhaseFail("Cannot open $filename for reading");
while (<FH>) {
$old .= $_;
foreach my $ref (@$replacements) {
my ($pat,$repl) = @$ref;
if ($_ =~ $pat) {
$_ =~ s/$pat/$repl/ee;
last;
}
}
$new .= $_;
pass:
}
close(FH);
PhaseSkip("No changes made")
if ($old eq $new);
# Stash old copy.
BackUpFileFatal($filename);
DeleteFileFatal($filename);
# and create new version.
CreateFileFatal($filename, $new);
return undef;
}
#
# Get the subject of a certificate, broken into the parts.
#
sub ParseCertificate($)
{
my ($filename) = @_;
my %results = ();
if (! -e $filename) {
return undef;
}
my $subject = `openssl x509 -subject -noout -in $filename`;
return undef
if ($?