Commit 9cf8f9c6 authored by Mike Hibler's avatar Mike Hibler

Rework client-side storage scripts to semi-coexist with mkextrafs uses.

Broke rc.storage into two phases, local blockstores and remote blockstores.
Setup of the former will also pick a best candidate for an old-school
"extrafs" and put the info in /var/emulab/boot/extrafs. This will be a
single line with one of DISK=foo, PART=foo, or FS=foo depending on whether
it found an available full disk, disk partition, or mounted filesystem
that we can use for mkextrafs (in the first two cases) or where we can
mooch off of (the last). This is only used in os_mountextrafs() right now;
i.e., I have NOT changed the mkextrafs script. So explicit invocations
by the user could still screw things up.

I have tested this with local blockstores and a non-nfs experiment
on both Linux and FreeBSD to make sure the most common sharing of space
works. I have not made any new images and I have not yet tested to make
sure I did not break non-blockstore, non-nfs experiments (i.e., where
we really should run mkextrafs).

So maybe don't make any new images til I get back, or else be prepared
to clean up after me.
parent 7999e43d
#
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -41,7 +41,8 @@ SCRIPTS = $(addprefix $(SRCDIR)/, \
rc.keys rc.trafgen rc.tarfiles rc.rpms rc.progagent \
rc.startcmd rc.simulator rc.topomap rc.firewall \
rc.tiptunnels rc.trace rc.motelog rc.fakejail \
rc.tpmsetup rc.blobs rc.diskagent rc.storage)
rc.tpmsetup rc.blobs rc.diskagent \
rc.storage rc.storagelocal rc.storageremote)
SUBBOSS_SCRIPTS = $(addprefix $(SRCDIR)/, \
rc.config rc.misc rc.mounts rc.accounts rc.route \
......
......@@ -123,19 +123,21 @@ elsif (WINDOWS()) {
}
else {
@bootscripts = ("rc.firewall", "rc.tpmsetup",
"rc.misc", "rc.localize", "rc.keys", "rc.mounts",
"rc.blobs",
"rc.topomap", "rc.accounts",
"rc.misc", "rc.localize", "rc.keys",
#
# local blockstore setup should happen before rc.mounts,
# rc.blobs and rc.trace.
#
"rc.storagelocal",
"rc.mounts", "rc.blobs", "rc.topomap", "rc.accounts",
"rc.route", "rc.tunnels", "rc.ifconfig", "rc.delays",
"rc.hostnames",
#
# rc.storage should be after rc.ifconfig (storage may
# come from a remote host; e.g., iSCSI) and rc.hostnames
# (iSCSI target identified with a short name). rc.storage
# handles its own mounts, so it is okay that it is called
# after rc.mounts.
# remote blockstore setup should be after rc.ifconfig
# and rc.hostnames. rc.storage handles its own mounts,
# so it is okay that it is called after rc.mounts.
#
"rc.storage",
"rc.storageremote",
"rc.trace", "rc.syncserver", "rc.trafgen",
"rc.tarfiles", "rc.rpms", "rc.progagent", "rc.linkagent",
"rc.tiptunnels", "rc.motelog", "rc.simulator",
......
......@@ -29,11 +29,14 @@ use Storable;
sub usage()
{
print "Usage: " .
scriptname() . " [-j vnodeid] boot|shutdown|reconfig|reset\n";
scriptname() . " [-LR] [-j vnodeid] boot|shutdown|reconfig|reset\n";
exit(1);
}
my $optlist = "j:";
my $optlist = "LRj:";
my $action = "boot";
my $dolocal = 1;
my $doremote = 1;
my $typestr = "";
# Turn off line buffering on output
$| = 1;
......@@ -47,10 +50,6 @@ if ($EUID != 0) {
" Must be root to run this script!\n");
}
# Script specific goo
my $OLDCONFIG = "$VARDIR/db/storage.conf";
my $STORAGEMAP = "$BOOTDIR/storagemap";
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
......@@ -61,6 +60,12 @@ use liblocstorage;
use libtmcc;
use librc;
# Script specific goo
my $OLDCONFIG = "$VARDIR/db/storage.conf";
my $STORAGEMAP = TMSTORAGEMAP();
my $DISKINFO = TMDISKINFO();
my $EXTRAFS = TMEXTRAFS();
#
# Not all clients support this.
#
......@@ -72,6 +77,7 @@ sub doboot();
sub doshutdown();
sub doreconfig();
sub docleanup($);
sub doinfo();
# Parse command line.
if (! getopts($optlist, \%options)) {
......@@ -81,6 +87,16 @@ if (defined($options{'j'})) {
my $vnodeid = $options{'j'};
libsetup_setvnodeid($vnodeid);
}
if (defined($options{'L'})) {
$dolocal = 1;
$doremote = 0;
$typestr = "local ";
}
if (defined($options{'R'})) {
$dolocal = 0;
$doremote = 1;
$typestr = "remote ";
}
# Allow default above.
if (@ARGV) {
$action = $ARGV[0];
......@@ -126,9 +142,14 @@ SWITCH: for ($action) {
};
# XXX non-standard, for debugging
/^fullreset$/i && do {
$dolocal = $doremote = 1;
docleanup(2);
last SWITCH;
};
/^info$/i && do {
doinfo();
last SWITCH;
};
fatal("Invalid action: $action\n");
}
exit(0);
......@@ -140,36 +161,57 @@ sub doboot()
{
my $bossip;
print STDOUT "Checking Testbed storage configuration ... \n";
print STDOUT "Checking Testbed ${typestr}storage configuration ... \n";
# XXX uncomment this for tmp testing with alternate tmcd
#configtmcc("portnum",7778);
my @cmds;
if (getstorageconfig(\@cmds) != 0) {
my @allcmds = ();
if (getstorageconfig(\@allcmds) != 0) {
fatal("Error grabbing storage config!");
}
my @cmds = ();
if (@allcmds > 0) {
foreach my $href (@allcmds) {
if ($dolocal && $href->{'CLASS'} eq "local") {
push(@cmds, $href);
}
if ($doremote && $href->{'CLASS'} ne "local") {
push(@cmds, $href);
}
}
}
#
# We could have just rebooted as the result of a swapmod operation.
# We read in any old config so we can see if we have added or removed
# any blockstores.
#
my $ocmdref = [];
my @ocmds = ();
if (-r "$OLDCONFIG") {
$ocmdref = eval { Storable::retrieve($OLDCONFIG); };
my $ocmdref = eval { Storable::retrieve($OLDCONFIG); };
if ($@ || !$ocmdref) {
warn "*** Could not read old config, ignoring...\n";
unlink($OLDCONFIG);
$ocmdref = [];
} else {
foreach my $ohref (@$ocmdref) {
if ($dolocal && $ohref->{'CLASS'} eq "local") {
push(@ocmds, $ohref);
}
if ($doremote && $ohref->{'CLASS'} ne "local") {
push(@ocmds, $ohref);
}
}
}
}
#
# No blockstores old or new.
#
if (!@cmds && !@$ocmdref) {
if (!@cmds && !@ocmds) {
#warn("*** No storageconfig output - nothing to do");
unlink($OLDCONFIG);
return;
}
......@@ -179,13 +221,13 @@ sub doboot()
# blockstores and remove them up front, in case new blockstores are
# counting on reusing their space.
#
if (@$ocmdref > 0) {
if (@ocmds > 0) {
my @dcmds = ();
#
# For each element of the old list, see if it exists in the new.
#
OUTER: foreach my $ohref (@$ocmdref) {
OUTER: foreach my $ohref (@ocmds) {
foreach my $href (@cmds) {
next if ($ohref->{'VOLNAME'} ne $href->{'VOLNAME'});
......@@ -237,9 +279,10 @@ sub doboot()
}
#
# Save config
# Save config. Note that we always write the full set regardless
# of whether we were processing local, remote or both.
#
my $ret = eval { Storable::store(\@cmds, $OLDCONFIG); };
my $ret = eval { Storable::store(\@allcmds, $OLDCONFIG); };
if ($@) {
fatal("$@");
}
......@@ -247,18 +290,30 @@ sub doboot()
fatal("Error stashing away storage config!");
}
#
# Write out some info for the benefit of other scripts.
#
my $dinfo = os_get_diskinfo($so);
#
# Stash mapping of block stores to local names for the convenience
# of the user.
# of the user. The storage map always gets all blockstores.
#
if (open(MAP, ">$STORAGEMAP")) {
foreach my $cmd (@cmds) {
foreach my $cmd (@allcmds) {
print MAP $cmd->{'VOLNAME'};
if (exists($cmd->{'LVDEV'})) {
print MAP " " . $cmd->{'LVDEV'};
}
if (exists($cmd->{'MOUNTPOINT'})) {
print MAP " " . $cmd->{'MOUNTPOINT'};
my $dev = $cmd->{'LVDEV'};
if ($dev) {
$dev =~ s/^\/dev\///;
if (exists($dinfo->{$dev})) {
$dinfo->{$dev}->{'mountpoint'} = $cmd->{'MOUNTPOINT'};
}
}
}
print MAP "\n";
}
......@@ -266,6 +321,86 @@ sub doboot()
} else {
warn("*** Could not create storage map: $STORAGEMAP\n");
}
#
# Since we went to a lot of trouble to compute it, also dump
# the diskinfo table for use by others.
#
if (open(MAP, ">$DISKINFO")) {
my $bdisk = $so->{'BOOTDISK'};
foreach my $dev (keys %$dinfo) {
my $type = $dinfo->{$dev}->{'type'};
my $lev = $dinfo->{$dev}->{'level'};
my $size = $dinfo->{$dev}->{'size'};
my $inuse = $dinfo->{$dev}->{'inuse'};
my $isbdisk = ($bdisk && $dev eq $bdisk) ? 1 : 0;
my $mpoint = exists($dinfo->{$dev}->{'mountpoint'}) ?
$dinfo->{$dev}->{'mountpoint'} : "none";
print MAP "NAME=$dev SIZE=$size TYPE=$type LEVEL=$lev INUSE=$inuse BOOTDISK=$isbdisk MOUNTPOINT=$mpoint\n";
}
close(MAP);
} else {
warn("*** Could not create diskinfo file: $DISKINFO\n");
}
#
# For the benefit of old scripts that use mkextrafs, identify
# an unused device or filesystem that can be used for extra space.
#
if ($dolocal && open(MAP, ">$EXTRAFS")) {
my $size;
#
# First see if there is an unused partition or disk.
# Remember the largest.
#
my $edev;
my $isdisk = 0;
$size = 0;
foreach my $dev (keys %$dinfo) {
my $href = $dinfo->{$dev};
if ($href->{'inuse'} == 0 && $href->{'size'} > $size) {
$edev = $dev;
if ($href->{'type'} eq "DISK") {
$isdisk = 1;
}
}
}
if ($edev) {
if ($isdisk) {
print MAP "DISK=$edev\n";
} else {
print MAP "PART=$edev\n";
}
}
#
# Otherwise we need to find a mounted local blockstore with space.
# Pick the largest one, including the root FS.
#
else {
$size = 0;
my $bdisk = $so->{'BOOTDISK'};
if ($bdisk) {
my $bpart = "${bdisk}s1a";
if (exists($dinfo->{$bpart})) {
$mntpt = "/";
$size = $dinfo->{$bpart}->{'size'};
}
}
foreach my $dev (keys %$dinfo) {
my $href = $dinfo->{$dev};
if (exists($href->{'mountpoint'}) &&
$href->{'type'} ne "iSCSI" && $href->{'size'} > $size) {
$mntpt = $href->{'mountpoint'};
$size = $href->{'size'};
}
}
if ($mntpt && $size) {
print MAP "FS=$mntpt\n";
}
}
close(MAP);
}
}
#
......@@ -288,6 +423,41 @@ sub doreconfig()
doboot();
}
sub doinfo()
{
my @allcmds = ();
if (getstorageconfig(\@allcmds) != 0) {
fatal("Error grabbing storage config!");
}
my @cmds = ();
if (@allcmds > 0) {
foreach my $href (@allcmds) {
if ($dolocal && $href->{'CLASS'} eq "local") {
push(@cmds, $href);
}
if ($doremote && $href->{'CLASS'} ne "local") {
push(@cmds, $href);
}
}
}
if (@cmds > 0) {
my $so = os_init_storage(\@cmds);
if (!$so) {
fatal("Could not initialize storage subsystem!");
}
print STDERR "Blockstores:\n";
foreach my $href (@cmds) {
print STDERR " #", $href->{'IDX'}, ": VOLNAME=", $href->{'VOLNAME'};
foreach my $k (sort keys %$href) {
next if ($k eq "VOLNAME" || $k eq "IDX");
print STDERR " $k=", $href->{$k};
}
print STDERR "\n";
}
os_show_storage($so);
}
}
#
# Node cleanup action (node is reset to clean state, as if just allocated).
#
......@@ -340,21 +510,34 @@ sub docleanup($)
print "Forcing teardown of storage, ignore errors...\n";
}
my @cmds = ();
foreach my $href (@$cmdref) {
if ($dolocal && $href->{'CLASS'} eq "local") {
push(@cmds, $href);
}
if ($doremote && $href->{'CLASS'} ne "local") {
push(@cmds, $href);
}
}
if (!@cmds) {
return;
}
#
# Process each command in turn. Already sorted.
# XXX do we need to reverse the order for teardown?
#
my $so = os_init_storage($cmdref);
my $so = os_init_storage(\@cmds);
if (!$so) {
fatal("Could not initialize storage subsystem!");
}
foreach my $cmd (@$cmdref) {
foreach my $cmd (@cmds) {
if (!process($so, $cmd, 0, $doteardown)) {
fatal("Could not process storage commands!");
}
}
unlink($STORAGEMAP);
unlink($STORAGEMAP, $DISKINFO);
}
#
......@@ -484,9 +667,9 @@ sub process($$$$)
if ($href->{'MOUNTPOINT'}) {
print ", ";
if ($href->{'FSTYPE'}) {
print $href->{'FSTYPE'} . " FS";
print $href->{'FSTYPE'} . " FS ";
}
print " mounted on " . $href->{'MOUNTPOINT'};
print "mounted on " . $href->{'MOUNTPOINT'};
} else {
print ", exists";
}
......
#!/usr/bin/perl -w
#
# Copyright (c) 2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
#
# Hack wrapper for rc.storage to do ONLY local storage.
#
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
my @args = ("$BINDIR/rc/rc.storage", "-L", @ARGV);
exec(@args);
exit(-1);
#!/usr/bin/perl -w
#
# Copyright (c) 2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
#
# Hack wrapper for rc.storage to do ONLY remote storage.
#
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
my @args = ("$BINDIR/rc/rc.storage", "-R", @ARGV);
exec(@args);
exit(-1);
......@@ -40,7 +40,8 @@ use Exporter;
gettraceconfig genhostsfile getmotelogconfig calcroutes fakejailsetup
getlocalevserver genvnodesetup getgenvnodeconfig stashgenvnodeconfig
getlinkdelayconfig getloadinfo getbootwhat getnodeattributes
copyfilefromnfs getnodeuuid getarpinfo getstorageconfig
copyfilefromnfs getnodeuuid getarpinfo
getstorageconfig getstoragediskinfo
getmanifest fetchmanifestblobs runbootscript runhooks
build_fake_macs getenvvars
......@@ -59,6 +60,7 @@ use Exporter;
TMROUTECONFIG TMLINKDELAY TMDELMAP TMTOPOMAP TMLTMAP TMLTPMAP
TMGATEDCONFIG TMSYNCSERVER TMKEYHASH TMNODEID TMNODEUUID TMEVENTKEY
TMCREATOR TMSWAPPER TMFWCONFIG TMGENVNODECONFIG
TMSTORAGEMAP TMDISKINFO TMEXTRAFS
INXENVM INVZVM
);
......@@ -285,6 +287,9 @@ sub ISDELAYNODEPATH() { "$BOOTDIR/isdelaynode"; }
sub TMTOPOMAP() { "$BOOTDIR/topomap";}
sub TMLTMAP() { "$BOOTDIR/ltmap";}
sub TMLTPMAP() { "$BOOTDIR/ltpmap";}
sub TMSTORAGEMAP() { "$BOOTDIR/storagemap";}
sub TMDISKINFO() { "$BOOTDIR/diskinfo";}
sub TMEXTRAFS() { "$BOOTDIR/extrafs";}
#
# This path is valid only *outside* the jail when its setup.
......@@ -565,6 +570,7 @@ sub cleanup_node ($) {
print STDOUT "Cleaning node; removing configuration files\n";
unlink TMUSESFS, TMROLE, ISSIMTRAFGENPATH, ISDELAYNODEPATH;
unlink TMSTORAGEMAP, TMDISKINFO, TMEXTRAFS;
#
# If scrubbing, also remove the password/group files and DBs so
......@@ -3571,6 +3577,43 @@ sub getstorageconfig($;$) {
return 0;
}
#
# If the storage subsystem is in use, read the contents of the diskinfo
# file it creates and create a hash of info keyed by device name.
#
sub getstoragediskinfo()
{
my $infofile = TMDISKINFO();
my %dinfo = ();
if (-f "$infofile" && open(FD, "<$infofile")) {
while ($line = <FD>) {
chomp($line);
my @kvs = split(/\s+/, $line);
my %thisone = ();
foreach my $kv (@kvs) {
my ($key,$val) = split(/=/, $kv);
if (!defined($val)) {
warn("*** WARNING: malformed key-val pair in diskinfo: '$kv'\n");
close(FD);
return undef;
}
$thisone{$key} = $val;
}
if (exists($thisone{"NAME"})) {
$dinfo{$thisone{"NAME"}} = \%thisone;
} else {
warn("*** WARNING: malformed diskinfo line: '$line'\n");
close(FD);
return undef;
}
}
close(FD);
return \%dinfo;
}
return undef;
}
#
# Fork a process to exec a command. Return the pid to wait on.
#
......
......@@ -1149,24 +1149,68 @@ sub os_nfsmount($$)
# already been done. Returns the resulting mount point (which may be
# different than what was specified as an argument if it already existed).
#
# Note that in FreeBSD, the extra partition is either:
# XXd0s4a (FreeBSD 10+)
# XXd0s4e (FreeBSD 9-)
#
sub os_mountextrafs($)
{
my $dir = shift;
my $mntpt = "";
my $log = "$VARDIR/logs/mkextrafs.log";
my $disk = "";
my $part = "";
#
# If the extrafs file was written, use the info from there.
#
my $extrafs = libsetup::TMEXTRAFS();
if (-f "$extrafs" && open(FD, "<$extrafs")) {
my $line = <FD>;
close(FD);
chomp($line);
if ($line =~ /^PART=(.*)/) {
$part = $1;
goto makeit;
}
if ($line =~ /^DISK=(.*)/) {
$disk = $1;
goto makeit;
}
if ($line =~ /^FS=(.*)/) {
$mntpt = $1;
}