Commit d746f6ca authored by Mike Hibler's avatar Mike Hibler

Linux+LVM clientside for node-local storage.

Also, refactored the OS specific stuff into a new liblocstorage.pm
which is included by rc.stoarge. Note that there is no OS-independent
libstorage.pm, at least for now.
parent c1d21b9a
......@@ -58,6 +58,7 @@ my $STORAGEMAP = "$BOOTDIR/storagemap";
#
use libsetup;
use liblocsetup;
use liblocstorage;
use libtmcc;
use librc;
......@@ -238,6 +239,14 @@ sub process($$$;$)
my ($so,$href,$dosetup,$doteardown) = @_;
my $class = $href->{'CLASS'};
#
# XXX get rid of any trailing slashes on the mountpoint so it
# doesn't cause grief for the OS-dependent backend.
#
if (exists($href->{'MOUNTPOINT'})) {
$href->{'MOUNTPOINT'} =~ s#/+$##;
}
if ($href->{'CMD'} eq "ELEMENT") {
# look up the host name and convert to IP
if (exists($href->{'HOSTID'})) {
......@@ -281,7 +290,7 @@ sub process($$$;$)
warn("*** Unknown storage slice class '$class'\n");
return 0;
}
if ($href->{'BSID'} !~ /^ALL_(SPACE|SYSVOL|NONSYSVOL)$/) {
if ($href->{'BSID'} !~ /^(ANY|SYSVOL|NONSYSVOL)$/) {
warn("*** Unknown storage slice bsid '".$href->{'BSID'}."'\n");
return 0;
}
......@@ -329,6 +338,7 @@ sub process($$$;$)
}
print "\n";
} else {
print " Deconfiguring '" . $href->{'VOLNAME'} . "'...\n";
if (!os_remove_storage($so, $href, $doteardown)) {
warn("*** Could not remove storage device '" .
$href->{'VOLNAME'} . "'\n");
......@@ -358,6 +368,7 @@ sub process($$$;$)
# If setting up, do it. Otherwise there is nothing to do.
#
if ($dosetup) {
print " Configuring '" . $href->{'VOLNAME'} . "'...\n";
if (!os_create_storage($so, $href)) {
warn("*** Could not create storage device '" .
$href->{'VOLNAME'} . "'\n");
......
......@@ -3400,11 +3400,11 @@ sub getarpinfo($;$)
# IDX :=
# \d+ -- monotonically increasing number indicating order of operations
# BSID :=
# (ALL_SPACE|ALL_SYSVOL|ALL_NONSYSVOL) -- i.e. where to take space from
# "ALL_SPACE" will take from any disk, possibly from multiple disks via
# (ANY|SYSVOL|NONSYSVOL) -- i.e. where to take space from
# "ANY" will take from any disk, possibly from multiple disks via
# use of a logical volume manager
# "ALL_SYSVOL" will take from any remaining space on the boot disk
# "ALL_NONSYSVOL" will take from any space on any non-boot disk, possibly
# "SYSVOL" will take from any remaining space on the boot disk
# "NONSYSVOL" will take from any space on any non-boot disk, possibly
# from multiple disks via a LVM.
# VOLNAME :=
# string -- Emulab name for the element
......
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -239,6 +239,7 @@ script-install: dir-install $(SCRIPTS)
$(INSTALL) -m 755 $(SRCDIR)/reboot_prepare $(BINDIR)/reboot_prepare
$(INSTALL) -m 755 $(SRCDIR)/ixpboot $(BINDIR)/ixpboot
$(INSTALL) -m 755 $(SRCDIR)/liblocsetup.pm $(BINDIR)/liblocsetup.pm
$(INSTALL) -m 755 $(SRCDIR)/liblocstorage.pm $(BINDIR)/liblocstorage.pm
$(INSTALL) -m 755 $(SRCDIR)/libvnode.pm $(BINDIR)/libvnode.pm
$(INSTALL) -m 755 $(SRCDIR)/rc.delayagent $(BINDIR)/rc/rc.delayagent
$(INSTALL) -m 755 $(SRCDIR)/rc.healthd $(BINDIR)/rc/rc.healthd
......
This diff is collapsed.
This diff is collapsed.
......@@ -231,6 +231,7 @@ sysetc-remove:
script-install: dir-install $(SCRIPTS)
$(INSTALL) -m 755 $(SRCDIR)/liblocsetup.pm $(BINDIR)/liblocsetup.pm
$(INSTALL) -m 755 $(SRCDIR)/liblocstorage.pm $(BINDIR)/liblocstorage.pm
$(INSTALL) -m 755 $(SRCDIR)/rc.ipod $(BINDIR)/rc/rc.ipod
$(INSTALL) -m 755 $(SRCDIR)/rc.kname $(BINDIR)/rc/rc.kname
$(INSTALL) -m 755 $(SRCDIR)/prepare $(BINDIR)/prepare
......
......@@ -39,7 +39,6 @@ use Exporter;
os_groupdel os_getnfsmounts os_islocaldir os_mountextrafs
os_fwconfig_line os_fwrouteconfig_line os_config_gre
os_get_disks os_get_disk_size os_get_partition_info os_nfsmount
os_init_storage os_check_storage os_create_storage os_remove_storage
os_get_ctrlnet_ip
os_getarpinfo os_createarpentry os_removearpentry
os_getstaticarp os_setstaticarp
......@@ -2458,431 +2457,4 @@ sub os_setstaticarp($$)
return 0;
}
#
#
# To find the block stores exported from a target portal:
#
# iscsiadm -m discovery -t sendtargets -p <storage-host>
#
# Display all data for a given node record:
#
# iscsiadm -m node -T <iqn> -p <storage-host>
#
# Here are the commands to add a remote iSCSI target, set it to be
# mounted at startup, and startup a session (login):
#
# iscsiadm -m node -T <iqn> -p <storage-host> -o new
# iscsiadm -m node -T <iqn> -p <storage-host> -o update \
# -n node.startup -v automatic
# iscsiadm -m node -T <iqn> -p <storage-host> -l
#
# To show active sessions:
#
# iscsiadm -m session
#
# To stop a specific session (logout) and kill its record:
#
# iscsiadm -m node -T <iqn> -p <storage-host> -u
# iscsiadm -m node -T <iqn> -p <storage-host> -o delete
#
# To stop all iscsi sessions and kill all records:
#
# iscsiadm -m node -U all
# iscsiadm -m node -o delete
#
# Once a blockstore is added, you have to use "fdisk -l" or possibly
# crap out in /proc to discover what the name of the disk is. I've been
# looking for uniform way to query the set of disks on a machine, but
# haven't quite figured this out yet. The closest thing I've found is
# "fdisk -l". There are some libraries and such, but there are enough
# of them that I'm not sure which one is best / most standard.
#
sub iscsi_to_dev($)
{
my ($session) = @_;
#
# XXX this is a total hack and maybe distro dependent?
#
my @lines = `ls -l /sys/block/sd? 2>&1`;
foreach (@lines) {
if (m#/sys/block/(sd.) -> ../devices/platform/host\d+/session(\d+)#) {
if ($2 == $session) {
return $1;
}
}
}
return undef;
}
sub serial_to_dev($)
{
my ($sn) = @_;
#
# XXX this is a total hack and maybe distro dependent?
#
my @lines = `ls -l /dev/disk/by-id/ 2>&1`;
foreach (@lines) {
if (m#.*_([^_\s]+) -> ../../(sd.)$#) {
if ($1 eq $sn) {
return $2;
}
}
}
return undef;
}
#
# Handle one-time operations.
# Return a cookie (object) with current state of storage subsystem.
#
sub os_init_storage($)
{
my ($lref) = @_;
my $gotlocal = 0;
my $gotnonlocal = 0;
my $gotelement = 0;
my $gotslice = 0;
my $gotiscsi = 0;
my $needavol = 0;
my $needall = 0;
my %so = ();
foreach my $href (@{$lref}) {
if ($href->{'CMD'} eq "ELEMENT") {
$gotelement++;
} elsif ($href->{'CMD'} eq "SLICE") {
$gotslice++;
if ($href->{'BSID'} eq "SYSVOL" ||
$href->{'BSID'} eq "ONSYSVOL") {
$needavol = 1;
} elsif ($href->{'BSID'} eq "ANY") {
$needall = 1;
}
}
if ($href->{'CLASS'} eq "local") {
$gotlocal++;
} else {
$gotnonlocal++;
if ($href->{'PROTO'} eq "iSCSI") {
$gotiscsi++;
}
}
}
# check for local storage incompatibility
if ($needall && $needavol) {
warn("*** storage: Incompatible local volumes.\n");
return undef;
}
# initialize volume manage if needed for local slices
if ($gotlocal && $gotslice) {
;
}
if ($gotiscsi) {
if (! -x "$ISCSI") {
warn("*** storage: $ISCSI does not exist, cannot continue\n");
return undef;
}
}
$so{'INITIALIZED'} = 1;
return \%so;
}
#
# os_check_storage(sobject,confighash)
#
# Determines if the storage unit described by confighash exists and
# is properly configured. Returns zero if it doesn't exist, 1 if it
# exists and is correct, -1 otherwise.
#
# Side-effect: Creates the hash member $href->{'LNAME'} with the /dev
# name of the storage unit.
#
sub os_check_storage($$)
{
my ($so,$href) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_check_storage_element($so,$href);
}
if ($href->{'CMD'} eq "SLICE") {
return os_check_storage_slice($so,$href);
}
return -1;
}
sub os_check_storage_element($$)
{
my ($so,$href) = @_;
my $CANDISCOVER = 0;
#
# iSCSI:
# make sure the IQN exists
# make sure a session exists
#
if ($href->{'PROTO'} eq "iSCSI") {
my $hostip = $href->{'HOSTIP'};
my $uuid = $href->{'UUID'};
my $bsid = $href->{'VOLNAME'};
my @lines;
#
# See if the block store exists on the indicated server.
# If not, something is very wrong, return -1.
#
# Note that the server may not support discovery. If not, we don't
# do it since it is only a sanity check anyway.
#
if ($CANDISCOVER) {
@lines = `$ISCSI -m discovery -t sendtargets -p $hostip 2>&1`;
if ($? != 0) {
warn("*** could not find exported iSCSI block stores\n");
return -1;
}
if (!grep(/$uuid/, @lines)) {
warn("*** could not find iSCSI block store '$uuid'\n");
return -1;
}
}
#
# It exists, are we connected to it?
# If not, we have not done the one-time initialization, return 0.
#
my $session;
@lines = `$ISCSI -m session 2>&1`;
foreach (@lines) {
if (/^tcp: \[(\d+)\].*$uuid$/) {
$session = $1;
last;
}
}
if (!defined($session)) {
return 0;
}
#
# We have a session, everything has been setup, return 1.
#
my $dev = iscsi_to_dev($session);
if (defined($dev)) {
$href->{'LNAME'} = $dev;
return 1;
}
#
# Otherwise, we are in some indeterminite state, return -1.
#
warn("*** $bsid: found iSCSI session but could not determine local device\n");
return -1;
}
#
# local disk:
# make sure disk exists
#
if ($href->{'PROTO'} eq "local") {
my $bsid = $href->{'VOLNAME'};
my $sn = $href->{'UUID'};
my $dev = serial_to_dev($sn);
if (defined($dev)) {
$href->{'LNAME'} = $dev;
return 1;
}
# for physical disks, there is no way to "create" it so return error
warn("*** $bsid: could not find HD with serial '$sn'\n");
return -1;
}
warn("*** Only support iSCSI now\n");
return -1;
}
#
# Return 0 if does not exist
# Return 1 if exists and correct
# Return -1 otherwise
#
sub os_check_storage_slice($$)
{
my ($so,$href) = @_;
warn("*** $bsid: unsupported class '" . $href->{'CLASS'} . "'\n");
return -1;
}
#
# os_create_storage(confighash)
#
# Create the storage unit described by confighash. Unit must not exist
# (os_check_storage should be called first to verify). Return one on
# success, zero otherwise.
#
sub os_create_storage($$)
{
my ($so,$href) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_create_storage_element($so, $href);
}
if ($href->{'CMD'} eq "SLICE") {
return os_create_storage_slice($so, $href);
}
return 0;
}
sub os_create_storage_element($$)
{
my ($so,$href) = @_;
#my $redir = "";
my $redir = ">/dev/null 2>&1";
if ($href->{'PROTO'} eq "iSCSI") {
my $hostip = $href->{'HOSTIP'};
my $uuid = $href->{'UUID'};
my $bsid = $href->{'VOLNAME'};
#
# Perform one time iSCSI operations
#
if (mysystem("$ISCSI -m node -T $uuid -p $hostip -o new $redir") ||
mysystem("$ISCSI -m node -T $uuid -p $hostip -o update -n node.startup -v automatic $redir") ||
mysystem("$ISCSI -m node -T $uuid -p $hostip -l $redir")) {
warn("*** Could not perform first-time initialization of block store $bsid (uuid=$uuid)\n");
return 0;
}
#
# Make sure we are connected
#
@lines = `$ISCSI -m session 2>&1`;
foreach (@lines) {
if (/^tcp: \[(\d+)\].*$uuid$/) {
$session = $1;
last;
}
}
if (!defined($session)) {
warn("*** Could not locate session for block store $bsid (uuid=$uuid)\n");
return 0;
}
#
# Map to a local device.
#
my $dev = iscsi_to_dev($session);
if (!defined($dev)) {
warn("*** $bsid: could not map iSCSI session to device\n");
return 0;
}
$href->{'LNAME'} = $dev;
return 1;
}
warn("*** Only support iSCSI now\n");
return 0;
}
sub os_create_storage_slice($$)
{
my ($so,$href) = @_;
warn("*** $bsid: unsupported class '" . $href->{'CLASS'} . "'\n");
return 0;
}
sub os_remove_storage($$$)
{
my ($so,$href,$teardown) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_remove_storage_element($so, $href, $teardown);
}
if ($href->{'CMD'} eq "SLICE") {
return os_remove_storage_slice($so, $href, $teardown);
}
return 0;
}
sub os_remove_storage_element($$$)
{
my ($so,$href,$teardown) = @_;
#my $redir = "";
my $redir = ">/dev/null 2>&1";
if ($href->{'PROTO'} eq "iSCSI") {
my $hostip = $href->{'HOSTIP'};
my $uuid = $href->{'UUID'};
my $bsid = $href->{'VOLNAME'};
#
# Logout of the session.
# XXX continue even if we could not logout.
#
if (mysystem("$ISCSI -m node -T $uuid -p $hostip -u $redir")) {
warn("*** $bsid: Could not logout iSCSI sesssion (uuid=$uuid)\n");
}
if ($teardown &&
mysystem("$ISCSI -m node -T $uuid -p $hostip -o delete $redir")) {
warn("*** $bsid: could not perform teardown of iSCSI block store (uuid=$uuid)\n");
return 0;
}
return 1;
}
#
# Nothing to do (yet) for a local disk
#
if ($href->{'PROTO'} eq "local") {
return 1;
}
warn("*** Only support iSCSI now\n");
return 0;
}
#
# teardown==0 means we are rebooting: unmount and shutdown gvinum
# teardown==1 means we are reconfiguring and will be destroying everything
#
sub os_remove_storage_slice($$$)
{
my ($so,$href,$teardown) = @_;
return 0;
}
sub mysystem($)
{
my ($cmd) = @_;
if (0) {
print STDERR "CMD: $cmd\n";
}
return system($cmd);
}
sub mybacktick($)
{
my ($cmd) = @_;
if (0) {
print STDERR "CMD: $cmd\n";
}
return `$cmd`;
}
1;
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment