Commit c1d21b9a authored by Mike Hibler's avatar Mike Hibler

First round of client-side support for node-local storage "slices".

Supports the three coarse-grained placements we decided on:

  "SYSVOL" is special. You can declare a single blockstore with this
       placement and it will create a "native" (ufs/ext) filesystem on
       the 4th partition of the boot disk. This is how you create an
       extra storage partition that can be captured in a custom image.
       We don't use a volume manager here because imagezip doesn't
       recognize any of them (lvm, zfs, vinum).

  "ANY" coalesces all "available" storage from all disks into a logical
        volume manager pool and dishes out storage from that for
        individual blockstores. Typically this would include, the 4th
    partition of the boot disk (if not in use) and the second hard
    drive. If the machine has more than 2 drives, it will include
    all the extra drives.

  "NONSYSVOL" coalesces all "available" storage that is NOT on the
       boot disk into a logical volume manager pool and dishes out
       storage from that for individual blockstores. This case is if
       you want to avoid interfere with the system disk.

Only implemented on FreeBSD 8/9 with "vinum" right now. It only creates
"concat" (JBOD) volumes right now.

This stuff will probably get split out into its own perl module(s) at
some point, as it is getting large.

Next up is LVM on Linux and then maybe ZFS on Freebsd.
parent 98dda6ef
......@@ -50,6 +50,7 @@ if ($EUID != 0) {
# Script specific goo
my $IQN_PREFIX = "iqn.2000-10.net.emulab";
my $OLDCONFIG = "$VARDIR/db/storage.conf";
my $STORAGEMAP = "$BOOTDIR/storagemap";
#
# Load the OS independent support library. It will load the OS dependent
......@@ -133,8 +134,12 @@ sub doboot()
# Process each command in turn. Already sorted by
# getstorageconfig().
#
my $so = os_init_storage(\@cmds);
if (!$so) {
fatal("Could not initialize storage subsystem!");
}
foreach my $cmd (@cmds) {
if (!process($cmd, 1)) {
if (!process($so, $cmd, 1)) {
fatal("Could not process storage commands!");
}
}
......@@ -149,6 +154,21 @@ sub doboot()
if (!$ret) {
fatal("Error stashing away storage config!");
}
#
# Stash mapping of block stores to local names for the convenience
# of the user.
#
if (open(MAP, ">$STORAGEMAP")) {
foreach my $cmd (@cmds) {
if (exists($cmd->{'LNAME'})) {
print MAP $cmd->{'VOLNAME'} . " /dev/" . $cmd->{'LNAME'} . "\n";
}
}
close(MAP);
} else {
warn("*** Could not create storage map: $STORAGEMAP\n");
}
}
#
......@@ -161,11 +181,10 @@ sub doshutdown()
#
# Node Reconfig Action (without rebooting).
# XXX don't do anything til we figure out what would be correct.
#
sub doreconfig()
{
docleanup(1);
doboot();
}
#
......@@ -196,11 +215,17 @@ sub docleanup($)
#
# Process each command in turn. Already sorted.
#
my $so = os_init_storage($cmdref);
if (!$so) {
fatal("Could not initialize storage subsystem!");
}
foreach my $cmd (@$cmdref) {
if (!process($cmd, 0, $doteardown)) {
if (!process($so, $cmd, 0, $doteardown)) {
fatal("Could not process storage commands!");
}
}
unlink($STORAGEMAP);
}
#
......@@ -208,49 +233,64 @@ sub docleanup($)
# This may wind up just being a call to the os-specific setup. But for now
# we do further validation of params based on what we currently implement.
#
sub process($$;$)
sub process($$$;$)
{
my ($href,$dosetup,$doteardown) = @_;
my ($so,$href,$dosetup,$doteardown) = @_;
my $class = $href->{'CLASS'};
if ($href->{'CMD'} ne "ELEMENT") {
warn("*** Only handle storage elements right now\n");
return 0;
}
if ($href->{'CMD'} eq "ELEMENT") {
# look up the host name and convert to IP
if (exists($href->{'HOSTID'})) {
my $hostip = gethostbyname($href->{'HOSTID'});
if (!defined($hostip)) {
warn("*** Cannot resolve hostname '" . $href->{'HOSTID'} . "'\n");
return 0;
}
$href->{'HOSTIP'} = inet_ntoa($hostip);
}
# look up the host name and convert to IP
my $hostip = gethostbyname($href->{'HOSTID'});
if (!defined($hostip)) {
warn("*** Cannot resolve hostname '" . $href->{'HOSTID'} . "'\n");
return 0;
}
$href->{'HOSTIP'} = inet_ntoa($hostip);
if ($class eq "SAN") {
if ($href->{'PROTO'} ne "iSCSI") {
warn("*** SAN protocol '" .
$href->{'PROTO'} . "' not implemented\n");
return 0;
}
if (!exists($href->{'HOSTID'})) {
warn("*** No iSCSI target portal specified\n");
return 0;
}
if ($href->{'UUID_TYPE'} ne "iqn" ||
$href->{'UUID'} !~ /^$IQN_PREFIX/) {
warn("*** Invalid iSCSI target name '".$href->{'UUID'}."'\n");
return 0;
}
if ($class eq "SAN") {
if ($href->{'PROTO'} ne "iSCSI") {
warn("*** SAN protocol '" .
$href->{'PROTO'} . "' not implemented\n");
}
elsif ($class eq "local") {
if ($href->{'HOSTID'} ne "localhost" ||
$href->{'UUID_TYPE'} ne "serial") {
warn("*** Unexpected parameters for local storage\n");
return 0;
}
} else {
warn("*** Unknown storage element class '$class'\n");
return 0;
}
if ($href->{'UUID_TYPE'} ne "iqn" ||
$href->{'UUID'} !~ /^$IQN_PREFIX/) {
warn("*** Invalid iSCSI target name '" . $href->{'UUID'} . "'\n");
} elsif ($href->{'CMD'} eq "SLICE") {
if ($class ne "local") {
warn("*** Unknown storage slice class '$class'\n");
return 0;
}
}
elsif ($class eq "local") {
if ($href->{'HOSTID'} ne "localhost" ||
$href->{'UUID_TYPE'} ne "serial") {
warn("*** Unexpected parameters for local storage\n");
if ($href->{'BSID'} !~ /^ALL_(SPACE|SYSVOL|NONSYSVOL)$/) {
warn("*** Unknown storage slice bsid '".$href->{'BSID'}."'\n");
return 0;
}
} else {
warn("*** Unknown storage class '$class'\n");
warn("*** Unrecognized storage command '".$href->{'CMD'}."'\n");
return 0;
}
my $exists = os_check_storage($href);
my $exists = os_check_storage($so, $href);
#
# Infrastructure failure or storage unit was partially configured.
......@@ -268,25 +308,46 @@ sub process($$;$)
#
if ($exists > 0) {
if ($dosetup) {
my $msg;
if ($href->{'PROTO'} eq "iSCSI") {
$msg = "iSCSI node";
} elsif ($href->{'PROTO'} eq "local") {
$msg = "local disk";
if ($href->{'CMD'} eq "ELEMENT") {
my $msg;
if ($href->{'CLASS'} eq "SAN" && $href->{'PROTO'} eq "iSCSI") {
$msg = "iSCSI node";
} elsif ($href->{'CLASS'} eq "local") {
$msg = "local disk";
}
print " " . $href->{'VOLNAME'} . ": $msg at /dev/" .
$href->{'LNAME'};
} elsif ($href->{'CMD'} eq "SLICE") {
if ($href->{'CLASS'} eq "local") {
print " " . $href->{'VOLNAME'} . ": /dev/" .
$href->{'LNAME'};
}
}
print " " . $href->{'VOLNAME'} . ": $msg at /dev/" .
$href->{'LNAME'} . "\n";
if ($href->{'MOUNTPOINT'}) {
print " mounted on " . $href->{'MOUNTPOINT'};
}
print "\n";
} else {
if (!os_remove_storage($href, $doteardown)) {
if (!os_remove_storage($so, $href, $doteardown)) {
warn("*** Could not remove storage device '" .
$href->{'VOLNAME'} . "'\n");
return 0;
}
if ($href->{'PROTO'} eq "iSCSI") {
if ($href->{'CLASS'} eq "SAN" && $href->{'PROTO'} eq "iSCSI") {
print " " . $href->{'VOLNAME'} .
": iSCSI node detached from /dev/" .
$href->{'LNAME'} . "\n";
} elsif ($href->{'CMD'} eq "SLICE") {
print " " . $href->{'VOLNAME'} . ": ";
if ($href->{'MOUNTPOINT'}) {
print "unmounted " . $href->{'MOUNTPOINT'} .
($doteardown ? " and " : " ");
}
if ($doteardown) {
print "destroyed /dev/" . $href->{'LNAME'};
}
print "\n";
}
}
return 1;
......@@ -297,17 +358,29 @@ sub process($$;$)
# If setting up, do it. Otherwise there is nothing to do.
#
if ($dosetup) {
if (!os_create_storage($href)) {
if (!os_create_storage($so, $href)) {
warn("*** Could not create storage device '" .
$href->{'VOLNAME'} . "'\n");
return 0;
}
if ($href->{'PROTO'} eq "iSCSI") {
print " " . $href->{'VOLNAME'} .
": iSCSI node attached as /dev/" .
$href->{'LNAME'} . "\n";
if ($href->{'CMD'} eq "ELEMENT") {
if ($href->{'CLASS'} eq "SAN" && $href->{'PROTO'} eq "iSCSI") {
print " " . $href->{'VOLNAME'} .
": iSCSI node attached as /dev/" .
$href->{'LNAME'};
}
}
elsif ($href->{'CMD'} eq "SLICE") {
if ($href->{'CLASS'} eq "local") {
print " " . $href->{'VOLNAME'} . ": /dev/" .
$href->{'LNAME'};
}
}
if ($href->{'MOUNTPOINT'}) {
print " mounted on " . $href->{'MOUNTPOINT'};
}
print "\n";
} else {
print " " . $href->{'VOLNAME'} . ": not configured\n";
}
......
......@@ -3347,13 +3347,13 @@ sub getarpinfo($;$)
# line into a hash, verifying the fields. Return sorted (by index)
# list of storage commands hashes.
#
# Format:
# ELEMENT format:
#
# CMD=ELEMENT IDX=<index> HOSTID=<some-storage-host> \
# CLASS=(SAN|local) PROTO=(iSCSI|local) \
# UUID=<unique-id> UUID_TYPE=<id-type> \
# VOLNAME=<id> VOLSIZE=<size-in-MiB> PERMS=<permissions>
#
# CLASS=(SAN|local) PROTO=(iSCSI|local) \
# UUID=<unique-id> UUID_TYPE=<id-type> \
# VOLNAME=<id> VOLSIZE=<size-in-MiB> PERMS=<permissions>
#
# Where:
#
# if CLASS=="SAN" && PROTO=="iSCSI" :
......@@ -3388,6 +3388,31 @@ sub getarpinfo($;$)
# \d+ -- size in mebibytes. Informational; could be used for sanity checking.
# PERMS :=
# <notpresent> -- this field will not show up for local elements
#
# SLICE format:
#
# CMD=SLICE IDX=<index> CLASS=local PROTO=<SAS|SCSI|SATA> \
# BSID=<local-disk-id> VOLNAME=<id> VOLSIZE=<size-in-MiB> MOUNTPOINT=<dir>
#
# Where:
#
# if CLASS=="local" :
# IDX :=
# \d+ -- monotonically increasing number indicating order of operations
# BSID :=
# (ALL_SPACE|ALL_SYSVOL|ALL_NONSYSVOL) -- i.e. where to take space from
# "ALL_SPACE" will take from any disk, possibly from multiple disks via
# use of a logical volume manager
# "ALL_SYSVOL" will take from any remaining space on the boot disk
# "ALL_NONSYSVOL" will take from any space on any non-boot disk, possibly
# from multiple disks via a LVM.
# VOLNAME :=
# string -- Emulab name for the element
# VOLSIZE :=
# \d+ -- size in mebibytes
# MOUNTPOINT :=
# If specified, implies the creation of a filesystem and mounting on
# the indicated directory.
#
sub getstorageconfig($;$) {
my ($rptr,$nocache) = @_;
......@@ -3409,8 +3434,9 @@ sub getstorageconfig($;$) {
'BSID' => '[-\w]+',
'CLASS' => '(SAN|local)',
'HOSTID' => '[-\w\.]+',
'MOUNTPOINT' => '\/[-\w\/\.]+',
'PERMS' => '(RO|RW)',
'PROTO' => '(iSCSI|local)',
'PROTO' => '(iSCSI|local|SCSI|SAS|SATA|PATA|IDE)',
'UUID' => '[-\w\.:]+',
'UUID_TYPE'=> '(iqn|serial)',
'VOLNAME' => '[-\w]+',
......
This diff is collapsed.
......@@ -39,7 +39,7 @@ use Exporter;
os_groupdel os_getnfsmounts os_islocaldir os_mountextrafs
os_fwconfig_line os_fwrouteconfig_line os_config_gre
os_get_disks os_get_disk_size os_get_partition_info os_nfsmount
os_check_storage os_create_storage os_remove_storage
os_init_storage os_check_storage os_create_storage os_remove_storage
os_get_ctrlnet_ip
os_getarpinfo os_createarpentry os_removearpentry
os_getstaticarp os_setstaticarp
......@@ -2537,7 +2537,68 @@ sub serial_to_dev($)
}
#
# os_check_storage(confighash)
# Handle one-time operations.
# Return a cookie (object) with current state of storage subsystem.
#
sub os_init_storage($)
{
my ($lref) = @_;
my $gotlocal = 0;
my $gotnonlocal = 0;
my $gotelement = 0;
my $gotslice = 0;
my $gotiscsi = 0;
my $needavol = 0;
my $needall = 0;
my %so = ();
foreach my $href (@{$lref}) {
if ($href->{'CMD'} eq "ELEMENT") {
$gotelement++;
} elsif ($href->{'CMD'} eq "SLICE") {
$gotslice++;
if ($href->{'BSID'} eq "SYSVOL" ||
$href->{'BSID'} eq "ONSYSVOL") {
$needavol = 1;
} elsif ($href->{'BSID'} eq "ANY") {
$needall = 1;
}
}
if ($href->{'CLASS'} eq "local") {
$gotlocal++;
} else {
$gotnonlocal++;
if ($href->{'PROTO'} eq "iSCSI") {
$gotiscsi++;
}
}
}
# check for local storage incompatibility
if ($needall && $needavol) {
warn("*** storage: Incompatible local volumes.\n");
return undef;
}
# initialize volume manage if needed for local slices
if ($gotlocal && $gotslice) {
;
}
if ($gotiscsi) {
if (! -x "$ISCSI") {
warn("*** storage: $ISCSI does not exist, cannot continue\n");
return undef;
}
}
$so{'INITIALIZED'} = 1;
return \%so;
}
#
# os_check_storage(sobject,confighash)
#
# Determines if the storage unit described by confighash exists and
# is properly configured. Returns zero if it doesn't exist, 1 if it
......@@ -2546,9 +2607,22 @@ sub serial_to_dev($)
# Side-effect: Creates the hash member $href->{'LNAME'} with the /dev
# name of the storage unit.
#
sub os_check_storage($)
sub os_check_storage($$)
{
my ($so,$href) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_check_storage_element($so,$href);
}
if ($href->{'CMD'} eq "SLICE") {
return os_check_storage_slice($so,$href);
}
return -1;
}
sub os_check_storage_element($$)
{
my ($href) = @_;
my ($so,$href) = @_;
my $CANDISCOVER = 0;
#
......@@ -2562,11 +2636,6 @@ sub os_check_storage($)
my $bsid = $href->{'VOLNAME'};
my @lines;
if (! -x "$ISCSI") {
warn("*** $ISCSI does not exist, cannot continue\n");
return -1;
}
#
# See if the block store exists on the indicated server.
# If not, something is very wrong, return -1.
......@@ -2641,6 +2710,19 @@ sub os_check_storage($)
return -1;
}
#
# Return 0 if does not exist
# Return 1 if exists and correct
# Return -1 otherwise
#
sub os_check_storage_slice($$)
{
my ($so,$href) = @_;
warn("*** $bsid: unsupported class '" . $href->{'CLASS'} . "'\n");
return -1;
}
#
# os_create_storage(confighash)
#
......@@ -2648,9 +2730,22 @@ sub os_check_storage($)
# (os_check_storage should be called first to verify). Return one on
# success, zero otherwise.
#
sub os_create_storage($)
sub os_create_storage($$)
{
my ($so,$href) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_create_storage_element($so, $href);
}
if ($href->{'CMD'} eq "SLICE") {
return os_create_storage_slice($so, $href);
}
return 0;
}
sub os_create_storage_element($$)
{
my ($href) = @_;
my ($so,$href) = @_;
#my $redir = "";
my $redir = ">/dev/null 2>&1";
......@@ -2701,9 +2796,30 @@ sub os_create_storage($)
return 0;
}
sub os_remove_storage($$)
sub os_create_storage_slice($$)
{
my ($href,$teardown) = @_;
my ($so,$href) = @_;
warn("*** $bsid: unsupported class '" . $href->{'CLASS'} . "'\n");
return 0;
}
sub os_remove_storage($$$)
{
my ($so,$href,$teardown) = @_;
if ($href->{'CMD'} eq "ELEMENT") {
return os_remove_storage_element($so, $href, $teardown);
}
if ($href->{'CMD'} eq "SLICE") {
return os_remove_storage_slice($so, $href, $teardown);
}
return 0;
}
sub os_remove_storage_element($$$)
{
my ($so,$href,$teardown) = @_;
#my $redir = "";
my $redir = ">/dev/null 2>&1";
......@@ -2740,6 +2856,17 @@ sub os_remove_storage($$)
return 0;
}
#
# teardown==0 means we are rebooting: unmount and shutdown gvinum
# teardown==1 means we are reconfiguring and will be destroying everything
#
sub os_remove_storage_slice($$$)
{
my ($so,$href,$teardown) = @_;
return 0;
}
sub mysystem($)
{
my ($cmd) = @_;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment