Skip to content
GitLab
Commit a2bccf93 authored by Mike Hibler's avatar Mike Hibler
Browse files

First cut at refactoring FreeNAS code. 

Move the command execution and storage-related stuff from
libvnode to libfreenas. The interface and vlan stuff is still in
libvnode since it is currently the only one that uses it.
parent 60eace23
Expand all Hide whitespace changes
Inline Side-by-side
clientside/tmcc/freenas8/bscontrol.proxy.pl
View file @ a2bccf93
......@@ -116,7 +116,7 @@ sub pools()
#
sub volumes()
{
my $vref = freenasVolumeList();
my $vref = freenasVolumeList(1);
foreach my $vol (keys %{$vref}) {
my $pool = $vref->{$vol}->{'pool'};
my $iname = $vref->{$vol}->{'iname'};
......
clientside/tmcc/freenas8/libfreenas.pm
View file @ a2bccf93
This diff is collapsed.
clientside/tmcc/freenas8/libvnode_blockstore.pm
View file @ a2bccf93
......@@ -96,15 +96,12 @@ use libgenvnode;
use libvnode;
use libtestbed;
use libsetup;
use libfreenas;
#
# Constants
#
my $GLOBAL_CONF_LOCK = "blkconf";
my $ZPOOL_CMD = "/sbin/zpool";
my $ZFS_CMD = "/sbin/zfs";
my $ZPOOL_STATUS_UNKNOWN = "unknown";
my $ZPOOL_STATUS_ONLINE = "online";
my $ZPOOL_LOW_WATERMARK = 2 * 2**10; # 2GiB, expressed in MiB
my $FREENAS_MNT_PREFIX = "/mnt";
my $ISCSI_GLOBAL_PORTAL = 1;
......@@ -122,28 +119,6 @@ my $BS_CLASS_SAN = "SAN";
my $BS_PROTO_ISCSI = "iSCSI";
my $BS_UUID_TYPE_IQN = "iqn";
# CLI stuff
my $FREENAS_CLI = "$BINDIR/freenas-config";
my $CLI_VERB_IFACE = "interface";
my $CLI_VERB_IST_EXTENT = "ist_extent";
my $CLI_VERB_IST_AUTHI = "ist_authinit";
my $CLI_VERB_IST_TARGET = "ist";
my $CLI_VERB_IST_ASSOC = "ist_assoc";
my $CLI_VERB_VLAN = "vlan";
my $CLI_VERB_VOLUME = "volume";
my $CLI_VERB_POOL = "pool";
my %cliverbs = (
$CLI_VERB_IFACE => 1,
$CLI_VERB_IST_EXTENT => 1,
$CLI_VERB_IST_AUTHI => 1,
$CLI_VERB_IST_TARGET => 1,
$CLI_VERB_IST_ASSOC => 1,
$CLI_VERB_VLAN => 1,
$CLI_VERB_VOLUME => 1,
$CLI_VERB_POOL => 1,
);
#
# Global variables
#
......@@ -154,14 +129,10 @@ my $debug = 0;
#
# Local Functions
#
sub parseFreeNASListing($);
sub runFreeNASCmd($$);
sub getSliceList();
sub parseSliceName($);
sub parseSlicePath($);
sub calcSliceSizes($);
sub getPoolList();
sub getVolumeList();
sub getIfConfig($);
sub getVlan($);
sub getNextAuthITag();
......@@ -535,86 +506,6 @@ sub vnodeUnmount($$$$)
# package-local functions
#
#
# Run a FreeNAS CLI command, checking for a return error and other
# such things. We check that the incoming verb is valid. Command line
# argument string needs to be untainted or this will fail.
#
# Throws exceptions (dies), passing along errors in $@.
#
sub runFreeNASCmd($$) {
my ($verb, $argstr) = @_;
my $errstate = 0;
my $message;
die "Invalid FreeNAS CLI verb: $verb"
unless exists($cliverbs{$verb});
print "DEBUG: blockstore_runFreeNASCmd:\n".
"\trunning: $verb $argstr\n" if $debug;
my $output = `$FREENAS_CLI $verb $argstr 2>&1`;
if ($? != 0) {
$errstate = 1;
$output =~ /^(.+Error: .+)$/;
$message = defined($1) ? $1 : "Error code: $?";
} elsif ($output =~ /"error": true/) {
$errstate = 1;
$output =~ /"message": "([^"]+)"/;
$message = $1;
}
if ($errstate) {
print STDERR $output if $debug;
die $message;
}
return 0;
}
# Run our custom FreeNAS CLI to extract info.
#
# Returns an array of hash references. Each hash contains info from
# one line of output. The hash keys are the field names from the
# header (first line of output). The hash values are the
# corresponding pieces of data at each field location in a line.
sub parseFreeNASListing($) {
my $verb = shift;
my @retlist = ();
# XXX: should check that a valid verb was passed in.
open(CLI, "$FREENAS_CLI $verb list |") or
die "Can't run FreeNAS CLI: $!";
my $header = <CLI>;
return @retlist
if !defined($header) or !$header;
chomp $header;
my @fields = split(/\t/, $header);
while (my $line = <CLI>) {
chomp $line;
my @lparts = split(/\t/, $line);
if (scalar(@lparts) != scalar(@fields)) {
warn("*** WARNING: blockstore_parseFreeNASListing: ".
"Bad output from CLI ($verb): $line");
next;
}
my %lineh = ();
for (my $i = 0; $i < scalar(@fields); $i++) {
$lineh{$fields[$i]} = $lparts[$i];
}
push @retlist, \%lineh;
}
close(CLI);
return @retlist;
}
#
# Run through list of storage commands and execute them, checking
# for errors. (Should have a lock before doing this.)
......@@ -642,23 +533,6 @@ sub runBlockstoreCmds($$$) {
return 0;
}
sub getVolumeList() {
my $vollist = {};
my @zvols = parseFreeNASListing($CLI_VERB_VOLUME);
foreach my $zvol (@zvols) {
my $vol = {};
if ($zvol->{'vol_name'} =~ /^([-\w]+)\/([-\w+]+)$/) {
$vol->{'pool'} = $1;
$vol->{'volume'} = $2;
$vol->{'size'} = convertToMebi($zvol->{'vol_size'});
$vollist->{$vol->{'volume'}} = $vol;
}
}
return $vollist;
}
# Yank information about blockstore slices out of FreeNAS.
# Note: this is an expensive call - may want to re-visit caching some of
# this later if performance becomes a problem.
......@@ -666,7 +540,7 @@ sub getSliceList() {
my $sliceshash = {};
# Grab list of slices (iscsi extents) from FreeNAS
my @slist = parseFreeNASListing($CLI_VERB_IST_EXTENT);
my @slist = freenasParseListing($FREENAS_CLI_VERB_IST_EXTENT);
# Just return if there are no slices.
return if !@slist;
......@@ -729,86 +603,40 @@ sub parseSlicePath($) {
sub calcSliceSizes($) {
my $sliceshash = shift;
my @zvols = parseFreeNASListing($CLI_VERB_VOLUME);
# Ugh... Have to look up size via the "volume" list for zvol slices.
my $zvollist = freenasVolumeList(0);
# Ugh... Have to look up size via the "volume" list for
# zvol slices.
foreach my $slice (values(%$sliceshash)) {
my $bsid = $slice->{'bsid'};
my $vnode_id = $slice->{'vnode_id'};
my $type = lc($slice->{'type'});
my $size = undef;
my $type = lc($slice->{'type'});
if ($type eq "zvol") {
foreach my $zvol (@zvols) {
#print STDERR "DEBUG: volume: ". $zvol->{'vol_name'} ."\n";
if ($zvol->{'vol_name'} eq "$bsid/$vnode_id") {
$size = $zvol->{'vol_size'};
last;
}
}
if (!defined($size)) {
if (!exists($zvollist->{$vnode_id})) {
warn("*** WARNING: blockstore_calcSliceList: ".
"Could not find matching volume entry for ".
"Could not find matching volume entry ($vnode_id) for ".
"zvol slice: $slice->{'name'}");
next;
}
$size .= "B"; # Fix up units (value is \d{1,3}[TGM]).
# already converted to Mebi
$slice->{'size'} = $zvollist->{$vnode_id}->{'size'};
} elsif ($type eq "file") {
$size = $slice->{'filesize'};
my $size = $slice->{'filesize'};
$size =~ s/B$/iB/; # re-write with correct units.
$slice->{'size'} = convertToMebi($size);
}
$slice->{'size'} = convertToMebi($size);
}
return;
}
# Return information on all of the volume pools available on this host.
sub getPoolList() {
my $poolh = {};
my @pools = parseFreeNASListing($CLI_VERB_POOL);
# Create hash with pool name as key. Stuff in some sentinel values
# in case we don't get a match from 'zpool list' below.
foreach my $pool (@pools) {
$pool->{'size'} = 0;
$pool->{'avail'} = 0;
$poolh->{$pool->{'volume_name'}} = $pool;
}
# Yuck - have to go after capacity and status info by calling the
# 'zfs' command line utility since the CLI doesn't return this info.
open(ZFS, "$ZFS_CMD list -H -o name,used,avail |") or
die "Can't run 'zfs list'!";
while (my $line = <ZFS>) {
chomp $line;
my ($pname, $pused, $pavail) = split(/\s+/, $line);
next if $pname =~ /\//; # filter out zvols.
if (exists($poolh->{$pname})) {
my $pool = $poolh->{$pname};
$pused = convertToMebi($pused);
$pavail = convertToMebi($pavail);
$pool->{'size'} = $pused + $pavail;
$pool->{'avail'} = $pavail;
} else {
warn("*** WARNING: blockstore_getPoolInfo: ".
"No FreeNAS entry for zpool: $pname");
}
}
return $poolh;
}
# Allocate a slice based on information from Emulab Central
# XXX: Do 'sliceconfig' parameter checking.
sub allocSlice($$$$) {
my ($vnode_id, $sconf, $vnconfig, $priv) = @_;
my $bsid = untaintHostname($sconf->{'BSID'});
my $bsid = $sconf->{'BSID'};
my $volname = $sconf->{'VOLNAME'};
$volname = "UNKNOWN" if (!$volname);
my $size = $sconf->{'VOLSIZE'};
#
# If this is a use of a persistent store, the BSID is a unique
......@@ -816,7 +644,7 @@ sub allocSlice($$$$) {
# sure it exists, but do nothing else.
#
if ($bsid =~ /^lease-\d+$/) {
my $volumes = getVolumeList();
my $volumes = freenasVolumeList(0);
if (exists($volumes->{$bsid})) {
$priv->{'pool'} = $volumes->{$bsid}->{'pool'};
$priv->{'volume'} = $volumes->{$bsid}->{'volume'};
......@@ -827,45 +655,9 @@ sub allocSlice($$$$) {
return -1;
}
#
# Otherwise this is a "transient" store that we must create.
# Here the pool is given by the BSID and the volume is named
# after the vnode.
#
my $pools = getPoolList();
# Does the requested pool exist?
my $destpool;
if (exists($pools->{$bsid})) {
$destpool = $pools->{$bsid};
$priv->{'pool'} = $bsid;
$priv->{'volume'} = $vnode_id;
} else {
warn("*** ERROR: blockstore_allocSlice: $volname: ".
"Requested pool not found: $bsid!");
return -1;
}
# Is there enough space on the requested blockstore? If not, there is
# a discrepancy between reality and the Emulab database.
my $size = untaintNumber($sconf->{'VOLSIZE'});
if ($size + $ZPOOL_LOW_WATERMARK > $destpool->{'avail'}) {
warn("*** ERROR: blockstore_allocSlice: $volname: ".
"Not enough space remaining in requested pool: $bsid");
return -1;
}
# Allocate slice in zpool
# XXX: check on size conversion.
eval { runFreeNASCmd($CLI_VERB_VOLUME,
"add $bsid $vnode_id ${size}MB off") };
if ($@) {
warn("*** ERROR: blockstore_allocSlice: $volname: ".
"slice allocation failed: $@");
return -1;
}
return 0;
$priv->{'pool'} = $bsid;
$priv->{'volume'} = $vnode_id;
return freenasVolumeCreate($bsid, $vnode_id, $size);
}
# Setup device export.
......@@ -925,7 +717,7 @@ sub exportSlice($$$$) {
my $iqn = $1; # untaint.
# Create iSCSI extent
eval { runFreeNASCmd($CLI_VERB_IST_EXTENT,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_EXTENT,
"add $iqn $pool/$volume") };
if ($@) {
warn("*** ERROR: blockstore_exportSlice: $volname: ".
......@@ -941,7 +733,7 @@ sub exportSlice($$$$) {
return -1;
}
$tag = $1; # untaint.
eval { runFreeNASCmd($CLI_VERB_IST_AUTHI,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_AUTHI,
"add $tag ALL $network/$cmask $vnode_id") };
if ($@) {
warn("*** ERROR: blockstore_exportSlice: $volname: ".
......@@ -951,7 +743,7 @@ sub exportSlice($$$$) {
# Create iSCSI target
my $serial = genSerial();
eval { runFreeNASCmd($CLI_VERB_IST_TARGET,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_TARGET,
"add $iqn $serial $ISCSI_GLOBAL_PORTAL ".
"$tag Auto -1") };
if ($@) {
......@@ -961,7 +753,7 @@ sub exportSlice($$$$) {
}
# Bind iSCSI target to slice (extent)
eval { runFreeNASCmd($CLI_VERB_IST_ASSOC,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_ASSOC,
"add $iqn $iqn") };
if ($@) {
warn("*** ERROR: blockstore_exportSlice: $volname: ".
......@@ -981,7 +773,7 @@ sub findAuthITag($) {
return undef
if !defined($vnode_id);
my @authentries = parseFreeNASListing($CLI_VERB_IST_AUTHI);
my @authentries = freenasParseListing($FREENAS_CLI_VERB_IST_AUTHI);
return undef
if !@authentries;
......@@ -998,7 +790,7 @@ sub findAuthITag($) {
# Helper function.
# Locate and return next unused tag ID for iSCSI initiator groups.
sub getNextAuthITag() {
my @authentries = parseFreeNASListing($CLI_VERB_IST_AUTHI);
my @authentries = freenasParseListing($FREENAS_CLI_VERB_IST_AUTHI);
my $maxtag = 1;
......@@ -1052,7 +844,7 @@ sub getVlan($) {
return undef
if (!defined($vtag) || $vtag !~ /^(\d+)$/);
my @vlans = parseFreeNASListing($CLI_VERB_VLAN);
my @vlans = freenasParseListing($FREENAS_CLI_VERB_VLAN);
my $retval = undef;
foreach my $vlan (@vlans) {
......@@ -1140,7 +932,7 @@ sub createVlanInterface($$) {
# vlan does not exist.
else {
# Create the vlan entry in FreeNAS.
eval { runFreeNASCmd($CLI_VERB_VLAN,
eval { freenasRunCmd($FREENAS_CLI_VERB_VLAN,
"add $piface $viface $vtag $lname") };
if ($@) {
warn("*** ERROR: blockstore_createVlanInterface: ".
......@@ -1149,7 +941,7 @@ sub createVlanInterface($$) {
}
# Create the vlan interface.
eval { runFreeNASCmd($CLI_VERB_IFACE,
eval { freenasRunCmd($FREENAS_CLI_VERB_IFACE,
"add $viface $lname") };
if ($@) {
warn("*** ERROR: blockstore_createVlanInterface: ".
......@@ -1243,7 +1035,7 @@ sub removeVlanInterface($$) {
if (!addressExists($viface)) {
# No more addresses: Delete the vlan interface.
eval { runFreeNASCmd($CLI_VERB_VLAN,
eval { freenasRunCmd($FREENAS_CLI_VERB_VLAN,
"del $viface") };
if ($@) {
warn("*** ERROR: blockstore_removeVlanInterface: ".
......@@ -1267,7 +1059,7 @@ sub unexportSlice($$$$) {
# Remove iSCSI target. This will also zap the target-to-extent
# association.
eval { runFreeNASCmd($CLI_VERB_IST_TARGET,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_TARGET,
"del $iqn") };
if ($@) {
warn("*** WARNING: blockstore_unexportSlice: $volname: ".
......@@ -1278,7 +1070,7 @@ sub unexportSlice($$$$) {
my $tag = findAuthITag($vnode_id);
if ($tag && $tag =~ /^(\d+)$/) {
$tag = $1; # untaint.
eval { runFreeNASCmd($CLI_VERB_IST_AUTHI,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_AUTHI,
"del $tag") };
if ($@) {
warn("*** WARNING: blockstore_unexportSlice: $volname: ".
......@@ -1287,7 +1079,7 @@ sub unexportSlice($$$$) {
}
# Remove iSCSI extent.
eval { runFreeNASCmd($CLI_VERB_IST_EXTENT,
eval { freenasRunCmd($FREENAS_CLI_VERB_IST_EXTENT,
"del $iqn") };
if ($@) {
warn("*** WARNING: blockstore_unexportSlice: $volname: ".
......@@ -1300,8 +1092,8 @@ sub unexportSlice($$$$) {
sub deallocSlice($$$$) {
my ($vnode_id, $sconf, $vnconfig, $priv) = @_;
my $bsid = $sconf->{'BSID'};
my $bsid = $sconf->{'BSID'};
my $volname = $sconf->{'VOLNAME'};
$volname = "UNKNOWN" if (!$volname);
......@@ -1311,7 +1103,7 @@ sub deallocSlice($$$$) {
# sure it exists, but do nothing else.
#
if ($bsid =~ /^lease-\d+$/) {
my $volumes = getVolumeList();
my $volumes = freenasVolumeList(0);
if (exists($volumes->{$bsid})) {
return 0;
}
......@@ -1320,56 +1112,7 @@ sub deallocSlice($$$$) {
return -1;
}
# Deallocate slice. Wrap in loop to enable retries.
my $count;
for ($count = 1; $count <= $MAX_RETRY_COUNT; $count++) {
eval { runFreeNASCmd($CLI_VERB_VOLUME,
"del $bsid $vnode_id") };
# Process exceptions thrown during deletion attempt. Retry on
# some errors.
if ($@) {
if ($@ =~ /dataset is busy/) {
warn("*** WARNING: blockstore_deallocSlice: $volname: ".
"Slice is busy. Waiting a bit before trying ".
"to free again (count=$count).");
sleep $SLICE_BUSY_WAIT;
}
elsif ($@ =~ /does not exist/) {
if ($count < $MAX_RETRY_COUNT) {
warn("*** WARNING: blockstore_deallocSlice: $volname: ".
"Blockstore slice seems to be gone, retrying.");
# Bump counter to just under termination to try once more.
$count = $MAX_RETRY_COUNT-1;
sleep $SLICE_GONE_WAIT;
} else {
warn("*** WARNING: blockstore_deallocSlice: $volname: ".
"Blockstore slice still seems to be gone.");
# Bail now because we don't want to report this as an
# error to the caller.
return 0;
}
}
else {
warn("*** ERROR: blockstore_deallocSlice: $volname: a".
"slice removal failed: $@");
return -1;
}
} else {
# No error condition - jump out of loop.
last;
}
}
# Note: Checks for lingering slices will be performed separately in
# consistency checking routines.
if ($count > $MAX_RETRY_COUNT) {
warn("*** WARNING: blockstore_deallocSlice: $volname: ".
"Could not free slice after several attempts!");
return -1;
}
return 0;
return freenasVolumeDestroy($bsid, $vnode_id);
}
# Required perl foo
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment