Commit 3d6d4aca authored by Mike Hibler's avatar Mike Hibler

Synchronize the activities of the OOB interface with the vnode interface.

Aka, add some locking, stupid!
parent c9faef6e
#!/usr/bin/perl -wT
#
# Copyright (c) 2013-2017 University of Utah and the Flux Group.
# Copyright (c) 2013-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -333,10 +333,10 @@ sub create($$$;$)
return 1;
}
my $rv = freenasVolumeCreate($pool, $vol, $size, $sparse);
my $rv = freenasVolumeCreate($pool, $vol, $size, $sparse, 1);
if ($rv == 0 && $fstype ne "none") {
$rv = freenasFSCreate($pool, $vol, $fstype);
if ($rv && freenasVolumeDestroy($pool, $vol)) {
$rv = freenasFSCreate($pool, $vol, $fstype, 1);
if ($rv && freenasVolumeDestroy($pool, $vol, 1)) {
print STDERR "bscontrol_proxy: could not destroy new volume ".
"after FS creation failure.\n";
}
......@@ -368,7 +368,7 @@ sub snapshot($$$)
return 1;
}
return freenasVolumeSnapshot($pool, $vol, $tstamp);
return freenasVolumeSnapshot($pool, $vol, $tstamp, 1);
}
sub desnapshot($$$)
......@@ -396,7 +396,7 @@ sub desnapshot($$$)
}
}
return freenasVolumeDesnapshot($pool, $vol, $tstamp, 0);
return freenasVolumeDesnapshot($pool, $vol, $tstamp, 0, 1);
}
sub desnapshotall($$)
......@@ -416,7 +416,7 @@ sub desnapshotall($$)
return 1;
}
return freenasVolumeDesnapshot($pool, $vol, undef, 1);
return freenasVolumeDesnapshot($pool, $vol, undef, 1, 1);
}
sub clone($$$;$)
......@@ -453,7 +453,7 @@ sub clone($$$;$)
$tstamp = 0;
}
return freenasVolumeClone($pool, $ovol, $nvol, $tstamp);
return freenasVolumeClone($pool, $ovol, $nvol, $tstamp, 1);
}
sub destroy($$$)
......@@ -473,7 +473,7 @@ sub destroy($$$)
return 1;
}
return freenasVolumeDestroy($pool, $vol);
return freenasVolumeDestroy($pool, $vol, 1);
}
sub declone($$$)
......@@ -493,5 +493,5 @@ sub declone($$$)
return 1;
}
return freenasVolumeDeclone($pool, $vol);
return freenasVolumeDeclone($pool, $vol, 1);
}
......@@ -50,6 +50,7 @@ use Exporter;
freenasVolumeSnapshot freenasVolumeClone
freenasVolumeDesnapshot freenasVolumeDeclone
freenasParseListing freenasRequest
freenasLock freenasUnlock
$FREENAS_API_RESOURCE_IFACE $FREENAS_API_RESOURCE_IST_EXTENT
$FREENAS_API_RESOURCE_IST_AUTHI $FREENAS_API_RESOURCE_IST_TARGET
$FREENAS_API_RESOURCE_IST_TGTGROUP
......@@ -119,28 +120,28 @@ my $BS_UUID_TYPE_IQN = "iqn";
#
# Global variables
#
my $debug = 0;
my $debug = 1;
my $auth;
my $server;
sub freenasPoolList();
sub freenasVolumeList($;$);
sub freenasVolumeCreate($$$;$);
sub freenasVolumeDestroy($$);
sub freenasFSCreate($$$);
sub freenasVolumeCreate($$$;$$);
sub freenasVolumeDestroy($$;$);
sub freenasFSCreate($$$;$);
sub freenasParseListing($);
sub freenasVolumeSnapshot($$;$);
sub freenasVolumeDesnapshot($$;$$);
sub freenasVolumeClone($$$;$);
sub freenasVolumeDeclone($$);
sub freenasVolumeSnapshot($$;$$);
sub freenasVolumeDesnapshot($$;$$$);
sub freenasVolumeClone($$$;$$);
sub freenasVolumeDeclone($$;$);
#
# Local Functions
#
sub listPools();
sub convertZfsToMebi($);
sub volumeDestroy($$$$);
sub volumeDestroy($$$$$);
sub snapshotHasClone($$);
sub getZvolsFromVolinfo($);
sub parseSliceName($);
......@@ -159,6 +160,38 @@ sub setDebug($)
if ($debug);
}
#
# Make sure we don't race with libvnode_blockstore operations.
#
sub freenasLock(;$)
{
my ($timo) = @_;
$timo = 900
if (!defined($timo)); # XXX same as libvnode_blockstore
print STDERR time() . ": Grabbing blockstore lock\n"
if ($debug);
my $locked = TBScriptLock($GLOBAL_CONF_LOCK, 0, $timo);
if ($locked != TBSCRIPTLOCK_OKAY()) {
print STDERR time() .
": Could not get blockstore lock after $timo seconds!\n";
return -1;
}
print STDERR time() . ": Got blockstore lock\n"
if ($debug);
return 0;
}
sub freenasUnlock()
{
print STDERR time() . ": Releasing blockstore lock\n"
if ($debug);
TBScriptUnlock();
}
#
# Make a request via the FreeNAS v1.0 API.
# $resourse is the resource path, e.g., "account/users"
......@@ -321,6 +354,12 @@ sub freenasRequest($;$$$$$)
return undef;
}
#
# Get a full listing of extant volume information.
#
# Note that we don't bother to lock here, the caller will have to
# lockout if it wants a consistent picture of affairs.
#
sub freenasVolumeList($;$)
{
my ($inameinfo,$snapinfo) = @_;
......@@ -468,9 +507,9 @@ sub freenasPoolList() {
#
# Create a ZFS zvol.
#
sub freenasVolumeCreate($$$;$)
sub freenasVolumeCreate($$$;$$)
{
my ($pool, $volname, $size, $sparse) = @_;
my ($pool, $volname, $size, $sparse, $dolock) = @_;
# Untaint arguments since they are passed to a command execution
$pool = untaintHostname($pool);
......@@ -482,6 +521,11 @@ sub freenasVolumeCreate($$$;$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
freenasLock()
if ($dolock);
# Does the requested pool exist?
my $pools = listPools();
......@@ -491,6 +535,8 @@ sub freenasVolumeCreate($$$;$)
} else {
warn("*** ERROR: freenasVolumeCreate: ".
"Requested pool not found: $pool!");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -499,6 +545,8 @@ sub freenasVolumeCreate($$$;$)
if ($size + $ZPOOL_LOW_WATERMARK > $destpool->{'avail'}) {
warn("*** ERROR: freenasVolumeCreate: ".
"Not enough space remaining in requested pool: $pool");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -523,15 +571,19 @@ sub freenasVolumeCreate($$$;$)
} else {
warn("*** ERROR: freenasVolumeCreate: volume creation failed");
}
freenasUnlock()
if ($dolock);
return -1;
}
freenasUnlock()
if ($dolock);
return 0;
}
sub freenasVolumeSnapshot($$;$)
sub freenasVolumeSnapshot($$;$$)
{
my ($pool, $volname, $tstamp) = @_;
my ($pool, $volname, $tstamp, $dolock) = @_;
# Untaint arguments that are passed to a command execution
$pool = untaintHostname($pool);
......@@ -546,6 +598,11 @@ sub freenasVolumeSnapshot($$;$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
freenasLock()
if ($dolock);
# Get volume and snapshot info
my $vollist = freenasVolumeList(0, 1);
......@@ -555,6 +612,8 @@ sub freenasVolumeSnapshot($$;$)
if (!$vref || $vref->{'pool'} ne $pool) {
warn("*** ERROR: freenasVolumeSnapshot: ".
"Base volume '$volname' does not exist in pool '$pool'");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -567,6 +626,8 @@ sub freenasVolumeSnapshot($$;$)
if ($snapshot eq $sname) {
warn("*** ERROR: freenasVolumeSnapshot: ".
"Snapshot '$snapshot' already exists");
freenasUnlock()
if ($dolock);
return -1;
}
}
......@@ -578,15 +639,19 @@ sub freenasVolumeSnapshot($$;$)
"name" => "$tstamp"});
if (!$res) {
warn("*** ERROR: freenasVolumeSnapshot: could not create snapshot");
freenasUnlock()
if ($dolock);
return -1;
}
freenasUnlock()
if ($dolock);
return 0;
}
sub freenasVolumeDesnapshot($$;$$)
sub freenasVolumeDesnapshot($$;$$$)
{
my ($pool, $volname, $tstamp, $force) = @_;
my ($pool, $volname, $tstamp, $force, $dolock) = @_;
# Untaint arguments that are passed to a command execution
$pool = untaintHostname($pool);
......@@ -601,6 +666,11 @@ sub freenasVolumeDesnapshot($$;$$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
freenasLock()
if ($dolock);
# Get volume and snapshot info
my $vollist = freenasVolumeList(0, ($force ? 2 : 1));
......@@ -610,6 +680,8 @@ sub freenasVolumeDesnapshot($$;$$)
if (!$vref || $vref->{'pool'} ne $pool) {
warn("*** ERROR: freenasVolumeDesnapshot: ".
"Base volume '$volname' does not exist in pool '$pool'");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -663,6 +735,8 @@ sub freenasVolumeDesnapshot($$;$$)
}
}
freenasUnlock()
if ($dolock);
return $rv;
}
......@@ -672,9 +746,9 @@ sub freenasVolumeDesnapshot($$;$$)
# $tag is interpreted as a timestamp. If $tag == 0, use the most recent
# (i.e., largest timestamp) snapshot.
#
sub freenasVolumeClone($$$;$)
sub freenasVolumeClone($$$;$$)
{
my ($pool, $ovolname, $nvolname, $tag) = @_;
my ($pool, $ovolname, $nvolname, $tag, $dolock) = @_;
# Untaint arguments that are passed to a command execution
$pool = untaintHostname($pool);
......@@ -690,6 +764,11 @@ sub freenasVolumeClone($$$;$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
freenasLock()
if ($dolock);
# Get volume and snapshot info
my $vollist = freenasVolumeList(0, 1);
......@@ -699,11 +778,15 @@ sub freenasVolumeClone($$$;$)
if (!$ovref || $ovref->{'pool'} ne $pool) {
warn("*** ERROR: freenasVolumeClone: ".
"Base volume '$ovolname' does not exist in pool '$pool'");
freenasUnlock()
if ($dolock);
return -1;
}
if (exists($vollist->{$nvolname})) {
warn("*** ERROR: freenasVolumeClone: ".
"Volume '$nvolname' already exists");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -711,6 +794,8 @@ sub freenasVolumeClone($$$;$)
if (!exists($ovref->{'snapshots'})) {
warn("*** ERROR: freenasVolumeClone: ".
"Base volume '$ovolname' has no snapshots");
freenasUnlock()
if ($dolock);
return -1;
}
my @snaps = split(',', $ovref->{'snapshots'});
......@@ -729,6 +814,8 @@ sub freenasVolumeClone($$$;$)
if (!$found) {
warn("*** ERROR: freenasVolumeClone: ".
"Snapshot '$snapshot' does not exist");
freenasUnlock()
if ($dolock);
return -1;
}
}
......@@ -750,15 +837,19 @@ sub freenasVolumeClone($$$;$)
{"name" => "$pool/$nvolname"}, 202);
if (!$res) {
warn("*** ERROR: freenasVolumeClone: could not create clone");
freenasUnlock()
if ($dolock);
return -1;
}
freenasUnlock()
if ($dolock);
return 0;
}
sub freenasVolumeDeclone($$)
sub freenasVolumeDeclone($$;$)
{
my ($pool, $volname) = @_;
my ($pool, $volname, $dolock) = @_;
# Untaint arguments since they are passed to a command execution
$pool = untaintHostname($pool);
......@@ -768,13 +859,15 @@ sub freenasVolumeDeclone($$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
return volumeDestroy($pool, $volname, 1, "freenasVolumeDeclone");
return volumeDestroy($pool, $volname, 1, "freenasVolumeDeclone", $dolock);
}
sub freenasVolumeDestroy($$)
sub freenasVolumeDestroy($$;$)
{
my ($pool, $volname) = @_;
my ($pool, $volname, $dolock) = @_;
# Untaint arguments since they are passed to a command execution
$pool = untaintHostname($pool);
......@@ -784,85 +877,111 @@ sub freenasVolumeDestroy($$)
"Invalid arguments");
return -1;
}
$dolock = 1
if (!defined($dolock));
return volumeDestroy($pool, $volname, 0, "freenasVolumeDestroy");
return volumeDestroy($pool, $volname, 0, "freenasVolumeDestroy", $dolock);
}
#
# The guts of destroy and declone
#
sub volumeDestroy($$$$) {
my ($pool, $volname, $declone, $tag) = @_;
sub volumeDestroy($$$$$) {
my ($pool, $volname, $declone, $tag, $dolock) = @_;
my $tries = 0;
retry:
if (++$tries > $MAX_RETRY_COUNT) {
warn("*** WARNING: $tag: ".
"Could not free volume after $MAX_RETRY_COUNT attempts!");
return -1;
}
freenasLock()
if ($dolock);
# Get volume and snapshot info
my $vollist = freenasVolumeList(0, 1);
#
# Volume must exist
# XXX let's not consider this an error if it disappears after we
# have tried once. It probably means that someone else removed it.
# Maybe we should not consider this an error even on the first try?
#
my $vref = $vollist->{$volname};
if (!$vref || $vref->{'pool'} ne $pool) {
warn("*** ERROR: $tag: ".
"Volume '$volname' does not exist in pool '$pool'");
return -1;
if ($tries > 1) {
warn("*** ERROR: $tag: ".
"Volume '$volname' does not exist in pool '$pool'");
freenasUnlock()
if ($dolock);
return -1;
}
warn("*** WARNING: $tag: ".
"Volume '$volname' in pool '$pool' disappeared while we slept");
freenasUnlock()
if ($dolock);
return 0;
}
# Volume must not have snapshots
#
# Volume must not have snapshots.
# Note that in the case of a clone volume, we are talking about snapshots
# of the clone itself, not the snapshot that the clone is based on.
# I.e., this is not inconsistant with the "If decloning" section below.
#
if (exists($vref->{'snapshots'})) {
warn("*** ERROR: $tag: ".
"Volume '$volname' has clones and/or snapshots, cannot destroy");
freenasUnlock()
if ($dolock);
return -1;
}
# Deallocate volume. Wrap in loop to enable retries.
my $count;
for ($count = 1; $count <= $MAX_RETRY_COUNT; $count++) {
my $resource = "$FREENAS_API_RESOURCE_VOLUME/$pool/datasets/$volname";
my $msg;
my $res = freenasRequest($resource, "DELETE", undef, undef,
undef, \$msg);
# Retry on some errors
if (!$res) {
if ($msg =~ /dataset is busy/) {
warn("*** WARNING: $tag: ".
"Volume is busy. ".
"Waiting $VOLUME_BUSY_WAIT seconds before trying again ".
"(count=$count).");
sleep $VOLUME_BUSY_WAIT;
}
elsif ($msg =~ /does not exist/) {
if ($count < $MAX_RETRY_COUNT) {
warn("*** WARNING: $tag: ".
"Volume seems to be gone, retrying.");
# Bump counter to just under termination to try once more.
$count = $MAX_RETRY_COUNT-1;
sleep $VOLUME_GONE_WAIT;
} else {
warn("*** WARNING: $tag: ".
"Volume still seems to be gone.");
# Bail now because we don't want to report this as an
# error to the caller.
return 0;
}
}
else {
$msg =~ s/\\n/\n /g;
warn("*** ERROR: $tag: ".
"Volume removal failed:\n$msg");
return -1;
}
} else {
# No error condition - jump out of loop.
last;
}
}
# Note: Checks for lingering volumes will be performed separately in
# consistency checking routines.
#
# Deallocate volume.
# If it fails, we retry on some errors up to MAX_RETRY times.
# Note that we release the lock between retries, so we must restart
# from scratch each time as the volume status might have changed while
# we slept.
#
my $resource = "$FREENAS_API_RESOURCE_VOLUME/$pool/datasets/$volname";
my $msg;
if ($count > $MAX_RETRY_COUNT) {
warn("*** WARNING: $tag: ".
"Could not free volume after several attempts!");
my $res = freenasRequest($resource, "DELETE", undef, undef, undef, \$msg);
if (!$res) {
if ($msg =~ /dataset is busy/) {
warn("*** WARNING: $tag: Volume is busy. ".
"Waiting $VOLUME_BUSY_WAIT seconds before trying again ".
"(tries=$tries).");
freenasUnlock()
if ($dolock);
sleep $VOLUME_BUSY_WAIT;
goto retry;
}
if ($msg =~ /does not exist/) {
if ($tries < $MAX_RETRY_COUNT) {
warn("*** WARNING: $tag: Volume seems to be gone, retrying.");
freenasUnlock()
if ($dolock);
# Bump counter to just under termination to try once more.
$tries = $MAX_RETRY_COUNT-1;
sleep $VOLUME_GONE_WAIT;
goto retry;
}
warn("*** WARNING: $tag: Volume still seems to be gone.");
freenasUnlock()
if ($dolock);
# Bail now because we don't want to report this as an
# error to the caller.
return 0;
}
$msg =~ s/\\n/\n /g;
warn("*** ERROR: $tag: Volume removal failed:\n$msg");
freenasUnlock()
if ($dolock);
return -1;
}
......@@ -886,6 +1005,8 @@ sub volumeDestroy($$$$) {
$msg =~ s/\\n/\n /g;
warn("*** ERROR: freenasVolumeDeclone: ".
"'del $pool/$snapshot' failed:\n$msg");
freenasUnlock()
if ($dolock);
return -1;
}
} else {
......@@ -894,6 +1015,8 @@ sub volumeDestroy($$$$) {
}
}
freenasUnlock()
if ($dolock);
return 0;
}
......@@ -910,8 +1033,8 @@ sub snapshotHasClone($$)
return 0;
}
sub freenasFSCreate($$$) {
my ($pool,$vol,$fstype) = @_;
sub freenasFSCreate($$$;$) {
my ($pool,$vol,$fstype,$dolock) = @_;
my $cmd;
if ($fstype =~ /^ext[234]$/) {
......@@ -923,11 +1046,17 @@ sub freenasFSCreate($$$) {
return -1;
}
my $redir = ">/dev/null 2>&1";
freenasLock()
if ($dolock);
if (system("$cmd /dev/zvol/$pool/$vol $redir") != 0) {
warn("*** WARNING: freenasFSCreate: '$cmd /dev/zvol/$pool/$vol' failed");
freenasUnlock()
if ($dolock);
return -1;
}
freenasUnlock()
if ($dolock);
return 0;
}
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2013-2017 University of Utah and the Flux Group.
# Copyright (c) 2013-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -651,7 +651,7 @@ sub allocSlice($$$$) {
$priv->{'pool'} = $bsid;
$priv->{'volume'} = $vnode_id;
return freenasVolumeCreate($bsid, $vnode_id, $size, $sparse);
return freenasVolumeCreate($bsid, $vnode_id, $size, $sparse, 0);
}
# Setup device export.
......@@ -736,16 +736,17 @@ sub exportSlice($$$$) {
# we assume another mapping has already gone through here and
# we just use the same snapshot.
#
# XXX could this race with another vnode setup? If so, we could
# wind up creating multiple snapshots for the same volume.
# That does not matter right now, but something to watch out for.
# N.B. the global lock serializes vnode_creates here so we do
# not have to worry about racing and creating multiple snapshots.
# However, the OOB path through bscontrol.proxy needs to respect
# the lock as well or we *will* wind up with multiple snapshots.
#
if (!exists($priv->{'lastsnapshot'})) {
# XXX this will be an error
warn("*** WARNING: blockstore_exportSlice: $volname: ".
"no snapshot found; created one for now");
my $tstamp = time();
if (freenasVolumeSnapshot($pool, $volume, $tstamp)) {
if (freenasVolumeSnapshot($pool, $volume, $tstamp, 0)) {
warn("*** ERROR: blockstore_exportSlice: $volname: ".
"Could not create snapshot for RO/Clone mapping");
return -1;
......@@ -760,7 +761,7 @@ sub exportSlice($$$$) {
# Clone will use the most recent snapshot (though there should
# only be one anyway).
#
if (freenasVolumeClone($pool, $volume, $vnode_id)) {
if (freenasVolumeClone($pool, $volume, $vnode_id, 0, 0)) {
warn("*** ERROR: blockstore_exportSlice: $volname: ".
"Could not create clone for RO/Clone mapping");
return -1;
......@@ -1403,9 +1404,9 @@ sub deallocSlice($$$$) {
# which could be a long time ago (and hence very stale).
#
if (defined($snaps) && $cloneof eq (split(',', $snaps))[0]) {
return freenasVolumeDestroy($pool, $vnode_id);
return freenasVolumeDestroy($pool, $vnode_id, 0);
}
return freenasVolumeDeclone($pool, $vnode_id);
return freenasVolumeDeclone($pool, $vnode_id, 0);
}
warn("*** WARNING: blockstore_deallocSlice: $volname: ".
"Found stale clone volume '$pool/$vnode_id'");
......@@ -1425,7 +1426,7 @@ sub deallocSlice($$$$) {
# to worry about keeping the latest snapshot as there will only be one
# and it should go away on last use.
#
return freenasVolumeDeclone($bsid, $vnode_id);
return freenasVolumeDeclone($bsid, $vnode_id, 0);
}
# Required perl foo
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment