Commit 2c0cdccf authored by Mike Hibler's avatar Mike Hibler

Finish off the "copystatus" blockstore server command.

A command to get the status of an ongoing copy. Will also let you
know if a copy has aborted for some reason. However, lots more work
will be required before we can gracefully recover (i.e., continue)
one of those as any failure on the server side causes the boss scripts
to tear down all the DB state or, leave the blockstore in a "failed"
state that requires a great deal of manual effort for resurrection.
parent 013443dd
......@@ -45,6 +45,8 @@ sub usage()
print STDERR " Create a clone of <pool>/<vol> called <nvol> from the snapshot at <tstamp> (most recent if not specified)\n";
print STDERR " copy <pool> <ovol> <nvol>\n";
print STDERR " Create a copy of <pool>/<vol> called <nvol>\n";
print STDERR " copystatus <pool> <vol>\n";
print STDERR " Print info about the status of a copy\n";
print STDERR " destroy <pool> <vol>\n";
print STDERR " Destroy <vol> in <pool>\n";
print STDERR " desnapshot <pool> <vol> [ <tstamp> ]\n";
......@@ -97,6 +99,7 @@ my %cmds = (
"assocs" => \&assocs,
"desnapshotall" => \&desnapshotall,
"copy" => \&copy,
"copystatus" => \&copystatus,
);
#
......@@ -152,10 +155,12 @@ sub volumes()
my $pool = $vref->{$vol}->{'pool'};
my $iname = $vref->{$vol}->{'iname'};
my $size = int($vref->{$vol}->{'size'});
my $used = int($vref->{$vol}->{'used'});
my $refer = int($vref->{$vol}->{'refer'});
my $snapshots = $vref->{$vol}->{'snapshots'};
my $cloneof = $vref->{$vol}->{'cloneof'};
print "volume=$vol pool=$pool size=$size";
print "volume=$vol pool=$pool size=$size used=$used refer=$refer";
if ($iname) {
print " iname=$iname";
}
......@@ -531,15 +536,28 @@ sub copy($$$)
#
# Report the progress of a copy.
#
# On boss, dataset copy is in progress (or did not complete) if lease
# "copyfrom" attribute is set. It is still in progress if a webtask
# exists?
#
# From the blockstore server perspecive, a copy is in progress (or did
# not complete) if the "receive_resume_token" property is set on zfs dataset.
# It is still in progress if send/recv processes exist. We should write some
# state to disk (a "pid file") to make this detection easier.
#
# The "referenced" attribute tells how much data has been copied, ala:
# zfs get -Hp referenced persist-1/lease-200
#
sub copystatus($$)
{
my ($pool,$vol) = @_;
if (defined($pool) && $pool =~ /^([-\w]+)$/) {
$pool = $1;
} else {
print STDERR "bscontrol_proxy: bogus pool arg\n";
return 1;
}
if (defined($vol) && $vol =~ /^([-\w]+)$/) {
$vol = $1;
} else {
print STDERR "bscontrol_proxy: bogus volume arg\n";
return 1;
}
my ($status, $size) = freenasVolumeCopyStatus($pool, $vol);
if (!defined($status)) {
return 1;
}
print "status=$status size=$size\n";
return 0;
}
......@@ -75,7 +75,7 @@ use Exporter;
freenasVolumeCreate freenasVolumeDestroy freenasFSCreate
freenasVolumeSnapshot freenasVolumeClone
freenasVolumeDesnapshot freenasVolumeDeclone
freenasVolumeCopy
freenasVolumeCopy freenasVolumeCopyStatus
freenasParseListing freenasRequest
freenasLock freenasUnlock
$FREENAS_API_RESOURCE_IFACE $FREENAS_API_RESOURCE_IST_EXTENT
......@@ -172,6 +172,7 @@ sub freenasVolumeDesnapshot($$;$$$);
sub freenasVolumeClone($$$;$$);
sub freenasVolumeDeclone($$;$);
sub freenasVolumeCopy($$$;$);
sub freenasVolumeCopyStatus($$);
#
# Local Functions
......@@ -436,7 +437,7 @@ sub freenasVolumeList($;$)
my %inames = (); # volume-name -> slice-name
my %snaps = (); # volume-name -> (snapshot1 snapshot2 ...)
my %clones = (); # clone-volume-name -> snapshot
my %zvolsizes = (); # volume-name -> volsize
my %zvolsizes = (); # volume-name -> (volsize, used, refer)
#
# Extract blockstores from the freenas volume info and augment
......@@ -507,18 +508,26 @@ sub freenasVolumeList($;$)
}
#
# XXX unbelievable: the storage/volume API does not return the volsize
# of a zvol! Gotta do it ourselves...
# XXX The new-ish /storage/volumes/<pool>/zvols API would get us all
# the remaining info we need, including sizes. But...we would have to
# call it individually for every pool, and we don't know what all the
# storage pools are without still more work.
#
# XXX we could now get this through storage/volume/<pool>/zvols for
# each pool, or storage/volume/<pool>/zvols/<volume> for each zvol.
# But for now, let's just stick with the ZFS command.
#
# For now we get size info (volsize,used,referenced) via the ZFS tool.
# The zvols API would do it, but if we were to start using that, we
# should use it to get all the volume info (see the note above)
# rather than making two API calls (which are expensive).
#
if (open(ZFS, "$ZFS_CMD get -t volume -o name,value -Hp volsize |")) {
# Random note: if volsize is 0, then the volume is being copied.
#
if (open(ZFS, "$ZFS_CMD get -t volume -o name,property,value -Hp volsize,used,referenced |")) {
while (my $line = <ZFS>) {
chomp $line;
my ($name, $val) = split(/\s+/, $line);
$zvolsizes{$name} = $val;
my ($name, $prop, $val) = split(/\s+/, $line);
$zvolsizes{$name} = () if (!exists($zvolsizes{$name}));
$zvolsizes{$name}->{$prop} = $val;
}
close(ZFS);
} else {
......@@ -540,11 +549,24 @@ sub freenasVolumeList($;$)
if ($volname =~ /^([-\w]+)\/([-\w+]+)$/) {
$vol->{'pool'} = $1;
$vol->{'volume'} = $2;
# fill in the size info
$vol->{'size'} = $vol->{'used'} = $vol->{'refer'} = 0;
if (exists($zvolsizes{$volname})) {
$vol->{'size'} = convertToMebi($zvolsizes{$volname});
if (exists($zvolsizes{$volname}->{'volsize'})) {
$vol->{'size'} =
int(convertToMebi($zvolsizes{$volname}->{'volsize'}));
}
if (exists($zvolsizes{$volname}->{'used'})) {
$vol->{'used'} =
int(convertToMebi($zvolsizes{$volname}->{'used'}));
}
if (exists($zvolsizes{$volname}->{'referenced'})) {
$vol->{'refer'} =
int(convertToMebi($zvolsizes{$volname}->{'referenced'}));
}
} else {
$vol->{'size'} = 0;
warn("*** WARNING: could not get volume size of $volname");
warn("*** WARNING: could not get sizes of $volname");
}
if ($inameinfo && exists($inames{$zvol->{'path'}})) {
......@@ -1026,20 +1048,40 @@ sub freenasVolumeCopy($$$;$)
#
# Do the send/recv pipeline.
#
# This could take a really, really long time so we leave things unlocked
# while we do it.
#
# We leave an "in progress" file in the volatile /var/run directory so
# we can tell if a supposedly active copy might have been terminated
# by a server reboot or crash of the bscontrol.proxy instance. The
# existance of the file detects the former, we write our pid into the
# file to detect the latter.
#
TBDebugTimeStampWithDate("freenasVolumeCopy: starting send/recv")
if ($debug);
my $pfile = "/var/run/$pool-$nvolname.copying";
if (open(FD, ">$pfile")) {
print FD "$PID\n";
close(FD);
}
if (system("$ZFS_CMD send -R $pool/$snapshot | $ZFS_CMD recv -Fs $pool/$nvolname")) {
TBDebugTimeStampWithDate("freenasVolumeCopy: send/recv FAILED")
if ($debug);
my $msg = "";
my $token =
`$ZFS_CMD get -Ho value receive_resume_token $pool/$nvolname`;
chomp($token);
if ($token ne "-") {
$msg = ", may be able to finish with:\n".
" $ZFS_CMD send -t $token | $ZFS_CMD recv -s $pool/$nvolname";
}
warn("*** ERROR: ".
"'$ZFS_CMD send -R $pool/$snapshot | $ZFS_CMD recv -Fs $pool/$nvolname' ".
"failed, may be able to finish with:\n".
"'$ZFS_CMD send -t <token> | $ZFS_CMD recv -s $nvolname'\n");
"'$ZFS_CMD send -R $pool/$snapshot | $ZFS_CMD recv -Fs $pool/$nvolname' failed$msg\n");
unlink($pfile);
return -1;
}
unlink($pfile);
TBDebugTimeStampWithDate("freenasVolumeCopy: finished send/recv")
if ($debug);
......@@ -1068,6 +1110,107 @@ sub freenasVolumeCopy($$$;$)
return 0;
}
#
# Determine if a volume copy is still running.
# Returns zero if not, non-zero if so.
#
# Note that we should only be called if the volume exists and has the
# receive_resume_token.
#
sub copyRunning($$)
{
my ($pool, $volname) = @_;
my $pfile = "/var/run/$pool-$volname.copying";
# if the pidfile does not exist, we must have rebooted
if (! -e $pfile) {
return 0;
}
# if we cannot open the file, be conservative and assume still running
if (!open(FD, "<$pfile")) {
warn("*** WARNING: $pool/$volname copy file exists but unreadable");
return 1;
}
my $dapid = <FD>;
close(FD);
chomp($dapid);
# ditto if the contents is malformed
if ($dapid !~ /^(\d+)$/) {
warn("*** WARNING: $pool/$volname copy file does not contain a pid");
return 1;
}
$dapid = $1;
return kill(0, $dapid);
}
#
# From the blockstore server perspecive, a copy is in progress (or did
# not complete) if the "receive_resume_token" property is set on zfs dataset.
# It is still in progress if send/recv processes exist. We should write some
# state to disk (a "pid file") to make this detection easier.
#
# The "referenced" attribute tells how much data has been copied, ala:
# zfs get -Hp referenced persist-1/lease-200
#
# Returns:
# * status=INVALID
# Volume cannot be found
# * status=INPROGRESS, size=<size-in-MiB>
# resume_token exists and send/recv pipeline is running
# * status=ABORTED, size=<size-in-MiB>
# resume_token exists and send/recv pipeline is not running
# * status=DONE, size=<size-in-MiB>
# None of the above are true (note that the volume might not
# even have been involved in a copy)
#
sub freenasVolumeCopyStatus($$)
{
my ($pool, $volname) = @_;
my $status = "UNKNOWN";
my $size = -1;
# Untaint arguments that are passed to a command execution
$pool = untaintHostname($pool);
$volname = untaintHostname($volname);
if (!$pool || !$volname) {
warn("*** ERROR: freenasVolumeCopyStatus: Invalid arguments");
return undef;
}
if (open(ZFS, "$ZFS_CMD get -o property,value -Hp referenced,receive_resume_token $pool/$volname 2>&1 |")) {
while (my $line = <ZFS>) {
chomp $line;
if ($line =~ /dataset does not exist/) {
$status = "INVALID";
next;
}
my ($name, $val) = split(/\s+/, $line);
if ($name eq "referenced") {
$size = int(convertToMebi($val));
} elsif ($name eq "receive_resume_token") {
if ($val eq "-") {
$status = "DONE";
} elsif (copyRunning($pool, $volname)) {
$status = "INPROGRESS";
} else {
$status = "ABORTED";
}
} else {
# just ignore unknown lines
next;
}
}
close(ZFS);
} else {
warn("*** WARNING: could not run 'zfs get' for zvol status info");
}
return ($status, $size);
}
sub freenasVolumeDeclone($$;$)
{
my ($pool, $volname, $dolock) = @_;
......@@ -1125,7 +1268,7 @@ sub volumeDestroy($$$$$) {
my $vollist = freenasVolumeList(0, 1);
#
# Volume must exist
# volume must exist
# XXX let's not consider this an error if it disappears after we
# have tried once. It probably means that someone else removed it.
# Maybe we should not consider this an error even on the first try?
......
......@@ -56,6 +56,9 @@ use Getopt::Std;
# could do server to server or pool to pool copies, but we don't
# right now.
#
# bscontrol [ -S server [ -P pool ] ] copystatus bsname
# Report on the status of a copy operation.
#
# bscontrol snapshot [ -S server [ -P pool ] ] bsname [ tstamp ]
# Create a snapshot of the named blockstore with the indicated
# timestamp. If timestamp is not provided, it will use the
......@@ -85,6 +88,7 @@ sub usage()
print STDERR "\nAdditional persistent blockstore commands.\n";
print STDERR "bscontrol [-S server [-P pool]] [-f fstype] -l leaseidx -s size -t type create bsname\n";
print STDERR "bscontrol [-S server [-P pool]] copy from-bsname to-bsname\n";
print STDERR "bscontrol [-S server [-P pool]] copystatus bsname\n";
print STDERR "bscontrol [-S server -P pool] destroy bsname\n";
print STDERR "bscontrol [-S server [-P pool]] snapshot bsname [tstamp]\n";
print STDERR "bscontrol [-S server [-P pool]] desnapshot bsname [tstamp]\n";
......@@ -108,6 +112,7 @@ sub bs_avail($$$@);
sub bs_info($$$@);
sub bs_create($$$@);
sub bs_copy($$$@);
sub bs_copystatus($$$@);
sub bs_snapshot($$$@);
sub bs_desnapshot($$$@);
sub bs_destory($$$@);
......@@ -161,6 +166,7 @@ my %cmds = (
"info" => \&bs_info,
"create" => \&bs_create,
"copy" => \&bs_copy,
"copystatus" => \&bs_copystatus,
"snapshot" => \&bs_snapshot,
"desnapshot" => \&bs_desnapshot,
"destroy" => \&bs_destroy,
......@@ -957,6 +963,98 @@ sub bs_copy($$$@)
return 0;
}
#
# On boss, a dataset copy is in progress (or did not complete) if lease
# "copyfrom" attribute is set. It is still in progress if a webtask
# exists?
#
sub bs_copystatus($$$@)
{
my ($srv,$pool,undef,$name) = @_;
if (defined($name) && $name =~ /^([-\w]+)$/) {
$name = $1;
} else {
fatal("copystatus: must specify a valid volume name");
}
#
# Find the blockstore based on info from the server(s).
#
my $volattrs;
my $volref = getvolumes($srv, $pool);
foreach my $vol (sort keys(%{$volref})) {
my $attrs = $volref->{$vol};
if ($name eq $attrs->{'volume'}) {
if ($volattrs) {
fatal("copy: multiple volumes match '$name', specify a server and pool");
}
$volattrs = $attrs;
}
}
if (!$volattrs) {
fatal("copy: no such volume '$name'");
}
if ($srv) {
if ($srv ne $volattrs->{'server'}) {
fatal("copy: found server is not the specified server!?");
}
} else {
$srv = $volattrs->{'server'};
}
if ($pool) {
if ($pool ne $volattrs->{'pool'}) {
fatal("copy: found pool is not the specified pool!?");
}
} else {
$pool = $volattrs->{'pool'};
}
#
# Figure out the source of the copy and how big it is.
# Source is an attribute of our lease (if we are a copy).
#
my $srcsize = -1;
if ($name =~ /lease-(\d+)$/) {
my $lease = Lease->Lookup($1);
if ($lease) {
my $srcbs = $lease->GetAttribute("copyfrom");
if ($srcbs) {
# find the source volume in the volume list
foreach my $vol (sort keys(%{$volref})) {
my $attrs = $volref->{$vol};
if ($srcbs eq $attrs->{'volume'} &&
$pool eq $attrs->{'pool'} &&
$srv eq $attrs->{'server'}) {
$srcsize = int($attrs->{'refer'});
last;
}
}
}
}
}
# Call out to the server to get the info
my $outref;
if (bsserver_cmd($srv, "$PROXYCMD copystatus $pool $name", 1, \$outref)) {
print STDERR "*** copystatus: could not get status of '$name' on $srv/$pool:\n";
foreach my $str (@$outref) {
print STDERR " $str\n";
}
exit(-1);
}
my $str = $outref->[0];
if ($srcsize >= 0 && $str =~ /status=(\w+),/) {
if ($1 eq "INPROGRESS" || $1 eq "ABORTED") {
$str .= " finalsize=$srcsize";
}
}
print "$str\n";
return 0;
}
sub dosnapshot($$$$$)
{
my ($create,$srv,$pool,$name,$tstamp) = @_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment