Skip to content

GitLab

Commit ce40efb6 authored by Kirk Webb's avatar Kirk Webb
Browse files
Options

Checkpoint libvnode_blockstore work.

parent 57469ba3
Hide whitespace changes
Inline Side-by-side
Showing with 747 additions and 4 deletions
clientside/tmcc/common/libgenvnode.pm
View file @ ce40efb6
......@@ -29,9 +29,13 @@ use Exporter;
@EXPORT = qw( VNODE_STATUS_RUNNING VNODE_STATUS_STOPPED VNODE_STATUS_BOOTING
VNODE_STATUS_INIT VNODE_STATUS_STOPPING VNODE_STATUS_UNKNOWN
VNODE_STATUS_MOUNTED
VNODE_PATH
findVirtControlNet
);
# Drag in path stuff
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
sub VNODE_STATUS_RUNNING() { return "running"; }
sub VNODE_STATUS_STOPPED() { return "stopped"; }
sub VNODE_STATUS_MOUNTED() { return "mounted"; }
......@@ -40,6 +44,10 @@ sub VNODE_STATUS_INIT() { return "init"; }
sub VNODE_STATUS_STOPPING(){ return "stopping"; }
sub VNODE_STATUS_UNKNOWN() { return "unknown"; }
# VM path stuff
my $VMPATH = "$VARDIR/vminfo";
sub VNODE_PATH() { return $VMPATH; }
#
# Magic control network config parameters.
#
......
clientside/tmcc/common/libsetup.pm
View file @ ce40efb6
......@@ -3403,16 +3403,17 @@ sub getstorageconfig($;$) {
}
my %fields = (
'CMD' => 'ELEMENT',
'CMD' => '(ELEMENT|EXPORT|SLICE)',
'IDX' => '\d+',
'BSID' => '[-\w]+',
'CLASS' => '(SAN|local)',
'PROTO' => '(iSCSI|local)',
'HOSTID' => '[-\w\.]+',
'PERMS' => '(RO|RW)',
'PROTO' => '(iSCSI|local)',
'UUID' => '[-\w\.:]+',
'UUID_TYPE'=> '(iqn|serial)',
'VOLNAME' => '[-\w]+',
'VOLSIZE' => '\d+',
'PERMS' => '(RO|RW)'
);
my @ops = ();
......
clientside/tmcc/common/libutil.pm
View file @ ce40efb6
......@@ -28,7 +28,7 @@ package libutil;
use Exporter;
@ISA = "Exporter";
@EXPORT = qw( ipToMac macAddSep fatal mysystem mysystem2
findDNS setState isRoutable findDomain
findDNS setState isRoutable findDomain convertToMebi
);
use libtmcc;
......@@ -122,6 +122,62 @@ sub findDomain()
return $domain;
}
#
# Convert most storage size specs to Mebibytes
#
sub convertToMebi($) {
my $insize = shift;
my $outsize;
if (!defined($insize) || !$insize) {
return -1;
}
CSIZE:
for ($insize) {
/^(\d+)B?$/ && do {
$outsize = $1 / 2**20;
last CSIZE;
};
/^(\d+(\.\d+)?)KB?$/ && do {
$outsize = $1 * 10**3 / 2**20;
last CSIZE;
};
/^(\d+(\.\d+)?)KiB?$/ && do {
$outsize = $1 / 2**10;
last CSIZE;
};
/^(\d+(\.\d+)?)MB?$/ && do {
$outsize = $1 * 10**6 / 2**20;
last CSIZE;
};
/^(\d+(\.\d+)?)MiB?$/ && do {
$outsize = $1;
last CSIZE;
};
/^(\d+(\.\d+)?)GB?$/ && do {
$outsize = $1 * 10**9 / 2**20;
last CSIZE;
};
/^(\d+(\.\d+)?)GiB?$/ && do {
$outsize = $1 * 2**10;
last CSIZE;
};
/^(\d+(\.\d+)?)TB?$/ && do {
$outsize = $1 * 10**12 / 2**20;
last CSIZE;
};
/^(\d+(\.\d+)?)TiB?$/ && do {
$outsize = $1 * 2**20;
last CSIZE;
};
# Default (bad size spec)
$outsize = -1;
}
return $outsize;
}
#
# Print error and exit.
#
......
clientside/tmcc/freenas8/libvnode_blockstore.pm 0 → 100644
View file @ ce40efb6
#!/usr/bin/perl -wT
#
# Copyright (c) 2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
# Implements the libvnode API for blockstore pseudo-VMs on FreeNAS 8
#
# Note that there is no distinguished first or last call of this library
# in the current implementation. Every vnode creation (through mkvnode.pl)
# will invoke all the root* and vnode* functions. It is up to us to make
# sure that "one time" operations really are executed only once.
#
package libvnode_blockstore;
use Exporter;
@ISA = "Exporter";
@EXPORT = qw( init setDebug rootPreConfig
rootPreConfigNetwork rootPostConfig
vnodeCreate vnodeDestroy vnodeState
vnodeBoot vnodePreBoot vnodeHalt vnodeReboot
vnodeUnmount
vnodePreConfig vnodePreConfigControlNetwork
vnodePreConfigExpNetwork vnodeConfigResources
vnodeConfigDevices vnodePostConfig vnodeExec vnodeTearDown
);
%ops = ( 'init' => \&init,
'setDebug' => \&setDebug,
'rootPreConfig' => \&rootPreConfig,
'rootPreConfigNetwork' => \&rootPreConfigNetwork,
'rootPostConfig' => \&rootPostConfig,
'vnodeCreate' => \&vnodeCreate,
'vnodeDestroy' => \&vnodeDestroy,
'vnodeTearDown' => \&vnodeTearDown,
'vnodeState' => \&vnodeState,
'vnodeBoot' => \&vnodeBoot,
'vnodeHalt' => \&vnodeHalt,
'vnodeUnmount' => \&vnodeUnmount,
'vnodeReboot' => \&vnodeReboot,
'vnodeExec' => \&vnodeExec,
'vnodePreConfig' => \&vnodePreConfig,
'vnodePreConfigControlNetwork' => \&vnodePreConfigControlNetwork,
'vnodePreConfigExpNetwork' => \&vnodePreConfigExpNetwork,
'vnodeConfigResources' => \&vnodeConfigResources,
'vnodeConfigDevices' => \&vnodeConfigDevices,
'vnodePostConfig' => \&vnodePostConfig,
);
use strict;
use English;
use Data::Dumper;
use Socket;
use File::Basename;
use File::Path;
use File::Copy;
# Pull in libvnode and other Emulab stuff
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
use libutil;
use libgenvnode;
use libvnode;
use libtestbed;
use libsetup;
#
# Constants
#
my $GLOBAL_CONF_LOCK = "blkconf";
my $FREENAS_CLI = "$BINDIR/freenas-config";
my $CLI_VERB_IST_EXTENT = "ist_extent";
my $CLI_VERB_VOLUME = "volume";
my $CLI_VERB_POOL = "pool";
my $ZPOOL_CMD = "/sbin/zpool";
my $ZFS_CMD = "/sbin/zfs";
my $ZPOOL_STATUS_UNKNOWN = "unknown";
my $ZPOOL_STATUS_ONLINE = "online";
my $ZPOOL_LOW_WATERMARK = 2 * 2**30; # 2GiB
my $FREENAS_MNT_PREFIX = "/mnt";
#
# Global variables
#
my $debug = 0;
#
# Local Functions
#
sub parseFreeNASListing($);
sub getSliceList();
sub parseSliceName($);
sub parseSlicePath($);
sub calcSliceSizes($);
sub getPoolList();
sub allocSlice($$);
sub createVlanInterface($$$$);
sub setVlanInterfaceIPAddress($$$$);
#
# Turn off line buffering on output
#
$| = 1;
sub setDebug($)
{
$debug = shift;
libvnode::setDebug($debug);
print "libvnode_blockstore: debug=$debug\n"
if ($debug);
}
#
# Called by mkvnode.pl shortly after the module is loaded.
#
sub init($) {
# XXX: doesn't seem to be passed in presently...
my ($pnode,) = @_;
# Nothing to do globally (yet).
return 0;
}
#
# Do once-per-hypervisor-boot activities.
#
# Note that this function is called for each VM, so use a marker to
# tell whether or not we've already been here, done that. Since
# FreeNAS uses memory filesystems for pretty much everything,
# we don't have to worry about removing the flag file on shutdown/reboot.
#
sub rootPreConfig() {
#
# Haven't been called yet, grab the lock and double check that someone
# didn't do it while we were waiting.
#
if (! -e "/var/run/blockstore.ready") {
my $locked = TBScriptLock($GLOBAL_CONF_LOCK,
TBSCRIPTLOCK_GLOBALWAIT(), 900);
if ($locked != TBSCRIPTLOCK_OKAY()) {
return 0
if ($locked == TBSCRIPTLOCK_IGNORE());
print STDERR "Could not get the blkinit lock after a long time!\n";
return -1;
}
}
if (-e "/var/run/blockstore.ready") {
TBScriptUnlock();
return 0;
}
print "Configuring root vnode context\n";
# XXX: nothing to do?
# XXX: Put in consistency checks.
mysystem("touch /var/run/blockstore.ready");
TBScriptUnlock();
return 0;
}
sub rootPreConfigNetwork($$$$)
{
my ($vnode_id, undef, $vnconfig, $private) = @_;
my @node_ifs = @{ $vnconfig->{'ifconfig'} };
my @node_lds = @{ $vnconfig->{'ldconfig'} };
if (TBScriptLock($GLOBAL_CONF_LOCK, 0, 900) != TBSCRIPTLOCK_OKAY()) {
print STDERR "Could not get the blknet lock after a long time!\n";
return -1;
}
# XXX: Nothing to do?
# XXX: Put in network consistency checks.
TBScriptUnlock();
return 0;
}
sub rootPostConfig($)
{
return 0;
}
sub vnodeState($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $err = 0;
my $out = VNODE_STATUS_UNKNOWN();
# if a mapping exists to a blockstore slice, then we are "running".
if (mappingExists($vnode_id)) {
$out = VNODE_STATUS_RUNNING();
}
return ($err, $out);
}
#
# Create the blockstore slice. We don't export it yet.
#
sub vnodeCreate($$$$)
{
my ($vnode_id, undef, $vnconfig, $private) = @_;
my $attributes = $vnconfig->{'attributes'};
my $vninfo = $private;
my $vmid;
if ($vnode_id =~ /^\w+\d+\-(\d+)$/) {
$vmid = $1;
}
else {
fatal("blockstore_vnodeCreate: bad vnode_id $vnode_id!");
}
$vninfo->{'vmid'} = $vmid;
$private->{'vndir'} = VNODE_PATH() . "/$vnode_id";
# Grab and stash away storageconfig stuff for this vnode.
# XXX: this bit should ultimately be moved into mkvnode.pl
my @tmp;
fatal("getstorageconfig($vnode_id): $!")
if (getstorageconfig(\@tmp));
$vnconfig->{"storageconfig"} = \@tmp;
return 0;
}
# Nothing to do presently.
sub vnodePreConfig($$$$$){
my ($vnode_id, $vmid, $vnconfig, $private, $callback) = @_;
my $vninfo = $private;
return 0;
}
# Blockstore pseudo-VMs do not have a control network to setup.
sub vnodePreConfigControlNetwork($$$$$$$$$$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private,
$ip,$mask,$mac,$gw, $vname,$longdomain,$shortdomain,$bossip) = @_;
my $vninfo = $private;
return 0;
}
# Here we actually do some work - setup the vlan interface.
sub vnodePreConfigExpNetwork($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $vninfo = $private;
my $ifconfigs = $vnconfig->{'ifconfig'};
if (@$ifconfigs != 1) {
fatal("blockstore_vnodePreConfigExpNetwork: Wrong number of ".
"network interfaces. There can be only one!");
}
my $ifc = $ifconfigs->[0];
if ($ifc->['ITYPE'] ne "vlan") {
fatal("blockstore_vnodePreConfigExpNetwork: ".
"interface type MUST be vlan!")
}
my $vtag = $ifc->['VTAG'];
my $pmac = $ifc->['PMAC'];
my $iface = $ifc->['IFACE'];
my $ip = $ifc->['IPADDR'];
my $mask = $ifc->['IPMASK'];
# First, create the vlan interface
if (createVlanInterface($vnode_id, $iface, $pmac, $vtag) != 0) {
fatal("blockstore:vnodePreConfigExpNetwork: ".
"could not create vlan interface: $iface");
}
# Next, setup its IP parameters
if (setVlanInterfaceIPAddress($vnode_id, $iface, $ip, $mask) != 0) {
fatal("blockstore:vnodePreConfigExpNetwork: ".
"could not set IP parameters on interface: $iface");
}
# XXX: not done.
return 0
}
# Tie the blockstore slice created earlier to the experiment's vlan
# (i.e., setup the export).
sub vnodeConfigResources($$$$){
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $attributes = $vnconfig->{'attributes'};
# XXX: implement
}
# Nothing to do (yet).
sub vnodeConfigDevices($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
return 0;
}
# The blockstore slice should be setup, the vlan interface created and
# plumbed, and the export in place by now. Just signal "ISUP".
sub vnodeBoot($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $vninfo = $private;
# notify Emulab that we are up. Have to go through the proper
# state transitions...
libutil::setState("BOOTING");
libutil::setState("ISUP");
return 0;
}
# Nothing to do.
sub vnodePostConfig($)
{
return 0;
}
# blockstores don't "reboot"
sub vnodeReboot($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
return 0;
}
# When this is called, we remove the blockstore export and zap the
# vlan interface.
sub vnodeTearDown($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
# XXX: implement.
}
# When this is called, we remove the blockstore slice altogether.
sub vnodeDestroy($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $vninfo = $private;
# XXX: implement.
}
# blockstores don't "halt"
sub vnodeHalt($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
return 0;
}
# What would I implement here?
sub vnodeExec($$$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private, $command) = @_;
return 0;
}
# On the surface it would seem like this might apply to blockstore pseudo-VMs.
# Teardown and destroy do the work that this might do.
sub vnodeUnmount($$$$)
{
my ($vnode_id, $vmid, $vnconfig, $private) = @_;
return 0;
}
#
# package-local functions
#
# Run our custom FreeNAS CLI to extract info.
#
# Returns an array of hash references. Each hash contains info from
# one line of output. The hash keys are the field names from the
# header (first line of output). The hash values are the
# corresponding pieces of data at each field location in a line.
sub parseFreeNASListing($) {
my $verb = shift;
my @retlist = ();
# XXX: should check that a valid verb was passed in.
open(CLI, "$FREENAS_CLI $verb list |") or
die "Can't run FreeNAS CLI: $!";
my $header = <CLI>;
chomp $header;
my @fields = split(/\t/, $header);
while (my $line = <CLI>) {
chomp $line;
my @lparts = split(/\t/, $line);
if (scalar(@lparts) != scalar(@fields)) {
warn("*** WARNING: blockstore_parseFreeNASListing: ".
"Bad output from CLI ($verb): $line");
next;
}
my %lineh = ();
for (my $i = 0; $i < scalar(@fields); $i++) {
$lineh{$fields[$i]} = $lparts[$i];
}
push @retlist, \%lineh;
}
close(CLI);
return @retlist;
}
# Yank information about blockstore slices out of FreeNAS.
sub getSliceList() {
my $sliceshash = {};
# Grab list of slices (iscsi extents) from FreeNAS
my @slist = parseFreeNASListing($CLI_VERB_IST_EXTENT);
# Just return if there are no slices.
return if !@slist;
# Go through each slice hash, culling out extra info.
# Save hash in global list. Throw out malformed stuff.
foreach my $slice (@slist) {
my ($pid,$eid,$volname) = parseSliceName($slice->{'name'});
my ($bsid, $vnode_id) = parseSlicePath($slice->{'path'});
if (!defined($pid) || !defined($bsid)) {
warn("*** WARNING: blockstore_getSliceList: ".
"malformed slice entry, skipping.");
next;
}
$slice->{'pid'} = $pid;
$slice->{'eid'} = $eid;
$slice->{'volname'} = $volname;
$slice->{'bsid'} = $bsid;
$slice->{'vnode_id'} = $vnode_id;
$sliceshash->{$vnode_id} = $slice;
}
# Do the messy work of getting slice size info into mebibytes.
calcSliceSizes($sliceshash);
return $sliceshash;
}
# helper function.
# Slice names look like: '<pid>:<eid>:<volname>'
sub parseSliceName($) {
my $name = shift;
my @parts = split(/:/, $name);
if (scalar(@parts) != 3) {
warn("*** WARNING: blockstore_parseSliceName: Bad slice name: $name");
return undef;
}
return @parts;
}
# helper function.
# Paths look like this: '/mnt/<blockstore_id>/<vnode_id>' for file-based
# extent (slice), and 'zvol/<blockstore_id>/<vnode_id>' for zvol extents.
sub parseSlicePath($) {
my $path = shift;
my @parts = split(/\//, $path);
shift @parts
if (scalar(@parts) == 4 && !$parts[0]); # chomp leading slash part
if (scalar(@parts) != 3 || $parts[0] !~ /^(mnt|zvol)$/i) {
warn("*** WARNING: blockstore_parseSlicePath: ".
"malformed slice path: $path");
return undef;
}