Commit e321b385 authored by Leigh B Stoller's avatar Leigh B Stoller

First cut at blockstore support. This example maps but does not

swap in yet:

set client [$ns node]
tb-set-node-os $client FEDORA15-STD

set d1 [$ns blockstore]
$d1 set-class "SAN"
$d1 set-protocol "iSCSI"
$d1 set-size 2GB

set san [$ns duplex-link $client $d1 10Kbps 0ms DropTail]
parent adb428d4
#!/usr/bin/perl -wT
#
# Copyright (c) 2012-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
package Blockstore;
use strict;
use Carp;
use Exporter;
use vars qw(@ISA @EXPORT $AUTOLOAD);
@ISA = qw(Exporter);
@EXPORT = qw ( );
use libdb;
use libtestbed;
use English;
use Data::Dumper;
use overload ('""' => 'Stringify');
my $debug = 0;
#
# Lookup a (physical) storage object type and create a class instance to
# return.
#
sub Lookup($$$)
{
my ($class, $nodeid, $bsid) = @_;
return undef
if (! ($nodeid =~ /^[-\w]+$/ && $bsid =~ /^[-\w]+$/));
my $query_result =
DBQueryWarn("select * from blockstores ".
"where node_id='$nodeid' and bs_id='$bsid'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
# To avoid writing out all the methods.
AUTOLOAD {
my $self = $_[0];
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
# A DB row proxy method call.
if (exists($self->{'DBROW'}->{$name})) {
return $self->{'DBROW'}->{$name};
}
# Local storage slot.
if ($name =~ /^_.*$/) {
if (scalar(@_) == 2) {
return $self->{'HASH'}->{$name} = $_[1];
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
}
carp("No such slot '$name' field in $self");
return undef;
}
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{"DBROW"} = undef;
$self->{"HASH"} = undef;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $node_id = $self->node_id();
return "[BlockStore:$bsidx, $bs_id, $node_id]";
}
#
# Blockstores are reserved to a pcvm; that is how we do the
# bookkeeping. When a node is released (nfree), we can find
# the reserved blockstores for that node, reset the capacity
# in the blockstore_state table, and delete the row(s).
#
sub Reserve($$$$$)
{
my ($self, $experiment, $vnode_id, $bs_name, $bs_size) = @_;
my $exptidx = $experiment->idx();
my $pid = $experiment->pid();
my $eid = $experiment->eid();
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $bs_node_id = $self->node_id();
my $remaining_capacity;
DBQueryWarn("lock tables blockstores read, ".
" reserved_blockstores write, ".
" blockstore_state write")
or return -1;
#
# Need the remaining size to make sure we can allocate it.
#
my $query_result =
DBQueryWarn("select remaining_capacity from blockstore_state ".
"where bsidx='$bsidx'");
goto bad
if (!$query_result);
#
# Just in case the state row is missing, okay to create it.
#
if (!$query_result->numrows) {
$remaining_capacity = $self->total_size();
DBQueryWarn("insert into blockstore_state set ".
" bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
" remaining_capacity='$remaining_capacity', 'ready=1'")
or goto bad;
}
else {
($remaining_capacity) = $query_result->fetchrow_array();
}
if ($bs_size > $remaining_capacity) {
print STDERR "Not enough remaining capacity on $bsidx\n";
goto bad;
}
#
# If we do not have a reservation row, create one with a zero
# size, to indicate nothing has actually been reserved in the
# blockstore_state table.
#
$query_result =
DBQueryWarn("select size from reserved_blockstores ".
"where exptidx='$exptidx' and bsidx='$bsidx' and ".
" vname='$bs_name'");
goto bad
if (!$query_result);
if (! $query_result->numrows) {
if (! DBQueryWarn("insert into reserved_blockstores set ".
" bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
" vname='$bs_name', pid='$pid', eid='$eid', ".
" size='0', vnode_id='$vnode_id', ".
" exptidx='$exptidx', rsrv_time=now()")) {
goto bad;
}
}
else {
my ($current_size) = $query_result->fetchrow_array();
#
# At the moment, I am not going to allow the blockstore
# to change size.
#
if ($current_size && $current_size != $bs_size) {
print STDERR "Not allowed to change size of existing store\n";
goto bad;
}
#
# If already have a reservation size, then this is most
# likely a swapmod, and we can just return without doing
# anything.
#
goto done
if ($current_size);
}
#
# Now do an atomic update that changes both tables.
#
if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
" remaining_capacity=remaining_capacity-${bs_size}, ".
" size='$bs_size' ".
"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
" blockstore_state.bs_id=reserved_blockstores.bs_id and ".
" reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")) {
goto bad;
}
done:
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return -1;
}
############################################################################
#
# Package to describe a specific reservation of a blockstore.
#
package Blockstore::Reservation;
use libdb;
use libtestbed;
use English;
use vars qw($AUTOLOAD);
use overload ('""' => 'Stringify');
#
# Lookup a blockstore reservation.
#
sub Lookup($$$$)
{
my ($class, $blockstore, $experiment, $vname) = @_;
return undef
if (! ($vname =~ /^[-\w]+$/ && ref($blockstore) && ref($experiment)));
my $exptidx = $experiment->idx();
my $bsidx = $blockstore->bsidx();
my $query_result =
DBQueryWarn("select * from reserved_blockstores ".
"where exptidx='$exptidx' and bsidx='$bsidx' and ".
" vname='$vname'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
#
# Look for the blockstore associated with a pcvm. At the present
# time, one blockstore is mapped to one pcvm.
#
sub LookupByNodeid($$)
{
my ($class, $vnode_id) = @_;
my $query_result =
DBQueryWarn("select * from reserved_blockstores ".
"where vnode_id='$vnode_id'");
return undef
if (!$query_result || !$query_result->numrows);
if ($query_result->numrows != 1) {
print STDERR "Too many blockstores for $vnode_id!\n";
return -1;
}
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
# To avoid writing out all the methods.
AUTOLOAD {
my $self = $_[0];
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
# A DB row proxy method call.
if (exists($self->{'DBROW'}->{$name})) {
return $self->{'DBROW'}->{$name};
}
carp("No such slot '$name' field in $self");
return undef;
}
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{"DBROW"} = undef;
$self->{"HASH"} = undef;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $node_id = $self->node_id();
my $vname = $self->vname();
return "[BlockStore:$bsidx, $bs_id, $node_id ($vname)]";
}
#
# Blockstores are reserved to a pcvm; that is how we do the
# bookkeeping. When a node is released (nfree), we can find
# the reserved blockstore for that node, reset the capacity
# in the blockstore_state table, and delete the row(s).
#
sub Release($)
{
my ($self) = @_;
my $exptidx = $self->exptidx();
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $bs_node_id = $self->node_id();
my $vnode_id = $self->vnode_id();
my $size = $self->size();
DBQueryWarn("lock tables blockstores read, ".
" reserved_blockstores write, ".
" blockstore_state write")
or return -1;
#
# Need the remaining size to deallocate.
#
my $query_result =
DBQueryWarn("select remaining_capacity from blockstore_state ".
"where bsidx='$bsidx'");
goto bad
if (!$query_result);
if (!$query_result->numrows) {
print STDERR "No blockstore state for $bsidx\n";
goto bad;
}
#
# We want to atomically uupdate update remaining_capacity and
# set the size in the reservation to zero, so that if we fail,
# nothing has changed.
#
if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
" remaining_capacity=remaining_capacity+size, ".
" size=0-size ".
"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
" blockstore_state.bs_id=reserved_blockstores.bs_id and ".
" reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")) {
goto bad;
}
# That worked, so now we can delete the reservation row.
DBQueryWarn("delete from reserved_blockstores ".
"where reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")
or goto bad;
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return -1;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -152,7 +152,7 @@ use vars qw(@ISA @EXPORT);
TBDB_FRISBEEMCBASEADDR
TBDB_RSRVROLE_NODE TBDB_RSRVROLE_VIRTHOST TBDB_RSRVROLE_DELAYNODE
TBDB_RSRVROLE_SIMHOST
TBDB_RSRVROLE_SIMHOST TBDB_RSRVROLE_STORAGEHOST
TBDB_EXPT_WORKDIR
TB_OSID_MBKERNEL
......@@ -544,6 +544,7 @@ sub TBDB_RSRVROLE_NODE() { "node"; }
sub TBDB_RSRVROLE_VIRTHOST() { "virthost"; }
sub TBDB_RSRVROLE_DELAYNODE() { "delaynode"; }
sub TBDB_RSRVROLE_SIMHOST() { "simhost"; }
sub TBDB_RSRVROLE_STORAGEHOST() { "storagehost"; }
# Interfaces roles.
sub TBDB_IFACEROLE_CONTROL() { "ctrl"; }
......
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -47,8 +47,8 @@ LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \
NodeType.pm Interface.pm User.pm Group.pm Project.pm \
Image.pm OSinfo.pm Archive.pm Logfile.pm Lan.pm emdbi.pm \
emdb.pm emutil.pm Firewall.pm VirtExperiment.pm libGeni.pm \
libEmulab.pm EmulabConstants.pm TraceUse.pm EmulabFeatures.pm \
Port.pm BlockstoreType.pm
libEmulab.pm EmulabConstants.pm TraceUse.pm \
EmulabFeatures.pm Port.pm BlockstoreType.pm Blockstore.pm
# Stuff installed on plastic.
USERSBINS = genelists.proxy dumperrorlog.proxy backup
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2005-2012 University of Utah and the Flux Group.
# Copyright (c) 2005-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -46,6 +46,7 @@ $JAILIPMASK = "@JAILIPMASK@";
use libdb;
use libtestbed;
use emutil;
use Blockstore;
use event;
use English;
use Socket;
......@@ -1748,6 +1749,29 @@ sub ReleaseSharedBandwidth($)
return 0;
}
#
# Relase the reserved blockstore, which requires updating the
# remaining_capacity on the underyling store. At the present
# time, one blockstore is mapped to one pcvm.
#
sub ReleaseBlockStore($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $node_id = $self->node_id();
my $blockstore = Blockstore::Reservation->LookupByNodeid($node_id);
return 0
if (!defined($blockstore));
return -1
if (!ref($blockstore));
return $blockstore->Release();
}
#
# Look up all interfaces for a node, return list of objects.
#
......@@ -2129,8 +2153,7 @@ sub CreateVnodes($$$)
my $sharing_mode = $node->sharing_mode();
if (! ($impotent || $isfednode)) {
my $reservation = $node->Reservation();
if (!defined($reservation)) {
if (!$node->IsReserved()) {
print STDERR "*** CreateVnodes: no reservation for $node!\n";
DBQueryFatal("unlock tables");
return -1;
......@@ -2142,7 +2165,11 @@ sub CreateVnodes($$$)
# is on and the pnode is in sharedmode. Locking in nfree and in
# the pool daemon prevents the race.
#
if (!$experiment->SameExperiment($reservation)) {
# Cause of locking above, we need to make the comparison directly
# using the slot data in the node.
#
if (! ($experiment->pid() eq $node->pid() &&
$experiment->eid() eq $node->eid())) {
if (! ($sharedokay && $sharing_mode)) {
print STDERR "*** CreateVnodes: $node is not shared!\n";
DBQueryFatal("unlock tables");
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -450,7 +450,7 @@ foreach my $node (@freed_nodes) {
}
#
# If the node is virtual, release the shared bandwidth it had
# If the node is virtual, release the shared resources it had
# reserved on the physical node.
#
if ($isvirt) {
......@@ -459,6 +459,8 @@ foreach my $node (@freed_nodes) {
DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'")
or $error++;
}
$node->ReleaseBlockStore() == 0
or $error++;
}
#
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -2009,4 +2009,41 @@ sub WaitDone($@)
return libossetup_virtnode::WaitDone($self, @nodelist);
}
#####################################################################
#
# Place holder for blockstore
#
package libossetup_blockstore;
use base qw(libossetup_subnode);
#
# A constructor for an object to handle all nodes of this type.
#
sub New($$) {
my ($class, $parent) = @_;
my $self = $class->SUPER::New("blockstore", $parent);
bless($self, $class);
return $self;
}
sub AddNode($$)
{
my ($self, $node) = @_;
print "Will skip blockstore $node ISUP wait.\n";
return $self->SUPER::AddNode($node);
}
#
# We do not currently do anything with blockstores; no waiting for ISUP.
#
sub Volunteers($)
{
my ($self) = @_;
return ();
}
1;
#!/usr/bin/perl -w
#
# Copyright (c) 2005-2012 University of Utah and the Flux Group.
# Copyright (c) 2005-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -37,11 +37,13 @@ use libdb;
use libtblog_simple;
use libtestbed;
use Experiment;
use EmulabConstants;
use VirtExperiment;
use Node;
use NodeType;
use Lan;
use OSinfo;
use Blockstore;
use Image;
use Port;
use English;
......@@ -107,6 +109,8 @@ sub Create($$$$)
$self->{'DELAYNODES'} = {};
$self->{'LANNODES'} = {};
$self->{'BRIDGES'} = {};
$self->{'BLOCKSTORES'}= {};
$self->{'SANHOSTS'} = {};
$self->{'VLANS'} = {};
$self->{'VPATHS'} = {};
$self->{'MEMBEROF'} = {};
......@@ -154,6 +158,8 @@ sub flags($) { return $_[0]->{'FLAGS'}; }
sub vnodes($) { return $_[0]->{'VNODES'}; }
sub delaynodes($) { return $_[0]->{'DELAYNODES'}; }
sub bridges($) { return $_[0]->{'BRIDGES'}; }
sub blockstores($) { return $_[0]->{'BLOCKSTORES'}; }
sub sanhosts($) { return $_[0]->{'SANHOSTS'}; }
sub lannodes($) { return $_[0]->{'LANNODES'}; }
sub vlans($) { return $_[0]->{'VLANS'}; }
sub memberof($) { return $_[0]->{'MEMBEROF'}; }
......@@ -199,10 +205,14 @@ sub virt_trafgens($) { return $_[0]->virt_table("virt_trafgens"); }
sub virt_lan_settings($){ return $_[0]->virt_table("virt_lan_settings"); }
sub virt_lan_member_settings($) {
return $_[0]->virt_table("virt_lan_member_settings"); }
sub virt_blockstores($) { return $_[0]->virt_table("virt_blockstores"); }
sub virt_blockstore_attributes($) {
return $_[0]->virt_table("virt_blockstore_attributes"); }
# Given a vname, is it a node in the topo (or something else like a delay).
sub isatoponode($$) { return exists($_[0]->vnodes()->{$_[1]}); }
sub isadelaynode($$) { return exists($_[0]->delaynodes()->{$_[1]}); }
sub isasanhost($$) { return exists($_[0]->sanhosts()->{$_[1]}); }
# Debug output.
sub verbose($) { return $_[0]->flags() & $VTOP_FLAGS_VERBOSE; }
......@@ -239,6 +249,7 @@ sub virtnodecount($) { return $_[0]->counter("virtcount"); }
sub simnodecount($) { return $_[0]->counter("simcount"); }
sub remotenodecount($) { return $_[0]->counter("remotecount"); }
sub sharednodecount($) { return $_[0]->counter("sharedcount"); }
sub bstorecount($) { return $_[0]->counter("bstorecount"); }
sub createLink($$$$$$$$$)
{
......@@ -1136,6 +1147,9 @@ sub LoadVirtNodes($)
$vnode->_sharedokay(0);
$vnode->_fixedvm(undef);
$vnode->_isbridge(($vnode->role() eq "bridge" ? 1 : 0));
$vnode->_blockstore(undef);
$vnode->_blockstore_attributes(undef);
$vnode->_sanhostname(undef);
#
# Explicit type request desire to match feature.
......@@ -1450,7 +1464,47 @@ sub LoadVirtNodes($)
$self->printdb("Fixing VM $vname to $fixnode, $osinfo\n");
}
#
# Load up the blockstore stuff since we have to spit out the
# attributes as desires.
#