First cut at blockstore support. This example maps but does not

swap in yet:

set client [$ns node]
tb-set-node-os $client FEDORA15-STD

set d1 [$ns blockstore]
$d1 set-class "SAN"
$d1 set-protocol "iSCSI"
$d1 set-size 2GB

set san [$ns duplex-link $client $d1 10Kbps 0ms DropTail]

db/Blockstore.pm

+#!/usr/bin/perl -wT
+#
+# Copyright (c) 2012-2013 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+package Blockstore;
+
+use strict;
+use Carp;
+use Exporter;
+use vars qw(@ISA @EXPORT $AUTOLOAD);
+
+@ISA = qw(Exporter);
+@EXPORT = qw ( );
+
+use libdb;
+use libtestbed;
+use English;
+use Data::Dumper;
+use overload ('""' => 'Stringify');
+
+my $debug	= 0;
+
+#
+# Lookup a (physical) storage object type and create a class instance to 
+# return.
+#
+sub Lookup($$$)
+{
+    my ($class, $nodeid, $bsid) = @_;
+
+    return undef
+	if (! ($nodeid =~ /^[-\w]+$/ && $bsid =~ /^[-\w]+$/));
+
+    my $query_result =
+	DBQueryWarn("select * from blockstores ".
+		    "where node_id='$nodeid' and bs_id='$bsid'");
+
+    return undef
+	if (!$query_result || !$query_result->numrows);
+
+    my $self         = {};
+    $self->{"HASH"}  = {};
+    $self->{"DBROW"} = $query_result->fetchrow_hashref();
+
+    bless($self, $class);
+    return $self;
+}
+
+# To avoid writing out all the methods.
+AUTOLOAD {
+    my $self  = $_[0];
+    my $name  = $AUTOLOAD;
+    $name =~ s/.*://;   # strip fully-qualified portion
+
+    # A DB row proxy method call.
+    if (exists($self->{'DBROW'}->{$name})) {
+	return $self->{'DBROW'}->{$name};
+    }
+
+    # Local storage slot.
+    if ($name =~ /^_.*$/) {
+	if (scalar(@_) == 2) {
+	    return $self->{'HASH'}->{$name} = $_[1];
+	}
+	elsif (exists($self->{'HASH'}->{$name})) {
+	    return $self->{'HASH'}->{$name};
+	}
+    }
+    carp("No such slot '$name' field in $self");
+    return undef;
+}
+
+# Break circular reference someplace to avoid exit errors.
+sub DESTROY {
+    my $self = shift;
+
+    $self->{"DBROW"}    = undef;
+    $self->{"HASH"}     = undef;
+}
+
+#
+# Stringify for output.
+#
+sub Stringify($)
+{
+    my ($self) = @_;
+    
+    my $bsidx   = $self->bsidx();
+    my $bs_id   = $self->bs_id();
+    my $node_id = $self->node_id();
+
+    return "[BlockStore:$bsidx, $bs_id, $node_id]";
+}
+
+#
+# Blockstores are reserved to a pcvm; that is how we do the
+# bookkeeping. When a node is released (nfree), we can find
+# the reserved blockstores for that node, reset the capacity
+# in the blockstore_state table, and delete the row(s).
+#
+sub Reserve($$$$$)
+{
+    my ($self, $experiment, $vnode_id, $bs_name, $bs_size) = @_;
+    my $exptidx    = $experiment->idx();
+    my $pid        = $experiment->pid();
+    my $eid        = $experiment->eid();
+    my $bsidx      = $self->bsidx();
+    my $bs_id      = $self->bs_id();
+    my $bs_node_id = $self->node_id();
+    my $remaining_capacity;
+
+    DBQueryWarn("lock tables blockstores read, ".
+		"            reserved_blockstores write, ".
+		"            blockstore_state write")
+	or return -1;
+
+    #
+    # Need the remaining size to make sure we can allocate it.
+    #
+    my $query_result =
+	DBQueryWarn("select remaining_capacity from blockstore_state ".
+		    "where bsidx='$bsidx'");
+    goto bad
+	if (!$query_result);
+
+    #
+    # Just in case the state row is missing, okay to create it.
+    #
+    if (!$query_result->numrows) {
+	$remaining_capacity = $self->total_size();
+
+	DBQueryWarn("insert into blockstore_state set ".
+		    "  bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
+		    "  remaining_capacity='$remaining_capacity', 'ready=1'")
+	    or goto bad;
+    }
+    else {
+	($remaining_capacity) = $query_result->fetchrow_array();
+    }
+    if ($bs_size > $remaining_capacity) {
+	print STDERR "Not enough remaining capacity on $bsidx\n";
+	goto bad;
+    }
+
+    #
+    # If we do not have a reservation row, create one with a zero
+    # size, to indicate nothing has actually been reserved in the
+    # blockstore_state table.
+    #
+    $query_result =
+	DBQueryWarn("select size from reserved_blockstores ".
+		    "where exptidx='$exptidx' and bsidx='$bsidx' and ".
+		    "       vname='$bs_name'");
+    goto bad
+	if (!$query_result);
+
+    if (! $query_result->numrows) {
+	if (! DBQueryWarn("insert into reserved_blockstores set ".
+	        "  bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
+	        "  vname='$bs_name', pid='$pid', eid='$eid', ".
+		"  size='0', vnode_id='$vnode_id', ".
+	        "  exptidx='$exptidx', rsrv_time=now()")) {
+	    goto bad;
+	}
+    }
+    else {
+	my ($current_size) = $query_result->fetchrow_array();
+
+	#
+	# At the moment, I am not going to allow the blockstore
+	# to change size. 
+	#
+	if ($current_size && $current_size != $bs_size) {
+	    print STDERR "Not allowed to change size of existing store\n";
+	    goto bad;
+	}
+
+	#
+	# If already have a reservation size, then this is most
+	# likely a swapmod, and we can just return without doing
+	# anything.
+	#
+	goto done
+	    if ($current_size);
+    }
+
+    #
+    # Now do an atomic update that changes both tables.
+    #
+    if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
+		"     remaining_capacity=remaining_capacity-${bs_size}, ".
+		"     size='$bs_size' ".
+		"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
+		"      blockstore_state.bs_id=reserved_blockstores.bs_id and ".
+		"      reserved_blockstores.bsidx='$bsidx' and ".
+		"      reserved_blockstores.exptidx='$exptidx' and ".
+		"      reserved_blockstores.vnode_id='$vnode_id'")) {
+	goto bad;
+    }
+done:
+    DBQueryWarn("unlock tables");
+    return 0;
+  bad:
+    DBQueryWarn("unlock tables");
+    return -1;
+}
+
+############################################################################
+#
+# Package to describe a specific reservation of a blockstore.
+#
+package Blockstore::Reservation;
+use libdb;
+use libtestbed;
+use English;
+use vars qw($AUTOLOAD);
+use overload ('""' => 'Stringify');
+
+#
+# Lookup a blockstore reservation.
+#
+sub Lookup($$$$)
+{
+    my ($class, $blockstore, $experiment, $vname) = @_;
+
+    return undef
+	if (! ($vname =~ /^[-\w]+$/ && ref($blockstore) && ref($experiment)));
+
+    my $exptidx = $experiment->idx();
+    my $bsidx   = $blockstore->bsidx();
+
+    my $query_result =
+	DBQueryWarn("select * from reserved_blockstores ".
+		    "where exptidx='$exptidx' and bsidx='$bsidx' and ".
+		    "       vname='$vname'");
+
+    return undef
+	if (!$query_result || !$query_result->numrows);
+
+    my $self         = {};
+    $self->{"HASH"}  = {};
+    $self->{"DBROW"} = $query_result->fetchrow_hashref();
+
+    bless($self, $class);
+    return $self;
+}
+
+#
+# Look for the blockstore associated with a pcvm. At the present
+# time, one blockstore is mapped to one pcvm. 
+#
+sub LookupByNodeid($$)
+{
+    my ($class, $vnode_id) = @_;
+
+    my $query_result =
+	DBQueryWarn("select * from reserved_blockstores ".
+		    "where vnode_id='$vnode_id'");
+
+    return undef
+	if (!$query_result || !$query_result->numrows);
+
+    if ($query_result->numrows != 1) {
+	print STDERR "Too many blockstores for $vnode_id!\n";
+	return -1;
+    }
+
+    my $self         = {};
+    $self->{"HASH"}  = {};
+    $self->{"DBROW"} = $query_result->fetchrow_hashref();
+
+    bless($self, $class);
+    return $self;
+}
+
+# To avoid writing out all the methods.
+AUTOLOAD {
+    my $self  = $_[0];
+    my $name  = $AUTOLOAD;
+    $name =~ s/.*://;   # strip fully-qualified portion
+
+    # A DB row proxy method call.
+    if (exists($self->{'DBROW'}->{$name})) {
+	return $self->{'DBROW'}->{$name};
+    }
+    carp("No such slot '$name' field in $self");
+    return undef;
+}
+
+# Break circular reference someplace to avoid exit errors.
+sub DESTROY {
+    my $self = shift;
+
+    $self->{"DBROW"}    = undef;
+    $self->{"HASH"}     = undef;
+}
+
+#
+# Stringify for output.
+#
+sub Stringify($)
+{
+    my ($self) = @_;
+    
+    my $bsidx   = $self->bsidx();
+    my $bs_id   = $self->bs_id();
+    my $node_id = $self->node_id();
+    my $vname   = $self->vname();
+
+    return "[BlockStore:$bsidx, $bs_id, $node_id ($vname)]";
+}
+
+#
+# Blockstores are reserved to a pcvm; that is how we do the
+# bookkeeping. When a node is released (nfree), we can find
+# the reserved blockstore for that node, reset the capacity
+# in the blockstore_state table, and delete the row(s).
+#
+sub Release($)
+{
+    my ($self)     = @_;
+    my $exptidx    = $self->exptidx();
+    my $bsidx      = $self->bsidx();
+    my $bs_id      = $self->bs_id();
+    my $bs_node_id = $self->node_id();
+    my $vnode_id   = $self->vnode_id();
+    my $size       = $self->size();
+
+    DBQueryWarn("lock tables blockstores read, ".
+		"            reserved_blockstores write, ".
+		"            blockstore_state write")
+	or return -1;
+
+    #
+    # Need the remaining size to deallocate.
+    #
+    my $query_result =
+	DBQueryWarn("select remaining_capacity from blockstore_state ".
+		    "where bsidx='$bsidx'");
+    goto bad
+	if (!$query_result);
+
+    if (!$query_result->numrows) {
+	print STDERR "No blockstore state for $bsidx\n";
+	goto bad;
+    }
+
+    #
+    # We want to atomically uupdate update remaining_capacity and
+    # set the size in the reservation to zero, so that if we fail,
+    # nothing has changed.
+    #
+    if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
+		"     remaining_capacity=remaining_capacity+size, ".
+		"     size=0-size ".
+		"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
+		"      blockstore_state.bs_id=reserved_blockstores.bs_id and ".
+		"      reserved_blockstores.bsidx='$bsidx' and ".
+		"      reserved_blockstores.exptidx='$exptidx' and ".
+		"      reserved_blockstores.vnode_id='$vnode_id'")) {
+	goto bad;
+    }
+    # That worked, so now we can delete the reservation row.
+    DBQueryWarn("delete from reserved_blockstores ".
+		"where reserved_blockstores.bsidx='$bsidx' and ".
+		"      reserved_blockstores.exptidx='$exptidx' and ".
+		"      reserved_blockstores.vnode_id='$vnode_id'")
+	or goto bad;
+
+    DBQueryWarn("unlock tables");
+    return 0;
+  bad:
+    DBQueryWarn("unlock tables");
+    return -1;
+}
+
+# _Always_ make sure that this 1 is at the end of the file...
+1;

db/EmulabConstants.pm.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2000-2012 University of Utah and the Flux Group.
+# Copyright (c) 2000-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -152,7 +152,7 @@ use vars qw(@ISA @EXPORT);
 	 TBDB_FRISBEEMCBASEADDR
 
 	 TBDB_RSRVROLE_NODE TBDB_RSRVROLE_VIRTHOST TBDB_RSRVROLE_DELAYNODE
-	 TBDB_RSRVROLE_SIMHOST
+	 TBDB_RSRVROLE_SIMHOST TBDB_RSRVROLE_STORAGEHOST
 
 	 TBDB_EXPT_WORKDIR
 	 TB_OSID_MBKERNEL 
@@ -544,6 +544,7 @@ sub TBDB_RSRVROLE_NODE()	{ "node"; }
 sub TBDB_RSRVROLE_VIRTHOST()	{ "virthost"; }
 sub TBDB_RSRVROLE_DELAYNODE()	{ "delaynode"; }
 sub TBDB_RSRVROLE_SIMHOST()	{ "simhost"; }
+sub TBDB_RSRVROLE_STORAGEHOST()	{ "storagehost"; }
 
 # Interfaces roles.
 sub TBDB_IFACEROLE_CONTROL()	{ "ctrl"; }

db/GNUmakefile.in

 #
-# Copyright (c) 2000-2012 University of Utah and the Flux Group.
+# Copyright (c) 2000-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -47,8 +47,8 @@ LIB_SCRIPTS     = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \
 		  NodeType.pm Interface.pm User.pm Group.pm Project.pm \
 		  Image.pm OSinfo.pm Archive.pm Logfile.pm Lan.pm emdbi.pm \
 		  emdb.pm emutil.pm Firewall.pm VirtExperiment.pm libGeni.pm \
-		  libEmulab.pm EmulabConstants.pm TraceUse.pm EmulabFeatures.pm \
-		  Port.pm BlockstoreType.pm
+		  libEmulab.pm EmulabConstants.pm TraceUse.pm \
+		  EmulabFeatures.pm Port.pm BlockstoreType.pm Blockstore.pm
 
 # Stuff installed on plastic.
 USERSBINS	= genelists.proxy dumperrorlog.proxy backup

db/Node.pm.in

 #!/usr/bin/perl -wT
 #
-# Copyright (c) 2005-2012 University of Utah and the Flux Group.
+# Copyright (c) 2005-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -46,6 +46,7 @@ $JAILIPMASK  = "@JAILIPMASK@";
 use libdb;
 use libtestbed;
 use emutil;
+use Blockstore;
 use event;
 use English;
 use Socket;
@@ -1748,6 +1749,29 @@ sub ReleaseSharedBandwidth($)
     return 0;
 }
 
+#
+# Relase the reserved blockstore, which requires updating the
+# remaining_capacity on the underyling store. At the present
+# time, one blockstore is mapped to one pcvm. 
+#
+sub ReleaseBlockStore($)
+{
+    my ($self) = @_;
+
+    # Must be a real reference. 
+    return -1
+	if (! ref($self));
+
+    my $node_id    = $self->node_id();
+    my $blockstore = Blockstore::Reservation->LookupByNodeid($node_id);
+    return 0
+	if (!defined($blockstore));
+    return -1
+	if (!ref($blockstore));
+
+    return $blockstore->Release();
+}
+
 #
 # Look up all interfaces for a node, return list of objects.
 #
@@ -2129,8 +2153,7 @@ sub CreateVnodes($$$)
     my $sharing_mode = $node->sharing_mode();
     
     if (! ($impotent || $isfednode)) {
-	my $reservation = $node->Reservation();
-	if (!defined($reservation)) {
+	if (!$node->IsReserved()) {
 	    print STDERR "*** CreateVnodes: no reservation for $node!\n";
 	    DBQueryFatal("unlock tables");
 	    return -1;
@@ -2142,7 +2165,11 @@ sub CreateVnodes($$$)
 	# is on and the pnode is in sharedmode. Locking in nfree and in
 	# the pool daemon prevents the race. 
 	#
-	if (!$experiment->SameExperiment($reservation)) {
+	# Cause of locking above, we need to make the comparison directly
+	# using the slot data in the node.
+	#
+	if (! ($experiment->pid() eq $node->pid() &&
+	       $experiment->eid() eq $node->eid())) {
 	    if (! ($sharedokay && $sharing_mode)) {
 		print STDERR "*** CreateVnodes: $node is not shared!\n";
 		DBQueryFatal("unlock tables");

db/nfree.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2000-2012 University of Utah and the Flux Group.
+# Copyright (c) 2000-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -450,7 +450,7 @@ foreach my $node (@freed_nodes) {
     }
 
     #
-    # If the node is virtual, release the shared bandwidth it had
+    # If the node is virtual, release the shared resources it had
     # reserved on the physical node.
     #
     if ($isvirt) {
@@ -459,6 +459,8 @@ foreach my $node (@freed_nodes) {
 	    DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'")
 		or $error++;
 	}
+	$node->ReleaseBlockStore() == 0
+	    or $error++;
     }
 
     #

tbsetup/libossetup.pm.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2000-2012 University of Utah and the Flux Group.
+# Copyright (c) 2000-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -2009,4 +2009,41 @@ sub WaitDone($@)
     return libossetup_virtnode::WaitDone($self, @nodelist);
 }
 
+#####################################################################
+#
+# Place holder for blockstore
+#
+package libossetup_blockstore;
+use base qw(libossetup_subnode);
+
+#
+# A constructor for an object to handle all nodes of this type.
+#
+sub New($$) {
+    my ($class, $parent) = @_;
+
+    my $self = $class->SUPER::New("blockstore", $parent);
+    bless($self, $class);
+    return $self;
+}
+
+sub AddNode($$)
+{
+    my ($self, $node) = @_;
+
+    print "Will skip blockstore $node ISUP wait.\n";
+
+    return $self->SUPER::AddNode($node);
+}
+
+#
+# We do not currently do anything with blockstores; no waiting for ISUP. 
+#
+sub Volunteers($)
+{
+    my ($self) = @_;
+
+    return ();
+}
+
 1;

tbsetup/libvtop_test.pm.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2005-2012 University of Utah and the Flux Group.
+# Copyright (c) 2005-2013 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -37,11 +37,13 @@ use libdb;
 use libtblog_simple;
 use libtestbed;
 use Experiment;
+use EmulabConstants;
 use VirtExperiment;
 use Node;
 use NodeType;
 use Lan;
 use OSinfo;
+use Blockstore;
 use Image;
 use Port;
 use English;
@@ -107,6 +109,8 @@ sub Create($$$$)
     $self->{'DELAYNODES'} = {};
     $self->{'LANNODES'}   = {};
     $self->{'BRIDGES'}    = {};
+    $self->{'BLOCKSTORES'}= {};
+    $self->{'SANHOSTS'}   = {};
     $self->{'VLANS'}      = {};
     $self->{'VPATHS'}     = {};
     $self->{'MEMBEROF'}   = {};
@@ -154,6 +158,8 @@ sub flags($)            { return $_[0]->{'FLAGS'}; }
 sub vnodes($)           { return $_[0]->{'VNODES'}; }
 sub delaynodes($)       { return $_[0]->{'DELAYNODES'}; }
 sub bridges($)          { return $_[0]->{'BRIDGES'}; }
+sub blockstores($)      { return $_[0]->{'BLOCKSTORES'}; }
+sub sanhosts($)         { return $_[0]->{'SANHOSTS'}; }
 sub lannodes($)         { return $_[0]->{'LANNODES'}; }
 sub vlans($)            { return $_[0]->{'VLANS'}; }
 sub memberof($)         { return $_[0]->{'MEMBEROF'}; }
@@ -199,10 +205,14 @@ sub virt_trafgens($)	{ return $_[0]->virt_table("virt_trafgens"); }
 sub virt_lan_settings($){ return $_[0]->virt_table("virt_lan_settings"); }
 sub virt_lan_member_settings($)	{
     return $_[0]->virt_table("virt_lan_member_settings"); }
+sub virt_blockstores($) { return $_[0]->virt_table("virt_blockstores"); }
+sub virt_blockstore_attributes($) {
+    return $_[0]->virt_table("virt_blockstore_attributes"); }
 
 # Given a vname, is it a node in the topo (or something else like a delay).
 sub isatoponode($$)     { return exists($_[0]->vnodes()->{$_[1]}); }
 sub isadelaynode($$)    { return exists($_[0]->delaynodes()->{$_[1]}); }
+sub isasanhost($$)      { return exists($_[0]->sanhosts()->{$_[1]}); }
 
 # Debug output.
 sub verbose($)		{ return $_[0]->flags() & $VTOP_FLAGS_VERBOSE; }
@@ -239,6 +249,7 @@ sub virtnodecount($)	{ return $_[0]->counter("virtcount"); }
 sub simnodecount($)	{ return $_[0]->counter("simcount"); }
 sub remotenodecount($)	{ return $_[0]->counter("remotecount"); }
 sub sharednodecount($)	{ return $_[0]->counter("sharedcount"); }
+sub bstorecount($)	{ return $_[0]->counter("bstorecount"); }
 
 sub createLink($$$$$$$$$)
 {
@@ -1136,6 +1147,9 @@ sub LoadVirtNodes($)
 	$vnode->_sharedokay(0);
 	$vnode->_fixedvm(undef);
 	$vnode->_isbridge(($vnode->role() eq "bridge" ? 1 : 0));
+	$vnode->_blockstore(undef);
+	$vnode->_blockstore_attributes(undef);
+	$vnode->_sanhostname(undef);
 
 	#
 	# Explicit type request desire to match feature.
@@ -1451,6 +1465,46 @@ sub LoadVirtNodes($)
 	$self->printdb("Fixing VM $vname to $fixnode, $osinfo\n");
     }
 
+    #
+    # Load up the blockstore stuff since we have to spit out the
+    # attributes as desires.
+