Commit e321b385 authored by Leigh B Stoller's avatar Leigh B Stoller

First cut at blockstore support. This example maps but does not

swap in yet:

set client [$ns node]
tb-set-node-os $client FEDORA15-STD

set d1 [$ns blockstore]
$d1 set-class "SAN"
$d1 set-protocol "iSCSI"
$d1 set-size 2GB

set san [$ns duplex-link $client $d1 10Kbps 0ms DropTail]
parent adb428d4
#!/usr/bin/perl -wT
#
# Copyright (c) 2012-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
package Blockstore;
use strict;
use Carp;
use Exporter;
use vars qw(@ISA @EXPORT $AUTOLOAD);
@ISA = qw(Exporter);
@EXPORT = qw ( );
use libdb;
use libtestbed;
use English;
use Data::Dumper;
use overload ('""' => 'Stringify');
my $debug = 0;
#
# Lookup a (physical) storage object type and create a class instance to
# return.
#
sub Lookup($$$)
{
my ($class, $nodeid, $bsid) = @_;
return undef
if (! ($nodeid =~ /^[-\w]+$/ && $bsid =~ /^[-\w]+$/));
my $query_result =
DBQueryWarn("select * from blockstores ".
"where node_id='$nodeid' and bs_id='$bsid'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
# To avoid writing out all the methods.
AUTOLOAD {
my $self = $_[0];
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
# A DB row proxy method call.
if (exists($self->{'DBROW'}->{$name})) {
return $self->{'DBROW'}->{$name};
}
# Local storage slot.
if ($name =~ /^_.*$/) {
if (scalar(@_) == 2) {
return $self->{'HASH'}->{$name} = $_[1];
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
}
carp("No such slot '$name' field in $self");
return undef;
}
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{"DBROW"} = undef;
$self->{"HASH"} = undef;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $node_id = $self->node_id();
return "[BlockStore:$bsidx, $bs_id, $node_id]";
}
#
# Blockstores are reserved to a pcvm; that is how we do the
# bookkeeping. When a node is released (nfree), we can find
# the reserved blockstores for that node, reset the capacity
# in the blockstore_state table, and delete the row(s).
#
sub Reserve($$$$$)
{
my ($self, $experiment, $vnode_id, $bs_name, $bs_size) = @_;
my $exptidx = $experiment->idx();
my $pid = $experiment->pid();
my $eid = $experiment->eid();
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $bs_node_id = $self->node_id();
my $remaining_capacity;
DBQueryWarn("lock tables blockstores read, ".
" reserved_blockstores write, ".
" blockstore_state write")
or return -1;
#
# Need the remaining size to make sure we can allocate it.
#
my $query_result =
DBQueryWarn("select remaining_capacity from blockstore_state ".
"where bsidx='$bsidx'");
goto bad
if (!$query_result);
#
# Just in case the state row is missing, okay to create it.
#
if (!$query_result->numrows) {
$remaining_capacity = $self->total_size();
DBQueryWarn("insert into blockstore_state set ".
" bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
" remaining_capacity='$remaining_capacity', 'ready=1'")
or goto bad;
}
else {
($remaining_capacity) = $query_result->fetchrow_array();
}
if ($bs_size > $remaining_capacity) {
print STDERR "Not enough remaining capacity on $bsidx\n";
goto bad;
}
#
# If we do not have a reservation row, create one with a zero
# size, to indicate nothing has actually been reserved in the
# blockstore_state table.
#
$query_result =
DBQueryWarn("select size from reserved_blockstores ".
"where exptidx='$exptidx' and bsidx='$bsidx' and ".
" vname='$bs_name'");
goto bad
if (!$query_result);
if (! $query_result->numrows) {
if (! DBQueryWarn("insert into reserved_blockstores set ".
" bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ".
" vname='$bs_name', pid='$pid', eid='$eid', ".
" size='0', vnode_id='$vnode_id', ".
" exptidx='$exptidx', rsrv_time=now()")) {
goto bad;
}
}
else {
my ($current_size) = $query_result->fetchrow_array();
#
# At the moment, I am not going to allow the blockstore
# to change size.
#
if ($current_size && $current_size != $bs_size) {
print STDERR "Not allowed to change size of existing store\n";
goto bad;
}
#
# If already have a reservation size, then this is most
# likely a swapmod, and we can just return without doing
# anything.
#
goto done
if ($current_size);
}
#
# Now do an atomic update that changes both tables.
#
if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
" remaining_capacity=remaining_capacity-${bs_size}, ".
" size='$bs_size' ".
"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
" blockstore_state.bs_id=reserved_blockstores.bs_id and ".
" reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")) {
goto bad;
}
done:
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return -1;
}
############################################################################
#
# Package to describe a specific reservation of a blockstore.
#
package Blockstore::Reservation;
use libdb;
use libtestbed;
use English;
use vars qw($AUTOLOAD);
use overload ('""' => 'Stringify');
#
# Lookup a blockstore reservation.
#
sub Lookup($$$$)
{
my ($class, $blockstore, $experiment, $vname) = @_;
return undef
if (! ($vname =~ /^[-\w]+$/ && ref($blockstore) && ref($experiment)));
my $exptidx = $experiment->idx();
my $bsidx = $blockstore->bsidx();
my $query_result =
DBQueryWarn("select * from reserved_blockstores ".
"where exptidx='$exptidx' and bsidx='$bsidx' and ".
" vname='$vname'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
#
# Look for the blockstore associated with a pcvm. At the present
# time, one blockstore is mapped to one pcvm.
#
sub LookupByNodeid($$)
{
my ($class, $vnode_id) = @_;
my $query_result =
DBQueryWarn("select * from reserved_blockstores ".
"where vnode_id='$vnode_id'");
return undef
if (!$query_result || !$query_result->numrows);
if ($query_result->numrows != 1) {
print STDERR "Too many blockstores for $vnode_id!\n";
return -1;
}
my $self = {};
$self->{"HASH"} = {};
$self->{"DBROW"} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
# To avoid writing out all the methods.
AUTOLOAD {
my $self = $_[0];
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
# A DB row proxy method call.
if (exists($self->{'DBROW'}->{$name})) {
return $self->{'DBROW'}->{$name};
}
carp("No such slot '$name' field in $self");
return undef;
}
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{"DBROW"} = undef;
$self->{"HASH"} = undef;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $node_id = $self->node_id();
my $vname = $self->vname();
return "[BlockStore:$bsidx, $bs_id, $node_id ($vname)]";
}
#
# Blockstores are reserved to a pcvm; that is how we do the
# bookkeeping. When a node is released (nfree), we can find
# the reserved blockstore for that node, reset the capacity
# in the blockstore_state table, and delete the row(s).
#
sub Release($)
{
my ($self) = @_;
my $exptidx = $self->exptidx();
my $bsidx = $self->bsidx();
my $bs_id = $self->bs_id();
my $bs_node_id = $self->node_id();
my $vnode_id = $self->vnode_id();
my $size = $self->size();
DBQueryWarn("lock tables blockstores read, ".
" reserved_blockstores write, ".
" blockstore_state write")
or return -1;
#
# Need the remaining size to deallocate.
#
my $query_result =
DBQueryWarn("select remaining_capacity from blockstore_state ".
"where bsidx='$bsidx'");
goto bad
if (!$query_result);
if (!$query_result->numrows) {
print STDERR "No blockstore state for $bsidx\n";
goto bad;
}
#
# We want to atomically uupdate update remaining_capacity and
# set the size in the reservation to zero, so that if we fail,
# nothing has changed.
#
if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ".
" remaining_capacity=remaining_capacity+size, ".
" size=0-size ".
"where blockstore_state.bsidx=reserved_blockstores.bsidx and ".
" blockstore_state.bs_id=reserved_blockstores.bs_id and ".
" reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")) {
goto bad;
}
# That worked, so now we can delete the reservation row.
DBQueryWarn("delete from reserved_blockstores ".
"where reserved_blockstores.bsidx='$bsidx' and ".
" reserved_blockstores.exptidx='$exptidx' and ".
" reserved_blockstores.vnode_id='$vnode_id'")
or goto bad;
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return -1;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2012 University of Utah and the Flux Group. # Copyright (c) 2000-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -152,7 +152,7 @@ use vars qw(@ISA @EXPORT); ...@@ -152,7 +152,7 @@ use vars qw(@ISA @EXPORT);
TBDB_FRISBEEMCBASEADDR TBDB_FRISBEEMCBASEADDR
TBDB_RSRVROLE_NODE TBDB_RSRVROLE_VIRTHOST TBDB_RSRVROLE_DELAYNODE TBDB_RSRVROLE_NODE TBDB_RSRVROLE_VIRTHOST TBDB_RSRVROLE_DELAYNODE
TBDB_RSRVROLE_SIMHOST TBDB_RSRVROLE_SIMHOST TBDB_RSRVROLE_STORAGEHOST
TBDB_EXPT_WORKDIR TBDB_EXPT_WORKDIR
TB_OSID_MBKERNEL TB_OSID_MBKERNEL
...@@ -544,6 +544,7 @@ sub TBDB_RSRVROLE_NODE() { "node"; } ...@@ -544,6 +544,7 @@ sub TBDB_RSRVROLE_NODE() { "node"; }
sub TBDB_RSRVROLE_VIRTHOST() { "virthost"; } sub TBDB_RSRVROLE_VIRTHOST() { "virthost"; }
sub TBDB_RSRVROLE_DELAYNODE() { "delaynode"; } sub TBDB_RSRVROLE_DELAYNODE() { "delaynode"; }
sub TBDB_RSRVROLE_SIMHOST() { "simhost"; } sub TBDB_RSRVROLE_SIMHOST() { "simhost"; }
sub TBDB_RSRVROLE_STORAGEHOST() { "storagehost"; }
# Interfaces roles. # Interfaces roles.
sub TBDB_IFACEROLE_CONTROL() { "ctrl"; } sub TBDB_IFACEROLE_CONTROL() { "ctrl"; }
......
# #
# Copyright (c) 2000-2012 University of Utah and the Flux Group. # Copyright (c) 2000-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -47,8 +47,8 @@ LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \ ...@@ -47,8 +47,8 @@ LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \
NodeType.pm Interface.pm User.pm Group.pm Project.pm \ NodeType.pm Interface.pm User.pm Group.pm Project.pm \
Image.pm OSinfo.pm Archive.pm Logfile.pm Lan.pm emdbi.pm \ Image.pm OSinfo.pm Archive.pm Logfile.pm Lan.pm emdbi.pm \
emdb.pm emutil.pm Firewall.pm VirtExperiment.pm libGeni.pm \ emdb.pm emutil.pm Firewall.pm VirtExperiment.pm libGeni.pm \
libEmulab.pm EmulabConstants.pm TraceUse.pm EmulabFeatures.pm \ libEmulab.pm EmulabConstants.pm TraceUse.pm \
Port.pm BlockstoreType.pm EmulabFeatures.pm Port.pm BlockstoreType.pm Blockstore.pm
# Stuff installed on plastic. # Stuff installed on plastic.
USERSBINS = genelists.proxy dumperrorlog.proxy backup USERSBINS = genelists.proxy dumperrorlog.proxy backup
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2005-2012 University of Utah and the Flux Group. # Copyright (c) 2005-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -46,6 +46,7 @@ $JAILIPMASK = "@JAILIPMASK@"; ...@@ -46,6 +46,7 @@ $JAILIPMASK = "@JAILIPMASK@";
use libdb; use libdb;
use libtestbed; use libtestbed;
use emutil; use emutil;
use Blockstore;
use event; use event;
use English; use English;
use Socket; use Socket;
...@@ -1748,6 +1749,29 @@ sub ReleaseSharedBandwidth($) ...@@ -1748,6 +1749,29 @@ sub ReleaseSharedBandwidth($)
return 0; return 0;
} }
#
# Relase the reserved blockstore, which requires updating the
# remaining_capacity on the underyling store. At the present
# time, one blockstore is mapped to one pcvm.
#
sub ReleaseBlockStore($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $node_id = $self->node_id();
my $blockstore = Blockstore::Reservation->LookupByNodeid($node_id);
return 0
if (!defined($blockstore));
return -1
if (!ref($blockstore));
return $blockstore->Release();
}
# #
# Look up all interfaces for a node, return list of objects. # Look up all interfaces for a node, return list of objects.
# #
...@@ -2129,8 +2153,7 @@ sub CreateVnodes($$$) ...@@ -2129,8 +2153,7 @@ sub CreateVnodes($$$)
my $sharing_mode = $node->sharing_mode(); my $sharing_mode = $node->sharing_mode();
if (! ($impotent || $isfednode)) { if (! ($impotent || $isfednode)) {
my $reservation = $node->Reservation(); if (!$node->IsReserved()) {
if (!defined($reservation)) {
print STDERR "*** CreateVnodes: no reservation for $node!\n"; print STDERR "*** CreateVnodes: no reservation for $node!\n";
DBQueryFatal("unlock tables"); DBQueryFatal("unlock tables");
return -1; return -1;
...@@ -2142,7 +2165,11 @@ sub CreateVnodes($$$) ...@@ -2142,7 +2165,11 @@ sub CreateVnodes($$$)
# is on and the pnode is in sharedmode. Locking in nfree and in # is on and the pnode is in sharedmode. Locking in nfree and in
# the pool daemon prevents the race. # the pool daemon prevents the race.
# #
if (!$experiment->SameExperiment($reservation)) { # Cause of locking above, we need to make the comparison directly
# using the slot data in the node.
#
if (! ($experiment->pid() eq $node->pid() &&
$experiment->eid() eq $node->eid())) {
if (! ($sharedokay && $sharing_mode)) { if (! ($sharedokay && $sharing_mode)) {
print STDERR "*** CreateVnodes: $node is not shared!\n"; print STDERR "*** CreateVnodes: $node is not shared!\n";
DBQueryFatal("unlock tables"); DBQueryFatal("unlock tables");
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2012 University of Utah and the Flux Group. # Copyright (c) 2000-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -450,7 +450,7 @@ foreach my $node (@freed_nodes) { ...@@ -450,7 +450,7 @@ foreach my $node (@freed_nodes) {
} }
# #
# If the node is virtual, release the shared bandwidth it had # If the node is virtual, release the shared resources it had
# reserved on the physical node. # reserved on the physical node.
# #
if ($isvirt) { if ($isvirt) {
...@@ -459,6 +459,8 @@ foreach my $node (@freed_nodes) { ...@@ -459,6 +459,8 @@ foreach my $node (@freed_nodes) {
DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'") DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'")
or $error++; or $error++;
} }
$node->ReleaseBlockStore() == 0
or $error++;
} }
# #
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2012 University of Utah and the Flux Group. # Copyright (c) 2000-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -2009,4 +2009,41 @@ sub WaitDone($@) ...@@ -2009,4 +2009,41 @@ sub WaitDone($@)
return libossetup_virtnode::WaitDone($self, @nodelist); return libossetup_virtnode::WaitDone($self, @nodelist);
} }
#####################################################################
#
# Place holder for blockstore
#
package libossetup_blockstore;
use base qw(libossetup_subnode);
#
# A constructor for an object to handle all nodes of this type.
#
sub New($$) {
my ($class, $parent) = @_;
my $self = $class->SUPER::New("blockstore", $parent);
bless($self, $class);
return $self;
}
sub AddNode($$)
{
my ($self, $node) = @_;
print "Will skip blockstore $node ISUP wait.\n";
return $self->SUPER::AddNode($node);
}
#
# We do not currently do anything with blockstores; no waiting for ISUP.
#
sub Volunteers($)
{
my ($self) = @_;
return ();
}
1; 1;
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2005-2012 University of Utah and the Flux Group. # Copyright (c) 2005-2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -37,11 +37,13 @@ use libdb; ...@@ -37,11 +37,13 @@ use libdb;
use libtblog_simple; use libtblog_simple;
use libtestbed; use libtestbed;
use Experiment; use Experiment;
use EmulabConstants;
use VirtExperiment; use VirtExperiment;
use Node; use Node;
use NodeType;