diff --git a/db/Blockstore.pm b/db/Blockstore.pm new file mode 100644 index 0000000000000000000000000000000000000000..48dddf0052e94f053e39cbc2ff524081c2b45486 --- /dev/null +++ b/db/Blockstore.pm @@ -0,0 +1,397 @@ +#!/usr/bin/perl -wT +# +# Copyright (c) 2012-2013 University of Utah and the Flux Group. +# +# {{{EMULAB-LICENSE +# +# This file is part of the Emulab network testbed software. +# +# This file is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This file is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +# License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this file. If not, see . +# +# }}} +# +package Blockstore; + +use strict; +use Carp; +use Exporter; +use vars qw(@ISA @EXPORT $AUTOLOAD); + +@ISA = qw(Exporter); +@EXPORT = qw ( ); + +use libdb; +use libtestbed; +use English; +use Data::Dumper; +use overload ('""' => 'Stringify'); + +my $debug = 0; + +# +# Lookup a (physical) storage object type and create a class instance to +# return. +# +sub Lookup($$$) +{ + my ($class, $nodeid, $bsid) = @_; + + return undef + if (! ($nodeid =~ /^[-\w]+$/ && $bsid =~ /^[-\w]+$/)); + + my $query_result = + DBQueryWarn("select * from blockstores ". + "where node_id='$nodeid' and bs_id='$bsid'"); + + return undef + if (!$query_result || !$query_result->numrows); + + my $self = {}; + $self->{"HASH"} = {}; + $self->{"DBROW"} = $query_result->fetchrow_hashref(); + + bless($self, $class); + return $self; +} + +# To avoid writing out all the methods. +AUTOLOAD { + my $self = $_[0]; + my $name = $AUTOLOAD; + $name =~ s/.*://; # strip fully-qualified portion + + # A DB row proxy method call. + if (exists($self->{'DBROW'}->{$name})) { + return $self->{'DBROW'}->{$name}; + } + + # Local storage slot. + if ($name =~ /^_.*$/) { + if (scalar(@_) == 2) { + return $self->{'HASH'}->{$name} = $_[1]; + } + elsif (exists($self->{'HASH'}->{$name})) { + return $self->{'HASH'}->{$name}; + } + } + carp("No such slot '$name' field in $self"); + return undef; +} + +# Break circular reference someplace to avoid exit errors. +sub DESTROY { + my $self = shift; + + $self->{"DBROW"} = undef; + $self->{"HASH"} = undef; +} + +# +# Stringify for output. +# +sub Stringify($) +{ + my ($self) = @_; + + my $bsidx = $self->bsidx(); + my $bs_id = $self->bs_id(); + my $node_id = $self->node_id(); + + return "[BlockStore:$bsidx, $bs_id, $node_id]"; +} + +# +# Blockstores are reserved to a pcvm; that is how we do the +# bookkeeping. When a node is released (nfree), we can find +# the reserved blockstores for that node, reset the capacity +# in the blockstore_state table, and delete the row(s). +# +sub Reserve($$$$$) +{ + my ($self, $experiment, $vnode_id, $bs_name, $bs_size) = @_; + my $exptidx = $experiment->idx(); + my $pid = $experiment->pid(); + my $eid = $experiment->eid(); + my $bsidx = $self->bsidx(); + my $bs_id = $self->bs_id(); + my $bs_node_id = $self->node_id(); + my $remaining_capacity; + + DBQueryWarn("lock tables blockstores read, ". + " reserved_blockstores write, ". + " blockstore_state write") + or return -1; + + # + # Need the remaining size to make sure we can allocate it. + # + my $query_result = + DBQueryWarn("select remaining_capacity from blockstore_state ". + "where bsidx='$bsidx'"); + goto bad + if (!$query_result); + + # + # Just in case the state row is missing, okay to create it. + # + if (!$query_result->numrows) { + $remaining_capacity = $self->total_size(); + + DBQueryWarn("insert into blockstore_state set ". + " bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ". + " remaining_capacity='$remaining_capacity', 'ready=1'") + or goto bad; + } + else { + ($remaining_capacity) = $query_result->fetchrow_array(); + } + if ($bs_size > $remaining_capacity) { + print STDERR "Not enough remaining capacity on $bsidx\n"; + goto bad; + } + + # + # If we do not have a reservation row, create one with a zero + # size, to indicate nothing has actually been reserved in the + # blockstore_state table. + # + $query_result = + DBQueryWarn("select size from reserved_blockstores ". + "where exptidx='$exptidx' and bsidx='$bsidx' and ". + " vname='$bs_name'"); + goto bad + if (!$query_result); + + if (! $query_result->numrows) { + if (! DBQueryWarn("insert into reserved_blockstores set ". + " bsidx='$bsidx', node_id='$bs_node_id', bs_id='$bs_id', ". + " vname='$bs_name', pid='$pid', eid='$eid', ". + " size='0', vnode_id='$vnode_id', ". + " exptidx='$exptidx', rsrv_time=now()")) { + goto bad; + } + } + else { + my ($current_size) = $query_result->fetchrow_array(); + + # + # At the moment, I am not going to allow the blockstore + # to change size. + # + if ($current_size && $current_size != $bs_size) { + print STDERR "Not allowed to change size of existing store\n"; + goto bad; + } + + # + # If already have a reservation size, then this is most + # likely a swapmod, and we can just return without doing + # anything. + # + goto done + if ($current_size); + } + + # + # Now do an atomic update that changes both tables. + # + if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ". + " remaining_capacity=remaining_capacity-${bs_size}, ". + " size='$bs_size' ". + "where blockstore_state.bsidx=reserved_blockstores.bsidx and ". + " blockstore_state.bs_id=reserved_blockstores.bs_id and ". + " reserved_blockstores.bsidx='$bsidx' and ". + " reserved_blockstores.exptidx='$exptidx' and ". + " reserved_blockstores.vnode_id='$vnode_id'")) { + goto bad; + } +done: + DBQueryWarn("unlock tables"); + return 0; + bad: + DBQueryWarn("unlock tables"); + return -1; +} + +############################################################################ +# +# Package to describe a specific reservation of a blockstore. +# +package Blockstore::Reservation; +use libdb; +use libtestbed; +use English; +use vars qw($AUTOLOAD); +use overload ('""' => 'Stringify'); + +# +# Lookup a blockstore reservation. +# +sub Lookup($$$$) +{ + my ($class, $blockstore, $experiment, $vname) = @_; + + return undef + if (! ($vname =~ /^[-\w]+$/ && ref($blockstore) && ref($experiment))); + + my $exptidx = $experiment->idx(); + my $bsidx = $blockstore->bsidx(); + + my $query_result = + DBQueryWarn("select * from reserved_blockstores ". + "where exptidx='$exptidx' and bsidx='$bsidx' and ". + " vname='$vname'"); + + return undef + if (!$query_result || !$query_result->numrows); + + my $self = {}; + $self->{"HASH"} = {}; + $self->{"DBROW"} = $query_result->fetchrow_hashref(); + + bless($self, $class); + return $self; +} + +# +# Look for the blockstore associated with a pcvm. At the present +# time, one blockstore is mapped to one pcvm. +# +sub LookupByNodeid($$) +{ + my ($class, $vnode_id) = @_; + + my $query_result = + DBQueryWarn("select * from reserved_blockstores ". + "where vnode_id='$vnode_id'"); + + return undef + if (!$query_result || !$query_result->numrows); + + if ($query_result->numrows != 1) { + print STDERR "Too many blockstores for $vnode_id!\n"; + return -1; + } + + my $self = {}; + $self->{"HASH"} = {}; + $self->{"DBROW"} = $query_result->fetchrow_hashref(); + + bless($self, $class); + return $self; +} + +# To avoid writing out all the methods. +AUTOLOAD { + my $self = $_[0]; + my $name = $AUTOLOAD; + $name =~ s/.*://; # strip fully-qualified portion + + # A DB row proxy method call. + if (exists($self->{'DBROW'}->{$name})) { + return $self->{'DBROW'}->{$name}; + } + carp("No such slot '$name' field in $self"); + return undef; +} + +# Break circular reference someplace to avoid exit errors. +sub DESTROY { + my $self = shift; + + $self->{"DBROW"} = undef; + $self->{"HASH"} = undef; +} + +# +# Stringify for output. +# +sub Stringify($) +{ + my ($self) = @_; + + my $bsidx = $self->bsidx(); + my $bs_id = $self->bs_id(); + my $node_id = $self->node_id(); + my $vname = $self->vname(); + + return "[BlockStore:$bsidx, $bs_id, $node_id ($vname)]"; +} + +# +# Blockstores are reserved to a pcvm; that is how we do the +# bookkeeping. When a node is released (nfree), we can find +# the reserved blockstore for that node, reset the capacity +# in the blockstore_state table, and delete the row(s). +# +sub Release($) +{ + my ($self) = @_; + my $exptidx = $self->exptidx(); + my $bsidx = $self->bsidx(); + my $bs_id = $self->bs_id(); + my $bs_node_id = $self->node_id(); + my $vnode_id = $self->vnode_id(); + my $size = $self->size(); + + DBQueryWarn("lock tables blockstores read, ". + " reserved_blockstores write, ". + " blockstore_state write") + or return -1; + + # + # Need the remaining size to deallocate. + # + my $query_result = + DBQueryWarn("select remaining_capacity from blockstore_state ". + "where bsidx='$bsidx'"); + goto bad + if (!$query_result); + + if (!$query_result->numrows) { + print STDERR "No blockstore state for $bsidx\n"; + goto bad; + } + + # + # We want to atomically uupdate update remaining_capacity and + # set the size in the reservation to zero, so that if we fail, + # nothing has changed. + # + if (!DBQueryWarn("update blockstore_state,reserved_blockstores set ". + " remaining_capacity=remaining_capacity+size, ". + " size=0-size ". + "where blockstore_state.bsidx=reserved_blockstores.bsidx and ". + " blockstore_state.bs_id=reserved_blockstores.bs_id and ". + " reserved_blockstores.bsidx='$bsidx' and ". + " reserved_blockstores.exptidx='$exptidx' and ". + " reserved_blockstores.vnode_id='$vnode_id'")) { + goto bad; + } + # That worked, so now we can delete the reservation row. + DBQueryWarn("delete from reserved_blockstores ". + "where reserved_blockstores.bsidx='$bsidx' and ". + " reserved_blockstores.exptidx='$exptidx' and ". + " reserved_blockstores.vnode_id='$vnode_id'") + or goto bad; + + DBQueryWarn("unlock tables"); + return 0; + bad: + DBQueryWarn("unlock tables"); + return -1; +} + +# _Always_ make sure that this 1 is at the end of the file... +1; diff --git a/db/EmulabConstants.pm.in b/db/EmulabConstants.pm.in index 852ff52375ea2c56e27ff270b75d36c324d0d038..0fce0f88de3c1e6cef4daa89e342f5cbdd118881 100644 --- a/db/EmulabConstants.pm.in +++ b/db/EmulabConstants.pm.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -152,7 +152,7 @@ use vars qw(@ISA @EXPORT); TBDB_FRISBEEMCBASEADDR TBDB_RSRVROLE_NODE TBDB_RSRVROLE_VIRTHOST TBDB_RSRVROLE_DELAYNODE - TBDB_RSRVROLE_SIMHOST + TBDB_RSRVROLE_SIMHOST TBDB_RSRVROLE_STORAGEHOST TBDB_EXPT_WORKDIR TB_OSID_MBKERNEL @@ -544,6 +544,7 @@ sub TBDB_RSRVROLE_NODE() { "node"; } sub TBDB_RSRVROLE_VIRTHOST() { "virthost"; } sub TBDB_RSRVROLE_DELAYNODE() { "delaynode"; } sub TBDB_RSRVROLE_SIMHOST() { "simhost"; } +sub TBDB_RSRVROLE_STORAGEHOST() { "storagehost"; } # Interfaces roles. sub TBDB_IFACEROLE_CONTROL() { "ctrl"; } diff --git a/db/GNUmakefile.in b/db/GNUmakefile.in index 7e5a042598510b02a49efadf9619949951dc9421..f0dec654a3b911ee048bd1ed032e74fdd1afcda5 100644 --- a/db/GNUmakefile.in +++ b/db/GNUmakefile.in @@ -1,5 +1,5 @@ # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -47,8 +47,8 @@ LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \ NodeType.pm Interface.pm User.pm Group.pm Project.pm \ Image.pm OSinfo.pm Archive.pm Logfile.pm Lan.pm emdbi.pm \ emdb.pm emutil.pm Firewall.pm VirtExperiment.pm libGeni.pm \ - libEmulab.pm EmulabConstants.pm TraceUse.pm EmulabFeatures.pm \ - Port.pm BlockstoreType.pm + libEmulab.pm EmulabConstants.pm TraceUse.pm \ + EmulabFeatures.pm Port.pm BlockstoreType.pm Blockstore.pm # Stuff installed on plastic. USERSBINS = genelists.proxy dumperrorlog.proxy backup diff --git a/db/Node.pm.in b/db/Node.pm.in index e678c0d2ca27504e93aa043fa358ab202a4a0ee3..b68b28a8dfb1fae43d73ae04c062a5b77291c756 100755 --- a/db/Node.pm.in +++ b/db/Node.pm.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -wT # -# Copyright (c) 2005-2012 University of Utah and the Flux Group. +# Copyright (c) 2005-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -46,6 +46,7 @@ $JAILIPMASK = "@JAILIPMASK@"; use libdb; use libtestbed; use emutil; +use Blockstore; use event; use English; use Socket; @@ -1748,6 +1749,29 @@ sub ReleaseSharedBandwidth($) return 0; } +# +# Relase the reserved blockstore, which requires updating the +# remaining_capacity on the underyling store. At the present +# time, one blockstore is mapped to one pcvm. +# +sub ReleaseBlockStore($) +{ + my ($self) = @_; + + # Must be a real reference. + return -1 + if (! ref($self)); + + my $node_id = $self->node_id(); + my $blockstore = Blockstore::Reservation->LookupByNodeid($node_id); + return 0 + if (!defined($blockstore)); + return -1 + if (!ref($blockstore)); + + return $blockstore->Release(); +} + # # Look up all interfaces for a node, return list of objects. # @@ -2129,8 +2153,7 @@ sub CreateVnodes($$$) my $sharing_mode = $node->sharing_mode(); if (! ($impotent || $isfednode)) { - my $reservation = $node->Reservation(); - if (!defined($reservation)) { + if (!$node->IsReserved()) { print STDERR "*** CreateVnodes: no reservation for $node!\n"; DBQueryFatal("unlock tables"); return -1; @@ -2142,7 +2165,11 @@ sub CreateVnodes($$$) # is on and the pnode is in sharedmode. Locking in nfree and in # the pool daemon prevents the race. # - if (!$experiment->SameExperiment($reservation)) { + # Cause of locking above, we need to make the comparison directly + # using the slot data in the node. + # + if (! ($experiment->pid() eq $node->pid() && + $experiment->eid() eq $node->eid())) { if (! ($sharedokay && $sharing_mode)) { print STDERR "*** CreateVnodes: $node is not shared!\n"; DBQueryFatal("unlock tables"); diff --git a/db/nfree.in b/db/nfree.in index f3cf06e84dab5df279eeb9b420dfc2293f9d5b14..0a88e34c2eb54efaf470663d64229d0028578e8c 100755 --- a/db/nfree.in +++ b/db/nfree.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -450,7 +450,7 @@ foreach my $node (@freed_nodes) { } # - # If the node is virtual, release the shared bandwidth it had + # If the node is virtual, release the shared resources it had # reserved on the physical node. # if ($isvirt) { @@ -459,6 +459,8 @@ foreach my $node (@freed_nodes) { DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'") or $error++; } + $node->ReleaseBlockStore() == 0 + or $error++; } # diff --git a/tbsetup/libossetup.pm.in b/tbsetup/libossetup.pm.in index 0b79514d367033a9192d0b4184b0e7353af64a29..721e4cfb0f3bb2893bf3836924b70c06526fac9e 100644 --- a/tbsetup/libossetup.pm.in +++ b/tbsetup/libossetup.pm.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -2009,4 +2009,41 @@ sub WaitDone($@) return libossetup_virtnode::WaitDone($self, @nodelist); } +##################################################################### +# +# Place holder for blockstore +# +package libossetup_blockstore; +use base qw(libossetup_subnode); + +# +# A constructor for an object to handle all nodes of this type. +# +sub New($$) { + my ($class, $parent) = @_; + + my $self = $class->SUPER::New("blockstore", $parent); + bless($self, $class); + return $self; +} + +sub AddNode($$) +{ + my ($self, $node) = @_; + + print "Will skip blockstore $node ISUP wait.\n"; + + return $self->SUPER::AddNode($node); +} + +# +# We do not currently do anything with blockstores; no waiting for ISUP. +# +sub Volunteers($) +{ + my ($self) = @_; + + return (); +} + 1; diff --git a/tbsetup/libvtop_test.pm.in b/tbsetup/libvtop_test.pm.in index c9375bc7f904b2d018698195119a9c4f633e5ecd..22ed6cf47c9335386ad76ef612115e5e35df2bf9 100755 --- a/tbsetup/libvtop_test.pm.in +++ b/tbsetup/libvtop_test.pm.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2005-2012 University of Utah and the Flux Group. +# Copyright (c) 2005-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -37,11 +37,13 @@ use libdb; use libtblog_simple; use libtestbed; use Experiment; +use EmulabConstants; use VirtExperiment; use Node; use NodeType; use Lan; use OSinfo; +use Blockstore; use Image; use Port; use English; @@ -107,6 +109,8 @@ sub Create($$$$) $self->{'DELAYNODES'} = {}; $self->{'LANNODES'} = {}; $self->{'BRIDGES'} = {}; + $self->{'BLOCKSTORES'}= {}; + $self->{'SANHOSTS'} = {}; $self->{'VLANS'} = {}; $self->{'VPATHS'} = {}; $self->{'MEMBEROF'} = {}; @@ -154,6 +158,8 @@ sub flags($) { return $_[0]->{'FLAGS'}; } sub vnodes($) { return $_[0]->{'VNODES'}; } sub delaynodes($) { return $_[0]->{'DELAYNODES'}; } sub bridges($) { return $_[0]->{'BRIDGES'}; } +sub blockstores($) { return $_[0]->{'BLOCKSTORES'}; } +sub sanhosts($) { return $_[0]->{'SANHOSTS'}; } sub lannodes($) { return $_[0]->{'LANNODES'}; } sub vlans($) { return $_[0]->{'VLANS'}; } sub memberof($) { return $_[0]->{'MEMBEROF'}; } @@ -199,10 +205,14 @@ sub virt_trafgens($) { return $_[0]->virt_table("virt_trafgens"); } sub virt_lan_settings($){ return $_[0]->virt_table("virt_lan_settings"); } sub virt_lan_member_settings($) { return $_[0]->virt_table("virt_lan_member_settings"); } +sub virt_blockstores($) { return $_[0]->virt_table("virt_blockstores"); } +sub virt_blockstore_attributes($) { + return $_[0]->virt_table("virt_blockstore_attributes"); } # Given a vname, is it a node in the topo (or something else like a delay). sub isatoponode($$) { return exists($_[0]->vnodes()->{$_[1]}); } sub isadelaynode($$) { return exists($_[0]->delaynodes()->{$_[1]}); } +sub isasanhost($$) { return exists($_[0]->sanhosts()->{$_[1]}); } # Debug output. sub verbose($) { return $_[0]->flags() & $VTOP_FLAGS_VERBOSE; } @@ -239,6 +249,7 @@ sub virtnodecount($) { return $_[0]->counter("virtcount"); } sub simnodecount($) { return $_[0]->counter("simcount"); } sub remotenodecount($) { return $_[0]->counter("remotecount"); } sub sharednodecount($) { return $_[0]->counter("sharedcount"); } +sub bstorecount($) { return $_[0]->counter("bstorecount"); } sub createLink($$$$$$$$$) { @@ -1136,6 +1147,9 @@ sub LoadVirtNodes($) $vnode->_sharedokay(0); $vnode->_fixedvm(undef); $vnode->_isbridge(($vnode->role() eq "bridge" ? 1 : 0)); + $vnode->_blockstore(undef); + $vnode->_blockstore_attributes(undef); + $vnode->_sanhostname(undef); # # Explicit type request desire to match feature. @@ -1450,7 +1464,47 @@ sub LoadVirtNodes($) $self->printdb("Fixing VM $vname to $fixnode, $osinfo\n"); } - + + # + # Load up the blockstore stuff since we have to spit out the + # attributes as desires. + # + foreach my $virt_bs ($self->virt_blockstores()->Rows()) { + # + # The blockstore is fixed to a node in the topo. + # + my $fixnode = $self->vnodes()->{$virt_bs->fixed()}; + if (!defined($fixnode)) { + tberror("Blockstore $virt_bs is fixed to non-existent node\n"); + return -1; + } + + # + # We need a place to hang the attributes, but they are split + # between the blockstore and the blockstore_attributes. Another + # wrinkle is that the blockstore is assigned to a node in the + # topo, and it is that node that we have to hang the desires off. + # + $self->blockstores()->{$virt_bs->vname()} = $fixnode; + $fixnode->_blockstore($virt_bs); + $fixnode->_sanhostname("sanhost-" . $virt_bs->vname()); + $fixnode->_blockstore_attributes({}); + # XXX Are these always in shared mode? + $fixnode->_sharedokay(1); + + $self->{'COUNTERS'}->{'bstorecount'}++; + } + foreach my $virt_bs_attr ($self->virt_blockstore_attributes()->Rows()) { + my $vname = $virt_bs_attr->vname(); + my $attrkey = $virt_bs_attr->attrkey(); + my $attrval = $virt_bs_attr->attrvalue(); + + next + if (! exists($self->blockstores()->{$vname})); + + my $fixnode = $self->blockstores()->{$vname}; + $fixnode->_blockstore_attributes()->{$attrkey} = $attrval; + } return 0; } @@ -1949,6 +2003,36 @@ sub GenVirtNodes($) if ($vnode->_issubnode()) { $others->{'subnode'} = $vnode->_parent(); } + if (defined($vnode->_blockstore())) { + my $virt_bs = $vnode->_blockstore(); + my $bs_vname = $virt_bs->vname(); + + # This is an additive desire. + $desires->{"bs-capacity"} = ['+', $virt_bs->size()]; + + # The rest come from the attributes. + foreach my $attrkey (keys(%{ $vnode->_blockstore_attributes() })) { + my $attrval = $vnode->_blockstore_attributes()->{$attrkey}; + + $desires->{"bs-${attrkey}-${attrval}"} = ['', 1.0]; + } + my $sanhost = $vnode->_sanhostname(); + + # + # Create a parent node for the sanhost. + # + $self->createNode($sanhost, $mycmurn, + "pc", '1', {"pcstorage" => ['', 1.0]}, undef); + $self->sanhosts()->{$sanhost} = $sanhost; + + # + # We must do this so that assign maps the blockhost and + # sanhost as a unit. Otherwise, it might allocate a subnode + # but pick some other physical node to be the parent. Note that + # ptop file also has subnode decls in it. + # + $others->{'subnode'} = $sanhost; + } foreach my $desirename (keys(%{ $vnode->_desires() })) { my $weight = $vnode->_desires()->{$desirename}; $desires->{$desirename} = ['', $weight]; @@ -1967,7 +2051,7 @@ sub GenVirtNodes($) # the OS it is going to run. However, if the OS is one with a # 'path' (like an OSKit kernel), we don't have an entry in # osidtoimageid for it, and thus we leave it off of the desire list - if (defined($vnode->_osinfo())) { + if (defined($vnode->_osinfo()) && !defined($vnode->_blockstore())) { my $osinfo = $vnode->_osinfo(); # # Support subOSes with a multi-OS desire. Since the pnodes @@ -2127,7 +2211,8 @@ sub GenFixNodes($) } # Normal nodes have a vnode but delay nodes do not. - if (!defined($vnode) && !$self->isadelaynode($vname)) { + if (!defined($vnode) && !$self->isadelaynode($vname) && + !$self->isasanhost($vname)) { tbinfo("GenFixNodes: No vnode for $vname\n"); } @@ -2147,7 +2232,8 @@ sub GenFixNodes($) if (defined($vnode) && $vnode->_isvirtnode() && defined($vnode->_fixedvm())); - if ($self->isatoponode($vname) || $self->isadelaynode($vname)) { + if ($self->isatoponode($vname) || $self->isadelaynode($vname) || + $self->isasanhost($vname)) { $self->createFixedNode($vname, $fixed); } } @@ -3137,9 +3223,15 @@ sub GenVirtLans($) if (defined($virtnode0->_fixedvm())) { $vname0 = $virtnode0->_fixedvm()->vname(); } + elsif (defined($virtnode0->_sanhostname())) { + $vname0 = $virtnode0->_sanhostname(); + } if (defined($virtnode1->_fixedvm())) { $vname1 = $virtnode1->_fixedvm()->vname(); } + elsif (defined($virtnode1->_sanhostname())) { + $vname1 = $virtnode1->_sanhostname(); + } $self->createLink($vname, $plink, [$virtnode0->_cmurn(), $virtnode1->_cmurn() ], @@ -3629,6 +3721,7 @@ sub CreateVtop($) $self->{'COUNTERS'}->{'simcount'} = 0; $self->{'COUNTERS'}->{'remotecount'} = 0; $self->{'COUNTERS'}->{'sharedcount'} = 0; + $self->{'COUNTERS'}->{'bstorecount'} = 0; $self->{'COUNTERS'}->{'physcount'} = 0; # @@ -3970,6 +4063,8 @@ sub solution_vethpatch($) { return $_[0]->{'SOLUTION'}->{'VETHPATCHES'}; } sub solution_portmap($) { return $_[0]->{'SOLUTION'}->{'PORTMAP'}; } sub solution_vifacemap($) { return $_[0]->{'SOLUTION'}->{'VIFACEMAP'}; } sub solution_ifacemap($) { return $_[0]->{'SOLUTION'}->{'IFACEMAP'}; } +sub solution_blockstores($){ return $_[0]->{'SOLUTION'}->{'BLOCKSTORES'}; } +sub solution_sanhosts($) { return $_[0]->{'SOLUTION'}->{'SANHOSTS'}; } sub MapResources($) { @@ -4014,6 +4109,8 @@ sub ClearSolution($) $self->{'SOLUTION'}->{'PORTMAP'} = {}; $self->{'SOLUTION'}->{'VIFACEMAP'} = {}; $self->{'SOLUTION'}->{'IFACEMAP'} = {}; + $self->{'SOLUTION'}->{'BLOCKSTORES'} = {}; + $self->{'SOLUTION'}->{'SANHOSTS'} = {}; return 0; } @@ -4049,6 +4146,25 @@ sub AddNodeToSolution($$$) (undef,undef,$physical) = GeniHRN::Parse($physical); } + # + # Watch for blockstore nodes; the physical nodes in the + # ptop file are fictitious. For example, dbox1:raid1 is + # the blockstore raid1 on physical host dbox1. The virtnode + # is then mapped to dbox1, and the blockstore is the raid1 + # entry in the physical blockstores table. + # + if ($virtnode && defined($virtnode->_blockstore())) { + my ($bphys,$bstore) = split(":", $physical); + my $virt_bs = $virtnode->_blockstore(); + my $bs_name = $virt_bs->vname(); + + # Record blockstore assignment for upload later. + $self->solution_blockstores()->{$bs_name} = $bstore; + + # Actual mapping. + $physical = $bphys; + } + # All we do in this stage is store the results. $self->solution()->{'V2P'}->{$virtual} = $physical; $self->solution()->{'P2V'}->{$physical} = [] @@ -4452,6 +4568,16 @@ sub InterpNodes($) # This might not exist, as for internal nodes. my $virtnode = $self->vnodes()->{$virtual}; + $self->printdb("$physical, $virtual" . + (defined($pnode) ? ", $pnode" : "") . + (defined($virtnode) ? ", $virtnode" : "") . "\n"); + + # + # Skip sanhosts; only there cause of the subnode relationship. + # + next + if (!defined($virtnode) && exists($self->sanhosts()->{$virtual})); + # # A node already allocated to this experiment, and still wanted. # We also have to watch for shared nodes; we will not have the @@ -4538,18 +4664,25 @@ sub InterpNodes($) if (!defined($virtnode)) { # - # Internally created node. At the moment, these should be - # delay nodes. + # Internally created node. # - if (! exists($self->delaynodes()->{$virtual})) { + if (! ($self->isadelaynode($virtual) || + $self->isasanhost($virtual))) { tbwarn("Unknown node $virtual on $physical\n"); return -1; } - if (!exists($self->solution_delaynodes()->{$physical})) { - $self->solution_delaynodes()->{$physical} = []; + if ($self->isadelaynode($virtual)) { + if (!exists($self->solution_delaynodes()->{$physical})) { + $self->solution_delaynodes()->{$physical} = []; + } + push(@{$self->solution_delaynodes()->{$physical}}, $virtual); + } + elsif ($self->isasanhost($virtual)) { + if (!exists($self->solution_sanhosts()->{$physical})) { + $self->solution_sanhosts()->{$physical} = []; + } + push(@{$self->solution_sanhosts()->{$physical}}, $virtual); } - push(@{$self->solution_delaynodes()->{$physical}}, $virtual); - } elsif ($virtnode->_isvirtnode()) { # @@ -4785,9 +4918,14 @@ sub AllocNodes($) $pnode->FlushReserved(); my $reservation = $pnode->Reservation(); - if (!defined($reservation) || - (! $reservation->SameExperiment($self->experiment()) && - ! $pnode->erole() eq "sharedhost")) { + + # We got the node we wanted. + next + if (defined($reservation) && + $reservation->SameExperiment($self->experiment())); + + if (! ($pnode->erole() eq "sharedhost" || + $pnode->erole() eq "storagehost")) { tbinfo("$pnode is not in shared mode.\n"); $rerun++; } @@ -5491,7 +5629,8 @@ sub InterpLinks($) or return -1; $self->UpLoadInterfaceSettings() == 0 or return -1; - + $self->UploadBlockstores() == 0 + or return -1; return 0; } @@ -6416,6 +6555,8 @@ sub InitializePhysNodes($) $pnode->FlushReserved(); my $reservation = $pnode->Reservation(); + print STDERR "$pnodename\n"; + # # We should never try to initialize a node that is not # allocated to the experiment. @@ -6429,6 +6570,11 @@ sub InitializePhysNodes($) $self->printdb("InitPnode: Skipping shared host $pnodename\n"); next; } + elsif (defined($pnode->erole()) && + $pnode->erole() eq TBDB_RSRVROLE_STORAGEHOST()) { + tbinfo("InitPnode: Skipping $pnodename; storage host\n"); + next; + } elsif (! ($self->impotent() && exists($self->solution()->{'TORESERVE'}->{$pnodename}))){ tbinfo("InitPnode: Skipping $pnodename; reserved elsewhere\n"); @@ -6455,6 +6601,11 @@ sub InitializePhysNodes($) # Need only one of the virtnodes to complete the initialization $vnodename = $vnodelist[0]; } + elsif (exists($self->solution_sanhosts()->{$pnodename})) { + $role = TBDB_RSRVROLE_STORAGEHOST; + # Need only one of the virtnodes to complete the initialization + $vnodename = $vnodelist[0]; + } elsif (exists($self->solution_virtnodes()->{$pnodename})) { $role = TBDB_RSRVROLE_VIRTHOST; # @@ -6544,11 +6695,17 @@ sub InitializePhysNode($$$) my $routertype; if ($role eq TBDB_RSRVROLE_DELAYNODE() || + $role eq TBDB_RSRVROLE_STORAGEHOST() || ($role eq TBDB_RSRVROLE_VIRTHOST() && $virtnode->_isvirtnode())) { # # One of our internally created nodes. # - if ($role eq TBDB_RSRVROLE_DELAYNODE()) { + if ($role eq TBDB_RSRVROLE_STORAGEHOST()) { + # + # A storage host. + # + } + elsif ($role eq TBDB_RSRVROLE_DELAYNODE()) { # # A delay node. # @@ -8506,6 +8663,47 @@ sub UpLoadInterfaceSettings($) return 0; } +sub UploadBlockstores($) +{ + my ($self) = @_; + my $experiment = $self->experiment(); + my $exptidx = $experiment->idx(); + my $pid = $experiment->pid(); + my $eid = $experiment->eid(); + + # + # vname = "blockstore-d1" + # bs_name = "d1" + # bstore = "raid1" + # vvnode = "dboxvm1-1" + # pnode = "dbox1" + # + foreach my $bs_name (keys(%{ $self->blockstores() })) { + my $virtnode = $self->blockstores()->{$bs_name}; + my $bstore = $self->solution_blockstores()->{$bs_name}; + my $vname = $virtnode->vname(); + my $vvnode = $self->solution_v2v()->{$vname}; + my $vpnode = $virtnode->_pnode(); + my $pnode = $vpnode->phys_nodeid(); + my $size = $virtnode->_blockstore()->size(); + + $self->printdb("BlockStore: ". + "$vname, $bstore, $size, $vvnode, $vpnode, $pnode\n"); + + my $blockstore = Blockstore->Lookup($pnode, $bstore); + if (!defined($blockstore)) { + tberror("Could not lookup blockstore $pnode:$bstore\n"); + return -1; + } + if (!$self->impotent() && + $blockstore->Reserve($experiment, $vvnode, $bs_name, $size)) { + tberror("Could not lookup reserve $pnode:$bstore $size\n"); + return -1; + } + } + return 0; +} + # getnodeport(s) # Takes a ports result from assign (mac0,mac1) and converts null strings to undef. sub getnodeport($) diff --git a/tbsetup/mapper.in b/tbsetup/mapper.in index a87000d3e12e83a08d5942f09d16ee17882deec2..a1f9961923cf7921af4e2b4de0eb411a8251e315 100644 --- a/tbsetup/mapper.in +++ b/tbsetup/mapper.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -611,6 +611,8 @@ sub RunAssign($$) if ($vtop->simnodecount()); $ptopargs .= "-h " if ($vtop->sharednodecount()); + $ptopargs .= "-b " + if ($vtop->bstorecount()); $ptopargs .= "-a " if ($precheck || $allnodesfree); $ptopargs .= "-c " . $experiment->delay_capacity() . " " diff --git a/tbsetup/ns2ir/blockstore.tcl b/tbsetup/ns2ir/blockstore.tcl index 4a2fa9742ec1bbcf60125748955ef06ec085b739..4ff9f5273d482d5466700ce80be85b9c4323d349 100644 --- a/tbsetup/ns2ir/blockstore.tcl +++ b/tbsetup/ns2ir/blockstore.tcl @@ -1,7 +1,7 @@ # -*- tcl -*- # -# Copyright (c) 2012 University of Utah and the Flux Group. +# Copyright (c) 2012-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -125,11 +125,10 @@ Blockstore instproc get_node {} { } # Allocate parent host and bind to it. - set hname "sanhost-${self}" + set hname "blockhost-${self}" uplevel "#0" "set $hname [$sim node]" - $hname set subnodehost 1 - $hname set subnodechild $self set node $hname + $node set_hwtype "blockstore" 0 1 0 # Return parent node object. return $hname diff --git a/tbsetup/ns2ir/lanlink.tcl b/tbsetup/ns2ir/lanlink.tcl index 1209b7672a4b0eafa7bca32b37c2751d0d8a64d9..5ab7eb43683a185e01feb9b3579faf6aa0d5bd46 100644 --- a/tbsetup/ns2ir/lanlink.tcl +++ b/tbsetup/ns2ir/lanlink.tcl @@ -1,6 +1,6 @@ # -*- tcl -*- # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -960,6 +960,7 @@ Link instproc updatedb {DB} { $self instvar bridge_links $self instvar settings $self instvar member_settings + $self instvar sanlan set vindex 0 $sim spitxml_data "virt_lan_lans" [list "vname" "failureaction"] [list $self $failureaction] @@ -974,6 +975,13 @@ Link instproc updatedb {DB} { $sim spitxml_data "virt_lan_settings" $fields $values } + # + # If this is a SAN, then nullify shaping + # + if {$sanlan == 1} { + set nobwshaping 1 + } + foreach nodeport $nodelist { set node [lindex $nodeport 0] if {$node == $src_node} { diff --git a/tbsetup/ns2ir/sim.tcl.in b/tbsetup/ns2ir/sim.tcl.in index 911b0d4d6f42e58f0892066e4eff029d6d005f9f..b5d1bbf35bfc2ae110c4da2f23987d15871757e1 100644 --- a/tbsetup/ns2ir/sim.tcl.in +++ b/tbsetup/ns2ir/sim.tcl.in @@ -1,6 +1,6 @@ # -*- tcl -*- # -# Copyright (c) 2000-2012 University of Utah and the Flux Group. +# Copyright (c) 2000-2013 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -225,11 +225,11 @@ Simulator instproc duplex-link {n1 n2 bw delay type args} { punsup "Arguments for duplex-link: $args" } set error 0 - if {! [$n1 info class Node]} { + if {! [$n1 info class Node] && ! [$n1 info class Blockstore] } { perror "\[duplex-link] $n1 is not a node." set error 1 } - if {! [$n2 info class Node]} { + if {! [$n2 info class Node] && ! [$n2 info class Blockstore] } { perror "\[duplex-link] $n2 is not a node." set error 1 } @@ -572,8 +572,6 @@ Simulator instproc run {} { if {[$child info class Node]} { set childtype [$child set type] $node add-desire "hosts-$childtype" 1.0 - } elseif {[$child info class Blockstore]} { - $node add-desire "sanhost" 1.0 } } } @@ -666,6 +664,11 @@ Simulator instproc run {} { $self spitxml_data "nseconfigs" [list "vname" "nseconfig" ] [list fullsim $nsecfg_script ] } + + # Make sure the blockstore parent nodes are created. + foreach blockstore [array names blockstore_list] { + set blockstore_node [$blockstore get_node] + } # Update the DB foreach node [lsort [array names node_list]] {