Commit dab52801 authored by Kirk Webb's avatar Kirk Webb

Add complete local node storage support from parser down to tcmd.

Doing this required adding columns to the virt and physical blockstores
tables to mark the attributes that will be considered for mapping.
Unmarked entries just flow through to the client-side.

This commit also introduces filesystem support in the form of passing
through a mount point to the client-side.  It is left to the client to
decide what filesystem and fs options to use to setup the space, including
any logical volume aggregation required to support the request.
parent d981ca72
...@@ -37,6 +37,7 @@ ...@@ -37,6 +37,7 @@
/* Blockstore protocols (a.k.a. bus type) */ /* Blockstore protocols (a.k.a. bus type) */
#define BS_PROTO_ISCSI "iSCSI" #define BS_PROTO_ISCSI "iSCSI"
#define BS_PROTO_SCSI "SCSI" #define BS_PROTO_SCSI "SCSI"
#define BS_PROTO_SAS "SAS"
/* Definitions related to iSCSI */ /* Definitions related to iSCSI */
#define BS_IQN_PREFIX "iqn.2000-10.net.emulab" /* XXX: don't hardcode. */ #define BS_IQN_PREFIX "iqn.2000-10.net.emulab" /* XXX: don't hardcode. */
...@@ -46,4 +47,10 @@ ...@@ -46,4 +47,10 @@
#define BS_PERMS_ISCSI_RW "RW" /* read/write */ #define BS_PERMS_ISCSI_RW "RW" /* read/write */
#define BS_PERMS_ISCSI_DEF BS_PERMS_ISCSI_RW #define BS_PERMS_ISCSI_DEF BS_PERMS_ISCSI_RW
/* Local placement directives */
#define BS_PLACEMENT_ALL "ALL_SPACE"
#define BS_PLACEMENT_SYSVOL "ALL_SYSVOL"
#define BS_PLACEMENT_NONSYS "ALL_NONSYSVOL"
#define BS_PLACEMENT_DEF BS_PLACEMENT_ALL
#endif /* BSDEFS_H */ #endif /* BSDEFS_H */
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2012 University of Utah and the Flux Group. # Copyright (c) 2012,2013 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -170,15 +170,16 @@ sub LoadAttributes($) ...@@ -170,15 +170,16 @@ sub LoadAttributes($)
my $type = $self->type(); my $type = $self->type();
my $query_result = my $query_result =
DBQueryWarn("select attrkey,attrvalue,attrtype ". DBQueryWarn("select attrkey,attrvalue,attrtype,isfeature ".
" from blockstore_type_attributes ". " from blockstore_type_attributes ".
"where type='$type'"); "where type='$type'");
$self->{"ATTRS"} = {}; $self->{"ATTRS"} = {};
while (my ($key,$val,$type) = $query_result->fetchrow_array()) { while (my ($key,$val,$type,$isfeature) = $query_result->fetchrow_array()) {
$self->{"ATTRS"}->{$key} = { "key" => $key, $self->{"ATTRS"}->{$key} = { "key" => $key,
"value" => $val, "value" => $val,
"type" => $type }; "type" => $type ,
"isfeature" => $isfeature };
} }
return 0; return 0;
} }
...@@ -199,9 +200,9 @@ sub Stringify($) ...@@ -199,9 +200,9 @@ sub Stringify($)
# #
# Look for an attribute. # Look for an attribute.
# #
sub GetAttribute($$;$$) sub GetAttribute($$;$$$)
{ {
my ($self, $attrkey, $pattrvalue, $pattrtype) = @_; my ($self, $attrkey, $pattrvalue, $pattrtype, $pattrfeature) = @_;
goto bad goto bad
if (!ref($self)); if (!ref($self));
...@@ -220,11 +221,13 @@ sub GetAttribute($$;$$) ...@@ -220,11 +221,13 @@ sub GetAttribute($$;$$)
# Return value instead if a $pattrvalue not provided. # Return value instead if a $pattrvalue not provided.
return $ref->{'value'} return $ref->{'value'}
if (!defined($pattrvalue)); if (!defined($pattrvalue));
$$pattrvalue = $ref->{'value'}; $$pattrvalue = $ref->{'value'};
$$pattrtype = $ref->{'type'} $$pattrtype = $ref->{'type'}
if (defined($pattrtype)); if (defined($pattrtype));
$$pattrfeature = $ref->{'isfeature'}
if (defined($pattrfeature));
return 0; return 0;
...@@ -252,16 +255,16 @@ sub GetAttributes($) ...@@ -252,16 +255,16 @@ sub GetAttributes($)
} }
# Shortcuts for typical attributes. # Shortcuts for typical attributes.
sub type($) { return $_[0]->{'TYPE'}; } sub type($) {return $_[0]->{'TYPE'}; }
sub class($;$) {return GetAttribute($_[0], "class", $_[1]); } sub class($;$) {return GetAttribute($_[0], "class", $_[1]); }
sub protocol($;$) {return GetAttribute($_[0], "protocol", $_[1]); } sub protocol($;$) {return GetAttribute($_[0], "protocol", $_[1]); }
# #
# Set the value of an attribute # Set the value of an attribute
# #
sub SetAttribute($$$;$) sub SetAttribute($$$;$$)
{ {
my ($self, $attrkey, $attrvalue, $attrtype) = @_; my ($self, $attrkey, $attrvalue, $attrtype, $attrfeature) = @_;
goto bad goto bad
if (!ref($self)); if (!ref($self));
...@@ -271,12 +274,14 @@ sub SetAttribute($$$;$) ...@@ -271,12 +274,14 @@ sub SetAttribute($$$;$)
$attrtype = "string" $attrtype = "string"
if (!defined($attrtype)); if (!defined($attrtype));
$attrfeature = (defined($attrfeature) && $attrfeature) ? 1 : 0;
my $safe_attrvalue = DBQuoteSpecial($attrvalue); my $safe_attrvalue = DBQuoteSpecial($attrvalue);
my $type = $self->type(); my $type = $self->type();
DBQueryWarn("replace into blockstore_type_attributes set ". DBQueryWarn("replace into blockstore_type_attributes set ".
" type='$type', attrkey='$attrkey', ". " type='$type', attrkey='$attrkey', ".
" attrtype='$attrtype', attrvalue=$safe_attrvalue") " attrtype='$attrtype', attrvalue=$safe_attrvalue, ".
" isfeature='$attrfeature'")
or return -1; or return -1;
$self->{"ATTRS"}->{$attrkey} = $attrvalue; $self->{"ATTRS"}->{$attrkey} = $attrvalue;
......
...@@ -197,6 +197,7 @@ CREATE TABLE `blockstore_type_attributes` ( ...@@ -197,6 +197,7 @@ CREATE TABLE `blockstore_type_attributes` (
`attrkey` varchar(32) NOT NULL default '', `attrkey` varchar(32) NOT NULL default '',
`attrvalue` tinytext NOT NULL, `attrvalue` tinytext NOT NULL,
`attrtype` enum('integer','float','boolean','string') default 'string', `attrtype` enum('integer','float','boolean','string') default 'string',
`isfeature` tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (`type`,`attrkey`) PRIMARY KEY (`type`,`attrkey`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1; ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
...@@ -4460,6 +4461,7 @@ CREATE TABLE `virt_blockstore_attributes` ( ...@@ -4460,6 +4461,7 @@ CREATE TABLE `virt_blockstore_attributes` (
`attrkey` varchar(32) NOT NULL default '', `attrkey` varchar(32) NOT NULL default '',
`attrvalue` tinytext NOT NULL, `attrvalue` tinytext NOT NULL,
`attrtype` enum('integer','float','boolean','string') default 'string', `attrtype` enum('integer','float','boolean','string') default 'string',
`isdesire` tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (`exptidx`,`vname`,`attrkey`) PRIMARY KEY (`exptidx`,`vname`,`attrkey`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1; ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
......
...@@ -1225,6 +1225,7 @@ REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','eid','text','redi ...@@ -1225,6 +1225,7 @@ REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','eid','text','redi
REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','vname','text','redirect','virt_blockstores:vname',0,0,NULL); REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','vname','text','redirect','virt_blockstores:vname',0,0,NULL);
REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','attrkey','text','regex','^[-\\w]+$',1,64,NULL); REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','attrkey','text','regex','^[-\\w]+$',1,64,NULL);
REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','attrvalue','text','regex','^[-\\w\\.\\+,\\s\\/:]+$',0,255,NULL); REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','attrvalue','text','regex','^[-\\w\\.\\+,\\s\\/:]+$',0,255,NULL);
REPLACE INTO table_regex VALUES ('virt_blockstore_attributes','isdesire','int','redirect','default:boolean',0,0,NULL);
REPLACE INTO table_regex VALUES ('emulab_sites','certificate','text','regex','^[\\012\\015\\040-\\176]*$',128,4096,NULL); REPLACE INTO table_regex VALUES ('emulab_sites','certificate','text','regex','^[\\012\\015\\040-\\176]*$',128,4096,NULL);
REPLACE INTO table_regex VALUES ('emulab_sites','url','text','redirect','default:tinytext',0,0,NULL); REPLACE INTO table_regex VALUES ('emulab_sites','url','text','redirect','default:tinytext',0,0,NULL);
......
#
# Add feature/desire marker flags to blockstore virt and physical type tables.
#
use strict;
use libdb;
my $impotent = 0;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if (!DBSlotExists("virt_blockstore_attributes", "isdesire")) {
DBQueryFatal("alter table virt_blockstore_attributes add ".
" `isdesire` tinyint(4) unsigned NOT NULL default '0'");
}
if (!DBSlotExists("blockstore_type_attributes", "isfeature")) {
DBQueryFatal("alter table blockstore_type_attributes add ".
" `isfeature` tinyint(4) unsigned NOT NULL default '0'");
}
DBQueryFatal("REPLACE INTO table_regex VALUES ".
" ('virt_blockstore_attributes','isdesire','int',".
" 'redirect','default:boolean',0,0,NULL)");
return 0;
}
# Local Variables:
# mode:perl
# End:
...@@ -1579,9 +1579,14 @@ sub LoadVirtNodes($) ...@@ -1579,9 +1579,14 @@ sub LoadVirtNodes($)
my $vname = $virt_bs_attr->vname(); my $vname = $virt_bs_attr->vname();
my $attrkey = $virt_bs_attr->attrkey(); my $attrkey = $virt_bs_attr->attrkey();
my $attrval = $virt_bs_attr->attrvalue(); my $attrval = $virt_bs_attr->attrvalue();
my $isdesire = $virt_bs_attr->isdesire();
# Skip any blockstores that don't have a corresponding entry in the
# 'blockstores' libvtop hash. Also skip any attributes that are not
# marked as a desire.
next next
if (! exists($self->blockstores()->{$vname})); if (! exists($self->blockstores()->{$vname}) ||
! $isdesire);
my $fixnode = $self->blockstores()->{$vname}; my $fixnode = $self->blockstores()->{$vname};
$fixnode->_blockstore_attributes()->{$attrkey} = $attrval; $fixnode->_blockstore_attributes()->{$attrkey} = $attrval;
......
...@@ -97,6 +97,35 @@ Blockstore instproc set-type {newtype} { ...@@ -97,6 +97,35 @@ Blockstore instproc set-type {newtype} {
return return
} }
Blockstore instproc set-placement {newplace} {
var_import ::TBCOMPAT::soplacements
$self instvar attributes
if {![info exists soplacements($newplace)]} {
perror("Invalid placement specified: $newplace")
return
}
$self set attributes(placement) $soplacements($newplace)
return
}
Blockstore instproc set-mount-point {newmount} {
$self instvar attributes
# Keep the mount point path rules simple but strict:
# * Must start with a forward slash (absolute path)
# * Directory names must only consist of characters in: [a-zA-Z0-9_]
# * Two forward slashes in a row not allowed
# * Optionally end with a forward slash
if {![regexp {^(/\w+){1,}/?$} $newmount]} {
perror "Bad mountpoint: $newmount"
}
$self set attributes(mountpoint) $newmount
return
}
Blockstore instproc set-size {newsize} { Blockstore instproc set-size {newsize} {
$self instvar node $self instvar node
...@@ -126,6 +155,13 @@ Blockstore instproc set-size {newsize} { ...@@ -126,6 +155,13 @@ Blockstore instproc set-size {newsize} {
return return
} }
#
# Alias for procedure below
#
Blockstore instproc set-node {pnode} {
return [$self set_fixed $pnode]
}
# #
# Explicitly fix a blockstore to a node. # Explicitly fix a blockstore to a node.
# #
...@@ -179,6 +215,7 @@ Blockstore instproc get_node {} { ...@@ -179,6 +215,7 @@ Blockstore instproc get_node {} {
Blockstore instproc updatedb {DB} { Blockstore instproc updatedb {DB} {
var_import ::GLOBALS::pid var_import ::GLOBALS::pid
var_import ::GLOBALS::eid var_import ::GLOBALS::eid
var_import ::TBCOMPAT::sodesires
$self instvar sim $self instvar sim
$self instvar node $self instvar node
$self instvar type $self instvar type
...@@ -199,7 +236,14 @@ Blockstore instproc updatedb {DB} { ...@@ -199,7 +236,14 @@ Blockstore instproc updatedb {DB} {
# Emit attributes. # Emit attributes.
foreach key [lsort [array names attributes]] { foreach key [lsort [array names attributes]] {
set val $attributes($key) set val $attributes($key)
$sim spitxml_data "virt_blockstore_attributes" [list "vname" "attrkey" "attrvalue"] [list $self $key $val] set vba_fields [list "vname" "attrkey" "attrvalue" "isdesire"]
set vba_values [list $self $key $val]
set isdesire [expr [info exists sodesires($key)] ? 1 : 0]
lappend vba_values $isdesire
$sim spitxml_data "virt_blockstore_attributes" $vba_fields $vba_values
} }
} }
...@@ -564,8 +564,6 @@ Simulator instproc run {} { ...@@ -564,8 +564,6 @@ Simulator instproc run {} {
# Go through the list of nodes, and find subnode hosts: # Go through the list of nodes, and find subnode hosts:
# - If the subnode is of class Node, we have to add a # - If the subnode is of class Node, we have to add a
# desire to have the hosts-<type-of-child> feature. # desire to have the hosts-<type-of-child> feature.
# - If it is of type Blockstore, we add a "sanhost"
# desire.
foreach node [lsort [array names node_list]] { foreach node [lsort [array names node_list]] {
if { [$node set subnodehost] == 1 } { if { [$node set subnodehost] == 1 } {
set child [$node set subnodechild] set child [$node set subnodechild]
......
...@@ -98,6 +98,17 @@ namespace eval TBCOMPAT { ...@@ -98,6 +98,17 @@ namespace eval TBCOMPAT {
variable sotypes variable sotypes
variable soclasses variable soclasses
variable soprotocols variable soprotocols
variable sodesires
array set sodesires {
"class" 1
"protocol" 1
}
variable soplacements
array set soplacements {
"all_remaining" "ALL_SPACE"
"sysvol_remaining" "ALL_SYSVOL"
"nonsysvol_remaining" "ALL_NONSYSVOL"
}
# NSE hack: sim type is not in DB. Just adding it now # NSE hack: sim type is not in DB. Just adding it now
set hwtypes(sim) 1 set hwtypes(sim) 1
......
...@@ -1368,21 +1368,18 @@ if ($useblockstore) { ...@@ -1368,21 +1368,18 @@ if ($useblockstore) {
"from blockstores as b ". "from blockstores as b ".
"left join blockstore_state as s ". "left join blockstore_state as s ".
"on b.bsidx=s.bsidx ". "on b.bsidx=s.bsidx ".
"where s.remaining_capacity is not null"); "where s.ready=1");
while (my ($nodeId, $bsId, $typeName, while (my ($nodeId, $bsId, $typeName,
$capacity) = $result->fetchrow_array) { $capacity) = $result->fetchrow_array) {
my $type = BlockstoreType->Lookup($typeName); my $type = BlockstoreType->Lookup($typeName);
if (defined($type)) { if (defined($type)) {
my @features = (); my @features = ();
my $class; foreach my $attr (values %{$type->GetAttributes()}) {
$type->GetAttribute("class", \$class); if (exists($attr->{'isfeature'}) && $attr->{'isfeature'}) {
if (defined($class)) { my $attrkey = $attr->{'key'};
push(@features, "bs-class-$class:0.5"); my $attrval = $attr->{'value'};
} push (@features, "bs-${attrkey}-${attrval}:0.5");
my $protocol; }
$type->GetAttribute("protocol", \$protocol);
if (defined($protocol)) {
push(@features, "bs-protocol-$protocol:0.5");
} }
my $divisible; my $divisible;
$type->GetAttribute("divisible", \$divisible); $type->GetAttribute("divisible", \$divisible);
......
...@@ -4066,6 +4066,7 @@ COMMAND_PROTOTYPE(dostorageconfig) ...@@ -4066,6 +4066,7 @@ COMMAND_PROTOTYPE(dostorageconfig)
* *
* - List of local storage elements to verify. * - List of local storage elements to verify.
* - List of remote storage elements to link up to. * - List of remote storage elements to link up to.
* - List of static slices to setup
* - ... XXX: Other stuff later (e.g., aggregates). * - ... XXX: Other stuff later (e.g., aggregates).
*/ */
...@@ -4147,6 +4148,37 @@ COMMAND_PROTOTYPE(dostorageconfig) ...@@ -4147,6 +4148,37 @@ COMMAND_PROTOTYPE(dostorageconfig)
mysql_free_result(res2); mysql_free_result(res2);
} }
mysql_free_result(res); mysql_free_result(res);
/*
* Send across local blockstore volumes (slices). These don't
* show up in the reserved table, existing entirely in the
* virt_blockstore* tables since local disk space is dedicated
* to the current experiment.
*/
res = mydb_query("select vname,size "
"from virt_blockstores "
"where exptidx=%d and "
"fixed='%s'",
2, reqp->exptidx, reqp->nickname);
if (!res) {
error("STORAGECONFIG: %s: DB Error getting virt_blockstore "
"info.\n",
mynodeid);
return 1;
}
nrows = (int) mysql_num_rows(res);
while (nrows--) {
row = mysql_fetch_row(res);
vname = row[0];
volsize = atoi(row[1]);
OUTPUT(buf, sizeof(buf),
"CMD=SLICE IDX=%d VOLNAME=%s VOLSIZE=%d",
cmdidx++, vname, volsize);
sendstoreconf(sock, tcp, reqp, buf, vname);
}
mysql_free_result(res);
/* /*
* Now to send the remote elements (a.k.a SAN disks). Figuring * Now to send the remote elements (a.k.a SAN disks). Figuring
...@@ -4230,9 +4262,10 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname) ...@@ -4230,9 +4262,10 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname)
MYSQL_RES *res; MYSQL_RES *res;
MYSQL_ROW row; MYSQL_ROW row;
char buf[MYBUFSIZE]; char buf[MYBUFSIZE];
char *bufp, *ebufp = &buf[sizeof(buf)];
char iqn[BS_IQN_MAXSIZE]; char iqn[BS_IQN_MAXSIZE];
char *mynodeid; char *mynodeid;
char *class, *protocol, *perms; char *class, *protocol, *perms, *placement, *mountpoint;
int nrows, nattrs; int nrows, nattrs;
/* Remember the nodeid we care about up front. */ /* Remember the nodeid we care about up front. */
...@@ -4253,7 +4286,7 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname) ...@@ -4253,7 +4286,7 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname)
/* Find out what type of blockstore we are dealing with and /* Find out what type of blockstore we are dealing with and
grab some additional attributes. */ grab some additional attributes. */
nrows = nattrs = (int) mysql_num_rows(res); nrows = nattrs = (int) mysql_num_rows(res);
class = protocol = perms = "\0"; class = protocol = perms = placement = mountpoint = "\0";
while (nrows--) { while (nrows--) {
char *key, *val; char *key, *val;
row = mysql_fetch_row(res); row = mysql_fetch_row(res);
...@@ -4265,6 +4298,10 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname) ...@@ -4265,6 +4298,10 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname)
protocol = val; protocol = val;
} else if (strcmp(key,"permissions") == 0) { } else if (strcmp(key,"permissions") == 0) {
perms = val; perms = val;
} else if (strcmp(key,"placement") == 0) {
placement = val;
} else if (strcmp(key,"mountpoint") == 0) {
mountpoint = val;
} }
} }
...@@ -4291,6 +4328,29 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname) ...@@ -4291,6 +4328,29 @@ sendstoreconf(int sock, int tcp, tmcdreq_t *reqp, char *bscmd, char *vname)
client_writeback(sock, buf, strlen(buf), tcp); client_writeback(sock, buf, strlen(buf), tcp);
} }
/* local disk. */
else if ((strcmp(class, BS_CLASS_LOCAL) == 0) &&
((strcmp(protocol, BS_PROTO_SAS) == 0) ||
(strcmp(protocol, BS_PROTO_SCSI) == 0))) {
/* Set default placement if not defined. */
placement = strlen(placement) ? placement : BS_PLACEMENT_DEF;
bufp = buf;
bufp += OUTPUT(bufp, ebufp-bufp,
"%s CLASS=%s PROTO=%s BSID=%s",
bscmd, class, protocol, placement);
/* Add the mountpoint to the buffer, if requested.*/
if (strlen(mountpoint)) {
bufp += OUTPUT(bufp, ebufp-bufp, " MOUNTPOINT=%s",
mountpoint);
}
bufp += OUTPUT(bufp, ebufp-bufp, "\n");
client_writeback(sock, buf, strlen(buf), tcp);
}
mysql_free_result(res); mysql_free_result(res);
return 0; return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment