Commit 997b21b5 authored by Leigh Stoller's avatar Leigh Stoller

This commit adds some simple support for using the Infiniband on the

Probe Cluster. The problem is that the IFB is a shared network that
every node attaches to, which can looks like an ethernet device that
can ifconfig'ed. In other words, one big lan.

But we still want the user to be able to create a lan so that they can
interact with it in thei NS file like any other network.

The NS syntax is:

	set lan2 [$ns make-lan "node1 node2 node3" * 0ms]
	tb-set-switch-fabric $lan2 "infiniband"

The switch fabric tells the backend to do IP assignment for the
specific global network. Yes, I tried to be a little but general
purpose. Lets see how this actually turns out.

This first commit treats the fabric as a single big lan on the same
subnet.

NOTE 1: Since the unroutable IP space is kinda small, but the Probe
Cluster is really big, we can easily run out of bits if we tried to do
assignment on virtual topos. Instead, fabrics get their IP allocation
at swapin time, and the allocations are deleted when the experiment is
swapped out. The rationale is that the number of swapped in
experiments is much much smaller then the number of possible topos
that can be loaded into the DB. Still might run out, but less likely.

The primary impact of above is that IP assignments can change from
one swap to another, but this is easy to deal with if the user is
scripting their experiment; the IP allocation is available via the
XMLRPC interface.

NOTE 2: The current code allocates from a single big network, which
makes it easy for users to mess each other up if they start doing
things by hand. Ultimately, we want each lan in each experinent to use
their own subnet, but that is going to take more work, so lets do it
in the second phase.

The definition of "network fabrics" is in the new network_fabrics
tables. As an example for probe:

	INSERT INTO `network_fabrics` set
		idx=NULL,
		name='ifband',
		created=now(),
		ipalloc=1, ipalloc_onenet=1,
		ipalloc_subnet='192.168.0.0',ipalloc_netmask='255.255.0.0'
parent 92137d2b
......@@ -16,6 +16,7 @@ use vars qw(@ISA @EXPORT $AUTOLOAD);
use libdb;
use EmulabConstants;
use libtestbed;
use Socket;
use Node;
use emutil;
use Logfile;
......@@ -3117,6 +3118,7 @@ sub BackupVirtualState($)
my $pid = $self->pid();
my $eid = $self->eid();
my $idx = $self->idx();
my $vstateDir = $self->WorkDir() . "/vstate";
my $errors = 0;
......@@ -3130,7 +3132,7 @@ sub BackupVirtualState($)
foreach my $table (@virtualTables) {
DBQueryWarn("SELECT * FROM $table ".
"WHERE pid='$pid' AND eid='$eid' ".
"WHERE exptidx='$idx' ".
"INTO OUTFILE '$vstateDir/$table' ")
or $errors++;
}
......@@ -3145,12 +3147,13 @@ sub RemoveVirtualState($)
return -1
if (! ref($self));
my $idx = $self->idx();
my $pid = $self->pid();
my $eid = $self->eid();
my $errors = 0;
foreach my $table (@virtualTables) {
DBQueryWarn("DELETE FROM $table WHERE pid='$pid' AND eid='$eid'")
DBQueryWarn("DELETE FROM $table WHERE exptidx='$idx'")
or $errors++;
}
return $errors;
......@@ -5367,5 +5370,259 @@ sub SharingVlans($)
return $query_result->numrows;
}
#
# Do IP allocation for switch network fabrics that require it.
#
sub SetupNetworkFabrics($)
{
my ($self) = @_;
my $idx = $self->idx();
#
# Find any lans with the fabric setting. Order them for consistent
# assignment.
#
my $query_result =
DBQueryWarn("select vname,capval from virt_lan_settings ".
"where exptidx='$idx' and ".
" (capkey='network_fabric' or ".
" capkey='switch_fabric') ".
"order by vname");
return -1
if (!defined($query_result));
if (!$query_result->numrows) {
#
# Be sure to delete stale assignments.
#
DBQueryWarn("delete from global_ipalloc where exptidx='$idx'")
or return -1;
return 0;
}
#
# Get the current assignment so we can delete stale ones.
#
my %old_ips = ();
my %new_ips = ();
my $ip_result =
DBQueryWarn("select ip,lanidx,member from global_ipalloc ".
"where exptidx='$idx'");
return -1
if (!$ip_result);
while (my ($ip,$lanidx,$member) = $ip_result->fetchrow_array()) {
$old_ips{"$lanidx:$member"} = $ip;
}
# Now get this after we are sure we need it.
my $virtexp = $self->GetVirtExperiment();
while (my ($lanname,$fabric) = $query_result->fetchrow_array()) {
my $safe_fabric = DBQuoteSpecial($fabric);
my $fabric_result =
DBQueryWarn("select * from network_fabrics ".
"where name=$safe_fabric");
return -1
if (!$fabric_result);
if (! $fabric_result->numrows) {
print STDERR "*** No such fabric $fabric for $lanname\n";
return -1;
}
my $rowref = $fabric_result->fetchrow_hashref();
# See if user is responsible.
next
if (! $rowref->{'ipalloc'});
my $fabidx = $rowref->{'idx'};
my $onenet = $rowref->{'ipalloc_onenet'};
my $subnet = $rowref->{'ipalloc_subnet'};
my $netmask = $rowref->{'ipalloc_netmask'};
my $submask = $rowref->{'ipalloc_submask'};
if (!$onenet) {
# Add this later.
print STDERR "*** Fabric $fabric is not onenet ipalloc!\n";
return -1;
}
DBQueryWarn("lock tables global_ipalloc write, ".
" global_ipalloc as u1 write, ".
" global_ipalloc as u2 write")
or return -1;
#
# This is so simplistic I could just call it moronic.
#
my $max = unpack("N", ~inet_aton($netmask)) - 1;
#
# Members for this lan.
#
my @members = ();
foreach my $member ($virtexp->Table("virt_lans")->Rows()) {
next
if ($member->vname() ne $lanname);
push(@members, $member);
}
#
# Order the rows so that we get consistent allocation.
#
@members = sort {$a->vindex() <=> $b->vindex()} @members;
# Need these below.
my $virtlan = $virtexp->Table("virt_lan_lans")->Find($lanname);
my $lanidx = $virtlan->idx();
# Process members.
foreach my $member (@members) {
my $ip = $member->ip();
my $midx = $member->vindex();
#
# If the virtual topology specifies an ip in the subnet then
# try to use that one. This is not likely to happen, but
# look for it anyway.
#
if (defined($ip) &&
inet_ntoa(inet_aton($netmask) & inet_aton($ip)) eq $subnet) {
my $ip_result =
DBQueryWarn("select lanidx,exptidx from global_ipalloc ".
"where ip='$ip' and fabric_idx='$fabidx'");
if (!$ip_result) {
DBQueryWarn("unlock tables");
return -1;
}
if (! $ip_result->numrows) {
# Safe to insert it;
goto insertip;
}
my ($lidx,$eidx) = $ip_result->fetchrow_array();
# Checks if this IP is used someplace else in the
# experiment. Do what the user wants since he owns it
# already.
goto reuseip
if ($lidx == $lanidx && $eidx == $idx);
# Some other experiment already has it.
print STDERR "*** IP $ip for $member already in use. ";
print STDERR "Allocating a new one for you.\n";
}
#
# Try to use existing IP, as for swapmod. Note that the
# virtual topo has been reloaded and previous assignment
# lost. So we have to go the global_ipalloc table to figure
# this out.
#
if (exists($old_ips{"$lanidx:$midx"})) {
$ip = $old_ips{"$lanidx:$midx"};
if (!exists($new_ips{$ip})) {
goto reuseip;
}
}
#
# Try to find an unused ip.
#
my $ip_result =
DBQueryWarn("select max(ipint) from global_ipalloc ".
"where fabric_idx='$fabidx'");
if (!$ip_result) {
DBQueryWarn("unlock tables");
return -1;
}
my ($curmax) = $ip_result->fetchrow_array();
if (!defined($curmax)) {
$ip = inet_ntoa(inet_aton($subnet) | pack("N", 1));
}
elsif ($curmax < $max - 1) {
$ip = inet_ntoa(inet_aton($subnet) | pack("N", $curmax + 1));
}
else {
#
# It is a pain to use mysql to find a free slot
# in a big range of numbers. This is about the worst
# way I could think of to do it.
#
$ip_result =
DBQueryWarn("select u1.ipint,u2.ipint ".
" from global_ipalloc as u1 ".
"left outer join global_ipalloc as u2 on ".
" u1.ipint-1=u2.ipint and ".
" u2.fabric_idx='$fabidx' ".
"where u2.ipint is NULL and ".
" u1.ipint<$max and u1.ipint>1 and ".
" u1.fabric_idx='$fabidx'");
if (!$ip_result) {
DBQueryWarn("unlock tables");
return -1;
}
if (!$ip_result->numrows) {
print STDERR "No free ip addresses for $member!";
DBQueryWarn("unlock tables");
return -1
}
my ($tmp) = $ip_result->fetchrow_array();
$ip = inet_ntoa(inet_aton($subnet) | pack("N", $tmp - 1));
}
insertip:
# Need to do the row insertion and then move on.
my $ipint = unpack("N", (~inet_aton($netmask)) & inet_aton($ip));
if (!DBQueryWarn("insert into global_ipalloc set ".
" fabric_idx='$fabidx', ipint=$ipint, ".
" member='$midx', ".
" exptidx=$idx, lanidx='$lanidx', ip='$ip'")) {
DBQueryWarn("unlock tables");
return -1;
}
reuseip:
$member->ip($ip);
$new_ips{$ip} = $ip;
}
DBQueryWarn("unlock tables");
}
#
# Need to delete stale ones.
#
foreach my $ip (values(%old_ips)) {
next
if (exists($new_ips{$ip}));
DBQueryWarn("delete from global_ipalloc ".
"where ip='$ip' and exptidx='$idx'");
}
$virtexp->Store();
return 0;
}
#
# This has to be done at swapout, but not during a swapmod since
# that would mess up the existing assignments.
#
sub ClearGlobalIPAllocation($)
{
my ($self) = @_;
my $idx = $self->idx();
DBQueryWarn("delete from global_ipalloc where exptidx=$idx")
or return -1;
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -1474,6 +1474,23 @@ CREATE TABLE `fs_resources` (
KEY `fileidx` (`fileidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `global_ipalloc`
--
DROP TABLE IF EXISTS `global_ipalloc`;
CREATE TABLE `global_ipalloc` (
`exptidx` int(11) NOT NULL default '0',
`lanidx` int(11) NOT NULL default '0',
`member` int(11) NOT NULL default '0',
`fabric_idx` int(11) NOT NULL default '0',
`ipint` int(11) unsigned NOT NULL default '0',
`ip` varchar(15) default NULL,
PRIMARY KEY (`exptidx`,`lanidx`,`ipint`),
UNIQUE KEY `fabip` (`fabric_idx`,`ipint`),
KEY `ipint` (`ipint`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `global_policies`
--
......@@ -2167,6 +2184,23 @@ CREATE TABLE `motelogfiles` (
PRIMARY KEY (`logfileid`,`pid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `network_fabrics`
--
DROP TABLE IF EXISTS `network_fabrics`;
CREATE TABLE `network_fabrics` (
`idx` int(11) NOT NULL auto_increment,
`name` varchar(64) NOT NULL default '',
`created` datetime default NULL,
`ipalloc` tinyint(1) NOT NULL default '0',
`ipalloc_onenet` tinyint(1) NOT NULL default '0',
`ipalloc_subnet` varchar(15) NOT NULL default '',
`ipalloc_netmask` varchar(15) NOT NULL default '',
`ipalloc_submask` varchar(15) default NULL,
PRIMARY KEY (`idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `new_interfaces`
--
......
......@@ -36,6 +36,9 @@ INSERT IGNORE INTO `interface_types` VALUES ('generic',100000,1,'Generic','Gener
INSERT IGNORE INTO `interface_types` VALUES ('generic_1G',1000000,1,'Generic GB','Generic GB',1,'RJ45');
INSERT IGNORE INTO `interface_capabilities` VALUES ('generic_1G','protocols','ethernet');
INSERT IGNORE INTO `interface_capabilities` VALUES ('generic_1G','ethernet_defspeed','1000000');
INSERT IGNORE INTO `interface_types` VALUES ('generic_10G',10000000,1,'Generic GB','Generic 10GB',1,'RJ45');
INSERT IGNORE INTO `interface_capabilities` VALUES ('generic_10G','protocols','ethernet');
INSERT IGNORE INTO `interface_capabilities` VALUES ('generic_10G','ethernet_defspeed','10000000');
-- We use these types for the ilo/drac management interfaces.
INSERT INTO `interface_types` VALUES ('ilo2',0,1,'HP','HP iLO 2',1,'RJ45');
......
#
# Add network_fabrics abd global_ipalloc. This stuff needs a lot more
# work!
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
#
# XXX ipsubnets table?
#
if (!DBTableExists("network_fabrics")) {
DBQueryFatal("CREATE TABLE `network_fabrics` ( ".
" `idx` int(11) NOT NULL auto_increment, ".
" `name` varchar(64) NOT NULL default '', ".
" `created` datetime default NULL, ".
" `ipalloc` tinyint(1) NOT NULL default '0', ".
" `ipalloc_onenet` tinyint(1) NOT NULL default '0', ".
" `ipalloc_subnet` varchar(15) NOT NULL default '', ".
" `ipalloc_netmask` varchar(15) NOT NULL default '', ".
" `ipalloc_submask` varchar(15) default NULL, ".
" PRIMARY KEY (`idx`) ".
") ENGINE=MyISAM DEFAULT CHARSET=latin1");
}
if (!DBTableExists("global_ipalloc")) {
DBQueryFatal("CREATE TABLE `global_ipalloc` ( ".
" `exptidx` int(11) NOT NULL default '0', ".
" `lanidx` int(11) NOT NULL default '0', ".
" `member` int(11) NOT NULL default '0', ".
" `fabric_idx` int(11) NOT NULL default '0', ".
" `ipint` int(11) unsigned NOT NULL default '0', ".
" `ip` varchar(15) default NULL, ".
" PRIMARY KEY (`exptidx`,`lanidx`,`ipint`), ".
" UNIQUE KEY `fabip` (`fabric_idx`,`ipint`), ".
" KEY `ipint` (`ipint`) ".
") ENGINE=MyISAM DEFAULT CHARSET=latin1");
}
DBQueryFatal("replace into `interface_types` values ".
"('generic_10G',10000000,1,'Generic 10GB', ".
" 'Generic 10GB',1,'RJ45')");
DBQueryFatal("replace into `interface_capabilities` values ".
"('generic_10G','protocols','ethernet')");
DBQueryFatal("replace into `interface_capabilities` values ".
"('generic_10G','ethernet_defspeed','10000000');");
return 0;
}
1;
# Local Variables:
# mode:perl
# End:
......@@ -2386,7 +2386,8 @@ sub GenVirtLans($)
}
# Figure out how many nodes support a feature
$nodesdo{"alias"}++
if ($osdoesmlink{$osid});
# All OSs support this.
if (1);
$nodesdo{"veth"}++
if ($osdoesveth{$osid});
$nodesdo{"veth-ne"}++
......@@ -2506,7 +2507,7 @@ sub GenVirtLans($)
#
# No longer support veth or alias encap; catch it now.
#
if (! ($globalencap =~ /^(default|veth-ne|vlan)$/)) {
if (! ($globalencap =~ /^(default|veth-ne|vlan|alias)$/)) {
tberror("unknown global encapsulation style '$globalencap'\n");
$errors++;
next;
......@@ -2709,6 +2710,7 @@ sub GenVirtLans($)
# treatment applies as well.
#
my $lanname = "fakelan/$vname";
$self->createNode($lanname, $mycmurn, $protocol, '1', undef, undef);
# So we ignore it when it comes back from assign.
......@@ -2716,6 +2718,13 @@ sub GenVirtLans($)
# XXX If not ethernet, assume wireless. Need more info someplace.
$self->exptstats()->{'wirelesslans'} += 1;
my $others = {};
if ($emulated) {
$others->{'emulated'} = 1;
# assign seems to require this, but it seems wrong.
$others->{'trivial_ok'} = 1;
}
foreach my $member (@members) {
my $plink = "fakelan/$vname/$member";
......@@ -2726,7 +2735,7 @@ sub GenVirtLans($)
{'virtual_node_id' => $vname,
'virtual_interface_id' => "$member" },
{'virtual_node_id' => $lanname },
$top_bw, $protocol);
$top_bw, $protocol, $others);
next;
}
}
......@@ -6301,8 +6310,7 @@ sub InterpLinksAux($)
}
elsif ($linktag eq "fakelan") {
#
# No trivial links, emulated links, delays, vlans. We do
# however need to come up with an ssid?
# No trivial links, delays, or vlans.
#
$self->printdb("FakeLan - $virtA - $nodeA:$portA\n");
......@@ -6312,8 +6320,20 @@ sub InterpLinksAux($)
if (!defined($protolan));
$protolan->SetType("fakelan");
$protolan->SetRole("link/lan");
$protolan->AddInterface($nodeA, $vnodeA, $vportA, $portA);
if ($member0->usevirtiface()) {
$protolan->AddMember($nodeA, $portA);
my $virtiface = $self->NewVirtIface($virtlan, $member0,
$nodeA, $portA);
return -1
if (!defined($virtiface));
$portA = $virtiface->viface();
}
else {
$protolan->AddInterface($nodeA, $vnodeA, $vportA, $portA);
}
$portmap{$virtA} = $portA;
#
......@@ -6325,7 +6345,7 @@ sub InterpLinksAux($)
my $portvlan = $virtlan->_portvlan();
#
# No trivial links, emulated links, delays, vlans.
# No trivial links, delays, vlans.
# This is a port in an existing lan in another experiment.
#
$self->printdb("PortLan - $virtA - $nodeA:$portA\n");
......
......@@ -927,10 +927,22 @@ Link instproc updatedb {DB} {
$self instvar ofenabled
$self instvar ofcontroller
$self instvar bridge_links
$self instvar settings
$self instvar member_settings
set vindex 0
$sim spitxml_data "virt_lan_lans" [list "vname" "failureaction"] [list $self $failureaction]
#
# Upload lan settings.
#
foreach setting [array names settings] {
set fields [list "vname" "capkey" "capval"]
set values [list $self $setting $settings($setting)]
$sim spitxml_data "virt_lan_settings" $fields $values
}
foreach nodeport $nodelist {
set node [lindex $nodeport 0]
if {$node == $src_node} {
......@@ -1091,6 +1103,20 @@ Link instproc updatedb {DB} {
set vindex [expr $vindex + 1]
$sim spitxml_data "virt_lans" $fields $values
foreach setting_key [array names member_settings] {
set foo [split $setting_key ","]
set thisnode [lindex $foo 0]
set capkey [lindex $foo 1]
if {$thisnode == $node} {
set fields [list "vname" "member" "capkey" "capval"]
set values [list $self $nodeportraw $capkey \
$member_settings($setting_key)]
$sim spitxml_data "virt_lan_member_settings" $fields $values
}
}
}
}
......@@ -1141,7 +1167,7 @@ Lan instproc updatedb {DB} {
$sim spitxml_data "virt_lan_lans" [list "vname" "failureaction"] [list $self $failureaction]
#
# Upload lan settings and them per-member settings
# Upload lan settings.
#
foreach setting [array names settings] {
set fields [list "vname" "capkey" "capval"]
......
......@@ -74,6 +74,7 @@ proc tb-use-ipassign {onoff} {}
proc tb-set-ipassign-args {args} {}
proc tb-set-lan-protocol {lanlink protocol} {}
proc tb-set-link-protocol {lanlink protocol} {}
proc tb-set-switch-fabric {lanlink fabric} {}
proc tb-set-lan-accesspoint {lanlink node} {}
proc tb-set-lan-setting {lanlink capkey capval} {}
proc tb-set-node-lan-setting {lanlink node capkey capval} {}
......
......@@ -1526,6 +1526,9 @@ proc tb-set-vlink-emulation {style} {
"vlan" {
set style "vlan"
}
"alias" {
set style "alias"
}
default {
perror "\[tb-set-encapsulate] one of: 'veth-ne', 'vlan'"
return
......@@ -1598,6 +1601,18 @@ proc tb-set-link-protocol {lanlink protocol} {
$lanlink set protocol $protocol
}
#
# Set the fabric. We change the protocol as well.
#
proc tb-set-switch-fabric {lanlink fabric} {
if {[$lanlink info class] != "Link" && [$lanlink info class] != "Lan"} {
perror "\[tb-set-lan-protocol] $lanlink is not a link or lan."
return
}
$lanlink set protocol $fabric
$lanlink set_setting "switch_fabric" $fabric
}
#
# XXX - We need to set the accesspoint for a wireless lan. I have no
# idea how this will eventually be done, but for now just do it manually.
......
......@@ -611,23 +611,28 @@ while (my ($osid) = $result->fetchrow()) {
if (defined($switchtouse) && ! defined($component_name)) {
# Should probably get the last four args out of the database, but I don't
# think we ever actually use this case...
print_switch($switchtouse,undef,undef,undef,undef);
print_switch($switchtouse,undef,undef,undef,undef,undef);
$switches{$switchtouse} = 1;
}
elsif (! defined($component_name)) {
$result =
DBQueryFatal("select n.node_id,n.uuid,country,latitude,longitude ".
DBQueryFatal("select n.node_id,n.uuid,country,latitude,longitude, ".
" na.attrvalue ".
" from nodes as n ".
"left join widearea_nodeinfo as wn ".
" on n.node_id=wn.node_id " .
"left join node_types as nt on ".
" nt.type=n.type ".
"left join node_type_attributes as na on ".
" na.type=n.type and ".
" na.attrkey='forwarding_protocols' ".
"where role='testswitch' or role='widearea_switch' or ".
" (role='testnodefoo' and nt.isswitch=1)");
while (($switch, $uuid, $country, $latitude, $longitude) =
while (($switch, $uuid, $country, $latitude, $longitude, $protocols) =
$result->fetchrow_array) {
print_switch($switch, $uuid, $country, $latitude, $longitude);
print_switch($switch, $uuid,
$country, $latitude, $longitude, $protocols);
$switches{$switch} = 1;
}
}
......@@ -1884,6 +1889,7 @@ sub print_switch
my $country = shift(@_);
my $latitude = shift(@_);
my $longitude = shift(@_);
my $protocols = shift(@_);
my $interfaces = [];
if ($do_xml && $genimode ne $NO_GENI)
{
......@@ -1896,6 +1902,20 @@ sub print_switch
if (!(defined($MAINSITE) && $MAINSITE && $name eq "procurve1")) {
push(@$types, "*lan:*");
}
#
# If there is are non-standard forwarding_protocol, add them.
#
if ($protocols) {
my @forwarding_protocols = split(",", $protocols);
foreach my $protocol (@forwarding_protocols) {
next
if ($protocol eq "ethernet");
push(@$types, "*${protocol}:*");
}
}
# This is for mixing real nodes with pnodes in a lan. The goal is to
# prevent assign from putting the lan node on a pc; always prefer a
# real switch. See corresponding change in libvtop.
......
......@@ -916,6 +916,7 @@ sub doSwapout($) {
# Special.
$experiment->ClearPortRegistration();
$experiment->ClearPortRange();
$experiment->ClearGlobalIPAllocation();
}
tblog_set_cleanup(0);
......@@ -938,6 +939,12 @@ sub doSwapin($) {
# Just the physnodes ...
my @deleted_pnodes = ();
# Special IP assignment. Must do before mapper runs and moves
# IPs from virt_lans to interfaces table.
if ($experiment->SetupNetworkFabrics()) {
return 1;
}
#
# assign_wrapper does all the virtual to physical mapping
# and updating the DB state.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment