Commit 12a41b7e authored by Mike Hibler's avatar Mike Hibler

Support for gathering and storing Infiniband interface GUIDs in the DB.

Since GUIDs are 16 bytes and our current interface MACs are only 12 bytes,
I agonized over whether to grow the mac column to 16 bytes and just treat
it as a unique identifier (which is all we use that column for anyway).
However, in the end I just added a new guid column as there were mac columns
in a variety of other tables and it wasn't clear what the relationship was
and what I might break.

So, the newnode MFS will now report back a GUID for interfaces it recognizes
as IB (FreeBSD-specific right now). The boss-side checkin code with stash
that value in new_interfaces (and later interfaces when added). For possible
backward compat, it will also generate a MAC address from that (possibly
Mellanox-specific) so that all entries in the interfaces table will have
a MAC (yes, it should really be the other way around--all interfaces should
always have a guid).

End of story. We don't do anything else with IB right now other than stash
an interface GUID.
parent 3add3502
......@@ -59,6 +59,7 @@ my $FORMAT = "/sbin/newfs_msdos";
my $MOUNTPOINT = "/mnt";
my $MOUNT = "/sbin/mount";
my $UNMOUNT = "/sbin/umount";
my $IBSTAT = "/usr/bin/ibstat";
#
# Error codes, and how many times we beep to indicate each one.
......@@ -90,6 +91,11 @@ my $BEEP_SLEEP = 3; # In seconds
my $FLOPPY_REMOVED = 0;
my $FLOPPY_INSERTED = 1;
my $FBSD_VERS = 8;
if (`uname -r` =~ /^(\d+)\.\d+/) {
$FBSD_VERS = $1;
}
#
# What type of client this node is. Possible values:
# testnode - The default, a regular testbed node
......@@ -106,7 +112,7 @@ my $headless = 0;
my $writefloppy = 0;
my $testing = 0;
my $bossnode = "";
my $checkib = 0;
my $uniquifier;
foreach my $arg (@ARGV) {
......@@ -116,6 +122,7 @@ foreach my $arg (@ARGV) {
/h/ && do { $headless = 1; last; };
/w/ && do { $writefloppy = 1; last; };
/t/ && do { $testing = 1; last; };
/I/ && do { $checkib = 1; last; };
/o/ && do { $client_type = "ops"; last; };
};
}
......@@ -138,6 +145,16 @@ sub usage {
1;
}
#
# XXX for now, auto detect whether we should check for IB.
# Otherwise you have to modify the FS of the MFS to add the option..messy!
#
if (system("sysctl -q sys.class.infiniband_verbs.abi_version") == 0) {
$checkib = 1;
} else {
$checkib = 0;
}
#
# Find out what our boss node was, so we don't have to hardcode it here - allow
# the user to provide one on the command line, though, so that we can run tests
......@@ -152,6 +169,33 @@ if (!$bossnode) {
}
}
#
# XXX we can only handle infiniband user FreeBSD 9 or above and
# only if we have the ibstat tool installed.
#
if ($checkib) {
if ($FBSD_VERS < 9 || ! -x "$IBSTAT") {
message("WARNING: not running IB-enabled newnode MFS, disabled IB collection.\n");
$checkib = 0;
}
#
# XXX if IB checking is enabled and we have a Mellanox VPI-enabled
# card, put the card in autodetect.
#
elsif (!$testing) {
my @out = `sysctl sys.device | grep mlx4_core`;
foreach my $line (@out) {
if ($line =~ /^(sys.device.mlx4_core0.mlx4_port)(\d+):\s+(\S+)/) {
if (($3 eq "ib" || $3 eq "eth") && system("sysctl $1$2=auto")) {
message("WARNING: Could not put mlx4 port $2 in auto\n");
} else {
sleep(1);
}
}
}
}
}
#
# Get physical information about this node
#
......@@ -355,14 +399,16 @@ sub freebsd_find_interfaces {
#
my @ifconfig_lines = `$IFCONFIG`;
my @ifaces;
my @ibifaces;
my $iface = "";
my $mac;
my $isib = 0;
my $status;
foreach my $line (@ifconfig_lines) {
chomp $line;
SWITCH: for ($line) {
#
# A line beginning a new interface
# A line beginning a new interface.
#
(/^(\w+):/) && do {
if ($iface) {
......@@ -370,13 +416,25 @@ sub freebsd_find_interfaces {
# We have an old interface to save away, but only if it's
# Ethernet
#
if ($mac) {
if ($mac && !$isib) {
push @ifaces, [$iface, $mac, $status];
}
}
$iface = $1;
$mac = $status = "";
#
# XXX we ignore IPoIB (ibN) here if we are checking for IB.
# We will use ibstat below to collect the info.
#
if ($checkib && $iface =~ /^ib\d+/) {
push @ibifaces, $iface;
$isib = 1;
} else {
$isib = 0;
}
last SWITCH;
};
......@@ -407,10 +465,30 @@ sub freebsd_find_interfaces {
#
# Get the last one
#
if ($iface && $mac) {
if ($iface && $mac && !$isib) {
push @ifaces, [$iface, $mac, $status];
}
#
# For infiniband, interfaces will show up as IPoIB (ibN) devices.
# We attempt to canonicalize the info here, by returning "ib?" and
# using ibstat to get the port GUID and returning that for the MAC.
#
if (@ibifaces > 0) {
my $ibix = 0;
my @ibout = `$IBSTAT -p`;
chomp @ibout;
foreach my $line (@ibout) {
if ($line =~ /^0x([0-9a-fA-F]{16})/) {
my $mac = $1;
my $iface = "ib$ibix";
$ibix++;
push @ifaces, [$iface, $mac, "down"];
next;
}
}
}
#
# Warn about any that don't have carrier
#
......@@ -600,7 +678,7 @@ sub linux_get_cpuspeed {
#
# Teach the switch where we are
# Teach the switch where we are.
#
sub teachswitch {
system "$TEACHSWITCH &" and message("Unable to start teachswitch: $!\n");
......
......@@ -2016,6 +2016,7 @@ CREATE TABLE `interfaces` (
`card` tinyint(3) unsigned NOT NULL default '0',
`port` tinyint(3) unsigned NOT NULL default '0',
`mac` varchar(12) NOT NULL default '000000000000',
`guid` varchar(16) default NULL,
`IP` varchar(15) default NULL,
`IPaliases` text,
`mask` varchar(15) default NULL,
......@@ -2437,6 +2438,7 @@ CREATE TABLE `new_interfaces` (
`card` int(11) NOT NULL default '0',
`port` tinyint(3) unsigned default NULL,
`mac` varchar(12) NOT NULL default '',
`guid` varchar(16) default NULL,
`interface_type` varchar(15) default NULL,
`switch_id` varchar(32) default NULL,
`switch_card` tinyint(3) default NULL,
......
#
# Add guid field to interfaces/new_interfaces specifically for InfiniBand.
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if (!DBSlotExists("interfaces", "guid")) {
DBQueryFatal("ALTER TABLE interfaces ADD ".
"`guid` varchar(16) default NULL ".
" AFTER mac");
}
if (!DBSlotExists("new_interfaces", "guid")) {
DBQueryFatal("ALTER TABLE new_interfaces ADD ".
"`guid` varchar(16) default NULL ".
" AFTER mac");
}
return 0;
}
# Local Variables:
# mode:perl
# End:
#!/usr/bin/perl -w
#
# Copyright (c) 2003-2013 University of Utah and the Flux Group.
# Copyright (c) 2003-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -158,7 +158,7 @@ my $IP = shift();
# Convert mac to DB representation.
if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) {
$mac = "$1$2$3$4$5$6";
$mac = lc("$1$2$3$4$5$6");
}
# Verify optional switch info
......
#!/usr/bin/perl -w
#
# Copyright (c) 2003-2013 University of Utah and the Flux Group.
# Copyright (c) 2003-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -237,7 +237,7 @@ NODE: foreach my $node_id (@node_ids) {
#
# Grab the node's MACs from the new_interfaces table
#
$query_result = DBQueryFatal("SELECT card, MAC, interface_type, " .
$query_result = DBQueryFatal("SELECT card, mac, guid, interface_type, " .
"switch_id, switch_card, switch_port, cable, len, role, port, IP " .
"FROM new_interfaces WHERE new_node_id='$new_node_id'");
if (!$query_result->num_rows()) {
......@@ -246,7 +246,7 @@ NODE: foreach my $node_id (@node_ids) {
}
my %interfaces;
while (my ($card, $MAC, $iface_type, $switch_id, $switch_card,
while (my ($card, $MAC, $guid, $iface_type, $switch_id, $switch_card,
$switch_port, $cable, $len, $ifrole, $port, $iface_IP) = $query_result->fetchrow()) {
#
# Get some more information about this interface type
......@@ -278,7 +278,7 @@ NODE: foreach my $node_id (@node_ids) {
#
# Stash it away...
#
$interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex,
$interfaces{$card} = [$MAC, $guid, $iface_type, $max_speed, $full_duplex,
$switch_id, $switch_card, $switch_port, $cable,
$len, $ifrole, $port, $iface_IP];
......@@ -451,8 +451,9 @@ NODE: foreach my $node_id (@node_ids) {
}
while (my ($card, $aref) = each %interfaces) {
my ($MAC, $iface_type, $max_speed, $duplex, $switch_id, $switch_card,
$switch_port, $cable, $len, $ifrole, $port, $iface_IP) = @$aref;
my ($MAC, $guid, $iface_type, $max_speed, $duplex,
$switch_id, $switch_card, $switch_port, $cable, $len,
$ifrole, $port, $iface_IP) = @$aref;
my $iface = "eth$card";
my $wire_type;
my $iface_role;
......@@ -485,15 +486,17 @@ NODE: foreach my $node_id (@node_ids) {
if (! defined($iface_IP)) {
$iface_IP = "";
}
my $guidstr = defined($guid) ? "guid='$guid'," : "";
if (!$useScriptFeature || !defined($port))
{
$port = 1;
}
DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " .
"card=$card, port='$port', mac='$MAC', IP='$iface_IP', " .
"interface_type='$iface_type', iface='$iface', uuid=UUID(), " .
"current_speed='$max_speed', duplex=$duplex, role='$iface_role'");
DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id'," .
"card=$card,port='$port',mac='$MAC',$guidstr".
"IP='$iface_IP',interface_type='$iface_type',".
"iface='$iface',uuid=UUID(),current_speed='$max_speed',".
"duplex=$duplex,role='$iface_role'");
DBQueryFatal("INSERT INTO interface_state SET node_id='$node_id', " .
"card=$card, port='$port', iface='$iface'");
......
......@@ -176,11 +176,15 @@ function guess_IP ($prefix, $number) {
# We want to be able to handle both numeric and character 'number's
# Figure out which we have
#
$ndigits = 0;
if (! is_numeric($number)) {
$using_char = 1;
$number = ord($number);
} else {
$using_char = 0;
if (preg_match("/^(0\d+)$/",$number)) {
$ndigits = strlen($number);
}
}
#
......@@ -193,6 +197,9 @@ function guess_IP ($prefix, $number) {
while ($i > 0) {
if ($using_char) {
$node = $prefix . chr($i);
} elseif ($ndigits) {
$fmt = "%0" . $ndigits . "d";
$node = $prefix . sprintf($fmt, $i);
} else {
$node = $prefix . $i;
}
......
......@@ -110,7 +110,7 @@ if (count($interfaces)) {
$query_result = DBQueryFatal("select n.node_id from " .
"nodes as n left join interfaces as i " .
"on n.node_id=i.node_id " .
"where i.mac='$testmac'");
"where i.mac='$testmac' or i.guid='$testmac'");
if (mysql_num_rows($query_result)) {
$row = mysql_fetch_array($query_result);
$node_id = $row["node_id"];
......@@ -128,8 +128,7 @@ if (count($interfaces)) {
$query_result = DBQueryFatal("select n.new_node_id, n.node_id from " .
"new_nodes as n left join new_interfaces as i " .
"on n.new_node_id=i.new_node_id " .
"where i.mac='$testmac'");
"where i.mac='$testmac' or i.guid='$testmac'");
if (mysql_num_rows($query_result)) {
$row = mysql_fetch_array($query_result);
$id = $row["new_node_id"];
......@@ -222,12 +221,23 @@ $new_node_id = $row[0];
echo "Node ID is $new_node_id\n";
foreach ($interfaces as $interface) {
$card = $interface["card"];
$mac = $interface["mac"];
$type = $interface["type"];
DBQueryFatal("insert into new_interfaces set " .
"new_node_id=$new_node_id, card=$card, mac='$mac', " .
"interface_type='$type'");
$card = $interface["card"];
$mac = $interface["mac"];
$type = $interface["type"];
$clause = "";
# XXX not a 6 byte value, assume it is a guid
# XXX probably should check interface_capabilities for the type
if (strlen($mac) != 12) {
$clause = ", guid='$mac'";
# XXX 16 bytes implies an Infiniband port GUID to us
# cons up what would be the mac
if (strlen($mac) == 16) {
$mac = substr($mac, 0, 6) . substr($mac, 10, 6);
}
}
DBQueryFatal("insert into new_interfaces set " .
"new_node_id=$new_node_id, card=$card, mac='$mac', " .
"interface_type='$type'$clause");
}
#
......@@ -292,6 +302,7 @@ function find_free_id($prefix) {
# First, check to see if there's a recent entry in new_nodes we can name
# this node after
#
$ndigits = 0;
$query_result = DBQueryFatal("select node_id from new_nodes " .
"order by created desc limit 1");
if (mysql_num_rows($query_result)) {
......@@ -300,7 +311,19 @@ function find_free_id($prefix) {
#
# Try to figure out if this is in some format we can increment
#
if (preg_match("/^(.*[^\d])(\d+)$/",$old_node_id,$matches)) {
if (preg_match("/^(.*[^\d])(0\d+)$/",$old_node_id,$matches)) {
$base = $matches[1];
$number = $matches[2];
$ndigits = strlen($number);
echo "Matches $ndigits-digit pcXXX format";
$fmt = "%0" . $ndigits . "d";
$number = sprintf($fmt, $number + 1);
$potential_name = $base . $number;
if (!check_node_exists($potential_name)) {
return array($base, $number);
}
$prefix = $base;
} elseif (preg_match("/^(.*[^\d])(\d+)$/",$old_node_id,$matches)) {
echo "Matches pcXXX format";
# pcXXX format
$base = $matches[1];
......@@ -309,6 +332,7 @@ function find_free_id($prefix) {
if (!check_node_exists($potential_name)) {
return array($base,($number +1));
}
$prefix = $base;
} elseif (preg_match("/^(.*)-([a-zA-Z])$/",$old_node_id,$matches)) {
# Something like WAIL's (type-rack-A) format
$base = $matches[1];
......@@ -318,6 +342,7 @@ function find_free_id($prefix) {
if (!check_node_exists($potential_name)) {
return array($base . '-', $newchar);
}
$prefix = $base;
}
}
......@@ -327,10 +352,16 @@ function find_free_id($prefix) {
# hasn't been used yet - put in a silly little guard to prevent an
# infinite loop in case of bugs.
#
if ($ndigits) {
$fmt = "%0" . $ndigits . "d";
}
$node_number = 0;
while ($node_number < 10000) {
$node_number++;
$potential_name = $prefix . $node_number;
$number = ++$node_number;
if ($ndigits) {
$number = sprintf($fmt, $number);
}
$potential_name = $prefix . $number;
if (!check_node_exists($potential_name)) {
break;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment