From 12a41b7ecc1aa0aa1c407e35210247f357b6b1f2 Mon Sep 17 00:00:00 2001 From: Mike Hibler Date: Mon, 2 Jun 2014 15:19:00 -0600 Subject: [PATCH] Support for gathering and storing Infiniband interface GUIDs in the DB. Since GUIDs are 16 bytes and our current interface MACs are only 12 bytes, I agonized over whether to grow the mac column to 16 bytes and just treat it as a unique identifier (which is all we use that column for anyway). However, in the end I just added a new guid column as there were mac columns in a variety of other tables and it wasn't clear what the relationship was and what I might break. So, the newnode MFS will now report back a GUID for interfaces it recognizes as IB (FreeBSD-specific right now). The boss-side checkin code with stash that value in new_interfaces (and later interfaces when added). For possible backward compat, it will also generate a MAC address from that (possibly Mellanox-specific) so that all entries in the interfaces table will have a MAC (yes, it should really be the other way around--all interfaces should always have a guid). End of story. We don't do anything else with IB right now other than stash an interface GUID. --- install/newclient | 88 ++++++++++++++++++++++++++++++++++++--- sql/database-create.sql | 2 + sql/updates/4/395 | 26 ++++++++++++ utils/management_iface.in | 4 +- utils/newnode.in | 23 +++++----- www/newnode-defs.php3 | 7 ++++ www/newnodecheckin.php | 55 ++++++++++++++++++------ 7 files changed, 176 insertions(+), 29 deletions(-) create mode 100644 sql/updates/4/395 diff --git a/install/newclient b/install/newclient index 64bcec18a..d36a97018 100644 --- a/install/newclient +++ b/install/newclient @@ -59,6 +59,7 @@ my $FORMAT = "/sbin/newfs_msdos"; my $MOUNTPOINT = "/mnt"; my $MOUNT = "/sbin/mount"; my $UNMOUNT = "/sbin/umount"; +my $IBSTAT = "/usr/bin/ibstat"; # # Error codes, and how many times we beep to indicate each one. @@ -90,6 +91,11 @@ my $BEEP_SLEEP = 3; # In seconds my $FLOPPY_REMOVED = 0; my $FLOPPY_INSERTED = 1; +my $FBSD_VERS = 8; +if (`uname -r` =~ /^(\d+)\.\d+/) { + $FBSD_VERS = $1; +} + # # What type of client this node is. Possible values: # testnode - The default, a regular testbed node @@ -106,7 +112,7 @@ my $headless = 0; my $writefloppy = 0; my $testing = 0; my $bossnode = ""; - +my $checkib = 0; my $uniquifier; foreach my $arg (@ARGV) { @@ -116,6 +122,7 @@ foreach my $arg (@ARGV) { /h/ && do { $headless = 1; last; }; /w/ && do { $writefloppy = 1; last; }; /t/ && do { $testing = 1; last; }; + /I/ && do { $checkib = 1; last; }; /o/ && do { $client_type = "ops"; last; }; }; } @@ -138,6 +145,16 @@ sub usage { 1; } +# +# XXX for now, auto detect whether we should check for IB. +# Otherwise you have to modify the FS of the MFS to add the option..messy! +# +if (system("sysctl -q sys.class.infiniband_verbs.abi_version") == 0) { + $checkib = 1; +} else { + $checkib = 0; +} + # # Find out what our boss node was, so we don't have to hardcode it here - allow # the user to provide one on the command line, though, so that we can run tests @@ -152,6 +169,33 @@ if (!$bossnode) { } } +# +# XXX we can only handle infiniband user FreeBSD 9 or above and +# only if we have the ibstat tool installed. +# +if ($checkib) { + if ($FBSD_VERS < 9 || ! -x "$IBSTAT") { + message("WARNING: not running IB-enabled newnode MFS, disabled IB collection.\n"); + $checkib = 0; + } + # + # XXX if IB checking is enabled and we have a Mellanox VPI-enabled + # card, put the card in autodetect. + # + elsif (!$testing) { + my @out = `sysctl sys.device | grep mlx4_core`; + foreach my $line (@out) { + if ($line =~ /^(sys.device.mlx4_core0.mlx4_port)(\d+):\s+(\S+)/) { + if (($3 eq "ib" || $3 eq "eth") && system("sysctl $1$2=auto")) { + message("WARNING: Could not put mlx4 port $2 in auto\n"); + } else { + sleep(1); + } + } + } + } +} + # # Get physical information about this node # @@ -355,14 +399,16 @@ sub freebsd_find_interfaces { # my @ifconfig_lines = `$IFCONFIG`; my @ifaces; + my @ibifaces; my $iface = ""; my $mac; + my $isib = 0; my $status; foreach my $line (@ifconfig_lines) { chomp $line; SWITCH: for ($line) { # - # A line beginning a new interface + # A line beginning a new interface. # (/^(\w+):/) && do { if ($iface) { @@ -370,13 +416,25 @@ sub freebsd_find_interfaces { # We have an old interface to save away, but only if it's # Ethernet # - if ($mac) { + if ($mac && !$isib) { push @ifaces, [$iface, $mac, $status]; } } $iface = $1; $mac = $status = ""; + + # + # XXX we ignore IPoIB (ibN) here if we are checking for IB. + # We will use ibstat below to collect the info. + # + if ($checkib && $iface =~ /^ib\d+/) { + push @ibifaces, $iface; + $isib = 1; + } else { + $isib = 0; + } + last SWITCH; }; @@ -407,10 +465,30 @@ sub freebsd_find_interfaces { # # Get the last one # - if ($iface && $mac) { + if ($iface && $mac && !$isib) { push @ifaces, [$iface, $mac, $status]; } + # + # For infiniband, interfaces will show up as IPoIB (ibN) devices. + # We attempt to canonicalize the info here, by returning "ib?" and + # using ibstat to get the port GUID and returning that for the MAC. + # + if (@ibifaces > 0) { + my $ibix = 0; + my @ibout = `$IBSTAT -p`; + chomp @ibout; + foreach my $line (@ibout) { + if ($line =~ /^0x([0-9a-fA-F]{16})/) { + my $mac = $1; + my $iface = "ib$ibix"; + $ibix++; + push @ifaces, [$iface, $mac, "down"]; + next; + } + } + } + # # Warn about any that don't have carrier # @@ -600,7 +678,7 @@ sub linux_get_cpuspeed { # -# Teach the switch where we are +# Teach the switch where we are. # sub teachswitch { system "$TEACHSWITCH &" and message("Unable to start teachswitch: $!\n"); diff --git a/sql/database-create.sql b/sql/database-create.sql index d4914d845..89198136e 100644 --- a/sql/database-create.sql +++ b/sql/database-create.sql @@ -2016,6 +2016,7 @@ CREATE TABLE `interfaces` ( `card` tinyint(3) unsigned NOT NULL default '0', `port` tinyint(3) unsigned NOT NULL default '0', `mac` varchar(12) NOT NULL default '000000000000', + `guid` varchar(16) default NULL, `IP` varchar(15) default NULL, `IPaliases` text, `mask` varchar(15) default NULL, @@ -2437,6 +2438,7 @@ CREATE TABLE `new_interfaces` ( `card` int(11) NOT NULL default '0', `port` tinyint(3) unsigned default NULL, `mac` varchar(12) NOT NULL default '', + `guid` varchar(16) default NULL, `interface_type` varchar(15) default NULL, `switch_id` varchar(32) default NULL, `switch_card` tinyint(3) default NULL, diff --git a/sql/updates/4/395 b/sql/updates/4/395 new file mode 100644 index 000000000..5d76da217 --- /dev/null +++ b/sql/updates/4/395 @@ -0,0 +1,26 @@ +# +# Add guid field to interfaces/new_interfaces specifically for InfiniBand. +# +use strict; +use libdb; + +sub DoUpdate($$$) +{ + my ($dbhandle, $dbname, $version) = @_; + + if (!DBSlotExists("interfaces", "guid")) { + DBQueryFatal("ALTER TABLE interfaces ADD ". + "`guid` varchar(16) default NULL ". + " AFTER mac"); + } + if (!DBSlotExists("new_interfaces", "guid")) { + DBQueryFatal("ALTER TABLE new_interfaces ADD ". + "`guid` varchar(16) default NULL ". + " AFTER mac"); + } + return 0; +} + +# Local Variables: +# mode:perl +# End: diff --git a/utils/management_iface.in b/utils/management_iface.in index 824d1d87b..d5012e1e0 100644 --- a/utils/management_iface.in +++ b/utils/management_iface.in @@ -1,6 +1,6 @@ #!/usr/bin/perl -w # -# Copyright (c) 2003-2013 University of Utah and the Flux Group. +# Copyright (c) 2003-2014 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -158,7 +158,7 @@ my $IP = shift(); # Convert mac to DB representation. if ($mac =~ /(\w\w):(\w\w):(\w\w):(\w\w):(\w\w):(\w\w)/) { - $mac = "$1$2$3$4$5$6"; + $mac = lc("$1$2$3$4$5$6"); } # Verify optional switch info diff --git a/utils/newnode.in b/utils/newnode.in index e57255ecc..099a607e1 100644 --- a/utils/newnode.in +++ b/utils/newnode.in @@ -1,7 +1,7 @@ #!/usr/bin/perl -w # -# Copyright (c) 2003-2013 University of Utah and the Flux Group. +# Copyright (c) 2003-2014 University of Utah and the Flux Group. # # {{{EMULAB-LICENSE # @@ -237,7 +237,7 @@ NODE: foreach my $node_id (@node_ids) { # # Grab the node's MACs from the new_interfaces table # - $query_result = DBQueryFatal("SELECT card, MAC, interface_type, " . + $query_result = DBQueryFatal("SELECT card, mac, guid, interface_type, " . "switch_id, switch_card, switch_port, cable, len, role, port, IP " . "FROM new_interfaces WHERE new_node_id='$new_node_id'"); if (!$query_result->num_rows()) { @@ -246,7 +246,7 @@ NODE: foreach my $node_id (@node_ids) { } my %interfaces; - while (my ($card, $MAC, $iface_type, $switch_id, $switch_card, + while (my ($card, $MAC, $guid, $iface_type, $switch_id, $switch_card, $switch_port, $cable, $len, $ifrole, $port, $iface_IP) = $query_result->fetchrow()) { # # Get some more information about this interface type @@ -278,7 +278,7 @@ NODE: foreach my $node_id (@node_ids) { # # Stash it away... # - $interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex, + $interfaces{$card} = [$MAC, $guid, $iface_type, $max_speed, $full_duplex, $switch_id, $switch_card, $switch_port, $cable, $len, $ifrole, $port, $iface_IP]; @@ -451,8 +451,9 @@ NODE: foreach my $node_id (@node_ids) { } while (my ($card, $aref) = each %interfaces) { - my ($MAC, $iface_type, $max_speed, $duplex, $switch_id, $switch_card, - $switch_port, $cable, $len, $ifrole, $port, $iface_IP) = @$aref; + my ($MAC, $guid, $iface_type, $max_speed, $duplex, + $switch_id, $switch_card, $switch_port, $cable, $len, + $ifrole, $port, $iface_IP) = @$aref; my $iface = "eth$card"; my $wire_type; my $iface_role; @@ -485,15 +486,17 @@ NODE: foreach my $node_id (@node_ids) { if (! defined($iface_IP)) { $iface_IP = ""; } + my $guidstr = defined($guid) ? "guid='$guid'," : ""; if (!$useScriptFeature || !defined($port)) { $port = 1; } - DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " . - "card=$card, port='$port', mac='$MAC', IP='$iface_IP', " . - "interface_type='$iface_type', iface='$iface', uuid=UUID(), " . - "current_speed='$max_speed', duplex=$duplex, role='$iface_role'"); + DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id'," . + "card=$card,port='$port',mac='$MAC',$guidstr". + "IP='$iface_IP',interface_type='$iface_type',". + "iface='$iface',uuid=UUID(),current_speed='$max_speed',". + "duplex=$duplex,role='$iface_role'"); DBQueryFatal("INSERT INTO interface_state SET node_id='$node_id', " . "card=$card, port='$port', iface='$iface'"); diff --git a/www/newnode-defs.php3 b/www/newnode-defs.php3 index 2a49851c5..ec0986dfe 100644 --- a/www/newnode-defs.php3 +++ b/www/newnode-defs.php3 @@ -176,11 +176,15 @@ function guess_IP ($prefix, $number) { # We want to be able to handle both numeric and character 'number's # Figure out which we have # + $ndigits = 0; if (! is_numeric($number)) { $using_char = 1; $number = ord($number); } else { $using_char = 0; + if (preg_match("/^(0\d+)$/",$number)) { + $ndigits = strlen($number); + } } # @@ -193,6 +197,9 @@ function guess_IP ($prefix, $number) { while ($i > 0) { if ($using_char) { $node = $prefix . chr($i); + } elseif ($ndigits) { + $fmt = "%0" . $ndigits . "d"; + $node = $prefix . sprintf($fmt, $i); } else { $node = $prefix . $i; } diff --git a/www/newnodecheckin.php b/www/newnodecheckin.php index e5b96147e..076788dde 100644 --- a/www/newnodecheckin.php +++ b/www/newnodecheckin.php @@ -110,7 +110,7 @@ if (count($interfaces)) { $query_result = DBQueryFatal("select n.node_id from " . "nodes as n left join interfaces as i " . "on n.node_id=i.node_id " . - "where i.mac='$testmac'"); + "where i.mac='$testmac' or i.guid='$testmac'"); if (mysql_num_rows($query_result)) { $row = mysql_fetch_array($query_result); $node_id = $row["node_id"]; @@ -128,8 +128,7 @@ if (count($interfaces)) { $query_result = DBQueryFatal("select n.new_node_id, n.node_id from " . "new_nodes as n left join new_interfaces as i " . "on n.new_node_id=i.new_node_id " . - "where i.mac='$testmac'"); - + "where i.mac='$testmac' or i.guid='$testmac'"); if (mysql_num_rows($query_result)) { $row = mysql_fetch_array($query_result); $id = $row["new_node_id"]; @@ -222,12 +221,23 @@ $new_node_id = $row[0]; echo "Node ID is $new_node_id\n"; foreach ($interfaces as $interface) { - $card = $interface["card"]; - $mac = $interface["mac"]; - $type = $interface["type"]; - DBQueryFatal("insert into new_interfaces set " . - "new_node_id=$new_node_id, card=$card, mac='$mac', " . - "interface_type='$type'"); + $card = $interface["card"]; + $mac = $interface["mac"]; + $type = $interface["type"]; + $clause = ""; + # XXX not a 6 byte value, assume it is a guid + # XXX probably should check interface_capabilities for the type + if (strlen($mac) != 12) { + $clause = ", guid='$mac'"; + # XXX 16 bytes implies an Infiniband port GUID to us + # cons up what would be the mac + if (strlen($mac) == 16) { + $mac = substr($mac, 0, 6) . substr($mac, 10, 6); + } + } + DBQueryFatal("insert into new_interfaces set " . + "new_node_id=$new_node_id, card=$card, mac='$mac', " . + "interface_type='$type'$clause"); } # @@ -292,6 +302,7 @@ function find_free_id($prefix) { # First, check to see if there's a recent entry in new_nodes we can name # this node after # + $ndigits = 0; $query_result = DBQueryFatal("select node_id from new_nodes " . "order by created desc limit 1"); if (mysql_num_rows($query_result)) { @@ -300,7 +311,19 @@ function find_free_id($prefix) { # # Try to figure out if this is in some format we can increment # - if (preg_match("/^(.*[^\d])(\d+)$/",$old_node_id,$matches)) { + if (preg_match("/^(.*[^\d])(0\d+)$/",$old_node_id,$matches)) { + $base = $matches[1]; + $number = $matches[2]; + $ndigits = strlen($number); + echo "Matches $ndigits-digit pcXXX format"; + $fmt = "%0" . $ndigits . "d"; + $number = sprintf($fmt, $number + 1); + $potential_name = $base . $number; + if (!check_node_exists($potential_name)) { + return array($base, $number); + } + $prefix = $base; + } elseif (preg_match("/^(.*[^\d])(\d+)$/",$old_node_id,$matches)) { echo "Matches pcXXX format"; # pcXXX format $base = $matches[1]; @@ -309,6 +332,7 @@ function find_free_id($prefix) { if (!check_node_exists($potential_name)) { return array($base,($number +1)); } + $prefix = $base; } elseif (preg_match("/^(.*)-([a-zA-Z])$/",$old_node_id,$matches)) { # Something like WAIL's (type-rack-A) format $base = $matches[1]; @@ -318,6 +342,7 @@ function find_free_id($prefix) { if (!check_node_exists($potential_name)) { return array($base . '-', $newchar); } + $prefix = $base; } } @@ -327,10 +352,16 @@ function find_free_id($prefix) { # hasn't been used yet - put in a silly little guard to prevent an # infinite loop in case of bugs. # + if ($ndigits) { + $fmt = "%0" . $ndigits . "d"; + } $node_number = 0; while ($node_number < 10000) { - $node_number++; - $potential_name = $prefix . $node_number; + $number = ++$node_number; + if ($ndigits) { + $number = sprintf($fmt, $number); + } + $potential_name = $prefix . $number; if (!check_node_exists($potential_name)) { break; } -- GitLab