Commit 94d97303 authored by Leigh Stoller's avatar Leigh Stoller

A set of changes to allow the newnode path (MFS and adding nodes) to

work inside ElabInElab.

The crux of this is to emulate what switchmac does when incorporating nodes
into the testbed. Rather then using snmpit to ask the switches for MAC info,
we can just ask outer emulab via the proxy to get that same info from the
DB. Note, there are no changes to the newnode MFS; it will boot and happily
report it after launching teachswitch, but those packets are harmless to the situation.

Another wrinkle is that we need the outer emulab to tell us which
interfaces are control and which are experimental, since inside there
is no real way to determine that. I did this by overloading the final
field in the return from switchmac, and using that to override the
"role" setting that utils/newnode would normally choose on its own.

There is another new routine in the XMLRPC server that is intended to
be used for bypassing the newnode path. This is not hooked into
anything yet, but the intent is that rather then using the web
interface to "add" nodes, we just return everything needed to seed the
new_nodes and new_interfaces table, and then run newnodes directly.
parent 5c1f78cd
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003, 2004 University of Utah and the Flux Group.
# Copyright (c) 2003, 2004, 2005 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -13,12 +13,17 @@
use lib '@prefix@/lib';
use libdb;
use libxmlrpc;
use English;
use Getopt::Std;
use strict;
my $TB = "@prefix@";
my $TB = "@prefix@";
my $ELABINELAB = @ELABINELAB@;
my $RPCSERVER = "@OUTERBOSS_NODENAME@";
my $RPCPORT = "@OUTERBOSS_XMLRPCPORT@";
my $RPCCERT = "@OUTERBOSS_SSLCERTNAME@";
my $switchmac = "$TB/sbin/switchmac";
my $os_load = "$TB/bin/os_load";
......@@ -184,7 +189,7 @@ NODE: foreach my $node_id (@node_ids) {
# Grab the node's MACs from the new_interfaces table
#
$query_result = DBQueryFatal("SELECT card, MAC, interface_type, " .
"switch_id, switch_card, switch_port, cable, len " .
"switch_id, switch_card, switch_port, cable, len, role " .
"FROM new_interfaces WHERE new_node_id='$new_node_id'");
if (!$query_result->num_rows()) {
warn "Node $node_id failed: Must have at least one interface!\n";
......@@ -193,7 +198,7 @@ NODE: foreach my $node_id (@node_ids) {
my %interfaces;
while (my ($card, $MAC, $iface_type, $switch_id, $switch_card,
$switch_port, $cable, $len) = $query_result->fetchrow()) {
$switch_port, $cable, $len, $role) = $query_result->fetchrow()) {
#
# Get some more information about this interface type
#
......@@ -225,7 +230,7 @@ NODE: foreach my $node_id (@node_ids) {
# Stash it away...
#
$interfaces{$card} = [$MAC, $iface_type, $max_speed, $full_duplex,
$switch_id, $switch_card, $switch_port, $cable, $len];
$switch_id, $switch_card, $switch_port, $cable, $len, $role];
}
......@@ -283,6 +288,49 @@ NODE: foreach my $node_id (@node_ids) {
$state = $STATE_INITIAL;
}
#
# When operating inside an inner elab, we need an extra bit of information
# from the outer boss; which the IP of the outer control network. This is
# strictly for bootinfo which has some really gross hacks in it.
#
my $outer_control_ip;
if ($ELABINELAB) {
libxmlrpc::Config({"server" => $RPCSERVER,
"verbose" => 0,
"cert" => $RPCCERT,
"portnum" => $RPCPORT});
my $rval = libxmlrpc::CallMethod("elabinelab", "newnode_info",
{"mac" => $interfaces{0}[0]});
if (!defined($rval)) {
print "XMPRPC server did not return control iface info!\n";
next NODE;
}
else {
#
# Grab what we need by searching the interface marked as
# the outer control network.
#
my %iface_array = %{ $rval->{"interfaces"} };
foreach my $mac (keys(%iface_array)) {
my $aref = $iface_array{$mac};
my $role = $aref->{"role"};
if ($role eq TBDB_IFACEROLE_OUTER_CONTROL()) {
$outer_control_ip = $aref->{"IP"};
last;
}
}
}
if (!defined($outer_control_ip)) {
print "Could not determine outer control interface IP!\n";
next NODE;
}
}
#
# Okay, time to actually add the node!
#
......@@ -348,20 +396,39 @@ NODE: foreach my $node_id (@node_ids) {
while (my ($card, $aref) = each %interfaces) {
my ($MAC, $iface_type, $max_speed, $duplex, $switch_id, $switch_card,
$switch_port, $cable, $len) = @$aref;
$switch_port, $cable, $len, $role) = @$aref;
my $iface = "eth$card";
my $iface_IP = "";
my $wire_type = "Node";
my $iface_role = TBDB_IFACEROLE_EXPERIMENT();
if ($iface eq $control_iface) {
$iface_IP = $IP;
$wire_type = "Control";
my $iface_IP;
my $wire_type;
my $iface_role;
# Role from new_interfaces overrides anything we decide here.
if (defined($role)) {
$iface_role = $role;
}
elsif ($iface eq $control_iface) {
$iface_role = TBDB_IFACEROLE_CONTROL();
}
else {
$iface_role = TBDB_IFACEROLE_EXPERIMENT();
}
# Okay, now set up some stuff for the inserts.
if ($iface_role eq TBDB_IFACEROLE_CONTROL()) {
$iface_IP = $IP;
$wire_type = "Control";
}
elsif ($iface_role eq TBDB_IFACEROLE_OUTER_CONTROL()) {
$iface_IP = $outer_control_ip;
$wire_type = "OuterControl";
}
else {
# Experimental interfaces start with no speed set.
$max_speed = 0;
$iface_IP = "";
$wire_type = "Node";
}
DBQueryFatal("INSERT INTO interfaces SET node_id='$node_id', " .
"card=$card, port=1, mac='$MAC', IP='$iface_IP', " .
"interface_type='$iface_type', iface='$iface', " .
......
......@@ -16,6 +16,7 @@ $TBWWW = "@TBWWW@";
$THISHOMEBASE = "@THISHOMEBASE@";
$ELABINELAB = @ELABINELAB@;
$WIKISUPPORT = @WIKISUPPORT@;
$CONTROL_NETWORK= "@CONTROL_NETWORK@";
$WIKIURL = "https://${USERNODE}/twiki/bin/newlogon";
$WIKICOOKIENAME = "WikiCookie";
......
<?PHP
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003 University of Utah and the Flux Group.
# Copyright (c) 2003, 2005 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -13,7 +13,7 @@
function find_switch_macs(&$mac_list) {
global $uid, $gid, $TBSUEXEC_PATH;
global $uid, $gid, $TBSUEXEC_PATH, $ELABINELAB;
#
# Avoid SIGPROF in child.
......@@ -40,11 +40,17 @@ function find_switch_macs(&$mac_list) {
$switch = $matches[1];
$card = $matches[2];
$port = $matches[3];
if ($mac_list[$MAC] && (!$mac_list[$MAC]["class"] ||
($mac_list[$MAC]["class"] == $class))) {
if ($mac_list[$MAC] &&
(is_null($mac_list[$MAC]["class"]) ||
!isset($mac_list[$MAC]["class"]) ||
($mac_list[$MAC]["class"] == $class))) {
$mac_list[$MAC]["switch"] = $switch;
$mac_list[$MAC]["switch_card"] = $card;
$mac_list[$MAC]["switch_port"] = $port;
if ($ELABINELAB) {
# We let switchmac tell us.
$mac_list[$MAC]["class"] = $class;
}
}
$line = fgets($macs);
}
......@@ -122,11 +128,12 @@ function make_node_type($type,$proc,$disk) {
}
function guess_IP ($prefix, $number) {
global $CONTROL_NETWORK;
$hostname = $prefix . $number;
#
# First, let's see if they've already added to to DNS - the PHP
# First, lets see if they've already added to to DNS - the PHP
# gethostbyname has a really dumb way to return failure - it just
# gives you the hostname back.
#
......@@ -181,8 +188,11 @@ function guess_IP ($prefix, $number) {
$i--;
}
#
# Lets start out as the first address on the given control network.
#
if ($i <= 0) {
return 0;
$IP = $CONTROL_NETWORK;
}
#
......
e<?php
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003 University of Utah and the Flux Group.
# Copyright (c) 2003, 2005 University of Utah and the Flux Group.
# All rights reserved.
#
require("defs.php3");
require("newnode-defs.php3");
include("xmlrpc.php3");
#
# Note - this script is not meant to be called by humans! It returns no useful
# information whatsoever, and expects the client to fill in all fields
# properly.
# Since this script does not cause any action to actually happen, so it's save
# Since this script does not cause any action to actually happen, so its save
# to leave 'in the open' - the worst someone can do is annoy the testbed admins
# with it!
#
......@@ -52,8 +53,8 @@ if (count($interfaces)) {
$testmac = $interfaces[0]["mac"];
#
# First, make sure it isn't a 'real boy' - we should let the operators know
# about this, because there may be some problem.
# First, make sure it is not a 'real boy' - we should let the operators
# know about this, because there may be some problem.
#
$query_result = DBQueryFatal("select n.node_id from " .
"nodes as n left join interfaces as i " .
......@@ -86,7 +87,7 @@ if (count($interfaces)) {
echo "Node ID is $id\n";
#
# Keep the temp. IP address around in case it's gotten a new one
# Keep the temp. IP address around in case its gotten a new one
#
DBQueryFatal("update new_nodes set temporary_IP='$tmpIP' " .
"where new_node_id=$id");
......@@ -116,11 +117,11 @@ if ($use_temp_IP) {
}
#
# Handle the node's type
# Handle the node type
#
if ($type) {
#
# If they gave us a type, let's see if that type exists or not
# If they gave us a type, lets see if that type exists or not
#
if (TBValidNodeType($type)) {
#
......@@ -198,6 +199,36 @@ function check_node_exists($node_id) {
}
function find_free_id($prefix) {
global $ELABINELAB, $TBADMINGROUP, $interfaces;
#
# When inside an inner emulab, we have to ask the outer emulab for
# our nodeid; we cannot just pick one out of a hat, at least not yet.
#
if ($ELABINELAB) {
$arghash = array();
$arghash["mac"] = $interfaces[0]["mac"];
$results = XMLRPC("nobody", $TBADMINGROUP,
"elabinelab.newnode_info", $arghash);
if (!$results || ! isset($results{'nodeid'})) {
echo "Could not get nodeid from XMLRPC server; quitting.\n";
exit;
}
elseif (preg_match("/^(.*[^\d])(\d+)$/",
$results{'nodeid'}, $matches)) {
$base = $matches[1];
$number = $matches[2];
return array($base, intval($number));
}
else {
$nodeid = $results{'nodeid'};
echo "Improper nodeid ($nodeid) from XMLRPC server; quitting.\n";
exit;
}
}
#
# First, check to see if there's a recent entry in new_nodes we can name
......
<?PHP
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003 University of Utah and the Flux Group.
# Copyright (c) 2003, 2005 University of Utah and the Flux Group.
# All rights reserved.
#
require("defs.php3");
......@@ -117,7 +117,7 @@ if ($create) {
#
if ($research) {
#
# Get the MACs we're supposed to be looking for
# Get the MACs we are supposed to be looking for
#
$query_result = DBQueryFatal("SELECT i.mac, i.new_node_id, n.node_id, " .
"i.card, i.card=t.control_net AS is_control " .
......@@ -127,10 +127,15 @@ if ($research) {
"WHERE $whereclause_qualified");
$mac_list = array();
while ($row = mysql_fetch_array($query_result)) {
if ($row["is_control"]) {
$class = "control";
} else {
$class = "experimental";
if ($ELABINELAB) {
# See find_switch_macs().
$class = NULL;
}
elseif ($row["is_control"]) {
$class = TBDB_IFACEROLE_CONTROL;
}
else {
$class = TBDB_IFACEROLE_EXPERIMENT;
}
$mac_list[$row["mac"]] = array( "new_node_id" => $row["new_node_id"],
"node_id" => $row["node_id"], "card" => $row["card"],
......@@ -141,7 +146,22 @@ if ($research) {
find_switch_macs($mac_list);
foreach ($mac_list as $mac => $switchport) {
if ($switchport["switch"]) {
DBQueryFatal("UPDATE new_interfaces SET " .
$extra_set = "";
if ($ELABINELAB) {
#
# The reason for these ELABINELAB tests is that we cannot
# use the node_types table to determine which interface is
# the control network, since assign can pick any old interface
# for each inner node. Generally speaking, we should not do
# this at all, but rely on an outside mechanism to tell us
# which interface is the control network. Anyway, I am using
# the "role" slot of the new_interfaces table to override
# what utils/newnode is going to set them too.
#
$extra_set = "role='$switchport[class]', ";
}
DBQueryFatal("UPDATE new_interfaces SET $extra_set " .
"switch_id='$switchport[switch]', " .
"switch_card='$switchport[switch_card]', " .
"switch_port='$switchport[switch_port]' ".
......
......@@ -3684,6 +3684,172 @@ class elabinelab:
argdict["imageid"])
return EmulabResponse(RESPONSE_SUCCESS, str(res[0][0]), output=output)
#
# Return the equivalent of what switchmac does. This is used so that
# nodes can be added to an inner emulab in mostly the same fashion as
# the outer emulab, but without having to talk to the switch directly.
#
def switchmac(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
verifyerror = self.verifystuff();
if (verifyerror):
return verifyerror
#
# Okay, lets grab what we need. This code is going to return the
# equiv of what the switchmac script does, but with a few minor
# changes.
#
dbres = DBQueryFatal("select r.node_id,i.mac,i.iface,i.role,"
" s.node_id2,s.card2,s.port2,i.IP "
"from reserved as r "
"left join interfaces as i on "
" i.node_id=r.node_id "
"left join wires as s on s.node_id1=i.node_id "
" and s.card1=i.card "
"where r.pid=%s and r.eid=%s and "
" r.inner_elab_role='node' "
"order by r.node_id,iface",
(self.pid,self.eid,))
result = {}
for res in dbres:
tmp = {}
#
# The current control network becomes the outer control network.
# The experimental network with an IP assigned becomes the
# inner control network.
#
role = res[3]
IP = res[7]
if role == "ctrl":
role = "outer_ctrl"
pass
elif role == "expt" and IP != "":
role = "ctrl"
pass
tmp["mac"] = res[1]
tmp["iface"] = res[2]
tmp["role"] = role
tmp["switch_id"] = res[4]
tmp["switch_card"] = res[5]
tmp["switch_port"] = res[6]
result[res[1]] = tmp;
pass
return EmulabResponse(RESPONSE_SUCCESS, result, output=str(result))
#
# Return some stuff about a newnode so that the inner newnode path
# can operate properly
#
def newnode_info(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
argerror = CheckRequiredArgs(argdict, ("mac",))
if (argerror):
return argerror
if not re.match("^[\w]*$", str(argdict["mac"])):
return EmulabResponse(RESPONSE_BADARGS,
output="Improperly formed mac value!")
verifyerror = self.verifystuff();
if (verifyerror):
return verifyerror
#
# First map the mac to a node by looking in the interfaces table.
#
dbres = DBQueryFatal("select node_id from interfaces "
"where mac=%s",
(argdict["mac"],))
# Hmm, something went wrong?
if len(dbres) == 0:
return EmulabResponse(RESPONSE_ERROR,
output="Cannot map MAC to nodeid")
nodeid = dbres[0][0]
#
# Okay, lets grab what we need. This code is going to return the
# equiv of what the switchmac script does, but with a few minor
# changes.
#
dbres = DBQueryFatal("select r.node_id,i.mac,i.iface,i.role,i.IP, "
" i.card,i.interface_type, "
" s.node_id2,s.card2,s.port2, "
" n.type "
"from reserved as r "
"left join nodes as n on n.node_id=r.node_id "
"left join interfaces as i on "
" i.node_id=r.node_id "
"left join wires as s on s.node_id1=i.node_id "
" and s.card1=i.card "
"where r.node_id=%s "
"order by r.node_id,iface",
(nodeid,))
result = {}
result["nodeid"] = nodeid
result["type"] = ""
interfaces = {}
for res in dbres:
tmp = {}
#
# The current control network becomes the outer control network.
# The experimental network with an IP assigned becomes the
# inner control network.
#
role = res[3]
IP = res[4]
ntype = res[10]
if role == "ctrl":
role = "outer_ctrl"
pass
elif role == "expt" and IP != "":
role = "ctrl"
pass
result["type"] = ntype
tmp["mac"] = res[1]
tmp["iface"] = res[2]
tmp["role"] = role
tmp["IP"] = IP
tmp["card"] = res[5]
tmp["type"] = res[6]
tmp["switch_id"] = res[7]
tmp["switch_card"] = res[9]
tmp["switch_port"] = res[9]
interfaces[res[1]] = tmp
pass
result["interfaces"] = interfaces
return EmulabResponse(RESPONSE_SUCCESS, result, output=str(result))
pass
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment