All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit 8c08c280 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Rework the previous hack stuff for virtual (multiplex nodes). Now its

a new hack for remote nodes! All remote node solving is done in the
new wanassign script. That script deals with both phys (ron) and
virtual (vrons), as well as fixed nodes of either type. The solution
is returned to assign_wrapper, which still does the tunnel stuff.
At some point it would be nice to get general purpose virtual nodes
(multiplex) working on either local or remote nodes, but thats a ways
off cause it means too many changes in assign_wrapper, which really
needs a total rewrite.
parent 16c998a9
......@@ -123,7 +123,8 @@ TBDebugTimeStamp("assign_wrapper started");
#
# Datastructures:
# nodes is indexed by virtual node and contains the node type.
# isvirtnode is indexed by virtual node and says whether the node is "virtual"
# isremotenode is indexed by virtual node and says whether the node is
# is remote. We let wanassign deal with those nodes.
# nodelans is indexed by virtual node and contains a list of
# <port>:<lan> that it is connected to.
# ips is indexed by nodeport and contains the IP addresses.
......@@ -180,24 +181,24 @@ printdb "\n";
printdb "Loading virt_nodes.\n";
$result =
DBQueryFatal("select distinct vname,ips,vn.type,fixed,nt.isvirtnode ".
DBQueryFatal("select distinct vname,ips,vn.type,fixed,nt.isremotenode ".
" from virt_nodes as vn ".
"left join node_types as nt on ".
" nt.type=vn.type or nt.class=vn.type ".
"where pid='$pid' and eid='$eid'");
while (($vname,$ips,$type,$fixed,$isvirt) = $result->fetchrow_array) {
while (($vname,$ips,$type,$fixed,$isremote) = $result->fetchrow_array) {
if (! defined($fixed)) {$fixed = "";}
# VIRTNODE HACK
# REMOTENODE HACK
#
# if its a vtype, no entry in node_types. vtypes break virtual nodes.
# if its a vtype, no entry in node_types. vtypes break remote nodes.
# Need to look inside the vtype and make sure no mixing of vnodes and
# physnodes. Later ...
#
if (! defined($isvirt)) {$isvirt = 0;}
if (! defined($isremote)) {$isremote = 0;}
printdb " $vname $type $ips $fixed $isvirt\n";
printdb " $vname $type $ips $fixed $isremote\n";
# We need to check the names to make sure they won't clash with
# our internal delay node names.
if (($vname =~ /^tbdelay\d+/) ||
......@@ -209,9 +210,9 @@ while (($vname,$ips,$type,$fixed,$isvirt) = $result->fetchrow_array) {
$nodes{$vname} = $type;
$nodelans{$vname} = [];
# VIRTNODE HACK
# REMOTENODE HACK
#
$isvirtnode{$vname} = $isvirt;
$isremotenode{$vname} = $isremote;
foreach $ipinfo (split(" ",$ips)) {
($port,$ip) = split(":",$ipinfo);
$ips{"$vname:$port"} = $ip;
......@@ -229,30 +230,30 @@ $result =
"from virt_lans where pid='$pid' and eid='$eid'");
#
# VIRTNODE HACK: Virtual nodes are special.
# REMOTENODE HACK: Remote nodes are special.
#
# Prepass the table looking for those virt_lans that have a virtual node
# Prepass the table looking for those virt_lans that have a remote node
# as a member. We want to ignore those (both sides) if its a duplex
# link, but if its in a lan then we barf since we cannot support putting
# a vnode into a lan yet. We might never support it ...
# a remote into a lan.
#
my %vnodelans = ();
my %rnodelans = ();
my %lancount = ();
my %tunnels = ();
while (($vname,$member) = $result->fetchrow_array) {
($node,$port) = split(":",$member);
if ($isvirtnode{$node}) {
$vnodelans{$vname} = 1;
if ($isremotenode{$node}) {
$rnodelans{$vname} = 1;
}
if (!defined($lancount{$vname})) {
$lancount{$vname} = 0;
}
$lancount{$vname} += 1;
if ($lancount{$vname} > 2 && defined($vnodelans{$vname})) {
if ($lancount{$vname} > 2 && defined($rnodelans{$vname})) {
die("*** $0:\n".
" Virtual node $node in lan $vname not allowed!\n");
" Remote node $node in lan $vname not allowed!\n");
}
}
$result->dataseek(0);
......@@ -265,13 +266,14 @@ while (($vname,$member,$delay,$bandwidth,$lossrate,
($node,$port) = split(":",$member);
#
# VIRTNODE HACK: Virtual nodes are special.
# REMOTENODE HACK:
#
# If its a duplex link involving a virtnode, skip it.
# If its a duplex link involving a remotenode, skip it.
# We do not want to have assign deal with these. The nodes are
# allocated as unconnected, and we deal with it later.
# allocated as unconnected by another program, and we deal with
# it later by creating tunnels.
#
if ($vnodelans{$vname}) {
if ($rnodelans{$vname}) {
if (! defined($tunnels{$vname})) {
$tunnels{$vname} = [];
}
......@@ -408,7 +410,8 @@ $delaynodes=0;
foreach $node (keys(%nodes)) {
# Shark hack
if (($nodes{$node} ne "shark") && ($nodes{$node} ne "dnard")) {
if (($nodes{$node} ne "shark") &&
($nodes{$node} ne "dnard") && !$isremotenode{$node}) {
print TOPFILE "node $node $nodes{$node}\n";
if ($nodes{$node} ne "shark-shelf") {
$nodes++;
......@@ -536,7 +539,9 @@ foreach $lan (keys(%lans)) {
# Print out fixed nodes
foreach $fixed (keys(%fixed_nodes)) {
print TOPFILE "fix-node $fixed $fixed_nodes{$fixed}\n";
if (! $isremotenode{$fixed}) {
print TOPFILE "fix-node $fixed $fixed_nodes{$fixed}\n";
}
}
close TOPFILE;
......@@ -748,6 +753,59 @@ while (1) {
TBDebugTimeStamp("assign_loop finished");
###########################################################################
# Step 2A
#
# We run the wanassigner to allocate remote nodes. We do this after cause
# it takes so long. We run it just once.
#
# wanassign does its own nalloc.
#
###########################################################################
if (scalar(keys(%isremotenode))) {
my $success = 0;
my $doremote = 0;
foreach my $value (values(%isremotenode)) {
if ($value) {
$doremote = 1;
}
}
if ($doremote) {
print "Running 'wanassign -d $pid $eid'\n";
open(WANFP,"wanassign -d $pid $eid 2>&1 | tee wanassign.log |") or
die("*** $0:\n".
" Failed to start wanassign: $!\n");
printdb "Reading wanassign results.\n";
while (<WANFP>) {
chop;
if ($_ =~ /(\S+) mapsto (\S+)/) {
$v2pmap{$1} = $2;
$p2vmap{$2} = $1;
printdb " $1 $2\n";
}
if ($_ =~ /^Success/) {
$success = 1;
}
# Skip other output. Usually its debugging output.
}
close(WANFP) or
die("*** $0:\n".
" wanassign: " . $? ? "exited with status: $?.\n" :
"error closing pipe: $!\n");
if (!$success) {
die("*** $0:\n".
" wanassign could not find a solution!\n");
}
TBDebugTimeStamp("wanassign finished");
}
}
#
# VIRTNODES HACK: physical nodes (from the nodes table) might really
# be a virtual node :-) Must get the underlying phys_nodeid.
......@@ -757,6 +815,7 @@ my %p2pmap = ();
foreach my $pnode (keys(%p2vmap)) {
my $phys_nodeid;
# Skip special LAN nodes.
if (defined($lannodes{$p2vmap{$pnode}})) {
next;
}
......@@ -1160,7 +1219,7 @@ foreach $vnodeport (keys(%portbw)) {
}
#
# Upload the tunnels table. These are built for virtual node links.
# Upload the tunnels table. These are built for remote node links.
#
# First need to assign ports to the servers. To do that need to figure
# out who is a server! We put the server on a real emulab node if
......@@ -1170,10 +1229,10 @@ foreach $vnodeport (keys(%portbw)) {
# per server node.
#
my %lantotunnelserver = ();
my %vnodetotunnelport = ();
my %rnodetotunnelport = ();
my $secretkey;
foreach $lan (keys(%vnodelans)) {
foreach $lan (keys(%rnodelans)) {
my $server;
my ($vnode,$vport);
......@@ -1183,7 +1242,7 @@ foreach $lan (keys(%vnodelans)) {
foreach $member (@{$tunnels{$lan}}) {
($vnode,$vport) = split(":",$member);
if (! $isvirtnode{$vnode} &&
if (! $isremotenode{$vnode} &&
! defined($lantotunnelserver{$lan})) {
$server = $vnode;
}
......@@ -1195,13 +1254,13 @@ foreach $lan (keys(%vnodelans)) {
$lantotunnelserver{$lan} = $server;
# Assign a port, but only the first time chosen.
if (! $vnodetotunnelport{$server}) {
$vnodetotunnelport{$server} = nextipportnum($p2pmap{$v2pmap{$server}});
if (! $rnodetotunnelport{$server}) {
$rnodetotunnelport{$server} = nextipportnum($p2pmap{$v2pmap{$server}});
}
}
printdb "Tunnels\n";
foreach $lan (keys(%vnodelans)) {
foreach $lan (keys(%rnodelans)) {
printdb " LAN: $lan\n";
my @members = @{$tunnels{$lan}};
......@@ -1210,7 +1269,7 @@ foreach $lan (keys(%vnodelans)) {
" Too many members for a tunnel!\n");
}
my $server = $lantotunnelserver{$lan};
my $ipport = $vnodetotunnelport{$server};
my $ipport = $rnodetotunnelport{$server};
# So we can figure out the peer ip.
my ($member0,$member1) = @members;
......@@ -1423,3 +1482,4 @@ sub nextipportnum($) {
return $port;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment