ptopgen.in

#!/usr/bin/perl -w

#
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#

use English;
use Getopt::Std;
use Math::BigInt;
use List::Util 'shuffle';

use lib "@prefix@/lib";
use libdb qw(TBGetSiteVar);
use libadminctrl;
use libptop;
use EmulabFeatures;

my @SAVEARGV = @ARGV;

my $top = libptop->Create();

my $PGENISUPPORT = @PROTOGENI_SUPPORT@;

$NO_GENI = "0";
$V_0_1 = "0.1";
$V_0_2 = "0.2";
$V_2 = "2";
$V_3 = "3";

$emulabns = "http://www.protogeni.net/resources/rspec/ext/emulab/1";
# XXX: This needs to be changed
$emulaburl = "http://www.protogeni.net/resources/rspec/ext/emulab/1/ptop_extension.xsd";
$sharedns = "http://www.geni.net/resources/rspec/ext/shared-vlan/1";
$sharedurl = "http://www.geni.net/resources/rspec/ext/shared-vlan/1/ad.xsd";
$stitchns = "http://hpn.east.isi.edu/rspec/ext/stitch/0.1/";
$stitchurl = "http://hpn.east.isi.edu/rspec/ext/stitch/0.1/stitch-schema.xsd";

sub usage()
{
    print("Usage: ptopgen [-v] [-s switch] [-p pid [-e eid]] [-m factor] " .
	  "[-n c/e] [-x] [-g (0.1 | 2)] [-c component-name]".
	  "[-l type-limit-file] [-z]\n" .
	  "       -p include nodes the project has permission to use\n".
	  "       -e include given experiments resources\n" .
	  "          in the ptopfile (as if they were free)\n" .
	  "       -v Include stuff for topologies with virtual nodes\n".
	  "       -r Include stuff for topologies with widearea nodes\n".
#	  "       -s Include stuff for topologies with simulated nodes\n".
	  "       -h Include stuff for topologies with shared nodes\n".
	  "       -a Include even reserved nodes\n".
	  "       -m Override multiplex_factor\n".
	  "       -u Prune unused interfaces of allocated nodes (-e)\n".
	  "       -c Delay capacity override\n".
	  "       -n Add in modelnet core and edge node features\n".
	  "       -x Output into the new xml ptop format\n".
	  "       -g version With -x, geni version. Must be '0', '0.1' or '2'\n".
	  "       -l specifies the location of the type limit file\n" .
          "       -1 Print an rspec containing only the node component-name\n" .
	  "       -z Randomize node output order\n" .
	  "       -Z Force old ptopgen\n");
        exit(-1);
}

my $optlist = "s:e:m:vp:rSan:c:uxg:h1:l:zZCb";
my $mfactor;
my $virtstuff = 0;
my $widearea  = 0;
my $simstuff  = 0;
my $allnodes  = 0;
my $mnetcores = 0;
my $mnetedges = 0;
my $prune     = 0;
my $do_xml    = 0;
my $genimode  = $NO_GENI;
my $useshared = 0;
my $useblockstore = 0;
my $component_name = undef;
my $randomize = 0;
my $usecontrol= 0;

my $OURDOMAIN = "@OURDOMAIN@";
my $MAINSITE  = @TBMAINSITE@;
my $cmuuid = TBGetSiteVar('protogeni/cm_uuid');
my $cmurn = "";
if ($PGENISUPPORT) {
    require GeniHRN;
    $cmurn = &GeniHRN::Generate($OURDOMAIN, "authority", "cm");
}

$opstate = <<'OPSTATE';
<rspec_opstate xmlns="http://www.geni.net/resources/rspec/ext/opstate/1"
  aggregate_manager_id="CMURN"
  start="geni_notready">
  <sliver_type name="raw-pc" />
  <sliver_type name="emulab-openvz" />
  <sliver_type name="emulab-xen" />

  <state name="geni_notready">
    <action name="geni_start" next="geni_configuring">
      <description>Boot the node</description>
    </action>
    <description>Raw PCs and VMs begin powered down or inactive. They
    must be explicitly booted before use.</description>
  </state>
  <state name="geni_configuring">
    <wait type="geni_success" next="geni_ready" />
    <wait type="geni_failure" next="geni_failed" />
    <description>Booting takes a significant amount of time, so it
    happens asynchronously while the node is in this
    state.</description>
  </state>
  <state name="geni_ready">
    <action name="geni_restart" next="geni_configuring">
      <description>Reboot the node</description>
    </action>
    <action name="geni_stop" next="geni_stopping">
      <description>Power down or stop the node.</description>
    </action>
    <description>The node is up and ready to use.</description>
  </state>
  <state name="geni_stopping">
    <wait type="geni_success" next="geni_notready" />
    <wait type="geni_failure" next="geni_failed" />
    <description>The node is being stopped or rebooted.</description>
  </state>
  <state name="geni_failed">
    <description>The node has failed and requires administrator
    intervention before it can be used. Please contact
    the administrator for assistance.</description>
  </state>
</rspec_opstate>
OPSTATE

$opstate =~ s/CMURN/$cmurn/g;


my $default_long = undef;
TBGetSiteVar('general/default_longitude', \$default_long);
my $default_lat = undef;
TBGetSiteVar('general/default_latitude', \$default_lat);
my $default_country = undef;
TBGetSiteVar('general/default_country', \$default_country);

my $delaycap_override;

#
# Turn off line buffering on output
#
$| = 1;

# Settings - Change these to your particular network.

#
# Also bogus. The interfaces table does not hold entries for the switch
# side of each link. It will have to at some point, when we have something
# other than 100Mbs ports (say, gbit ports).
#
# Speed in in Kbs!
#
my $default_switchport_speed    = 100000;

#
# Yet more bogosity - we hardcode this weight, which is given to wide-area
# (primarily plab) nodes to prefer spreading across sites
#
my $site_weight = 0.99;

######################################################################

my $TBROOT = "@prefix@";
use lib '@prefix@/lib';
require exitonwarn;
use libdb;
use libtblog;
use Experiment;
use Node;
use NodeType;
use Lan;
use BlockstoreType;

tblog_stop_capture('stdout');

my $TRACK_INTERSWITCH_BANDWIDTH = "@TRACK_INTERSWITCH_BANDWIDTH@";

my %switches	  = ();
my %permissions   = ();
my %typemap       = ();
my %auxtypemap    = ();
my %areamap       = ();
my %globalcounts  = ();
my %interfacestate= ();
my %vinterfaces   = ();
my %rusagedata    = ();

my $DEADPID = NODEDEAD_PID();
my $DEADEID = NODEDEAD_EID();

my $pid;
my $exempt_eid;
my $switchtouse;
my $experiment;

my $typelimitfile = "";

sub fatal($);

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV) {
    usage();
}
if (defined($options{"s"})) {
    $switchtouse = $options{"s"};
}
if (defined($options{"m"})) {
    $mfactor = $options{"m"};
}
if (defined($options{"h"})) {
    $useshared = 1;
}
if (defined($options{"b"})) {
    $useblockstore = 1;
}
if (defined($options{"v"})) {
    $virtstuff = 1;
}
if (defined($options{"r"})) {
    $widearea = 1;
}
if (defined($options{"S"})) {
    $simstuff = 1;
}
if (defined($options{"p"})) {
    $pid = $options{"p"};
}
if (defined($options{"a"})) {
    $allnodes = 1;
}
if (defined($options{"u"})) {
    $prune = 1;
}
if (defined($options{"c"})) {
    $delaycap_override = $options{"c"};
}
if (defined($options{"C"})) {
    $usecontrol = 1;
}
if (defined($options{"n"})) {
    if ($options{"n"} =~ /(\d*),(\d*)/) {
	$mnetcores = $1;
	$mnetedges = $2;
    }
    else {
	usage();
    }
}
if (defined($options{"e"})) {
    $exempt_eid = $options{"e"};
    usage()
	if (!defined($pid));
    $experiment = Experiment->Lookup($pid, $exempt_eid);
    die("Could not look up experiment $pid,$exempt_eid\n")
	if (!defined($experiment));
}
if (defined($options{"x"})) {
    if (defined($options{"g"}) && $PGENISUPPORT) {
	$useshared = 1;
	$virtstuff = 1;
	$genimode = $options{"g"};
    }
    $do_xml = 1;
}
if (defined($options{"l"})) {
    $typelimitfile = $options{"l"};
}
if (defined($options{"1"})) {
    $component_name = $options{"1"};
}
if (defined($options{"z"})) {
    $randomize = 1;
}

usage()
    if ($prune && !defined($exempt_eid)
	|| ($genimode ne $NO_GENI
	    && $genimode ne $V_0_1
	    && $genimode ne $V_0_2
	    && $genimode ne $V_2
	    && $genimode ne $V_3));

if (defined($pid) && ! defined($options{"Z"})) {
    my $group = Group->Lookup($pid, $pid);
    if (defined($group)) {
	$EmulabFeatures::verbose = 0;
	my $newptopgen = EmulabFeatures->FeatureEnabled("NewPtopgen", undef,
							$group, undef);
	if ($newptopgen) {
	    my $newpath = "$TBROOT/libexec/ptopgen_new";
	    print STDERR "Invoking new ptopgen from $newpath\n"
		if (!$genimode);
	    exec $newpath, @SAVEARGV;
	    die("*** $0:\n".
		"    Could not exec $newpath: $!");
	}
    }
}

$fake_inet_switch = "internet";
$fake_inet_iface = "(null)";
$fake_air_switch = "airswitch";
$fake_air_iface = "(null)";
if ($genimode ne $NO_GENI) {
    $fake_inet_iface = "border";
    $fake_air_iface = "air";
}

my %nodetointerface;
my %interfaceroles = ();
my %interfacetypes = ();
my %interfaceips = ();

# Read interfaces
my $result =
    DBQueryFatal("SELECT node_id,card,port,iface,interface_type,role,IP" .
		 " from interfaces where logical=0 " .
		 ($genimode ne $NO_GENI ? "and role!='gw'" : ""));
		 
while (($node,$card,$port,$iface,$type,$role,$ip) = $result->fetchrow_array) {
    push @{ $nodetointerface{"$node"} }, $iface;
    $interfacemap{"$node:$card:$port"} = $iface;
    if ((defined $type) && ($type ne "")) {
	$interfacetypes{"$node:$card:$port"} = $type;
    }
    if ((defined $role) && ($role ne "")) {
	$interfaceroles{"$node:$iface"} = $role;
    }
    if ((defined $ip) && ($ip ne "")) {
	$interfaceips{"$node:$iface"} = $ip;
    }
}

my %external_nodes;
my %external_managers;
my %external_ifaces;
my %external_links;
my %contact_nodes;
my %contact_ifaces;
my %stitch_points;

$result = DBQueryFatal("select w.node_id1, w.card1, w.port1, i1.iface, ".
		       "w.node_id2, w.card2, w.port2, i2.iface, ".
		       "w.external_interface, w.external_wire, ".
		       "e.node_id, e.min_vlan, e.max_vlan, ".
		       "e.external_manager, e.network_id, e.external_interface, e.external_wire ".
		       "from wires as w ".
		       "left join external_networks as e ".
		       "on w.node_id1=e.node_id or w.node_id2=e.node_id ".
		       "left join interfaces as i1 on w.node_id1=i1.node_id and w.card1=i1.card and w.port1=i1.port ".
		       "left join interfaces as i2 on w.node_id2=i2.node_id and w.card2=i2.card and w.port2=i2.port ".
		       "where e.node_id is not null");
while (my ($cnode, $ccard, $cport, $ciface, $enode, $ecard, $eport, $eiface,
	   $external_iface_urn, $external_link_urn, $external, $minLan,
	   $maxLan, $external_manager_urn,
	   $external_network_id, $external_network_iface_urn,
       $external_network_link_urn) = $result->fetchrow_array) {
    if ($external eq $cnode) {
	my $temp;
	$temp = $cnode; $cnode = $enode; $enode = $temp;
	$temp = $ccard; $ccard = $ecard; $ecard = $temp;
	$temp = $cport; $cport = $eport; $eport = $temp;
	$temp = $ciface; $ciface = $eiface; $eiface = $temp;
    }
    $stitch_points{"$cnode:$enode"} = 1;
    $enode = $external_network_id;
    if (! defined($eiface)) {
	$eiface = "$ecard.$eport";
    }
    if (! defined($ciface)) {
	$ciface = "$ccard.$cport";
    }
    if (! defined($external_iface_urn)) {
	$external_iface_urn = $external_network_iface_urn;
    }
    if (! defined($external_link_urn)) {
	$external_link_urn = $external_network_link_urn;
    }
    $external_nodes{$enode} = "$minLan-$maxLan";
    $external_managers{"$cnode:$enode"} = $external_manager_urn;
    $external_ifaces{"$cnode:$enode"} = $external_iface_urn;
    $external_links{"$cnode:$enode"} = $external_link_urn;
    if (exists($contact_nodes{$cnode})) {
	push(@{ $contact_nodes{$cnode} }, $enode);
    } else {
	$contact_nodes{$cnode} = [$enode];
    }
    $contact_ifaces{"$cnode:$enode"} = GeniHRN::GenerateInterface($OURDOMAIN,
								  $cnode,
								  $ciface);
}

print_header();

my %nodetouuid;
my %nodetoavailable;

$result = DBQueryFatal("SELECT n.node_id, n.eventstate, n.role, n.uuid, " .
		       "nt.isremotenode, " .
		       "dedicated_wa_types.attrvalue, b.erole, " .
		       "n.reserved_pid, b.eid " .
		       "from nodes as n " .
		       "left join reserved as b on n.node_id=b.node_id " .
		       "left join node_types as nt on nt.type=n.type " .
		       "left outer join " .
		       "  (select type, attrvalue " .
		       "   from node_type_attributes " .
		       "   where attrkey='dedicated_widearea' " .
		       "   group by type) as dedicated_wa_types " .
		       "  on nt.type=dedicated_wa_types.type " .
		       "where nt.isvirtnode = 0 or nt.isvirtnode is null;");
while (($node,$eventstate, $role, $uuid, $isremotenode,
	$wa_attrvalue, $erole,
	$reserved_pid, $reserved_eid) = $result->fetchrow_array) {
    if (defined($uuid) && $uuid ne "")
    {
	$nodetouuid{$node} = $uuid;
    }
    my $islocal = $role eq 'testnode'
	&& ((! defined($isremotenode) || $isremotenode == 0)
	    || (defined($wa_attrvalue) && $wa_attrvalue == 1));
    my $isup = defined($eventstate)
	&& ($eventstate eq TBDB_NODESTATE_ISUP
	    || $eventstate eq TBDB_NODESTATE_PXEWAIT
	    || $eventstate eq TBDB_NODESTATE_POWEROFF
	    || $eventstate eq TBDB_NODESTATE_ALWAYSUP);
    my $isshared = (defined($erole)
		    && $erole eq "sharedhost"
		    && $useshared
		    && $isup);
    my $isblockstore = (defined($erole)
			&& $erole eq "storagehost"
			&& $useblockstore
			&& $isup);
    my $isreserved = (defined($reserved_eid)
		      || (defined($reserved_pid)
			  && (! defined($pid) || $pid ne $reserved_pid)));
    my $isfree = (!$islocal
		  || (! $isreserved && $isup)
		  || $isshared || $isblockstore);
    $nodetoavailable{$node} = $isfree;
}

# Read class/type maps
$result =
    DBQueryFatal("select class,type,isvirtnode from node_types");

while (my ($class,$type,$isvirt) = $result->fetchrow_array) {
    $map = {};
    $map->{'CLASS'}    = $class;
    $map->{'ISVIRT'}   = $isvirt;
    my $typeinfo       = NodeType->Lookup($type);
    $map->{'DELAYCAP'} = $typeinfo->delay_capacity();
    $map->{'VIRTCAP'}  = $typeinfo->virtnode_capacity();
    $map->{'SIMCAP'}   = $typeinfo->simnode_capacity();
    $map->{'SPEED'}    = $typeinfo->frequency();
    $map->{'RAM'}      = $typeinfo->memory();
    $map->{'OSID'}     = $typeinfo->default_osid();
    $map->{'IMAGEABLE'}= $typeinfo->imageable();
    $map->{'TRIVSPEED'}= $typeinfo->trivlink_maxspeed();
    $map->{'SHARED'}   = $typeinfo->shared();
    $map->{'TYPEINFO'} = $typeinfo;
    $map->{'FEATURES'} = [];
    $map->{'AUXTYPES'} = {};
    $map->{'OSLIST'} = [];
    $typemap{$type} = $map;

    # Default is permission to use all types,classes. 
    $permissions{$class} = 1;
    $permissions{$type}  = 1;
}

# Figure out which nodes will use a USB dongle to boot. As a proxy, we
# look for the pxe_boot_path pointing to the tpm version of grub.
our %node_usb;

$result = DBQueryFatal("select node_id from node_attributes ".
		       "where attrkey='pxe_boot_path' and ".
		       "      attrvalue='/tftpboot/pxeboot_tpm'");
while (($node) = $result->fetchrow_array) {
    $node_usb{$node} = 1;
}

#
# Physical RAM overrides.
#
my %node_ram;

$result = DBQueryFatal("select node_id,attrvalue from node_attributes ".
		       "where attrkey='physical_ram'");
while (my ($nodeid,$ram) = $result->fetchrow_array) {
    $node_ram{$nodeid} = $ram;
}

# Read node_startloc
$result = DBQueryFatal("select node_id,building from node_startloc");
while (($node,$building) = $result->fetchrow_array) {
    # TODO: This screws up the meaning of the @features list. Now it
    # is not just a list of feature strings, but a list of strings
    # which might themselves be space-delimited lists of feature
    # strings. Fix this.
    $areamap{$node} .= " area-$building:0";
}

#
# Read the features table for each type.
# 
$result = DBQueryFatal("select type, feature, weight from node_type_features");
while (my ($type, $feature, $weight) = $result->fetchrow()) {
    push(@{$typemap{$type}->{'FEATURES'}}, "$feature:$weight");
}

#
# Read the auxtypes for each type.
# 
$result = DBQueryFatal("select auxtype,type from node_types_auxtypes");
while (my ($auxtype,$type) = $result->fetchrow()) {
    $typemap{$type}->{'AUXTYPES'}->{$auxtype} = 1;
    $auxtypemap{$auxtype} = $type;
}

#
# Read the features table for each individual node
#
$result = DBQueryFatal("select node_id, feature, weight from node_features");
while (my ($node_id, $feature, $weight) = $result->fetchrow()) {
    if (! defined($features{$node_id})) {
	$features{$node_id} = ["$feature:$weight"];
	next;
    } else {
	push @{$features{$node_id}}, "$feature:$weight";
    }
}

#
# Read in the node_auxtypes table for each node.
#
$result = DBQueryFatal("select node_id, type, count from node_auxtypes");
while (my ($node_id, $type, $count) = $result->fetchrow()) {
    if (! defined($auxtypes{$node_id})) {
	$auxtypes{$node_id} = ["$type:$count"];
	next;
    }
    push @{$auxtypes{$node_id}}, "$type:$count";
}
#
# Read the permission table if given a pid. If there is an entry in
# the table for a type/class, then permission is granted only if there
# is a record with the pid. If not, remove the permission granted above.
# 
if (defined($pid)) {
    $result =
	DBQueryFatal("select type from nodetypeXpid_permissions");
    
    while (my ($type) = $result->fetchrow_array) {
	$permissions{$type} = 0;
    }
    
    $result =
	DBQueryFatal("select type from nodetypeXpid_permissions ".
		     "where pid='$pid'");
    
    while (my ($type) = $result->fetchrow_array) {
	$permissions{$type} = 1;
    }
}

#
# Read the table of which image types are supported on which hardware - we
# limit this to global images and ones that match the PID (if given) We do this
# limiting for two reasons:
# 1) To avoid an explosion in the number of features for nodes
# 2) To avoid information leaks, allowing projects to see each other's images
# 
my $osidquery = "select distinct o.osid, oi.type, o.osname, o.pid, o.OS, o.version, o.description,o.protogeni_export, o.osfeatures from os_info as o " .
	"left join osidtoimageid as oi on o.osid = oi.osid " .
        "left join images as i on oi.imageid = i.imageid ";
if ($pid) {
    $osidquery .= "left join image_permissions as p1 on p1.imageid=i.imageid and p1.permission_type='group' ".
	"left join groups as g on p1.permission_idx=g.gid_idx ";
}
$osidquery .= "where i.global = 1 ";
if ($pid) {
    $osidquery .= " or i.pid='$pid' ".
	" or (g.pid is not null and g.pid='$pid')";
}

my $defaultosidquery = 'select distinct o.osid, t.type, o.osname, o.pid, o.OS, o.version, o.description, o.protogeni_export, o.osfeatures '.
    'from os_info as o left join node_type_attributes as t '.
    'on (o.osid=t.attrvalue) where t.attrkey="default_osid"';

#
# For subOS support (i.e., vnode OSes running atop vhost OSes), we have to
# check both the subosid and all the parent_osid it can run on.
#
my $subosidquery = "select distinct o.osid,o.parent_osid from os_submap as o " .
	"left join osidtoimageid as oi1 on o.osid = oi1.osid " .
	"left join osidtoimageid as oi2 on o.parent_osid = oi2.osid " .
	"left join images as i1 on oi1.imageid = i1.imageid ".
	"left join images as i2 on oi2.imageid = i2.imageid ".
	"where (i1.imageid is null or i1.global = 1";
if ($pid) {
    $subosidquery .= " or i1.pid='$pid'";
}
$subosidquery .= ") and (i2.global = 1";
if ($pid) {
    $subosidquery .= " or i2.pid='$pid'";
}
$subosidquery .= ")";

our %node_type_osids;
our %osid_node_types;
our %osid_subosids;
our %osid_name;
our %osid_pid;
our %osid_os;
our %osid_version;
our %osid_description;
our %osid_avoid_usb;
our %node_countries;
our %node_latitudes;
our %node_longitudes;

our $openvzid;

$result = DBQueryFatal($osidquery);
processOs($result);

$result = DBQueryFatal($defaultosidquery);
processOs($result);

sub processOs
{
    my $result = shift(@_);
    while (my ($osid,$type,$osname,$ospid,$osos, $osversion,
	       $osdescription,$geni,$osfeatures) = $result->fetchrow()) {
	if ($osname eq "OPENVZ-STD") {
	    $openvzid = $osid;
	}
	if ($typemap{$type}) {
	    my $default = $typemap{$type}->{'OSID'};
	    if ($geni eq 1 ||
		(defined($default) && $default eq $osid)) {
		push(@{ $typemap{$type}->{'OSLIST'} }, $osid);
		$osid_name{$osid} = $osname;
		$osid_pid{$osid} = $ospid;
		$osid_os{$osid} = $osos;
		$osid_version{$osid} = $osversion;
		$osid_description{$osid} = $osdescription;
	    }
	}
	if ($node_type_osids{$type}) {
	    push @{$node_type_osids{$type}}, $osid;
	} else {
	    $node_type_osids{$type} = [$osid];
	}
	if (defined($osfeatures) && $osfeatures =~ /no-usb-boot/) {
	    $osid_avoid_usb{$osid} = 1;
	}
	#
	# We have to maintain a data structure telling us which types an OSID could
	# be on for use below with generic OSIDs
	#
	if ($osid_node_types{$osid}) {
	    push @{$osid_node_types{$osid}}, $type;
	} else {
	    $osid_node_types{$osid} = [$type];
	}
    }
}
#
# XXX Note below that for now, subosids cannot redirect to other subosids.
#
$result = DBQueryFatal($subosidquery);
while (my ($subosid,$osid) = $result->fetchrow()) {
    if ($osid_subosids{$osid}) {
	push @{$osid_subosids{$osid}}, $subosid;
    } else {
	$osid_subosids{$osid} = [$subosid];
    }
}

#
# We also have to resolve the 'generic' OSIDs, which use the nextosid field to
# redirect to another OSID
#
$result = DBQueryFatal("select osid,nextosid from os_info where " .
    "nextosid is not null");
while (my ($osid,$nextosid) = $result->fetchrow()) {
    if (defined($openvzid) && $osid == $openvzid) {
	$openvzid = $nextosid;
    }
    #
    # Check to see if they were allowed to use the real OSID
    #
    my $realosid = TBResolveNextOSID($osid, $pid, $exempt_eid);
    if (defined($realosid) && $osid_node_types{$realosid}) {
	foreach my $type (@{$osid_node_types{$realosid}}) {
	    push @{$node_type_osids{$type}}, $osid;
	}
    }
}

# Print switches
if (defined($switchtouse) && ! defined($component_name)) {
    # Should probably get the last four args out of the database, but I don't
    # think we ever actually use this case...
    print_switch($switchtouse,undef,undef,undef,undef,undef);
    $switches{$switchtouse} = 1;
}
elsif (! defined($component_name)) {
    $result =
	DBQueryFatal("select n.node_id,n.uuid,country,latitude,longitude, ".
		     "       na.attrvalue ".
                     "   from nodes as n ".
		     "left join widearea_nodeinfo as wn ".
                     "     on n.node_id=wn.node_id " .
		     "left join node_types as nt on ".
		     "     nt.type=n.type ".
		     "left join node_type_attributes as na on ".
		     "     na.type=n.type and ".
		     "     na.attrkey='forwarding_protocols' ".
		     "where ".
		     ($usecontrol ?
		      "role='ctrlswitch' and nt.isswitch=1" :
		      "role='testswitch' or role='widearea_switch' or ".
		      "n.type='external-switch' or ".
		      "      (role='testnodefoo' and nt.isswitch=1)"));

    while (($switch, $uuid, $country, $latitude, $longitude, $protocols) =
            $result->fetchrow_array) {
	print_switch($switch, $uuid,
		     $country, $latitude, $longitude, $protocols);
	$switches{$switch} = 1;
    }
}

#
# Get the global counts.
#
$result =
    DBQueryFatal("select phys_nodeid,count(phys_nodeid) from reserved as r ".
		 "left join nodes as n on n.node_id=r.node_id ".
		 "where n.node_id!=n.phys_nodeid ".
		 (defined($exempt_eid) ?
		  "and not (pid='$pid' and eid='$exempt_eid') " : " ") .
		 "group by phys_nodeid");
while (my ($node_id,$count) = $result->fetchrow_array) {
    $globalcounts{$node_id} = $count;
}

#
# Get the memory usage on each physical node. 
# This will be subtracted from whatever the type/node info says.
# Note that we have to exclude nodes from this experiment
# that are already mapped to the node since the user already owns that memory.
#
my %node_ramusage;

if ($virtstuff) {
    $result =
	DBQueryFatal("select n.phys_nodeid,n.reserved_memory from nodes as n ".
		     "left join reserved as r on r.node_id=n.node_id ".
		     "where n.node_id!=n.phys_nodeid ".
		     (defined($experiment) ?
		      "and r.exptidx!=" . $experiment->idx() : ""));

    while (my ($pnode,$memory) = $result->fetchrow_array()) {
	$node_ramusage{$pnode} = 0
	    if (!exists($node_ramusage{$pnode}));
	
	$node_ramusage{$pnode} += $memory;
    }

    #
    # When running xen and there is a dom0mem attribute, we need to
    # subtract that too since it is not available. But we do not know
    # for sure that the node is running XEN. Need to addres this at
    # some point.
    #
    $result = DBQueryFatal("select node_id,attrvalue from node_attributes ".
			   "where attrkey='dom0mem'");
    while (my ($pnode,$ram) = $result->fetchrow_array) {
	$node_ramusage{$pnode} = 0
	    if (!exists($node_ramusage{$pnode}));
	
	$node_ramusage{$pnode} += $memory;
    }
}

# Find available nodes.
#
# This first query deals with just local nodes. Local nodes can host
# virtnodes, according to the delay_capacity in the types table. 
#

# the ordinary free condition for a local node.
my $free_condition = "(b.node_id is null and ".
                     " (np.eventstate='" . TBDB_NODESTATE_ISUP . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_PXEWAIT . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_POWEROFF . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_ALWAYSUP . "')) ";

if (defined($pid)) {
    $free_condition = "($free_condition and ".
	"(np.reserved_pid is null or np.reserved_pid='$pid'))";
}

# if the user has specified an exempt pid/eid, 
# then view any node which is reserved to that experiment as available.
if (defined($exempt_eid)) {
    $free_condition = "($free_condition or ".
	"(b.pid='$pid' and b.eid='$exempt_eid'))"; 
}

# In shared mode, allow allocated nodes whose sharing_mode is set.
if ($useshared) {
    $free_condition = "($free_condition or ".
	"(b.node_id is not null && b.erole='sharedhost' && ".
	" np.eventstate='" . TBDB_NODESTATE_ISUP . "'))";
}

# In blockstore mode, allow allocated nodes who are storagehosts.
if ($useblockstore) {
    $free_condition = "($free_condition or ".
	"(b.node_id is not null && b.erole='storagehost' && ".
	" np.eventstate='" . TBDB_NODESTATE_ISUP . "'))";
}

# If the user wants all nodes, we consider everything to be free (this
# overrides the other possible free conditions
if ($allnodes) {
    $free_condition = "1";
}

# By default, include no wide area nodes when setting up the data structures.
#$free_condition = "($free_condition and ".
#    "(t.isremotenode=0 or ".
#    "dedicated_wa_types.attrvalue=1))";

# But if they have asked for widearea, we want this prepass to include
# these nodes.
#if ($widearea) {
#    $free_condition = "($free_condition or ".
#	"(t.isremotenode=1 and ".
#	"t.isvirtnode=0 and t.type!='pcfedphys' and ".
#	"dedicated_wa_types.attrvalue is NULL))";
#}


# In genimode exclude nodes with exclusion attribute.
if ($genimode ne $NO_GENI) {
    $free_condition = "($free_condition and ".
	"(nat1.attrvalue is null or nat1.attrvalue=0))";
}

if (defined($component_name)) {
    $free_condition = "(a.node_id = \"$component_name\")";
}

$result =
    DBQueryFatal("select a.node_id,a.type,a.phys_nodeid,t.class,t.issubnode," .
		 "a.def_boot_osid,(b.pid is not null and b.eid is not null), ".
		 "  np.reserved_pid,np.eventstate, ".
		 "  np.battery_percentage,np.uuid,b.sharing_mode, ".
		 "  ru.load_1min, ru.load_5min, ru.status_timestamp, ".
		 "  a.def_boot_osid, nat2.attrvalue, wn.country, " .
                 "  wn.latitude, wn.longitude, t.isremotenode, b.erole, ".
		 "  nat3.attrvalue, so.osfeatures ".
		 "from nodes as a ".
		 "left join reserved as b on a.node_id=b.node_id ".
		 "left join reserved as m on a.phys_nodeid=m.node_id ".
		 "left join nodes as np on a.phys_nodeid=np.node_id ".
		 "left join node_types as t on t.type=a.type ".
		 "left join node_attributes as nat1 on ".
		 "     nat1.node_id=a.node_id and ".
		 "     nat1.attrkey='protogeni_exclude' ".
		 "left join node_attributes as nat2 on ".
		 "     nat2.node_id=a.node_id and ".
		 "     nat2.attrkey='shared_weight' ".
		 "left join node_attributes as nat3 on ".
		 "     nat3.node_id=a.node_id and ".
		 "     nat3.attrkey='allowed_projects' ".
		 "left join node_rusage as ru on ru.node_id=a.node_id ".
		 "left outer join ". 
		 "  (select type,attrvalue ".
		 "   from node_type_attributes ".
		 "   where attrkey='dedicated_widearea' ".
		 "   group by type) as dedicated_wa_types ".
		 "  on t.type=dedicated_wa_types.type ".
                 "left join widearea_nodeinfo as wn on a.node_id=wn.node_id ".
		 "left join os_info as so on a.def_boot_osid=so.osid ".
		 "where $free_condition and ".
		 "   (a.role='testnode' and (t.isremotenode=0 or ".
		 "                           dedicated_wa_types.attrvalue=1))");

%storage_node = ();

#
# Scan the results, checking permissions and adding to the list
# You get to use a node type if no pid was specified (that is, you get
# to use all nodes), or if there is no entry in the perms table for
# the type/class of node.
#
while (($node,$type,$physnode,$class,$issubnode,$def_boot_osid,$reserved,
	$prereserved,$eventstate,$battery_perc,$uuid,$sharing_mode,
	$load_1min,$load_5min,$load_tstamp,$osid,$weight,$country,$latitude,
        $longitude,$isremote,$erole,
	$allowed_projects,$osfeatures) = $result->fetchrow_array) {
    my $current = $top->CreateNode($node);
    $current->SetSubnode($physnode, $issubnode);
    $current->SetRemote($isremote);

    #
    # Look for a specific node restriction. Easier to to do here
    # then trying to do this up above in the already bizarre query.
    #
    if (defined($pid) && defined($allowed_projects)) {
	my @allowed = split(",", $allowed_projects);
	next
	    if (! grep {$_ eq $pid} @allowed);	
    }

    $nodes{$node} = $type
	if (!defined($pid) ||
	    ($permissions{$type} && $permissions{$class}));
    $node_def_boot_osid{$node} = $def_boot_osid;

    if ($reserved) {
	$is_reserved{$node} = 1;
    } else {
	$is_reserved{$node} = 0;
    }
    if ($useshared && $sharing_mode) {
	$sharing_mode{$node} = { "load_1min"   => $load_1min,
				 "load_5min"   => $load_5min,
				 "load_tstamp" => $load_tstamp,
				 "osid"        => $osid,
				 "osfeatures"  => $osfeatures,
				 "weight"      => $weight };
    } else {
	$sharing_mode{$node} = 0;
    }

    if (defined($erole) && $erole eq "storagehost") {
	$storage_node{$node} = 1;
    }

    if (defined($prereserved) && !(defined($pid) && $prereserved eq $pid)) {
	$is_prereserved{$node} = $prereserved;
    }

    $curr_state{$node} = $eventstate;
    $curr_batt{$node} = $battery_perc;

    $node_countries{$node} = $country;
    $node_latitudes{$node} = $latitude;
    $node_longitudes{$node} = $longitude;
}

#
# Find out which nodes are connected to which, so that we can add some special
# features
#
$result = DBQueryFatal("SELECT DISTINCT node_id1, node_id2 " .
                       "  FROM wires where logical=0 and type!='Unused'");
my %connections = ();