ptopgen.in

#!/usr/bin/perl -w

#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# All rights reserved.
#

use English;
use Getopt::Std;
use Math::BigInt;
use List::Util 'shuffle';

use lib "@prefix@/lib";
use libdb qw(TBGetSiteVar);
use libadminctrl;
use libptop;
use EmulabFeatures;

my @SAVEARGV = @ARGV;

my $top = libptop->Create();

my $PGENISUPPORT = @PROTOGENI_SUPPORT@;

$NO_GENI = "0";
$V_0_1 = "0.1";
$V_0_2 = "0.2";
$V_2 = "2";
$V_3 = "3";

$emulabns = "http://www.protogeni.net/resources/rspec/ext/emulab/1";
# XXX: This needs to be changed
$emulaburl = "http://www.protogeni.net/resources/rspec/ext/emulab/1/ptop_extension.xsd";
$sharedns = "http://www.geni.net/resources/rspec/ext/shared-vlan/1";
$sharedurl = "http://www.geni.net/resources/rspec/ext/shared-vlan/1/ad.xsd";
$stitchns = "http://hpn.east.isi.edu/rspec/ext/stitch/0.1/";
$stitchurl = "http://hpn.east.isi.edu/rspec/ext/stitch/0.1/stitch-schema.xsd";

if ($PGENISUPPORT) {
  require GeniHRN;
}

sub usage()
{
    print("Usage: ptopgen [-v] [-s switch] [-p pid [-e eid]] [-m factor] " .
	  "[-n c/e] [-x] [-g (0.1 | 2)] [-c component-name]".
	  "[-l type-limit-file] [-z]\n" .
	  "       -p include nodes the project has permission to use\n".
	  "       -e include given experiments resources\n" .
	  "          in the ptopfile (as if they were free)\n" .
	  "       -v Include stuff for topologies with virtual nodes\n".
	  "       -r Include stuff for topologies with widearea nodes\n".
#	  "       -s Include stuff for topologies with simulated nodes\n".
	  "       -h Include stuff for topologies with shared nodes\n".
	  "       -a Include even reserved nodes\n".
	  "       -m Override multiplex_factor\n".
	  "       -u Prune unused interfaces of allocated nodes (-e)\n".
	  "       -c Delay capacity override\n".
	  "       -n Add in modelnet core and edge node features\n".
	  "       -x Output into the new xml ptop format\n".
	  "       -g version With -x, geni version. Must be '0', '0.1' or '2'\n".
	  "       -l specifies the location of the type limit file\n" .
          "       -1 Print an rspec containing only the node component-name\n" .
	  "       -z Randomize node output order\n" .
	  "       -Z Force old ptopgen\n");
        exit(-1);
}

my $optlist = "s:e:m:vp:rSan:c:uxg:h1:l:zZ";
my $mfactor;
my $virtstuff = 0;
my $widearea  = 0;
my $simstuff  = 0;
my $allnodes  = 0;
my $mnetcores = 0;
my $mnetedges = 0;
my $prune     = 0;
my $do_xml    = 0;
my $genimode  = $NO_GENI;
my $useshared = 0;
my $component_name = undef;
my $randomize = 0;

my $OURDOMAIN = "@OURDOMAIN@";
my $MAINSITE  = @TBMAINSITE@;
my $cmuuid = TBGetSiteVar('protogeni/cm_uuid');
my $cmurn = "";
if ($PGENISUPPORT) {
    $cmurn = GeniHRN::Generate($OURDOMAIN, "authority", "cm");
}

my $default_long = undef;
TBGetSiteVar('general/default_longitude', \$default_long);
my $default_lat = undef;
TBGetSiteVar('general/default_latitude', \$default_lat);
my $default_country = undef;
TBGetSiteVar('general/default_country', \$default_country);

my $delaycap_override;

#
# Turn off line buffering on output
#
$| = 1;

# Settings - Change these to your particular network.

#
# Also bogus. The interfaces table does not hold entries for the switch
# side of each link. It will have to at some point, when we have something
# other than 100Mbs ports (say, gbit ports).
#
# Speed in in Kbs!
#
my $default_switchport_speed    = 100000;

#
# Yet more bogosity - we hardcode this weight, which is given to wide-area
# (primarily plab) nodes to prefer spreading across sites
#
my $site_weight = 0.99;

######################################################################

my $TBROOT = "@prefix@";
use lib '@prefix@/lib';
require exitonwarn;
use libdb;
use libtblog;
use Experiment;
use Node;
use NodeType;
use Lan;

tblog_stop_capture('stdout');

my $TRACK_INTERSWITCH_BANDWIDTH = "@TRACK_INTERSWITCH_BANDWIDTH@";

my %switches	  = ();
my %permissions   = ();
my %typemap       = ();
my %auxtypemap    = ();
my %areamap       = ();
my %globalcounts  = ();
my %interfacestate= ();
my %vinterfaces   = ();
my %rusagedata    = ();

my $DEADPID = NODEDEAD_PID();
my $DEADEID = NODEDEAD_EID();

my $pid;
my $exempt_eid;
my $switchtouse;
my $experiment;

my $typelimitfile = "";

sub fatal($);

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV) {
    usage();
}
if (defined($options{"s"})) {
    $switchtouse = $options{"s"};
}
if (defined($options{"m"})) {
    $mfactor = $options{"m"};
}
if (defined($options{"h"})) {
    $useshared = 1;
}
if (defined($options{"v"})) {
    $virtstuff = 1;
}
if (defined($options{"r"})) {
    $widearea = 1;
}
if (defined($options{"S"})) {
    $simstuff = 1;
}
if (defined($options{"p"})) {
    $pid = $options{"p"};
}
if (defined($options{"a"})) {
    $allnodes = 1;
}
if (defined($options{"u"})) {
    $prune = 1;
}
if (defined($options{"c"})) {
    $delaycap_override = $options{"c"};
}
if (defined($options{"n"})) {
    if ($options{"n"} =~ /(\d*),(\d*)/) {
	$mnetcores = $1;
	$mnetedges = $2;
    }
    else {
	usage();
    }
}
if (defined($options{"e"})) {
    $exempt_eid = $options{"e"};
    usage()
	if (!defined($pid));
    $experiment = Experiment->Lookup($pid, $exempt_eid);
    die("Could not look up experiment $pid,$exempt_eid\n")
	if (!defined($experiment));
}
if (defined($options{"x"})) {
    if (defined($options{"g"}) && $PGENISUPPORT) {
	$useshared = 1;
	$virtstuff = 1;
	$genimode = $options{"g"};
    }
    $do_xml = 1;
}
if (defined($options{"l"})) {
    $typelimitfile = $options{"l"};
}
if (defined($options{"1"})) {
    $component_name = $options{"1"};
}
if (defined($options{"z"})) {
    $randomize = 1;
}

usage()
    if ($prune && !defined($exempt_eid)
	|| ($genimode ne $NO_GENI
	    && $genimode ne $V_0_1
	    && $genimode ne $V_0_2
	    && $genimode ne $V_2
	    && $genimode ne $V_3));

if (defined($pid) && ! defined($options{"Z"})) {
    my $group = Group->Lookup($pid, $pid);
    if (defined($group)) {
	$EmulabFeatures::verbose = 0;
	my $newptopgen = EmulabFeatures->FeatureEnabled("NewPtopgen", undef,
							$group, undef);
	if ($newptopgen) {
	    my $newpath = "$TBROOT/libexec/ptopgen_new";
	    print STDERR "Invoking new ptopgen from $newpath\n"
		if (!$genimode);
	    exec $newpath, @SAVEARGV;
	    die("*** $0:\n".
		"    Could not exec $newpath: $!");
	}
    }
}

$fake_inet_switch = "internet";
$fake_inet_iface = "(null)";
$fake_air_switch = "airswitch";
$fake_air_iface = "(null)";
if ($genimode ne $NO_GENI) {
    $fake_inet_iface = "border";
    $fake_air_iface = "air";
}

print_header();

my %nodetointerface;
my %interfaceroles = ();
my %interfacetypes = ();
my %interfaceips = ();

# Read interfaces
my $result =
    DBQueryFatal("SELECT node_id,card,port,iface,interface_type,role,IP" .
		 " from interfaces where logical=0 " .
		 ($genimode ne $NO_GENI ? "and role!='gw'" : ""));
		 
while (($node,$card,$port,$iface,$type,$role,$ip) = $result->fetchrow_array) {
    push @{ $nodetointerface{"$node"} }, $iface;
    $interfacemap{"$node:$card:$port"} = $iface;
    if ((defined $type) && ($type ne "")) {
	$interfacetypes{"$node:$card:$port"} = $type;
    }
    if ((defined $role) && ($role ne "")) {
	$interfaceroles{"$node:$iface"} = $role;
    }
    if ((defined $ip) && ($ip ne "")) {
	$interfaceips{"$node:$iface"} = $ip;
    }
}

my %external_nodes;
my %external_ifaces;
my %contact_nodes;
my %contact_ifaces;
my %contact_links;

$result = DBQueryFatal("select w.node_id1, w.card1, w.port1, i1.iface, ".
		       "w.node_id2, w.card2, w.port2, i2.iface, ".
		       "e.node_id, e.min_vlan, e.max_vlan ".
		       "from wires as w ".
		       "left join external_networks as e ".
		       "on w.node_id1=e.node_id or w.node_id2=e.node_id ".
		       "left join interfaces as i1 on w.node_id1=i1.node_id and w.card1=i1.card and w.port1=i1.port ".
		       "left join interfaces as i2 on w.node_id2=i2.node_id and w.card2=i2.card and w.port2=i2.port ".
		       "where e.node_id is not null");
while (my ($cnode, $ccard, $cport, $ciface, $enode, $ecard, $eport, $eiface,
	   $external, $minLan, $maxLan) = $result->fetchrow_array) {
    if ($external eq $cnode) {
	my $temp;
	$temp = $cnode; $cnode = $enode; $enode = $temp;
	$temp = $ccard; $ccard = $ecard; $ecard = $temp;
	$temp = $cport; $cport = $eport; $eport = $temp;
	$temp = $ciface; $ciface = $eiface; $eiface = $temp;
    }
    if (! defined($eiface)) {
	$eiface = "$ecard.$eport";
    }
    if (! defined($ciface)) {
	$ciface = "$ccard.$cport";
    }
    $external_nodes{$enode} = "$minLan-$maxLan";
    $external_ifaces{"$cnode:$enode"} = $eiface;
    $contact_nodes{$cnode} = $enode;
    $contact_ifaces{"$cnode:$enode"} = $ciface;
}

my %nodetouuid;
my %nodetoavailable;

$result = DBQueryFatal("SELECT n.node_id, n.eventstate, n.role, n.uuid, " .
		       "nt.isremotenode, " .
		       "dedicated_wa_types.attrvalue, b.erole, " .
		       "n.reserved_pid, b.eid " .
		       "from nodes as n " .
		       "left join reserved as b on n.node_id=b.node_id " .
		       "left join node_types as nt on nt.type=n.type " .
		       "left outer join " .
		       "  (select type, attrvalue " .
		       "   from node_type_attributes " .
		       "   where attrkey='dedicated_widearea' " .
		       "   group by type) as dedicated_wa_types " .
		       "  on nt.type=dedicated_wa_types.type " .
		       "where nt.isvirtnode = 0 or nt.isvirtnode is null;");
while (($node,$eventstate, $role, $uuid, $isremotenode,
	$wa_attrvalue, $erole,
	$reserved_pid, $reserved_eid) = $result->fetchrow_array) {
    if (defined($uuid) && $uuid ne "")
    {
	$nodetouuid{$node} = $uuid;
    }
    my $islocal = $role eq 'testnode'
	&& ((! defined($isremotenode) || $isremotenode == 0)
	    || (defined($wa_attrvalue) && $wa_attrvalue == 1));
    my $isup = defined($eventstate)
	&& ($eventstate eq TBDB_NODESTATE_ISUP
	    || $eventstate eq TBDB_NODESTATE_PXEWAIT
	    || $eventstate eq TBDB_NODESTATE_POWEROFF
	    || $eventstate eq TBDB_NODESTATE_ALWAYSUP);
    my $isshared = (defined($erole)
		    && $erole eq "sharedhost"
		    && $useshared
		    && $isup);
    my $isreserved = (defined($reserved_eid)
		      || (defined($reserved_pid)
			  && (! defined($pid) || $pid ne $reserved_pid)));
    my $isfree = (!$islocal
		  || (! $isreserved && $isup)
		  || $isshared);
    $nodetoavailable{$node} = $isfree;
}

# Read class/type maps
$result =
    DBQueryFatal("select class,type,isvirtnode from node_types");

while (my ($class,$type,$isvirt) = $result->fetchrow_array) {
    $map = {};
    $map->{'CLASS'}    = $class;
    $map->{'ISVIRT'}   = $isvirt;
    my $typeinfo       = NodeType->Lookup($type);
    $map->{'DELAYCAP'} = $typeinfo->delay_capacity();
    $map->{'VIRTCAP'}  = $typeinfo->virtnode_capacity();
    $map->{'SIMCAP'}   = $typeinfo->simnode_capacity();
    $map->{'SPEED'}    = $typeinfo->frequency();
    $map->{'RAM'}      = $typeinfo->memory();
    $map->{'OSID'}     = $typeinfo->default_osid();
    $map->{'IMAGEABLE'}= $typeinfo->imageable();
    $map->{'TRIVSPEED'}= $typeinfo->trivlink_maxspeed();
    $map->{'SHARED'}   = $typeinfo->shared();
    $map->{'TYPEINFO'} = $typeinfo;
    $map->{'FEATURES'} = [];
    $map->{'AUXTYPES'} = {};
    $map->{'OSLIST'} = [];
    $typemap{$type} = $map;

    # Default is permission to use all types,classes. 
    $permissions{$class} = 1;
    $permissions{$type}  = 1;
}

# Read node_startloc
$result = DBQueryFatal("select node_id,building from node_startloc");
while (($node,$building) = $result->fetchrow_array) {
    # TODO: This screws up the meaning of the @features list. Now it
    # is not just a list of feature strings, but a list of strings
    # which might themselves be space-delimited lists of feature
    # strings. Fix this.
    $areamap{$node} .= " area-$building:0";
}

#
# Read the features table for each type.
# 
$result = DBQueryFatal("select type, feature, weight from node_type_features");
while (my ($type, $feature, $weight) = $result->fetchrow()) {
    push(@{$typemap{$type}->{'FEATURES'}}, "$feature:$weight");
}

#
# Read the auxtypes for each type.
# 
$result = DBQueryFatal("select auxtype,type from node_types_auxtypes");
while (my ($auxtype,$type) = $result->fetchrow()) {
    $typemap{$type}->{'AUXTYPES'}->{$auxtype} = 1;
    $auxtypemap{$auxtype} = $type;
}

#
# Read the features table for each individual node
#
$result = DBQueryFatal("select node_id, feature, weight from node_features");
while (my ($node_id, $feature, $weight) = $result->fetchrow()) {
    if (! defined($features{$node_id})) {
	$features{$node_id} = ["$feature:$weight"];
	next;
    } else {
	push @{$features{$node_id}}, "$feature:$weight";
    }
}
#
# Read in the node_auxtypes table for each node.
#
$result = DBQueryFatal("select node_id, type, count from node_auxtypes");
while (my ($node_id, $type, $count) = $result->fetchrow()) {
    if (! defined($auxtypes{$node_id})) {
	$auxtypes{$node_id} = ["$type:$count"];
	next;
    }
    push @{$auxtypes{$node_id}}, "$type:$count";
}
#
# Read the permission table if given a pid. If there is an entry in
# the table for a type/class, then permission is granted only if there
# is a record with the pid. If not, remove the permission granted above.
# 
if (defined($pid)) {
    $result =
	DBQueryFatal("select type from nodetypeXpid_permissions");
    
    while (my ($type) = $result->fetchrow_array) {
	$permissions{$type} = 0;
    }
    
    $result =
	DBQueryFatal("select type from nodetypeXpid_permissions ".
		     "where pid='$pid'");
    
    while (my ($type) = $result->fetchrow_array) {
	$permissions{$type} = 1;
    }
}

#
# Read the table of which image types are supported on which hardware - we
# limit this to global images and ones that match the PID (if given) We do this
# limiting for two reasons:
# 1) To avoid an explosion in the number of features for nodes
# 2) To avoid information leaks, allowing projects to see each other's images
# 
my $osidquery = "select distinct o.osid, oi.type, o.osname, o.pid, o.OS, o.version, o.description,o.protogeni_export from os_info as o " .
	"left join osidtoimageid as oi on o.osid = oi.osid " .
        "left join images as i on oi.imageid = i.imageid ";
if ($pid) {
    $osidquery .= "left join image_permissions as p1 on p1.imageid=i.imageid and p1.permission_type='group' ".
	"left join groups as g on p1.permission_idx=g.gid_idx ";
}
$osidquery .= "where i.global = 1 ";
if ($pid) {
    $osidquery .= " or i.pid='$pid' ".
	" or (g.pid is not null and g.pid='$pid')";
}

my $defaultosidquery = 'select distinct o.osid, t.type, o.osname, o.pid, o.OS, o.version, o.description, o.protogeni_export '.
    'from os_info as o left join node_type_attributes as t '.
    'on (o.osid=t.attrvalue) where t.attrkey="default_osid"';

#
# For subOS support (i.e., vnode OSes running atop vhost OSes), we have to
# check both the subosid and all the parent_osid it can run on.
#
my $subosidquery = "select distinct o.osid,o.parent_osid from os_submap as o " .
	"left join osidtoimageid as oi1 on o.osid = oi1.osid " .
	"left join osidtoimageid as oi2 on o.parent_osid = oi2.osid " .
	"left join images as i1 on oi1.imageid = i1.imageid ".
	"left join images as i2 on oi2.imageid = i2.imageid ".
	"where (i1.imageid is null or i1.global = 1";
if ($pid) {
    $subosidquery .= " or i1.pid='$pid'";
}
$subosidquery .= ") and (i2.global = 1";
if ($pid) {
    $subosidquery .= " or i2.pid='$pid'";
}
$subosidquery .= ")";

our %node_type_osids;
our %osid_node_types;
our %osid_subosids;
our %osid_name;
our %osid_pid;
our %osid_os;
our %osid_version;
our %osid_description;
our %node_countries;
our %node_latitudes;
our %node_longitudes;

$result = DBQueryFatal($osidquery);
processOs($result);

$result = DBQueryFatal($defaultosidquery);
processOs($result);

sub processOs
{
    my $result = shift(@_);
    while (my ($osid,$type,$osname,$ospid,$osos,
	       $osversion,$osdescription,$geni) = $result->fetchrow()) {
	if ($typemap{$type}) {
	    my $default = $typemap{$type}->{'OSID'};
	    if ($geni eq 1 ||
		(defined($default) && $default eq $osid)) {
		push(@{ $typemap{$type}->{'OSLIST'} }, $osid);
		$osid_name{$osid} = $osname;
		$osid_pid{$osid} = $ospid;
		$osid_os{$osid} = $osos;
		$osid_version{$osid} = $osversion;
		$osid_description{$osid} = $osdescription;
	    }
	}
	if ($node_type_osids{$type}) {
	    push @{$node_type_osids{$type}}, $osid;
	} else {
	    $node_type_osids{$type} = [$osid];
	}
	#
	# We have to maintain a data structure telling us which types an OSID could
	# be on for use below with generic OSIDs
	#
	if ($osid_node_types{$osid}) {
	    push @{$osid_node_types{$osid}}, $type;
	} else {
	    $osid_node_types{$osid} = [$type];
	}
    }
}
#
# XXX Note below that for now, subosids cannot redirect to other subosids.
#
$result = DBQueryFatal($subosidquery);
while (my ($subosid,$osid) = $result->fetchrow()) {
    if ($osid_subosids{$osid}) {
	push @{$osid_subosids{$osid}}, $subosid;
    } else {
	$osid_subosids{$osid} = [$subosid];
    }
}

#
# We also have to resolve the 'generic' OSIDs, which use the nextosid field to
# redirect to another OSID
#
$result = DBQueryFatal("select osid from os_info where " .
    "nextosid is not null");
while (my ($osid) = $result->fetchrow()) {
    #
    # Check to see if they were allowed to use the real OSID
    #
    my $realosid = TBResolveNextOSID($osid, $pid, $exempt_eid);
    if (defined($realosid) && $osid_node_types{$realosid}) {
	foreach my $type (@{$osid_node_types{$realosid}}) {
	    push @{$node_type_osids{$type}}, $osid;
	}
    }
}

# Print switches
if (defined($switchtouse) && ! defined($component_name)) {
    # Should probably get the last four args out of the database, but I don't
    # think we ever actually use this case...
    print_switch($switchtouse,undef,undef,undef,undef);
    $switches{$switchtouse} = 1;
}
elsif (! defined($component_name)) {
    $result =
	DBQueryFatal("select n.node_id,n.uuid,country,latitude,longitude ".
                     "   from nodes as n ".
		     "left join widearea_nodeinfo as wn ".
                     "     on n.node_id=wn.node_id " .
		     "left join node_types as nt on ".
		     "     nt.type=n.type ".
		     "where role='testswitch' or role='widearea_switch' or ".
		     "      (role='testnodefoo' and nt.isswitch=1)");

    while (($switch, $uuid, $country, $latitude, $longitude) =
            $result->fetchrow_array) {
	print_switch($switch, $uuid, $country, $latitude, $longitude);
	$switches{$switch} = 1;
    }
}

#
# Get the global counts.
#
$result =
    DBQueryFatal("select phys_nodeid,count(phys_nodeid) from reserved as r ".
		 "left join nodes as n on n.node_id=r.node_id ".
		 "where n.node_id!=n.phys_nodeid ".
		 (defined($exempt_eid) ?
		  "and not (pid='$pid' and eid='$exempt_eid') " : " ") .
		 "group by phys_nodeid");
while (my ($node_id,$count) = $result->fetchrow_array) {
    $globalcounts{$node_id} = $count;
}

# Find available nodes.
#
# This first query deals with just local nodes. Local nodes can host
# virtnodes, according to the delay_capacity in the types table. 
#

# the ordinary free condition for a local node.
my $free_condition = "(b.node_id is null and ".
                     " (np.eventstate='" . TBDB_NODESTATE_ISUP . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_PXEWAIT . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_POWEROFF . "' or ".
                     "  np.eventstate='" . TBDB_NODESTATE_ALWAYSUP . "')) ";

if (defined($pid)) {
    $free_condition = "($free_condition and ".
	"(np.reserved_pid is null or np.reserved_pid='$pid'))";
}

# if the user has specified an exempt pid/eid, 
# then view any node which is reserved to that experiment as available.
if (defined($exempt_eid)) {
    $free_condition = "($free_condition or ".
	"(b.pid='$pid' and b.eid='$exempt_eid'))"; 
}

# In shared mode, allow allocated nodes whose sharing_mode is set.
if ($useshared) {
    $free_condition = "($free_condition or ".
	"(b.node_id is not null && b.erole='sharedhost' && ".
	" np.eventstate='" . TBDB_NODESTATE_ISUP . "'))";
}

# If the user wants all nodes, we consider everything to be free (this
# overrides the other possible free conditions
if ($allnodes) {
    $free_condition = "1";
}

# By default, include no wide area nodes when setting up the data structures.
#$free_condition = "($free_condition and ".
#    "(t.isremotenode=0 or ".
#    "dedicated_wa_types.attrvalue=1))";

# But if they have asked for widearea, we want this prepass to include
# these nodes.
#if ($widearea) {
#    $free_condition = "($free_condition or ".
#	"(t.isremotenode=1 and ".
#	"t.isvirtnode=0 and t.type!='pcfedphys' and ".
#	"dedicated_wa_types.attrvalue is NULL))";
#}


# In genimode exclude nodes with exclusion attribute.
if ($genimode ne $NO_GENI) {
    $free_condition = "($free_condition and ".
	"(nat1.attrvalue is null or nat1.attrvalue=0))";
}

if (defined($component_name)) {
    $free_condition = "(a.node_id = \"$component_name\")";
}

$result =
    DBQueryFatal("select a.node_id,a.type,a.phys_nodeid,t.class,t.issubnode," .
		 "a.def_boot_osid,(b.pid is not null and b.eid is not null), ".
		 "  np.reserved_pid is not null,np.eventstate, ".
		 "  np.battery_percentage,np.uuid,b.sharing_mode, ".
		 "  ru.load_1min, ru.load_5min, ru.status_timestamp, ".
		 "  a.def_boot_osid, nat2.attrvalue, wn.country, " .
                 "  wn.latitude, wn.longitude, t.isremotenode ".
		 "from nodes as a ".
		 "left join reserved as b on a.node_id=b.node_id ".
		 "left join reserved as m on a.phys_nodeid=m.node_id ".
		 "left join nodes as np on a.phys_nodeid=np.node_id ".
		 "left join node_types as t on t.type=a.type ".
		 "left join node_attributes as nat1 on ".
		 "     nat1.node_id=a.node_id and ".
		 "     nat1.attrkey='protogeni_exclude' ".
		 "left join node_attributes as nat2 on ".
		 "     nat2.node_id=a.node_id and ".
		 "     nat2.attrkey='shared_weight' ".
		 "left join node_rusage as ru on ru.node_id=a.node_id ".
		 "left outer join ". 
		 "  (select type,attrvalue ".
		 "   from node_type_attributes ".
		 "   where attrkey='dedicated_widearea' ".
		 "   group by type) as dedicated_wa_types ".
		 "  on t.type=dedicated_wa_types.type ".
                 "left join widearea_nodeinfo as wn on a.node_id=wn.node_id ".
		 "where $free_condition and ".
#		 "      a.role='testnode'"
		 "   (a.role='testnode' and (t.isremotenode=0 or ".
		 "                           dedicated_wa_types.attrvalue=1))");

#
# Scan the results, checking permissions and adding to the list
# You get to use a node type if no pid was specified (that is, you get
# to use all nodes), or if there is no entry in the perms table for
# the type/class of node.
#
while (($node,$type,$physnode,$class,$issubnode,$def_boot_osid,$reserved,
	$prereserved,$eventstate,$battery_perc,$uuid,$sharing_mode,
	$load_1min,$load_5min,$load_tstamp,$osid,$weight,$country,$latitude,
        $longitude,$isremote) = $result->fetchrow_array) {
    my $current = $top->CreateNode($node);
    $current->SetSubnode($physnode, $issubnode);
    $current->SetRemote($isremote);

#    next
#	if ($isremote);

    $nodes{$node} = $type
	if (!defined($pid) ||
	    ($permissions{$type} && $permissions{$class}));
    $node_def_boot_osid{$node} = $def_boot_osid;

    if ($reserved) {
	$is_reserved{$node} = 1;
    } else {
	$is_reserved{$node} = 0;
    }
    if ($useshared && $sharing_mode) {
	$sharing_mode{$node} = { "load_1min"   => $load_1min,
				 "load_5min"   => $load_5min,
				 "load_tstamp" => $load_tstamp,
				 "osid"        => $osid,
				 "weight"      => $weight};
    }
    else {
	$sharing_mode{$node} = 0;
    }

    $is_prereserved{$node} = $prereserved;

    $curr_state{$node} = $eventstate;
    $curr_batt{$node} = $battery_perc;

    $node_countries{$node} = $country;
    $node_latitudes{$node} = $latitude;
    $node_longitudes{$node} = $longitude;
}

#
# Find out which nodes are connected to which, so that we can add some special
# features
#
$result = DBQueryFatal("SELECT DISTINCT node_id1, node_id2 " .
                       "  FROM wires where logical=0 and type!='Unused'");
my %connections = ();
while (my ($node_id1, $node_id2) = $result->fetchrow()) {
    foreach my $nodes ([$node_id1, $node_id2], [$node_id2, $node_id1]) {
        my ($node1, $node2) = @$nodes;
        if ($connections{$node1}) {
            # Check to see if we've already added this one (possible if the
            # nodes appeared in node1,node2 order, then node2,node1
            if (!grep(/^$node2$/, @{$connections{$node1}})) {
                push @{$connections{$node1}}, $node2;
            }
        } else {
            $connections{$node1} = [$node2];
        }
    }
}

@nodenames = keys(%nodes);
if ($randomize)
{
    @nodenames = shuffle(@nodenames);
}

#
# Loop through and print out all nodes
#
foreach $node (@nodenames) {
    my $current = $top->nodes()->{$node};

    # Wide area nodes are handled below
#    next
#	if ($current->is_remote());

    my $type  = $nodes{$node};
    my $class = $typemap{$type}->{'CLASS'};
    my $delay_capacity = $typemap{$type}->{'DELAYCAP'};
    my $simnode_capacity = $typemap{$type}->{'SIMCAP'};
    my $cpu_speed = $typemap{$type}->{'SPEED'};
    my $ram = $typemap{$type}->{'RAM'};
    my $trivspeed = $typemap{$type}->{'TRIVSPEED'};
    my $uuid = $nodetouuid{$node};
    
    my @types = ("$type:1");
    my @features = ();
    my @flags;
    my $needvirtgoo = 0;

    my ($latitude, $longitude, $country);

    # XXX temporary hack until node reboot avoidance 
    # is available. Nodes running the FBSD-NSE image
    # will have a feature def-osid-fbsd-nse 0.0
    # This is used by assign to prefer these pnodes
    # first before using others.
    if($node_def_boot_osid{$node} && ($node_def_boot_osid{$node} eq 
	TBOSID(TB_OPSPID, "FBSD-NSE"))) { 
	push(@features, "FBSD-NSE:0.0");
    }

    #
    # XXX: Temporary hack - don't march switches that are testnodes
    # as having class 'switch' - assign treats those specially. We
    # use the knowledge that 'real' switches don't hit this point!
    #
    if ($class eq "switch") {
	$class = $type;
    }
        
    # Might be equal, which assign would sum as two, not one!
    if ($type ne $class) {
	push(@types, "$class:1");
    }

    if (defined($delay_capacity) && $delay_capacity > 0) {
	# Comes from the NS file;
	$delay_capacity = $delaycap_override
	    if (defined($delaycap_override) &&
		$delaycap_override > 0 &&
		$delaycap_override < $delay_capacity);
	
	push @types, "delay:$delay_capacity";
	push @types, "delay-${type}:$delay_capacity";
    }

    #
    # Prototype shared mode.
    #
    if ($sharing_mode{$node}) {
	#
	# Add a feature that says this node should not be picked
	# unless the cooresponding desire is in the vtop. This
	# allows the node to be picked, subject to other type constraints.
	#
	push(@features, "pcshared:1.0");

	#
	# Add a first-free feature which spreads VMs amongs shared nodes.
	#
	push(@features, "*&spreadshared:0.6");

	#
	# The pool daemon may override the share weight.
	#
	if (defined($sharing_mode{$node}->{"weight"})) {
	    my $weight = $sharing_mode{$node}->{"weight"};
	    
	    push(@features, "shareweight:$weight");
	}
	else {
	    #
	    # The point of this feature is to have assign favor shared nodes
	    # that already have nodes on them, so that they are well packed.
	    # Shared nodes with just a few vnodes on them are avoided so that
	    # they will free up eventually. 
	    #
	    my $maxvnodes = 10;
	    my $weight    = 0.5;
	    my $gcount    = $globalcounts{$node} || 0.0;

	    foreach my $auxinfo (@{$auxtypes{$node}}) {
		my ($auxtype,$count) = split(":", $auxinfo);
		
		if ($auxtype eq "pcvm") {
		    $maxvnodes = $count;
		    last;
		}
	    }
	    #
	    # No point in the feature if no room left. 
	    #
	    if ($maxvnodes > $gcount) {
		my $factor = ($gcount / $maxvnodes);
		if ($factor < 0.25) {
		    $weight = 0.8;
		}
		elsif ($factor > 0.75) {
		    $weight = 0.1;
		}
		else {
		    $weight = 0.3;
		}
		#push(@features, "shareweight:$weight");
	    }
	}
    }

    #
    # Add any auxiliary types
    #
    foreach my $auxinfo (@{$auxtypes{$node}}) {
	my ($auxtype,$count) = split(":", $auxinfo);
	my $realtype;

	# Map an auxtype back to its real type, unless it is a real type.
	if (defined($auxtypemap{$auxtype})) {
	    $realtype = $auxtypemap{$auxtype};
	}
	else {
	    $realtype = $auxtype;
	}

	if ($typemap{$realtype}->{'ISVIRT'} && $count > 0) {
	    next
		if (! $virtstuff);

	    #
	    # If the node is shared, must subtract the current global count
	    # from the max first, to see if there is any room left.
	    #
	    if ($sharing_mode{$node} && exists($globalcounts{$node})) {
		$count -= $globalcounts{$node};
	    }
	    if (defined($mfactor) && $mfactor <= $count) {
		$count = $mfactor;
	    }
	    $auxinfo = "$auxtype:$count";
	    $needvirtgoo = 1;

	    #
	    # Add in machine specific auxtypes that use the same count.
	    #
	    push(@types, "${type}-vm:$count");

	    # And a legacy type.
	    my $legacy_type = $type;
	    if (($legacy_type =~ s/pc/pcvm/)) {
		push(@types, "${legacy_type}:$count");
	    }
	}
	push(@types, $auxinfo);
    }

    if (defined($areamap{$node})) {
	push @features, $areamap{$node};
    }

    my $cpu_ram_features_present = 0;
    #
    # This stuff is extra goo for local virtual nodes.
    # 
    if ($needvirtgoo) {
	push @types, "*lan:*";
	# Add trivial bw spec., but only if the node type has it
	if ($trivspeed) {
	    push @flags, "trivial_bw:$trivspeed";
	}
	if (! $sharing_mode{$node}) {
	    # Add CPU and RAM information
	    $cpu_ram_features_present++;
	    # This number can be use for fine-tuning packing
	    push @features, "?+virtpercent:100";
	}
	# Put this silly feature in so that we can try to keep vnodes
	# on the same pnode they were before - but only if updating
        if ($exempt_eid) {
            push @features, "${node}:0.0";
        }
    }

    if ($simstuff && defined($simnode_capacity) && $simnode_capacity > 0) {
	#
	# Use user specified multiplex factor
	#
	if (defined($mfactor) && $mfactor <= $simnode_capacity) {
	    push @types, "sim:$mfactor";
	}
	else {
	    push @types, "sim:$simnode_capacity";
	}
	# Add CPU and RAM information
	$cpu_ram_features_present++;
	push @types, "*lan:*";
	# Add trivial bw spec.
	push @flags, "trivial_bw:100000";
    }
    if ($cpu_ram_features_present) {
	# Add CPU and RAM information
	push @features, "?+cpu:$cpu_speed"
	    if (defined($cpu_speed));
	push @features, "?+ram:$ram"
	    if (defined($ram));
	push @features, "?+cpupercent:92"; # XXX Hack
	push @features, "?+rampercent:80"; # XXX Hack
    }

    # Add features
    if (! $sharing_mode{$node}) {
	push(@features, @{$typemap{$type}->{'FEATURES'}});
	if (defined($features{$node})) {
	    push @features, @{$features{$node}};
	}
    }

    #
    # Add in OS features.
    #
    if ($sharing_mode{$node}) {
	#
	# A shared node is running just one OS, and we put that in
	# so that the user can specify which of the current VM types
	# is wanted.
	#
	my $osid = $sharing_mode{$node}->{'osid'};
	push(@features, "OS-$osid:0.5");
	# Add any subOSes the shared node osid can support
	if (defined($osid_subosids{$osid})) {
	    push (@features, map "OS-$osid-$_:0", @{$osid_subosids{$osid}});
	}
    }
    elsif ($node_type_osids{$type}) {
	#
	# Add in features for all of the OSes that this node (as
	# evidenced by its type) can support
	#
	foreach my $o1 (@{$node_type_osids{$type}}) {
	    push @features, "OS-$o1:0";
	    foreach my $o2 (@{$osid_subosids{$o1}}) {
		push @features, "OS-$o1-$o2:0";
	    }
	}
    }
    elsif (! $typemap{$type}->{'IMAGEABLE'} &&
	   defined($typemap{$type}->{'OSID'})) {
	#
	# If node is not imageable (and thus no entries in osidtoimageid,
	# then assume it always has its default OSID loaded and ready to
	# go, so that assign will agree to the allocation (assign_wrapper
	# adds a desire that says it has to be running the OSID the user
	# has selected, or the default OSID from the node_types table).
	#
	push @features, map "OS-$_:0", $typemap{$type}->{'OSID'};
    }

    # Add features indicating what this node has connections to
    if ($connections{$node} && !$sharing_mode{$node}) {
        my @connected_to  = @{$connections{$node}};
        foreach my $other_end (@connected_to) {
            push(@features,"connected-to-$other_end:0.0");
        }
    }

    # This is for the case that we are modifying an existing experiment - tell
    # assign to prefer nodes the user has already allocated
    if ($exempt_eid && !$allnodes && $is_reserved{$node}) {
	push(@features,"already_reserved:0");
    }

    if ($is_prereserved{$node}) {
	push(@features,"prereserved:0.9");
    }

    # For robots, prefer ones that are already powered on and/or won't need to
    # be recharged soon.
    if (defined($curr_state{$node}) && ($curr_state{$node} eq TBDB_NODESTATE_POWEROFF)) {
	push(@features,"poweroff:0.9");
    }
    if (defined($curr_batt{$node})) {
	push(@features,"powerneeded:" . (1.0 - $curr_batt{$node} / 100.0));
    }

    #
    # Handle subnodes
    #
    # XXX This breaks if subnodes can themselves have subnodes
    if ($current->is_subnode()) {
	# We don't want to include subnodes unless their parent node is going
	# to be in the ptop file too
	if (! exists($nodes{$current->subnode_of()}) ||
	    $sharing_mode{$current->subnode_of()} ||
	    !$nodetoavailable{$current->subnode_of()}) {
	    # In fact, nuke it from %nodes so that we don't include its links,
	    # either
	    $nodetoavailable{$node} = 0;
	    if (! $allnodes || ! exists($nodes{$current->subnode_of()})) {