GeniCM.pm.in

#!/usr/bin/perl -wT
#
# GENIPUBLIC-COPYRIGHT
# Copyright (c) 2008-2009 University of Utah and the Flux Group.
# All rights reserved.
#
package GeniCM;

#
# The server side of the CM interface on remote sites. Also communicates
# with the GMC interface at Geni Central as a client.
#
use strict;
use Exporter;
use vars qw(@ISA @EXPORT);

@ISA    = "Exporter";
@EXPORT = qw ( );

# Must come after package declaration!
use lib '@prefix@/lib';
use GeniDB;
use Genixmlrpc;
use GeniResponse;
use GeniTicket;
use GeniCredential;
use GeniCertificate;
use GeniSlice;
use GeniAggregate;
use GeniAuthority;
use GeniSliver;
use GeniUser;
use GeniRegistry;
use GeniUtil;
use GeniHRN;
use libtestbed qw(SENDMAIL);
use emutil;
# Hate to import all this crap; need a utility library.
use libdb qw(TBGetSiteVar EXPTSTATE_SWAPPED EXPTSTATE_ACTIVE TBOPSPID
	     TBDB_NODESTATE_TBFAILED);
use User;
use Node;
use OSinfo;
use Image;
use Interface;
use English;
use Data::Dumper;
use XML::Simple;
use Date::Parse;
use POSIX qw(strftime tmpnam);
use Time::Local;
use Experiment;
use VirtExperiment;
use Firewall;
use Compress::Zlib;
use MIME::Base64;

# Configure variables
my $TB		   = "@prefix@";
my $TBOPS          = "@TBOPSEMAIL@";
my $TBAPPROVAL     = "@TBAPPROVALEMAIL@";
my $TBAUDIT   	   = "@TBAUDITEMAIL@";
my $BOSSNODE       = "@BOSSNODE@";
my $OURDOMAIN      = "@OURDOMAIN@";
my $PGENIDOMAIN    = "@PROTOGENI_DOMAIN@";
my $CREATEEXPT     = "$TB/bin/batchexp";
my $ENDEXPT        = "$TB/bin/endexp";
my $NALLOC	   = "$TB/bin/nalloc";
my $NFREE	   = "$TB/bin/nfree";
my $AVAIL	   = "$TB/sbin/avail";
my $PTOPGEN	   = "$TB/libexec/ptopgen";
my $TBSWAP	   = "$TB/bin/tbswap";
my $SWAPEXP	   = "$TB/bin/swapexp";
my $PLABSLICE	   = "$TB/sbin/plabslicewrapper";
my $NAMEDSETUP     = "$TB/sbin/named_setup";
my $EXPORTS_SETUP  = "$TB/sbin/exports_setup";
my $VNODESETUP     = "$TB/sbin/vnode_setup";
my $GENTOPOFILE    = "$TB/libexec/gentopofile";
my $TARFILES_SETUP = "$TB/bin/tarfiles_setup";
my $MAPPER         = "$TB/bin/mapper";
my $VTOPGEN        = "$TB/bin/vtopgen";
my $SNMPIT         = "$TB/bin/snmpit";
my $PRERENDER      = "$TB/libexec/vis/prerender";
my $EMULAB_PEMFILE = "@prefix@/etc/genicm.pem";

my $API_VERSION = 1;

#
# Tell the client what API revision we support.  The correspondence
# between revision numbers and API features is to be specified elsewhere.
# No credentials are required.
#
sub GetVersion()
{
    return GeniResponse->Create( GENIRESPONSE_SUCCESS, $API_VERSION );
}

# Look up a node by an identifier of unspecified type (perhaps a URN, an
# (obsolete) UUID, or an old-style HRN.  Ultimately, all IDs should be
# URNs and this mess will go away, but for now we try not to make
# any assumptions, because of backward compatibility constraints.
sub LookupNode($)
{
    my ($nodeid) = @_;

    if( GeniHRN::IsValid( $nodeid ) ) {
	# Looks like a URN.
	my ($auth,$t,$id) = GeniHRN::Parse( $nodeid );

	return undef if $auth ne $OURDOMAIN or $t ne "node";

	return Node->Lookup( $id );
    }
 
    #
    # Looks like an old HRN, but we only want the last token for node lookup.
    #
    if ($nodeid =~ /\./) {
	($nodeid) = ($nodeid =~ /\.([-\w]*)$/);

	return Node->Lookup($nodeid);
    }
    
    # Assume it's a UUID, and pass it on as is.
    return Node->Lookup($nodeid);
}

#
# Respond to a Resolve request. 
#
sub Resolve($)
{
    my ($argref) = @_;
    my $uuid       = $argref->{'uuid'};
    my $cred       = $argref->{'credential'};
    my $type       = lc( $argref->{'type'} );
    my $hrn        = $argref->{'hrn'};

    if (! defined($cred)) {
	return GeniResponse->MalformedArgsResponse();
    }
    if (defined($uuid) && GeniHRN::IsValid($uuid)) {
	$hrn  = $uuid;
	$uuid = undef;
    }
    if( defined( $hrn ) && GeniHRN::IsValid( $hrn ) ) {
	my ($auth,$t,$id) = GeniHRN::Parse( $hrn );

	return GeniResponse->Create( GENIRESPONSE_ERROR, undef,
				     "Authority mismatch" )
	    if( $auth ne $OURDOMAIN );

	$type = lc( $t );
	
	$hrn = $id;	
    }
    if (! (defined($type) && ($type =~ /^(node)$/))) {
	return GeniResponse->MalformedArgsResponse();
    }
    # Allow lookup by uuid or hrn.
    if (! defined($uuid) && !defined( $hrn ) ) {
	return GeniResponse->MalformedArgsResponse();
    }
    if (defined($uuid) && !($uuid =~ /^[-\w]*$/)) {
	return GeniResponse->MalformedArgsResponse();
    }

    my $credential = GeniCredential->CreateFromSigned($cred);
    if (!defined($credential)) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "Could not create GeniCredential object");
    }
    
    #
    # Make sure the credential was issued to the caller, but no special
    # permission required to resolve component resources.
    #
    if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "This is not your credential!");
    }
    if ($type eq "node") {
	my $node;
	
	if (defined($uuid)) {
	    $node= LookupNode($uuid);
	}
	else {
	    $node= LookupNode($hrn);
	}
	if (! defined($node)) {
	    return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED,
					undef, "Nothing here by that name");
	}

	my $rspec = GetAdvertisement(0, $node->node_id());
	if (! defined($rspec)) {
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					"Could not start avail");
	}
	
	# Return a blob.
	my $blob = { "hrn"          => "${PGENIDOMAIN}." . $node->node_id(),
		     "uuid"         => $node->uuid(),
		     "role"	    => $node->role(),
		     "hostname"     => $node->node_id() . ".${OURDOMAIN}",
		     "physctrl"     => 
			 Interface->LookupControl( $node->phys_nodeid() )->IP(),
		     "urn"          => GeniHRN::Generate( $OURDOMAIN,
							  "node",
							  $node->node_id() ),
		     "rspec"        => $rspec
		   };

	return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
    }
    return GeniResponse->Create(GENIRESPONSE_UNSUPPORTED);
}

#
# Discover resources on this component, returning a resource availablity spec
#
sub DiscoverResources($)
{
    my ($argref) = @_;
    my $credential = $argref->{'credential'};
    my $available = $argref->{'available'} || 0;
    my $compress = $argref->{'compress'} || 0;
    my $user_uuid  = $ENV{'GENIUSER'};

    $credential = GeniCredential->CreateFromSigned($credential);
    if (!defined($credential)) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "Could not create GeniCredential object");
    }
    # The credential owner/slice has to match what was provided.
    if ($user_uuid ne $credential->owner_uuid()) {
	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
				    "Invalid credentials for operation");
    }
    return DiscoverResourcesAux($available, $compress);
}
# Helper function for V2.
sub DiscoverResourcesAux($$)
{
    my ($available, $compress) = @_;
    my $user_uuid  = $ENV{'GENIUSER'};

    # Oh, for $*%(s sake.  Frontier::RPC2 insists on representing a
    # Boolean as its own object type -- which Perl always interprets as
    # true, regardless of the object's value.  Undo all of that silliness.
    if (defined($available) && ref($available) eq 'Frontier::RPC2::Boolean') {
	$available = $available->value;
    }
    if (defined($compress) && ref($compress) eq 'Frontier::RPC2::Boolean') {
	$compress = $compress->value;
    }

    #
    # A sitevar controls whether external users can get any nodes.
    #
    my $allow_externalusers = 0;
    if (!TBGetSiteVar('protogeni/allow_externalusers', \$allow_externalusers)){
	# Cannot get the value, say no.
	$allow_externalusers = 0;
    }
    if (!$allow_externalusers) {
	my $user = GeniUser->Lookup($user_uuid, 1);
	# No record means the user is remote.
	if (!defined($user) || !$user->IsLocal()) {
	    return GeniResponse->Create(GENIRESPONSE_UNAVAILABLE, undef,
					"External users temporarily denied");
	}
    }

    #
    # Acquire the advertisement from ptopgen and compress it if requested.
    #
    my $xml = GetAdvertisement($available, undef);
    if (! defined($xml)) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "Could not start avail");
    }

    if( $compress ) {
	my $coder = Frontier::RPC2->new();
	my $base64 = encode_base64( compress( $xml ) );
	$xml = $coder->base64( $base64 );	
    }

    return GeniResponse->Create(GENIRESPONSE_SUCCESS, $xml);
}

#
# Use ptopgen in xml mode to spit back an xml file. 
#
sub GetAdvertisement($$)
{
    my ($available, $pc) = @_;
    my $xml = undef;

    my $invocation = "$PTOPGEN -x -g -r -p GeniSlices";
    $invocation .= " -a" unless $available;
    if (defined($pc)) {
	$invocation .= " -1 $pc";
    }
    if (open(AVAIL, "$invocation |")) {
	$xml = "";
	while (<AVAIL>) {
	    $xml .= $_;
	}
	close(AVAIL);
    }
    return $xml;
}

#
# Update a ticket with a new rspec.
#
sub UpdateTicket($)
{
    my ($argref) = @_;

    return GetTicket($argref, 1);
}

#
# Respond to a GetTicket request. 
#
sub GetTicket($;$)
{
    my ($argref, $isupdate) = @_;
    my $rspecstr   = $argref->{'rspec'};
    my $impotent   = $argref->{'impotent'};
    my $credstr    = $argref->{'credential'};
    my $tickstr    = $argref->{'ticket'};
    my $ticket;

    # Default to no update
    $isupdate = 0
	if (!defined($isupdate));
    $impotent = 0
	if (!defined($impotent));

    if (! defined($credstr)) {
	return GeniResponse->MalformedArgsResponse();
    }
    if (!defined($rspecstr)) {
	return GeniResponse->MalformedArgsResponse();
    }
    if (! ($rspecstr =~ /^[\040-\176\012\015\011]+$/)) {
	return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				    "Improper characters in rspec");
    }
    my $credential = GeniCredential->CreateFromSigned($credstr);
    if (!defined($credential)) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "Could not create GeniCredential object");
    }
    #
    # Make sure the credential was issued to the caller.
    #
    if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				    "This is not your credential!");
    }
    if ($isupdate) {
	$ticket = GeniTicket->CreateFromSignedTicket($tickstr);
	if (!defined($ticket)) {
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					"Could not create GeniTicket object");
	}
	$ticket->SetSlice($credential->target_uuid());
    }
    return GetTicketAux($credential,
			$rspecstr, $isupdate, $impotent, 0, 1, $ticket);
}

sub GetTicketAux($$$$$$$)
{
    my ($credential, $rspecstr, $isupdate, $impotent, $v2, $level,
	$ticket) = @_;
    
    defined($credential) &&
	($credential->HasPrivilege( "pi" ) or
	 $credential->HasPrivilege( "instantiate" ) or
	 $credential->HasPrivilege( "bind" ) or
	 return GeniResponse->Create( GENIRESPONSE_FORBIDDEN, undef,
				      "Insufficient privilege" ));
    
    my $slice_uuid = $credential->target_uuid();
    my $user_uuid  = $credential->owner_uuid();
    
    #
    # Create slice from the certificate.
    #
    my $slice = GeniSlice->Lookup($slice_uuid);
    if (!defined($slice)) {
	if ($isupdate) {
	    print STDERR "Could not locate slice $slice_uuid for Update\n";
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					"No slice found for UpdateTicket");
	}
	$slice = CreateSliceFromCertificate($credential);
	if (!defined($slice)) {
	    print STDERR "Could not create $slice_uuid\n";
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					"Could not create slice");
	}
    }

    #
    # Ditto the user.
    #
    my $user = GeniUser->Lookup($user_uuid, 1);
    if (!defined($user)) {
	if ($isupdate) {
	    print STDERR "Could not locate $user_uuid for UpdateTicket\n";
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					"No user found for UpdateTicket");
	}
	$user = CreateUserFromCertificate($credential->owner_cert());
	if (!defined($user)) {
	    print STDERR "No user $user_uuid in the ClearingHouse\n";
	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
			    "Could not get user info from ClearingHouse");
	}
    }
    return GetTicketAuxAux($slice, $user, $rspecstr,
			   $isupdate, $impotent, $v2, $level, $ticket);
}
sub GetTicketAuxAux($$$$$$$$)
{
    my ($slice, $user,
	$rspecstr, $isupdate, $impotent, $v2, $level, $ticket) = @_;
    my $response    = undef;
    my $restorevirt = 0;	# Flag to restore virtual state
    my $restorephys = 0;	# Flag to restore physical state

    #
    # We need this below to sign the ticket.
    #
    my $authority = GeniCertificate->LoadFromFile($EMULAB_PEMFILE);
    if (!defined($authority)) {
	print STDERR " Could not get uuid from $EMULAB_PEMFILE\n";
	return GeniResponse->Create(GENIRESPONSE_ERROR);
    }

    my $rspec =
	eval { XMLin($rspecstr, KeyAttr => [],
		     ForceArray => ["node", "link", "interface",
				    "interface_ref", "linkendpoints"]) };
    if ($@) {
	print STDERR "XMLin error: $@\n";
	return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				    "XML error in rspec");
    }

    #
    # A sitevar controls whether external users can get any nodes.
    #
    my $allow_externalusers = 0;
    if (!TBGetSiteVar('protogeni/allow_externalusers', \$allow_externalusers)){
	# Cannot get the value, say no.
	$allow_externalusers = 0;
    }
    if (!$allow_externalusers && !$user->IsLocal()) {
	return GeniResponse->Create(GENIRESPONSE_UNAVAILABLE, undef,
				    "External users temporarily denied");
    }
    
    #
    # For now all tickets expire very quickly (minutes), but once the
    # ticket is redeemed, it will expire according to the rspec request.
    #
    if (exists($rspec->{'valid_until'})) {
	my $expires = $rspec->{'valid_until'};

	if (! ($expires =~ /^[-\w:.\/]+/)) {
	    return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					"Illegal valid_until in rspec");
	}
	# Convert to a localtime.
	my $when = timegm(strptime($expires));
	if (!defined($when)) {
	    return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					"Could not parse valid_until");
	}
	
	#
	# Do we need a policy limit?
	#
	my $diff = $when - time();
	if ($diff < (60 * 5) || $diff > (3600 * 24 * 100)) {
	    return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					"valid_until out of range");
	}
    }
    else {
	# Give it a reasonable default for later when the ticket is redeemed.
	$rspec->{'valid_until'} =
	    POSIX::strftime("20%y-%m-%dT%H:%M:%S", gmtime(time() + (3600*1)));
    }

    #
    # Lock the ticket so it cannot be released.
    #
    if (defined($ticket) && $ticket->stored() && $ticket->Lock() != 0) {
	return GeniResponse->BusyResponse("ticket");
    }
    
    #
    #
    # Lock the slice from further access.
    #
    if ($slice->Lock() != 0) {
	$ticket->UnLock()
	    if (defined($ticket) && $ticket->stored());
	return GeniResponse->BusyResponse("slice");
    }
    # Shutdown slices get nothing.
    if ($slice->shutdown()) {
	$slice->UnLock();
	$ticket->UnLock()
	    if (defined($ticket) && $ticket->stored());
	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
				    "Slice has been shutdown");
    }

    #
    # For now, there can be only a single toplevel aggregate per slice.
    # The existence of an aggregate means the slice is active here. 
    #
    my $aggregate = GeniAggregate->SliceAggregate($slice);
    if (!$isupdate) {
	if (defined($aggregate)) {
	    $response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				     "Already have an aggregate for slice");
	    goto bad;
	}
    }
    elsif ($v2 && $level && !defined($ticket) && !defined($aggregate)) {
	print STDERR "No aggregate for $slice in version two API\n";
	$response = GeniResponse->Create(GENIRESPONSE_ERROR);
	goto bad;
    }

    #
    # Firewall hack; just a flag in the rspec for now.
    #
    if (exists($rspec->{'needsfirewall'}) && $rspec->{'needsfirewall'}) {
	if ($slice->SetFirewallFlag($rspec->{'needsfirewall'}) != 0) {
	    $response = GeniResponse->Create(GENIRESPONSE_ERROR);
	    goto bad;
	}
    }

    #
    # We need this now so we can form a virtual topo.
    #
    my $slice_experiment = GeniExperiment($slice);
    if (!defined($slice_experiment)) {
	print STDERR "Could not create new Geni slice experiment!\n";
	$response = GeniResponse->Create(GENIRESPONSE_ERROR);
	goto bad;
    }
    my $pid = $slice_experiment->pid();
    my $eid = $slice_experiment->eid();

    #
    # Mark the experiment locally as coming from the cooked interface.
    # This changes what tmcd returns to the local nodes.
    #
    if (exists($rspec->{'generated_by'}) &&
	$rspec->{'generated_by'} eq "libvtop") {
	$slice_experiment->Update({"geniflags" =>
				       $Experiment::EXPT_GENIFLAGS_EXPT|
				       $Experiment::EXPT_GENIFLAGS_COOKED});
    }
    
    #
    # Create a virt topology object. We are going to load this up as we
    # process the rspec.
    #
    my $virtexperiment = VirtExperiment->CreateNew($slice_experiment);
    if (!defined($virtexperiment)) {
	print STDERR "Could not create VirtExperiment object!\n";
	$response = GeniResponse->Create(GENIRESPONSE_ERROR);
	goto bad;
    }
    # Turn off fixnode; we will control this on the commandline.
    $virtexperiment->allowfixnode(0);

    # This is where nodes are parked until a ticket is redeemed.
    # This experiment no longer has to exist.
    my $reserved_holding = Experiment->Lookup("GeniSlices", "reservations");

    #
    # An rspec is a structure that requests specific nodes. If those
    # nodes are available, then reserve it. Otherwise the ticket
    # cannot be granted.
    #
    my %namemap  = ();
    my %colomap  = ();
    my %ifacemap = ();
    my %nodemap  = ();
    my @nodeids  = ();
    my %lannodes = ();
    my @dealloc;

    #
    # If this is a ticket update, we want to seed the namemap with
    # existing nodes. This is cause the rspec might refer to wildcards
    # that were already bound in a previous call. We also want to know
    # what nodes are currently reserved in case we have to release some.
    #
    if ($isupdate) {
	$slice_experiment->ClearBackupState();
	if ($slice_experiment->BackupVirtualState()) {
	    print STDERR "Could not backup virtual state!\n";
	    $response = GeniResponse->Create(GENIRESPONSE_ERROR);
	    goto bad;
	}
	if ($slice_experiment->RemoveVirtualState()) {
	    print STDERR "Could not remove virtual state!\n";
	    $response = GeniResponse->Create(GENIRESPONSE_ERROR);
	    goto bad;
	}
	$restorevirt = 1;

	if ($slice_experiment->BackupPhysicalState()) {
	    print STDERR "Could not backup physical state!\n";
	    $response = GeniResponse->Create(GENIRESPONSE_ERROR);
	    goto bad;
	}
	my $oldrspec;
	if ($v2 && defined($aggregate)) {
	    $oldrspec = $aggregate->GetManifest(0);
	}
	else {
	    $oldrspec = $ticket->rspec();
	}
	
	foreach my $ref (@{$oldrspec->{'node'}}) {
	    my $resource_uuid = $ref->{'component_uuid'} || $ref->{'uuid'};
	    my $manager_uuid  = $ref->{'component_manager_uuid'};
	    my $node_nickname = $ref->{'virtual_id'} || $ref->{'nickname'};
	    my $colocate      = $ref->{'colocate'} || $ref->{'phys_nickname'};

	    # Let remote nodes pass through.
	    next
		if (defined($manager_uuid) &&
		    !GeniHRN::Equal( $manager_uuid, $ENV{'MYURN'} ) &&
		    $manager_uuid ne $ENV{'MYUUID'});

	    # Skip lan nodes; they are fake.
	    next
		if (exists($ref->{'node_type'}) &&
		    exists($ref->{'node_type'}->{'type_name'}) &&
		    $ref->{'node_type'}->{'type_name'} eq "lan");

	    my $node = LookupNode($resource_uuid);
	    if (!defined($node)) {
		$response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				 "Bad resource $resource_uuid in ticket");
		goto bad;
	    }

	    #
	    # Grab the reservation. For backwards compatibility, we want
	    # to find nodes in the reservations holding area, and move them
	    # into the slice experiment. The holding area is no longer going
	    # to be used, at least not until we have a reservations system.
	    #
	    my $reservation = $node->Reservation();
	    if (defined($reservation) &&
		defined($reserved_holding) &&
		$reservation->SameExperiment($reserved_holding)) {
		if ($node->MoveReservation($slice_experiment)) {
		    print STDERR "Could not move $node to $slice_experiment\n";
		    goto bad;
		}
		$node->Refresh();
	    }
	    $namemap{$node_nickname} = $node;
	    $colomap{$colocate} = $node
		if (defined($colocate));
	}
    }

    print STDERR Dumper($rspec);

    foreach my $ref (@{$rspec->{'node'}}) {
	my $resource_uuid = $ref->{'component_uuid'} || $ref->{'uuid'};
	my $manager_uuid  = $ref->{'component_manager_uuid'};
	my $node_nickname = $ref->{'virtual_id'} || $ref->{'nickname'};
	my $colocate      = $ref->{'colocate'} || $ref->{'phys_nickname'};
	my $subnode_of    = $ref->{'subnode_of'};
	my $virtualization_type    = $ref->{'virtualization_type'};
	my $virtualization_subtype = $ref->{'virtualization_subtype'};
	my $exclusive     = $ref->{'exclusive'};
	my $pctype;
	my $osname;
	my $node;

	# Let remote nodes pass through.
	next
	    if (defined($manager_uuid) &&
		!GeniHRN::Equal( $manager_uuid, $ENV{'MYURN'} ) &&
		$manager_uuid ne $ENV{'MYUUID'});

	#
	# Lan nodes are fake and do not go into the virt topo. Need
	# to remember them though, for when we do the links below.
	# They are still in the returned ticket though. 
	#
	if (exists($ref->{'node_type'}) &&
	    exists($ref->{'node_type'}->{'type_name'}) &&
	    $ref->{'node_type'}->{'type_name'} eq "lan") {
	    $lannodes{$node_nickname} = $ref;
	    next;
	}

	if (defined($virtualization_type)) {
	    if ($virtualization_type eq "emulab-vnode") {
		if (defined($virtualization_subtype)) {
		    if ($virtualization_subtype eq "emulab-jail") {
			$osname = "FBSD-JAIL";
		    }
		    elsif ($virtualization_subtype eq "emulab-openvz") {
			$osname = "OPENVZ-STD";
		    }
		    $pctype = "pcvm";
		}
		else {
		    goto raw;
		}
	    }
	    else {
	      raw:
		# Lets force to exclusive real node.
		$ref->{'exclusive'} = $exclusive = 1;
		$ref->{'virtualization_type'} = "raw";
	    }
	}
	else {
	    $response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				     "Must provide a virtualization_type");
	    goto bad;

	}
	if (!defined($node_nickname)) {
	    $response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				     "Must provide a virtual_id for nodes");
	    goto bad;
	}

	#
	# Allow wildcarding.
	#
	if (!defined($resource_uuid) || $resource_uuid eq "*") {
	    if (defined($colocate) && exists($colomap{$colocate})) {
		$node = $colomap{$colocate};
	    }
	    elsif ($isupdate && exists($namemap{$node_nickname})) {
		$node = $namemap{$node_nickname};
	    }
	}
	else {
	    $node = LookupNode($resource_uuid);

	    if (!defined($node)) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					 "Bad resource $resource_uuid");
		goto bad;
	    }
	    $pctype = $node->type()
		if (!defined($pctype));
	}
	#
	# If no osname by this point, try for the default.
	#
	if (defined($node) && !defined($osname)) {
	    if (defined($node->default_osid())) {	    
		my $osinfo = OSinfo->Lookup($node->default_osid());
		$osname = $osinfo->osname()
		    if (defined($osinfo));
	    }
	}
	# The slot does not like to be NULL.
	$osname = ""
	    if (!defined($osname));
	
	# Need some kind of default.
	$pctype = "pc"
	    if (!defined($pctype));
	
	my $nodeblob = {"vname"   => $node_nickname,
			"type"    => $pctype,
			"osname"  => $osname,
			"ips"     => '', # deprecated
			"cmd_line"=> '', # bogus
			"fixed"   => (defined($subnode_of) ? $subnode_of :
				      defined($node) ? $node->node_id() : ""),
			};

	# Tarball and startup command.
	if (exists($ref->{'startup_command'})) {
	    my $startupcmd = $ref->{'startup_command'};
	    
	    if (! TBcheck_dbslot($startupcmd, "virt_nodes", "startupcmd",
			 TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					 "Invalid startup command");
		goto bad;
	    }
	    $nodeblob->{'startupcmd'} = $startupcmd;
	}
	if (exists($ref->{'tarfiles'})) {
	    my $tarfiles = $ref->{'tarfiles'};
	    
	    if (! TBcheck_dbslot($tarfiles, "virt_nodes", "tarfiles",
			 TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
					 "Invalid tarfiles");
		goto bad;
	    }
	    $nodeblob->{'tarfiles'} = $tarfiles;
	}

	my $virtnode = $virtexperiment->NewTableRow("virt_nodes", $nodeblob);
	if (!defined($virtnode)) {
	    $response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
					     "Error creating virtnode");
	    goto bad;
	}

	$virtexperiment->NewTableRow("virt_node_desires",
				     {"vname"    => $node_nickname,
				      "desire"   => "pcshared",
				      "weight"   => 0.95})
	    if (!defined($exclusive) || !$exclusive);

	# Store reference so we can munge it below. 
	$nodemap{$node_nickname} = {"rspec"    => $ref,
				    "virtnode" => $virtnode};
	
	#
	# Look for interface forward declarations that will be used later
	# in the link specifications. 
	#
	next
	    if (!exists($ref->{'interface'}));
	
	foreach my $linkref (@{$ref->{'interface'}}) {
	    my $component_id = $linkref->{"component_id"};
	    my $virtual_id   = $linkref->{"virtual_id"};

	    if (!defined($virtual_id)) {
		$response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
			     "Must provide a virtual_id for interfaces");
		goto bad;
	    }
	    
	    $ifacemap{$node_nickname} = {}
	        if (!exists($ifacemap{$node_nickname}));

	    # port counter.
	    my $vport = scalar(keys(%{ $ifacemap{$node_nickname} }));

	    # Store reference so we can munge it below. 
	    $ifacemap{$node_nickname}->{$virtual_id} = {"rspec" => $linkref,
							"vport" => $vport};

	    # This is used after the mapper runs since it uses vname:vport.
	    $ifacemap{"$node_nickname:$vport"} = $linkref;
	}
    }

    goto skiplinks
	if (!exists($rspec->{'link'}));
    
    #
    # Now deal with links for wildcarded nodes. We need to fill in the
    # node_uuid.
    #
    my $linknum = 1;
    
    foreach my $linkref (@{$rspec->{'link'}}) {
	my $lanname    = $linkref->{"nickname"} || $linkref->{"virtual_id"};
	my $istunnel   = (exists($linkref->{'link_type'}) &&
			  $linkref->{'link_type'} eq "tunnel");
	my $interfaces = $linkref->{'linkendpoints'} ||
	    $linkref->{'interface_ref'};
	my $ifacenum   = 1;

	if (!defined($lanname)) {
	    $response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
				     "Must provide a virtual_id for links");
	    goto bad;
	}

	#
	# Ick. Before we create the virt_lan_lans entry, we have to check
	# inside to see if one of the interfaces is connected to a lan
	# node. In this case, we want to reuse (if its been created) the
	# lan name, rather then a bunch of links with one interface, which
	# would result in a bogus topology. 
	#
	if (!$istunnel) {
	    foreach my $ref (@{ $interfaces }) {
		my $node_nickname = $ref->{'virtual_node_id'} ||
		    $ref->{'node_nickname'};

		if (exists($lannodes{$node_nickname})) {
		    $lanname = $node_nickname;
		}
	    }
	    if (!defined($virtexperiment->Find("virt_lan_lans", $lanname))) {
		$virtexperiment->NewTableRow("virt_lan_lans",
					     {"vname" => $lanname});
	    }
	}
	
	foreach my $ref (@{ $interfaces }) {
	    my $node_nickname = $ref->{'virtual_node_id'} ||
		$ref->{'node_nickname'};
	    my $iface_id     = $ref->{'virtual_interface_id'} ||
		$ref->{'iface_name'};

	    if (!defined($node_nickname)) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				 "$lanname: Need node id for links");
		goto bad;
	    }
	    if (!defined($iface_id)) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				 "$lanname: Need interface id for links");
		goto bad;
	    }

	    #
	    # Look for links that are really lans; one of the interfaces
	    # is on a fake lan node, which we caught above. Just skip it
	    # since in the virt topo, a lan is just a link with more then
	    # two nodes.
	    #
	    next
		if (exists($lannodes{$node_nickname}));

	    if ($istunnel) {
		# Might be the other side. Skip for now; might bite later.
		next
		    if (!exists($namemap{$node_nickname}));

		# Not doing anything else.
		next;
	    }
	    if (!exists($ifacemap{$node_nickname})) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				 "$lanname: No such virtual_node_id: ".
				 "$node_nickname");
		goto bad;
	    }
	    
	    #
	    # Sanity check the interface.
	    #
	    if (!exists($ifacemap{$node_nickname}->{$iface_id})) {
		$response =
		    GeniResponse->Create(GENIRESPONSE_ERROR, undef,
				 "$lanname: No such interface on component: ".
				 "$node_nickname:$iface_id");
		goto bad;
	    }
	    my $iface_ref   = $ifacemap{$node_nickname}->{$iface_id}->{"rspec"};
	    my $iface_name  = $iface_ref->{"component_id"} || "";
	    if( GeniHRN::IsValid( $iface_name ) ) {
		my ($urn_authority,$urn_node,$urn_iface) =
		    GeniHRN::ParseInterface( $iface_name );
		$iface_name = $urn_iface;
	    }
	    my $iface_vport = $ifacemap{$node_nickname}->{$iface_id}->{"vport"};

	    # XXX
	    my $ip     = "10.10.${linknum}.${ifacenum}";
	    my $mask   = "255.255.255.0";
	    my $member = "$node_nickname:$iface_vport";
	    my $bandwidth = 100000;

	    # Let user override.
	    $bandwidth = $linkref->{'bandwidth'}
	        if (exists($linkref->{'bandwidth'}));
	    
	    $virtexperiment->NewTableRow("virt_lans",
					 {"vname"       => $lanname,
					  "vnode"       => $node_nickname,
					  "vport"       => $iface_vport,
					  "trivial_ok"  => 1,
					  "ip"          => $ip,
					  "delay"       => 0.0,
					  "bandwidth"   => $bandwidth, # kbps
					  "lossrate"    => 0.0,
					  "member"      => $member,
					  "mask"        => $mask,
					  "rdelay"      => 0.0,
					  "rbandwidth"  => $bandwidth, # kbps
					  "rlossrate"   => 0.0,
					  "fixed_iface" => $iface_name});
	    $ifacenum++;
	}
	$linknum++;
    }
  skiplinks:
    $virtexperiment->Dump();
    if ($virtexperiment->Store()) {
	$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef);
	goto bad;
    }

    # Must chdir to the work directory for the mapper.
    if (! chdir($slice_experiment->WorkDir())) {
	$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef);
	goto bad;
    }

    # Do a render cause its nice to have on the show experiment page.
    system("$PRERENDER -r $pid $eid");
    system("$PRERENDER -t $pid $eid");