From 4bf3c2bff27e03c01745767f8e58dc9064ace0e8 Mon Sep 17 00:00:00 2001
From: Leigh B Stoller <stoller@flux.utah.edu>
Date: Tue, 23 Feb 2010 15:53:46 -0700
Subject: [PATCH] Merge in my cooked mode changes

---
 db/Experiment.pm.in                 |   4 +-
 db/Interface.pm.in                  |  23 ++
 db/Node.pm.in                       |  22 +-
 db/geni_control.in                  |  20 +-
 db/libGeni.pm.in                    | 206 ++++++++---
 event/lib/event.c                   | 302 +++++++++++----
 event/lib/event.h                   |  16 +-
 os/install-rpm                      |   4 +-
 os/install-tarfile                  |   4 +-
 protogeni/etc/protogeni.sql         |   1 +
 protogeni/lib/GeniCredential.pm.in  |   4 +-
 protogeni/lib/GeniEmulab.pm.in      |  91 +++--
 protogeni/lib/GeniRegistry.pm.in    |  24 +-
 protogeni/lib/GeniResource.pm.in    | 555 ++++++++++++++++++++++------
 protogeni/lib/GeniSA.pm.in          | 145 ++++----
 protogeni/lib/GeniSlice.pm.in       |   9 +-
 protogeni/scripts/cleanupslice.in   |   6 +-
 protogeni/scripts/sa_daemon.in      |  89 ++++-
 protogeni/xmlrpc/protogeni-cm.pl.in |   2 +-
 protogeni/xmlrpc/protogeni-sa.pl.in |   1 -
 tbsetup/endexp.in                   |   2 +-
 tbsetup/libvtop.pm.in               | 125 ++++++-
 tbsetup/tbend.in                    |  11 +-
 tmcd/common/GNUmakefile.in          |  16 +-
 tmcd/common/config/rc.ifconfig      |   2 +-
 tmcd/common/config/rc.progagent     |  11 +-
 tmcd/common/config/rc.rpms          |   4 +-
 tmcd/common/config/rc.tarfiles      |   4 +-
 tmcd/common/config/rc.topomap       |  15 +-
 tmcd/common/libsetup.pm             |  95 ++++-
 tmcd/common/libtmcc.pm              |  15 +-
 tmcd/common/runstartup              |   4 +-
 tmcd/tmcd.c                         | 184 +++++----
 www/nodessh.php3                    |   7 +-
 www/showslice.php                   |   5 +-
 35 files changed, 1542 insertions(+), 486 deletions(-)

diff --git a/db/Experiment.pm.in b/db/Experiment.pm.in
index 0a4087a12..79c30ae59 100644
--- a/db/Experiment.pm.in
+++ b/db/Experiment.pm.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2005-2009 University of Utah and the Flux Group.
+# Copyright (c) 2005-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package Experiment;
@@ -324,6 +324,8 @@ sub security_level($)   { return field($_[0], 'security_level');}
 sub linktest_pid($)     { return field($_[0], 'linktest_pid');}
 sub linktest_level($)   { return field($_[0], 'linktest_level');}
 sub logfile($)          { return field($_[0], 'logfile');}
+sub eventkey($)         { return field($_[0], 'eventkey');}
+sub keyhash($)          { return field($_[0], 'keyhash');}
 sub paniced($)          { return field($_[0], 'paniced');}
 sub cpu_usage($)        { return field($_[0], 'cpu_usage');}
 sub encap_style($)      { return field($_[0], 'encap_style');}
diff --git a/db/Interface.pm.in b/db/Interface.pm.in
index cce5e9f89..a20bb0f8b 100644
--- a/db/Interface.pm.in
+++ b/db/Interface.pm.in
@@ -518,6 +518,29 @@ sub Update($$)
     return Refresh($self);
 }
 
+#
+# Lookup a specific attribute in the type capabilities table. 
+#
+sub TypeCapability($$$)
+{
+    my ($self, $capkey, $pcapval) = @_;
+    
+    return -1
+	if (!ref($self));
+
+    my $itype = $self->interface_type();
+
+    my $query_result =
+	DBQueryWarn("select capval from interface_capabilities ".
+		    "where type='$itype' and capkey=$capkey");
+
+    return -1
+	if (!$query_result || !$query_result->numrows());
+    my ($capval) = $query_result->fetchrow_array();
+    $pcapval = $capval;
+    return 0;
+}
+
 ##############################################################################
 package Interface::VInterface;
 use libdb;
diff --git a/db/Node.pm.in b/db/Node.pm.in
index 6907231cc..6cbcf4a8e 100644
--- a/db/Node.pm.in
+++ b/db/Node.pm.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2005-2009 University of Utah and the Flux Group.
+# Copyright (c) 2005-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package Node;
@@ -2458,5 +2458,25 @@ sub GetOsids($) {
 	    $self->{"DBROW"}{"next_boot_osid"});
 }
 
+#
+# Look for a widearea node by its external node id.
+#
+sub LookupWideArea($$)
+{
+    my ($class, $external_node_id) = @_;
+    my $safe_id = DBQuoteSpecial($external_node_id);
+
+    my $query_result =
+	DBQueryWarn("select node_id from widearea_nodeinfo ".
+		    "where external_node_id=$safe_id");
+
+    return undef
+	if (!defined($query_result) || !$query_result->numrows);
+
+    my ($node_id) = $query_result->fetchrow_array();
+
+    return Node->Lookup($node_id);
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;
diff --git a/db/geni_control.in b/db/geni_control.in
index d543cf9d7..86b421252 100755
--- a/db/geni_control.in
+++ b/db/geni_control.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2009 University of Utah and the Flux Group.
+# Copyright (c) 2009, 2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use strict;
@@ -52,6 +52,7 @@ sub ClearAll();
 sub StartAll();
 sub WaitAll();
 sub PurgeAll();
+sub RenewAll();
 
 #
 # Turn off line buffering on output
@@ -82,7 +83,8 @@ my $pid    = shift;
 my $eid	   = shift;
 my $action = shift;
 
-if ($action =~ /^(alloc|free|clear|wait|purge|start|register|unregister)$/) {
+if ($action =~
+    /^(alloc|free|clear|wait|purge|start|register|unregister|renew)$/) {
     $action = $1;
 }
 else {
@@ -166,6 +168,10 @@ SWITCH: for ($action) {
 	PurgeAll();
 	last SWITCH;
     };
+    /^renew$/ && do {
+	RenewAll();
+	last SWITCH;
+    };
     fatal("Unknown action $action");
 }
 exit($exitval);
@@ -229,8 +235,10 @@ sub PurgeAll()
     foreach my $resource (@resources) {
 	$resource->Purge($this_user) == 0
 	    or fatal("Could not purge resources from $resource");
+	$resource->Delete() == 0
+	    or fatal("Could not delete $resource");
     }
-
+    UnRegister();
     return 0;
 }
 
@@ -246,6 +254,12 @@ sub WaitAll()
 	fatal("Cannot wait on slivers!\n");
 }
 
+sub RenewAll()
+{
+    libGeni::RenewSlivers($experiment, 1) == 0 or
+	fatal("Cannot renew resources");
+}
+
 #
 # Register.
 #
diff --git a/db/libGeni.pm.in b/db/libGeni.pm.in
index d8613f4ab..3527a4732 100644
--- a/db/libGeni.pm.in
+++ b/db/libGeni.pm.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2009 University of Utah and the Flux Group.
+# Copyright (c) 2009, 2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package libGeni;
@@ -50,23 +50,19 @@ sub UnRegister($)
     return GeniEmulab::UnRegisterExperiment($experiment);
 }
 
-#
-# Quickie solution.
-#
-sub RenewAllSlivers()
+sub RenewSlivers($;$)
 {
-    #
-    # Hand this off ...
-    #
-    return GeniResource::RenewAll();
+    my ($experiment, $force) = @_;
+
+    return GeniResource::RenewExperimentResources($experiment, $force);
 }
 
 #
 # Map rspec to resources using assign. 
 #
-sub MapResources($$$)
+sub MapResources($$$$)
 {
-    my ($experiment, $user, $rspec) = @_;
+    my ($experiment, $user, $rspec, $verbose) = @_;
     my %cm_urns   = ();
     my %fragments = ();
     my %nodemap   = ();
@@ -112,6 +108,7 @@ sub MapResources($$$)
 	$node_cms{$ref->{'virtual_id'}} = $cm;
 	$nodemap{$ref->{'virtual_id'}}  = $ref;
     }
+
     #
     # As above, need to split the interfaces into the correct fragments.
     #
@@ -119,6 +116,15 @@ sub MapResources($$$)
 	foreach my $ref (@{ $rspec->{'link'} }) {
 	    my $linkname   = $ref->{'virtual_id'};
 
+	    # Skip tunnels until rspec stitching in place.
+	    next
+		if (exists($ref->{'link_type'}) &&
+		    $ref->{'link_type'} eq "tunnel");
+	    next
+		if (0 && exists($ref->{'link_type'}) &&
+		    exists($ref->{'link_type'}->{'type_name'}) &&
+		    $ref->{'link_type'}->{'type_name'} eq "tunnel");
+
 	    foreach my $ifaceref (@{ $ref->{'interface_ref'} }) {
 		my $virtual_node_id = $ifaceref->{'virtual_node_id'};
 
@@ -140,7 +146,7 @@ sub MapResources($$$)
 	    }
 	}
     }
-	
+
     #
     # Get the resource objects.
     #
@@ -206,12 +212,16 @@ sub MapResources($$$)
 	my $solution =
 	    eval { XMLin($soln, KeyAttr => [],
 			 ForceArray => ["node", "link", "interface",
-					"interface_ref", "linkendpoints"]) };
+					"interface_ref", "linkendpoints",
+					"component_manager"]) };
 	if ($@) {
 	    print STDERR "XMLin error reading $soln: $@\n";
 	    return -1;
 	}
-#	print STDERR Dumper($solution);
+	if ($verbose) {
+	    print STDERR "Solution for $resource\n";
+	    print STDERR Dumper($solution);
+	}
 
 	foreach my $ref (@{ $solution->{'node'} }) {
 	    my $virtual_id = $ref->{'virtual_id'};
@@ -233,6 +243,11 @@ sub MapResources($$$)
 		    my $iface_id = $ifaceref->{'virtual_id'};
 		    my $compid   = $ifaceref->{'component_id'};
 
+		    # Not supposed to happen, but does cause of issues
+		    # with tunnels and rspec stitching. 
+		    next
+			if (!defined($compid));
+
 		    foreach my $oref (@{ $noderef->{'interface'} }) {
 			if ($oref->{'virtual_id'} eq $iface_id) {
 			    # write the solution back into the original rspec. 
@@ -244,14 +259,36 @@ sub MapResources($$$)
 	    }
 	}
     }
-#    print STDERR Dumper($rspec);
+    if (exists($rspec->{'link'})) {
+	foreach my $ref (@{ $rspec->{'link'} }) {
+	    my %cms = ();
+	    my @cms = ();
+
+	    foreach my $ifaceref (@{ $ref->{'interface_ref'} }) {
+		my $virtual_node_id = $ifaceref->{'virtual_node_id'};
+		my $node_cm = $node_cms{$virtual_node_id};
+
+		if (!exists($cms{$node_cm})) {
+		    push(@cms, {'id' => $node_cm});
+		    $cms{$node_cm} = $node_cm;
+		}
+	    }
+	    $ref->{'component_manager'} = [@cms];
+	}
+    }
+
+    if ($verbose) {
+	print STDERR "Final rspec:\n";
+	print STDERR Dumper($rspec);
+    }
     return 0;
 }
 
 sub GetTickets($$$$)
 {
     my ($experiment, $impotent, $user, $rspec) = @_;
-    my %cm_urns = ();
+    my %cm_urns  = ();
+    my %node_cms = ();
 
     Register($experiment, $user) == 0
 	or return -1;
@@ -272,6 +309,7 @@ sub GetTickets($$$$)
 	my $cm = GeniHRN::Generate($auth, "authority", "cm");
 
 	$cm_urns{$cm} = $cm;
+	$node_cms{$ref->{'virtual_id'}} = $cm;
 
 	#
 	# This is how we get the client side to do cooked mode properly.
@@ -279,7 +317,6 @@ sub GetTickets($$$$)
 	$ref->{'tarfiles'} = "/usr/local/etc/emulab ".
 	    "$TBDOCBASE/downloads/geniclient.tar";
     }
-    #print STDERR Dumper($rspec);
 
     #
     # Get the resource objects.
@@ -295,6 +332,7 @@ sub GetTickets($$$$)
 	}
 	$cm_urns{$cm} = $resource;
     }
+
     #
     # Ask for tickets.
     #
@@ -343,9 +381,11 @@ sub RedeemTickets($$$)
 # Map the local nodes to the external nodes. This just sets some DB
 # state for now.
 #
-sub MapNodes($)
+sub MapNodes($$)
 {
-    my ($experiment) = @_;
+    my ($experiment, $verbose) = @_;
+    my $eventkey = $experiment->eventkey();
+    my $keyhash  = $experiment->keyhash();
     my %ifacemap = ();
 
     #
@@ -361,11 +401,21 @@ sub MapNodes($)
 	return -1
 	    if (!defined($manifest));
 
+	print STDERR Dumper($manifest);
+	
 	foreach my $ref (@{ $manifest->{'node'} }) {
 	    my $sliver_urn = $ref->{'sliver_urn'};
 	    my $vname      = $ref->{'virtual_id'};
-	    my $node       = $experiment->VnameToNode($vname);
 
+	    #
+	    # The manifest can include nodes from other CMs. There will not
+	    # be a sliver urn in that case.
+	    #
+	    my $component_manager_urn = $ref->{'component_manager_urn'};
+	    next
+		if (!defined($sliver_urn) && !defined($component_manager_urn));
+	    
+	    my $node = $experiment->VnameToNode($vname);
 	    if (!defined($node)) {
 		print STDERR
 		    "MapNodes: Could not locate node $vname in $experiment\n";
@@ -385,8 +435,9 @@ sub MapNodes($)
 	    #
 	    # This is how we get the client side to do cooked mode properly.
 	    #
-	    $ref->{'startup_command'} = "/usr/local/etc/emulab/rc.pgeni ".
-		"-s $BOSSNODE -u '$sliver_urn' boot";
+	    $ref->{'startup_command'} =
+		"sudo /usr/local/etc/emulab/rc/rc.pgeni ".
+		"-s $BOSSNODE -k $eventkey,$keyhash -u '$sliver_urn' boot";
 
 	    # Interface map for loop below.
 	    if (exists($ref->{'interface'})) {
@@ -401,6 +452,20 @@ sub MapNodes($)
 	foreach my $ref (@{ $manifest->{'link'} }) {
 	    my $linkname   = $ref->{"virtual_id"};
 	    my $interfaces = $ref->{'interface_ref'};
+	    my %managers   = map { $_->{'id'} => $_->{'id'} }
+	                        @{ $ref->{'component_manager'} };
+
+	    # Skip tunnels in this loop for now.
+	    next
+		if (exists($ref->{'link_type'}) &&
+		    $ref->{'link_type'} eq "tunnel");
+
+	    #
+	    # The manifest can include links for other CMs. Skip those
+	    # for now.
+	    #
+	    next 
+		if (!exists($managers{$resource->manager_urn()}));
 
 	    foreach my $ifaceref (@{ $interfaces }) {
 		my $vname    = $ifaceref->{'virtual_node_id'};
@@ -418,18 +483,18 @@ sub MapNodes($)
 		if (!defined($iface)) {
 		    print STDERR "Could not determine iface for" .
 			"$vname,$iface_id\n";
-		    print Dumper($manifest);
+		    print STDERR Dumper($manifest);
 		    return -1;
 		}
 		my $interface = Interface->LookupByIface($node,$iface);
 		if (!defined($interface)) {
 		    print STDERR "Could not map iface for $node,$iface\n";
-		    print Dumper($manifest);
+		    print STDERR Dumper($manifest);
 		    return -1;
 		}
 		if (! ($MAC =~ /^[\w]*$/)) {
 		    print STDERR "Bad mac '$MAC' for $node,$iface\n";
-		    print Dumper($manifest);
+		    print STDERR Dumper($manifest);
 		    return -1;
 		}
 		if ($interface->Update({"mac" => "$MAC"})) {
@@ -439,6 +504,11 @@ sub MapNodes($)
 		}
 	    }
 	}
+	# The manifest was changed above.
+	if ($resource->UpdateManifest($manifest)) {
+	    print STDERR "Could not store manifest for $resource\n";
+	    return -1;
+	}
     }
     return 0;
 }
@@ -511,16 +581,45 @@ sub WaitForSlivers($$@)
     }
 
     #
-    # Build a map of the nodes.
+    # Build a map of the nodes. I made a real mess of this in Version 1.
     #
-    my @nodelist = $experiment->NodeList(0, 1);
-    foreach my $node (@nodelist) {
-	next
-	    if (!defined($node->external_resource_id()) ||
-		$node->external_resource_id() eq "");
-	
-	$nodemap{$node->external_resource_id()} = $node;
-	$node->Refresh();
+    foreach my $resource (@resources) {
+	my $manifest = $resource->Manifest();
+	return -1
+	    if (!defined($manifest));
+
+	foreach my $ref (@{ $manifest->{'node'} }) {
+	    my $vname      = $ref->{'virtual_id'};
+	    my $urn        = $ref->{'component_urn'};
+	    my $node       = $experiment->VnameToNode($vname);
+
+	    #
+	    # The manifest can include nodes from other CMs. There will not
+	    # be a sliver urn in that case.
+	    #
+	    my $sliver_urn = $ref->{'sliver_urn'};
+	    next
+		if (!defined($sliver_urn));
+	    
+	    if (!defined($node)) {
+		print STDERR "WaitForSlivers: ".
+		    "Could not locate node $vname in $experiment\n";
+		return -1;
+	    }
+	    if ($resource->ManagerVersion() == 1.0) {
+		my ($domain,undef,$node_id) = GeniHRN::Parse($urn);
+		$urn = GeniHRN::Generate($domain, "sliver", $node_id);
+		$nodemap{$urn} = $node;
+	    }
+	    else {
+		next
+		    if (!defined($node->external_resource_id()) ||
+			$node->external_resource_id() eq "");
+
+		$nodemap{$node->external_resource_id()} = $node;
+	    }
+	    $node->Refresh();
+	}
     }
 
     #
@@ -541,9 +640,23 @@ sub WaitForSlivers($$@)
 	}
 	print STDERR Dumper($ref);
 
-	foreach my $key (keys(%{ $ref->{'detailsNew'} })) {
-	    my $val  = $ref->{'detailsNew'}->{$key};
-	    my $node = $nodemap{$key};
+	foreach my $key (keys(%{ $ref->{'details'} })) {
+	    my $val  = $ref->{'details'}->{$key};
+	    my ($status, $node);
+
+	    if ($resource->ManagerVersion() == 1.0) {
+		$node   = $nodemap{$key};
+		$status = $val;
+	    }
+	    elsif ($resource->ManagerVersion() == 2.0) {
+		$node   = $nodemap{$key};
+		$status = $val->{'status'};
+	    }
+	    else {
+		print STDERR
+		    "*** WaitForSlivers: Unknown version on $resource\n";
+		next;
+	    }
 
 	    if (!defined($node)) {
 		print STDERR "No node in map for $key ($resource)\n";
@@ -555,14 +668,14 @@ sub WaitForSlivers($$@)
 	    #
 	    # Only send on state change to avoid multiple events.
 	    #
-	    if ($val eq "ready" && !$node->IsUp()) {
-		print STDERR "    Sending ISUP event.\n";
-		$node->SetEventState(TBDB_NODESTATE_ISUP());
+	    if ($status eq "ready" && !$node->IsUp()) {
+#		print STDERR "    Sending ISUP event.\n";
+#		$node->SetEventState(TBDB_NODESTATE_ISUP());
 	    }
-	    elsif ($val eq "failed" &&
+	    elsif ($status eq "failed" &&
 		   $node->eventstate() ne TBDB_NODESTATE_TBFAILED) {
-		print STDERR "    Sending TBFAILED event.\n";
-		$node->SetEventState(TBDB_NODESTATE_TBFAILED());
+#		print STDERR "    Sending TBFAILED event.\n";
+#		$node->SetEventState(TBDB_NODESTATE_TBFAILED());
 	    }
 	    else {
 		$notready++;
@@ -646,8 +759,11 @@ sub DeleteAllSlivers($$)
     foreach my $resource (@resources) {
 	print STDERR "Deleting sliver for $resource\n";
 	if ($resource->Clear($user)) {
-	    print STDERR
-		"DeleteSlivers: Could not delete sliver for $resource\n";
+	    print STDERR "DeleteSlivers: Could not clear $resource\n";
+	    return -1;
+	}
+	if ($resource->Delete()) {
+	    print STDERR "DeleteSlivers: Could not delete $resource\n";
 	    return -1;
 	}
     }
diff --git a/event/lib/event.c b/event/lib/event.c
index 7fc62bdc2..0a348db06 100644
--- a/event/lib/event.c
+++ b/event/lib/event.c
@@ -1,6 +1,6 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2008 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2010 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
@@ -583,6 +583,29 @@ event_notification_alloc(event_handle_t handle, address_tuple_t tuple)
     notification->pubsub_notification = pubsub_notification;
     notification->has_hmac = 0;
 
+    /*
+     * Event version number
+     */
+    if (!event_notification_set_version(handle, notification, "1.0")) {
+        ERROR("pubsub_notification_alloc failed to set version number\n");
+	event_notification_free(handle, notification);
+        return NULL;
+    }
+#ifdef ELVIN_COMPAT
+    if (!event_notification_set_elvincompat(handle, notification)) {
+        ERROR("pubsub_notification_alloc failed to set elvin compat\n");
+	event_notification_free(handle, notification);
+        return NULL;
+    }
+#endif
+#ifdef ELVIN_COMPAT0
+    notification->hashtable = elvin_hashtable_alloc(0, &handle->status);
+    if (notification->hashtable == NULL) {
+        ERROR("pubsub_notification_alloc failed to allocate hashtable\n");
+	event_notification_free(handle, notification);
+        return NULL;
+    }
+#endif
     if (tuple == NULL)
 	    return notification;
 
@@ -638,6 +661,11 @@ event_notification_free(event_handle_t handle,
 
     pubsub_notification_free(handle->server, notification->pubsub_notification,
 			     &handle->status);
+#ifdef ELVIN_COMPAT0
+    if (notification->hashtable) {
+        elvin_hashtable_free(notification->hashtable);
+    }
+#endif
     free(notification);
 
     return 1;
@@ -670,10 +698,13 @@ event_notification_clone(event_handle_t handle,
 	free(clone);
         return 0;
     }
+#ifdef ELVIN_COMPAT0
+    notification->hashtable = elvin_hashtable_clone(notification->hashtable,
+						    &handle->status);
+#endif
     clone->has_hmac = notification->has_hmac;
 
     return clone;
-
 }
 
 
@@ -910,7 +941,6 @@ event_notification_put_double(event_handle_t handle,
         pubsub_error_fprintf(stderr, &handle->status);
         return 0;
     }
-
     return 1;
 }
 
@@ -1487,14 +1517,13 @@ hmac_traverse(void *rock, char *name,
 	if (!strcmp(name, "__hmac__")) 
 		return 1;
 
-#ifdef ELVIN_COMPAT
 	/*
-	 * The elvin gateway sticks this flag in, but we need to ignore it
-	 * when doing hmac computation.
+	 * Never include this in hmac computation. See elvin_gateway and
+	 * the elvin compat code below. 
 	 */
 	if (!strcmp(name, "___elvin_ordered___"))
 		return 1;
-#endif
+
 	switch (type) {
 	case INT32_TYPE:
 		HMAC_Update(ctx,
@@ -1535,60 +1564,60 @@ hmac_traverse(void *rock, char *name,
 
 #ifdef ELVIN_COMPAT
 static int
-hmac_fill_hash(void *rock, char *name,
-	       pubsub_type_t type, pubsub_value_t value,
-	       pubsub_error_t *status)
+notify_order_traverse(void *arg, char *name,
+		      pubsub_type_t type, pubsub_value_t value,
+		      pubsub_error_t *error)
 {
-	struct elvin_hashtable	*table = (struct elvin_hashtable *) rock;
+	pubsub_notification_t	*pubsub_notification =
+		(pubsub_notification_t *) arg;
 
-	if (elvin_hashtable_add(table, name, value, type, status) == -1)
-		return 0;
-	
+	switch (type) {
+	case STRING_TYPE:
+		pubsub_notification_add_string(pubsub_notification,
+					       name, value.pv_string, error);
+		break;
+	case INT32_TYPE:
+		pubsub_notification_add_int32(pubsub_notification,
+					      name, value.pv_int32, error);
+		break;
+	case INT64_TYPE:
+		pubsub_notification_add_int64(pubsub_notification,
+					      name, value.pv_int64, error);
+		break;
+	case REAL64_TYPE:
+		pubsub_notification_add_real64(pubsub_notification,
+					       name, value.pv_real64, error);
+		break;
+
+	case OPAQUE_TYPE:
+		pubsub_notification_add_opaque(pubsub_notification,
+					       name,
+					       (char *)(value.pv_opaque.data),
+					       value.pv_opaque.length,
+					       error);
+		break;
+		       
+	default:
+		break;
+	}
 	return 1;
 }
-#endif
 
 static int
-notification_hmac(pubsub_notification_t *notification, HMAC_CTX *ctx,
-		  pubsub_error_t *status)
+hmac_fill_hash(void *rock, char *name,
+	       pubsub_type_t type, pubsub_value_t value,
+	       pubsub_error_t *status)
 {
-	int retval = 0;
-#ifdef ELVIN_COMPAT
-	struct elvin_hashtable  *table;
-	int			elvin_ordered;
-
-	if (pubsub_notification_get_int32(notification,
-					  "___elvin_ordered___",
-					  &elvin_ordered, status) == 0) {
-		if (!pubsub_notification_traverse(notification, hmac_traverse,
-						  ctx, status)) {
-			return -1;
-		}
-		return 0;
-	}
-	if ((table = elvin_hashtable_alloc(0, status)) == NULL) {
-		return -1;
-	}
-	else if (!pubsub_notification_traverse(notification, hmac_fill_hash,
-					       table, status)) {
-		retval = -1;
-	}
-	else if (!elvin_hashtable_traverse(table, hmac_traverse,
-					   ctx, status)) {
-		retval = -1;
-	}
+	struct elvin_hashtable	*table = (struct elvin_hashtable *) rock;
 
-	elvin_hashtable_free(table);
-	table = NULL;
-#else
-	if (!pubsub_notification_traverse(notification, hmac_traverse,
-					  ctx, status)) {
-		return -1;
+	if (elvin_hashtable_add(table, name, value, type, status) == -1) {
+	    ERROR("hmac_fill_hash failure %s: ", name);
+	    pubsub_error_fprintf(stderr, status);
+	    return 0;
 	}
-#endif
-	
-	return retval;
+	return 1;
 }
+#endif
 
 int
 event_notification_insert_hmac(event_handle_t handle,
@@ -1597,8 +1626,11 @@ event_notification_insert_hmac(event_handle_t handle,
 	HMAC_CTX	ctx;
 	unsigned char	mac[EVP_MAX_MD_SIZE];
 	int		i, len = EVP_MAX_MD_SIZE;
-
-	if (0)
+#ifdef ELVIN_COMPAT
+	struct elvin_hashtable  *hashtable;
+	pubsub_notification_t   *notecopy;
+#endif
+	if (1)
 	    INFO("event_notification_insert_hmac: %d %s\n",
 		 handle->keylen, handle->keydata);
 
@@ -1608,12 +1640,73 @@ event_notification_insert_hmac(event_handle_t handle,
 	}
 #ifdef  ELVIN_COMPAT
 	/*
-	 * Remove this so we recompute the elvin ordering above, since the
-	 * notification might have changed, and the exiting linear order
-	 * will no longer correspond to elvin ordering.
+	 * The point of this is to convert the ordering of the pubsub
+	 * notification into elvin ordering according to its hash
+	 * function, and then create a new notification with that
+	 * order as returned by pubsub_traverse (which is linear). Then
+	 * insert the ___elvin_ordered___ flag. At the client:
+	 *
+	 * 1) With Elvin Compat: That the pubsub notification is in the
+         *     correct order to generate the hmac using pubsub traverse.
+	 *     Without ___elvin_ordered___, the client will convert the
+	 *     ordering to elvin hash order to generate the hmac.
+	 * 2) No Elvin Compat: The client always uses pubsub traverse, so
+	 *     it will get the correct hmac.
+	 *
+	 * This is backwards compatabile with clients linked with old
+	 * versions of the event library, and new versions of the event
+	 * library.
+	 *
+	 * XXX This impl is just a prototype. It would be much more efficient
+	 * to do this as items are added to the notification. Later.
 	 */
+	notecopy = pubsub_notification_alloc(handle->server, &handle->status);
+	if (notecopy == NULL) {
+	    ERROR("event_notification_insert_hmac failed: notecopy alloc\n");
+	    return -1;
+	}
+	hashtable = elvin_hashtable_alloc(0, &handle->status);
+	if (hashtable == NULL) {
+	    ERROR("event_notification_insert_hmac failed: hashtable alloc\n");
+	    pubsub_notification_free(handle->server,
+				     notecopy, &handle->status);
+	    return -1;
+	}
+	
 	pubsub_notification_remove(notification->pubsub_notification,
 				   "___elvin_ordered___", &handle->status);
+
+	/*
+	 * Pubsub puts this in when ELVIN_COMPAT is on. But it it causes
+	 * a duplicate entry when trying to copy the notification. Seems
+	 * that Pubsub should not put this in, since it is not doing
+	 * anything else wrt ELVIN_COMPAT. Still pondering this.
+	 */
+	pubsub_notification_remove(notification->pubsub_notification,
+				   "___PUBSUB___", &handle->status);
+
+	if (!pubsub_notification_traverse(notification->pubsub_notification,
+					  hmac_fill_hash,
+					  hashtable, &handle->status)) {
+	    ERROR("event_notification_insert_hmac failed: hmac_fill_hash\n");
+	    pubsub_notification_free(handle->server,
+				     notecopy, &handle->status);
+	    elvin_hashtable_free(hashtable);
+	    return -1;
+	}
+	if (!elvin_hashtable_traverse(hashtable, notify_order_traverse,
+				      notecopy, &handle->status)) {
+	    ERROR("event_notification_insert_hmac failed: notify_traverse\n");
+	    pubsub_notification_free(handle->server,
+				     notecopy, &handle->status);
+	    elvin_hashtable_free(hashtable);
+	    return -1;
+	}
+	pubsub_notification_free(handle->server,
+				 notification->pubsub_notification,
+				 &handle->status);
+	notification->pubsub_notification = notecopy;
+	elvin_hashtable_free(hashtable);
 #endif	
 	memset(&ctx, 0, sizeof(ctx));
 #if (OPENSSL_VERSION_NUMBER < 0x0090703f)
@@ -1622,15 +1715,17 @@ event_notification_insert_hmac(event_handle_t handle,
 	HMAC_CTX_init(&ctx);
 	HMAC_Init_ex(&ctx, handle->keydata, handle->keylen, EVP_sha1(), NULL);
 #endif
-	if (notification_hmac(notification->pubsub_notification,
-			      &ctx, &handle->status) == -1) {
-		HMAC_cleanup(&ctx);
-		return 1;
+	if (!pubsub_notification_traverse(notification->pubsub_notification,
+					  hmac_traverse,
+					  &ctx, &handle->status)) {
+	    ERROR("event_notification_insert_hmac failed: hmac_traverse\n");
+	    HMAC_cleanup(&ctx);
+	    return 1;
 	}
 	HMAC_Final(&ctx, mac, &len);
 	HMAC_cleanup(&ctx);
 
-	if (0) {
+	if (1) {
 		unsigned char   *up;
 		
 		INFO("event_notification_insert_hmac: %d\n", len);
@@ -1650,6 +1745,15 @@ event_notification_insert_hmac(event_handle_t handle,
 		pubsub_error_fprintf(stderr, &handle->status);
 		return 1;
 	}
+#ifdef ELVIN_COMPAT
+	/*
+	 * Add a flag to indicate this notification is in "elvin order."
+	 * See above. 
+	 */
+	pubsub_notification_add_int32(notification->pubsub_notification,
+				      "___elvin_ordered___", 1,
+				      &handle->status);
+#endif
 	notification->has_hmac = 1;
     	return 0;
 }
@@ -1665,8 +1769,12 @@ event_notification_check_hmac(event_handle_t handle,
 	unsigned char	srcmac[EVP_MAX_MD_SIZE], mac[EVP_MAX_MD_SIZE];
 	char		*pmac;
 	int		i, srclen, len = EVP_MAX_MD_SIZE;
+	int		oldelvin, newelvin, elvin_ordered;
+#ifdef ELVIN_COMPAT
+	struct elvin_hashtable  *hashtable;
+#endif
 
-	if (0)
+	if (1)
 	    INFO("event_notification_check_hmac: %d %s\n",
 		 handle->keylen, handle->keydata);
 		
@@ -1682,7 +1790,7 @@ event_notification_check_hmac(event_handle_t handle,
 	assert(srclen <= EVP_MAX_MD_SIZE);
 	memcpy(srcmac, pmac, srclen);
 
-	if (0) {
+	if (1) {
 		unsigned char   *up;
 		
 		INFO("event_notification_check_hmac1: %d\n", srclen);
@@ -1693,6 +1801,39 @@ event_notification_check_hmac(event_handle_t handle,
 		fprintf(stderr, "\n");
 	}
 	
+	/*
+	 * Look to see if the notification is from an elvin compatabile
+	 * client. Its either old or new, but if the current client is
+	 * not compiled with ELVIN_COMPAT, then we can say something useful
+	 * instead of just dying with an hmac error.
+	 */
+	if (pubsub_notification_get_int32(notification->pubsub_notification,
+					  "___PUBSUB___",
+					  &oldelvin, &handle->status)) {
+	    oldelvin = 0;
+	}
+	if (event_notification_get_elvincompat(handle, notification,
+					       &newelvin)) {
+	    newelvin = 0;
+	}
+	else if (newelvin) {
+	    oldelvin = 1;
+	}
+	if (pubsub_notification_get_int32(notification->pubsub_notification,
+					  "___elvin_ordered___",
+					  &elvin_ordered, &handle->status)) {
+	    elvin_ordered = 0;
+	}
+#ifndef ELVIN_COMPAT
+	if (oldelvin && !elvin_ordered) {
+	    ERROR("Client says its old elvin compatabile, but not ordered!\n");
+	    return -1;
+	}
+	if (newelvin && !elvin_ordered) {
+	    ERROR("Client says its new elvin compatabile, but not ordered!\n");
+	    return -1;
+	}
+#endif
 	memset(&ctx, 0, sizeof(ctx));
 #if (OPENSSL_VERSION_NUMBER < 0x0090703f)
 	HMAC_Init(&ctx, handle->keydata, handle->keylen, EVP_sha1());
@@ -1700,17 +1841,42 @@ event_notification_check_hmac(event_handle_t handle,
 	HMAC_CTX_init(&ctx);
 	HMAC_Init_ex(&ctx, handle->keydata, handle->keylen, EVP_sha1(), NULL);
 #endif
-	
-	/* Compute the MAC */
-	if (notification_hmac(notification->pubsub_notification,
-			      &ctx, &handle->status) == -1) {
+#ifdef ELVIN_COMPAT
+	if ((newelvin || oldelvin) && !elvin_ordered) {
+	    hashtable = elvin_hashtable_alloc(0, &handle->status);
+	    if (hashtable == NULL) {
+		HMAC_cleanup(&ctx);
+	        return -1;
+	    }
+	    if (!pubsub_notification_traverse(notification->pubsub_notification,
+					      hmac_fill_hash,
+					      hashtable, &handle->status)) {
+	        elvin_hashtable_free(hashtable);
 		HMAC_cleanup(&ctx);
 		return -1;
+	    }
+	    if (!elvin_hashtable_traverse(hashtable, hmac_traverse,
+					  &ctx, &handle->status)) {
+
+	        elvin_hashtable_free(hashtable);
+		HMAC_cleanup(&ctx);
+		return -1;
+	    }
+	    elvin_hashtable_free(hashtable);
+	}
+	else
+#endif
+	if (!pubsub_notification_traverse(notification->pubsub_notification,
+					  hmac_traverse,
+					  &ctx, &handle->status)) {
+	    HMAC_cleanup(&ctx);
+	    return -1;
 	}
+
 	HMAC_Final(&ctx, mac, &len);
 	HMAC_cleanup(&ctx);
 
-	if (0) {
+	if (1) {
 		unsigned char   *up;
 		
 		INFO("event_notification_check_hmac2: %d\n", len);
diff --git a/event/lib/event.h b/event/lib/event.h
index 98db2f134..7733de85f 100644
--- a/event/lib/event.h
+++ b/event/lib/event.h
@@ -1,6 +1,6 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2008 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2010 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
@@ -135,6 +135,20 @@ int		address_tuple_free(address_tuple_t);
         event_notification_put_string(handle, note, "___SENDER___", buf)
 #define event_notification_clr_sender(handle, note) \
         event_notification_remove(handle, note, "___SENDER___")
+
+#define event_notification_get_version(handle, note, buf, len) \
+        event_notification_get_string(handle, note, "___VERSION___", buf, len)
+#define event_notification_set_version(handle, note, buf) \
+        event_notification_put_string(handle, note, "___VERSION___", buf)
+#define event_notification_clr_version(handle, note) \
+        event_notification_remove(handle, note, "___VERSION___")
+
+#define event_notification_get_elvincompat(handle, note, buf) \
+        event_notification_get_int32(handle, note, "___ELVIN___", buf)
+#define event_notification_set_elvincompat(handle, note) \
+        event_notification_put_int32(handle, note, "___ELVIN___", 1)
+#define event_notification_clr_elvincompat(handle, note) \
+        event_notification_remove(handle, note, "___ELVIN___")
 #endif /* ifndef NO_EVENT_MACROS */
 #endif /* ifndef SWIG */
 
diff --git a/os/install-rpm b/os/install-rpm
index 2199dc585..ff85435d3 100644
--- a/os/install-rpm
+++ b/os/install-rpm
@@ -1,7 +1,7 @@
-#!/usr/bin/perl -wT
+#!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2005 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
diff --git a/os/install-tarfile b/os/install-tarfile
index 2841f8786..55b77eba4 100755
--- a/os/install-tarfile
+++ b/os/install-tarfile
@@ -1,7 +1,7 @@
-#!/usr/bin/perl -wT
+#!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2005 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
diff --git a/protogeni/etc/protogeni.sql b/protogeni/etc/protogeni.sql
index e3c3d701e..2a5df1c01 100644
--- a/protogeni/etc/protogeni.sql
+++ b/protogeni/etc/protogeni.sql
@@ -197,6 +197,7 @@ CREATE TABLE `geni_resources` (
   `idx` mediumint(8) unsigned NOT NULL default '0',
   `manager_urn` tinytext,
   `created` datetime default NULL,
+  `expires` datetime default NULL,
   `updated` datetime default NULL,
   `slice_idx` mediumint(8) unsigned NOT NULL default '0',
   `credential_idx` mediumint(8) unsigned NOT NULL default '0',
diff --git a/protogeni/lib/GeniCredential.pm.in b/protogeni/lib/GeniCredential.pm.in
index 817372977..606d82102 100644
--- a/protogeni/lib/GeniCredential.pm.in
+++ b/protogeni/lib/GeniCredential.pm.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # GENIPUBLIC-COPYRIGHT
-# Copyright (c) 2008-2009 University of Utah and the Flux Group.
+# Copyright (c) 2008-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package GeniCredential;
@@ -586,7 +586,7 @@ sub Store($)
 
     if (defined($expires)) {
 	my $safe_expires = DBQuoteSpecial($expires);
-	push(@insert_data, "expires=$safe_expires");
+	push(@insert_data, "valid_until=$safe_expires");
     }
 
     # Insert into DB.
diff --git a/protogeni/lib/GeniEmulab.pm.in b/protogeni/lib/GeniEmulab.pm.in
index 4079d7cd7..cc6547653 100644
--- a/protogeni/lib/GeniEmulab.pm.in
+++ b/protogeni/lib/GeniEmulab.pm.in
@@ -42,6 +42,8 @@ use XML::Simple;
 use Data::Dumper;
 use Experiment;
 use libdb qw(TBDB_IFACEROLE_CONTROL TBDB_IFACEROLE_EXPERIMENT);
+use POSIX qw(strftime);
+use Carp;
 
 # Configure variables
 my $TB		   = "@prefix@";
@@ -96,7 +98,7 @@ sub RegisterExperiment($$)
     }
 
     #
-    # Create and register the slice.
+    # Create and register the slice. Slice is returned locked
     #
     print STDERR "Creating new slice for $experiment\n";
     $slice = GeniSlice->CreateFromLocal($experiment, $user);
@@ -104,12 +106,28 @@ sub RegisterExperiment($$)
 	print STDERR "Could not create local slice from $experiment\n";
 	return -1;
     }
+    #
+    # We want this slice to have a long expiration and let Emulab
+    # deal with aging it out.
+    #
+    my $expires = time() + (3600 * 24 * 90);
+    if ($slice->SetExpiration($expires)) {
+	print STDERR "Could not set expiration for $slice\n";
+
+	if ($slice->UnRegister() != 0) {
+	    print STDERR "Could not unregister $slice for $experiment\n";
+	    return -1;
+	}
+	$slice->Delete();
+	return -1;
+    }
     print STDERR "Registering $slice at the ClearingHouse.\n";
     if ($slice->Register() != 0) {
 	$slice->Delete();
-	print STDERR "Could not register slice for $experiment\n";
+	print STDERR "Could not register slice $slice\n";
 	return -1;
     }
+    $slice->UnLock();
     return 0;
 }
 
@@ -135,17 +153,28 @@ sub UnRegisterExperiment($)
     my $context = Genixmlrpc->Context($certificate);
     if (!defined($context)) {
 	print STDERR "Could not create context to talk to clearinghouse\n";
+	return -1;
     }
     #
     # Set the default RPC context. 
     #
     Genixmlrpc->SetContext($context);
 
+    #
+    # Its possible the sa_daemon has the slice locked.
+    #
+    if ($slice->Lock() != 0) {
+	print STDERR "Could not lock slice\n";
+	return -1;
+    }
     print STDERR "Unregistering $slice at the ClearingHouse.\n";
     if ($slice->UnRegister() != 0) {
 	print STDERR "Could not unregister $slice for $experiment\n";
 	return -1;
     }
+    # Needs to move.
+    GeniRegistry::ClientSliver->SliceDelete($slice);
+    
     if ($slice->Delete()) {
 	print STDERR "Could not delete $slice for $experiment\n";
 	return -1;
@@ -159,15 +188,18 @@ sub UnRegisterExperiment($)
 sub CreatePhysNode($)
 {
     my ($node_urn) = @_;
-    my $blob;
     my @ifaces;
     my $ctrliface;
 
+    if (!defined($node_urn) || ! GeniHRN::IsValid($node_urn)) {
+	carp("Not a proper node urn: $node_urn\n");
+	return -1;
+    }
+    print STDERR "$node_urn\n";
+
     my ($auth,$type,$node_id) = GeniHRN::Parse($node_urn);
     my $manager_urn = GeniHRN::Generate($auth, "authority", "cm");
 
-#    print STDERR "$node_urn\n";
-
     #
     # Load the SA cert to act as caller context.
     #
@@ -191,40 +223,24 @@ sub CreatePhysNode($)
 	print STDERR "Could not lookup $manager_urn at ClearingHouse\n";
 	return undef;
     }
-
-    #
-    # Until this urn stuff is done.
-    #
-    my ($translated)  = $authority->hrn() =~ /^([-\w]+)\..*/;
-    my $node_hrn      = $translated . "." . $node_id;
-
-#    print STDERR "$node_hrn\n";
+    my $manager_version = $authority->Version();
+    return undef
+	if (!defined($manager_version));
 
     my $component = GeniComponent->CreateFromRegistry($node_urn);
     if (!defined($component)) {
 	print STDERR "Could not lookup $node_urn at ClearingHouse\n";
 	return undef;
     }
-    my $node = Node->Lookup($component->uuid());
+    my $node = Node->LookupWideArea($node_urn);
     return $node
 	if (defined($node));
 
     print STDERR "Creating local node for $node_urn\n";
 
-    my $credential = GeniRegistry::Client->CreateCredential($component);
-    if (!defined($credential)) {
-	print STDERR "Could not create a credential for $component\n";
-	return undef;
-    }
-    my $registry = GeniRegistry::Client->Create($component,undef,$credential);
-    if (!defined($registry)) {
-	print STDERR "Could not create a registry client for $component\n";
-	return undef;
-    }
-    if ($registry->Resolve($component->uuid(), "Node", \$blob)) {
-	print STDERR "Could not resolve $component at $registry\n";
-	return undef;
-    }
+    my $blob     = $component->Resolve();
+    return undef
+	if (!defined($blob));
 
     my $hrn      = $blob->{'hrn'};
     my $IP       = $blob->{'physctrl'};
@@ -315,8 +331,16 @@ sub CreatePhysNode($)
 						$noderef)->get_nodelist()) {
 		my $component_id = GeniXML::GetText("component_id", $ref);
 		my $role = GeniXML::GetText("role", $ref);
-		if (! defined($role)) {
-		    $role = "expt";
+		if (!defined($role) ||
+		    $role eq "experimental" || $role eq "expt") {
+		    $role = TBDB_IFACEROLE_EXPERIMENT();
+		}
+		elsif ($role eq "ctrl" || $role eq "control") {
+		    $role = TBDB_IFACEROLE_CONTROL();
+		}
+		else {
+		    print STDERR "Unknown role $role for $node_urn!\n";
+		    goto bad;
 		}
 		my $MAC  = "00000000000" . $count;
 
@@ -332,10 +356,10 @@ sub CreatePhysNode($)
 		};
 		push(@ifaces, $ifaceargs);
 
-		if ($role eq "control") {
+		if ($role eq TBDB_IFACEROLE_CONTROL()) {
+		    my $ipv4 = GeniXML::GetText("public_ipv4", $ref);
 		    $ctrliface = $ifaceargs;
-		    $ifaceargs->{'IP'}   = $ref->{'public_ipv4'};
-		    $ifaceargs->{'role'} = TBDB_IFACEROLE_CONTROL();
+		    $ifaceargs->{'IP'}   = $ipv4;
 		}
 		$count++;
 		#print Dumper($ifaceargs);
@@ -353,7 +377,6 @@ sub CreatePhysNode($)
     my $newnode = Node->Create($node_id, undef,
 			       {"role"      => "testnode",
 				"type"      => "pcfedphys",
-				"uuid"      => $uuid,
 				"hostname"  => $hostname,
 				"external_node_id" => $node_urn,
 				"IP"        => $IP});
diff --git a/protogeni/lib/GeniRegistry.pm.in b/protogeni/lib/GeniRegistry.pm.in
index 7ca839cf1..380ea9e13 100644
--- a/protogeni/lib/GeniRegistry.pm.in
+++ b/protogeni/lib/GeniRegistry.pm.in
@@ -129,9 +129,9 @@ sub LOCKED($)           { return $_[0]->{'LOCKED'}; }
 #
 # Class function to create new Geni slice and return the object.
 #
-sub Create($$$$;$)
+sub Create($$$$;$$)
 {
-    my ($class, $certificate, $creator_uuid, $authority, $exptidx) = @_;
+    my ($class, $certificate, $creator_uuid, $authority, $exptidx, $lock) = @_;
     my @insert_data = ();
 
     # Every slice gets a new unique index.
@@ -150,6 +150,8 @@ sub Create($$$$;$)
     push(@insert_data, "sa_uuid='$sa_uuid'");
     push(@insert_data, "exptidx=$exptidx")
 	if (defined($exptidx));
+    push(@insert_data, "locked=now()")
+	if (defined($lock) && $lock);
     
     my $safe_hrn   = DBQuoteSpecial($certificate->hrn());
     my $safe_uuid  = DBQuoteSpecial($certificate->uuid());
@@ -410,6 +412,24 @@ sub Delete($)
     return 0;
 }
 
+#
+# Delete all for a slice.
+#
+sub SliceDelete($$)
+{
+    my ($self, $slice) = @_;
+
+    return -1
+	if (! ref($self));
+
+    my $slice_idx = $self->slice_idx();
+
+    DBQueryWarn("delete from client_slivers where slice_idx='$slice_idx'")
+	or return -1;
+    
+    return 0;
+}
+
 #
 # Stringify for output.
 #
diff --git a/protogeni/lib/GeniResource.pm.in b/protogeni/lib/GeniResource.pm.in
index f011ae8ba..5f5cf9997 100644
--- a/protogeni/lib/GeniResource.pm.in
+++ b/protogeni/lib/GeniResource.pm.in
@@ -26,7 +26,6 @@ use GeniAuthority;
 use GeniComponent;
 use GeniUser;
 use GeniHRN;
-use GeniXML;
 use emutil qw(TBGetUniqueIndex);
 use User;
 use Project;
@@ -35,6 +34,8 @@ use Node;
 use Interface;
 use English;
 use XML::Simple;
+use Date::Parse;
+use Data::Dumper;
 use POSIX qw(strftime);
 use overload ('""' => 'Stringify');
 
@@ -79,6 +80,9 @@ sub Lookup($$;$)
     my $self              = {};
     $self->{'DBROW'}      = $query_result->fetchrow_hashref();
     $self->{'NEWTICKET'}  = undef;
+    $self->{'manager_version'}    = undef;
+    $self->{'rspec_fragment'}     = undef;
+    $self->{'last_rpcerror'}	  = 0;
     bless($self, $class);
 
     return $self;
@@ -115,6 +119,33 @@ sub LookupAll($$)
     return @result;
 }
 
+sub SliceResources($)
+{
+    my ($class, $slice) = @_;
+    my $slice_idx = $slice->idx();
+
+    my $query_result =
+	DBQueryWarn("select idx from geni_resources ".
+		    "where slice_idx='$slice_idx'");
+
+    return undef
+	if (!$query_result);
+    return ()
+	if (!$query_result->numrows);
+
+    my @result = ();
+
+    while (my ($idx) = $query_result->fetchrow_array()) {
+	my $resource = GeniResource->Lookup($idx);
+	if (!defined($resource)) {
+	    print STDERR "Could not get GeniResource for $idx\n";
+	    return undef
+	}
+	push(@result, $resource);
+    }
+    return @result;
+}
+
 #
 # Stringify for output.
 #
@@ -175,15 +206,18 @@ sub pid($)		{ return field($_[0], "pid"); }
 sub eid($)		{ return field($_[0], "eid"); }
 sub created($)          { return field($_[0], "created"); }
 sub updated($)          { return field($_[0], "updated"); }
+sub expires($)          { return field($_[0], "expires"); }
 sub manager_urn($)      { return field($_[0], "manager_urn"); }
 sub slice_idx($)        { return field($_[0], "slice_idx"); }
 sub credential_idx($)   { return field($_[0], "credential_idx"); }
 sub manifest_idx($)     { return field($_[0], "manifest_idx"); }
 sub ticket_idx($)       { return field($_[0], "ticket_idx"); }
 sub newticket_idx($)    { return field($_[0], "newticket_idx"); }
+sub manager_version($)  { return $_[0]->{'manager_version'}; }
+sub last_rpcerror($)    { return $_[0]->{'last_rpcerror'}; }
 
 #
-# Delete a component from the DB.
+# Delete a resource record from the DB.
 #
 sub Delete($)
 {
@@ -260,6 +294,41 @@ sub Update($$)
     return Refresh($self);
 }
 
+#
+# Set the expiration time.
+#
+sub SetExpiration($$)
+{
+    my ($self, $expires) = @_;
+    my $idx = $self->idx();
+
+    if (!defined($expires)) {
+	$expires = "NULL";
+    }
+    elsif ($expires =~ /^\d+$/) {
+	$expires = "FROM_UNIXTIME($expires)";
+    }
+    else {
+	$expires = "'$expires'";
+    }
+    my $query_result =
+	DBQueryWarn("update geni_resources set expires=$expires " .
+		    "where idx='$idx'");
+    
+    return -1
+	if (!$query_result);
+
+    # Has to be in the correct format.
+    $query_result =
+	DBQueryWarn("select expires from geni_resources where idx='$idx'");
+    return -1
+	if (!$query_result || !$query_result->numrows);
+    ($expires) = $query_result->fetchrow_array();
+    
+    $self->{'DBROW'}->{'expires'} = $expires;
+    return 0;
+}
+
 #
 # Return the ticket.
 #
@@ -304,7 +373,10 @@ sub Manifest($)
 	return undef;
     }
     my ($manifest_string) = $query_result->fetchrow_array();
-    my $manifest = GeniXML::Parse($manifest_string);
+    my $manifest = XMLin($manifest_string, KeyAttr => [],
+			 ForceArray => ["node", "link", "interface",
+					"interface_ref", "linkendpoints",
+					"component_manager"]);
 
     $self->{'MANIFEST'}    = $manifest;
     $self->{'MANIFESTSTR'} = $manifest_string;
@@ -320,8 +392,13 @@ sub UpdateManifest($$)
 	return -1;
     }
     my $manifest_idx = $self->manifest_idx();
-    my $manifest_string = $manifest->toStringC14N();
-
+    my $manifest_string =
+	eval { XMLout($manifest, "NoAttr" => 1, RootName => "manifest") };
+    if ($@) {
+	print STDERR "UpdateManifest: XMLout error: $@\n";
+	print STDERR Dumper($manifest);
+	return -1;
+    }
     my $query_result =
 	DBQueryWarn("update geni_manifests set ".
 		    "  manifest=". DBQuoteSpecial($manifest_string) . ", " .
@@ -335,6 +412,47 @@ sub UpdateManifest($$)
     return 0;
 }
 
+#
+# Get Version. Ask the CM what version it is running. 
+#
+sub ManagerVersion($)
+{
+    my ($self) = @_;
+
+    return $self->manager_version()
+	if (defined($self->manager_version()));
+
+    my $manager_urn = $self->manager_urn();
+
+    #
+    # Load the SA cert to act as caller context.
+    #
+    my $certificate = GeniCertificate->LoadFromFile($SACERT);
+    if (!defined($certificate)) {
+	print STDERR "*** Could not load certificate from $SACERT\n";
+	return undef;
+    }
+    my $context = Genixmlrpc->Context($certificate);
+    if (!defined($context)) {
+	print STDERR "*** Could not create context to talk to clearinghouse\n";
+	return undef;
+    }
+    #
+    # Set the default RPC context. 
+    #
+    Genixmlrpc->SetContext($context);
+
+    my $authority = GeniAuthority->Lookup($manager_urn);
+    if (!defined($authority)) {
+	$authority = GeniAuthority->CreateFromRegistry("cm", $manager_urn);
+	if (!defined($authority)) {
+	    print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
+	    return undef;
+	}
+    }
+    $self->{'manager_version'} = $authority->Version();
+    return $self->{'manager_version'};
+}
 
 #
 # Add Resources. We get an rspec to replace the current rspec.
@@ -353,17 +471,13 @@ sub GetTicket($$$$)
     }
     my $manager_urn = $self->manager_urn();
 
-    #
-    # Lets give it a reasonable default time.
-    #
-    my $valid_until = GeniXML::GetText("valid_until", $rspec);
-    if (! defined($valid_until)) {
-	GeniXML::SetText("valid_until", $rspec,
-			 POSIX::strftime("20%y-%m-%dT%H:%M:%S",
-					 gmtime(time() + (3600*6))));
+    my $rspec_string = eval { XMLout($rspec, "NoAttr" => 1) };
+    if ($@) {
+	print STDERR "GetTicket: XMLout error: $@\n";
+	print STDERR Dumper($rspec);
+	return -1;
     }
-
-    my $rspec_string = $rspec->toStringC14N();
+    $rspec_string =~ s/opt\>/rspec\>/g;
 
     #
     # Load the SA cert to act as caller context.
@@ -388,6 +502,7 @@ sub GetTicket($$$$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -428,6 +543,9 @@ sub GetTicket($$$$)
 	}
     }
 
+    my $method_name = undef;
+    my $method_args = {};
+
     #
     # Already have a credential for a sliver, then we want to update
     # the existing ticket we also have on file, and then redeem that.
@@ -446,26 +564,34 @@ sub GetTicket($$$$)
 	    print STDERR "*** Could not get ticket for $self.\n";
 	    return -1;
 	}
-	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-			   $usercontext, "UpdateTicket",
-			   { "credential" => $slice_credential->asString(),
-			     "ticket"     => $ticket->asString(),
-			     "rspec"      => $rspec_string,
-			     "impotent"   => $impotent});
+	$method_name = "UpdateTicket";
+	$method_args->{'ticket'} = $ticket->asString();
     }
     else {
-	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-				   $usercontext, "GetTicket",
-			   { "credential" => $slice_credential->asString(),
-			     "rspec"      => $rspec_string,
-			     "impotent"   => $impotent});
+	$method_name = "GetTicket";
     }
+    $method_args->{'impotent'} = $impotent;
+    $method_args->{'rspec'}    = $rspec_string;
+    if ($manager_version == 1.0) {
+	$method_args->{'credential'} = $slice_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$slice_credential->asString()];
+	$method_args->{'slice_urn'}   = $slice->urn();
+    }
+    else {
+	print STDERR "GeniResource::GetTicket Unknown version at $authority\n";
+	return -1;
+    }
+    $response =
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR "*** Internal error getting/updating ticket for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS) {
 	if ($ticket) {
 	    print STDERR "Could not update ticket $ticket. Error: ";
@@ -531,6 +657,7 @@ sub RedeemTicket($$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -598,27 +725,34 @@ sub RedeemTicket($$)
     #
     # Redeem it. 
     #
-    if (defined($sliver_credential)) {
-	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-				   $usercontext, "UpdateSliver",
-			{ "credential" => $sliver_credential->asString(),
-			  "ticket"     => $ticket->asString(),
-			  "keys"       => \@keys });
+    my $method_name = "RedeemTicket";
+    my $method_args = { "ticket"     => $ticket->asString(),
+			"keys"       => \@keys };
+
+    if ($manager_version == 1.0) {
+	$method_name = "UpdateSliver"
+	    if (defined($sliver_credential));
+	$method_args->{'credential'} = $slice_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$slice_credential->asString()];
+	$method_args->{'slice_urn'}   = $slice->urn();
     }
     else {
-	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-				   $usercontext, "RedeemTicket",
-			{ "credential" => $slice_credential->asString(),
-			  "ticket"     => $ticket->asString(),
-			  "keys"       => \@keys });
+	print STDERR
+	    "GeniResource::RedeemTicket Unknown version at $authority\n";
+	return -1;
     }
+    $response =
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR
 	    "*** Internal error redeeming/updating sliver for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS) {
 	if ($sliver_credential) {
 	    print STDERR "Could not update sliver $sliver_credential Error: ";
@@ -629,6 +763,12 @@ sub RedeemTicket($$)
 	print STDERR "  " . $response->output() . "\n";
 	return -1;
     }
+    #
+    # It worked, so mark the expiration time for the resource.
+    #
+    if ($self->SetExpiration($slice->expires())) {
+	print STDERR "RenewAll: Could not set expiration for $self!\n";
+    }
 
     #
     # Okay, so that worked. Store the new credential and the ticket and
@@ -720,6 +860,7 @@ sub Clear($$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -735,6 +876,20 @@ sub Clear($$)
 	return -1;
     }
 
+    #
+    # Generate a slice credential for the user. 
+    #
+    my $slice_credential =
+	GeniCredential->CreateSigned($slice,
+				     $geniuser,
+				     $GeniCredential::LOCALSA_FLAG);
+    if (!defined($slice_credential)) {
+	print STDERR
+	    "*** Could not create a slice credential for ".
+	    "$slice/$geniuser!\n";
+	return -1;
+    }
+
     #
     # Load sliver credential if we have it.
     #
@@ -764,15 +919,31 @@ sub Clear($$)
     }
 
     if (defined($sliver_credential)) {
-	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-			   $usercontext, "DeleteSliver",
-			   { "credential" => $sliver_credential->asString()});
+	my $method_name = "DeleteSlice";
+	my $method_args = undef;
 
+	if ($manager_version == 1.0) {
+	    $method_args->{'credential'} = $slice_credential->asString();
+	}
+	elsif ($manager_version == 2.0) {
+	    $method_args->{'credentials'} = [$slice_credential->asString()];
+	    $method_args->{'slice_urn'}   = $slice->urn();
+	}
+	else {
+	    print STDERR
+		"GeniResource::Clear Unknown version at $authority\n";
+	    return -1;
+	}
+	print STDERR "Deleting sliver on $authority\n";
+	$response =
+	    Genixmlrpc::CallMethod($authority->url(), $usercontext,
+				   $method_name, $method_args);
 	if (!defined($response)) {
+	    $self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	    print STDERR "*** Internal error deleting sliver for $self\n";
 	    return -1;
 	}
+	$self->{'last_rpcerror'} = $response->code();
 	if ($response->code() != GENIRESPONSE_SUCCESS &&
 	    $response->code() != GENIRESPONSE_SEARCHFAILED) {
 	    print STDERR "Could not delete sliver for $self. Error: ";
@@ -781,31 +952,45 @@ sub Clear($$)
 	}
 	$sliver_credential->Delete();
 	$self->Update({"credential_idx" => 0});
-    }
 
-    if (defined($newticket)) {
 	#
-	# Generate a slice credential for the user. 
+	# Delete this now; no point in waiting for the CM to tell us.
 	#
-	my $slice_credential =
-	    GeniCredential->CreateSigned($slice,
-					 $geniuser,
-					 $GeniCredential::LOCALSA_FLAG);
-	if (!defined($slice_credential)) {
+	# Needs to move elsewhere I think.
+	#
+	my $clientsliver =
+	    GeniRegistry::ClientSliver->LookupByAuthority($slice,
+							  $manager_urn);
+	$clientsliver->Delete()
+	    if (defined($clientsliver));
+    }
+
+    if (defined($newticket)) {
+	my $method_name = "ReleaseTicket";
+	my $method_args = { "ticket" => $newticket->asString() };
+
+	if ($manager_version == 1.0) {
+	    $method_args->{'credential'} = $slice_credential->asString();
+	}
+	elsif ($manager_version == 2.0) {
+	    $method_args->{'credentials'} = [$slice_credential->asString()];
+	    $method_args->{'slice_urn'}   = $slice->urn();
+	}
+	else {
 	    print STDERR
-		"*** Could not create a slice credential for ".
-		"$slice/$geniuser!\n";
+		"GeniResource::Clear Unknown version at $authority\n";
 	    return -1;
 	}
+	print STDERR "Deleting $newticket on $authority\n";
 	$response =
-	    Genixmlrpc::CallMethod($authority->url(),
-			   $usercontext, "ReleaseTicket",
-			   { "ticket" => $newticket->asString(),
-			     "credential" => $slice_credential->asString()});
+	    Genixmlrpc::CallMethod($authority->url(), $usercontext,
+				   $method_name, $method_args);
 	if (!defined($response)) {
+	    $self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	    print STDERR "*** Internal error releasing ticket for $self\n";
 	    return -1;
 	}
+	$self->{'last_rpcerror'} = $response->code();
 	if ($response->code() != GENIRESPONSE_SUCCESS &&
 	    $response->code() != GENIRESPONSE_SEARCHFAILED) {
 	    print STDERR "Could not release ticket for $self. Error: ";
@@ -853,6 +1038,7 @@ sub Purge($$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -914,22 +1100,46 @@ sub Purge($$)
 	
 	$self->Update({"manifest_idx" => 0});
     }
+    my $method_name = "DeleteSlice";
+    my $method_args = {};
 
+    if ($manager_version == 1.0) {
+	$method_args->{'credential'} = $slice_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$slice_credential->asString()];
+	$method_args->{'slice_urn'}   = $slice->urn();
+    }
+    else {
+	print STDERR
+	    "GeniResource::Purge Unknown version at $authority\n";
+	return -1;
+    }
     $response =
-	Genixmlrpc::CallMethod($authority->url(),
-			       $usercontext, "DeleteSlice",
-			   { "credential" => $slice_credential->asString()});
-
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR "*** Internal error deleting sliver for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS &&
 	$response->code() != GENIRESPONSE_SEARCHFAILED) {
 	print STDERR "Could not delete sliver for $self. Error: ";
 	print STDERR "  " . $response->output() . "\n";
 	return -1;
     }
+    #
+    # Delete this now; no point in waiting for the CM to tell us.
+    #
+    # Needs to move elsewhere I think.
+    #
+    my $clientsliver =
+	GeniRegistry::ClientSliver->LookupByAuthority($slice,
+						      $manager_urn);
+    $clientsliver->Delete()
+	if (defined($clientsliver));
     return 0;
 }
 
@@ -968,6 +1178,7 @@ sub StartSliver($$$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -995,7 +1206,7 @@ sub StartSliver($$$)
 	    "*** Could not create a slice credential for $slice/$geniuser!\n";
 	return -1;
     }
-    if (!defined(self->Manifest())) {
+    if (!defined($self->Manifest())) {
 	print STDERR "*** Could not load manifest\n";
 	return -1;
     }
@@ -1003,16 +1214,31 @@ sub StartSliver($$$)
     #
     # Do it. 
     #
+    my $method_name = "StartSliver";
+    my $method_args = { "manifest" => $self->{'MANIFESTSTR'} };
+
+    if ($manager_version == 1.0) {
+	$method_args->{'credential'} = $sliver_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$slice_credential->asString()];
+	$method_args->{'slice_urn'}   = $slice->urn();
+    }
+    else {
+	print STDERR
+	    "GeniResource::StartSliver Unknown version at $authority\n";
+	return -1;
+    }
     $response =
-	Genixmlrpc::CallMethod($authority->url(),
-			       $usercontext, "StartSliver",
-		       { "credential" => $sliver_credential->asString(),
-			 "manifest"   => $self->{'MANIFESTSTR'} });
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR
 	    "*** Internal error starting sliver for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS) {
 	print STDERR "Could not start sliver $sliver_credential Error: ";
 	print STDERR "  " . $response->output() . "\n";
@@ -1027,7 +1253,7 @@ sub StartSliver($$$)
 sub SliverStatus($$$$)
 {
     my ($self, $user, $pref, $urn) = @_;
-    my $response;
+    my ($response);
 
     #
     # Load sliver credential if we have it.
@@ -1056,6 +1282,7 @@ sub SliverStatus($$$$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -1074,22 +1301,47 @@ sub SliverStatus($$$$)
     #
     # Do it. 
     #
-    $response =
-	Genixmlrpc::CallMethod($authority->url(),
-			       $usercontext, "SliverStatus",
-		       { "credential" => $sliver_credential->asString() });
+    my $method_name = "SliverStatus";
+    my $method_args = {};
 
+    if ($manager_version == 1.0) {
+	$method_args->{'credential'} = $sliver_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$sliver_credential->asString()];
+	$method_args->{'slice_urn'}   = $slice->urn();
+    }
+    else {
+	print STDERR
+	    "GeniResource::SliverStatus Unknown version at $authority\n";
+	return -1;
+    }
+    $response =
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR
 	    "*** Internal error getting sliver status for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS) {
 	print STDERR "Could not get sliver status $sliver_credential Error: ";
 	print STDERR "  " . $response->output() . "\n";
 	return -1;
-    }
-    $$pref = $response->value()
+    } 
+    my $blob = {
+	"state"   => "unknown",
+	"status"  => $response->value()->{'status'},
+    };
+    if ($manager_version == 1.0) {
+	$blob->{'details'} = $response->value()->{'detailsNew'};
+    }
+    elsif ($manager_version == 2.0) {
+	$blob->{'details'} = $response->value()->{'details'};
+    }
+    $$pref = $blob
 	if (defined($pref));
     
     return 0;
@@ -1105,7 +1357,7 @@ sub Discover($$$)
 
     my $slice = GeniSlice->Lookup($self->slice_idx());
     if (!defined($slice)) {
-	print STDERR "*** SliverStatus: No slice for $self\n";
+	print STDERR "*** Discover: No slice for $self\n";
 	return -1;
     }
     my $manager_urn = $self->manager_urn();
@@ -1114,6 +1366,7 @@ sub Discover($$$)
 	print STDERR "*** Could not find $manager_urn at ClearingHouse\n";
 	return -1;
     }
+    my $manager_version = $self->ManagerVersion();
 
     #
     # Create a Geni user from current user doing the operation.
@@ -1145,17 +1398,30 @@ sub Discover($$$)
     #
     # Do it. 
     #
-    $response =
-	Genixmlrpc::CallMethod($authority->url(),
-			       $usercontext, "DiscoverResources",
-		       { "credential" => $slice_credential->asString(),
-			 "available"  => "true" });
+    my $method_name = "DiscoverResources";
+    my $method_args = { "available" => "true" };
 
+    if ($manager_version == 1.0) {
+	$method_args->{'credential'} = $slice_credential->asString();
+    }
+    elsif ($manager_version == 2.0) {
+	$method_args->{'credentials'} = [$slice_credential->asString()];
+    }
+    else {
+	print STDERR
+	    "GeniResource::DiscoverResources Unknown version at $authority\n";
+	return -1;
+    }
+    $response =
+	Genixmlrpc::CallMethod($authority->url(), $usercontext,
+			       $method_name, $method_args);
     if (!defined($response)) {
+	$self->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	print STDERR
 	    "*** Internal error doing DiscoverResources for $self\n";
 	return -1;
     }
+    $self->{'last_rpcerror'} = $response->code();
     if ($response->code() != GENIRESPONSE_SUCCESS) {
 	print STDERR "Could not discover resources $slice_credential Error: ";
 	print STDERR "  " . $response->output() . "\n";
@@ -1168,16 +1434,82 @@ sub Discover($$$)
 }
 
 #
-# Class method to renew all resources in use.
+# Class method to renew resources for an experiment/slice.
 #
-sub RenewAll()
+sub RenewExperimentResources($;$)
 {
-    my $query_result =
-	DBQueryWarn("select idx from geni_resources");
-    return -1
-	if (!$query_result);
+    my ($experiment, $force) = @_;
+
+    my $slice = GeniSlice->LookupByExperiment($experiment);
+    if (!defined($slice)) {
+	print STDERR
+	    "*** RenewExperimentResources: No slice for $experiment\n";
+	return -1;
+    }
+    return RenewSliceResources($slice, $force);
+}
+
+sub RenewSliceResources($;$)
+{
+    my ($slice, $force) = @_;
+
+    $force = 0
+	if (!defined($force));
+
+    #
+    # Get the resource objects.
+    #
+    my @resources = GeniResource->SliceResources($slice);
     return 0
-	if (!$query_result->numrows);
+	if (!@resources);
+
+    #
+    # If the expiration is more then 30 days out, do not bother.
+    #
+    my $when = str2time($slice->expires());
+    return 0
+	if (!$force && $when > time() + (3600 * 24 * 30));
+
+    # Good default
+    my $expires = time() + (3600 * 24 * 90);
+
+    if ($slice->SetExpiration($expires)) {
+	print STDERR "RenewSliceResources: ".
+	    "Could not set expiration for $slice!\n";
+	return -1
+    }
+
+    #
+    # Cull out resources that are more then 30 days out.
+    #
+    if (!$force) {
+	my @tmp = ();
+	foreach my $resource (@resources) {
+	    # Nothing to renew yet
+	    next
+		if ($resource->credential_idx());
+	    
+	    if (!defined($resource->expires())) {
+		push(@tmp, $resource);
+		next;
+	    }
+	    my $resource_expires = str2time($resource->expires());
+	    push(@tmp, $resource)
+		if (time() + (3600 * 24 * 30) > $resource_expires);
+	}
+	@resources = @tmp;
+	return 0
+	    if (!@resources);
+    }
+    return RenewResources(@resources);
+}
+
+#
+# Class method to renew a set of resources.
+#
+sub RenewResources(@)
+{
+    my @resources = @_;
 
     #
     # Load the SA cert to act as caller context.
@@ -1198,19 +1530,10 @@ sub RenewAll()
     #
     Genixmlrpc->SetContext($context);
 
-    # Good default
-    my $valid_until =
-	POSIX::strftime("20%y-%m-%dT%H:%M:%S", gmtime(time() + (3600*24)));
-
     # Hash the slice credentials so we generate them only once.
     my %credentials;
 
-    while (my ($idx) = $query_result->fetchrow_array()) {
-	my $resource = GeniResource->Lookup($idx);
-	if (!defined($resource)) {
-	    print STDERR "RenewAll: Could not lookup resource $idx\n";
-	    next;
-	}
+    foreach my $resource (@resources) {
 	# No sliver, skip.
 	next
 	    if (! $resource->credential_idx());
@@ -1227,6 +1550,7 @@ sub RenewAll()
 	    print STDERR "RenewAll: Could not locate $manager_urn\n";
 	    next;
 	}
+	my $manager_version = $resource->ManagerVersion();
 
 	#
 	# Generate a slice credential.
@@ -1252,22 +1576,45 @@ sub RenewAll()
 	#
 	print STDERR "Renewing $slice on $authority\n";
 	
-	my $response =
-	    Genixmlrpc::CallMethod($authority->url(),
-				   undef, "RenewSliver",
-			   { "credential"  => $slice_credential->asString(),
-			     "valid_until" => $valid_until });
+	my $method_name = "RenewSlice";
+	my $method_args = { };
 
+	if ($manager_version == 1.0) {
+	    $method_args->{'credential'} = $slice_credential->asString();
+	}
+	elsif ($manager_version == 2.0) {
+	    $method_args->{'credentials'} = [$slice_credential->asString()];
+	    $method_args->{'slice_urn'}   = $slice->urn();
+	}
+	else {
+	    print STDERR
+		"GeniResource::RenewAll Unknown version at $authority\n";
+	    return -1;
+	}
+	my $response =
+	    Genixmlrpc::CallMethod($authority->url(), $context,
+				   $method_name, $method_args);
 	if (!defined($response)) {
+	    $resource->{'last_rpcerror'} = GENIRESPONSE_RPCERROR();
 	    print STDERR
 		"RenewAll: Internal error doing renew for $resource\n";
 	    next;
 	}
+	$resource->{'last_rpcerror'} = $response->code();
 	if ($response->code() != GENIRESPONSE_SUCCESS) {
-	    print STDERR "RenewAll: Could renew $slice_credential Error: ";
+	    print STDERR "RenewAll: Could not renew $slice_credential Error: ";
 	    print STDERR "  " . $response->output() . "\n";
 	    next;
 	}
+	#
+	# It worked, so mark this resource as renewed. Since an individual
+	# resource could fail to renew, we want to know which ones to come
+	# back to later.
+	#
+	if ($resource->SetExpiration($slice->expires())) {
+	    print STDERR "RenewAll: Could not set expiration for $resource!\n";
+	    next;
+	}
     }
     return 0;
 }
diff --git a/protogeni/lib/GeniSA.pm.in b/protogeni/lib/GeniSA.pm.in
index d896036ac..823d4b118 100644
--- a/protogeni/lib/GeniSA.pm.in
+++ b/protogeni/lib/GeniSA.pm.in
@@ -159,14 +159,19 @@ sub GetCredential($)
     #
     if ($type eq "Slice") {
 	my $slice = GeniSlice->Lookup( defined( $urn ) ? $urn : $uuid );
+
 	return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
 				    "No such Slice")
 	    if (!defined($slice));
-	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
-				    "Not your slice!")
-	    if ($slice->creator_uuid() ne $this_user->uuid() &&
-		!$slice->IsBound($this_user));
-	
+	if ($slice->Lock() != 0) {
+	    return GeniResponse->BusyResponse("slice");
+	}
+	if ($slice->creator_uuid() ne $this_user->uuid() &&
+	    !$slice->IsBound($this_user)) {
+	    $slice->UnLock();
+	    return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
+					"Not your slice!");
+	}
 	#
 	# Return a credential for the slice.
 	#
@@ -174,9 +179,11 @@ sub GetCredential($)
 	    GeniCredential->CreateSigned($slice,
 					 $this_user,
 					 $GeniCredential::LOCALSA_FLAG);
-	return GeniResponse->Create(GENIRESPONSE_ERROR)
-	    if (!defined($slice_credential));
-
+	if (!defined($slice_credential)) {
+	    $slice->UnLock();
+	    return GeniResponse->Create(GENIRESPONSE_ERROR);
+	}
+	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_SUCCESS,
 				    $slice_credential->asString());
     }
@@ -317,7 +324,9 @@ sub Resolve($)
 	    return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
 					"No such slice registered here");
 	}
-
+	if ($slice->Lock() != 0) {
+	    return GeniResponse->BusyResponse("slice");
+	}
 	# Return a blob.
 	my $blob = { "hrn"          => $slice->hrn(),
 		     "uuid"         => $slice->uuid(),
@@ -327,6 +336,7 @@ sub Resolve($)
 							  "slice",
 							  $slice->slicename() )
 		 };
+	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
     }
     
@@ -504,16 +514,19 @@ sub Register($)
 	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 					"$hrn or $uuid already registered");
 	}
+	# Slice is created as locked.
 	my $slice = GeniSlice->Create($certificate,
 				      $credential->owner_uuid(),
-				      $authority);
+				      $authority, undef, 1);
 	if (!defined($slice)) {
 	    $certificate->Delete();
 	    print STDERR "Could not create new slice object\n";
 	    return GeniResponse->Create(GENIRESPONSE_ERROR);
 	}
+	
 	if (defined($expires) && $slice->SetExpiration($expires) != 0) {
 	    print STDERR "Could not set slice expiration to $expires\n";
+	    $slice->Delete();
 	    return GeniResponse->Create(GENIRESPONSE_ERROR);
 	}
 
@@ -529,16 +542,6 @@ sub Register($)
 	    return GeniResponse->Create(GENIRESPONSE_ERROR);
 	}
 
-	#
-	# Add the bindings locally before registration.
-	# 
-	if (0 && defined($userbindings)) {
-	    foreach my $binding_uuid (@{ $userbindings }) {
-		my $binding_user = GeniUser->Lookup($binding_uuid, 1);
-		$slice->BindUser($binding_user);
-	    }
-	}
-
 	#
 	# Register new slice at the clearinghouse.
 	#
@@ -549,7 +552,7 @@ sub Register($)
 	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 			"Could not register new slice at clearinghouse");
 	}
-
+	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_SUCCESS,
 				    $slice_credential->asString());
     }
@@ -662,21 +665,37 @@ sub Remove($)
 	    return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
 					"No such slice");
 	}
+	if ($slice->Lock() != 0) {
+	    return GeniResponse->BusyResponse("slice");
+	}
+	#
+	# Not allowed to delete a cooked mode slice via this interface.
+	#
+	if ($slice->exptidx()) {
+	    $slice->UnLock();
+	    return GeniResponse->Create(GENIRESPONSE_REFUSED, undef,
+					"Cooked mode Slice");
+	}
+	
 	#
 	# Not allowed to delete a slice that has not expired since
 	# that would make it impossible to control any existing
 	# slivers.
 	#
 	if (! $slice->IsExpired()) {
+	    $slice->UnLock();
 	    return GeniResponse->Create(GENIRESPONSE_REFUSED, undef,
 					"Slice has not expired");
 	}
-
+	# Needs to move.
+	GeniRegistry::ClientSliver->SliceDelete($slice);
+	
 	#
 	# Remove from the clearing house.
 	#
 	if ($slice->UnRegister()) {
 	    print STDERR "Could not delete $slice from clearinghouse!\n";
+	    $slice->UnLock();
 	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 					(defined($uuid)?$uuid:$hrn)
 					. " could not be unregistered");
@@ -693,60 +712,6 @@ sub Remove($)
     return GeniResponse->Create(GENIRESPONSE_UNSUPPORTED);
 }
 
-#
-# Discover resources for one of our slices. Just return the list of
-# components that the clearinghouse gives us. Ug.
-#
-sub DiscoverResources($)
-{
-    my ($argref) = @_;
-    my $cred  = $argref->{'credential'};
-    my $rspec = $argref->{'rspec'};
-
-    if (! defined($cred)) {
-	return GeniResponse->MalformedArgsResponse();
-    }
-    if (! (defined($rspec) && ($rspec =~ /^[-\w]+$/))) {
-	GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
-			     "Improper rspec");	
-    }
-    my $credential = GeniCredential->CreateFromSigned($cred);
-    if (!defined($credential)) {
-	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-				    "Could not create GeniCredential object");
-    }
-
-    #
-    # Make sure the credential was issued to the caller.
-    #
-    if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
-	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-				    "This is not your credential!");
-    }
-
-    my $slice = GeniSlice->Lookup($credential->target_uuid());
-    if (!defined($slice)) {
-	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-				    "Unknown slice for this credential");
-    }
-
-    $credential->HasPrivilege( "authority" ) or
-	$credential->HasPrivilege( "resolve" ) or
-	return GeniResponse->Create( GENIRESPONSE_FORBIDDEN, undef,
-				     "Insufficient privilege" );
-
-    #
-    # Ask clearing house for a list of components.
-    #
-    my @components;
-    if ($slice->DiscoverResources(\@components)) {
-	print STDERR "Could not DiscoverResources for $slice\n";
-	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-			    "Could not discover resources at clearinghouse");
-    }
-    return GeniResponse->Create(GENIRESPONSE_SUCCESS, 0);
-}
-
 #
 # Return ssh keys.
 #
@@ -856,10 +821,15 @@ sub BindToSlice($)
 	return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED,
 				    undef, "No such user here");
     }
+    if ($slice->Lock() != 0) {
+	return GeniResponse->BusyResponse("slice");
+    }
     if ($slice->BindUser($target_user) != 0) {
 	print STDERR "Could not bind $target_user to $slice\n";
+	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_ERROR);
     }
+    $slice->UnLock();
     return GeniResponse->Create(GENIRESPONSE_SUCCESS);
 }
 
@@ -938,6 +908,13 @@ sub RenewSlice($)
 	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 				    "Unknown slice for this credential");
     }
+    #
+    # Not allowed to renew a cooked mode slice via this interface.
+    #
+    if ($slice->exptidx()) {
+	return GeniResponse->Create(GENIRESPONSE_REFUSED, undef,
+				    "Cooked mode Slice");
+    }
     
     $credential->HasPrivilege( "pi" ) or
 	$credential->HasPrivilege( "bind" ) or
@@ -971,10 +948,15 @@ sub RenewSlice($)
 	$message = "Expiration is in the past";
 	goto bad;
     }
+    if ($slice->Lock() != 0) {
+	return GeniResponse->BusyResponse("slice");
+    }
     if ($slice->SetExpiration($when) != 0) {
 	$message = "Could not set expiration time";
+	$slice->UnLock();
 	goto bad;
     }
+    $slice->UnLock();
     return GeniResponse->Create(GENIRESPONSE_SUCCESS);
 
   bad:
@@ -1049,7 +1031,9 @@ sub RegisterSliver($)
 	print STDERR "No URN in $credential\n";
 	return GeniResponse->Create(GENIRESPONSE_ERROR);
     }
-        
+    if ($slice->Lock() != 0) {
+	return GeniResponse->BusyResponse("slice");
+    }
     #
     # See if one already exists; overwrite it.
     #
@@ -1063,8 +1047,10 @@ sub RegisterSliver($)
     if (!defined($clientsliver)) {
 	print STDERR "Could not register sliver for $slice_urn\n";
 	print STDERR Dumper($blob);
+	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_ERROR);
     }
+    $slice->UnLock();
     return GeniResponse->Create(GENIRESPONSE_SUCCESS);
 }
 
@@ -1124,7 +1110,9 @@ sub UnRegisterSliver($)
 	print STDERR "No URN in $credential\n";
 	return GeniResponse->Create(GENIRESPONSE_ERROR);
     }
-        
+    if ($slice->Lock() != 0) {
+	return GeniResponse->BusyResponse("slice");
+    }
     #
     # See if one already exists; overwrite it.
     #
@@ -1134,6 +1122,7 @@ sub UnRegisterSliver($)
     $clientsliver->Delete()
 	if (defined($clientsliver));
     
+    $slice->UnLock();
     return GeniResponse->Create(GENIRESPONSE_SUCCESS);
 }
 
diff --git a/protogeni/lib/GeniSlice.pm.in b/protogeni/lib/GeniSlice.pm.in
index 48ca555ad..6b9970cc7 100644
--- a/protogeni/lib/GeniSlice.pm.in
+++ b/protogeni/lib/GeniSlice.pm.in
@@ -86,12 +86,12 @@ sub Lookup($$)
 #
 # Class function to create new Geni slice and return the object.
 #
-sub Create($$$$;$)
+sub Create($$$$;$$)
 {
-    my ($class, $certificate, $creator_uuid, $authority, $exptidx) = @_;
+    my ($class, $certificate, $creator_uuid, $authority, $exptidx, $lock) = @_;
 
     my $slice = GeniRegistry::GeniSlice->Create($certificate, $creator_uuid,
-						$authority, $exptidx);
+						$authority, $exptidx, $lock);
 
     return undef
 	if (!defined($slice));
@@ -274,8 +274,9 @@ sub CreateFromLocal($$$)
 	    "Could not generate new certificate $experiment\n";
 	return undef;
     }
+    # Create the slice as locked.
     my $slice = GeniSlice->Create($certificate, $user->uuid(),
-				  $authority, $experiment->idx());
+				  $authority, $experiment->idx(), 1);
     $certificate->Delete()
 	if (!defined($slice));
 
diff --git a/protogeni/scripts/cleanupslice.in b/protogeni/scripts/cleanupslice.in
index b3611ff86..1deaf9bfc 100644
--- a/protogeni/scripts/cleanupslice.in
+++ b/protogeni/scripts/cleanupslice.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # GENIPUBLIC-COPYRIGHT
-# Copyright (c) 2008-2009 University of Utah and the Flux Group.
+# Copyright (c) 2008-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use strict;
@@ -43,6 +43,7 @@ $| = 1;
 use lib "@prefix@/lib";
 use GeniDB;
 use GeniSlice;
+use GeniRegistry;
 use GeniUtil;
 use GeniCM;
 use Genixmlrpc;
@@ -152,6 +153,9 @@ sub DoSA()
 	
 	print STDERR "Deleting locally anyway ...\n";
     }
+    # Needs to move.
+    GeniRegistry::ClientSliver->SliceDelete($slice);
+
     if ($slice->Delete()) {
 	fatal("Could not delete $slice from slice authority DB!");
     }
diff --git a/protogeni/scripts/sa_daemon.in b/protogeni/scripts/sa_daemon.in
index 1795c93d3..6b7cde00c 100644
--- a/protogeni/scripts/sa_daemon.in
+++ b/protogeni/scripts/sa_daemon.in
@@ -82,6 +82,7 @@ require GeniUtil;
 require GeniSlice;
 require GeniCertificate;
 require GeniUser;
+require libGeni;
 use Experiment;
 use Node;
 use libtestbed;
@@ -122,11 +123,16 @@ my $grace   = $hours * 3600;
 #
 sub WarnSlices()
 {
+    #
+    # This query purposely skips slices that are attached to cooked
+    # mode experiments; those that have a non-null exptidx of a local
+    # experiment.
+    #
     my $query_result =
 	GeniDB::DBQueryWarn("select idx,expires,UNIX_TIMESTAMP(expires), ".
 			    "   UNIX_TIMESTAMP(created) ".
 			    "  from geni_slices ".
-			    "where shutdown is null");
+			    "where shutdown is null and exptidx is null");
 
     while (my ($idx,$expires,$stamp,$created) =
 	   $query_result->fetchrow_array()) {
@@ -229,11 +235,16 @@ sub WarnSlices()
 #
 sub ExpireSlices()
 {
+    #
+    # This query purposely skips slices that are attached to cooked
+    # mode experiments; those that have a non-null exptidx of a local
+    # experiment.
+    #
     my $query_result =
 	GeniDB::DBQueryWarn("select idx from geni_slices ".
 			    "where UNIX_TIMESTAMP(now()) > ".
 			    "      UNIX_TIMESTAMP(expires) and ".
-			    "      shutdown is null");
+			    "      shutdown is null and exptidx is null");
 
     while (my ($idx) = $query_result->fetchrow_array()) {
 	my $slice = GeniSlice->Lookup($idx);
@@ -281,6 +292,79 @@ sub ExpireSlices()
     }
 }
 
+#
+# Renew slices and slivers for local cooked mode experiments.
+#
+sub RenewSlices()
+{
+    my $query_result =
+	GeniDB::DBQueryWarn("select idx,UNIX_TIMESTAMP(expires), ".
+			    "  from geni_slices ".
+			    "where shutdown is null and exptidx is not null");
+
+    while (my ($idx,$stamp) = $query_result->fetchrow_array()) {
+	#
+	# Wait until within $hours of expiration.
+	#
+	if ($stamp > time() + $grace) {
+	    next;
+	}
+	my $slice = GeniSlice->Lookup($idx);
+
+	if (!defined($slice)) {
+	    # Slice is gone, lets not worry.
+	    next;
+	}
+	if ($slice->Lock() != 0) {
+	    print STDERR "Could not lock slice $slice.\n";
+	    goto skip;
+	}
+	#
+	# There should be an experiment.
+	#
+	my $experiment = $slice->GetExperiment();
+	if (!defined($experiment)) {
+	    print STDERR "No experiment for $slice.\n";
+	    $slice->UnLock();
+	    goto skip;
+	}
+	#
+	# And we want to take the lock to prevent it from changing.
+	#
+	if ($experiment->LockTables()) {
+	    print STDERR "Could not lock tables for $experiment\n";
+	    $slice->UnLock();
+	    goto skip;
+	}
+	if ($experiment->Lock(undef, 1)) {
+	    print STDERR "Could not lock $experiment\n";
+	    $slice->UnLock();
+	    goto skip;
+	}
+	if ($impotent) {
+	    print STDERR "Would renew $slice\n";
+	    $experiment->Unlock();
+	    $slice->UnLock();
+	    goto skip;
+	}
+	
+	if (libGeni::RenewSlivers($experiment)) {
+	    print STDERR "Could not renew $slice/$experiment\n";
+
+	    SENDMAIL($TBOPS,
+		     "SA daemon renew failure",
+		     "Could not renew $slice/$experiment\n",
+		     $TBOPS);
+	}
+	$experiment->Unlock();
+	$slice->UnLock();
+	
+      skip:
+	$slice->Flush()
+	    if (defined($slice));
+    }
+}
+
 #
 # Setup a signal handler for newsyslog.
 #
@@ -297,6 +381,7 @@ while (1) {
 
     WarnSlices();
     ExpireSlices();
+#    RenewSlices();
 
     # Be certain stale info is gone.
     GeniUtil::FlushCaches();
diff --git a/protogeni/xmlrpc/protogeni-cm.pl.in b/protogeni/xmlrpc/protogeni-cm.pl.in
index 4f79f3f70..3ba637de4 100644
--- a/protogeni/xmlrpc/protogeni-cm.pl.in
+++ b/protogeni/xmlrpc/protogeni-cm.pl.in
@@ -33,7 +33,7 @@ BEGIN { $GENI_DBNAME = "geni-cm"; }
 # Configure variables
 my $EMULAB_PEMFILE = "@prefix@/etc/genicm.pem";
 my $MAINSITE 	   = @TBMAINSITE@;
-my $VERSION	   = "1.0";
+my $VERSION	   = "2.0";
 
 # Testbed libraries.
 use lib '@prefix@/lib';
diff --git a/protogeni/xmlrpc/protogeni-sa.pl.in b/protogeni/xmlrpc/protogeni-sa.pl.in
index dd79da435..779e57834 100755
--- a/protogeni/xmlrpc/protogeni-sa.pl.in
+++ b/protogeni/xmlrpc/protogeni-sa.pl.in
@@ -142,7 +142,6 @@ my $responder = Frontier::Responder->new( "methods" => {
     "Resolve"           => \&GeniSA::Resolve,
     "Register"          => \&GeniSA::Register,
     "Remove"            => \&GeniSA::Remove,
-    "DiscoverResources" => \&GeniSA::DiscoverResources,
     "GetKeys"	        => \&GeniSA::GetKeys,
     "BindToSlice"	=> \&GeniSA::BindToSlice,
     "Shutdown"          => \&GeniSA::Shutdown,
diff --git a/tbsetup/endexp.in b/tbsetup/endexp.in
index 02fa41e85..19e6e2cd9 100755
--- a/tbsetup/endexp.in
+++ b/tbsetup/endexp.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
diff --git a/tbsetup/libvtop.pm.in b/tbsetup/libvtop.pm.in
index f2837fd83..41442e901 100644
--- a/tbsetup/libvtop.pm.in
+++ b/tbsetup/libvtop.pm.in
@@ -1243,14 +1243,16 @@ sub GenVirtNodes($)
 	    if (@ifaces) {
 		my @refs = ();
 		foreach my $vlanmember (@ifaces) {
-		    my $iface = {'virtual_id'   => "$vlanmember"};
+		    my $virtlan = $vlanmember->virt_lan();
+		    my $iface   = {'virtual_id'   => "$vlanmember"};
 		    
 		    if ($vlanmember->fixed_iface() ne "") {
 			$iface->{'component_id'}  = $vlanmember->fixed_iface();
 		    }
 		    push(@refs, $iface);
 		}
-		$ref->{'interface'} = [ @refs ];
+		$ref->{'interface'} = [ @refs ]
+		    if (@refs);
 	    }
 	    if (!defined($self->rspec())) {
 		$self->{'RSPEC'} = {
@@ -1451,6 +1453,82 @@ sub GenVirtLans($)
     my $pid    = $experiment->pid();
     my $eid    = $experiment->eid();
 
+    #
+    # It would be nice if "tunnels" could be handled as a case in the
+    # logic below, but there are too many special things about
+    # tunnels, primary being that they are assumed to be connected
+    # over the control interfaces of the nodes. There should really be
+    # no reason why we cannot build a tunnel on top of an experimental
+    # link since it is just another form of encapsulation. 
+    #
+    foreach my $vname (sort(keys(%{ $self->{'VLANS'} }))) {
+	my $vlan = $self->vlans()->{$vname};
+
+	next
+	    if (!$vlan->_tunnel());
+
+	my @members = $vlan->memberlist();
+	$self->printdb("$vname (tunnel): " . join(" ",@members) . "\n");
+
+	if (scalar(@members) != 2) {
+	    tberror("Too many members in tunnel $vname\n");
+	    return -1;
+	}
+
+	my ($member0,$member1) = @members;
+	my $virtnode0 = $member0->virt_node();
+	my $virtnode1 = $member1->virt_node();
+	my $vname0    = $virtnode0->vname();
+	my $vname1    = $virtnode1->vname();
+	my $ip0       = $member0->ip();
+	my $ip1       = $member1->ip();
+
+	$vlan->_geninodes(1)
+	    if ($virtnode0->_isgeninode() || $virtnode0->_isgeninode());
+	
+	my ($delay0,$bw0,$ebw0,$backfill0,$loss0,
+	    $rdelay0,$rbw0,$rebw0,$rbackfill0,$rloss0) =
+		@{$member0->_delayinfo()};
+	my ($delay1,$bw1,$ebw1,$backfill1,$loss1,
+	    $rdelay1,$rbw1,$rebw1,$rbackfill1,$rloss1) =
+		@{$member1->_delayinfo()};
+
+	my $bw = min($bw0,$rbw1);
+
+	#
+	# If the link has geninode members, need to extend the rspec.
+	#
+	# XXX We eventually want to support tunnels between a local
+	# cluster node and a protogeni node. Should be doable, but might
+	# be easier once assign can take a single rspec for the entire
+	# topology.
+	#
+	if ($vlan->_geninodes()) {
+	    my $ref = {
+		'virtual_id'    => $vname,
+		# The list references are so XML::Simple does not
+		# turn them into attributes. Need a better solution.
+		'bandwidth'     => [$bw],
+		'latency'       => [0],
+		'packet_loss'   => [0],
+#		'link_type'     => {"type_name" => "tunnel"},
+		'link_type'     => "tunnel",
+		'interface_ref' =>
+		    [ {'virtual_node_id' => $vname0,
+		       'virtual_interface_id' => "$member0",
+		       'tunnel_ip' => $ip0 },
+		      {'virtual_node_id' => $vname1,
+		       'virtual_interface_id' => "$member1",
+		       'tunnel_ip' => $ip1 },
+		      ],
+	    };
+	    if (!exists($self->rspec()->{'link'})) {
+		$self->rspec()->{'link'} = [];
+	    }
+	    push(@{ $self->rspec()->{'link'} }, $ref);
+	}
+    }
+
     my $errors	         = 0;
     my %osdoesveth       = ();
     my %osdoesvethEN     = ();	# Encapsulated veth
@@ -1946,7 +2024,8 @@ sub GenVirtLans($)
 		    'bandwidth'     => [$bw],
 		    'latency'       => [0],
 		    'packet_loss'   => [0],
-		    'link_type'     => {"type_name" => "ethernet"},
+#		    'link_type'     => {"type_name" => "ethernet"},
+		    'link_type'     => "ethernet",
 		    'interface_ref' =>
 			[ {'virtual_node_id' => $vname0,
 			   'virtual_interface_id' => "$member0" },
@@ -2224,7 +2303,8 @@ sub GenVirtLans($)
 			   # turn them into attributes. Need a better solution.
 			   'latency'       => [0],
 			   'packet_loss'   => [0],
-			   'link_type'     => {"type_name" => "ethernet"},
+#			   'link_type'     => {"type_name" => "ethernet"},
+			   'link_type'     => "ethernet",
 			   'bandwidth'     => [$bw],
 			   'interface_ref' =>
 			       [
@@ -2920,7 +3000,8 @@ sub MapResources($)
 	$self->printdb("Mapping geni resources ...\n");
 	
 	if (libGeni::MapResources($self->experiment(),
-				  $self->user(), $self->rspec())) {
+				  $self->user(), $self->rspec(),
+				  $self->verbose())) {
 	    tberror("Could not map Geni resources\n");
 	    return -1;
 	}
@@ -3204,6 +3285,11 @@ sub ReadRspecSolution($$)
 	foreach my $linkref (@{$ref->{'interface'}}) {
 	    my $component_id  = $linkref->{"component_id"};
 	    my $virtual_id    = $linkref->{"virtual_id"};
+	    my $virtlan       = $self->memberof()->{$virtual_id};
+
+	    # Skip tunnels until assign can handle multiple rspecs
+	    next
+		if ($virtlan->_tunnel());
 
 	    if (GeniHRN::IsValid($component_id)) {
 		my ($urn_authority,$urn_node,$urn_iface) =
@@ -3216,17 +3302,16 @@ sub ReadRspecSolution($$)
     $self->printdb("Links:\n");
     foreach my $ref (@{ $self->rspec()->{'link'} }) {
 	my $virtual_id = $ref->{"virtual_id"};
-	my $istunnel   = (exists($ref->{'link_type'}) &&
-			  $ref->{'link_type'} eq "tunnel");
-
-	next
-	    if ($istunnel);
-	    
-	my $virtlan = $self->vlans()->{$virtual_id};
+	my $virtlan    = $self->vlans()->{$virtual_id};
+	
 	if (!defined($virtlan)) {
 	    tberror("Could not find lan $virtual_id\n");
 	    return -1;
 	}
+	# Skip tunnels until assign can handle multiple rspecs
+	next
+	    if ($virtlan->_tunnel());
+	
 	my @ifacerefs = @{ $ref->{'interface_ref'} };
 
 	if (scalar(@ifacerefs) == 2) {
@@ -3635,7 +3720,7 @@ sub AllocNodes($)
     if (defined($self->rspec()) && !$self->impotent()) {
 	$self->printdb("Requesting geni tickets ...\n");
 	
-	if (libGeni::GetTickets($self->experiment(), $self->impotent,
+	if (libGeni::GetTickets($self->experiment(), $self->impotent(),
 				$self->user(), $self->rspec())) {
 	    tberror("Could not allocate Geni Tickets\n");
 	    return -1;
@@ -4805,7 +4890,7 @@ sub InitializePhysNodes($)
     # XXX This is here cause vnames were not set until now. Need to move.
     #
     if (defined($self->rspec()) && !$self->impotent()) {
-	if (libGeni::MapNodes($self->experiment())) {
+	if (libGeni::MapNodes($self->experiment(), $self->verbose())) {
 	    tberror("Could not map geni nodes to local nodes.\n");
 	    return -1;
 	}
@@ -5987,6 +6072,8 @@ sub UpLoadTunnels($)
 	
 	next
 	    if (! $virtlan->_tunnel());
+	next
+	    if ($virtlan->_geninodes());
 
 	my @members = $virtlan->memberlist();
 	if (@members != 2) {
@@ -6019,6 +6106,8 @@ sub UpLoadTunnels($)
     foreach my $virtlan (values(%{ $self->vlans() })) {
 	next
 	    if (! $virtlan->_tunnel());
+	next
+	    if ($virtlan->_geninodes());
 
 	my $server    = $lantotunnelserver{$virtlan};
 	my $ipport    = $rnodetotunnelport{$server};
@@ -6039,17 +6128,19 @@ sub UpLoadTunnels($)
 			  $self->solution_v2p()->{$virtnode1->vname()});
 	my $pnode0 = $self->pnodes()->{$pnodename0};
 	my $pnode1 = $self->pnodes()->{$pnodename1};
-	my ($peerip0,$peerip1);
     
 	#
 	# Need to map the server to the control net interface of the
 	# physical node.
 	#
-	if (! TBControlNetIP($pnodename0, \$peerip0)) {
+	my $peerip0 = Interface->LookupControl($pnode0);
+	my $peerip1 = Interface->LookupControl($pnode1);
+
+	if (!defined($peerip0)) {
 	    tberror("No Control Network IP for $pnodename0!\n");
 	    return -1;
 	}
-	if (! TBControlNetIP($pnodename1, \$peerip1)) {
+	if (!defined($peerip1)) {
 	    tberror("No Control Network IP for $pnodename1!\n");
 	    return -1;
 	}
diff --git a/tbsetup/tbend.in b/tbsetup/tbend.in
index b7479203d..9923d7784 100644
--- a/tbsetup/tbend.in
+++ b/tbsetup/tbend.in
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2007 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use strict;
@@ -29,6 +29,7 @@ my $force   = 0;
 my $TBROOT       = "@prefix@";
 my $OPSDBSUPPORT = @OPSDBSUPPORT@;
 my $dbcontrol    = "$TBROOT/sbin/opsdb_control";
+my $PGENISUPPORT = @PROTOGENI_SUPPORT@;
 
 # Untaint the path
 $ENV{'PATH'} = "/usr/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
@@ -47,6 +48,9 @@ use libdb;
 use libtestbed;
 use libtblog;
 use Experiment;
+if ($PGENISUPPORT) {
+    require libGeni;
+}
 
 #
 # Parse command arguments. Once we return from getopts, all that should be
@@ -84,6 +88,11 @@ if (!$force && $state ne EXPTSTATE_TERMINATING) {
     tbdie("Experiment is in the wrong state: $state.\n");
 }
 
+if ($PGENISUPPORT) {
+    if (libGeni::UnRegister($experiment)) {
+	tbdie("Could not delete protogeni slice\n");
+    }
+}
 print "Clearing out virtual state.\n";
 $experiment->RemoveVirtualState() == 0 or $errors++;
 
diff --git a/tmcd/common/GNUmakefile.in b/tmcd/common/GNUmakefile.in
index 6d2594807..164e15710 100644
--- a/tmcd/common/GNUmakefile.in
+++ b/tmcd/common/GNUmakefile.in
@@ -1,6 +1,6 @@
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -102,6 +102,7 @@ local-script-install:	common-script-install
 	$(INSTALL) -m 755 $(SRCDIR)/rc.canaryd $(RCDIR)/rc.canaryd
 	$(INSTALL) -m 755 $(SRCDIR)/rc.linktest $(RCDIR)/rc.linktest
 	$(INSTALL) -m 755 $(SRCDIR)/rc.inelab $(RCDIR)/rc.inelab
+	$(INSTALL) -m 755 $(SRCDIR)/rc.pgeni $(RCDIR)/rc.pgeni
 	# Symlink this cause we invoke it from boss, and its too much
 	# of a hassle to worry about right now.
 	rm -f $(ETCDIR)/update
@@ -120,5 +121,18 @@ control-script-install:	dir-install bossnode
 	$(INSTALL) -m 755 $(SRCDIR)/config/librc.pm $(BINDIR)/librc.pm
 	$(INSTALL) bossnode $(ETCDIR)/bossnode
 
+PGENIFILES = rc.ifconfig rc.topomap rc.progagent rc.pgeni \
+		rc.tarfiles rc.rpms
+
+pgeni-tarball:
+	-rm -f $(TBROOT)/www/downloads/geniclient.tar
+	rm -rf /tmp/mk1
+	($(MAKE) DESTDIR=/tmp/mk1 local-install)
+	tar cf $(TBROOT)/www/downloads/geniclient.tar \
+                -C /tmp/mk1/usr/local/etc/emulab \
+		libsetup.pm libtmcc.pm runstartup \
+		$(addprefix rc/, $(PGENIFILES))
+	rm -rf /tmp/mk1
+
 bossnode:	GNUmakefile
 	echo >$@ "$(BOSSNODE)"
diff --git a/tmcd/common/config/rc.ifconfig b/tmcd/common/config/rc.ifconfig
index 4068eb852..c238de712 100755
--- a/tmcd/common/config/rc.ifconfig
+++ b/tmcd/common/config/rc.ifconfig
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2004-2009 University of Utah and the Flux Group.
+# Copyright (c) 2004-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
diff --git a/tmcd/common/config/rc.progagent b/tmcd/common/config/rc.progagent
index cd2f49676..d73c6b820 100755
--- a/tmcd/common/config/rc.progagent
+++ b/tmcd/common/config/rc.progagent
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2004-2008 University of Utah and the Flux Group.
+# Copyright (c) 2004-2008, 2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
@@ -189,6 +189,15 @@ sub doboot()
 
 	my $server = getlocalevserver();
 
+	if (SHADOW()) {
+	    my $boss = $ENV{'BOSSNAME'};
+	    if (!defined($boss)) {
+		fatal("BOSSNAME is not set in the environment");
+	    }
+	    my ($domain) = ($boss =~ /^[^\.]+\.(.*)$/);
+	    $server = "event-server.${domain}";
+	}
+
 	#
 	# PlanetLab nodes don't have a fixed elvind port, so we need to
 	# look that up and pass it in.  Also for plab, we always restart
diff --git a/tmcd/common/config/rc.rpms b/tmcd/common/config/rc.rpms
index 44afcb47d..19433145b 100755
--- a/tmcd/common/config/rc.rpms
+++ b/tmcd/common/config/rc.rpms
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2004, 2005 University of Utah and the Flux Group.
+# Copyright (c) 2004-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
@@ -112,7 +112,7 @@ sub doboot()
     # Note: Windows also uses the -t option since root does not have access
     # to NFS mounted FSes.
     # 
-    my $installoption = ((REMOTE() || WINDOWS()) ? "-t" : "-c");
+    my $installoption = ((REMOTE() || SHADOW() || WINDOWS()) ? "-t" : "-c");
 
     foreach my $rpm (@rpms) {
 	if ($rpm =~ /RPM=(.+)/) {
diff --git a/tmcd/common/config/rc.tarfiles b/tmcd/common/config/rc.tarfiles
index 1f85e9d2b..ed262d734 100755
--- a/tmcd/common/config/rc.tarfiles
+++ b/tmcd/common/config/rc.tarfiles
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2004, 2005 University of Utah and the Flux Group.
+# Copyright (c) 2004-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
@@ -112,7 +112,7 @@ sub doboot()
     # Note: Windows also uses the -t option since root does not have access
     # to NFS mounted FSes.
     # 
-    my $installoption = ((REMOTE() || WINDOWS()) ? "-t" : "-c");
+    my $installoption = ((REMOTE() || SHADOW() || WINDOWS()) ? "-t" : "-c");
 
     foreach my $tarball (@tarballs) {
 	if ($tarball =~ /DIR=(.+)\s+TARBALL=(.+)/) {
diff --git a/tmcd/common/config/rc.topomap b/tmcd/common/config/rc.topomap
index 8c6692994..20bcf0033 100755
--- a/tmcd/common/config/rc.topomap
+++ b/tmcd/common/config/rc.topomap
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2004-2008 University of Utah and the Flux Group.
+# Copyright (c) 2004-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
@@ -15,6 +15,7 @@ sub usage()
 }
 my $optlist = "j:";
 my $action  = "boot";
+my $vnodeid;
 
 # Turn off line buffering on output
 $| = 1;
@@ -54,7 +55,7 @@ if (! getopts($optlist, \%options)) {
     usage();
 }
 if (defined($options{'j'})) {
-    my $vnodeid = $options{'j'};
+    $vnodeid = $options{'j'};
     libsetup_setvnodeid($vnodeid);
 }
 # Allow default above.
@@ -173,12 +174,16 @@ sub dotopomap()
 	    }
 	}
     }
+    my $optarg = (defined($vnodeid) ? "-n $vnodeid" : "");
+    my $urnarg = (exists($ENV{'URN'}) ? "URN=" . $ENV{'URN'} : "");
     
     if (! -e $tempmap) {
 	#
 	# Try using tmcc. It returns compressed data so do it directly.
 	#
-	system(TMCCBIN() . " topomap > $tempmap");
+	print "foo $optarg $urnarg\n";
+	
+	system(TMCCBIN() . " $optarg $urnarg topomap > $tempmap");
 	#
 	# tmcc does not really return useful error codes.
 	#
@@ -191,7 +196,7 @@ sub dotopomap()
 	#
 	# Try using tmcc. It returns compressed data so do it directly.
 	#
-	system(TMCCBIN() . " ltmap > $templt");
+	system(TMCCBIN() . " $optarg $urnarg ltmap > $templt");
 	#
 	# tmcc does not really return useful error codes.
 	#
@@ -204,7 +209,7 @@ sub dotopomap()
 	#
 	# Try using tmcc. It returns compressed data so do it directly.
 	#
-	system(TMCCBIN() . " ltpmap > $templtp");
+	system(TMCCBIN() . " $optarg $urnarg ltpmap > $templtp");
 	#
 	# tmcc does not really return useful error codes.
 	#
diff --git a/tmcd/common/libsetup.pm b/tmcd/common/libsetup.pm
index f4d6dc611..2c9bda54e 100644
--- a/tmcd/common/libsetup.pm
+++ b/tmcd/common/libsetup.pm
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -wT
+#!/usr/bin/perl -w
 
 #
 # EMULAB-COPYRIGHT
@@ -26,7 +26,8 @@ use Exporter;
 
 	 TBDebugTimeStamp TBDebugTimeStampsOn
 
-	 MFS REMOTE REMOTEDED CONTROL WINDOWS JAILED PLAB LOCALROOTFS IXP USESFS 
+	 MFS REMOTE REMOTEDED CONTROL WINDOWS JAILED PLAB LOCALROOTFS IXP
+	 USESFS SHADOW
 
 	 SIMTRAFGEN SIMHOST ISDELAYNODEPATH JAILHOST DELAYHOST STARGATE
 	 ISFW FAKEJAILED LINUXJAILED GENVNODE GENVNODETYPE GENVNODEHOST
@@ -118,6 +119,12 @@ my $inplab;
 #
 my $inixp;
 
+#
+# Shadow mode. Run the client side against a remote tmcd.
+#
+my $shadow;
+my $SHADOWDIR = "$VARDIR/shadow";
+
 #
 # The role of this pnode
 #
@@ -132,6 +139,7 @@ BEGIN
     }
     require "/etc/emulab/paths.pm";
     import emulabpaths;
+    $SHADOWDIR = "$VARDIR/shadow";
 
     # Make sure these exist! They will not exist on a PLAB vserver initially.
     mkdir("$VARDIR", 0775);
@@ -141,11 +149,28 @@ BEGIN
     mkdir("$VARDIR/logs", 0775);
     mkdir("$VARDIR/boot", 0775);
     mkdir("$VARDIR/lock", 0775);
+    mkdir("$SHADOWDIR", 0775);
+    mkdir("$SHADOWDIR/db", 0755);
+    mkdir("$SHADOWDIR/logs", 0775);
+    mkdir("$SHADOWDIR/boot", 0775);
+    mkdir("$SHADOWDIR/lock", 0775);
 
+    #
+    # Shadow mode allows the client side to run against remote tmcd.
+    #
+    if (exists($ENV{'SHADOW'})) {
+	$shadow = $ENV{'SHADOW'};
+	my ($server,$urn) = split(',', $shadow);
+	
+	# The cache needs to go in a difference location.
+	libtmcc::configtmcc("cachedir", $SHADOWDIR);
+	libtmcc::configtmcc("server", $server);
+	libtmcc::configtmcc("urn", $urn);
+    }
     #
     # Determine if running inside a jail. This affects the paths below.
     #
-    if (-e "$BOOTDIR/jailname") {
+    elsif (-e "$BOOTDIR/jailname") {
 	open(VN, "$BOOTDIR/jailname");
 	my $vid = <VN>;
 	close(VN);
@@ -256,6 +281,8 @@ sub LOCALROOTFS() {
 # jailed node, the code that sets it up needs a different per-vnode path.
 #
 sub CONFDIR() {
+    return "$SHADOWDIR/boot"
+	if ($shadow);
     return $BOOTDIR
 	if ($injail || $inplab || $ingenvnode);
     return GENVNODEDIR() 
@@ -266,6 +293,8 @@ sub CONFDIR() {
 }
 # Cause of fakejails, we want log files in the right place.
 sub LOGDIR() {
+    return "$SHADOWDIR/logs"
+	if ($shadow);
     return $LOGDIR
 	if ($injail || $inplab || $ingenvnode);
     return GENVNODEDIR() 
@@ -396,6 +425,9 @@ sub SHAREDHOST()   { return ($role eq "sharedhost" ? 1 : 0); }
 # A delay host?  Either a delay node or a node using linkdelays
 sub DELAYHOST()	{ if (-e ISDELAYNODEPATH()) { return 1; } else { return 0; } }
 
+# A shadow?
+sub SHADOW()	   { return (defined($shadow) ? 1 : 0); }
+
 #
 # Is this node using SFS. Several scripts need to know this.
 #
@@ -1808,6 +1840,60 @@ sub bootsetup()
     return ($pid, $eid, $vname);
 }
 
+#
+# Shadow mode setup. The server argument is the remote boss we talk to.
+#
+sub shadowsetup($$)
+{
+    my ($server, $urn) = @_;
+
+    $shadow = 1;
+
+    # This changes where tmcc is going to store the data.
+    libtmcc::configtmcc("cachedir", $SHADOWDIR);
+    libtmcc::configtmcc("server", $server);
+    libtmcc::configtmcc("urn", $urn);
+
+    # Tell children.
+    $ENV{'SHADOW'} = "$server,$urn";
+    $ENV{'URN'}    = $urn;
+
+    # Tell libtmcc to forget anything it knows.
+    tmccclrconfig();
+    
+    #
+    # Check allocation. Exit now if not allocated.
+    #
+    if (! check_status()) {
+	return undef;
+    } 
+    print STDOUT "  Allocated! $pid/$eid/$vname\n";
+
+    #
+    # Tell libtmcc to get the full config. 
+    #
+    tmccgetconfig();
+    
+    #
+    # Get the role of this node from tmcc which can be one of
+    # "node", "virthost", "delaynode" or "simhost".
+    # Mainly useful for simulation (nse) stuff
+    # Hopefully, this will come out of the tmcc cache and will not
+    # be expensive.
+    #
+    dorole();
+
+    #
+    # And the nodeid.
+    #
+    donodeid();
+
+    my $eiddir = "/proj/$pid/exp/$eid/tbdata";
+    os_mkdir($eiddir, "0777");
+
+    return ($pid, $eid, $vname);
+}
+
 #
 # This happens inside a jail. 
 #
@@ -2026,6 +2112,9 @@ sub getgenvnodeconfig($;$)
 		$vconfig{"CTRLIP"} = $1;
 		$vconfig{"CTRLMASK"} = $2;
 	    }
+	    else {
+		$vconfig{$1} = $2;
+	    }
 	}
     }
 
diff --git a/tmcd/common/libtmcc.pm b/tmcd/common/libtmcc.pm
index fd7961528..9d0135b91 100644
--- a/tmcd/common/libtmcc.pm
+++ b/tmcd/common/libtmcc.pm
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -85,6 +85,8 @@ my $beproxy     = 0;
       "clrcache"        => 0,
       "noproxy"         => 0,
       "nossl"           => 0,
+      "cachedir"        => undef,
+      "urn"             => undef,
       "usetpm"          => 0,
     );
 
@@ -267,6 +269,12 @@ sub configtmcc($$)
 	             "    Invalid libtmcc option: $opt/$val\n";
 	return -1;
     }
+    if ($opt eq "cachedir") {
+	$CACHEDIR = $val;
+    }
+    elsif ($opt eq "server") {
+	$ENV{'BOSSNAME'} = $val;
+    }
     $config{$opt} = $val;
 }
 
@@ -352,6 +360,11 @@ sub runtmcc ($;$$%)
     $options = optionstring($options, %optconfig)
 	if (%optconfig);
 
+    # Must be last option, before command
+    if (defined($config{"urn"})) {
+	$options .= " URN=" . $config{"urn"};
+    }
+
     if (!defined($args)) {
 	$args = "";
     }
diff --git a/tmcd/common/runstartup b/tmcd/common/runstartup
index 48965c926..b7e922533 100755
--- a/tmcd/common/runstartup
+++ b/tmcd/common/runstartup
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 use English;
@@ -40,7 +40,7 @@ my $logtemplate;
 # Get the config parameters so we can open up a log file in the proper
 # location. See below.
 #
-if (JAILED() || GENVNODE() || PLAB()) {
+if (JAILED() || GENVNODE() || PLAB() || SHADOW()) {
     $logtemplate = "$VARDIR/logs/runlog.XXXXXX";
 }
 else {
diff --git a/tmcd/tmcd.c b/tmcd/tmcd.c
index bedc63f20..8d36f85da 100644
--- a/tmcd/tmcd.c
+++ b/tmcd/tmcd.c
@@ -68,6 +68,10 @@
 #define RUNASGROUP	"nobody"
 #define NTPSERVER       "ntp1"
 #define PROTOUSER	"elabman"
+#define PRIVKEY_LEN	128
+#define URN_LEN		128
+#define XSTRINGIFY(s)   STRINGIFY(s)
+#define STRINGIFY(s)	#s
 
 /* socket read/write timeouts in ms */
 #define READTIMO	3000
@@ -110,7 +114,6 @@ static char     dbname[DBNAME_SIZE];
 static struct in_addr myipaddr;
 static char	fshostid[HOSTID_SIZE];
 static int	nodeidtoexp(char *nodeid, char *pid, char *eid, char *gid);
-static int	checkprivkey(struct in_addr, char *);
 static void	tcpserver(int sock, int portnum);
 static void	udpserver(int sock, int portnum);
 static int      handle_request(int, struct sockaddr_in *, char *, int);
@@ -192,7 +195,8 @@ typedef struct {
 	char		sfshostid[TBDB_FLEN_SFSHOSTID];
 	char		testdb[TBDB_FLEN_TINYTEXT];
 	char		sharing_mode[TBDB_FLEN_TINYTEXT];
-	char            privkey[TBDB_FLEN_PRIVKEY+1];
+	char            privkey[PRIVKEY_LEN+1];
+	char            urn[URN_LEN+1];
 } tmcdreq_t;
 static int	iptonodeid(struct in_addr, tmcdreq_t *, char*);
 static int	checkdbredirect(tmcdreq_t *);
@@ -932,9 +936,10 @@ static int
 handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 {
 	struct sockaddr_in redirect_client;
-	int		   redirect = 0, havekey = 0;
+	int		   redirect = 0;
 	char		   buf[BUFSIZ], *bp, *cp;
-	char		   privkey[TBDB_FLEN_PRIVKEY];
+	char		   privkeybuf[PRIVKEY_LEN];
+	char		   *privkey = (char *) NULL;
 	int		   i, overbose = 0, err = 0;
 	int		   version = DEFAULT_VERSION;
 	tmcdreq_t	   tmcdreq, *reqp = &tmcdreq;
@@ -960,7 +965,11 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 		/*
 		 * Look for PRIVKEY. 
 		 */
-		if (sscanf(bp, "PRIVKEY=%64s", buf)) {
+		if (sscanf(bp, "PRIVKEY=%" XSTRINGIFY(PRIVKEY_LEN) "s", buf)) {
+			if (strlen(buf) < 16) {
+				info("tmcd client provided short privkey");
+				goto skipit;
+			}
 			for (i = 0; i < strlen(buf); i++){
 				if (! isxdigit(buf[i])) {
 					info("tmcd client provided invalid "
@@ -968,8 +977,8 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 					goto skipit;
 				}
 			}
-			havekey = 1;
-			strncpy(privkey, buf, sizeof(privkey));
+			strncpy(privkeybuf, buf, sizeof(privkeybuf));
+			privkey = privkeybuf;
 
 			if (debug) {
 				info("%s: PRIVKEY %s\n", reqp->nodeid, buf);
@@ -1016,8 +1025,6 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 		 * Look for VNODE. This is used for virtual nodes.
 		 * It indicates which of the virtual nodes (on the physical
 		 * node) is talking to us. Currently no perm checking.
-		 * Very temporary approach; should be done via a per-vnode
-		 * cert or a key.
 		 */
 		if (sscanf(bp, "VNODEID=%30s", buf)) {
 			for (i = 0; i < strlen(buf); i++){
@@ -1037,6 +1044,28 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 			continue;
 		}
 
+		/*
+		 * Look for URN. 
+		 */
+		if (sscanf(bp, "URN=%" XSTRINGIFY(URN_LEN) "s", buf)) {
+			for (i = 0; i < strlen(buf); i++){
+				if (! (isalnum(buf[i]) ||
+				       buf[i] == '_' || buf[i] == '-' ||
+				       buf[i] == '.' || buf[i] == '/' ||
+				       buf[i] == ':' || buf[i] == '+')) {
+					info("tmcd client provided invalid "
+					     "characters in URN");
+					goto skipit;
+				}
+			}
+			strncpy(reqp->urn, buf, sizeof(reqp->urn));
+
+			if (debug) {
+				info("URN %s\n", buf);
+			}
+			continue;
+		}
+
 		/*
 		 * An empty token (two delimiters next to each other)
 		 * is indicated by a null string. If nothing matched,
@@ -1057,24 +1086,26 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 	/*
 	 * Map the ip to a nodeid.
 	 */
-	if (havekey) {
-		if ((err = iptonodeid(client->sin_addr, reqp, privkey))) {
+	if ((err = iptonodeid(client->sin_addr, reqp, privkey))) {
+		if (privkey) {
 			error("No such node with wanode_key [%s]\n", privkey);
-			goto skipit;
 		}
-	}
-	else {
-		if ((err = iptonodeid(client->sin_addr, reqp, NULL))) {
-			if (reqp->isvnode) {
-				error("No such vnode %s associated with %s\n",
-				      reqp->vnodeid, inet_ntoa(client->sin_addr));
-			}
-			else {
-				error("No such node: %s\n",
-				      inet_ntoa(client->sin_addr));
-			}
-			goto skipit;
+		else if (reqp->urn[0]) {
+			if (reqp->isvnode)
+				error("No such vnode %s with urn %s\n",
+				      reqp->vnodeid, reqp->urn);
+			else
+				error("No such node with urn %s\n", reqp->urn);
+		}
+		else if (reqp->isvnode) {
+			error("No such vnode %s associated with %s\n",
+			      reqp->vnodeid, inet_ntoa(client->sin_addr));
+		}
+		else {
+			error("No such node: %s\n",
+			      inet_ntoa(client->sin_addr));
 		}
+		goto skipit;
 	}	
 
 	/*
@@ -1152,31 +1183,6 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
 		goto skipit;
 	}
 
-	/*
-	 * Do private key check. (Non-plab) widearea nodes must report a
-	 * private key. It comes over ssl of course. At present we skip
-	 * this check for ron nodes.
-	 *
-	 * LBS: I took this test out cause its silly. Will kill the code at
-	 * some point.
-	 */
-	if (0 &&
-	    (!reqp->islocal && !(reqp->isplabdslice || reqp->isplabsvc))) {
-		if (!havekey) {
-			error("%s: No privkey sent!\n", reqp->nodeid);
-			/*
-			 * Skip. Okay, the problem is that the nodes out
-			 * there are not reporting the key!
-			goto skipit;
-			 */
-		}
-		else if (checkprivkey(client->sin_addr, privkey)) {
-			error("%s: privkey mismatch: %s!\n",
-			      reqp->nodeid, privkey);
-			goto skipit;
-		}
-	}
-
 	/*
 	 * Figure out what command was given.
 	 */
@@ -4602,14 +4608,27 @@ iptonodeid(struct in_addr ipaddr, tmcdreq_t *reqp, char* nodekey)
 				 "   WHERE attrkey='dedicated_widearea' "
 				 "   GROUP BY type) AS dedicated_wa_types "
 				 "  ON n.type=dedicated_wa_types.type "
-				"WHERE n.node_id IN "
-                                        "(SELECT node_id FROM widearea_nodeinfo WHERE privkey='%s') "
+				 "WHERE n.node_id IN "
+				 "     (SELECT node_id FROM widearea_nodeinfo "
+				 "      WHERE privkey='%s') "
 				 "  AND nobootinfo_types.attrvalue IS NULL",
 				 34, nodekey);
-
 	}
-
 	else if (reqp->isvnode) {
+		char	clause[BUFSIZ];
+		
+		if (reqp->urn[0]) {
+			sprintf(clause,
+				"r.external_resource_id is not null and "
+				"r.external_resource_id='%s'",
+				reqp->urn);
+		}
+		else {
+			sprintf(clause,
+				"(i.IP='%s' and i.role='ctrl') or "
+				"nv.jailip='%s'",
+				inet_ntoa(ipaddr), inet_ntoa(ipaddr));
+		}
 		res = mydb_query("select vt.class,vt.type,np.node_id,"
 				 " nv.jailflag,r.pid,r.eid,r.vname, "
 				 " e.gid,e.testdb,nv.update_accounts, "
@@ -4642,13 +4661,23 @@ iptonodeid(struct in_addr ipaddr, tmcdreq_t *reqp, char* nodekey)
 				 " nk.node_id=nv.node_id "
 				 "left join users as u on "
 				 " u.uid_idx=e.swapper_idx "
-				 "where nv.node_id='%s' and "
-				 " ((i.IP='%s' and i.role='ctrl') or "
-				 "  nv.jailip='%s')",
-				 34, reqp->vnodeid,
-				 inet_ntoa(ipaddr), inet_ntoa(ipaddr));
+				 "where nv.node_id='%s' and (%s)",
+				 34, reqp->vnodeid, clause);
 	}
 	else {
+		char	clause[BUFSIZ];
+		
+		if (reqp->urn[0]) {
+			sprintf(clause,
+				"r.external_resource_id is not null and "
+				"r.external_resource_id='%s'",
+				reqp->urn);
+		}
+		else {
+			sprintf(clause,
+				"i.IP='%s' and i.role='ctrl'",
+				inet_ntoa(ipaddr));
+		}
 		res = mydb_query("select t.class,t.type,n.node_id,n.jailflag,"
 				 " r.pid,r.eid,r.vname,e.gid,e.testdb, "
 				 " n.update_accounts,n.role, "
@@ -4687,9 +4716,9 @@ iptonodeid(struct in_addr ipaddr, tmcdreq_t *reqp, char* nodekey)
 				 "   where attrkey='dedicated_widearea' "
 				 "   group by type) as dedicated_wa_types "
 				 "  on n.type=dedicated_wa_types.type "
-				 "where i.IP='%s' and i.role='ctrl' "
+				 "where (%s) "
 				 "  and nobootinfo_types.attrvalue is NULL",
-				 34, inet_ntoa(ipaddr));
+				 34, clause);
 	}
 
 	if (!res) {
@@ -4716,7 +4745,7 @@ iptonodeid(struct in_addr ipaddr, tmcdreq_t *reqp, char* nodekey)
 	strncpy(reqp->ptype,  row[15], sizeof(reqp->ptype));
 	strncpy(reqp->nodeid, row[2],  sizeof(reqp->nodeid));
 	if(nodekey != NULL) {
-		strncpy(reqp->privkey, nodekey, TBDB_FLEN_PRIVKEY); 
+		strncpy(reqp->privkey, nodekey, PRIVKEY_LEN); 
 	}
 	else {
 		strcpy(reqp->privkey, ""); 
@@ -4882,37 +4911,6 @@ checkdbredirect(tmcdreq_t *reqp)
 	return 0;
 }
 
-/*
- * Check private key. 
- */
-static int
-checkprivkey(struct in_addr ipaddr, char *privkey)
-{
-	MYSQL_RES	*res;
-	MYSQL_ROW	row;
-
-	res = mydb_query("select privkey from widearea_privkeys "
-			 "where IP='%s'",
-			 1, inet_ntoa(ipaddr));
-	
-	if (!res) {
-		error("checkprivkey: %s: DB Error getting privkey!\n",
-		      inet_ntoa(ipaddr));
-		return 1;
-	}
-
-	if (! (int)mysql_num_rows(res)) {
-		mysql_free_result(res);
-		return 1;
-	}
-	row = mysql_fetch_row(res);
-	mysql_free_result(res);
-	if (! row[0] || !row[0][0])
-		return 1;
-
-	return strcmp(privkey, row[0]);
-}
- 
 #ifdef EVENTSYS
 /*
  * Connect to the event system. It's not an error to call this function if
@@ -7819,7 +7817,7 @@ COMMAND_PROTOTYPE(dobootwhat)
 
 	if(strlen(reqp->privkey) > 1) { /* We have a private key, so prepare bootinfo for it. */
 		boot_info.opcode = BIOPCODE_BOOTWHAT_KEYED_REQUEST;
-		strncpy(boot_info.data, reqp->privkey, TBDB_FLEN_PRIVKEY);
+		strncpy(boot_info.data, reqp->privkey, PRIVKEY_LEN);
 	}
 
 	if (bootinfo(reqp->client, (reqp->isvnode) ? reqp->nodeid : NULL,
diff --git a/www/nodessh.php3 b/www/nodessh.php3
index 05a31468b..4fcd2ac4b 100644
--- a/www/nodessh.php3
+++ b/www/nodessh.php3
@@ -1,7 +1,7 @@
 <?php
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2004, 2006, 2007 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 include("defs.php3");
@@ -31,7 +31,7 @@ $query_result =
     DBQueryFatal("select n.jailflag,n.jailip,n.sshdport, ".
 		 "       r.vname,r.pid,r.eid, ".
 		 "       t.isvirtnode,t.isremotenode,t.isplabdslice, ".
-		 "       t.issubnode,t.class ".
+		 "       t.issubnode,t.isfednode,t.class ".
 		 " from nodes as n ".
 		 "left join reserved as r on n.node_id=r.node_id ".
 		 "left join node_types as t on t.type=n.type ".
@@ -54,6 +54,7 @@ $isremote = $row["isremotenode"];
 $isplab   = $row["isplabdslice"];
 $issubnode= $row["issubnode"];
 $class    = $row["class"];
+$isfednode= $row["isfednode"];
 
 #
 # XXX hack to determine if target node is on a routable network
@@ -78,7 +79,7 @@ if ($isvirt) {
 	# Remote nodes run sshd on another port since they so not
 	# have per-jail IPs. Of course, might not even be jailed!
 	#
-	if ($jailflag || $isplab) {
+	if ($jailflag || $isplab || $isfednode) {
 	    echo "port: $sshdport\n";
 	}
     }
diff --git a/www/showslice.php b/www/showslice.php
index e144dc54e..f50140a66 100644
--- a/www/showslice.php
+++ b/www/showslice.php
@@ -1,7 +1,7 @@
 <?php
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 include("defs.php3");
@@ -55,6 +55,9 @@ $rows[] = array("hrn"      => $slice->hrn());
 $rows[] = array("uuid"     => $slice->uuid());
 $rows[] = array("created"  => $slice->created());
 $rows[] = array("expires"  => $slice->expires());
+if ($slice->locked()) {
+    $rows[] = array("locked"  => $slice->locked());
+}
 if (($manifest = $slice->GetManifest())) {
     $json = new Services_JSON();
     $manifest = $json->encode($manifest);
-- 
GitLab