A slew of new support for cooked mode. Swapmod now works pretty well,

although the default is currently to do cooked mode in "basic" mode,
rather then "full" mode.

protogeni/lib/GeniAggregate.pm.in

@@ -239,6 +239,17 @@ sub GetCertificate($)   { return $_[0]->{'CERTIFICATE'}; }
 # An alias so that slivers look like aggregates.
 sub resource_type($)	{ return field($_[0], "type"); }
 
+# A place to stash a temporary rspec.
+sub rspec($;$)
+{
+    my ($self, $rspec) = @_;
+
+    if (defined($rspec)) {
+	$self->{'RSPEC'} = $rspec;
+    }
+    return $self->{'RSPEC'};
+}
+
 # Return the URN.
 sub urn($)
 {
@@ -770,6 +781,8 @@ sub Start($$$)
 	    return -1;
 	}
 	if ($reservation->SameExperiment($experiment)) {
+	    my $vnode;
+	    
 	    #
 	    # Since this is an aggregate, some slivers may already be
 	    # in the started state. Skip those, unless doing a restart.
@@ -778,14 +791,28 @@ sub Start($$$)
 		if ($sliver->state() eq "started" && !$restart);
 	    
 	    if ($node->isvirtnode()) {
-		$vnodes{$node->node_id} = $node;
-
-		# A virtnode on a shared physical node needs nothing else.
-		next
-		    if ($node->sharing_mode());
-
-		# But if non-shared, have to make sure that the phys node
-		# gets loaded.
+		# A virtnode on a shared physical node needs reboot or setup
+		if ($node->sharing_mode()) {
+		    if ($restart && $sliver->state() eq "started") {
+			$reboots{$node->node_id} = $node;
+		    }
+		    else {
+			$vnodes{$node->node_id} = $node;
+		    }
+		    next;
+		}
+		# See below.
+		$vnode = $node;
+		
+		#
+		# Now it gets messy. Do not want to mess with the physnode
+		# if its running other vnodes, and we just need to fire up
+		# a new one. But if the physnode is going to get rebooted,
+		# then there is no need to do anything with the vnodes; they
+		# will boot up with the physnode. 
+		#
+		# But, have to make sure that the phys node gets setup.
+		#
 		my $physnodeid = $node->phys_nodeid();
 		next
 		    if (exists($poweron{$physnodeid}) ||
@@ -798,21 +825,22 @@ sub Start($$$)
 		}
 	    }
 	    #
-	    # Look to see if local physical node was stopped (powered off).
+	    # If the node is not imageable, then there is not much to
+	    # do except turn it on or reboot it. I am assuming that a
+	    # a non imageable node is always in raw mode.
 	    #
-	    if (!$node->isremotenode() &&
-		($sliver->state() eq "stopped")) {
-		$poweron{$node->node_id} = $node;
-	    }
-	    else {
-		# node_reboot is smart enough to know that if a pnode
-		# is rebooted it can ignore the vnodes on it, so do
-		# not optimize this here.
-		$reboots{$node->node_id} = $node;
+	    if (!$node->imageable()) {
+		if ($sliver->state() eq "stopped") {
+		    $poweron{$node->node_id} = $node;
+		}
+		else  {
+		    $reboots{$node->node_id} = $node;
+		}
+		next;
 	    }
-	    next
-		if (!$node->imageable());
-
+	    #
+	    # See if the node is running the requested OS.
+	    #
 	    my $osinfo = OSinfo->Lookup($node->def_boot_osid());
 	    if (!defined($osinfo)) {
 		print STDERR "Could not get osinfo for $node\n";
@@ -852,6 +880,14 @@ sub Start($$$)
 		    $reloads{$image->imageid()} = [ ];
 		} 
 		push(@{ $reloads{$image->imageid()} }, $node);
+		
+		# Reload means reboot or power on.
+		if (!defined($vnode) && $sliver->state() eq "stopped") {
+		    $poweron{$node->node_id} = $node;
+		}
+		else  {
+		    $reboots{$node->node_id} = $node;
+		}
 	    }
 	    else {
 		#
@@ -861,6 +897,33 @@ sub Start($$$)
 		    print STDERR "  Could not os_select $node to $osinfo\n";
 		    return -1;
 		}
+		#
+		# If the node is going to get rebooted, then do not need
+		# to worry about the vnodes on it. But if the node is ready
+		# to go, then we have to do the vnodes. Remember, we do not
+		# reboot the physnode since we might only be adding a new node
+		# in which case, a full reboot is wrong.
+		#
+		if (! $node->IsUp()) {
+		    if ($sliver->state() eq "stopped" && !defined($vnode)) {
+			$poweron{$node->node_id} = $node;
+		    }
+		    else {
+			$reboots{$node->node_id} = $node;
+		    }
+		}
+		elsif ($restart && !defined($vnode)) {
+		    # Just a physnode that needs restarting.
+		    $reboots{$node->node_id} = $node;
+		}
+		elsif (defined($vnode)) {
+		    if ($sliver->state() eq "started") {
+			$reboots{$vnode->node_id} = $vnode;
+		    }
+		    else {
+			$vnodes{$vnode->node_id} = $vnode;
+		    }
+		}
 	    }
 	}
 	else {

protogeni/lib/GeniCM.pm.in

@@ -150,7 +150,7 @@ sub Resolve($)
 					undef, "Nothing here by that name");
 	}
 
-	my $rspec = GetAdvertisement(0, $node->node_id(), "0.1");
+	my $rspec = GetAdvertisement(0, $node->node_id(), "0.1", undef);
 	if (! defined($rspec)) {
 	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 					"Could not start avail");
@@ -237,10 +237,29 @@ sub DiscoverResourcesAux($$$)
 	}
     }
 
+    #
+    # See if one of the credentials is a slice credential. If it is, and
+    # that slice is active, pass it to ptopgen so that it includes the current
+    # resources as available.
+    #
+    my $experiment = undef;
+    foreach my $credential (@$credentials) {
+	my ($auth, $type, $id) = GeniHRN::Parse($credential->target_urn());
+	if ($type eq "slice") {
+	    # Might not exist here yet.
+	    my $slice = GeniSlice->Lookup($credential->target_urn());
+	    if (defined($slice)) {
+		# See if the local experiment exists yet.
+		$experiment = Experiment->Lookup($slice->uuid());
+	    }
+	    last;
+	}
+    }
+
     #
     # Acquire the advertisement from ptopgen and compress it if requested.
     #
-    my $xml = GetAdvertisement($available, undef, "0.2");
+    my $xml = GetAdvertisement($available, undef, "0.2", $experiment);
     if (! defined($xml)) {
 	return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 				    "Could not start avail");
@@ -258,12 +277,16 @@ sub DiscoverResourcesAux($$$)
 #
 # Use ptopgen in xml mode to spit back an xml file. 
 #
-sub GetAdvertisement($$$)
+sub GetAdvertisement($$$$)
 {
-    my ($available, $pc, $version) = @_;
+    my ($available, $pc, $version, $experiment) = @_;
     my $xml = undef;
 
     my $invocation = "$PTOPGEN -x -g $version -r -p GeniSlices";
+    if (defined($experiment)) {
+	my $eid = $experiment->eid();
+	$invocation .= " -e $eid";
+    }
     $invocation .= " -a" unless $available;
     if (defined($pc)) {
 	$invocation .= " -1 $pc";
@@ -1535,6 +1558,7 @@ sub SliverWorkAux($$$$$$$)
     my $response;
     my $ticket;
     my $rspec;
+    my $oldmanifest;
     require Interface;
 
     # V2 API support.
@@ -1665,10 +1689,18 @@ sub SliverWorkAux($$$$$$$)
 
     print GeniXML::Serialize($rspec);
 
+    # The Manifest starts out as a copy of the rspec. 
+    my $manifest = $rspec->cloneNode(1);
+
     #
     # Find current slivers and save.
     #
     if (defined($aggregate)) {
+	$oldmanifest = $aggregate->GetManifest(0);
+	if (!defined($oldmanifest)) {
+	    $message = "Internal error getting manifest";
+	    goto bad;
+	}
 	my @slivers;
 	if ($aggregate->SliverList(\@slivers) != 0) {
 	    $message = "Could not get sliverlist for $aggregate";
@@ -2040,8 +2072,22 @@ sub SliverWorkAux($$$$$$$)
 	}
 
 	# Already in the aggregate?
-	next
-	    if (grep {$_ eq $virtual_id} keys(%nodemap));
+	if (grep {$_ eq $virtual_id} keys(%nodemap)) {
+	    #
+	    # Need to find the old manifest entry and make sure it
+	    # gets into the new manifest.
+	    #
+	    if (defined($oldmanifest)) {
+		my $oldnode = GeniXML::GetNodeByVirtualId($virtual_id,
+							  $oldmanifest);
+		my $curnode = GeniXML::GetNodeByVirtualId($virtual_id,
+							  $manifest);
+		if (defined($oldnode) && defined($curnode)) {
+		    GeniXML::ReplaceNode($curnode, $oldnode);
+		}
+	    }
+	    next;
+	}
 	
 	my $node = GeniUtil::LookupNode($resource_id);
 	if (!defined($node)) {
@@ -2080,6 +2126,13 @@ sub SliverWorkAux($$$$$$$)
 	    }
 	    push(@plabnodes, $vnode);
 	}
+	# Store the updated rspec for later.
+	goto bad
+	    if ($sliver->UpdateRspec($ref));
+
+	# And store into the new manifest.
+	my $oldnode = GeniXML::GetNodeByVirtualId($virtual_id, $manifest);
+	GeniXML::ReplaceNode($oldnode, $ref);
     }
 
     #
@@ -2126,8 +2179,22 @@ sub SliverWorkAux($$$$$$$)
 	# Do not worry about modifying a link that is setup, unless a tunnel.
 	# Need to deal with this later.
 	if (grep {$_ eq $linkname} keys(%linkmap)) {
-	    next
-		if ($is_tunnel);
+	    if (!$is_tunnel) {
+		#
+		# Need to find the old manifest entry and make sure it
+		# gets into the new manifest.
+		#
+		if (defined($oldmanifest)) {
+		    my $oldlink = GeniXML::GetLinkByVirtualId($linkname,
+							      $oldmanifest);
+		    my $curlink = GeniXML::GetLinkByVirtualId($linkname,
+							      $manifest);
+		    if (defined($oldlink) && defined($curlink)) {
+			GeniXML::ReplaceNode($curlink, $oldlink);
+		      }
+		}
+		next;
+	    }
 
 	    #
 	    # Gack, we have to recreate all the tunnel physical state, but
@@ -2201,7 +2268,7 @@ sub SliverWorkAux($$$$$$$)
 		$message = "Could not set aggregate for $tunnel to $aggregate";
 		goto bad;
 	    }
-	    next;
+	    goto manifest;
 	}
 
 	my $linkaggregate = GeniAggregate::Link->Create($slice, $owner,
@@ -2340,6 +2407,11 @@ sub SliverWorkAux($$$$$$$)
 		goto bad;
 	    }
 	}
+      manifest:
+	# And store into the new manifest.
+	my $oldlink = GeniXML::GetLinkByVirtualId($linkname, $manifest);
+	GeniXML::ReplaceNode($oldlink, $linkref);
+	
     }
   skiplinks:
     #
@@ -2445,14 +2517,12 @@ sub SliverWorkAux($$$$$$$)
 	}
     }
 
-    # The Manifest.
-    my $manifest = GeniXML::Serialize($rspec);
-
     #
     # Move this elsewhere.
     #
+    my $manifest_string = GeniXML::Serialize($manifest);
     DBQueryWarn("replace into geni_manifests set ".
-		"  manifest=". DBQuoteSpecial($manifest) . ", " .
+		"  manifest=". DBQuoteSpecial($manifest_string) . ", " .
 		"  idx=NULL, slice_uuid='$slice_uuid', created=now()");
 
     if (GeniUsage->NewManifest($aggregate, $rspec)) {
@@ -2485,7 +2555,7 @@ sub SliverWorkAux($$$$$$$)
 	$slice->UnLock();
 	return GeniResponse->Create(GENIRESPONSE_SUCCESS,
 				    [$sliver_credential->asString(),
-				     $manifest]);
+				     $manifest_string]);
     }
     #
     # Free any slivers that were no longer wanted.
@@ -2505,9 +2575,9 @@ sub SliverWorkAux($$$$$$$)
     if ($v2) {
 	return GeniResponse->Create(GENIRESPONSE_SUCCESS,
 				    [$sliver_credential->asString(),
-				     $manifest]);
+				     $manifest_string]);
     } else {
-	return GeniResponse->Create(GENIRESPONSE_SUCCESS, $manifest);
+	return GeniResponse->Create(GENIRESPONSE_SUCCESS, $manifest_string);
     }
 
   bad:

protogeni/lib/GeniCMV2.pm.in

@@ -125,7 +125,7 @@ sub Resolve($)
     
     if ($type eq "node") {
 	my $node  = $object;
-	my $rspec = GeniCM::GetAdvertisement(0, $node->node_id(), "0.1");
+	my $rspec = GeniCM::GetAdvertisement(0, $node->node_id(), "0.1", undef);
 	if (! defined($rspec)) {
 	    return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
 					"Error getting advertisement");
@@ -585,6 +585,8 @@ sub SliverAction($$$$$)
 {
     my ($action, $slice_urn, $sliver_urns, $credentials, $manifest) = @_;
     my $response;
+
+    print STDERR "fooey\n";
     
     if (! (defined($credentials) &&
 	   (defined($slice_urn) || defined($sliver_urns)))) {

protogeni/lib/GeniSliver.pm.in

@@ -321,6 +321,26 @@ sub GetManifest($$)
     return $xml;
 }
 
+#
+# Store the rspec/manifest string.
+#
+sub UpdateRspec($$)
+{
+    my ($self, $rspec) = @_;
+
+    my $idx          = $self->idx();
+    my $rspec_string = GeniXML::Serialize($rspec);
+    my $safe_rspec   = DBQuoteSpecial($rspec_string);
+
+    return -1
+	if (!DBQueryWarn("update geni_slivers set ".
+			 "  rspec_string=$safe_rspec ".
+			 "where idx='$idx'"));
+    
+    $self->{'RSPEC'} = $rspec;
+    return 0;
+}
+
 #
 # Set the aggregate for a sliver.
 #

tbsetup/libossetup.pm.in

@@ -1645,14 +1645,45 @@ sub LightUpNodes($@)
     my $experiment = $parent->experiment();
     my $pid        = $experiment->pid();
     my $eid        = $experiment->eid();
+    my @rebootlist = ();
 
+    #
+    # What nodes need reboot.
+    #
+    foreach my $node (@nodelist) {
+	push (@rebootlist, $node)
+	    if ($node->allocstate() eq TBDB_ALLOCSTATE_RES_REBOOT());
+    }
+    
     TBDebugTimeStamp("Starting Geni setup.");
+    #
+    # This will start newly added slivers, but not existing.
+    #
     if (libGeni::StartSlivers($experiment,
-			      $parent->user(), 0, $parent->debug())) {
+			      $parent->user(), $parent->debug())) {
 	print STDERR "*** Could not start Geni slivers\n";
 	return 1;
     }
     TBDebugTimeStamp("Geni slivers have been started.");
+
+    # Now reboot the rest.
+    if (@rebootlist) {
+	if (libGeni::RestartNodes($parent->user(), $parent->debug(), @rebootlist)) {
+	    tbdie("Could not restart protogeni nodes\n");
+	}
+	#
+	# We always do a wait for geni nodes since for nodes in "basic"
+	# cooked mode, nothing will be reporting a state change from the
+	# node. We have to go poll it to make sure that the node is alive,
+	# and so we can report ISUP for it. This is a bit of a violation of
+	# the default reboot model, which is fire and forget when waitmode
+	# is not set, but no way around it.
+	#
+	if (libGeni::WaitForNodes($parent->user(),
+				  $parent->debug(), undef, @rebootlist)) {
+	    tbdie("Error in waiting for protogeni nodes\n");
+	}
+    }
     return 0;
 }
 

tbsetup/mapper.in

@@ -321,9 +321,9 @@ sub AssignLoop()
 
     TBDebugTimeStamp("mapper loop started");
     while (1) {
-	chat("Assign run $currentrun\n");
+	chat("Mapper loop $currentrun\n");
 
-	my $prefix = ($debug || $regression ? "$pid-$eid" : "$pid-$eid-$$");
+	my $prefix = "$pid-$eid-$$";
 
 	#
 	# When precheck is on, we only do one run in impotent mode and exit.
@@ -469,9 +469,24 @@ sub RunAssign($$)
 	return -1;
     }
     close(VTOPFILE);
+    system("/bin/cp -fp $vtopfile ${pid}-${eid}.vtop") if ($debug);
     return 0
 	if ($toponly);
 
+    if (! ($impotent || $regression || $alloconly)) {
+	if ($experiment->Update({"maximum_nodes" => $vtop->maximum_nodes(),
+				 "minimum_nodes" => $vtop->minimum_nodes(),
+				 "virtnode_count"=> $vtop->virtnodecount() })){
+	    tberror("Could not update min/max/virt nodes for $experiment\n");
+	    return -1;
+	}
+    }
+    # New solution each time.
+    $vtop->ClearSolution();
+
+    goto skipassign
+	if ($vtop->nodecount() == $vtop->genicount());
+
     # Debugging hack for regression mode. Avoid really long assign runs.
     if ($regression && $noassign) {
 	if (! -e "assign.log") {
@@ -550,17 +565,9 @@ sub RunAssign($$)
 	tberror("Failure in ptopgen\n");
 	return -1;
     }
+    system("/bin/cp -fp $ptopfile ${pid}-${eid}.ptop") if ($debug);
     TBDebugTimeStamp("ptopgen finished");
 
-    if (! ($impotent || $regression || $alloconly)) {
-	if ($experiment->Update({"maximum_nodes" => $vtop->maximum_nodes(),
-				 "minimum_nodes" => $vtop->minimum_nodes(),
-				 "virtnode_count"=> $vtop->virtnodecount() })){
-	    tberror("Could not update min/max/virt nodes for $experiment\n");
-	    return -1;
-	}
-    }
-
     # Run assign
     my $cmd  = "assign";
     $cmd .= "-new"
@@ -650,7 +657,8 @@ sub RunAssign($$)
     # purposes We do not call it .log though, since we do not want it
     # copied out to the user directory every swapin. See Experiment.pm
     #
-    system("/bin/cp assign.log ${prefix}.assign");
+    system("/bin/cp -fp assign.log ${prefix}.assign");
+    system("/bin/cp -fp assign.log ${pid}-${eid}.assign") if ($debug);
 
     if ($assignexitcode) {
 	print "Assign exited with $assignexitcode\n" if ($debug);
@@ -680,9 +688,6 @@ sub RunAssign($$)
 	print("Could not open assign logfile! $!\n");
 	return -1;
     }
-    # New solution each time.
-    $vtop->ClearSolution();
-
     TBDebugTimeStamp("ReadSolution started");
     if ($vtop->ReadTextSolution(*ASSIGNFP) != 0) {
 	print("Could not parse assign logfile! $!\n");
@@ -690,8 +695,9 @@ sub RunAssign($$)
     }
     close(ASSIGNFP);
     TBDebugTimeStamp("ReadSolution ended");
+  skipassign:
     
-    if (defined($vtop->genirspec())) {
+    if (defined($vtop->genicount())) {
 	TBDebugTimeStamp("Map Geni Resources Started");
 	if ($vtop->MapResources() != 0) {
 	    print("Could not map external resources! $!\n");

tbsetup/tbswap.in

@@ -1021,7 +1021,7 @@ sub doSwapin($) {
 		if ($isvirt) {
 		    push(@virtnodes, $node);
 		}
-		elsif (!$isrem) {
+		else {
 		    push(@physnodes, $node);
 		}
 	    }
@@ -1234,16 +1234,19 @@ sub doSwapin($) {
 			 "where r.pid='$pid' and r.eid='$eid'");
 
 	while (my ($node,$allocstate) = $db_result->fetchrow_array) {
+	    print STDERR "A: $node, $allocstate\n";
+	    
 	    #
-	    # If the node is INIT_CLEAN, leave it alone. It will still get
-	    # rebooted, but will not falsely be tagged as dirty. This is
-	    # important for vnodes too, where INIT_CLEAN indicated the vnode
-	    # does not even exist yet (plab nodes).
+	    # INIT_CLEAN and INIT_DIRTY indicate brand new nodes. Leave
+	    # the state alone for os_setup so it knows. REBOOT means libvtop
+	    # knows something we do not, so do not change that either.
 	    #
-	    if ($allocstate ne TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
+	    if ($allocstate ne TBDB_ALLOCSTATE_RES_INIT_CLEAN() &&
+		$allocstate ne TBDB_ALLOCSTATE_RES_INIT_DIRTY() &&
+		$allocstate ne TBDB_ALLOCSTATE_RES_REBOOT()) {
 		TBSetNodeAllocState($node,
 				    ($needreboot ?
-				     TBDB_ALLOCSTATE_RES_INIT_DIRTY() :
+				     TBDB_ALLOCSTATE_RES_REBOOT() :
 				     TBDB_ALLOCSTATE_RES_RECONFIG()));
 	    }
 	}