Merge branch 'master' of git-public.flux.utah.edu:/flux/git/emulab-devel

install/update-testbed.in

@@ -143,11 +143,13 @@ my @INCDIRS = ("-I${objdir}", "-I${objdir}/../tbsetup",
 	       );
 
 # Chicken or Egg.
-DBQueryFatal("INSERT INTO sitevariables VALUES ".
+# XXX only set specific fields as the ns_include field may not be there yet.
+DBQueryFatal("INSERT INTO sitevariables (name,value,defaultvalue,description)".
+	     " VALUES ".
 	     " ('general/testbed_shutdown',NULL,'0', ".
 	     "  'Non-zero value indicates that the testbed is shutdown ".
 	     "and scripts should not do anything when they run. ".
-	     "DO NOT SET THIS BY HAND!', 0)")
+	     "DO NOT SET THIS BY HAND!')")
     if (!SiteVarExists("general/testbed_shutdown"));
 
 #

install/updates/5/4

@@ -20,14 +20,14 @@ sub InstallUpdate($$)
 	Phase "p5-Crypt-X509", "Checking for port p5-Crypt-X509", sub {
 	    DoneIfPackageInstalled("p5-Crypt-X509");
 	    ExecQuietFatal("cd $PORTSDIR/security/p5-Crypt-X509; ".
-			   "make -DBATCH install");
+			   "make MASTER_SITE_FREEBSD=1 -DBATCH install");
 	};
     
 	Phase "p5-Crypt-OpenSSL-X509",
 	      "Checking for port p5-Crypt-OpenSSL-X509", sub {
 	    DoneIfPackageInstalled("p5-Crypt-OpenSSL-X509");
 	    ExecQuietFatal("cd $PORTSDIR/security/p5-Crypt-OpenSSL-X509; ".
-			   "make -DBATCH install");
+			   "make MASTER_SITE_FREEBSD=1 -DBATCH install");
 	};
     }
     

install/updates/5/8

@@ -20,7 +20,7 @@ sub InstallUpdate($$)
 		DoneIfPackageInstalled("xerces");
 		ExecQuietFatal("cd $PORTSDIR/textproc/xerces-c2; ".
 			       "make clean; make rmconfig; ".
-			       "make -DBATCH WITH_DEBUG=on install");
+			       "make MASTER_SITE_FREEBSD=1 -DBATCH WITH_DEBUG=on install");
 	    };
 	    Phase "reconfig", "Checking to see if reconfigure needed", sub {
 		PhaseSkip("No reconfig needed")

protogeni/lib/GeniAggregate.pm.in

@@ -29,7 +29,6 @@ use GeniComponent;
 use GeniHRN;
 use GeniXML;
 use emutil;
-use Lan;
 use Data::Dumper;
 use English;
 use overload ('""' => 'Stringify');
@@ -730,6 +729,8 @@ sub ProcessManifest($$)
 sub Start($$$)
 {
     my ($self, $version, $restart) = @_;
+    require Lan;
+    require OSinfo;
 
     return -1
 	if (! ref($self));

protogeni/lib/GeniAuthority.pm.in

@@ -95,6 +95,8 @@ sub Lookup($$)
     my $self              = {};
     $self->{'AUTHORITY'}  = $query_result->fetchrow_hashref();
     $self->{'version'}    = undef;
+    $self->{'apilevel'}   = undef;
+    $self->{'api'}        = undef;
     bless($self, $class);
 
     #
@@ -178,7 +180,9 @@ sub url($)		{ return field($_[0], "url"); }
 sub hrn($)		{ return field($_[0], "hrn"); }
 sub type($)		{ return field($_[0], "type"); }
 sub disabled($)		{ return field($_[0], "disabled"); }
-sub version($)		{ return field($_[0], "version"); }
+sub version($)		{ return $_[0]->{"version"}; }
+sub apilevel($)		{ return $_[0]->{"apilevel"}; }
+sub api($)		{ return $_[0]->{"api"}; }
 sub cert($)		{ return $_[0]->{'CERT'}->cert(); }
 sub GetCertificate($)   { return $_[0]->{'CERT'}; }
 
@@ -329,13 +333,53 @@ sub Version($)
 	return undef;
     }
     if (ref($response->value())) {
-	$self->{'version'} = $response->value()->{'api'};
+	# Look for the AM interface.
+	if (exists($response->value()->{'geni_api'})) {
+	    $self->{'version'}  = $response->value()->{'geni_api'};
+	    # This was wrong; it should be 2.0 not 1.0
+	    $self->{'version'}  = 2.0 if ($self->{'version'} == 1.0);
+	    $self->{'apilevel'} = 0;
+	    $self->{'api'}      = "AM";
+	}
+	else {
+	    $self->{'version'}  = $response->value()->{'api'};
+	    $self->{'apilevel'} = $response->value()->{'level'};
+	    $self->{'apilevel'} = 0;
+	    $self->{'api'}      = "CM";
+	}
     }
     else {
-	$self->{'version'} = $response->value();
+	$self->{'version'}  = $response->value();
+	$self->{'apilevel'} = 1;
+	$self->{'api'}      = "CM";
     }
     return $self->{'version'};
 }
+# Ditto for the API level
+sub ApiLevel($)
+{
+    my ($self) = @_;
+
+    return $self->apilevel()
+	if (defined($self->apilevel()));
+
+    return undef
+	if (!defined($self->Version()));
+
+    return $self->apilevel();
+}
+sub Api($)
+{
+    my ($self) = @_;
+
+    return $self->api()
+	if (defined($self->api()));
+
+    return undef
+	if (!defined($self->Version()));
+
+    return $self->api();
+}
 
 #
 # Check that the authority is the issuer of the given certificate.

protogeni/lib/GeniCMV2.pm.in

@@ -20,34 +20,24 @@ use vars qw(@ISA @EXPORT);
 # Must come after package declaration!
 use lib '@prefix@/lib';
 use GeniDB;
-use Genixmlrpc;
 use GeniResponse;
 use GeniTicket;
 use GeniCredential;
 use GeniCertificate;
 use GeniSlice;
 use GeniAggregate;
-use GeniAuthority;
 use GeniSliver;
-use GeniUser;
-use GeniRegistry;
 use GeniUtil;
 use GeniCM;
 use GeniHRN;
 use GeniXML;
-use libtestbed qw(SENDMAIL);
 use emutil;
-# Hate to import all this crap; need a utility library.
-use libdb qw(TBGetSiteVar EXPTSTATE_SWAPPED EXPTSTATE_ACTIVE TBOPSPID
-	     TBDB_NODESTATE_TBFAILED);
-use Node;
 use English;
 use Data::Dumper;
 use XML::Simple;
 use Date::Parse;
 use POSIX qw(strftime tmpnam);
 use Time::Local;
-use Experiment;
 use Compress::Zlib;
 use MIME::Base64;
 
@@ -79,8 +69,7 @@ my $PRERENDER      = "$TB/libexec/vis/prerender";
 my $EMULAB_PEMFILE = "@prefix@/etc/genicm.pem";
 # Just one of these, at Utah.
 my $GENICH_PEMFILE = "@prefix@/etc/genich.pem";
-
-my $API_VERSION = 2;
+my $API_VERSION    = 2;
 
 #
 # Tell the client what API revision we support.  The correspondence
@@ -97,7 +86,7 @@ sub GetVersion()
 	"output_rspec" => "0.1"
     };
 
-    return GeniResponse->Create( GENIRESPONSE_SUCCESS, $blob );
+    return GeniResponse->Create( GENIRESPONSE_SUCCESS, $blob);
 }
 
 #
@@ -268,6 +257,8 @@ sub CreateSliver($)
     my $credentials  = $argref->{'credentials'};
     my $keys         = $argref->{'keys'};
     my $impotent     = $argref->{'impotent'} || 0;
+    require Node;
+    require Experiment;
     
     # For now, I am not worrying about the slice_urn argument.
     if (! (defined($credentials) &&
@@ -699,24 +690,21 @@ sub SliverAction($$$$$)
     my $PerformAction = sub {
 	my ($object, $action) = @_;
 
+	my $exitval = 0;
+
 	if ($action eq "start") {
-	    if ($object->Start($API_VERSION, 0) != 0) {
-		return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-					    "Could not start sliver");
-	    }
+	    $exitval = $object->Start($API_VERSION, 0);
 	}
 	elsif ($action eq "stop") {
-	    if ($object->Stop($API_VERSION) != 0) {
-		return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-					    "Could not stop sliver");
-	    }
+	    $exitval = $object->Stop($API_VERSION);
 	}
 	elsif ($action eq "restart") {
-	    if ($object->Start($API_VERSION, 1) != 0) {
-		return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
-					    "Could not restart sliver");
-	    }	    
+	    $exitval = $object->Start($API_VERSION, 1);
 	}
+	return GeniResponse->Create(GENIRESPONSE_ERROR, 
+				    "Could not $action sliver")
+	    if ($exitval);
+	
 	return 0;
     };
 
@@ -887,6 +875,7 @@ sub Shutdown($)
     my $slice_urn    = $argref->{'slice_urn'};
     my $clear        = $argref->{'clear'} || 0;
     my $credentials  = $argref->{'credentials'};
+    require libtestbed;
 
     if (! (defined($credentials) && defined($slice_urn))) {
 	return GeniResponse->MalformedArgsResponse("Missing arguments");
@@ -946,8 +935,8 @@ sub Shutdown($)
 	    return GeniResponse->BusyResponse();
 	}
 	if (GeniCM::CleanupDeadSlice($slice, 0) != 0) {
-	    SENDMAIL($TBOPS, "Emergency Shutdown failed",
-		     "Emergency shutdown failed on $slice\n");
+	    libtestbed::SENDMAIL($TBOPS, "Emergency Shutdown failed",
+				 "Emergency shutdown failed on $slice\n");
 	    print STDERR "Could not shutdown $slice!\n";
 	    # Lets call this a non-error since the local admin person
 	    # is going to have to deal with it anyway. 
@@ -1365,7 +1354,7 @@ sub ReleaseTicket($)
 
     my ($slice, $aggregate) = Credential2SliceAggregate($credential);
     if (! (defined($slice))) {
-	return GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
+	return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
 				    "No slice here");
     }
     if ($slice_urn ne $slice->urn()) {

protogeni/lib/GeniEmulab.pm.in

@@ -62,8 +62,19 @@ sub RegisterExperiment($$)
     my ($experiment, $user) = @_;
 
     my $slice = GeniSlice->LookupByExperiment($experiment);
-    return 0
-	if (defined($slice));
+
+    #
+    # If we have a slice for it, check to see if its expired. We just
+    # go ahead and renew it.
+    #
+    if (defined($slice)) {
+	if ($slice->IsExpired() &&
+	    $slice->SetExpiration(time() + (24 * 3600 * 30))) {
+	    print STDERR "Could not reset slice expiration\n";
+	    return -1;
+	}
+	return 0;
+    }
 
     #
     # Load the SA cert to act as caller context.

protogeni/lib/GeniRegistry.pm.in

@@ -933,7 +933,7 @@ sub GetCredential($;$$)
     my $clearinghousecred =
 	GeniCredential->Lookup($clearinghousecert->uuid(), $myuuid);
 
-    #print "$clearinghousecred\n";
+    #print "$clearinghousecred\n" if (defined($clearinghousecred));
 
     if (defined($clearinghousecred) &&
 	defined($clearinghousecred->expires())) {

protogeni/lib/GeniSliver.pm.in

@@ -24,7 +24,6 @@ use GeniUsage;
 use GeniHRN;
 use GeniXML;
 use emutil;
-use OSinfo;
 use Node;
 use English;
 use XML::Simple;
@@ -605,7 +604,7 @@ sub component_urn($)
 sub Create($$$$$$)
 {
     # $rspec is a LibXML element representing a single node.
-    my ($class, $slice, $user, $resource_uuid, $rspec) = @_;
+    my ($class, $slice, $user, $node, $rspec) = @_;
     my $virtualization_type = GeniXML::GetVirtualizationType($rspec);
     if (!defined($virtualization_type)) {
 	print STDERR "Node does not contain a virtualization_type\n";
@@ -618,17 +617,6 @@ sub Create($$$$$$)
 	return undef;
     }
 
-    #
-    # the node is already allocated to the sliver, but still need to enter
-    # a virt_nodes entry, and possibly more virt table entries, so that the
-    # node will boot properly, and is otherwise controllable.
-    #
-    my $node = Node->Lookup($resource_uuid);
-    if (!defined($node)) {
-	print STDERR "Could not map node $resource_uuid to its object\n";
-	return undef;
-    }
-
     #
     # An artifact of Emulab is that for shared/remote nodes, the physical
     # node is already allocated, but not to the current experiment.
@@ -651,32 +639,16 @@ sub Create($$$$$$)
 	print STDERR "Node does not contain a virtual_id\n";
 	return undef;
     }
-    my $hrn;
-    my $sshdport = 22;
-    my $hostname;
-    my $phostname;
-    my $resource_id;
+    my $resource_uuid = $node->uuid();
+    my $resource_id   = $node->node_id();
+    my $hrn           = "${PGENIDOMAIN}." . $node->node_id();
+    my $sshdport      = 22;
+    my $hostname      = GeniUtil::FindHostname($node->node_id());
+    my $phostname     = $hostname;
 
-    #
-    # The resource UUID refers to the physical node, but the virtualization
-    # type might require a vnode.
-    # 
-    if ($virtualization_type eq "emulab-vnode") {
-	my $vnode = $experiment->VnameToNode($nickname);
-	if (!defined($vnode)) {
-	    print STDERR "Could not lookup node $nickname in $experiment\n";
-	    return undef;
-	}
-	$hrn = "${PGENIDOMAIN}." . $vnode->node_id();
-	$resource_id = $vnode->node_id();
-	$hostname = GeniUtil::FindHostname($vnode->node_id());
-	$phostname = GeniUtil::FindHostname($vnode->phys_nodeid());
-	$sshdport = $vnode->sshdport();
-    }
-    else {
-	$hrn = "${PGENIDOMAIN}." . $node->node_id();
-	$resource_id = $node->node_id();
-	$phostname = $hostname = GeniUtil::FindHostname($node->node_id());
+    if ($node->isvirtnode()) {
+	$sshdport  = $node->sshdport();
+	$phostname = GeniUtil::FindHostname($node->phys_nodeid());
     }
 
     #
@@ -796,7 +768,7 @@ sub UnProvision($;$)
     my $experiment = $self->GetExperiment();
     if (!defined($experiment)) {
 	print STDERR "Could not map $self to its experiment\n";
-	return -1;
+	return 0;
     }
     my $node_id = $self->resource_id();
     return 0

protogeni/lib/GeniTicket.pm.in

@@ -777,6 +777,7 @@ sub Release($$)
     my $eid     = $experiment->eid();
     my @nodeids = ();
     my @nodes   = ();
+    my @vhosts  = ();
 
     foreach my $ref (GeniXML::FindNodes("n:node",
 					$self->rspec())->get_nodelist()) {
@@ -795,7 +796,6 @@ sub Release($$)
 	    return -1;
 	}
 
-	# Virtual nodes not created until ticket redeemed.
 	my $node = GeniUtil::LookupNode($resource_id);
 	next
 	    if (!defined($node));
@@ -804,10 +804,6 @@ sub Release($$)
 	next
 	    if (!defined($reservation));
 
-	# Watch for duplicates, as in multiple vnodes on a pnode.
-	next
-	    if (grep {$_ eq $node->node_id()} @nodeids);
-
 	#
 	# If the node is still in the experiment and not incorporated,
 	# release it. genisliver_idx is not defined until ticket redeemed.
@@ -819,6 +815,13 @@ sub Release($$)
 		 $restable->{'genisliver_idx'} == 0)) {
 		push(@nodeids, $node->node_id());
 		push(@nodes, $node);
+
+		# So we can tell if we need to deallocate a physical
+		# host once all the VMs on it are gone. 
+		if ($node->isvirtnode() && !$node->sharing_mode()) {
+		    push(@vhosts, $node->phys_nodeid())
+			if (! grep {$_ eq $node->phys_nodeid()} @vhosts);
+		}
 	    }
 	}
     }
@@ -828,10 +831,50 @@ sub Release($$)
 	#
 	#system("export NORELOAD=1; $NFREE -x -q $pid $eid @nodeids");
 	system("$NFREE -x -q $pid $eid @nodeids");
+	return -1
+	    if ($?);
+    }
+    foreach my $node (@nodes) {
+	$node->Refresh();
+    }
+
+    @nodes = @nodeids = ();
+    #
+    # Now go through the vhosts that belong to us, and see if we can
+    # free them cause there are no longer VMs on them.
+    #
+    foreach my $pnode_id (@vhosts) {
+	my $pnode = Node->Lookup($pnode_id);
+	if (!defined($pnode)) {
+	    print STDERR "ReleaseTicket: *** No such pnode $pnode_id\n";
+	    next;
+	}
+	
+	my $reservation = $pnode->Reservation();
+	next
+	    if (!defined($reservation) ||
+		!$reservation->SameExperiment($experiment));
+	
+	my @vnodes = ();
+	if ($pnode->VirtualNodes(\@vnodes) != 0) {
+	    print STDERR
+		"ReleaseTicket: *** Could not get vnode list for $pnode_id\n";
+	    return -1;
+	}
+	if (!@vnodes) {
+	    push(@nodeids, $pnode->node_id());
+	    push(@nodes, $pnode);
+	}
+    }
+    if (@nodeids) {
+	system("$NFREE -x -q $pid $eid @nodeids");
+	return -1
+	    if ($?);
     }
     foreach my $node (@nodes) {
 	$node->Refresh();
     }
+    
     $self->Delete($flag);
     return 0;
 }

protogeni/lib/GeniUtil.pm.in

@@ -44,8 +44,11 @@ sub GENI_PURGEFLAG()	{ return 1; }
 # there is an emulab user context, or many of the scripts we invoke
 # will complain and croak.
 #
-sub FlipToGeniUser()
+sub FlipToGeniUser(;$)
 {
+    my ($default_gid) = @_;
+    my $glist;
+    
     my $unix_uid = getpwnam("$user") or
 	die("*** $0:\n".
 	    "    No such user $user\n");
@@ -53,14 +56,29 @@ sub FlipToGeniUser()
 	die("*** $0:\n".
 	    "    No such group $group\n");
 
-    $GID            = $unix_gid;
-    $EGID           = "$unix_gid $unix_gid";
+    if (defined($default_gid) && $default_gid != $unix_gid) {
+	$glist = "$default_gid $default_gid $unix_gid";
+    }
+    else {
+	$default_gid = $unix_gid;
+	$glist = "$unix_gid $unix_gid";
+    }
+    $GID            = $default_gid;
+    $EGID           = $glist;
     $EUID = $UID    = $unix_uid;
     $ENV{'USER'}    = $user;
     $ENV{'LOGNAME'} = $user;
     return 0;
 }
 
+sub ResetGroups($)
+{
+    my ($default_gid) = @_;
+    
+    $EUID = 0;
+    return FlipToGeniUser($default_gid);
+}
+
 #
 # Store up the list of caches to flush
 #
@@ -104,6 +122,7 @@ sub NewUUID()
 sub LookupNode($)
 {
     my ($nodeid) = @_;
+    require Node;
 
     if( GeniHRN::IsValid( $nodeid ) ) {
 	# Looks like a URN.
@@ -138,5 +157,24 @@ sub FindHostname($)
     return $nodeid . "." . $prefix;    
 }
 
+#
+# Run a command, being sure to capture all output. 
+#
+sub ExecQuiet($)
+{
+    #
+    # Use a pipe read, so that we save away the output
+    #
+    my ($command) = @_;
+    my $output    = "";
+    
+    open(PIPE,"$command 2>&1 |") or return -1;
+    while (<PIPE>) {
+	$output .= $_;
+    }
+    close(PIPE);
+    return $output;
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

protogeni/lib/GeniXML.pm.in

@@ -461,11 +461,12 @@ sub GetText($$)
 
 # Converts the XML representation of a node to a UTF-8 string and
 # outputs it as a complete XML document.
-sub Serialize($)
+sub Serialize($;$)
 {
-    my ($node) = @_;
+    my ($node, $format) = @_;
+    $format = 0 if (!defined($format));
     my $newnode = $node->cloneNode(1);
-    return $newnode->toString();
+    return $newnode->toString($format);
 }
 
 # Create a new XML document with a given namespace URI and document

protogeni/test/createsliver.py

@@ -68,45 +68,20 @@ mykeys = response["value"]
 if debug: print str(mykeys)
 
 #
-# Lookup slice.
+# Lookup slice and get credential.
 #
-params = {}
-params["credential"] = mycredential
-params["type"]       = "Slice"
-params["hrn"]        = SLICENAME
-rval,response = do_method("sa", "Resolve", params)
-if rval:
-    #
-    # Create a slice. 
-    #
-    print "Creating new slice called " + SLICENAME
-    params = {}
-    params["credential"] = mycredential
-    params["type"]       = "Slice"
-    params["hrn"]        = SLICENAME
-    rval,response = do_method("sa", "Register", params)
-    if rval:
-        Fatal("Could not create new slice")
-        pass
-    myslice = response["value"]
-    print "New slice created"
-    pass
-else:
-    #
-    # Get the slice credential.
-    #
-    print "Asking for slice credential for " + SLICENAME
-    myslice = response["value"]
-    myslice = get_slice_credential( myslice, mycredential )
-    print "Got the slice credential"
-    pass
+myslice = resolve_slice( SLICEURN, mycredential )
+
+print "Asking for slice credential for " + SLICENAME
+slicecredential = get_slice_credential( myslice, mycredential )
+print "Got the slice credential"
 
 #
 # Create the sliver.
 #
 print "Creating the Sliver ..."
 params = {}
-params["credentials"] = (myslice,)
+params["credentials"] = (slicecredential,)
 params["slice_urn"]   = SLICEURN
 params["rspec"]       = rspec
 params["keys"]        = mykeys
@@ -117,5 +92,5 @@ if rval:
     pass
 sliver,manifest = response["value"]
 print "Created the sliver"
-print str(manifest)
+#print str(manifest)
 

protogeni/test/getticket.py

@@ -55,37 +55,13 @@ mycredential = get_self_credential()
 print "Got my SA credential, looking up " + SLICENAME
 
 #
-# Lookup slice.
+# Lookup slice and get credential.
 #
-params = {}
-params["credential"] = mycredential
-params["type"]       = "Slice"
-params["hrn"]        = SLICENAME
-rval,response = do_method("sa", "Resolve", params)
-if rval:
-    #
-    # Create a slice. 
-    #
-    print "Creating new slice called " + SLICENAME
-    params = {}
-    params["credential"] = mycredential
-    params["type"]       = "Slice"
-    params["hrn"]        = SLICENAME
-    rval,response = do_method("sa", "Register", params)
-    if rval:
-        Fatal("Could not create new slice")
-        pass
-    myslice = response["value"]
-    print "New slice created"
-    pass
-else: