Start using apt_aggregates table to drive what clusters are part of the

Portal, what clusters are usable by each portal, ditto datasets. Remove
all hardwired cluster urns and urls.

apt/APT_Aggregate.pm.in

+#!/usr/bin/perl -wT
+#
+# Copyright (c) 2007-2015 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+package APT_Aggregate;
+
+use strict;
+use English;
+use Data::Dumper;
+use Carp;
+use Exporter;
+use vars qw(@ISA @EXPORT $AUTOLOAD);
+
+@ISA    = "Exporter";
+@EXPORT = qw ( );
+
+# Must come after package declaration!
+use EmulabConstants;
+use emdb;
+use emutil;
+use libtestbed;
+use GeniHRN;
+use Genixmlrpc;
+use GeniResponse;
+use GeniCertificate;
+use GeniAuthority;
+use GeniCredential;
+use overload ('""' => 'Stringify');
+
+# Configure variables
+my $TB		  = "@prefix@";
+my $TBOPS         = "@TBOPSEMAIL@";
+my $OURDOMAIN     = "@OURDOMAIN@";
+
+#
+# Lookup by uuid.
+#
+sub Lookup($$)
+{
+    my ($class, $token) = @_;
+    my $query_result;
+
+    if (GeniHRN::IsValid($token)) {
+	my $safe_urn = DBQuoteSpecial($token);
+
+	$query_result =
+	    DBQueryWarn("select * from apt_aggregates where urn=$safe_urn");
+    }
+    else {
+	return undef;
+    }
+    return undef
+	if (!$query_result || !$query_result->numrows);
+
+    my $self             = {};
+    $self->{'AGGREGATE'} = $query_result->fetchrow_hashref();
+
+    bless($self, $class);
+    return $self;
+}
+
+AUTOLOAD {
+    my $self  = $_[0];
+    my $type  = ref($self) or croak "$self is not an object";
+    my $name  = $AUTOLOAD;
+    $name =~ s/.*://;   # strip fully-qualified portion
+
+    # A DB row proxy method call.
+    if (exists($self->{'AGGREGATE'}->{$name})) {
+	return $self->{'AGGREGATE'}->{$name};
+    }
+    carp("No such slot '$name' field in class $type");
+    return undef;
+}
+
+# Break circular reference someplace to avoid exit errors.
+sub DESTROY {
+    my $self = shift;
+
+    $self->{'AGGREGATE'} = undef;
+}
+
+#
+# Refresh a class instance by reloading from the DB.
+#
+sub Refresh($)
+{
+    my ($self) = @_;
+
+    return -1
+	if (! ref($self));
+
+    my $safe_urn = DBQuoteSpecial($self->urn());
+    
+    my $query_result =
+	DBQueryWarn("select * from apt_aggregates where urn=$safe_urn");
+
+    return -1
+	if (!$query_result || !$query_result->numrows);
+
+    $self->{'AGGREGATE'} = $query_result->fetchrow_hashref();
+
+    return 0;
+}
+
+#
+# Stringify for output.
+#
+sub Stringify($)
+{
+    my ($self) = @_;
+    my $urn    = $self->urn();
+
+    return "[APT_Aggregate: $urn]";
+}
+
+# _Always_ make sure that this 1 is at the end of the file...
+1;

apt/APT_Dataset.pm.in

@@ -39,6 +39,7 @@ use emdb;
 use emutil;
 use libtestbed;
 use APT_Geni;
+use APT_Aggregate;
 use GeniHRN;
 use Genixmlrpc;
 use GeniResponse;
@@ -132,17 +133,11 @@ sub DESTROY {
 sub ValidBlockstoreBackend($)
 {
     my ($authority) = @_;
-    my $hrn = GeniHRN->new($authority);
+    my $aggregate   = APT_Aggregate->Lookup($authority);
     return -1
-	if (!defined($hrn));
-    my $domain = $hrn->domain();
-    
-    return 1
-	if ($domain eq "emulab.net" ||
-	    $domain eq "apt.emulab.net" ||
-	    $domain eq "clemson.cloudlab.us");
+	if (!defined($aggregate));
 
-    return 0;
+    return $aggregate->has_datasets();
 }
 
 #

apt/APT_Instance.pm.in

@@ -78,21 +78,22 @@ my $usemydevtree  = 0;
 #
 # Lookup by uuid.
 #
-sub Lookup($$;$)
+sub Lookup($$)
 {
-    my ($class, $uuid) = @_;
-
-    if ($uuid !~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
-	return undef
-    }
+    my ($class, $token) = @_;
+    my $query_result;
 
-    # Look in cache first
-    return $instances{"$uuid"}
-        if (exists($instances{"$uuid"}));
-    
-    my $query_result =
-	DBQueryWarn("select * from apt_instances where uuid='$uuid'");
+    if ($token =~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
+	# Look in cache first
+	return $instances{$token}
+            if (exists($instances{$token}));
 
+	$query_result =
+	    DBQueryWarn("select * from apt_instances where uuid='$token'");
+    }
+    else {
+	return undef;
+    }
     return undef
 	if (!$query_result || !$query_result->numrows);
 
@@ -118,7 +119,7 @@ sub Lookup($$;$)
     $self->{'AGGREGATES'} = $aggregates;
     
     # Add to cache. 
-    $instances{"$uuid"} = $self;
+    $instances{$self->uuid()} = $self;
     
     return $self;
 }
@@ -1033,7 +1034,7 @@ sub Stringify($)
     my $uuid  = $self->uuid();
     my $urn   = $self->aggregate_urn();
 
-    return "[APT_Aggregate: $uuid, $urn]";
+    return "[APT_Instance::Aggregate: $uuid, $urn]";
 }
 
 #
@@ -1914,6 +1915,48 @@ sub Panic($$)
     return undef;
 }
 
+#
+# Control Linktest
+#
+sub RunLinktest($$$)
+{
+    my ($self, $action, $level) = @_;
+    my $authority = $self->GetGeniAuthority();
+    my $urn       = $self->aggregate_urn();
+    my $geniuser  = $self->instance()->GetGeniUser();
+    my $slice     = $self->instance()->GetGeniSlice();
+    my $context   = APT_Geni::GeniContext();
+    return undef
+	if (! (defined($geniuser) && defined($authority) &&
+	       defined($slice) && defined($context)));
+
+    my ($slice_credential, $speaksfor_credential) =
+	APT_Geni::GenCredentials($slice, $geniuser);
+    return undef
+	if (! (defined($speaksfor_credential) &&
+	       defined($slice_credential)));
+
+    my $args = {
+	"slice_urn"   => $slice->urn(),
+	"credentials" => [$slice_credential->asString(),
+			  $speaksfor_credential->asString()],
+    };
+    if ($action eq "kill") {
+	$args->{"stop"} = 1;
+    }
+    else {
+	$args->{"level"} = $level;
+    }
+    my $cmurl = $authority->url();
+    $cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
+
+    my $response = Genixmlrpc::CallMethod($cmurl,
+					  $context, "RunLinktest", $args);
+    return $response;
+  bad:
+    return undef;
+}
+
 #
 # Ask for Image Imfo
 #

apt/GNUmakefile.in

@@ -33,7 +33,8 @@ SUBDIRS		=
 BIN_SCRIPTS	= manage_profile manage_instance manage_dataset \
                   create_instance rungenilib
 SBIN_SCRIPTS	= apt_daemon
-LIB_SCRIPTS     = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm
+LIB_SCRIPTS     = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm \
+		  APT_Aggregate.pm
 WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
                   webcreate_instance webrungenilib
 WEB_SBIN_SCRIPTS=

apt/create_instance.in

@@ -46,14 +46,12 @@ sub usage()
 my @optlist = ('d', 'v', 'u=s', 'a=s', 'S');
 my $debug   = 0;
 my $verbose = 1;
-my $DEFAULT_URN  = "urn:publicid:IDN+apt.emulab.net+authority+cm";
 my $xmlfile;
 my $webtask;
 my $webtask_id;
 my $localuser  = 0;
 my $usestitcher= 0;
 my $quickuuid;
-my $default_aggregate_urn = $DEFAULT_URN;
 my $this_user;
 my $xmlparse;
 my $instance;
@@ -93,6 +91,8 @@ my $UPDATEGENIUSER= "$TB/sbin/protogeni/updategeniuser";
 my $STITCHER      = "$TB/gcf/src/stitcher.py";
 my $OPENSSL       = "/usr/bin/openssl";
 my $MANAGEINSTANCE= "$TB/bin/manage_instance";
+my $DEFAULT_URN   = "urn:publicid:IDN+${OURDOMAIN}+authority+cm";
+my $default_aggregate_urn = $DEFAULT_URN;
 
 # un-taint path
 $ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';

apt/manage_dataset.in

@@ -160,10 +160,10 @@ sub DoCreate()
 	print STDERR "Usage: manage_dataset create ".
 	    "[-t type] [-f fstype] [-e expiration] ".
 	    "[-R global|project] [-W creator|project] ".
-	    "[-a am_urn] -s size pid/name\n";
+	    "-a am_urn -s size pid/name\n";
 	exit(-1);
     };
-    my $aggregate_urn = "urn:publicid:IDN+apt.emulab.net+authority+cm";
+    my $aggregate_urn;
     my $errmsg;
     my $pid;
     my $expires;
@@ -186,6 +186,9 @@ sub DoCreate()
     if (defined($options{"a"})) {
 	$aggregate_urn = $options{"a"};
     }
+    else {
+	&$usage();
+    }
     if (defined($options{"t"})) {
 	$type = $options{"t"};
 	&$usage()

apt/manage_instance.in

@@ -46,6 +46,7 @@ sub usage()
     print("Usage: manage_instance monitor instance\n");
     print("Usage: manage_instance lockdown instance set|clear user|admin\n");
     print("Usage: manage_instance panic instance set|clear\n");
+    print("Usage: manage_instance linktest instance [-k | level]\n");
     print("Usage: manage_instance writecreds instance directory\n");
     exit(-1);
 }
@@ -108,6 +109,7 @@ sub DoReload();
 sub DoLockdown();
 sub DoPanic();
 sub DoManifests();
+sub DoLinktest();
 sub WriteCredentials();
 sub StartMonitor();
 sub StartMonitorInternal(;$);
@@ -173,6 +175,9 @@ elsif ($action eq "lockdown") {
 elsif ($action eq "panic") {
     DoPanic()
 }
+elsif ($action eq "linktest") {
+    DoLinktest()
+}
 elsif ($action eq "writecreds") {
     WriteCredentials()
 }
@@ -839,7 +844,8 @@ sub DoImageTrackerStuff($$$$$$)
     # What happens if the user is doing a snapshot on the cluster where
     # the image lives? The copyback (import) makes no sense in that case,
     # but what if its the same cluster but different projects? In this case
-    # a copyback is not wrong, but sure is inefficient.
+    # we want a standard image clone, and we use whatever URN the cluster
+    # hands back to us.
     #
     # Aside; should we allow snapshots (in the web ui) across projects?
     #
@@ -1380,8 +1386,8 @@ sub DoRefresh()
 	
 	if ($code) {
 	    $errmsg = "Some slivers could not be refreshed";
-	    if ($agg->output()) {
-		$errmsg .= ": " . $agg->output();
+	    if ($agg->webtask()->output()) {
+		$errmsg .= ": " . $agg->webtask()->output();
 	    }
 	    goto bad;
 	}
@@ -1979,6 +1985,124 @@ sub DoPanic()
     exit(-1);
 }
 
+#
+# Linktest
+# 
+sub DoLinktest()
+{
+    my $action = "start";
+    my $level  = 1;
+    my $errmsg;
+    
+    if (@ARGV) {
+	my $arg = shift(@ARGV);
+	if ($arg eq "-k") {
+	    $action = "kill";
+	}
+	elsif ($arg =~ /^\d$/ && $arg >= 1 && $arg <= 4) {
+	    $level = $arg;
+	}
+	else {
+	    usage();
+	}
+    }
+    if ($action eq "start") {
+	if ($instance->status() ne "ready") {
+	    fatal("Must be ready to run linktest!");
+	}
+    }
+    else {
+	if ($instance->status() ne "linktest") {
+	    fatal("Linktest is not running!");
+	}
+    }
+    my $slice = $instance->GetGeniSlice();
+    if (!defined($slice)) {
+	fatal("No slice for instance!");
+    }
+    # The web interface (and in the future the xmlrpc interface) sets this.
+    my $this_user = User->ImpliedUser();
+    if (! defined($this_user)) {
+	$this_user = User->ThisUser();
+    }
+
+    #
+    # Lock the slice in case it is doing something else, like taking
+    # a disk image.
+    #
+    if ($slice->Lock()) {
+	fatal("Slice is busy, cannot lock it");
+    }
+    #
+    # Create the webtask object, but AFTER locking the slice so we do
+    # not destroy one in use.
+    #
+    if (defined($webtask_id)) {
+	$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
+	# Convenient.
+	$webtask->AutoStore(1);
+    }
+
+    #
+    # And tell the backend clusters to run linktest
+    #
+    my $coderef = sub {
+	my ($sliver) = @_;
+	my $webtask  = $sliver->webtask();
+
+	my $response = $sliver->RunLinktest($action, $level);
+	if (!defined($response)) {
+	    print STDERR "RPC Error calling linktest on $sliver\n";
+	    return -1;
+	}
+	if ($response->code() != GENIRESPONSE_SUCCESS) {
+	    print STDERR "Could not run linktest on sliver: ".
+		$response->output() . "\n";
+	    $webtask->output($response->output());
+	    $webtask->Exited(1);
+	    return 1;
+	}
+	print STDERR "foo\n" . $response->output() . "\n";
+	$webtask->output($response->output());
+	$webtask->Exited(0);
+	return 0;
+    };
+    my @return_codes = ();
+    my @agglist =      $instance->AggregateList();
+    if (ParRun({"maxwaittime" => 99999,
+		"maxchildren" => scalar(@agglist)},
+	       \@return_codes, $coderef, @agglist)) {
+	$errmsg = "Internal error calling Lockdown()";
+	goto bad;
+    }
+    #
+    # Check the exit codes. 
+    #
+    foreach my $agg (@agglist) {
+	my $code = shift(@return_codes);
+	if ($code) {
+	    $errmsg = "Could not run linktest on some slivers";
+	    if ($agg->webtask()->output()) {
+		$errmsg .= ": " . $agg->webtask()->output();
+	    }
+	    goto bad;
+	}
+	if (!defined($webtask) && $agg->webtask()->output()) {
+	    print $agg->webtask()->output();
+	}
+    }
+    $slice->UnLock();
+    exit(0);
+  bad:
+    $slice->UnLock();
+    print STDERR $errmsg . "\n";
+    if (defined($webtask)) {
+	$webtask->output($errmsg);
+	$webtask->Exited(1);
+    }
+    exit(1);
+}
+
 #
 # Write instance credentials to files.
 #

sql/database-create.sql

@@ -64,9 +64,13 @@ CREATE TABLE `apt_aggregates` (
   `name` varchar(32) NOT NULL default '',
   `nickname` varchar(32) NOT NULL default '',
   `abbreviation` varchar(16) NOT NULL default '',
+  `adminonly` tinyint(1) NOT NULL default '0',
+  `isfederate` tinyint(1) NOT NULL default '0',
   `noupdate` tinyint(1) NOT NULL default '0',
   `updated` datetime NOT NULL default '0000-00-00 00:00:00',
   `weburl` tinytext,
+  `has_datasets` tinyint(1) NOT NULL default '0',
+  `portals` set('emulab','aptlab','cloudlab','phantomnet') default NULL,
   `jsondata` text,
   PRIMARY KEY  (`urn`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;

sql/updates/4/491

+#
+# APT aggregates table.
+#
+use strict;
+use libdb;
+
+sub DoUpdate($$$)
+{
+    my ($dbhandle, $dbname, $version) = @_;
+
+    if (!DBSlotExists("apt_aggregates", "adminonly")) {
+        DBQueryFatal("alter table apt_aggregates add ".
+		     "  `adminonly` tinyint(1) NOT NULL default '0' ".
+		     "after abbreviation");
+    }
+    if (!DBSlotExists("apt_aggregates", "isfederate")) {
+        DBQueryFatal("alter table apt_aggregates add ".
+		     "  `isfederate` tinyint(1) NOT NULL default '0' ".
+		     "after adminonly");
+    }
+    if (!DBSlotExists("apt_aggregates", "has_datasets")) {
+        DBQueryFatal("alter table apt_aggregates add ".
+		     "  `has_datasets` tinyint(1) NOT NULL default '0' ".
+		     "after weburl");
+    }
+    if (!DBSlotExists("apt_aggregates", "portals")) {
+        DBQueryFatal("alter table apt_aggregates add ".
+		     "  `portals` set('emulab','aptlab','cloudlab',".
+		     "     'phantomnet') default NULL ".
+		     "after has_datasets");
+    }
+    return 0;
+}
+
+# Local Variables:
+# mode:perl
+# End:

www/aptui/aggregate_defs.php

+<?php
+#
+# Copyright (c) 2006-2015 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+#
+
+class Aggregate
+{
+    var	$aggregate;
+    
+    #
+    # Constructor by lookup by urn
+    #
+    function Aggregate($urn) {
+	$safe_urn = addslashes($urn);
+
+	$query_result =
+	    DBQueryWarn("select * from apt_aggregates where urn='$safe_urn'");
+
+	if (!$query_result || !mysql_num_rows($query_result)) {
+	    $this->aggregate = null;
+	    return;
+	}
+	$this->aggregate = mysql_fetch_array($query_result);
+    }
+    # accessors
+    function field($name) {
+	return (is_null($this->aggregate) ? -1 : $this->aggregate[$name]);
+    }
+    function name()	    { return $this->field('name'); }
+    function urn()	    { return $this->field('urn'); }
+    function nickname()	    { return $this->field('nickname'); }
+    function abbreviation() { return $this->field('abbreviation'); }
+    function weburl()	    { return $this->field('weburl'); }
+    function has_datasets() { return $this->field('has_datasets'); }
+    function isfederate()   { return $this->field('isfederate'); }
+    function portals()      { return $this->field('portals'); }
+
+    # Hmm, how does one cause an error in a php constructor?
+    function IsValid() {
+	return !is_null($this->aggregate);
+    }
+
+    # Lookup up by urn,
+    function Lookup($urn) {
+	$foo = new Aggregate($urn);
+
+	if ($foo->IsValid()) {
+	    return $foo;
+	}	
+	return null;
+    }
+
+    #
+    # Generate the free nodes URL from the web url.
+    #
+    function FreeNodesURL() {
+        return $this->weburl() . "/n