Commit df6bcbec authored by Leigh Stoller's avatar Leigh Stoller

Start using apt_aggregates table to drive what clusters are part of the

Portal, what clusters are usable by each portal, ditto datasets. Remove
all hardwired cluster urns and urls.
parent b9964038
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2015 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
package APT_Aggregate;
use strict;
use English;
use Data::Dumper;
use Carp;
use Exporter;
use vars qw(@ISA @EXPORT $AUTOLOAD);
@ISA = "Exporter";
@EXPORT = qw ( );
# Must come after package declaration!
use EmulabConstants;
use emdb;
use emutil;
use libtestbed;
use GeniHRN;
use Genixmlrpc;
use GeniResponse;
use GeniCertificate;
use GeniAuthority;
use GeniCredential;
use overload ('""' => 'Stringify');
# Configure variables
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $OURDOMAIN = "@OURDOMAIN@";
#
# Lookup by uuid.
#
sub Lookup($$)
{
my ($class, $token) = @_;
my $query_result;
if (GeniHRN::IsValid($token)) {
my $safe_urn = DBQuoteSpecial($token);
$query_result =
DBQueryWarn("select * from apt_aggregates where urn=$safe_urn");
}
else {
return undef;
}
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'AGGREGATE'} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
AUTOLOAD {
my $self = $_[0];
my $type = ref($self) or croak "$self is not an object";
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
# A DB row proxy method call.
if (exists($self->{'AGGREGATE'}->{$name})) {
return $self->{'AGGREGATE'}->{$name};
}
carp("No such slot '$name' field in class $type");
return undef;
}
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{'AGGREGATE'} = undef;
}
#
# Refresh a class instance by reloading from the DB.
#
sub Refresh($)
{
my ($self) = @_;
return -1
if (! ref($self));
my $safe_urn = DBQuoteSpecial($self->urn());
my $query_result =
DBQueryWarn("select * from apt_aggregates where urn=$safe_urn");
return -1
if (!$query_result || !$query_result->numrows);
$self->{'AGGREGATE'} = $query_result->fetchrow_hashref();
return 0;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $urn = $self->urn();
return "[APT_Aggregate: $urn]";
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -39,6 +39,7 @@ use emdb;
use emutil;
use libtestbed;
use APT_Geni;
use APT_Aggregate;
use GeniHRN;
use Genixmlrpc;
use GeniResponse;
......@@ -132,17 +133,11 @@ sub DESTROY {
sub ValidBlockstoreBackend($)
{
my ($authority) = @_;
my $hrn = GeniHRN->new($authority);
my $aggregate = APT_Aggregate->Lookup($authority);
return -1
if (!defined($hrn));
my $domain = $hrn->domain();
return 1
if ($domain eq "emulab.net" ||
$domain eq "apt.emulab.net" ||
$domain eq "clemson.cloudlab.us");
if (!defined($aggregate));
return 0;
return $aggregate->has_datasets();
}
#
......
......@@ -78,21 +78,22 @@ my $usemydevtree = 0;
#
# Lookup by uuid.
#
sub Lookup($$;$)
sub Lookup($$)
{
my ($class, $uuid) = @_;
if ($uuid !~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
return undef
}
my ($class, $token) = @_;
my $query_result;
# Look in cache first
return $instances{"$uuid"}
if (exists($instances{"$uuid"}));
my $query_result =
DBQueryWarn("select * from apt_instances where uuid='$uuid'");
if ($token =~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
# Look in cache first
return $instances{$token}
if (exists($instances{$token}));
$query_result =
DBQueryWarn("select * from apt_instances where uuid='$token'");
}
else {
return undef;
}
return undef
if (!$query_result || !$query_result->numrows);
......@@ -118,7 +119,7 @@ sub Lookup($$;$)
$self->{'AGGREGATES'} = $aggregates;
# Add to cache.
$instances{"$uuid"} = $self;
$instances{$self->uuid()} = $self;
return $self;
}
......@@ -1033,7 +1034,7 @@ sub Stringify($)
my $uuid = $self->uuid();
my $urn = $self->aggregate_urn();
return "[APT_Aggregate: $uuid, $urn]";
return "[APT_Instance::Aggregate: $uuid, $urn]";
}
#
......@@ -1914,6 +1915,48 @@ sub Panic($$)
return undef;
}
#
# Control Linktest
#
sub RunLinktest($$$)
{
my ($self, $action, $level) = @_;
my $authority = $self->GetGeniAuthority();
my $urn = $self->aggregate_urn();
my $geniuser = $self->instance()->GetGeniUser();
my $slice = $self->instance()->GetGeniSlice();
my $context = APT_Geni::GeniContext();
return undef
if (! (defined($geniuser) && defined($authority) &&
defined($slice) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($slice, $geniuser);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
my $args = {
"slice_urn" => $slice->urn(),
"credentials" => [$slice_credential->asString(),
$speaksfor_credential->asString()],
};
if ($action eq "kill") {
$args->{"stop"} = 1;
}
else {
$args->{"level"} = $level;
}
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
my $response = Genixmlrpc::CallMethod($cmurl,
$context, "RunLinktest", $args);
return $response;
bad:
return undef;
}
#
# Ask for Image Imfo
#
......
......@@ -33,7 +33,8 @@ SUBDIRS =
BIN_SCRIPTS = manage_profile manage_instance manage_dataset \
create_instance rungenilib
SBIN_SCRIPTS = apt_daemon
LIB_SCRIPTS = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm
LIB_SCRIPTS = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm \
APT_Aggregate.pm
WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
webcreate_instance webrungenilib
WEB_SBIN_SCRIPTS=
......
......@@ -46,14 +46,12 @@ sub usage()
my @optlist = ('d', 'v', 'u=s', 'a=s', 'S');
my $debug = 0;
my $verbose = 1;
my $DEFAULT_URN = "urn:publicid:IDN+apt.emulab.net+authority+cm";
my $xmlfile;
my $webtask;
my $webtask_id;
my $localuser = 0;
my $usestitcher= 0;
my $quickuuid;
my $default_aggregate_urn = $DEFAULT_URN;
my $this_user;
my $xmlparse;
my $instance;
......@@ -93,6 +91,8 @@ my $UPDATEGENIUSER= "$TB/sbin/protogeni/updategeniuser";
my $STITCHER = "$TB/gcf/src/stitcher.py";
my $OPENSSL = "/usr/bin/openssl";
my $MANAGEINSTANCE= "$TB/bin/manage_instance";
my $DEFAULT_URN = "urn:publicid:IDN+${OURDOMAIN}+authority+cm";
my $default_aggregate_urn = $DEFAULT_URN;
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
......
......@@ -160,10 +160,10 @@ sub DoCreate()
print STDERR "Usage: manage_dataset create ".
"[-t type] [-f fstype] [-e expiration] ".
"[-R global|project] [-W creator|project] ".
"[-a am_urn] -s size pid/name\n";
"-a am_urn -s size pid/name\n";
exit(-1);
};
my $aggregate_urn = "urn:publicid:IDN+apt.emulab.net+authority+cm";
my $aggregate_urn;
my $errmsg;
my $pid;
my $expires;
......@@ -186,6 +186,9 @@ sub DoCreate()
if (defined($options{"a"})) {
$aggregate_urn = $options{"a"};
}
else {
&$usage();
}
if (defined($options{"t"})) {
$type = $options{"t"};
&$usage()
......
......@@ -46,6 +46,7 @@ sub usage()
print("Usage: manage_instance monitor instance\n");
print("Usage: manage_instance lockdown instance set|clear user|admin\n");
print("Usage: manage_instance panic instance set|clear\n");
print("Usage: manage_instance linktest instance [-k | level]\n");
print("Usage: manage_instance writecreds instance directory\n");
exit(-1);
}
......@@ -108,6 +109,7 @@ sub DoReload();
sub DoLockdown();
sub DoPanic();
sub DoManifests();
sub DoLinktest();
sub WriteCredentials();
sub StartMonitor();
sub StartMonitorInternal(;$);
......@@ -173,6 +175,9 @@ elsif ($action eq "lockdown") {
elsif ($action eq "panic") {
DoPanic()
}
elsif ($action eq "linktest") {
DoLinktest()
}
elsif ($action eq "writecreds") {
WriteCredentials()
}
......@@ -839,7 +844,8 @@ sub DoImageTrackerStuff($$$$$$)
# What happens if the user is doing a snapshot on the cluster where
# the image lives? The copyback (import) makes no sense in that case,
# but what if its the same cluster but different projects? In this case
# a copyback is not wrong, but sure is inefficient.
# we want a standard image clone, and we use whatever URN the cluster
# hands back to us.
#
# Aside; should we allow snapshots (in the web ui) across projects?
#
......@@ -1380,8 +1386,8 @@ sub DoRefresh()
if ($code) {
$errmsg = "Some slivers could not be refreshed";
if ($agg->output()) {
$errmsg .= ": " . $agg->output();
if ($agg->webtask()->output()) {
$errmsg .= ": " . $agg->webtask()->output();
}
goto bad;
}
......@@ -1979,6 +1985,124 @@ sub DoPanic()
exit(-1);
}
#
# Linktest
#
sub DoLinktest()
{
my $action = "start";
my $level = 1;
my $errmsg;
if (@ARGV) {
my $arg = shift(@ARGV);
if ($arg eq "-k") {
$action = "kill";
}
elsif ($arg =~ /^\d$/ && $arg >= 1 && $arg <= 4) {
$level = $arg;
}
else {
usage();
}
}
if ($action eq "start") {
if ($instance->status() ne "ready") {
fatal("Must be ready to run linktest!");
}
}
else {
if ($instance->status() ne "linktest") {
fatal("Linktest is not running!");
}
}
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
# The web interface (and in the future the xmlrpc interface) sets this.
my $this_user = User->ImpliedUser();
if (! defined($this_user)) {
$this_user = User->ThisUser();
}
#
# Lock the slice in case it is doing something else, like taking
# a disk image.
#
if ($slice->Lock()) {
fatal("Slice is busy, cannot lock it");
}
#
# Create the webtask object, but AFTER locking the slice so we do
# not destroy one in use.
#
if (defined($webtask_id)) {
$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
# Convenient.
$webtask->AutoStore(1);
}
#
# And tell the backend clusters to run linktest
#
my $coderef = sub {
my ($sliver) = @_;
my $webtask = $sliver->webtask();
my $response = $sliver->RunLinktest($action, $level);
if (!defined($response)) {
print STDERR "RPC Error calling linktest on $sliver\n";
return -1;
}
if ($response->code() != GENIRESPONSE_SUCCESS) {
print STDERR "Could not run linktest on sliver: ".
$response->output() . "\n";
$webtask->output($response->output());
$webtask->Exited(1);
return 1;
}
print STDERR "foo\n" . $response->output() . "\n";
$webtask->output($response->output());
$webtask->Exited(0);
return 0;
};
my @return_codes = ();
my @agglist = $instance->AggregateList();
if (ParRun({"maxwaittime" => 99999,
"maxchildren" => scalar(@agglist)},
\@return_codes, $coderef, @agglist)) {
$errmsg = "Internal error calling Lockdown()";
goto bad;
}
#
# Check the exit codes.
#
foreach my $agg (@agglist) {
my $code = shift(@return_codes);
if ($code) {
$errmsg = "Could not run linktest on some slivers";
if ($agg->webtask()->output()) {
$errmsg .= ": " . $agg->webtask()->output();
}
goto bad;
}
if (!defined($webtask) && $agg->webtask()->output()) {
print $agg->webtask()->output();
}
}
$slice->UnLock();
exit(0);
bad:
$slice->UnLock();
print STDERR $errmsg . "\n";
if (defined($webtask)) {
$webtask->output($errmsg);
$webtask->Exited(1);
}
exit(1);
}
#
# Write instance credentials to files.
#
......
......@@ -64,9 +64,13 @@ CREATE TABLE `apt_aggregates` (
`name` varchar(32) NOT NULL default '',
`nickname` varchar(32) NOT NULL default '',
`abbreviation` varchar(16) NOT NULL default '',
`adminonly` tinyint(1) NOT NULL default '0',
`isfederate` tinyint(1) NOT NULL default '0',
`noupdate` tinyint(1) NOT NULL default '0',
`updated` datetime NOT NULL default '0000-00-00 00:00:00',
`weburl` tinytext,
`has_datasets` tinyint(1) NOT NULL default '0',
`portals` set('emulab','aptlab','cloudlab','phantomnet') default NULL,
`jsondata` text,
PRIMARY KEY (`urn`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
......
#
# APT aggregates table.
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if (!DBSlotExists("apt_aggregates", "adminonly")) {
DBQueryFatal("alter table apt_aggregates add ".
" `adminonly` tinyint(1) NOT NULL default '0' ".
"after abbreviation");
}
if (!DBSlotExists("apt_aggregates", "isfederate")) {
DBQueryFatal("alter table apt_aggregates add ".
" `isfederate` tinyint(1) NOT NULL default '0' ".
"after adminonly");
}
if (!DBSlotExists("apt_aggregates", "has_datasets")) {
DBQueryFatal("alter table apt_aggregates add ".
" `has_datasets` tinyint(1) NOT NULL default '0' ".
"after weburl");
}
if (!DBSlotExists("apt_aggregates", "portals")) {
DBQueryFatal("alter table apt_aggregates add ".
" `portals` set('emulab','aptlab','cloudlab',".
" 'phantomnet') default NULL ".
"after has_datasets");
}
return 0;
}
# Local Variables:
# mode:perl
# End:
<?php
#
# Copyright (c) 2006-2015 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
#
class Aggregate
{
var $aggregate;
#
# Constructor by lookup by urn
#
function Aggregate($urn) {
$safe_urn = addslashes($urn);
$query_result =
DBQueryWarn("select * from apt_aggregates where urn='$safe_urn'");
if (!$query_result || !mysql_num_rows($query_result)) {
$this->aggregate = null;
return;
}
$this->aggregate = mysql_fetch_array($query_result);
}
# accessors
function field($name) {
return (is_null($this->aggregate) ? -1 : $this->aggregate[$name]);
}
function name() { return $this->field('name'); }
function urn() { return $this->field('urn'); }
function nickname() { return $this->field('nickname'); }
function abbreviation() { return $this->field('abbreviation'); }
function weburl() { return $this->field('weburl'); }
function has_datasets() { return $this->field('has_datasets'); }
function isfederate() { return $this->field('isfederate'); }
function portals() { return $this->field('portals'); }
# Hmm, how does one cause an error in a php constructor?
function IsValid() {
return !is_null($this->aggregate);
}
# Lookup up by urn,
function Lookup($urn) {
$foo = new Aggregate($urn);
if ($foo->IsValid()) {
return $foo;
}
return null;
}
#
# Generate the free nodes URL from the web url.
#
function FreeNodesURL() {
return $this->weburl() . "/node_usage/freenodes.svg";
}
#
# Return a list of aggregates supporting datasets.
#
function SupportsDatasetsList() {
$result = array();
$query_result =
DBQueryFatal("select urn from apt_aggregates ".
"where has_datasets!=0");
while ($row = mysql_fetch_array($query_result)) {
$urn = $row["urn"];
if (! ($aggregate = Aggregate::Lookup($urn))) {
TBERROR("Aggregate::SupportsDatasetsList: ".
"Could not load aggregate $urn!", 1);
}
$result[] = $aggregate;
}
return $result;
}
#
# Return the list of allowed aggregates based on the portal in use.
#
function DefaultAggregateList() {
global $PORTAL_GENESIS;
$am_array = array();
$query_result =
DBQueryFatal("select urn,name,adminonly from apt_aggregates ".
"where FIND_IN_SET('$PORTAL_GENESIS', portals)");
while ($row = mysql_fetch_array($query_result)) {
$urn = $row["urn"];
$name = $row["name"];
$adminonly = $row["adminonly"];
if ($adminonly && !ISADMIN()) {
continue;
}
$am_array[$name] = $urn;
}
return $am_array;
}
}
#
# We use this in a lot of places, so build it all the time.
#
$urn_mapping = array();
$query_result =
DBQueryFatal("select urn,abbreviation from apt_aggregates");
while ($row = mysql_fetch_array($query_result)) {
$urn_mapping[$row["urn"]] = $row["abbreviation"];
}
?>
......@@ -96,10 +96,13 @@ function SPITFORM($formfields, $errors)
echo htmlentities(json_encode($instance_array));
echo "</script>\n";
#
# Ask the DB for the list of aggregates that do datasets.
#
$amlist = array();
$amlist["urn:publicid:IDN+apt.emulab.net+authority+cm"] = "APT";
$amlist["urn:publicid:IDN+emulab.net+authority+cm"] = "Emulab";
$amlist["urn:publicid:IDN+clemson.cloudlab.us+authority+cm"] = "Clemson";
foreach (Aggregate::SupportsDatasetsList() as $aggregate) {
$amlist[$aggregate->urn()] = $aggregate->nickname();
}
echo "<script type='text/plain' id='amlist-json'>\n";
echo htmlentities(json_encode($amlist));
echo "</script>\n";
......
......@@ -22,30 +22,7 @@
# }}}
#
#
$urn_mapping =
array("urn:publicid:IDN+utah.cloudlab.us+authority+cm" => "Utah",
"urn:publicid:IDN+wisc.cloudlab.us+authority+cm" => "Wisc",
"urn:publicid:IDN+clemson.cloudlab.us+authority+cm" => "Clem",
"urn:publicid:IDN+apt.emulab.net+authority+cm" => "APT",
"urn:publicid:IDN+emulab.net+authority+cm" => "Emulab",
"urn:publicid:IDN+utahddc.geniracks.net+authority+cm" => "DDC",
"urn:publicid:IDN+stitch.geniracks.net+authority+cm" => "UStitch",
"urn:publicid:IDN+al2s.internet2.edu+authority+am" => "AL2S",
"urn:publicid:IDN+uky.emulab.edu+authority+cm" => "UKY",
"urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm" => "Wall2");
$freenodes_mapping =
array("urn:publicid:IDN+utah.cloudlab.us+authority+cm" =>
"https://www.utah.cloudlab.us/node_usage/freenodes.svg",
"urn:publicid:IDN+wisc.cloudlab.us+authority+cm" =>
"https://www.wisc.cloudlab.us/node_usage/freenodes.svg",
"urn:publicid:IDN+clemson.cloudlab.us+authority+cm" =>
"https://www.clemson.cloudlab.us/node_usage/freenodes.svg",
"urn:publicid:IDN+apt.emulab.net+authority+cm" =>
"https://www.apt.emulab.net/node_usage/freenodes.svg",
"urn:publicid:IDN+emulab.net+authority+cm" =>
"https://www.emulab.net/node_usage/freenodes.svg");
include_once("aggregate_defs.php");
$geni_response_codes =
array("Success",
......@@ -349,49 +326,9 @@ class Instance
# Return aggregate based on the current user.
#
function DefaultAggregateList() {
global $ISAPT, $ISCLOUD, $ISPNET, $ISEMULAB;
if ($ISAPT) {