All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit 0adc340f authored by Leigh B Stoller's avatar Leigh B Stoller

Lots of dataset changes.

Project leases are now per-group, so we build a sub authority certificate
for a remote dataset so that on the remote side, it is created inside the
group named by the project on the local side.

Many bug fixes.
parent 0002fc0f
......@@ -39,14 +39,18 @@ use emdb;
use emutil;
use libtestbed;
use APT_Geni;
use GeniHRN;
use Genixmlrpc;
use GeniResponse;
use GeniCertificate;
use GeniAuthority;
use GeniCredential;
use overload ('""' => 'Stringify');
# Configure variables
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $OURDOMAIN = "@OURDOMAIN@";
#
# Lookup by uuid.
......@@ -100,6 +104,17 @@ sub DESTROY {
$self->{'DATASET'} = undef;
}
# Valid Blockstore backend.
sub ValidBlockstoreBackend($)
{
my ($authority) = @_;
return 1
if ($authority eq "emulab.net" || $authority eq "apt.emulab.net");
return 0;
}
#
# Refresh a class instance by reloading from the DB.
#
......@@ -167,7 +182,13 @@ sub Create($$)
return undef;
}
DBQueryWarn("unlock tables");
return Lookup($class, $uuid);
my $dataset = Lookup($class, $uuid);
return undef
if (!defined($dataset));
return undef
if ($dataset->CreateCertificate());
return $dataset;
}
#
......@@ -216,6 +237,10 @@ sub Delete($)
my $uuid = $self->uuid();
my $certificate = $self->GetCertificate();
$certificate->Delete()
if (defined($certificate));
DBQueryWarn("delete from apt_datasets where uuid='$uuid'") or
return -1;
......@@ -343,6 +368,45 @@ sub WarnExpiring($$)
return 0;
}
#
# Create a certificate we can use for the credential. We want this
# certificate to be a subauthority certificate so that the backend puts
# the dataset in an SA subgroup.
#
sub CreateCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
# Kill stale certificate.
my $certificate = GeniCertificate->Lookup($urn);
$certificate->Delete()
if (defined($certificate));
$certificate = GeniCertificate->Create({"urn" => $urn,
"email" => "$TBOPS",
"hrn" => "$OURDOMAIN.$pid.$id"});
return -1
if (!defined($certificate));
# We want to save until we delete the dataset.
$certificate->Store() == 0
or return -1;
return 0;
}
sub GetCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
return GeniCertificate->Lookup($urn);
}
#
# Create a dataset on the remote aggregate.
#
......@@ -352,29 +416,31 @@ sub CreateDataset($)
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $context = APT_Geni::GeniContext();
my $cert = $self->GetCertificate();
return undef
if (! (defined($geniuser) && defined($authority) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($geniuser, $geniuser, ["blockstores"]);
if (! (defined($geniuser) && defined($authority) &&
defined($context) && defined($cert)));
my ($credential, $speaksfor_credential) =
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"]);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
defined($credential)));
my $args = {
"size" => $self->size(),
"name" => $self->dataset_id(),
"type" => $self->type(),
"credentials" => [$slice_credential->asString(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
$args->{"fstype"} = $self->fstype()
if ($self->fstype() ne "none");
$args->{"expires"} = TBDateStringGMT($self->expires())
$args->{"expires"} = emutil::TBDateStringGMT($self->expires())
if (defined($self->expires()));
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/;
# $cmurl =~ s/protogeni/protogeni\/stoller/;
return Genixmlrpc::CallMethod($cmurl, $context, "CreateDataset", $args);
}
......@@ -388,22 +454,24 @@ sub DeleteDataset($)
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $context = APT_Geni::GeniContext();
my $cert = $self->GetCertificate();
return undef
if (! (defined($geniuser) && defined($authority) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($geniuser, $geniuser, ["blockstores"]);
if (! (defined($geniuser) && defined($authority) &&
defined($context) && defined($cert)));
my ($credential, $speaksfor_credential) =
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"]);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"credentials" => [$slice_credential->asString(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/;
# $cmurl =~ s/protogeni/protogeni\/stoller/;
return Genixmlrpc::CallMethod($cmurl, $context, "DeleteDataset", $args);
}
......@@ -417,22 +485,24 @@ sub DescribeDataset($)
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $context = APT_Geni::GeniContext();
my $cert = $self->GetCertificate();
return undef
if (! (defined($geniuser) && defined($authority) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($geniuser, $geniuser, ["blockstores"]);
if (! (defined($geniuser) && defined($authority) &&
defined($context) && defined($cert)));
my ($credential, $speaksfor_credential) =
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"]);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"credentials" => [$slice_credential->asString(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/;
# $cmurl =~ s/protogeni/protogeni\/stoller/;
return Genixmlrpc::CallMethod($cmurl, $context, "DescribeDataset", $args);
}
......
......@@ -30,16 +30,18 @@ include $(OBJDIR)/Makeconf
SUBDIRS =
BIN_SCRIPTS = manage_profile manage_instance manage_dataset create_instance
BIN_SCRIPTS = manage_profile manage_instance manage_dataset \
create_instance rungenilib
SBIN_SCRIPTS =
LIB_SCRIPTS = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm
WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
webcreate_instance
webcreate_instance webrungenilib
WEB_SBIN_SCRIPTS=
LIBEXEC_SCRIPTS = $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS)
USERLIBEXEC = rungenilib.proxy
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS =
SETUID_BIN_SCRIPTS = rungenilib
SETUID_SBIN_SCRIPTS =
SETUID_SUEXEC_SCRIPTS=
......@@ -48,7 +50,7 @@ SETUID_SUEXEC_SCRIPTS=
# configure if the .in file is changed.
#
all: $(BIN_SCRIPTS) $(SBIN_SCRIPTS) $(LIBEXEC_SCRIPTS) $(SUBDIRS) \
$(LIB_SCRIPTS) all-subdirs
$(LIB_SCRIPTS) $(USERLIBEXEC) all-subdirs
subboss:
......@@ -58,6 +60,8 @@ install: $(addprefix $(INSTALL_BINDIR)/, $(BIN_SCRIPTS)) \
$(addprefix $(INSTALL_SBINDIR)/, $(SBIN_SCRIPTS)) \
$(addprefix $(INSTALL_LIBDIR)/, $(LIB_SCRIPTS)) \
$(addprefix $(INSTALL_LIBEXECDIR)/, $(LIBEXEC_SCRIPTS)) \
$(addprefix $(INSTALL_DIR)/opsdir/libexec/, $(USERLIBEXEC))
boss-install: install install-subdirs
......@@ -90,4 +94,9 @@ clean: clean-subdirs
@$(MAKE) -C $(dir $@) $(basename $(notdir $@))
%-subdirs: $(addsuffix /%.MAKE,$(SUBDIRS)) ;
$(INSTALL_DIR)/opsdir/libexec/%: %
@echo "Installing $<"
-mkdir -p $(INSTALL_DIR)/opsdir/libexec
$(INSTALL) $< $@
.PHONY: $(SUBDIRS) install
......@@ -234,7 +234,7 @@ sub DoCreate()
};
$blob->{"fstype"} = $fstype
if (defined($fstype));
$blob->{"expires"} = $expires
$blob->{"expires"} = TBDateStringLocal($expires)
if (defined($expires));
my $dataset = APT_Dataset->Create($blob);
......@@ -337,7 +337,8 @@ sub DoDelete()
fatal("dataset is busy, cannot lock it");
}
my $response = $dataset->DeleteDataset();
if ($response->code() != GENIRESPONSE_SUCCESS) {
if ($response->code() != GENIRESPONSE_SUCCESS &&
$response->code() != GENIRESPONSE_SEARCHFAILED) {
$errmsg = "DeleteDataset failed: ". $response->output() . "\n";
goto failed;
}
......
......@@ -6772,11 +6772,20 @@ sub HandleBlockstore($$$$)
$message = "Missing blockstore mount point";
goto bad;
}
if (defined($leasename) &&
!TBcheck_dbslot($leasename, "project_leases", "lease_id",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
$message = "Illegal blockstore lease name for $bsname";
goto bad;;
if (defined($leasename)) {
if (GeniHRN::IsValid($leasename)) {
my (undef,$type,$id) = GeniHRN::Parse($leasename);
if ($type ne "dataset") {
$message = "Illegal persistent urn for $bsname";
goto bad;
}
$leasename = $id;
}
if (!TBcheck_dbslot($leasename, "project_leases", "lease_id",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
$message = "Illegal persistent name for $bsname";
goto bad;
}
}
if (!defined($readonly)) {
$readonly = 0;
......@@ -6818,10 +6827,9 @@ sub HandleBlockstore($$$$)
my $lease = Lease->Lookup($experiment->pid(), $leasename);
if (!defined($lease)) {
#
# Need to create the dataset. Eventually this needs to
# be its own API, but for now it needs to already exist.
# Dataset must already exist.
#
$message = "Unknown lease for $bsname: $leasename";
$message = "No such persistent dataset for $bsname: $leasename";
goto bad;
}
my $blockstore = Blockstore->LookupByLease($lease->lease_idx());
......
......@@ -2512,6 +2512,10 @@ sub CreateImage($)
if (exists($argref->{'global'})) {
$opt .= " -g " . ($argref->{'global'} ? "1" : "0");
}
else {
# Force shared (within project).
$opt .= " -r 1";
}
my $output =
GeniUtil::ExecQuiet("$CLONEIMAGE $opt -s $imagename $node_id");
# Not a typical op, so always print debugging info;
......@@ -3230,7 +3234,7 @@ sub CreateDataset($)
$credential->HasPrivilege("blockstores")));
# Get the project and group. This needs more thought.
my $group = GeniUtil::GetHoldingProject($user->urn(), $user);
my $group = GeniUtil::GetHoldingProject($credential->target_urn(), $user);
return $group
if (GeniResponse::IsResponse($group));
......@@ -3239,7 +3243,7 @@ sub CreateDataset($)
GeniResponse->Create(GENIRESPONSE_ALREADYEXISTS);
}
my $cmd = "$CREATEDATASET -b -s $size ";
my $cmd = "$CREATEDATASET -C -b -s $size ";
if (exists($argref->{'type'})) {
my $type = $argref->{'type'};
return GeniResponse->MalformedArgsResponse("Bad type")
......@@ -3262,9 +3266,9 @@ sub CreateDataset($)
if (!str2time($expiration)) {
return GeniResponse->MalformedArgsResponse("Bad expiration");
}
$cmd .= " -e '" . libtestbed::TBDateStringLocal($expiration) . "' ";
$cmd .= " -e '" . emutil::TBDateStringLocal($expiration) . "' ";
}
$cmd .= " " . $group->pid() . "/" . $dataset;
$cmd .= " " . $group->pid() . "/" . $group->gid() . "/" . $dataset;
my $output = GeniUtil::ExecQuiet($cmd);
# Not a typical op, so always print debugging info;
print STDERR $output;
......@@ -3275,7 +3279,7 @@ sub CreateDataset($)
# Grab the lease to see if its been approved, we want to tell
# the user something.
#
my $lease = Lease->Lookup($group->pid(), $dataset);
my $lease = Lease->Lookup($group->pid(), $group->gid(), $dataset);
if (!defined($lease)) {
print STDERR "Could not lookup lease after createdataset\n";
GeniResponse->Create(GENIRESPONSE_ERROR);
......@@ -3323,11 +3327,11 @@ sub DeleteDataset($)
$credential->HasPrivilege("blockstores")));
# Get the project and group. This needs more thought.
my $group = GeniUtil::GetHoldingProject($user->urn(), $user);
my $group = GeniUtil::GetHoldingProject($credential->target_urn(), $user);
return $group
if (GeniResponse::IsResponse($group));
my $lease = Lease->Lookup($group->pid(), $dataset);
my $lease = Lease->Lookup($group->pid(), $group->gid(), $dataset);
if (!defined($lease)) {
GeniResponse->Create(GENIRESPONSE_SEARCHFAILED);
}
......@@ -3337,7 +3341,8 @@ sub DeleteDataset($)
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN)
if (0 && $user->uid() ne $lease->owner());
my $cmd = "$DELETEDATASET -b -f " . $group->pid() . "/" . $dataset;
my $cmd = "$DELETEDATASET -b -f " .
$group->pid() . "/" . $group->gid() . "/" . $dataset;
my $output = GeniUtil::ExecQuiet($cmd);
# Not a typical op, so always print debugging info;
print STDERR $output;
......@@ -3360,7 +3365,6 @@ sub DescribeDataset($)
my $dataset = $argref->{'name'};
require Lease;
require Blockstore;
require libtestbed;
if (! (defined($credentials) && defined($dataset))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
......@@ -3382,11 +3386,11 @@ sub DescribeDataset($)
$credential->HasPrivilege("blockstores")));
# Get the project and group. This needs more thought.
my $group = GeniUtil::GetHoldingProject($user->urn(), $user);
my $group = GeniUtil::GetHoldingProject($credential->target_urn(), $user);
return $group
if (GeniResponse::IsResponse($group));
my $lease = Lease->Lookup($group->pid(), $dataset);
my $lease = Lease->Lookup($group->pid(), $group->gid(), $dataset);
if (!defined($lease)) {
GeniResponse->Create(GENIRESPONSE_SEARCHFAILED);
}
......@@ -3394,9 +3398,9 @@ sub DescribeDataset($)
$blob->{'state'} = $lease->state();
$blob->{'type'} = $lease->type();
$blob->{"busy"} = $lease->locked() ? 1 : 0;
$blob->{'created'} = libtestbed::TBDateStringGMT($lease->inception());
$blob->{'expires'} = libtestbed::TBDateStringGMT($lease->lease_end());
$blob->{'lastused'} = libtestbed::TBDateStringGMT($lease->last_used());
$blob->{'created'} = emutil::TBDateStringGMT($lease->inception());
$blob->{'expires'} = emutil::TBDateStringGMT($lease->lease_end());
$blob->{'lastused'} = emutil::TBDateStringGMT($lease->last_used());
my $bstore = Blockstore->LookupByLease($lease->idx());
......
......@@ -35,6 +35,7 @@ function Do_CreateDataSet()
global $this_user;
global $ajax_args;
global $DBFieldErrstr, $TBDIR, $APTBASE, $embedded;
global $suexec_output, $suexec_output_array;
$this_idx = $this_user->uid_idx();
$this_uid = $this_user->uid();
......@@ -247,6 +248,48 @@ function Do_ApproveDataset()
SPITAJAX_RESPONSE("$APTBASE/show-dataset.php?uuid=$lease_uuid");
}
function Do_RefreshDataset()
{
global $this_user;
global $ajax_args;
global $suexec_output, $suexec_output_array, $APTBASE;
$this_idx = $this_user->uid_idx();
$this_uid = $this_user->uid();
if (!isset($ajax_args["uuid"])) {
SPITAJAX_ERROR(1, "Missing uuid");
return;
}
$uuid = $ajax_args["uuid"];
$dataset = Dataset::Lookup($uuid);
if (!$dataset) {
SPITAJAX_ERROR(1, "Unknown dataset");
return;
}
if ($this_uid != $dataset->owner_uid() && !ISADMIN()) {
SPITAJAX_ERROR(1, "Not enough permission");
return;
}
#
# Invoke backend.
#
$retval = SUEXEC($this_uid, $dataset->pid(),
"webmanage_dataset refresh " .
$dataset->pid() . "/" . $dataset->id(),
SUEXEC_ACTION_CONTINUE);
if ($retval != 0) {
$error = "Transient error; please try again later";
if ($retval && count($suexec_output_array)) {
$error = $suexec_output_array[0];
}
SPITAJAX_ERROR(1, $error);
return;
}
SPITAJAX_RESPONSE(0);
}
# Local Variables:
# mode:php
# End:
......
......@@ -84,7 +84,7 @@ class Dataset
function field($name) {
return (is_null($this->dataset) ? -1 : $this->dataset[$name]);
}
function idx() { return $this->field("_idx"); }
function idx() { return $this->field("idx"); }
function dataset_id() { return $this->field("dataset_id"); }
function id() { return $this->field("dataset_id"); }
function creator_uid() { return $this->field("creator_uid"); }
......@@ -92,6 +92,8 @@ class Dataset
function uuid() { return $this->field("uuid"); }
function pid() { return $this->field("pid"); }
function pid_idx() { return $this->field("pid_idx"); }
function gid() { return $this->pid(); }
function aggregate_urn() { return $this->field("aggregate_urn"); }
function type() { return $this->field("type"); }
function fstype() { return $this->field("fstype"); }
function created() { return $this->field("created"); }
......@@ -101,6 +103,7 @@ class Dataset
function size() { return $this->field("size"); }
function locked() { return $this->field("locked"); }
function locker_pid() { return $this->field("locker_pid"); }
function islocal() { return 0; }
#
# This is incomplete.
......@@ -117,5 +120,16 @@ class Dataset
}
return 0;
}
#
# Form a URN for the dataset.
#
function URN() {
if (!preg_match("/^([^\+]+)\+([^\+]+)/",
$this->aggregate_urn(), $matches)) {
return "";
}
return $matches[1] . "+" . $matches[2] . "+dataset+" . $this->id();
}
}
?>
......@@ -149,6 +149,7 @@ function (_, sup, mainString)
return;
}
sup.SpitOops("oops", json.value);
return;
}
// Now do the actual create.
if (checkonly) {
......
......@@ -45,6 +45,13 @@ function (sup)
// pressing escape to cancel the search
$.tablesorter.filter.bindSearch( table, $('#dataset_search') );
}
// Format dates with moment before display.
$('.format-date').each(function() {
var date = $.trim($(this).html());
if (date != "") {
$(this).html(moment($(this).html()).format("lll"));
}
});
//
// When embedded, we want the Show link to go through the outer
......
require(window.APT_OPTIONS.configObject,
['underscore', 'js/quickvm_sup',
['underscore', 'js/quickvm_sup', 'moment',
'js/lib/text!template/show-dataset.html',
'jquery-ui'],
function (_, sup, mainString)
function (_, sup, moment, mainString)
{
'use strict';
var mainTemplate = _.template(mainString);
var dataset_uuid = null;
var embedded = 0;
var canrefresh = 0;
function initialize()
{
window.APT_OPTIONS.initialize(sup);
dataset_uuid = window.UUID;
embedded = window.EMBEDDED;
canrefresh = window.CANREFRESH;
var fields = JSON.parse(_.unescape($('#fields-json')[0].textContent));
......@@ -22,10 +24,18 @@ function (_, sup, mainString)
formfields: fields,
candelete: window.CANDELETE,
canapprove: window.CANAPPROVE,
canrefresh: window.CANREFRESH,
embedded: embedded,
title: window.TITLE,
});
$('#main-body').html(html);
// Format dates with moment before display.
$('.format-date').each(function() {
var date = $.trim($(this).html());
if (date != "") {
$(this).html(moment($(this).html()).format("lll"));
}
});
//
// When embedded, we want the links to go through the outer
......@@ -40,6 +50,11 @@ function (_, sup, mainString)
return false;
});
}
// Refresh.
$('#dataset_refresh_button').click(function (event) {
event.preventDefault();
RefreshDataset();
});
// Confirm Delete profile.
$('#delete-confirm').click(function (event) {
......@@ -75,6 +90,26 @@ function (_, sup, mainString)
xmlthing.done(callback);
}
//
// Refresh
//
function RefreshDataset()
{
var callback = function(json) {
sup.HideModal('#waitwait');
if (json.code) {
sup.SpitOops("oops", json.value);
return;
}
window.location.reload(true);
}
sup.ShowModal("#waitwait");
var xmlthing = sup.CallServerMethod(null,
"dataset",
"refresh",
{"uuid" : dataset_uuid});
xmlthing.done(callback);
}
//
// Approve dataset.
//
function ApproveDataset()
......
......@@ -165,7 +165,7 @@ while ($row = mysql_fetch_array($query_result)) {
$name = $row["dataset_id"];
$pid = $row["pid"];
$creator = $row["creator_uid"];
$created = $row["created"];
$created = DateStringGMT($row["created"]);
$state = $row["state"];
}
......@@ -182,7 +182,7 @@ while ($row = mysql_fetch_array($query_result)) {
echo "<td>$creator</td>";
}
echo " <td style='white-space:nowrap'>$pid</td>