Commit 4f84041b authored by Leigh B Stoller's avatar Leigh B Stoller

Checkpoint new dataset code before I rip apart the credential handling,

not happy with that part yet.
parent 28cd4e6f
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2015 University of Utah and the Flux Group.
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -25,6 +25,7 @@ package APT_Dataset;
use strict;
use English;
use Date::Parse;
use Data::Dumper;
use Carp;
use Exporter;
......@@ -54,7 +55,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $OURDOMAIN = "@OURDOMAIN@";
# Debugging
my $usemydevtree = 0;
my $usemydevtree = 1;
#
# Lookup by uuid.
......@@ -81,6 +82,7 @@ sub Lookup($$;$)
my $self = {};
$self->{'DATASET'} = $query_result->fetchrow_hashref();
$self->{'HASH'} = {};
bless($self, $class);
......@@ -118,6 +120,16 @@ AUTOLOAD {
if (exists($self->{'DATASET'}->{$name})) {
return $self->{'DATASET'}->{$name};
}
# Or it is for a local storage slot.
elsif ($name =~ /^_.*$/) {
if (scalar(@_) == 2) {
return $self->{'HASH'}->{$name} = $_[1];
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
return undef;
}
carp("No such slot '$name' field in class $type");
return undef;
}
......@@ -127,6 +139,7 @@ sub DESTROY {
my $self = shift;
$self->{'DATASET'} = undef;
$self->{'HASH'} = undef;
}
# Valid Blockstore backend.
......@@ -293,6 +306,22 @@ sub SetStatus($$)
}
#
# Is dataset expired?
#
sub IsExpired($)
{
my ($self) = @_;
my $expires = $self->expires();
return 0
if (!defined($expires));
$expires = str2time($expires);
return (time() >= $expires);
}
#
# Load the project object for an experiment.
#
......@@ -421,8 +450,16 @@ sub CreateCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $gid = $self->gid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
my $type = $self->type();
# New domain format for datasets.
my $domain = $OURDOMAIN;
$domain .= ":${pid}";
$domain .= ":${gid}" if ($pid ne $gid);
my $urn = GeniHRN::Generate($domain, $type, $id);
# Kill stale certificate.
my $certificate = GeniCertificate->Lookup($urn);
......@@ -430,6 +467,7 @@ sub CreateCertificate($)
if (defined($certificate));
$certificate = GeniCertificate->Create({"urn" => $urn,
"uuid" => $self->uuid(),
"email" => "$TBOPS",
"hrn" => "$OURDOMAIN.$pid.$id"});
return -1
......@@ -444,11 +482,15 @@ sub CreateCertificate($)
sub GetCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
return GeniCertificate->Lookup($urn);
my $cert = GeniCertificate->Lookup($self->uuid());
# Old style
if (!defined($cert)) {
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
$cert = GeniCertificate->Lookup($urn);
}
return $cert;
}
#
......@@ -473,10 +515,10 @@ sub CreateDataset($)
my $args = {
"size" => $self->size(),
"name" => $self->dataset_id(),
"type" => $self->type(),
"read_access" => $self->read_access(),
"write_access"=> $self->write_access(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
......@@ -512,7 +554,7 @@ sub DeleteDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
......@@ -543,7 +585,7 @@ sub ModifyDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"read_access" => $self->read_access(),
"write_access"=> $self->write_access(),
"credentials" => [$credential->asString(),
......@@ -576,7 +618,7 @@ sub ExtendDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
"extend" => 1,
......@@ -602,15 +644,17 @@ sub DescribeDataset($)
defined($context) && defined($cert)));
my ($credential, $speaksfor_credential) =
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"]);
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"], 1);
return undef
if (! (defined($speaksfor_credential) &&
defined($credential)));
if (!defined($credential));
my $credentials = [$credential->asString()];
if (defined($speaksfor_credential)) {
$credentials = [@$credentials, $speaksfor_credential->asString()];
}
my $args = {
"name" => $self->dataset_id(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
"dataset_urn" => $cert->urn(),
"credentials" => $credentials,
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
......@@ -618,5 +662,35 @@ sub DescribeDataset($)
return Genixmlrpc::CallMethod($cmurl, $context, "DescribeDataset", $args);
}
#
# Get a new credential so we can operate on the dataset.
#
sub GetCredential($)
{
my ($self) = @_;
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $context = APT_Geni::GeniContext();
my $cert = $self->GetCertificate();
return undef
if (! (defined($geniuser) && defined($authority) &&
defined($context) && defined($cert)));
my ($credential) =
APT_Geni::GenAuthCredential($cert, ["blockstores"]);
return undef
if (!defined($credential));
my $args = {
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString()],
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
return Genixmlrpc::CallMethod($cmurl, $context,
"GetDatasetCredential", $args);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -290,6 +290,8 @@ sub Create($$$$$$)
my $name = DBQuoteSpecial($argref->{'name'});
my $pid = $project->pid();
my $pid_idx = $project->pid_idx();
my $gid = $project->pid();
my $gid_idx = $project->pid_idx();
my $uid = $creator->uid();
my $uid_idx = $creator->uid_idx();
......@@ -911,12 +913,11 @@ sub CheckNodeConstraints($$$)
}
#
# Check blockstores.
# Check dataset validity.
#
sub CheckDatasets($$$)
sub CheckDatasets($$)
{
my ($xml, $ppid, $pmsg) = @_;
my $pid = $ppid;
my ($xml, $pmsg) = @_;
my $rspec = GeniXML::Parse($xml);
if (! defined($rspec)) {
......@@ -963,63 +964,60 @@ sub CheckDatasets($$$)
$$pmsg = "Persistent dataset is not a valid URN";
return 1;
}
my $dataset_urn = $dataset_id;
my ($dataset_authority, $type, $id) = GeniHRN::Parse($dataset_urn);
my ($dataset_domain) = split(":", $dataset_authority);
my $dataset_urn = GeniHRN->new($dataset_id);
#
# Separate project from name; this is how the rspec specifies
# the dataset they want, since it might be in another project
# For leases, the domain of the dataset has to match the
# domain of aggregate, but for image backed datasets, we now
# to transfer them as needed.
#
if ($id =~ /^([-\w]+)\/\/(.+)$/) {
$pid = $1;
$id = $2;
}
#
# The domain of the dataset has to match the domain of aggregate.
# We also use this when creating a profile, so rspec might not
# be bound.
#
if (defined($manager_urn)) {
my ($manager_authority) = GeniHRN::Parse($manager_urn);
my ($manager_domain) = split(":", $manager_authority);
if ($class ne "local") {
if (defined($manager_urn)) {
$manager_urn = GeniHRN->new($manager_urn);
if ($manager_domain ne $dataset_domain) {
$$pmsg = "Dataset $id is not located on $manager_authority";
if ($manager_urn->domain() ne $dataset_urn->domain()) {
$$pmsg = "$dataset_urn is not located on $manager_urn";
return 1;
}
}
#
# Not all backends have blockstore (lease) support.
#
if (!APT_Dataset::ValidBlockstoreBackend($dataset_urn)) {
$$pmsg = "Dataset $dataset_urn is not on a valid aggregate";
return 1;
}
}
next
if ($class eq "local");
#
# Not all backends have blockstore support.
# So the rspec will refer to the dataset by the remote URN.
# And it might be a classic dataset (so not in the apt_datasets
# table). We want to be able to handle either of these cases.
#
if (!APT_Dataset::ValidBlockstoreBackend($dataset_urn)) {
$$pmsg = "Dataset $id is not on a valid aggregate";
return 1;
}
my $dataset = APT_Dataset->Lookup("$pid/$id");
my $dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
$dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
# If it is for a local dataset, see if it plain lease
# created via the classic interface or command line.
# The backend can find those.
if ($dataset_domain eq $OURDOMAIN) {
$dataset = Lease->Lookup($pid, $id);
#
# Local image backed dataset or lease.
#
if ($dataset_urn->domain() eq $OURDOMAIN) {
my $pid = $dataset_urn->project();
my $id = $dataset_urn->id();
if ($dataset_urn->type() eq "imdataset") {
$dataset = Image->Lookup($pid, $id);
if ($dataset && !$dataset->isdataset()) {
$$pmsg = "$dataset_urn is an image not a dataset ";
return 1;
}
}
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$pid/$id' does not exist";
return 1;
else {
$dataset = Lease->Lookup($pid, $id);
}
}
}
#
# XXX Need basic frontend permission checks?
#
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$dataset_urn' does not exist";
return 1;
}
}
}
return 0;
......
......@@ -344,15 +344,11 @@ $usestitcher = 1 if ($needstitcher);
#
# Look for datasets; need to verify that the datasets being referenced
# still exist and are still permissible to use, and we have to generate
# credentials for those datasets (if not a global dataset). The tricky
# aspect is that while a dataset and a profile have project permissions,
# the experiment has no project association, so if the profile/dataset
# perms are okay, then we send over a credential that tells the CM to
# allow this experiment to use that dataset in that project.
# actually exists, in so far as we can check. We check permissions
# below when we generate the credentials.
#
$errmsg = "Bad dataset";
if (APT_Profile::CheckDatasets($rspecstr, $profile->pid(), \$errmsg)) {
if (APT_Profile::CheckDatasets($rspecstr, \$errmsg)) {
UserError($errmsg);
}
......@@ -617,8 +613,7 @@ if (!$debug) {
# can access the datasets.
my @dataset_credentials = ();
if (defined($profile)) {
my $retval = CreateDatasetCreds($rspecstr,
$profile->pid(), $geniuser,
my $retval = CreateDatasetCreds($rspecstr, $project, $geniuser,
\$errmsg, \@dataset_credentials);
if ($retval) {
($retval < 0 ? fatal($errmsg) : UserError($errmsg));
......@@ -1020,7 +1015,7 @@ exit(0);
#
sub CreateDatasetCreds($$$$$)
{
my ($xml, $pid, $user, $pmsg, $pref) = @_;
my ($xml, $project, $geniuser, $pmsg, $pref) = @_;
my @credentials = ();
my $rspec = GeniXML::Parse($xml);
......@@ -1029,8 +1024,9 @@ sub CreateDatasetCreds($$$$$)
return -1;
}
foreach my $ref (GeniXML::FindNodes("n:node", $rspec)->get_nodelist()) {
foreach my $blockref (GeniXML::FindNodesNS("n:blockstore",
$ref,
my $manager_urn = GetManagerId($ref);
foreach my $blockref (GeniXML::FindNodesNS("n:blockstore", $ref,
$GeniXML::EMULAB_NS)->get_nodelist()) {
my $dataset_id = GeniXML::GetText("persistent", $blockref);
if (!defined($dataset_id)) {
......@@ -1049,54 +1045,78 @@ sub CreateDatasetCreds($$$$$)
if (!defined($class)) {
$class = "remote";
}
# Image backed. No checking since the image has to be global
# anyway. Needs more thought.
# Image backed referenced by URL. No checking since the
# image has to be global anyway. Needs more thought.
next
if ($class eq "local");
my ($authority, $type, $id) = GeniHRN::Parse($dataset_id);
my ($dataset_domain) = split(":", $authority);
#
# Separate project from name; this is how the rspec specifies
# the dataset they want, since it might be in another project
#
if ($id =~ /^([-\w]+)\/\/(.+)$/) {
$pid = $1;
$id = $2;
}
my $dataset = APT_Dataset->Lookup("$pid/$id");
if ($class eq "local" && $dataset_id =~ /^(http|https):/);
my $dataset_urn = GeniHRN->new($dataset_id);
my $dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
$dataset = APT_Dataset->LookupByRemoteURN($dataset_id);
if (!defined($dataset)) {
# If it is for a local dataset, see if it plain lease
# created via the classic interface or command line.
# The backend can find those.
if ($dataset_domain eq $OURDOMAIN) {
$dataset = Lease->Lookup($pid, $id);
if (defined($dataset)) {
# No need for a credential.
next;
if ($dataset_urn->domain() eq $OURDOMAIN) {
#
# Local image backed dataset or lease.
#
my ($image,$lease);
my $pid = $dataset_urn->project();
my $id = $dataset_urn->id();
if ($dataset_urn->type() eq "imdataset") {
$image = Image->Lookup($pid, $id);
if ($image && !$image->isdataset()) {
$$pmsg = "$dataset_urn is an image not a dataset ";
return 1;
}
#
# Do a partial permission check here to catch
# errors early. The CM will do its own check
# of course.
#
if (!$image->global() &&
$PROTOGENI_LOCALUSER && $geniuser->IsLocal() &&
!$image->AccessCheck($geniuser->emulab_user(),
TB_IMAGEID_ACCESS())) {
$$pmsg = "No permission to use $dataset_urn";
return 1;
}
}
else {
$lease = Lease->Lookup($pid, $id);
}
# We cannot generate a credential for "legacy" datasets.
# So if it is not global, it cannot be transferred. Maybe
# this is okay, we will find out. We could generate a
# credential if we needed to.
next
if ($image || $lease);
}
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$pid/$id' does not exist";
return 1;
}
$$pmsg = "Dataset '$dataset_urn' does not exist";
return 1;
}
my $certificate = $dataset->GetCertificate();
if (!defined($certificate)) {
$$pmsg = "No certificate for dataset '$pid/$id'";
return -1;
}
my $credential =
APT_Geni::GenCredentials($certificate, $geniuser,
["blockstores"]);
if (!defined($credential)) {
$$pmsg = "Could not create credential for dataset '$pid/$id'";
#
# We do not need a credential for leases, only real users
# can use those, and so standard emulab permission checks
# are applied at the CM.
#
next
if ($dataset->type() ne "imdataset");
#
# For image backed datasets, we need to send along a credential
# that allows the remote CM to securely download the dataset if
# it does not already have it. To do that we need to send it a
# credential from the CM where the dataset lives. We do that by
# requesting a credential, and delegating it to the target CM.
#
my $pid = $dataset->pid();
my $id = $dataset->dataset_id();
my $cmd = "$MANAGEDATASET getcredential -a $manager_urn $pid/$id";
my $credential = `$cmd`;
if ($?) {
$$pmsg = "Could not generate credential for $dataset_urn";
return -1;
}
push(@credentials, $credential->asString());
push(@credentials, $credential);
}
}
@$pref = @credentials;
......
......@@ -40,6 +40,7 @@ sub usage()
print STDERR "Usage: manage_dataset [options --] modify ...\n";
print STDERR "Usage: manage_dataset [options --] extend ...\n";
print STDERR "Usage: manage_dataset [options --] snapshot ...\n";
print STDERR "Usage: manage_dataset [options --] getcredential ...\n";
exit(-1);
}
my $optlist = "dt:";
......@@ -54,6 +55,7 @@ my $geniuser;
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $SACERT = "$TB/etc/genisa.pem";
#
# Untaint the path
......@@ -74,6 +76,7 @@ use libtestbed;
use EmulabConstants;
use emdb;
use emutil;
use libEmulab;
use User;
use Project;
use APT_Dataset;
......@@ -83,19 +86,25 @@ use Blockstore;
use GeniResponse;
use GeniXML;
use GeniUser;
use GeniAuthority;
use GeniCertificate;
use GeniCredential;
use GeniImage;
# Protos
sub fatal($);
sub uerror($);
sub uerror($;$);
sub DoCreate();
sub DoDelete();
sub DoRefresh();
sub DoRefreshInternal($$);
sub DoGetCredential();
sub DoModify();
sub DoExtend();
sub DoSnapshot();
sub DoSnapShotInternal($$$$$);
sub PollDatasetStatus($$);
sub PollDatasetStatus($$$);
sub DoImageTrackerStuff($$$$);
#
# Parse command arguments. Once we return from getopts, all that should be
......@@ -152,6 +161,9 @@ elsif ($action eq "extend") {
elsif ($action eq "snapshot") {
exit(DoSnapshot());
}
elsif ($action eq "getcredential") {
exit(DoGetCredential());
}
else {
usage();
}
......@@ -301,6 +313,8 @@ sub DoCreate()
"dataset_id" => $name,
"pid" => $project->pid(),
"pid_idx" => $project->pid_idx,
"gid" => $project->pid(),
"gid_idx" => $project->pid_idx,
"creator_uid" => $this_user->uid(),
"creator_idx" => $this_user->uid_idx(),
"aggregate_urn" => $aggregate_urn,
......@@ -330,16 +344,15 @@ sub DoCreate()
}
}
#
# For IM datasets always create a webtask for tracking imaging status.
# Always create a webtask for tracking image or allocation status.
#
if ($type eq "imdataset") {
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$webtask->AutoStore(1);
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$webtask->AutoStore(1);
#
# Ask the aggregate to create the dataset.
#
......@@ -377,8 +390,7 @@ sub DoCreate()
# Handoff to snapshot if an imdataset.
#
if ($type eq "imdataset" &&
DoSnapShotInternal($dataset, $aggregate,
$bsname, $nodeid, \$errmsg)) {
DoSnapShotInternal($dataset, $aggregate, $bsname, $nodeid, \$errmsg)) {
$response = $dataset->DeleteDataset();
if ($response->code() == GENIRESPONSE_SUCCESS ||
$response->code() == GENIRESPONSE_SEARCHFAILED) {
......@@ -391,7 +403,7 @@ sub DoCreate()
# This will set the webtask, see below.
fatal($errmsg);
}
if (PollDatasetStatus($dataset, \$errmsg)) {
if (PollDatasetStatus($dataset, $aggregate, \$errmsg)) {
# Exit and let child poll
exit(0);
}
......@@ -466,16 +478,16 @@ sub DoRefresh()
if ($dataset->Lock()) {
uerror("dataset is busy, cannot lock it");
}
if (DoRefreshInternal($dataset, \$errmsg)) {
goto failed;
}
my $errcode = DoRefreshInternal($dataset, \$errmsg);
goto failed
if ($errcode);
$dataset->Unlock();
return 0;
failed:
$dataset->Unlock();
# This will set the webtask, see below.