Commit 4f84041b authored by Leigh Stoller's avatar Leigh Stoller

Checkpoint new dataset code before I rip apart the credential handling,

not happy with that part yet.
parent 28cd4e6f
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2015 University of Utah and the Flux Group.
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -25,6 +25,7 @@ package APT_Dataset;
use strict;
use English;
use Date::Parse;
use Data::Dumper;
use Carp;
use Exporter;
......@@ -54,7 +55,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $OURDOMAIN = "@OURDOMAIN@";
# Debugging
my $usemydevtree = 0;
my $usemydevtree = 1;
#
# Lookup by uuid.
......@@ -81,6 +82,7 @@ sub Lookup($$;$)
my $self = {};
$self->{'DATASET'} = $query_result->fetchrow_hashref();
$self->{'HASH'} = {};
bless($self, $class);
......@@ -118,6 +120,16 @@ AUTOLOAD {
if (exists($self->{'DATASET'}->{$name})) {
return $self->{'DATASET'}->{$name};
}
# Or it is for a local storage slot.
elsif ($name =~ /^_.*$/) {
if (scalar(@_) == 2) {
return $self->{'HASH'}->{$name} = $_[1];
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
return undef;
}
carp("No such slot '$name' field in class $type");
return undef;
}
......@@ -127,6 +139,7 @@ sub DESTROY {
my $self = shift;
$self->{'DATASET'} = undef;
$self->{'HASH'} = undef;
}
# Valid Blockstore backend.
......@@ -293,6 +306,22 @@ sub SetStatus($$)
}
#
# Is dataset expired?
#
sub IsExpired($)
{
my ($self) = @_;
my $expires = $self->expires();
return 0
if (!defined($expires));
$expires = str2time($expires);
return (time() >= $expires);
}
#
# Load the project object for an experiment.
#
......@@ -421,8 +450,16 @@ sub CreateCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $gid = $self->gid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
my $type = $self->type();
# New domain format for datasets.
my $domain = $OURDOMAIN;
$domain .= ":${pid}";
$domain .= ":${gid}" if ($pid ne $gid);
my $urn = GeniHRN::Generate($domain, $type, $id);
# Kill stale certificate.
my $certificate = GeniCertificate->Lookup($urn);
......@@ -430,6 +467,7 @@ sub CreateCertificate($)
if (defined($certificate));
$certificate = GeniCertificate->Create({"urn" => $urn,
"uuid" => $self->uuid(),
"email" => "$TBOPS",
"hrn" => "$OURDOMAIN.$pid.$id"});
return -1
......@@ -444,11 +482,15 @@ sub CreateCertificate($)
sub GetCertificate($)
{
my ($self) = @_;
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
return GeniCertificate->Lookup($urn);
my $cert = GeniCertificate->Lookup($self->uuid());
# Old style
if (!defined($cert)) {
my $pid = $self->pid();
my $id = $self->dataset_id();
my $urn = GeniHRN::Generate("$OURDOMAIN:$pid", "dataset", $id);
$cert = GeniCertificate->Lookup($urn);
}
return $cert;
}
#
......@@ -473,10 +515,10 @@ sub CreateDataset($)
my $args = {
"size" => $self->size(),
"name" => $self->dataset_id(),
"type" => $self->type(),
"read_access" => $self->read_access(),
"write_access"=> $self->write_access(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
......@@ -512,7 +554,7 @@ sub DeleteDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
};
......@@ -543,7 +585,7 @@ sub ModifyDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"read_access" => $self->read_access(),
"write_access"=> $self->write_access(),
"credentials" => [$credential->asString(),
......@@ -576,7 +618,7 @@ sub ExtendDataset($)
defined($credential)));
my $args = {
"name" => $self->dataset_id(),
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
"extend" => 1,
......@@ -602,15 +644,17 @@ sub DescribeDataset($)
defined($context) && defined($cert)));
my ($credential, $speaksfor_credential) =
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"]);
APT_Geni::GenCredentials($cert, $geniuser, ["blockstores"], 1);
return undef
if (! (defined($speaksfor_credential) &&
defined($credential)));
if (!defined($credential));
my $credentials = [$credential->asString()];
if (defined($speaksfor_credential)) {
$credentials = [@$credentials, $speaksfor_credential->asString()];
}
my $args = {
"name" => $self->dataset_id(),
"credentials" => [$credential->asString(),
$speaksfor_credential->asString()],
"dataset_urn" => $cert->urn(),
"credentials" => $credentials,
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
......@@ -618,5 +662,35 @@ sub DescribeDataset($)
return Genixmlrpc::CallMethod($cmurl, $context, "DescribeDataset", $args);
}
#
# Get a new credential so we can operate on the dataset.
#
sub GetCredential($)
{
my ($self) = @_;
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $context = APT_Geni::GeniContext();
my $cert = $self->GetCertificate();
return undef
if (! (defined($geniuser) && defined($authority) &&
defined($context) && defined($cert)));
my ($credential) =
APT_Geni::GenAuthCredential($cert, ["blockstores"]);
return undef
if (!defined($credential));
my $args = {
"dataset_urn" => $cert->urn(),
"credentials" => [$credential->asString()],
};
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
return Genixmlrpc::CallMethod($cmurl, $context,
"GetDatasetCredential", $args);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -290,6 +290,8 @@ sub Create($$$$$$)
my $name = DBQuoteSpecial($argref->{'name'});
my $pid = $project->pid();
my $pid_idx = $project->pid_idx();
my $gid = $project->pid();
my $gid_idx = $project->pid_idx();
my $uid = $creator->uid();
my $uid_idx = $creator->uid_idx();
......@@ -911,12 +913,11 @@ sub CheckNodeConstraints($$$)
}
#
# Check blockstores.
# Check dataset validity.
#
sub CheckDatasets($$$)
sub CheckDatasets($$)
{
my ($xml, $ppid, $pmsg) = @_;
my $pid = $ppid;
my ($xml, $pmsg) = @_;
my $rspec = GeniXML::Parse($xml);
if (! defined($rspec)) {
......@@ -963,63 +964,60 @@ sub CheckDatasets($$$)
$$pmsg = "Persistent dataset is not a valid URN";
return 1;
}
my $dataset_urn = $dataset_id;
my ($dataset_authority, $type, $id) = GeniHRN::Parse($dataset_urn);
my ($dataset_domain) = split(":", $dataset_authority);
my $dataset_urn = GeniHRN->new($dataset_id);
#
# Separate project from name; this is how the rspec specifies
# the dataset they want, since it might be in another project
# For leases, the domain of the dataset has to match the
# domain of aggregate, but for image backed datasets, we now
# to transfer them as needed.
#
if ($id =~ /^([-\w]+)\/\/(.+)$/) {
$pid = $1;
$id = $2;
}
#
# The domain of the dataset has to match the domain of aggregate.
# We also use this when creating a profile, so rspec might not
# be bound.
#
if (defined($manager_urn)) {
my ($manager_authority) = GeniHRN::Parse($manager_urn);
my ($manager_domain) = split(":", $manager_authority);
if ($class ne "local") {
if (defined($manager_urn)) {
$manager_urn = GeniHRN->new($manager_urn);
if ($manager_domain ne $dataset_domain) {
$$pmsg = "Dataset $id is not located on $manager_authority";
if ($manager_urn->domain() ne $dataset_urn->domain()) {
$$pmsg = "$dataset_urn is not located on $manager_urn";
return 1;
}
}
#
# Not all backends have blockstore (lease) support.
#
if (!APT_Dataset::ValidBlockstoreBackend($dataset_urn)) {
$$pmsg = "Dataset $dataset_urn is not on a valid aggregate";
return 1;
}
}
next
if ($class eq "local");
#
# Not all backends have blockstore support.
# So the rspec will refer to the dataset by the remote URN.
# And it might be a classic dataset (so not in the apt_datasets
# table). We want to be able to handle either of these cases.
#
if (!APT_Dataset::ValidBlockstoreBackend($dataset_urn)) {
$$pmsg = "Dataset $id is not on a valid aggregate";
return 1;
}
my $dataset = APT_Dataset->Lookup("$pid/$id");
my $dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
$dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
# If it is for a local dataset, see if it plain lease
# created via the classic interface or command line.
# The backend can find those.
if ($dataset_domain eq $OURDOMAIN) {
$dataset = Lease->Lookup($pid, $id);
#
# Local image backed dataset or lease.
#
if ($dataset_urn->domain() eq $OURDOMAIN) {
my $pid = $dataset_urn->project();
my $id = $dataset_urn->id();
if ($dataset_urn->type() eq "imdataset") {
$dataset = Image->Lookup($pid, $id);
if ($dataset && !$dataset->isdataset()) {
$$pmsg = "$dataset_urn is an image not a dataset ";
return 1;
}
}
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$pid/$id' does not exist";
return 1;
else {
$dataset = Lease->Lookup($pid, $id);
}
}
}
#
# XXX Need basic frontend permission checks?
#
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$dataset_urn' does not exist";
return 1;
}
}
}
return 0;
......
......@@ -344,15 +344,11 @@ $usestitcher = 1 if ($needstitcher);
#
# Look for datasets; need to verify that the datasets being referenced
# still exist and are still permissible to use, and we have to generate
# credentials for those datasets (if not a global dataset). The tricky
# aspect is that while a dataset and a profile have project permissions,
# the experiment has no project association, so if the profile/dataset
# perms are okay, then we send over a credential that tells the CM to
# allow this experiment to use that dataset in that project.
# actually exists, in so far as we can check. We check permissions
# below when we generate the credentials.
#
$errmsg = "Bad dataset";
if (APT_Profile::CheckDatasets($rspecstr, $profile->pid(), \$errmsg)) {
if (APT_Profile::CheckDatasets($rspecstr, \$errmsg)) {
UserError($errmsg);
}
......@@ -617,8 +613,7 @@ if (!$debug) {
# can access the datasets.
my @dataset_credentials = ();
if (defined($profile)) {
my $retval = CreateDatasetCreds($rspecstr,
$profile->pid(), $geniuser,
my $retval = CreateDatasetCreds($rspecstr, $project, $geniuser,
\$errmsg, \@dataset_credentials);
if ($retval) {
($retval < 0 ? fatal($errmsg) : UserError($errmsg));
......@@ -1020,7 +1015,7 @@ exit(0);
#
sub CreateDatasetCreds($$$$$)
{
my ($xml, $pid, $user, $pmsg, $pref) = @_;
my ($xml, $project, $geniuser, $pmsg, $pref) = @_;
my @credentials = ();
my $rspec = GeniXML::Parse($xml);
......@@ -1029,8 +1024,9 @@ sub CreateDatasetCreds($$$$$)
return -1;
}
foreach my $ref (GeniXML::FindNodes("n:node", $rspec)->get_nodelist()) {
foreach my $blockref (GeniXML::FindNodesNS("n:blockstore",
$ref,
my $manager_urn = GetManagerId($ref);
foreach my $blockref (GeniXML::FindNodesNS("n:blockstore", $ref,
$GeniXML::EMULAB_NS)->get_nodelist()) {
my $dataset_id = GeniXML::GetText("persistent", $blockref);
if (!defined($dataset_id)) {
......@@ -1049,54 +1045,78 @@ sub CreateDatasetCreds($$$$$)
if (!defined($class)) {
$class = "remote";
}
# Image backed. No checking since the image has to be global
# anyway. Needs more thought.
# Image backed referenced by URL. No checking since the
# image has to be global anyway. Needs more thought.
next
if ($class eq "local");
my ($authority, $type, $id) = GeniHRN::Parse($dataset_id);
my ($dataset_domain) = split(":", $authority);
#
# Separate project from name; this is how the rspec specifies
# the dataset they want, since it might be in another project
#
if ($id =~ /^([-\w]+)\/\/(.+)$/) {
$pid = $1;
$id = $2;
}
my $dataset = APT_Dataset->Lookup("$pid/$id");
if ($class eq "local" && $dataset_id =~ /^(http|https):/);
my $dataset_urn = GeniHRN->new($dataset_id);
my $dataset = APT_Dataset->LookupByRemoteURN($dataset_urn);
if (!defined($dataset)) {
$dataset = APT_Dataset->LookupByRemoteURN($dataset_id);
if (!defined($dataset)) {
# If it is for a local dataset, see if it plain lease
# created via the classic interface or command line.
# The backend can find those.
if ($dataset_domain eq $OURDOMAIN) {
$dataset = Lease->Lookup($pid, $id);
if (defined($dataset)) {
# No need for a credential.
next;
if ($dataset_urn->domain() eq $OURDOMAIN) {
#
# Local image backed dataset or lease.
#
my ($image,$lease);
my $pid = $dataset_urn->project();
my $id = $dataset_urn->id();
if ($dataset_urn->type() eq "imdataset") {
$image = Image->Lookup($pid, $id);
if ($image && !$image->isdataset()) {
$$pmsg = "$dataset_urn is an image not a dataset ";
return 1;
}
#
# Do a partial permission check here to catch
# errors early. The CM will do its own check
# of course.
#
if (!$image->global() &&
$PROTOGENI_LOCALUSER && $geniuser->IsLocal() &&
!$image->AccessCheck($geniuser->emulab_user(),
TB_IMAGEID_ACCESS())) {
$$pmsg = "No permission to use $dataset_urn";
return 1;
}
}
else {
$lease = Lease->Lookup($pid, $id);
}
# We cannot generate a credential for "legacy" datasets.
# So if it is not global, it cannot be transferred. Maybe
# this is okay, we will find out. We could generate a
# credential if we needed to.
next
if ($image || $lease);
}
if (!defined($dataset)) {
$$pmsg = "Persistent dataset '$pid/$id' does not exist";
return 1;
}
$$pmsg = "Dataset '$dataset_urn' does not exist";
return 1;
}
my $certificate = $dataset->GetCertificate();
if (!defined($certificate)) {
$$pmsg = "No certificate for dataset '$pid/$id'";
return -1;
}
my $credential =
APT_Geni::GenCredentials($certificate, $geniuser,
["blockstores"]);
if (!defined($credential)) {
$$pmsg = "Could not create credential for dataset '$pid/$id'";
#
# We do not need a credential for leases, only real users
# can use those, and so standard emulab permission checks
# are applied at the CM.
#
next
if ($dataset->type() ne "imdataset");
#
# For image backed datasets, we need to send along a credential
# that allows the remote CM to securely download the dataset if
# it does not already have it. To do that we need to send it a
# credential from the CM where the dataset lives. We do that by
# requesting a credential, and delegating it to the target CM.
#
my $pid = $dataset->pid();
my $id = $dataset->dataset_id();
my $cmd = "$MANAGEDATASET getcredential -a $manager_urn $pid/$id";
my $credential = `$cmd`;
if ($?) {
$$pmsg = "Could not generate credential for $dataset_urn";
return -1;
}
push(@credentials, $credential->asString());
push(@credentials, $credential);
}
}
@$pref = @credentials;
......
This diff is collapsed.
......@@ -326,7 +326,7 @@ elsif (!$project->AccessCheck($this_user, TB_PROJECT_MAKEIMAGEID())) {
# Check datasets.
if (defined($rspec)) {
my $errmsg = "Bad dataset";
if (APT_Profile::CheckDatasets($rspec, $project->pid(), \$errmsg)) {
if (APT_Profile::CheckDatasets($rspec, \$errmsg)) {
$errors{"error"} = $errmsg;
UserError();
}
......
......@@ -75,11 +75,23 @@ sub urn($)
my ($self) = @_;
return undef
if (! $PGENISUPPORT);
if (! $PGENISUPPORT);
require GeniHRN;
return GeniHRN::Generate($OURDOMAIN, "image",
$self->pid() . "//" . $self->imagename());
my $pid = $self->pid();
my $gid = $self->gid();
my $name = $self->imagename();
my $domain = $OURDOMAIN;
if ($self->isdataset()) {
$domain .= ":${pid}";
$domain .= ":${gid}" if ($pid ne $gid);
return GeniHRN::Generate($domain, "imdataset", $name);
}
else {
return GeniHRN::Generate($domain, "image", $pid . "//" . $name);
}
}
# Little helper and debug function.
......@@ -411,7 +423,7 @@ sub LookupByOriginUUID($$$)
my $query_result =
DBQueryWarn("select imageid from image_versions ".
"where pid=$safe_pid ".
"where pid=$safe_pid and ".
" origin_uuid=$safe_uuid and deleted is null");
return undef
if (!$query_result || !$query_result->numrows);
......@@ -420,6 +432,32 @@ sub LookupByOriginUUID($$$)
return Image->Lookup($imageid);
}
#
# Lookup by Origin URN.
#
sub LookupByOriginURN($$)
{
my ($class, $urn) = @_;
return undef
if (! $PGENISUPPORT);
require GeniHRN;
return undef
if (!GeniHRN::IsValid($urn));
my $safe_urn = DBQuoteSpecial($urn);
my $query_result =
DBQueryWarn("select imageid,version from image_versions ".
"where origin_urn=$safe_urn and deleted is null");
return undef
if (!$query_result || !$query_result->numrows);
my ($imageid,$version) = $query_result->fetchrow_array();
return Image->Lookup($imageid, $version);
}
#
# Get a list of all running frisbee images.
# XXX if this is actually used, it will have to be fixed; DB no longer
......@@ -899,7 +937,7 @@ sub NewVersion($$$$)
" parent_imageid=$parent_imageid,".
" parent_version=$parent_version, ".
" origin_neednotify=0,origin_needupdate=0, ".
" origin_uuid=NULL,origin_name=NULL, ".
" origin_uuid=NULL,origin_name=NULL,credential_string=NULL, ".
" hash=null,deltahash=null,size=0,deltasize=0, ".
" updater='$uid',updater_idx='$uid_idx' $updater_urn ".
"where imageid='$osid'")
......@@ -2438,6 +2476,35 @@ sub LocalVersionURL($)
return "$TBBASE/image_metadata.php?uuid=$uuid";
}
sub HasCredential($)
{
my ($self) = @_;
return undef
if (! $PGENISUPPORT);
return $self->credential_string() && $self->credential_string() ne "";
}
sub GetCredential($)
{
my ($self) = @_;
return undef
if (! ($PGENISUPPORT && $self->HasCredential()));
require GeniCredential;
return GeniCredential->CreateFromSigned($self->credential_string());
}
sub SetCredential($$)
{
my ($self, $credstr) = @_;
return -1
if (! $PGENISUPPORT);
return $self->Update({"credential_string" => $credstr});
}
#
# Path and Directory stuff.
#
......
......@@ -34,6 +34,8 @@ use vars qw(@ISA @EXPORT);
use vars qw($TB $BSCONTROL);
$TB = "@prefix@";
$BSCONTROL = "$TB/sbin/bscontrol";
my $PGENISUPPORT = @PROTOGENI_SUPPORT@;
my $OURDOMAIN = "@OURDOMAIN@";
use libdb;
use libtestbed;
......@@ -204,6 +206,8 @@ sub lease_end($) {return str2time($_[0]->{'DBROW'}->{'lease_end'}); }
sub expiration($) {return $_[0]->lease_end(); }
sub last_used($) {return str2time($_[0]->{'DBROW'}->{'last_used'}); }
sub last_checked($) {return str2time($_[0]->{'DBROW'}->{'last_checked'}); }
sub origin_urn($) {return $_[0]->{'DBROW'}->{'origin_urn'}; }
sub origin_uuid($) {return $_[0]->{'DBROW'}->{'origin_uuid'}; }
sub state($) {return $_[0]->{'DBROW'}->{'state'}; }
sub statestamp($) {return str2time($_[0]->{'DBROW'}->{'statestamp'}); }
sub renewals($) {return $_[0]->{'DBROW'}->{'renewals'}; }
......@@ -462,6 +466,24 @@ sub Delete($) {
return 0
}
sub urn($)
{
my ($self) = @_;
return undef
if (! $PGENISUPPORT);
require GeniHRN;
my $pid = $self->pid();
my $gid = $self->gid();
my $name = $self->lease_id();
my $domain = $OURDOMAIN;
$domain .= ":${pid}";
$domain .= ":${gid}" if ($pid ne $gid);
return GeniHRN::Generate($domain, $self->type(), $name);
}
#
# Is the underlying blockstore resource mapped in RW currently?
# XXX: blockstore-specific lease function. Should be in a subclass.
......@@ -1761,6 +1783,37 @@ sub RevokeAccess($$)
return 0;
}
# Convience functions for Geni interface.
sub IsPermRO($)
{
my ($self) = @_;
my $idx = $self->idx();
my $perm_idx = GLOBAL_PERM_ANON_RO_IDX();
my $perm_type = "global";
my $qres =
DBQueryWarn("select * from lease_permissions ".
"where lease_idx=$idx and permission_idx='$perm_idx' and ".
" permission_type='$perm_type'");
return 0
if (!$qres || !$qres->numrows);
return 1;
}
sub IsAnonRO($)
{
my ($self) = @_;
my $idx = $self->idx();
my $perm_idx = GLOBAL_PERM_USER_RO_IDX();
my $perm_type = "global";
my $qres =
DBQueryWarn("select * from lease_permissions ".
"where lease_idx=$idx and permission_idx='$perm_idx' and ".
" permission_type='$perm_type'");
return 0
if (!$qres || !$qres->numrows);
return 1;
}
#
# Load attributes if not already loaded.
#
......@@ -2064,6 +2117,32 @@ sub GetProject($)
return $project;
}
#
# Lookup by origin, as for portal created leases.
#
sub LookupByOriginURN($$)
{
my ($class, $urn) = @_;
return undef
if (! $PGENISUPPORT);