All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit c1f9523f authored by Leigh B Stoller's avatar Leigh B Stoller

A big cleanup/rework of the webtask code to stop leakage and to detect

when they have leaked.
parent 355d7015
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
# Copyright (c) 2007-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -87,6 +87,27 @@ sub Lookup($$;$)
bless($self, $class);
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
$self->Update({"webtask_id" => $webtask->task_id()}) == 0
or return undef;
}
$self->{'WEBTASK'} = $webtask;
return $self;
}
......@@ -134,6 +155,7 @@ AUTOLOAD {
carp("No such slot '$name' field in class $type");
return undef;
}
sub webtask($) { return $_[0]->{'WEBTASK'}; }
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
......@@ -141,6 +163,7 @@ sub DESTROY {
$self->{'DATASET'} = undef;
$self->{'HASH'} = undef;
$self->{'WEBTASK'} = undef;
}
# Valid Blockstore backend.
......@@ -290,9 +313,9 @@ sub Delete($)
my $certificate = $self->GetCertificate();
$certificate->Delete()
if (defined($certificate));
DBQueryWarn("delete from web_tasks where object_uuid='$uuid'") or
return -1;
$self->webtask()->Delete()
if ($self->webtask());
DBQueryWarn("delete from apt_datasets where uuid='$uuid'") or
return -1;
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
# Copyright (c) 2007-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -149,6 +149,27 @@ sub Lookup($$)
APT_Instance::Aggregate->GenTemp($self)};
}
$self->{'AGGREGATES'} = $aggregates;
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
$self->Update({"webtask_id" => $webtask->task_id()}) == 0
or return undef;
}
$self->{'WEBTASK'} = $webtask;
# Add to cache.
$instances{$self->uuid()} = $self;
......@@ -182,6 +203,7 @@ sub Brand($) { return $_[0]->{'BRAND'}; }
sub isAPT($) { return $_[0]->Brand()->isAPT() ? 1 : 0; }
sub isCloud($) { return $_[0]->Brand()->isCloud() ? 1 : 0; }
sub isPNet($) { return $_[0]->Brand()->isPNet() ? 1 : 0; }
sub webtask($) { return $_[0]->{'WEBTASK'}; }
sub AggregateList($) { return values(%{ $_[0]->{'AGGREGATES'} }); }
sub AggregateHash($) { return $_[0]->{'AGGREGATES'}; }
......@@ -193,6 +215,7 @@ sub DESTROY {
$self->{'BRAND'} = undef;
$self->{'AGGREGATES'} = undef;
$self->{'HASH'} = undef;
$self->{'WEBTASK'} = undef;
}
#
......@@ -369,10 +392,6 @@ sub Update($$)
return Refresh($self);
}
#
# NOTE: We should delete the webtask, but the web UI needs it to
# report status back to the user when an experiment is terminated.
#
sub Delete($)
{
my ($self) = @_;
......@@ -382,6 +401,7 @@ sub Delete($)
$agg->Delete() == 0
or return -1;
}
$self->webtask()->Delete();
DBQueryWarn("delete from apt_instance_extension_info where uuid='$uuid'") or
return -1;
DBQueryWarn("delete from apt_instances where uuid='$uuid'") or
......@@ -967,7 +987,7 @@ sub UpdateImageStatus($$)
if ($self->status() ne "imaging") {
goto done;
}
my $webtask = WebTask->LookupByObject($self->uuid());
my $webtask = $self->webtask();
if (!defined($webtask)) {
goto done;
}
......@@ -1267,7 +1287,7 @@ sub isAL2S($) { return $_[0]->{'ISAL2S'}; }
sub GenTemp($$)
{
my ($class, $instance) = @_;
my $webtask = WebTask->LookupByObject($instance->uuid());
my $webtask = $instance->webtask();
if (!defined($webtask)) {
$webtask = WebTask->Create($instance->uuid());
}
......@@ -1351,6 +1371,8 @@ sub Create($$$)
my $instance_uuid = $instance->uuid();
my $instance_name = $instance->name();
# XXX Anonymous is the wrong thing to do here, but we do not have
# a unique uuid to use.
my $webtask = WebTask->Create(undef);
return undef
if (!defined($webtask));
......@@ -1374,7 +1396,9 @@ sub Delete($)
my $uuid = $self->uuid();
my $urn = $self->aggregate_urn();
$self->webtask()->Delete();
$self->webtask()->Delete()
if ($self->webtask()->Delete());
return 0
if ($self->{'FAKE'});
......
......@@ -45,6 +45,7 @@ use vars qw(@ISA @EXPORT $AUTOLOAD);
# Must come after package declaration!
use EmulabConstants;
use emutil;
use WebTask;
use emdb;
use APT_Dataset;
use GeniXML;
......@@ -54,6 +55,7 @@ use Lease;
use English;
use Data::Dumper;
use File::Basename;
use URI:URL;
use File::Temp qw(tempfile :mktemp tmpnam :POSIX);
use overload ('""' => 'Stringify');
......@@ -85,8 +87,32 @@ sub BlessRow($$)
my $self = {};
$self->{'DBROW'} = $row;
bless($self, $class);
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
my $profileid = $self->profileid();
my $webtask_id = $webtask->task_id();
DBQueryWarn("update apt_profiles set webtask_id='$webtask_id' ".
"where profileid='$profileid'")
or return undef;
}
$self->{'WEBTASK'} = $webtask;
return $self;
}
......@@ -253,7 +279,8 @@ AUTOLOAD {
sub DESTROY {
my $self = shift;
$self->{'DBROW'} = undef;
$self->{'WEBTASK'} = undef;
$self->{'DBROW'} = undef;
}
sub IsRepoBased($) {
......@@ -261,6 +288,7 @@ sub IsRepoBased($) {
return (defined($self->repourl()) ? 1 : 0);
}
sub webtask($) { return $_[0]->{'WEBTASK'}; }
#
# Refresh a class instance by reloading from the DB.
......@@ -301,27 +329,33 @@ sub Create($$$$$$)
my $gid_idx = $project->pid_idx();
my $uid = $creator->uid();
my $uid_idx = $creator->uid_idx();
my $puuid = NewUUID();
my $vuuid = NewUUID();
my $webtask = WebTask->Create();
return undef
if (!defined($webtask));
#
# The pid/imageid has to be unique, so lock the table for the check/insert.
#
DBQueryWarn("lock tables apt_profiles write, apt_profile_versions write, ".
" emulab_indicies write")
or return undef;
if (!DBQueryWarn("lock tables apt_profiles write, ".
" apt_profile_versions write, ".
" emulab_indicies write")) {
$webtask->Delete();
return undef;
}
my $query_result =
DBQueryWarn("select name from apt_profiles ".
"where pid_idx='$pid_idx' and name=$name");
if ($query_result->numrows) {
DBQueryWarn("unlock tables");
$webtask->Delete();
$$usrerr_ref = "Profile already exists in project!";
return undef;
}
my $profileid = TBGetUniqueIndex("next_profile", undef, 1);
my $puuid = NewUUID();
my $vuuid = NewUUID();
my $rspec = DBQuoteSpecial($argref->{'rspec'});
my $cquery = "";
my $vquery = "";
......@@ -357,6 +391,7 @@ sub Create($$$$$$)
# Back to the main table.
$cquery .= ",uuid='$puuid'";
$cquery .= ",webtask_id=" . DBQuoteSpecial($webtask->task_id());
$cquery .= ",public=1"
if (exists($argref->{'public'}) && $argref->{'public'});
$cquery .= ",listed=1"
......@@ -370,6 +405,7 @@ sub Create($$$$$$)
if (! DBQueryWarn("insert into apt_profiles set $cquery")) {
DBQueryWarn("unlock tables");
tberror("Error inserting new apt_profiles record!");
$webtask->Delete();
return undef;
}
# And the versions entry.
......@@ -377,6 +413,7 @@ sub Create($$$$$$)
DBQueryWarn("delete from apt_profiles where profileid='$profileid'");
DBQueryWarn("unlock tables");
tberror("Error inserting new apt_profile_versions record!");
$webtask->Delete();
return undef;
}
DBQueryWarn("unlock tables");
......@@ -562,11 +599,14 @@ sub Delete($$)
DBQueryWarn("update apt_profile_versions set deleted=now() ".
"where profileid='$profileid'")
or goto bad;
# Delete any leftover webtasks.
# Delete any leftover webtasks. These are old ones.
DBQueryWarn("delete web_tasks from apt_profile_versions ".
"left join web_tasks on ".
" web_tasks.object_uuid=apt_profile_versions.uuid ".
"where apt_profile_versions.profileid='$profileid'");
# Primary webtask.
$self->webtask()->Delete()
if ($self->webtask());
}
DBQueryWarn("delete from apt_profile_favorites ".
"where profileid='$profileid'")
......
......@@ -822,9 +822,22 @@ if (defined($project)) {
$blob->{"gid"} = $group->gid();
$blob->{"gid_idx"} = $group->gid_idx();
}
#
# Create a webtask so that we can store additional information about
# the sliver while we wait.
#
$webtask = WebTask->Create($quickvm_uuid);
if (!defined($webtask)) {
fatal("Could not create a webtask!");
}
$webtask_id = $webtask->task_id();
$webtask->AutoStore(1);
$blob->{"webtask_id"} = $webtask_id;
$errmsg = undef;
$instance = APT_Instance->Create($blob, \$errmsg);
if (!defined($instance)) {
$webtask->Delete();
fatal(defined($errmsg) ? $errmsg :
"Could not create instance record for $quickvm_uuid");
}
......@@ -873,17 +886,6 @@ foreach my $aggregate_urn (@aggregate_urns) {
# To keep stuff happy until multisite support finished.
$instance->Update({'aggregate_urn' => $aggregate_urns[0]});
#
# Create a webtask so that we can store additional information about
# the sliver while we wait.
#
$webtask = WebTask->Create($instance->uuid());
if (!defined($webtask)) {
fatal("Could not create a webtask!");
}
$webtask_id = $webtask->task_id();
$webtask->AutoStore(1);
print STDERR "\n";
print STDERR "User: $user_urn\n";
print STDERR "Email: $user_email" . (!$localuser ? " (guest)" : "") . "\n";
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -120,9 +120,9 @@ if (defined($options{"d"})) {
}
if (defined($options{"t"})) {
$webtask_id = $options{"t"};
$webtask = WebTask->LookupOrCreate(undef, $webtask_id);
$webtask = WebTask->Lookup($webtask_id);
if (!defined($webtask)) {
fatal("Could not create webtask object");
fatal("Could not get webtask object");
}
$webtask->AutoStore(1);
}
......@@ -337,8 +337,21 @@ sub DoCreate()
$blob->{"write_access"} = $write_access
if (defined($write_access));
#
# Always create a webtask for tracking image or allocation status.
# This is an internal webtask, not the one used on the command line.
#
my $dwebtask = WebTask->Create();
if (!defined($dwebtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$dwebtask->AutoStore(1);
$blob->{"webtask_id"} = $dwebtask->task_id();
my $dataset = APT_Dataset->Create($blob);
if (!defined($dataset)) {
$dwebtask->Delete();
fatal("Error creating dataset object");
}
......@@ -350,16 +363,6 @@ sub DoCreate()
goto failed;
}
}
#
# Always create a webtask for tracking image or allocation status.
#
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$webtask->AutoStore(1);
#
# Ask the aggregate to create the dataset.
#
......@@ -527,24 +530,22 @@ sub DoRefreshInternal($$)
if ($blob->{"busy"}) {
$dataset->Update({"state" => "busy"});
if ($dataset->type() eq "imdataset") {
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
$dataset->webtask()->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$dataset->webtask()->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
$dataset->webtask()->Store();
}
}
else {
$dataset->Update({"state" => $blob->{"state"}});
if ($dataset->type() eq "imdataset") {
$dataset->Update({"size" => $blob->{"size"}});
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
$dataset->webtask()->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$dataset->webtask()->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
$dataset->webtask()->Store();
}
}
return 0;
......@@ -726,36 +727,24 @@ sub DoSnapshot()
$dataset->Unlock();
uerror("instance is busy, cannot lock it");
}
#
# Always create a webtask for tracking imaging status. Must be
# associated with the object.
#
if (defined($webtask)) {
if ($webtask->object_uuid() ne $dataset->uuid()) {
$errmsg = "Webtask not associated with dataset!";
goto failed;
}
}
else {
$webtask = WebTask->LookupByObject($dataset->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object!";
goto failed;
}
}
$webtask->AutoStore(1);
}
$webtask->Reset();
# Clear the webtask, starting a new snapshot.
$dataset->webtask()->Reset();
# These three are convenience for the web server to give feedback.
$dataset->webtask()->aggregate_urn($aggregate->aggregate_urn());
$dataset->webtask()->client_id($nodeid);
$dataset->webtask()->instance($instance->uuid());
if (defined($copyback_uuid)) {
# Tell the imaging modal.
$webtask->copyback_uuid($copyback_uuid);
$dataset->webtask()->copyback_uuid($copyback_uuid);
# For polling below.
$dataset->_copyback_uuid($copyback_uuid);
$dataset->_sha1hash("$sha1hash");
$dataset->_copying(0);
}
$dataset->webtask()->Store();
$dataset->webtask()->AutoStore(1);
if (DoSnapShotInternal($dataset, $aggregate, $bsname, $nodeid, \$errmsg)) {
goto failed;
}
......@@ -863,8 +852,7 @@ sub PollDatasetStatus($$$)
# Let parent exit;
sleep(2);
}
$webtask->SetProcessID($PID)
if (defined($webtask));
$dataset->webtask()->SetProcessID($PID);
print "State: " . $dataset->state() . "\n";
if (defined($dataset->_copyback_uuid())) {
......@@ -897,17 +885,13 @@ sub PollDatasetStatus($$$)
"Your dataset is now ready to use",
"Dataset '$dname' is now ready to use.\n",
$project->LogsEmailAddress(), undef, $logfile);
$webtask->Exited(0)
if (defined($webtask));
$dataset->webtask()->Exited(0);
last;
}
sleep($interval);
}
$webtask->Exited(-1)
if (defined($webtask) && $seconds <= 0);
# unlink($logfile)
# if (defined($logfile));
$dataset->webtask()->Exited(-1)
if ($seconds <= 0);
return 0;
}
......@@ -1052,16 +1036,18 @@ sub PollImageStatus($$$)
print STDERR $perrmsg . "\n";
# Give up.
$dataset->Update({"state" => "valid"});
$webtask->image_status("ready");
$dataset->webtask()->image_status("ready");
}
if ("$sha1hash" eq $dataset->_sha1hash()) {
# Done!
$dataset->Update({"state" => "valid"});
$webtask->image_status("ready");
$dataset->webtask()->image_status("ready");
}
return 0;
}
else {
print "Getting Image Info\n";
my $response = $aggregate->ImageInfo($image_urn);
if ($response->code() != GENIRESPONSE_SUCCESS &&
$response->code() != GENIRESPONSE_RPCERROR &&
......@@ -1076,15 +1062,15 @@ sub PollImageStatus($$$)
$response->code() == GENIRESPONSE_RPCERROR);
my $blob = $response->value();
print Dumper($blob);
print Dumper($response->value());
$webtask->image_size($blob->{'size'})
$dataset->webtask()->image_size($blob->{'size'})
if (exists($blob->{'size'}));
$webtask->image_status($blob->{'status'})
$dataset->webtask()->image_status($blob->{'status'})
if (exists($blob->{'status'}));
if ($blob->{'status'} eq "ready") {
if ($copyback_uuid) {
$webtask->image_status("copying");
$dataset->webtask()->image_status("copying");
$dataset->_copying(1);
}
else {
......
This diff is collapsed.
......@@ -162,26 +162,12 @@ elsif ($action ne "create") {
my $xmlfile = shift(@ARGV);
#
# Create the webtask object, even though we do not have a profile
# object yet, we will set it below. We use the webtask to pass the
# errors back to the web interface before the profile is created.
# Grab the webtask object.
#
# Note that if we fail, we want to leave the webtask around for the
# web interface, it will need to delete it.
#
# If doing a snapshot, we always create one since that is the easiest
# way to communicate with the manage_instance script, even if we ran
# this from the command line.
#
if (defined($webtask_id) || $snap) {
if (defined($webtask_id)) {
$webtask = WebTask->Lookup($webtask_id);
}
else {
$webtask = WebTask->Create(undef);
}
if (defined($webtask_id)) {
$webtask = WebTask->Lookup($webtask_id);
if (!defined($webtask)) {
fatal("Could not create webtask for profile");
fatal("Could not lookup/create webtask for profile");
}
$webtask->AutoStore(1);
}
......@@ -530,10 +516,6 @@ else {
}
fatal("Could not create new profile");
}
# Had to wait to do this, see comment above.
$webtask->SetObject($profile->uuid())
if (defined($webtask));
if (!$this_user->IsAdmin()) {
$profile->Publish();
}
......@@ -546,13 +528,14 @@ if (defined($instance)) {
my $apt_uuid = $instance->uuid();
my $imagename = $profile->name();
my $new_uuid = $profile->uuid();
# We want to use the webtask associated with the new profile.
my $pwebtask = $profile->webtask();
if ($profile->Lock()) {
$profile->Delete(1);
fatal("Could not lock new profile");
}
my $command = "$MANAGEINSTANCE -t " . $webtask->task_id() . " -- ".
my $command = "$MANAGEINSTANCE -t " . $pwebtask->task_id() . " ".
"snapshot $apt_uuid -c $new_uuid -n $node_id -i $imagename";
#
......@@ -566,8 +549,6 @@ if (defined($instance)) {
my $stat = $? >> 8;
$profile->Delete(1);
$webtask->Delete()
if (!defined($webtask_id));
print STDERR $output . "\n";
if ($stat < 0) {
fatal("Failed to create disk image!");
......@@ -577,25 +558,26 @@ if (defined($instance)) {
#
# The script helpfully put the new image urn in the webtask.
#
$webtask->Refresh();
$pwebtask->Refresh();
my $newimage;
if (GetSiteVar("protogeni/use_imagetracker") &&
EmulabFeatures->FeatureEnabled("APT_UseImageTracker",
$this_user, $project)) {
$newimage = $webtask->image_urn();