Commit c1f9523f authored by Leigh Stoller's avatar Leigh Stoller

A big cleanup/rework of the webtask code to stop leakage and to detect

when they have leaked.
parent 355d7015
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
# Copyright (c) 2007-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -87,6 +87,27 @@ sub Lookup($$;$)
bless($self, $class);
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
$self->Update({"webtask_id" => $webtask->task_id()}) == 0
or return undef;
}
$self->{'WEBTASK'} = $webtask;
return $self;
}
......@@ -134,6 +155,7 @@ AUTOLOAD {
carp("No such slot '$name' field in class $type");
return undef;
}
sub webtask($) { return $_[0]->{'WEBTASK'}; }
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
......@@ -141,6 +163,7 @@ sub DESTROY {
$self->{'DATASET'} = undef;
$self->{'HASH'} = undef;
$self->{'WEBTASK'} = undef;
}
# Valid Blockstore backend.
......@@ -290,9 +313,9 @@ sub Delete($)
my $certificate = $self->GetCertificate();
$certificate->Delete()
if (defined($certificate));
DBQueryWarn("delete from web_tasks where object_uuid='$uuid'") or
return -1;
$self->webtask()->Delete()
if ($self->webtask());
DBQueryWarn("delete from apt_datasets where uuid='$uuid'") or
return -1;
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2016 University of Utah and the Flux Group.
# Copyright (c) 2007-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -149,6 +149,27 @@ sub Lookup($$)
APT_Instance::Aggregate->GenTemp($self)};
}
$self->{'AGGREGATES'} = $aggregates;
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
$self->Update({"webtask_id" => $webtask->task_id()}) == 0
or return undef;
}
$self->{'WEBTASK'} = $webtask;
# Add to cache.
$instances{$self->uuid()} = $self;
......@@ -182,6 +203,7 @@ sub Brand($) { return $_[0]->{'BRAND'}; }
sub isAPT($) { return $_[0]->Brand()->isAPT() ? 1 : 0; }
sub isCloud($) { return $_[0]->Brand()->isCloud() ? 1 : 0; }
sub isPNet($) { return $_[0]->Brand()->isPNet() ? 1 : 0; }
sub webtask($) { return $_[0]->{'WEBTASK'}; }
sub AggregateList($) { return values(%{ $_[0]->{'AGGREGATES'} }); }
sub AggregateHash($) { return $_[0]->{'AGGREGATES'}; }
......@@ -193,6 +215,7 @@ sub DESTROY {
$self->{'BRAND'} = undef;
$self->{'AGGREGATES'} = undef;
$self->{'HASH'} = undef;
$self->{'WEBTASK'} = undef;
}
#
......@@ -369,10 +392,6 @@ sub Update($$)
return Refresh($self);
}
#
# NOTE: We should delete the webtask, but the web UI needs it to
# report status back to the user when an experiment is terminated.
#
sub Delete($)
{
my ($self) = @_;
......@@ -382,6 +401,7 @@ sub Delete($)
$agg->Delete() == 0
or return -1;
}
$self->webtask()->Delete();
DBQueryWarn("delete from apt_instance_extension_info where uuid='$uuid'") or
return -1;
DBQueryWarn("delete from apt_instances where uuid='$uuid'") or
......@@ -967,7 +987,7 @@ sub UpdateImageStatus($$)
if ($self->status() ne "imaging") {
goto done;
}
my $webtask = WebTask->LookupByObject($self->uuid());
my $webtask = $self->webtask();
if (!defined($webtask)) {
goto done;
}
......@@ -1267,7 +1287,7 @@ sub isAL2S($) { return $_[0]->{'ISAL2S'}; }
sub GenTemp($$)
{
my ($class, $instance) = @_;
my $webtask = WebTask->LookupByObject($instance->uuid());
my $webtask = $instance->webtask();
if (!defined($webtask)) {
$webtask = WebTask->Create($instance->uuid());
}
......@@ -1351,6 +1371,8 @@ sub Create($$$)
my $instance_uuid = $instance->uuid();
my $instance_name = $instance->name();
# XXX Anonymous is the wrong thing to do here, but we do not have
# a unique uuid to use.
my $webtask = WebTask->Create(undef);
return undef
if (!defined($webtask));
......@@ -1374,7 +1396,9 @@ sub Delete($)
my $uuid = $self->uuid();
my $urn = $self->aggregate_urn();
$self->webtask()->Delete();
$self->webtask()->Delete()
if ($self->webtask()->Delete());
return 0
if ($self->{'FAKE'});
......
......@@ -45,6 +45,7 @@ use vars qw(@ISA @EXPORT $AUTOLOAD);
# Must come after package declaration!
use EmulabConstants;
use emutil;
use WebTask;
use emdb;
use APT_Dataset;
use GeniXML;
......@@ -54,6 +55,7 @@ use Lease;
use English;
use Data::Dumper;
use File::Basename;
use URI:URL;
use File::Temp qw(tempfile :mktemp tmpnam :POSIX);
use overload ('""' => 'Stringify');
......@@ -85,8 +87,32 @@ sub BlessRow($$)
my $self = {};
$self->{'DBROW'} = $row;
bless($self, $class);
#
# Grab the webtask. Backwards compat mode, see if there is one associated
# with the object, use that. Otherwise create a new one.
#
my $webtask;
if (defined($self->webtask_id())) {
$webtask = WebTask->Lookup($self->webtask_id());
}
if (!defined($webtask)) {
$webtask = WebTask->LookupByObject($self->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create();
return undef
if (!defined($webtask));
}
my $profileid = $self->profileid();
my $webtask_id = $webtask->task_id();
DBQueryWarn("update apt_profiles set webtask_id='$webtask_id' ".
"where profileid='$profileid'")
or return undef;
}
$self->{'WEBTASK'} = $webtask;
return $self;
}
......@@ -253,7 +279,8 @@ AUTOLOAD {
sub DESTROY {
my $self = shift;
$self->{'DBROW'} = undef;
$self->{'WEBTASK'} = undef;
$self->{'DBROW'} = undef;
}
sub IsRepoBased($) {
......@@ -261,6 +288,7 @@ sub IsRepoBased($) {
return (defined($self->repourl()) ? 1 : 0);
}
sub webtask($) { return $_[0]->{'WEBTASK'}; }
#
# Refresh a class instance by reloading from the DB.
......@@ -301,27 +329,33 @@ sub Create($$$$$$)
my $gid_idx = $project->pid_idx();
my $uid = $creator->uid();
my $uid_idx = $creator->uid_idx();
my $puuid = NewUUID();
my $vuuid = NewUUID();
my $webtask = WebTask->Create();
return undef
if (!defined($webtask));
#
# The pid/imageid has to be unique, so lock the table for the check/insert.
#
DBQueryWarn("lock tables apt_profiles write, apt_profile_versions write, ".
" emulab_indicies write")
or return undef;
if (!DBQueryWarn("lock tables apt_profiles write, ".
" apt_profile_versions write, ".
" emulab_indicies write")) {
$webtask->Delete();
return undef;
}
my $query_result =
DBQueryWarn("select name from apt_profiles ".
"where pid_idx='$pid_idx' and name=$name");
if ($query_result->numrows) {
DBQueryWarn("unlock tables");
$webtask->Delete();
$$usrerr_ref = "Profile already exists in project!";
return undef;
}
my $profileid = TBGetUniqueIndex("next_profile", undef, 1);
my $puuid = NewUUID();
my $vuuid = NewUUID();
my $rspec = DBQuoteSpecial($argref->{'rspec'});
my $cquery = "";
my $vquery = "";
......@@ -357,6 +391,7 @@ sub Create($$$$$$)
# Back to the main table.
$cquery .= ",uuid='$puuid'";
$cquery .= ",webtask_id=" . DBQuoteSpecial($webtask->task_id());
$cquery .= ",public=1"
if (exists($argref->{'public'}) && $argref->{'public'});
$cquery .= ",listed=1"
......@@ -370,6 +405,7 @@ sub Create($$$$$$)
if (! DBQueryWarn("insert into apt_profiles set $cquery")) {
DBQueryWarn("unlock tables");
tberror("Error inserting new apt_profiles record!");
$webtask->Delete();
return undef;
}
# And the versions entry.
......@@ -377,6 +413,7 @@ sub Create($$$$$$)
DBQueryWarn("delete from apt_profiles where profileid='$profileid'");
DBQueryWarn("unlock tables");
tberror("Error inserting new apt_profile_versions record!");
$webtask->Delete();
return undef;
}
DBQueryWarn("unlock tables");
......@@ -562,11 +599,14 @@ sub Delete($$)
DBQueryWarn("update apt_profile_versions set deleted=now() ".
"where profileid='$profileid'")
or goto bad;
# Delete any leftover webtasks.
# Delete any leftover webtasks. These are old ones.
DBQueryWarn("delete web_tasks from apt_profile_versions ".
"left join web_tasks on ".
" web_tasks.object_uuid=apt_profile_versions.uuid ".
"where apt_profile_versions.profileid='$profileid'");
# Primary webtask.
$self->webtask()->Delete()
if ($self->webtask());
}
DBQueryWarn("delete from apt_profile_favorites ".
"where profileid='$profileid'")
......
......@@ -822,9 +822,22 @@ if (defined($project)) {
$blob->{"gid"} = $group->gid();
$blob->{"gid_idx"} = $group->gid_idx();
}
#
# Create a webtask so that we can store additional information about
# the sliver while we wait.
#
$webtask = WebTask->Create($quickvm_uuid);
if (!defined($webtask)) {
fatal("Could not create a webtask!");
}
$webtask_id = $webtask->task_id();
$webtask->AutoStore(1);
$blob->{"webtask_id"} = $webtask_id;
$errmsg = undef;
$instance = APT_Instance->Create($blob, \$errmsg);
if (!defined($instance)) {
$webtask->Delete();
fatal(defined($errmsg) ? $errmsg :
"Could not create instance record for $quickvm_uuid");
}
......@@ -873,17 +886,6 @@ foreach my $aggregate_urn (@aggregate_urns) {
# To keep stuff happy until multisite support finished.
$instance->Update({'aggregate_urn' => $aggregate_urns[0]});
#
# Create a webtask so that we can store additional information about
# the sliver while we wait.
#
$webtask = WebTask->Create($instance->uuid());
if (!defined($webtask)) {
fatal("Could not create a webtask!");
}
$webtask_id = $webtask->task_id();
$webtask->AutoStore(1);
print STDERR "\n";
print STDERR "User: $user_urn\n";
print STDERR "Email: $user_email" . (!$localuser ? " (guest)" : "") . "\n";
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -120,9 +120,9 @@ if (defined($options{"d"})) {
}
if (defined($options{"t"})) {
$webtask_id = $options{"t"};
$webtask = WebTask->LookupOrCreate(undef, $webtask_id);
$webtask = WebTask->Lookup($webtask_id);
if (!defined($webtask)) {
fatal("Could not create webtask object");
fatal("Could not get webtask object");
}
$webtask->AutoStore(1);
}
......@@ -337,8 +337,21 @@ sub DoCreate()
$blob->{"write_access"} = $write_access
if (defined($write_access));
#
# Always create a webtask for tracking image or allocation status.
# This is an internal webtask, not the one used on the command line.
#
my $dwebtask = WebTask->Create();
if (!defined($dwebtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$dwebtask->AutoStore(1);
$blob->{"webtask_id"} = $dwebtask->task_id();
my $dataset = APT_Dataset->Create($blob);
if (!defined($dataset)) {
$dwebtask->Delete();
fatal("Error creating dataset object");
}
......@@ -350,16 +363,6 @@ sub DoCreate()
goto failed;
}
}
#
# Always create a webtask for tracking image or allocation status.
#
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object";
goto failed;
}
$webtask->AutoStore(1);
#
# Ask the aggregate to create the dataset.
#
......@@ -527,24 +530,22 @@ sub DoRefreshInternal($$)
if ($blob->{"busy"}) {
$dataset->Update({"state" => "busy"});
if ($dataset->type() eq "imdataset") {
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
$dataset->webtask()->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$dataset->webtask()->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
$dataset->webtask()->Store();
}
}
else {
$dataset->Update({"state" => $blob->{"state"}});
if ($dataset->type() eq "imdataset") {
$dataset->Update({"size" => $blob->{"size"}});
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
$dataset->webtask()->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$dataset->webtask()->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
$dataset->webtask()->Store();
}
}
return 0;
......@@ -726,36 +727,24 @@ sub DoSnapshot()
$dataset->Unlock();
uerror("instance is busy, cannot lock it");
}
#
# Always create a webtask for tracking imaging status. Must be
# associated with the object.
#
if (defined($webtask)) {
if ($webtask->object_uuid() ne $dataset->uuid()) {
$errmsg = "Webtask not associated with dataset!";
goto failed;
}
}
else {
$webtask = WebTask->LookupByObject($dataset->uuid());
if (!defined($webtask)) {
$webtask = WebTask->Create($dataset->uuid());
if (!defined($webtask)) {
$errmsg = "Could not create webtask object!";
goto failed;
}
}
$webtask->AutoStore(1);
}
$webtask->Reset();
# Clear the webtask, starting a new snapshot.
$dataset->webtask()->Reset();
# These three are convenience for the web server to give feedback.
$dataset->webtask()->aggregate_urn($aggregate->aggregate_urn());
$dataset->webtask()->client_id($nodeid);
$dataset->webtask()->instance($instance->uuid());
if (defined($copyback_uuid)) {
# Tell the imaging modal.
$webtask->copyback_uuid($copyback_uuid);
$dataset->webtask()->copyback_uuid($copyback_uuid);
# For polling below.
$dataset->_copyback_uuid($copyback_uuid);
$dataset->_sha1hash("$sha1hash");
$dataset->_copying(0);
}
$dataset->webtask()->Store();
$dataset->webtask()->AutoStore(1);
if (DoSnapShotInternal($dataset, $aggregate, $bsname, $nodeid, \$errmsg)) {
goto failed;
}
......@@ -863,8 +852,7 @@ sub PollDatasetStatus($$$)
# Let parent exit;
sleep(2);
}
$webtask->SetProcessID($PID)
if (defined($webtask));
$dataset->webtask()->SetProcessID($PID);
print "State: " . $dataset->state() . "\n";
if (defined($dataset->_copyback_uuid())) {
......@@ -897,17 +885,13 @@ sub PollDatasetStatus($$$)
"Your dataset is now ready to use",
"Dataset '$dname' is now ready to use.\n",
$project->LogsEmailAddress(), undef, $logfile);
$webtask->Exited(0)
if (defined($webtask));
$dataset->webtask()->Exited(0);
last;
}
sleep($interval);
}
$webtask->Exited(-1)
if (defined($webtask) && $seconds <= 0);
# unlink($logfile)
# if (defined($logfile));
$dataset->webtask()->Exited(-1)
if ($seconds <= 0);
return 0;
}
......@@ -1052,16 +1036,18 @@ sub PollImageStatus($$$)
print STDERR $perrmsg . "\n";
# Give up.
$dataset->Update({"state" => "valid"});
$webtask->image_status("ready");
$dataset->webtask()->image_status("ready");
}
if ("$sha1hash" eq $dataset->_sha1hash()) {
# Done!
$dataset->Update({"state" => "valid"});
$webtask->image_status("ready");
$dataset->webtask()->image_status("ready");
}
return 0;
}
else {
print "Getting Image Info\n";
my $response = $aggregate->ImageInfo($image_urn);
if ($response->code() != GENIRESPONSE_SUCCESS &&
$response->code() != GENIRESPONSE_RPCERROR &&
......@@ -1076,15 +1062,15 @@ sub PollImageStatus($$$)
$response->code() == GENIRESPONSE_RPCERROR);
my $blob = $response->value();
print Dumper($blob);
print Dumper($response->value());
$webtask->image_size($blob->{'size'})
$dataset->webtask()->image_size($blob->{'size'})
if (exists($blob->{'size'}));
$webtask->image_status($blob->{'status'})
$dataset->webtask()->image_status($blob->{'status'})
if (exists($blob->{'status'}));
if ($blob->{'status'} eq "ready") {
if ($copyback_uuid) {
$webtask->image_status("copying");
$dataset->webtask()->image_status("copying");
$dataset->_copying(1);
}
else {
......
This diff is collapsed.
......@@ -162,26 +162,12 @@ elsif ($action ne "create") {
my $xmlfile = shift(@ARGV);
#
# Create the webtask object, even though we do not have a profile
# object yet, we will set it below. We use the webtask to pass the
# errors back to the web interface before the profile is created.
# Grab the webtask object.
#
# Note that if we fail, we want to leave the webtask around for the
# web interface, it will need to delete it.
#
# If doing a snapshot, we always create one since that is the easiest
# way to communicate with the manage_instance script, even if we ran
# this from the command line.
#
if (defined($webtask_id) || $snap) {
if (defined($webtask_id)) {
$webtask = WebTask->Lookup($webtask_id);
}
else {
$webtask = WebTask->Create(undef);
}
if (defined($webtask_id)) {
$webtask = WebTask->Lookup($webtask_id);
if (!defined($webtask)) {
fatal("Could not create webtask for profile");
fatal("Could not lookup/create webtask for profile");
}
$webtask->AutoStore(1);
}
......@@ -530,10 +516,6 @@ else {
}
fatal("Could not create new profile");
}
# Had to wait to do this, see comment above.
$webtask->SetObject($profile->uuid())
if (defined($webtask));
if (!$this_user->IsAdmin()) {
$profile->Publish();
}
......@@ -546,13 +528,14 @@ if (defined($instance)) {
my $apt_uuid = $instance->uuid();
my $imagename = $profile->name();
my $new_uuid = $profile->uuid();
# We want to use the webtask associated with the new profile.
my $pwebtask = $profile->webtask();
if ($profile->Lock()) {
$profile->Delete(1);
fatal("Could not lock new profile");
}
my $command = "$MANAGEINSTANCE -t " . $webtask->task_id() . " -- ".
my $command = "$MANAGEINSTANCE -t " . $pwebtask->task_id() . " ".
"snapshot $apt_uuid -c $new_uuid -n $node_id -i $imagename";
#
......@@ -566,8 +549,6 @@ if (defined($instance)) {
my $stat = $? >> 8;
$profile->Delete(1);
$webtask->Delete()
if (!defined($webtask_id));
print STDERR $output . "\n";
if ($stat < 0) {
fatal("Failed to create disk image!");
......@@ -577,25 +558,26 @@ if (defined($instance)) {
#
# The script helpfully put the new image urn in the webtask.
#
$webtask->Refresh();
$pwebtask->Refresh();
my $newimage;
if (GetSiteVar("protogeni/use_imagetracker") &&
EmulabFeatures->FeatureEnabled("APT_UseImageTracker",
$this_user, $project)) {
$newimage = $webtask->image_urn();
$newimage = $pwebtask->image_urn();
}
else {
$newimage = $webtask->image_url();
$newimage = $pwebtask->image_url();
}
if (!defined($newimage) ||
$profile->UpdateDiskImage($node_id, $newimage, 0)) {
$webtask->Delete()
if (!defined($webtask_id));
$profile->Delete(1);
fatal("Could not update image URN in rspec");
}
# Tell web interface cloning has started.
$pwebtask->cloning(1);
#
# Exit and leave child to poll.
#
......@@ -622,33 +604,34 @@ if (defined($instance)) {
while (1) {
sleep(10);
$webtask->Refresh();
$pwebtask->Refresh();
last
if (defined($webtask->exited()));
if (defined($pwebtask->exited()));
#
# See if the process is still running. If not then it died badly.
# Mark the webtask as exited.
#
my $pid = $webtask->process_id();
my $pid = $pwebtask->process_id();
if (! kill(0, $pid)) {
# Check again in case it just exited.
$webtask->Refresh();
if (! defined($webtask->exited())) {
$webtask->Exited(-1);
$pwebtask->Refresh();
if (! defined($pwebtask->exited())) {
$pwebtask->Exited(-1);
}
last;
}
}
if ($webtask->exitcode()) {
# When the profile is deleted, the web task will be deleted. The
# web interface will see that of course and return an error to the
# client JS code.
if ($pwebtask->exitcode()) {
$profile->Delete(1);
$webtask->Delete()
if (!defined($webtask_id));
exit(1);
}
# Tell web interface cloning has finished
$pwebtask->cloning(0);
$profile->Unlock();
$webtask->Delete()
if (!defined($webtask_id));
exit(0);
}
......@@ -665,24 +648,15 @@ $project->SendEmail($portalLogs, "New Profile Created",
"UUID: ". $profile->uuid() . "\n".
"URL: ". $profile->AdminURL() . "\n");
# Make sure we delete Anonymous webtask,
$webtask->Delete()
if (defined($webtask) && !defined($webtask_id));
exit(0);
sub fatal($)
{
my ($mesg) = @_;
if (defined($webtask)) {
# Anonymous webtask, delete.
if (!defined($webtask_id)) {
$webtask->Delete();
}
else {
$webtask->output($mesg);
$webtask->Exited(-1);
}
if (defined($webtask_id)) {
$webtask->output($mesg);
$webtask->Exited(-1);
}
print STDERR "*** $0:\n".