Skip to content

GitLab

Commit 2bdbfa16 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files
Options

Hooks in the imaging progress modal to image backed dataset creation. 

Checkpoint, still needs some work.
parent 8fe52045
Hide whitespace changes
Inline Side-by-side
Showing with 243 additions and 83 deletions
apt/APT_Dataset.pm.in
View file @ 2bdbfa16
......@@ -266,6 +266,8 @@ sub Delete($)
$certificate->Delete()
if (defined($certificate));
DBQueryWarn("delete from web_tasks object_uuid='$uuid'") or
return -1;
DBQueryWarn("delete from apt_datasets where uuid='$uuid'") or
return -1;
......@@ -291,6 +293,23 @@ sub SetStatus($$)
}
#
# Load the project object for an experiment.
#
sub GetProject($)
{
my ($self) = @_;
require Project;
my $project = Project->Lookup($self->pid_idx());
if (! defined($project)) {
print("*** WARNING: Could not lookup project object for $self!\n");
return undef;
}
return $project;
}
#
# Lock and Unlock
#
......
apt/manage_dataset.in
View file @ 2bdbfa16
......@@ -91,6 +91,7 @@ sub DoModify();
sub DoExtend();
sub DoSnapshot();
sub DoSnapShotInternal($$$$$);
sub PollDatasetStatus($$);
#
# Parse command arguments. Once we return from getopts, all that should be
......@@ -121,18 +122,6 @@ if (! defined($this_user)) {
}
my $action = shift(@ARGV);
#
# Create the webtask object if coming from the web interface.
#
if (defined($webtask_id)) {
$webtask = WebTask->Create(undef, $webtask_id);
if (!defined($webtask)) {
fatal("Could not create webtask");
}
# Convenient.
$webtask->AutoStore(1);
}
if ($action eq "create") {
exit(DoCreate());
}
......@@ -169,7 +158,6 @@ sub DoCreate()
exit(-1);
};
my $aggregate_urn = "urn:publicid:IDN+apt.emulab.net+authority+cm";
my $logfile;
my $errmsg;
my $pid;
my $expires;
......@@ -303,6 +291,15 @@ sub DoCreate()
if (!defined($dataset)) {
fatal("Internal error creating dataset object");
}
# Now we can create.
if (defined($webtask_id)) {
$webtask = WebTask->Create($dataset->uuid(), $webtask_id);
if (defined($webtask)) {
# Convenient.
$webtask->AutoStore(1);
}
}
# new dataset is returned locked. If we have instance, try to lock
# that now, else its a failure.
if ($type eq "imdataset" && defined($instance)) {
......@@ -351,56 +348,17 @@ sub DoCreate()
$instance->Unlock();
goto failed;
}
#
# If busy, then allocation is in progress. We leave it locked and
# poll in the background for a while, hoping for it to eventually
# stop being busy. Eventually might have to replace this, since
# polling got any non-small length of time will lead to trouble.
#
if (! $debug) {
$logfile = TBMakeLogname("createdataset");
if (my $childpid = TBBackGround($logfile)) {
# Parent exits normally, web interface watches.
exit(0);
}
# Let parent exit;
sleep(2);
}
$webtask->SetProcessID($PID)
if (defined($webtask));
my $seconds = 1200;
my $interval = 30;
while ($seconds > 0) {
sleep($interval);
$seconds -= $interval;
if (DoRefreshInternal($dataset, \$errmsg)) {
print STDERR $errmsg;
next;
}
if ($dataset->state() eq "valid") {
$project->SendEmail($this_user->email(),
"Your dataset is now ready to use",
"Dataset '$name' is now allocated and ready to use.\n",
$project->OpsEmailAddress());
last;
}
if (PollDatasetStatus($dataset, \$errmsg)) {
# Exit and let child poll
exit(0);
}
done:
$dataset->Unlock();
$instance->Unlock() if (defined($instance));
unlink($logfile)
if (defined($logfile));
return 0;
failed:
$dataset->Delete()
if (defined($dataset));
unlink($logfile)
if (defined($logfile));
# This will set the webtask, see below.
fatal($errmsg);
}
......@@ -483,16 +441,32 @@ sub DoRefreshInternal($$)
return -1;
}
my $blob = $response->value();
print STDERR Dumper($blob);
$dataset->Update({"last_used" => TBDateStringLocal($blob->{"lastused"}),
"expires" => TBDateStringLocal($blob->{"expires"})});
if ($blob->{"busy"}) {
$dataset->Update({"state" => "busy"});
if ($dataset->type() eq "imdataset") {
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
}
}
else {
$dataset->Update({"state" => $blob->{"state"}});
if ($dataset->type() eq "imdataset") {
$dataset->Update({"size" => $blob->{"size"}});
if (defined($webtask)) {
$webtask->image_size($blob->{'image_size'})
if (exists($blob->{'image_size'}));
$webtask->image_status($blob->{'image_status'})
if (exists($blob->{'image_status'}));
}
}
}
return 0;
......@@ -657,9 +631,19 @@ sub DoSnapshot()
$errmsg = "instance is busy, cannot lock it";
goto failed
}
# Create a webtask for the caller, might be ignored but thats okay.
$webtask = WebTask->LookupOrCreate($dataset->uuid(), $webtask_id);
if (defined($webtask)) {
# Convenient.
$webtask->AutoStore(1);
}
if (DoSnapShotInternal($dataset, $instance, $bsname, $nodeid, \$errmsg)) {
goto failed;
}
if (PollDatasetStatus($dataset, \$errmsg)) {
# Exit and let child poll
exit(0);
}
$instance->Unlock();
$dataset->Unlock();
return 0;
......@@ -724,6 +708,64 @@ sub DoSnapShotInternal($$$$$)
$$perrmsg = $errmsg;
return -1;
}
#
# Poll for snapshot status.
#
sub PollDatasetStatus($$)
{
my ($dataset, $perrmsg) = @_;
my $project = $dataset->GetProject();
my $dname = $dataset->dataset_id();
my $logfile;
#
# If busy, then allocation is in progress. We leave it locked and
# poll in the background for a while, hoping for it to eventually
# stop being busy. Eventually might have to replace this, since
# polling got any non-small length of time will lead to trouble.
#
if (! $debug) {
$logfile = TBMakeLogname("createdataset");
if (my $childpid = TBBackGround($logfile)) {
return $childpid;
}
# Let parent exit;
sleep(2);
}
$webtask->SetProcessID($PID)
if (defined($webtask));
my $seconds = 1200;
my $interval = 5;
while ($seconds > 0) {
$seconds -= $interval;
if (DoRefreshInternal($dataset, $perrmsg)) {
print STDERR $$perrmsg;
sleep($interval);
next;
}
if ($dataset->state() eq "valid") {
$project->SendEmail($this_user->email(),
"Your dataset is now ready to use",
"Dataset '$dname' is now allocated and ready to use.\n",
$project->OpsEmailAddress(), undef, $logfile);
$webtask->Exited(0)
if (defined($webtask));
last;
}
sleep($interval);
}
$webtask->Exited(-1)
if (defined($webtask) && $seconds <= 0);
# unlink($logfile)
# if (defined($logfile));
return 0;
}
sub fatal($)
{
my ($mesg) = @_;
......
protogeni/lib/GeniCMV2.pm.in
View file @ 2bdbfa16
......@@ -2648,7 +2648,9 @@ dataset:
$slice->UnLock();
if ($user->email()) {
libtestbed::SENDMAIL($user->email(),
"Failed to clone image",
($image->isdataset() ?
"Failed to snapshot dataset" :
"Failed to clone image"),
"$output\n",
$user->email(),
"Bcc: $TBOPS");
......@@ -2656,15 +2658,17 @@ dataset:
return -1;
}
if ($user->email()) {
libtestbed::SENDMAIL($user->email(),
"Finished cloning image",
"Image URN: $image_urn\n".
"Image URL: $image_url\n".
"\n".
"-----------------------------------------\n".
"$output\n",
$user->email(),
"Bcc: $TBOPS");
libtestbed::SENDMAIL($user->email(),
($image->isdataset() ?
"Finished taking snapshot of dataset" :
"Finished cloning image"),
"Image URN: $image_urn\n".
"Image URL: $image_url\n".
"\n".
"-----------------------------------------\n".
"$output\n",
$user->email(),
"Bcc: $TBOPS");
}
return 0;
}
......@@ -3620,6 +3624,7 @@ sub DescribeDataset($)
require Lease;
require Blockstore;
require EmulabConstants;
require WebTask;
if (! (defined($credentials) && defined($dataset))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
......@@ -3676,14 +3681,45 @@ sub DescribeDataset($)
!defined($image->creator_urn()) ||
$image->creator_urn() ne $user->urn());
$blob->{'state'} = ($image->size() ? "valid" : "new");
print STDERR Dumper($image);
$blob->{'state'} = "valid";
$blob->{'type'} = "imdataset";
$blob->{"busy"} = $image->locked() ? 1 : 0;
$blob->{"busy"} = 0;
$blob->{'size'} = 0;
$blob->{'created'} = emutil::TBDateStringGMT($image->created());
$blob->{'updated'} = emutil::TBDateStringGMT($image->updated());
$blob->{'expires'} = "";
$blob->{'lastused'} = "";
#
# Is there an active webtask, then we are taking a snapshot, so
# report that info.
#
my $webtask = WebTask->LookupByObject($image->uuid());
if (defined($webtask)) {
$blob->{'image_size'} = $webtask->imagesize() . "KB";
$blob->{'image_status'} = $webtask->status();
#
# We have reported status, kill it. So if two parties are
# trying to determine the status, one gets nothing. Maybe
# we need a notion of staleness.
#
if ($webtask->HasExited()) {
# Final size.
if ($image->size()) {
my $kbytes = int(($image->lba_high() -
$image->lba_low() + 1) /
(1024 / $image->lba_size()));
$blob->{'image_size'} = $kbytes . "KB";
}
$webtask->Delete();
}
else {
$blob->{'state'} = "allocating";
$blob->{'busy'} = 1;
}
}
if ($image->size()) {
my $kbytes = int(($image->lba_high() -
$image->lba_low() + 1) /
......@@ -3691,6 +3727,7 @@ sub DescribeDataset($)
$blob->{'size'} =
Blockstore::ConvertToMebi("$kbytes" . "KB");
}
print STDERR Dumper($blob);
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
}
......
www/aptui/dataset.ajax
View file @ 2bdbfa16
......@@ -26,6 +26,7 @@ include_once("lease_defs.php");
include_once("imageid_defs.php");
include_once("blockstore_defs.php");
include_once("node_defs.php");
include_once("webtask.php");
chdir("apt");
include_once("dataset_defs.php");
include_once("instance_defs.php");
......@@ -57,6 +58,12 @@ function Do_CreateDataSet()
}
else {
$command = "webmanage_dataset create ";
# We are going to track imaging status.
if ($formfields["dataset_type"] == "imdataset") {
$webtask_id = WebTask::GenerateID();
$command .= " -t " . $webtask_id . " -- ";
}
}
$required = array("dataset_pid", "dataset_name", "dataset_type",
"dataset_fstype", "dataset_read", "dataset_modify");
......@@ -227,6 +234,14 @@ function Do_CreateDataSet()
$retval = SUEXEC($this_uid, $pid, $command, SUEXEC_ACTION_CONTINUE);
if ($retval) {
if (isset($webtask_id)) {
$webtask = WebTask::Lookup($webtask_id);
if ($webtask) {
$webtask->Delete();
SPITAJAX_ERROR(1, $webtask->TaskValue("output"));
return;
}
}
SPITAJAX_ERROR(1, $suexec_output);
return;
}
......@@ -395,22 +410,26 @@ function Do_ModifyDataSet()
if (isset($formfields["dataset_read"])) {
$perm = $formfields["dataset_read"];
$retval = SUEXEC($this_uid, $dataset->pid(),
"$command -R $perm $leaseid",
SUEXEC_ACTION_CONTINUE);
if ($retval) {
SPITAJAX_ERROR(1, $suexec_output);
return;
if ($perm != $dataset->read_access()) {
$retval = SUEXEC($this_uid, $dataset->pid(),
"$command -R $perm $leaseid",
SUEXEC_ACTION_CONTINUE);
if ($retval) {
SPITAJAX_ERROR(1, $suexec_output);
return;
}
}
}
if (isset($formfields["dataset_write"])) {
$perm = $formfields["dataset_write"];
$retval = SUEXEC($this_uid, $dataset->pid(),
"$command -W $perm $leaseid",
SUEXEC_ACTION_CONTINUE);
if ($retval) {
SPITAJAX_ERROR(1, $suexec_output);
return;
if ($perm != $dataset->write_access()) {
$retval = SUEXEC($this_uid, $dataset->pid(),
"$command -W $perm $leaseid",
SUEXEC_ACTION_CONTINUE);
if ($retval) {
SPITAJAX_ERROR(1, $suexec_output);
return;
}
}
}
if ($dataset->type() == "imdataset" && $nodeid) {
......@@ -672,7 +691,31 @@ function Do_GetInfo()
$blob["state"] = $dataset->state();
}
$blob["size"] = $dataset->size() ? $dataset->size() : "0";
#
# If we were generating an image.
#
$webtask = WebTask::LookupByObject($dataset->uuid());
if ($webtask) {
$taskdata = $webtask->TaskData();
#
# Size is in KB to avoid bigint problems. But kill the KB.
#
if (isset($taskdata["image_size"])) {
if (preg_match("/^(\d+)KB$/",
$taskdata["image_size"], $matches)) {
$taskdata["image_size"] = $matches[1];
}
$blob["image_size"] = $taskdata["image_size"];
}
else {
$blob["image_size"] = 0;
}
$blob["image_status"] = $taskdata["image_status"];
if ($webtask->exited()) {
$webtask->Delete();
}
}
SPITAJAX_RESPONSE($blob);
}
......
www/aptui/js/show-dataset.js
View file @ 2bdbfa16
require(window.APT_OPTIONS.configObject,
['underscore', 'js/quickvm_sup', 'moment',
'js/lib/text!template/show-dataset.html',
'js/lib/text!template/show-dataset.html', 'js/image',
'jquery-ui'],
function (_, sup, moment, mainString)
function (_, sup, moment, mainString, ShowImagingModal)
{
'use strict';
var mainTemplate = _.template(mainString);
......@@ -85,7 +85,7 @@ function (_, sup, moment, mainString)
*/
if (fields.dataset_state == "busy" ||
fields.dataset_state == "allocating") {
StateWatch();
ShowProgressModal();
}
}
......@@ -114,6 +114,21 @@ function (_, sup, moment, mainString)
xmlthing.done(callback);
}
function ShowProgressModal()
{
ShowImagingModal(function()
{
return sup.CallServerMethod(null,
"dataset",
"getinfo",
{"uuid" :
dataset_uuid});
},
function(failed)
{
});
}
//
// Delete dataset.
//
......
www/aptui/show-dataset.php
View file @ 2bdbfa16
......@@ -137,6 +137,9 @@ echo " window.CANAPPROVE = $canapprove;\n";
echo " window.CANREFRESH = $canrefresh;\n";
echo "</script>\n";
SPITREQUIRE("show-dataset");
# For progress bubbles in the imaging modal.
echo "<link rel='stylesheet' href='css/progress.css'>\n";
echo "<link rel='stylesheet' href='css/codemirror.css'>\n";
SPITFOOTER();
?>
www/aptui/template/show-dataset.html
View file @ 2bdbfa16
......@@ -197,4 +197,5 @@
</div>
</div>
</div>
<div id='imaging_div'></div>
</div>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment