Commit 4e97553c authored by Leigh Stoller's avatar Leigh Stoller

Improve admin extend page by getting rid of all the locking in the back

end so that everything can run in parallel. And, so the extend
button (among others) is not locked out from working while it loads
everything.

Also, do not copy the massive idledata strings through the database
(twice). Sometimes a file is just a good idea.
parent 13ee8406
......@@ -29,6 +29,7 @@ use Data::Dumper;
use CGI;
use POSIX ":sys_wait_h";
use POSIX qw(setsid strftime ceil floor);
use File::Temp qw(tempfile tmpnam);
use Date::Parse;
use JSON;
......@@ -3830,17 +3831,12 @@ sub DoUtilization()
my $exitcode = -1;
my $response;
my @aggregates = ();
my $results = {};
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
if ($slice->WaitForLock(10)) {
$errcode = GENIRESPONSE_BUSY;
$exitcode = 1;
$errmsg = "Experiment is busy, cannot lock it. Try again later.";
goto bad;
}
#
# Get the nodeid to client id mapping
......@@ -3918,12 +3914,12 @@ sub DoUtilization()
if ($debug) {
print Dumper($blob);
}
$agg->webtask()->results($blob);
$results->{$agg->aggregate_urn()} = $blob;
}
goto bad
if ($errcode);
$slice->UnLock();
$webtask->results($results);
exit(0);
bad:
print STDERR $errmsg . "\n";
......@@ -3931,7 +3927,6 @@ sub DoUtilization()
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
$slice->UnLock();
exit($exitcode);
}
......@@ -4029,17 +4024,27 @@ sub DoIdleData()
my $exitcode = -1;
my $response;
my @aggregates = ();
my %tempfiles = ();
my $optlist = "o:";
my $filename;
my $outfp;
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"o"})) {
$filename = $options{"o"}
}
usage()
if (@ARGV);
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
if ($slice->WaitForLock(10)) {
$errcode = GENIRESPONSE_BUSY;
$exitcode = 1;
$errmsg ="Experiment is busy, cannot lock it. Try again later.";
goto bad;
}
#
# Cull out any aggregates with no nodes.
#
......@@ -4049,9 +4054,41 @@ sub DoIdleData()
}
#
# Helper callback; if we get back the idledata lets set it here (in
# the child) and avoid the copy through the temp webtask/database.
# Clear the response structure to prevent that.
# Open the output file for writing.
#
if (defined($filename)) {
if (!open(FOUT, "> $filename")) {
fatal("Could not open $filename for writing: $!");
}
$outfp = *FOUT;
}
else {
$outfp = *STDOUT;
}
#
# If more then one aggregate, we pass the data back from the
# parrun children via temp files, rather then go through the DB
# via a webtask (the data can be very big). With one aggregate we
# do not fork, so we can put the data directly into the webtask
# that comes in on the command line.
#
if (@aggregates > 0) {
foreach my $aggregate (@aggregates) {
my ($fp, $filename) = tempfile("/tmp/idledata.XXXXX", UNLINK => 1);
if (!defined($fp)) {
fatal("Could not create temporary file for idle data");
}
$tempfiles{$aggregate->aggregate_urn()} = [$fp, $filename];
}
}
#
# Helper callback; if we get back the idledata write it to the
# temporary file for the parent to pick up.
#
# Clear the response structure to prevent the data from going to
# database.
#
my $coderef = sub {
my ($sliver) = @_;
......@@ -4062,8 +4099,11 @@ sub DoIdleData()
if ($debug) {
print STDERR $json . "\n";
}
# This might be an empty string if no nodes allocated.
$sliver->webtask()->idledata($json);
if (@aggregates > 0) {
my ($fp) = @{ $tempfiles{$sliver->aggregate_urn()} };
print $fp $json . "\n";
close($fp);
}
$response->value(0);
}
return $response;
......@@ -4092,13 +4132,28 @@ sub DoIdleData()
else {
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
}
next;
# Give up now.
last;
}
print $outfp "" . $agg->aggregate_urn() . "\n";
if (@aggregates > 0) {
my (undef, $filename) = @{ $tempfiles{$agg->aggregate_urn()} };
if (open(TEMP, $filename)) {
while (<TEMP>) {
print $outfp $_;
}
close(TEMP);
}
else {
$exitcode = -1;
$errcode = -1;
$errmsg = "Could not open temporary file to read idle data";
}
}
}
goto bad
if ($errcode);
$slice->UnLock();
exit(0);
bad:
print STDERR $errmsg . "\n";
......@@ -4106,7 +4161,6 @@ sub DoIdleData()
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
$slice->UnLock();
exit($exitcode);
}
......@@ -4984,10 +5038,15 @@ sub CallMethodOnAggregates($$$@)
#
# Create anonymous webtasks to return the value. Note that for the
# single aggregate case (no parrun) we still create a webtask, but
# do not set the AutoStore flag, which prevents the value from
# actually going to the db. Which makes the interface a little
# bit more consistent regardless of the number of aggregates.
# single aggregate case (no parrun) we still create a webtask,
# which which makes the interface a little bit more consistent
# regardless of the number of aggregates. Also note that we do
# not set AutoStore, in case the callee messes with it, we flush it
# by hand below.
#
# NOTE: This also means we are independent of other calls that
# are using anonymous webtasks instead of the instance webtask.
# or per-aggregate webtask. No locking required.
#
my @webtasks = ();
foreach my $agg (@aggregates) {
......@@ -4996,8 +5055,6 @@ sub CallMethodOnAggregates($$$@)
print STDERR "Could not create an anonymous webtask!\n";
return -1;
}
$webtask->AutoStore(1)
if (@aggregates > 1);
push(@webtasks, $webtask);
}
......@@ -5070,6 +5127,7 @@ sub CallMethodOnAggregates($$$@)
}
# Need unblessed ref to store into webtask.
$webtask->response($response->Unbless());
$webtask->Store();
return ($response->code() == GENIRESPONSE_SUCCESS ? 0 : -1);
};
my @return_codes = ();
......@@ -5105,8 +5163,6 @@ sub CallMethodOnAggregates($$$@)
foreach my $agg (@aggregates) {
my $webtask = shift(@webtasks);
print "$agg\n";
# No need to refresh if we did not use ParRun above.
$webtask->Refresh() if (@aggregates > 1);
push(@return_values, GeniResponse->Bless($webtask->response()));
......
<?php
#
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
# Copyright (c) 2000-2019 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -97,6 +97,7 @@ if (!ISADMIN()) {
SPITUSERERROR("You do not have permission to view this information!");
return;
}
$started = $instance->started() ? "true" : "false";
echo "<link rel='stylesheet'
href='css/tablesorter.css'>\n";
......@@ -106,6 +107,7 @@ echo " window.UUID = '" . $uuid . "';\n";
echo " window.PID = '" . $pid . "';\n";
echo " window.CREATOR = '" . $creator . "';\n";
echo " window.HOURS = $hours;\n";
echo " window.STARTED = $started;\n";
echo "</script>\n";
echo "<link rel='stylesheet'
......
......@@ -34,18 +34,13 @@ $(function ()
extensionsTemplate = _.template(historyString);
listTemplate = _.template(templates["reservation-list"]);
ReloadFirstRow(function () {
if (expinfo.started) {
$('#extension-controls').removeClass("hidden");
// Need to serialize this stuff cause of locking in the backend.
LoadUtilization(function () {
LoadIdleData(function () {
LoadOpenStack();
});
});
}
});
ReloadFirstRow();
if (window.STARTED) {
$('#extension-controls').removeClass("hidden");
LoadIdleData();
LoadUtilization();
LoadOpenStack();
}
// Second row is the user/project usage summarys. We make two calls
// and use jquery "when" to wait for both to finish before running
......@@ -277,7 +272,7 @@ $(function ()
.format("MMM D, YYYY h:mm A"));
}
});
if (!expinfo.started) {
if (!window.STARTED) {
// Disable the flags.
$('#lockout-checkbox, #user-lockdown-checkbox, ' +
'#admin-lockdown-checkbox, #quarantine-checkbox')
......@@ -319,7 +314,7 @@ $(function ()
SetupAdminNotes();
}
function ReloadFirstRow(continuation)
function ReloadFirstRow()
{
sup.CallServerMethod(null, "status", "ExpInfo",
{"uuid" : window.UUID},
......@@ -328,14 +323,12 @@ $(function ()
if (json.code == 0) {
expinfo = json.value;
LoadFirstRow();
continuation();
}
});
}
function LoadUtilization(continuation) {
console.info("LoadUtilization", continuation);
if (!expinfo.started) {
function LoadUtilization() {
if (!window.STARTED) {
return;
}
var utilizationTemplate = _.template(utilizationString);
......@@ -343,10 +336,6 @@ $(function ()
var callback = function(json) {
console.info("LoadUtilization", json);
// Fire off the next part.
if (continuation !== undefined) {
continuation();
}
if (json.code) {
console.info("Could not load utilization");
$("#thirdrow .thirdrow-error .well")
......@@ -592,11 +581,11 @@ $(function ()
//
// Get Max Extension and update the table.
//
function DoMaxExtension(expires, continuation)
function DoMaxExtension(expires)
{
console.info("DoMaxExtension", expires, continuation);
console.info("DoMaxExtension", expires);
if (! expinfo.started) {
if (! window.STARTED) {
$('#max-extension').html("<span class='text-warning'>" +
"Not Started Yet</span>");
return;
......@@ -604,9 +593,6 @@ $(function ()
// Warn if changing days violates max extension.
var callback = function(json) {
if (continuation !== undefined) {
continuation();
}
$("#howlong").on("keyup", function (event) {
if (!maxextension) {
$('#max-extension-nomax').removeClass("hidden");
......@@ -797,12 +783,9 @@ $(function ()
//
// Slothd graphs.
//
function LoadIdleData(continuation)
function LoadIdleData()
{
console.info("LoadIdleData", continuation);
var callback = function (status, json) {
console.info("LoadIdleData callback");
if (status <= 0) {
if (status == 0) {
// No data.
......@@ -816,9 +799,6 @@ $(function ()
$('#idledata-error').removeClass("hidden");
}
}
if (continuation !== undefined) {
continuation();
}
};
ShowIdleGraphs({"uuid" : window.UUID,
"showwait" : false,
......@@ -833,8 +813,6 @@ $(function ()
//
function LoadOpenStack()
{
console.info("LoadIdleData");
var callback = function(json) {
if (json.code) {
return;
......
......@@ -384,6 +384,7 @@ window.ShowIdleGraphs = (function ()
}
return;
}
//console.info("rpc", json);
_.each(json.value, function(data, name) {
// No data, skip
if (data == "") {
......@@ -391,7 +392,7 @@ window.ShowIdleGraphs = (function ()
}
rawData[name] = JSON.parse(data);
});
console.info("raw", rawData);
//console.info("raw", rawData);
// No data, tell caller and done.
if (Object.keys(rawData).length == 0) {
......
......@@ -1727,19 +1727,17 @@ function Do_Utilization()
$webtask->Delete();
return;
}
$webtask->Delete();
$results = $webtask->TaskValue("results");
$blob = array();
# Look at per sliver.
foreach ($instance->slivers() as $sliver) {
if ($sliver->webtask_id() &&
$webtask = WebTask::Lookup($sliver->webtask_id())) {
if (!($sliver->physnode_count() || $sliver->virtnode_count())) {
continue;
}
$blob[$urn_mapping[$sliver->aggregate_urn()]] =
$webtask->TaskValue("results");
if (!($sliver->physnode_count() || $sliver->virtnode_count())) {
continue;
}
$blob[$urn_mapping[$sliver->aggregate_urn()]] =
$results[$sliver->aggregate_urn()];
}
$webtask->Delete();
SPITAJAX_RESPONSE($blob);
}
......@@ -1760,11 +1758,20 @@ function Do_IdleData()
SPITAJAX_ERROR(1, "no slice for instance");
return 1;
}
#
# This can be a lot of data, so lets not go through the DB
# (via the webtask). Pass a temp file name to the backend.
#
$filename = tempnam("/tmp", "idledata");
$fp = fopen($filename, "w+");
chmod($filename, 0666);
$webtask = WebTask::CreateAnonymous();
# XXX Need to do this as elabman cause of emulab.key. Pondering.
$retval = SUEXEC("elabman", $TBADMINGROUP,
"webmanage_instance -t " . $webtask->task_id() . " -- ".
" idledata $uuid",
" idledata $uuid -o $filename",
SUEXEC_ACTION_IGNORE);
$webtask->Refresh();
......@@ -1779,27 +1786,35 @@ function Do_IdleData()
$webtask->Delete();
return;
}
$webtask->Delete();
$blob = array();
# Look at per sliver.
foreach ($instance->slivers() as $sliver) {
if ($sliver->webtask_id() &&
$webtask = WebTask::Lookup($sliver->webtask_id())) {
if (!($sliver->physnode_count() || $sliver->virtnode_count())) {
continue;
}
$json = $webtask->TaskValue("idledata");
# We might not get any data if no nodes at the target cluster.
# This happens occasionally.
if (!$json || $json == "") {
continue;
while (($urn = fgets($fp, 0x1000)) !== false) {
$json = "";
#
# First line is an aggregate urn. Now suck up the next very
# big line until the newline.
#
while (($buffer = fgets($fp, 0x4000)) !== false) {
$json = $json . $buffer;
if ($buffer[strlen($buffer)-1] == "\n") {
break;
}
# Send back the raw json.
$blob[$urn_mapping[$sliver->aggregate_urn()]] = $json;
}
$urn = rtrim($urn);
$json = rtrim($json);
# This happens occasionally.
if ($json == "") {
continue;
}
# Send back the raw json.
$blob[$urn_mapping[$urn]] = $json;
}
fclose($fp);
#unlink($filename);
SPITAJAX_RESPONSE($blob);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment