Commit 2a9a5840 authored by Leigh B Stoller's avatar Leigh B Stoller

Fix up race condition when dealing with multi aggregate experiments.

This race always existed cause there was no locking, it just didn't
manifest since there are not many multi-aggregate experiments.
parent 7edea7b4
......@@ -1447,12 +1447,29 @@ sub DoExtend()
$errcode = 1;
goto bad;
}
if (!TBcheck_dbslot($wanted, "default", "int",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
$errmsg = "Illegal integer for length";
$errcode = 1;
goto bad;
# Lets allow a date too.
if ($wanted !~ /^\d+$/) {
my $when = str2time($wanted);
if (!$when) {
$errmsg = "Illegal number of days or date";
$errcode = 1;
goto bad;
}
if ($when < time() + 3600) {
$errmsg = "Expiration is too soon";
$errcode = 1;
goto bad;
}
my $hours = int(($when - time()) / 3600);
if ($hours < 24) {
$wanted = $hours;
$inhours = 1;
}
else {
$wanted = int($hours / 24);
}
}
# Helper function.
my $needAdminApproval = sub {
my ($wanted, $granted, $reason, $message) = @_;
......@@ -3649,7 +3666,18 @@ sub DoUtilization()
{
my $utilization;
my $errmsg;
my $errcode;
my @aggregates = ();
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
if ($slice->Lock()) {
$errcode = GENIRESPONSE_BUSY;
$errmsg ="Experiment is busy, cannot lock it. Try again later.";
goto bad;
}
#
# Get the nodeid to client id mapping
......@@ -3688,7 +3716,7 @@ sub DoUtilization()
$client_ids{$aggregate->aggregate_urn()}->{$node_id} = $client_id;
}
}
my $errcode = CallAggregateMethod("Utilization",\$utilization,@aggregates);
$errcode = CallAggregateMethod("Utilization",\$utilization,@aggregates);
if ($errcode) {
$errmsg = $utilization;
goto bad;
......@@ -3707,6 +3735,7 @@ sub DoUtilization()
}
$aggregate->webtask()->results($blob);
}
$slice->UnLock();
exit(0);
bad:
print STDERR $errmsg . "\n";
......@@ -3714,6 +3743,7 @@ sub DoUtilization()
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
$slice->UnLock();
exit($errcode);
}
......@@ -3807,8 +3837,18 @@ sub DoIdleData()
{
my $idledata;
my $errmsg;
my $errcode;
my @aggregates = ();
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
if ($slice->Lock()) {
$errcode = GENIRESPONSE_BUSY;
$errmsg ="Experiment is busy, cannot lock it. Try again later.";
goto bad;
}
#
# Cull out any aggregates with no nodes.
#
......@@ -3816,7 +3856,7 @@ sub DoIdleData()
push(@aggregates, $aggregate)
if ($aggregate->physnode_count() || $aggregate->virtnode_count());
}
my $errcode = CallAggregateMethod("IdleData", \$idledata, @aggregates);
$errcode = CallAggregateMethod("IdleData", \$idledata, @aggregates);
if ($errcode) {
$errmsg = $idledata;
goto bad;
......@@ -3829,6 +3869,7 @@ sub DoIdleData()
}
$aggregate->webtask()->idledata($json);
}
$slice->UnLock();
exit(0);
bad:
print STDERR $errmsg . "\n";
......@@ -3836,6 +3877,7 @@ sub DoIdleData()
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
$slice->UnLock();
exit($errcode);
}
......
......@@ -31,10 +31,14 @@ $(function ()
secondrowTemplate = _.template(secondrowString);
extensionsTemplate = _.template(historyString);
LoadUtilization();
LoadIdleData();
LoadFirstRow();
LoadOpenStack();
// Need to serialize this stuff cause of locking in the backend.
LoadFirstRow(function () {
LoadUtilization(function () {
LoadIdleData(function () {
LoadOpenStack();
});
});
});
// Second row is the user/project usage summarys. We make two calls
// and use jquery "when" to wait for both to finish before running
......@@ -161,7 +165,7 @@ $(function ()
}
// First Row is the experiment summary info.
function LoadFirstRow() {
function LoadFirstRow(continuation) {
sup.CallServerMethod(null, "status", "ExpInfo", {"uuid" : window.UUID},
function (json) {
console.info(json);
......@@ -210,18 +214,24 @@ $(function ()
.attr("disabled", "disabled");
}
// Update the Max Extension
DoMaxExtension(json.value.expires);
DoMaxExtension(json.value.expires,
continuation);
SetupAdminNotes();
}
});
}
function LoadUtilization() {
function LoadUtilization(continuation) {
console.info("LoadUtilization", continuation);
var utilizationTemplate = _.template(utilizationString);
var summaryTemplate = _.template(summaryString);
var callback = function(json) {
console.info(json);
console.info("LoadUtilization", json);
// Fire off the next part.
if (continuation !== undefined) {
continuation();
}
if (json.code) {
console.info("Could not load utilization");
return;
......@@ -392,10 +402,15 @@ $(function ()
//
// Get Max Extension and update the table.
//
function DoMaxExtension(expires)
function DoMaxExtension(expires, continuation)
{
console.info("DoMaxExtension", expires, continuation);
// Warn if changing days violates max extension.
var callback = function(json) {
if (continuation !== undefined) {
continuation();
}
$("#days").on("keyup", function (event) {
if (!maxextension) {
$('#max-extension-nomax').removeClass("hidden");
......@@ -497,13 +512,22 @@ $(function ()
//
// Slothd graphs.
//
function LoadIdleData()
function LoadIdleData(continuation)
{
console.info("LoadIdleData", continuation);
var callback = function (gotdata) {
console.info("LoadIdleData callback");
if (continuation !== undefined) {
continuation();
}
};
ShowIdleGraphs({"uuid" : window.UUID,
"showwait" : false,
"loadID" : "#loadavg-panel-div",
"ctrlID" : "#ctrl-traffic-panel-div",
"exptID" : "#expt-traffic-panel-div"});
"exptID" : "#expt-traffic-panel-div",
"callback" : callback});
}
//
......@@ -511,6 +535,8 @@ $(function ()
//
function LoadOpenStack()
{
console.info("LoadIdleData");
var callback = function(json) {
if (json.code) {
return;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment