Commit 6762e839 authored by Leigh Stoller's avatar Leigh Stoller

Two unrelated changes.

1. Use information from sliverstatus to indicate that nodes have startup
   execution services running, and then tell then when they have finished,
   and they exited with non-zero status, indicate that they failed. We also
   hold saying the "ready" in the upper panel until all the services have
   exited, we say "booted" instead, and also say that nodes are running
   startup services.

2. For snapshot, when we know an image has to be copied back to its origin
   cluster, tell the web interface, so that we can add another step to the
   imaging modal ("copying"). We know the copy is done when the origin
   cluster has posted the new image data to the IMS, so we do an additional
   poll in the backend waiting for the image server to get the data, and
   then we mark the image as ready for use.
parent fbd055d8
......@@ -49,6 +49,7 @@ my $verbose = 1;
my $DEFAULT_URN = "urn:publicid:IDN+apt.emulab.net+authority+cm";
my $xmlfile;
my $webtask;
my $webtask_id;
my $localuser = 0;
my $usestitcher= 0;
my $quickuuid;
......@@ -62,7 +63,7 @@ my $usetracker = 0;
my @aggregate_urns = ();
# Debugging
my $usemydevtree = 0;
my $usemydevtree = 1;
# Protos
sub fatal($);
......@@ -91,6 +92,7 @@ my $ADDPUBKEY = "$TB/sbin/addpubkey";
my $UPDATEGENIUSER= "$TB/sbin/protogeni/updategeniuser";
my $STITCHER = "$TB/gcf/src/stitcher.py";
my $OPENSSL = "/usr/bin/openssl";
my $MANAGEINSTANCE= "$TB/bin/manage_instance";
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
......@@ -751,6 +753,7 @@ $webtask = WebTask->Create($instance->uuid());
if (!defined($webtask)) {
fatal("Could not create a webtask!");
}
$webtask_id = $webtask->task_id();
$webtask->AutoStore(1);
print STDERR "\n";
......@@ -937,6 +940,9 @@ if (ParRun({"maxwaittime" => 99999, "maxchildren" => scalar(@aggregate_list)},
}
print "$slice_urn\n";
# Count up nodes running a startup service.
my $startuprunning = 0;
#
# Check the exit codes; any failure is a total failure (for now).
#
......@@ -954,7 +960,6 @@ foreach my $aggobj (@aggregate_list) {
$webtask->output($aggobj->webtask()->output());
$webtask->Exited($aggobj->webtask()->exitcode());
print $aggobj->webtask()->output() . "\n";
}
else {
$webtask->output("WaitforSliver Failure at " .
......@@ -962,6 +967,18 @@ foreach my $aggobj (@aggregate_list) {
$webtask->Exited(1);
}
}
else {
my $statusblob = $aggobj->webtask()->sliverstatus();
print Dumper($statusblob);
foreach my $details (values(%{ $statusblob })) {
# Startup command is still running.
$startuprunning++
if (exists($details->{'execute_state'}) &&
$details->{'execute_state'} ne "exited");
}
}
if (defined($aggobj->public_url())) {
print $aggobj->public_url() . "\n";
}
......@@ -976,7 +993,19 @@ if ($failed) {
}
else {
$instance->SetStatus("ready");
$webtask->Exited(0);
#
# If there are still execute services running, lets keep polling
# using the monitor.
#
if ($startuprunning) {
print "$MANAGEINSTANCE -t $webtask_id monitor $quickvm_uuid -w\n";
system("$MANAGEINSTANCE -t $webtask_id monitor $quickvm_uuid -w")
}
else {
$webtask->Exited(0);
}
}
exit(0);
......
......@@ -108,6 +108,7 @@ sub DoLockdown();
sub DoManifests();
sub WriteCredentials();
sub StartMonitor();
sub StartMonitorInternal(;$);
sub DoImageTrackerStuff($$$$$$);
#
......@@ -481,15 +482,28 @@ sub DoSnapshot()
if (defined($webtask)) {
$webtask->image_urn($version_urn);
$webtask->image_url($version_url);
my $image_name;
# DoImageTrackerStuff determined that we use whatever the cluster
# tells us, cause it is the home of the image.
my $copyback_urn = $version_urn
if ($usetracker && !defined($copyback_urn));
if ($usetracker) {
# DoImageTrackerStuff determined that we use whatever the cluster
# tells us, cause it is the home of the image.
if (!defined($copyback_urn)) {
$image_name = $version_urn;
}
else {
$image_name = $copyback_urn;
}
}
else {
$image_name = $version_url;
}
$webtask->image_name($image_name);
# For the web interface.
$webtask->image_name((defined($copyback_urn) ?
$copyback_urn : $version_url));
# We tell the web interface that the image has to be copied
# back,
if (defined($copyback_uuid)) {
$webtask->copyback_uuid($copyback_uuid);
}
}
else {
print "$image_urn,$image_url\n";
......@@ -590,8 +604,20 @@ sub DoSnapshot()
if (defined($webtask)) {
$webtask->image_size($blob->{'size'})
if (exists($blob->{'size'}));
$webtask->image_status($blob->{'status'})
if (exists($blob->{'status'}));
if (exists($blob->{'status'})) {
#
# If the image is ready, but needs to be copied back to
# its origin, hold of ready till later. We will wait for
# the copyback to finish, see below.
#
if (defined($copyback_uuid)) {
$webtask->image_status("copying");
}
else {
$webtask->image_status($blob->{'status'});
}
}
}
if ($blob->{'status'} eq "ready") {
$ready = 1;
......@@ -639,6 +665,45 @@ sub DoSnapshot()
($update_profile eq "all" ? 1 : 0));
}
$instance->SetStatus("ready");
#
# If there is a copyback_uuid, we want to wait for that to finish.
#
if (defined($copyback_uuid)) {
#
# We know the copyback is done when the IMS has the info.
#
my $copied = 0;
$seconds = 1000;
while ($seconds > 0) {
sleep($interval);
$seconds -= $interval;
#
# It would clearly be more more efficient to just look in
# the IMS database.
#
Genixmlrpc->SetContext(APT_Geni::GeniContext());
my $blob = GeniImage::GetImageData($copyback_urn, \$errmsg);
Genixmlrpc->SetContext(undef);
# We get back undefined if the image is not posted yet.
if (defined($blob)) {
$copied = 1;
last;
}
sleep($interval);
}
# Tell the web interface.
if (!$copied) {
$errmsg = "Failed to copy image back to its origin cluster";
$errcode = 1;
goto bad;
}
elsif (defined($webtask)) {
$webtask->image_status("ready");
}
}
# We garbage collect these later, so anyone waiting has a chance
# to see the exit status
$webtask->Exited(0)
......@@ -656,7 +721,7 @@ sub DoSnapshot()
# Image is ready, but sliver is not. Start a monitor so that
# web interface is updated.
#
StartMonitor();
StartMonitorInternal();
}
exit(0);
bad:
......@@ -665,7 +730,7 @@ sub DoSnapshot()
# Image is ready, but sliver is not. Start a monitor so that
# web interface is updated.
#
StartMonitor();
StartMonitorInternal();
}
$instance->SetStatus("ready");
if (defined($logfile)) {
......@@ -1452,7 +1517,7 @@ sub DoRebootOrReload($)
# has actually come back up.
#
# XXX This will not return unless a monitor is already running.
StartMonitor();
StartMonitorInternal();
exit(0);
killit:
$instance->RecordHistory($RECORDHISTORY_TERMINATED);
......@@ -1530,8 +1595,32 @@ sub DoManifests()
#
sub StartMonitor()
{
my $waitforstartup = 0;
if (@ARGV && $ARGV[0] eq "-w") {
$waitforstartup = 1;
}
if (defined($webtask_id)) {
$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
if (!defined($webtask)) {
fatal("Could not lookup/create webtask for $webtask_id");
}
# Convenient.
$webtask->AutoStore(1);
}
return StartMonitorInternal($waitforstartup)
}
sub StartMonitorInternal(;$)
{
my ($waitforstartup) = @_;
my $logfile;
my $signaled = 0;
# Wait for the startup command to finish.
$waitforstartup = 0
if (!defined($waitforstartup));
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
......@@ -1562,7 +1651,7 @@ sub StartMonitor()
#
sleep(30);
my $seconds = 1500;
my $seconds = ($waitforstartup ? 3600 : 900);
my $interval = 15;
# Shorten default timeout now.
Genixmlrpc->SetTimeout(30);
......@@ -1578,6 +1667,7 @@ sub StartMonitor()
return GENIRESPONSE_RPCERROR;
}
if (($response->code() != GENIRESPONSE_SUCCESS &&
$response->code() != GENIRESPONSE_SERVER_UNAVAILABLE &&
$response->code() != GENIRESPONSE_BUSY)) {
print STDERR "SliverStatus failed";
print STDERR ": " . $response->output() . "\n";
......@@ -1603,13 +1693,19 @@ sub StartMonitor()
# Convert to something smaller, with info the web interface
# cares about.
#
my $statusblob = {};
my $statusblob = {};
my $executing = 0;
foreach my $urn (keys(%{$blob->{'details'}})) {
my $details = $blob->{'details'}->{$urn};
my $node_id = $details->{'client_id'};
$statusblob->{$node_id} = $details;
# Startup command is still running.
$executing++
if (exists($details->{'execute_state'}) &&
$details->{'execute_state'} ne "exited");
}
if ($debug) {
if (1) {
print STDERR Dumper($statusblob);
}
$webtask->sliverstatus($statusblob);
......@@ -1618,7 +1714,8 @@ sub StartMonitor()
# We poll until the status goes ready. Might not be a good idea.
#
if ($blob->{'status'} eq "ready") {
return 0;
return 0
if (!$executing || !$waitforstartup);
}
# Not done yet.
return 1;
......@@ -1672,8 +1769,8 @@ sub StartMonitor()
sleep($interval);
}
unlink($logfile)
if (defined($logfile));
# unlink($logfile)
# if (defined($logfile) && !$debug);
exit(0);
}
......
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="305.002px" height="305.002px" viewBox="0 0 305.002 305.002" style="enable-background:new 0 0 305.002 305.002;"
xml:space="preserve">
<g>
<g>
<g>
<path d="M152.502,305.001C68.411,305.001,0,236.59,0,152.501s68.411-152.5,152.502-152.5c84.089,0,152.5,68.411,152.5,152.5
S236.591,305.001,152.502,305.001z M152.502,25.001C82.197,25.001,25,82.197,25,152.501c0,70.305,57.197,127.5,127.502,127.5
c70.304,0,127.5-57.195,127.5-127.5C280.002,82.197,222.806,25.001,152.502,25.001z"/>
</g>
<g>
<path d="M225.998,165.001H79.006c-6.903,0-12.5-5.598-12.5-12.5c0-6.903,5.597-12.5,12.5-12.5h146.992
c6.903,0,12.5,5.597,12.5,12.5C238.498,159.403,232.901,165.001,225.998,165.001z"/>
</g>
</g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
</svg>
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="305.002px" height="305.002px" viewBox="0 0 305.002 305.002" style="enable-background:new 0 0 305.002 305.002;"
xml:space="preserve">
<g>
<g>
<path d="M152.502,0.001C68.412,0.001,0,68.412,0,152.501s68.412,152.5,152.502,152.5c84.089,0,152.5-68.411,152.5-152.5
S236.591,0.001,152.502,0.001z M152.502,280.001C82.197,280.001,25,222.806,25,152.501c0-70.304,57.197-127.5,127.502-127.5
c70.304,0,127.5,57.196,127.5,127.5C280.002,222.806,222.806,280.001,152.502,280.001z"/>
<path d="M170.18,152.5l43.13-43.129c4.882-4.882,4.882-12.796,0-17.678c-4.881-4.882-12.796-4.881-17.678,0l-43.13,43.13
l-43.131-43.131c-4.882-4.881-12.796-4.881-17.678,0c-4.881,4.882-4.881,12.796,0,17.678l43.13,43.13l-43.131,43.131
c-4.881,4.882-4.881,12.796,0,17.679c2.441,2.44,5.64,3.66,8.839,3.66c3.199,0,6.398-1.221,8.839-3.66l43.131-43.132
l43.131,43.132c2.441,2.439,5.64,3.66,8.839,3.66s6.398-1.221,8.839-3.66c4.882-4.883,4.882-12.797,0-17.679L170.18,152.5z"/>
</g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
</svg>
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="305.002px" height="305.002px" viewBox="0 0 305.002 305.002" style="enable-background:new 0 0 305.002 305.002;"
xml:space="preserve">
<g>
<g>
<path d="M152.502,0.001C68.412,0.001,0,68.412,0,152.501s68.412,152.5,152.502,152.5c84.089,0,152.5-68.411,152.5-152.5
S236.591,0.001,152.502,0.001z M152.502,280.001C82.197,280.001,25,222.806,25,152.501c0-70.304,57.197-127.5,127.502-127.5
c70.304,0,127.5,57.196,127.5,127.5C280.002,222.806,222.806,280.001,152.502,280.001z"/>
<path d="M218.473,93.97l-90.546,90.547l-41.398-41.398c-4.882-4.881-12.796-4.881-17.678,0c-4.881,4.882-4.881,12.796,0,17.678
l50.237,50.237c2.441,2.44,5.64,3.661,8.839,3.661c3.199,0,6.398-1.221,8.839-3.661l99.385-99.385
c4.881-4.882,4.881-12.796,0-17.678C231.269,89.089,223.354,89.089,218.473,93.97z"/>
</g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
</svg>
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="512px" height="512px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve">
<g>
<path d="M256,0C114.609,0,0,114.609,0,256c0,141.391,114.609,256,256,256c141.391,0,256-114.609,256-256
C512,114.609,397.391,0,256,0z M256,472c-119.297,0-216-96.703-216-216S136.703,40,256,40s216,96.703,216,216S375.297,472,256,472z
"/>
<path d="M336,256c0,44.188-35.812,80-80,80c-44.188,0-80-35.812-80-80c0-44.188,35.812-80,80-80C300.188,176,336,211.812,336,256z"
/>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
</svg>
......@@ -77,11 +77,21 @@ define(['underscore', 'js/quickvm_sup', 'filesize',
$('#tracker-finishing').removeClass('progtrckr-todo');
$('#tracker-finishing').addClass('progtrckr-done');
}
else if (status == "copying") {
$('#tracker-imaging').removeClass('progtrckr-todo');
$('#tracker-imaging').addClass('progtrckr-done');
$('#tracker-finishing').removeClass('progtrckr-todo');
$('#tracker-finishing').addClass('progtrckr-done');
$('#tracker-copying').removeClass('progtrckr-todo');
$('#tracker-copying').addClass('progtrckr-done');
}
else if (status == "ready") {
$('#tracker-imaging').removeClass('progtrckr-todo');
$('#tracker-imaging').addClass('progtrckr-done');
$('#tracker-finishing').removeClass('progtrckr-todo');
$('#tracker-finishing').addClass('progtrckr-done');
$('#tracker-copying').removeClass('progtrckr-todo');
$('#tracker-copying').addClass('progtrckr-done');
$('#tracker-ready').removeClass('progtrckr-todo');
$('#tracker-ready').addClass('progtrckr-done');
$('#imaging-spinner').addClass("hidden");
......@@ -108,10 +118,17 @@ define(['underscore', 'js/quickvm_sup', 'filesize',
$('#tracker-imaging').removeClass('progtrckr-todo');
$('#tracker-imaging').addClass('progtrckr-failed');
}
if (laststatus == "imaging" || laststatus == "preparing") {
if (laststatus == "imaging" ||
laststatus == "preparing") {
$('#tracker-finishing').removeClass('progtrckr-todo');
$('#tracker-finishing').addClass('progtrckr-failed');
}
if (laststatus == "imaging" ||
laststatus == "preparing" ||
laststatus == "finishing") {
$('#tracker-copying').removeClass('progtrckr-todo');
$('#tracker-copying').addClass('progtrckr-failed');
}
$('#tracker-ready').removeClass('progtrckr-todo');
$('#tracker-ready').addClass('progtrckr-failed');
......@@ -150,11 +167,21 @@ define(['underscore', 'js/quickvm_sup', 'filesize',
if (imagingTemplate == null) {
imagingTemplate = _.template(imagingString);
}
var imaging_html = imagingTemplate({});
$('#imaging_div').html(imaging_html);
imaging_modal_display = true;
ShowImagingModal();
var callback = function(json) {
var value = json.value;
console.log("ShowImagingModal Startup");
console.log(json);
var imaging_html = imagingTemplate({
needcopy : _.has(json.value, "copyback_uuid")});
$('#imaging_div').html(imaging_html);
imaging_modal_display = true;
ShowImagingModal();
};
var $xmlthing = status_callback();
$xmlthing.done(callback);
}
}
);
......@@ -335,7 +335,7 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
// Call back for above.
function StatusWatchCallBack(uuid, json)
{
//console.info(json);
console.info(json);
// Flag to indicate that we have seen ready and do not
// need to do initial stuff. We need this cause the
......@@ -383,7 +383,14 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
else if (status == 'ready') {
bgtype = "panel-success";
status_message = "Your experiment is ready!";
status_html = "<font color=green>ready</font>";
if (servicesExecuting(json.value)) {
status_html = "<font color=green>booted</font>";
status_html += " (startup services are still running)";
}
else {
status_html = "<font color=green>ready</font>";
}
if ($("#status_progress_div").length) {
$("#status_progress_div").removeClass("progress-striped");
$("#status_progress_div").removeClass("active");
......@@ -450,6 +457,17 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
.addClass(bgtype);
$("#quickvm_status").html(status_html);
}
else if (lastStatus == "ready" && status == "ready") {
if (servicesExecuting(json.value)) {
status_html = "<font color=green>booted</font>";
status_html += " (startup services are still running)";
}
else {
status_html = "<font color=green>ready</font>";
}
$("#quickvm_status").html(status_html);
}
//
// Look for a sliverstatus blob.
//
......@@ -687,6 +705,17 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
{"uuid" : uuid});
xmlthing.done(callback);
}
var svgimg = document.createElementNS('http://www.w3.org/2000/svg','image');
svgimg.setAttributeNS(null,'class','node-status-icon');
svgimg.setAttributeNS(null,'height','15');
svgimg.setAttributeNS(null,'width','15');
svgimg.setAttributeNS('http://www.w3.org/1999/xlink','href',
'fonts/record8.svg');
svgimg.setAttributeNS(null,'x','13');
svgimg.setAttributeNS(null,'y','-23');
svgimg.setAttributeNS(null, 'visibility', 'visible');
// Helper for above and called from the status callback.
function UpdateSliverStatus(oblob)
{
......@@ -707,17 +736,68 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
$('#' + jacksIDs[node_id] + ' .node .nodebox')
.css("fill", "#fcf8e3");
}
var html =
"<table class='table table-condensed'><tbody> " +
"<tr><td>Node:</td><td>" +
details.component_urn + "</td></tr>" +
"<tr><td>Status:</td><td>" +
details.status + "</td></tr>" +
"<tr><td>State:</td><td>" +
details.state + "</td></tr>" +
"<tr><td>Raw State:</td><td>" +
details.rawstate + "</td></tr>" +
"</tbody></table>";
details.rawstate + "</td></tr>";
if (_.has(details, "execute_state")) {
var tag;
var icon;
if (details.execute_state == "running") {
tag = "Running";
icon = "record8.svg";
}
else if (details.execute_state == "exited") {
if (details.execute_status != 0) {
tag = "Exited (" + details.execute_status + ")";
icon = "cancel22.svg";
}
else {
tag = "Finished";
icon = "check64.svg";
}
}
else {
tag = "Pending";
icon = "button14.svg"
}
html += "<tr><td>Startup Service:</td><td>" +
tag + "</td></tr>";
$('#' + jacksIDs[node_id] + ' .node .node-status')
.css("visibility", "visible");
if (!$('#' + jacksIDs[node_id] +
' .node .node-status-icon').length) {
$('#' + jacksIDs[node_id] + ' .node .node-status')
.append(svgimg);
}
$('#' + jacksIDs[node_id] + ' .node .node-status-icon')
.attr("href", "fonts/" + icon);
if ($('#' + jacksIDs[node_id] + ' .node .node-status-icon')
.data("bs.tooltip")) {
$('#' + jacksIDs[node_id] + ' .node .node-status-icon')
.data("bs.tooltip").options.title = tag;
}
else {
$('#' + jacksIDs[node_id] + ' .node .node-status-icon')
.tooltip({"title" : tag,
"trigger" : "hover",
"html" : true,
"container" : "body",
"placement" : "auto right",
});
}
}
html += "</tbody></table>";
if ($('#' + jacksIDs[node_id]).data("bs.popover")) {
$('#' + jacksIDs[node_id])
......@@ -742,7 +822,43 @@ function (_, sup, moment, marked, UriTemplate, ShowImagingModal,
}
});
});
}
}
//
// Check the status blob to see if any nodes have execute services
// still running.
//
function servicesExecuting(blob)
{
if (_.has(blob, "sliverstatus")) {
for (var urn in blob.sliverstatus) {
var nodes = blob.sliverstatus[urn];
for (var nodeid in nodes) {
var status = nodes[nodeid];
if (_.has(status, "execute_state") &&
status.execute_state != "exited") {
return 1;
}
}
}
}
return 0;
}
function hasExecutionServices(blob)
{
if (_.has(blob, "sliverstatus")) {
for (var urn in blob.sliverstatus) {
var nodes = blob.sliverstatus[urn];
for (var nodeid in nodes) {
var status = nodes[nodeid];
if (_.has(status, "execute_state")) {
return 1;
}
}
}
}