Commit 9f333d41 authored by Leigh Stoller's avatar Leigh Stoller

Show recent error details on the dashboard page. Hover over the

reason to see the error message, click on the log to see the gory
details.
parent fee57b6e
......@@ -39,10 +39,14 @@ $scales = array("Day" => (24 * 3600),
#
function Do_GetStats()
{
global $scales;
global $scales, $geni_response_codes;
$started = array();
$errors = array();
$started = array();
$errors = array();
$breakdown = array();
$breakdown["timedout"] = array();
$breakdown["mapping"] = array();
$breakdown["other"] = array();
foreach ($scales as $key => $age) {
$query_result =
......@@ -59,9 +63,35 @@ function Do_GetStats()
" where created > DATE_SUB(now(),interval $age second)");
$row = mysql_fetch_array($query_result);
$errors[$key] = $row[0] + $row[1];
$query_result =
DBQueryFatal("select exitcode,count(exitcode) as count ".
" from apt_instance_failures ".
"where created>DATE_SUB(now(),interval $age second)".
"group by exitcode");
while ($row = mysql_fetch_array($query_result)) {
$exitcode = $row[0];
$count = $row[1];
if ($exitcode == 8) {
$breakdown["timedout"][$key] = $count;
}
elseif ($exitcode >= 24 && $exitcode < 30) {
if (!isset($breakdown["mapping"][$key])) {
$breakdown["mapping"][$key] = 0;
}
$breakdown["mapping"][$key] += $count;
}
else {
if (!isset($breakdown["other"][$key])) {
$breakdown["other"][$key] = 0;
}
$breakdown["other"][$key] += $count;
}
}
}
$dashboard = array("started" => $started,
"errors" => $errors);
$dashboard = array("started" => $started,
"errors" => $errors,
"breakdown" => $breakdown);
#
# What are most recent swapins
......@@ -153,6 +183,45 @@ function Do_GetStats()
}
$dashboard["heaviest_profiles"] = $profiles;
#
# Detailed info on the 10 most recent errors.
#
$error_details = array();
$query_result =
DBQueryFatal("select f.name,exitcode,f.created,f.public_url,f.uuid,".
" IF(i.uuid,1,0) as active,f.exitmessage ".
"from apt_instance_failures as f ".
"left join apt_instances as i on ".
" i.uuid=f.uuid ".
"left join apt_instance_history as h on ".
" h.uuid=f.uuid ".
"order by f.created desc limit 10");
while ($row = mysql_fetch_array($query_result)) {
$name = $row["name"];
$exitcode = $row["exitcode"];
$url = $row["public_url"];
$active = $row["active"] ? 1 : 0;
$uuid = $row["uuid"];
$created = $row["created"];
$message = $row["exitmessage"];
$reason = $exitcode;
if ($exitcode >= 0 && $exitcode <= count($geni_response_codes)) {
$reason = $geni_response_codes[$exitcode];
}
$error_details[$uuid] = array("name" => $name,
"uuid" => $uuid,
"exitcode" => $exitcode,
"active" => $active,
"created" => $created,
"reason" => $reason,
"message" => $message,
"url" => $url);
}
$dashboard["error_details"] = $error_details;
SPITAJAX_RESPONSE($dashboard);
}
......
......@@ -54,6 +54,38 @@ $freenodes_mapping =
"urn:publicid:IDN+emulab.net+authority+cm" =>
"https://www.emulab.net/node_usage/freenodes.svg");
$geni_response_codes =
array("Success",
"Bad Arguments",
"Error",
"Forbidden",
"Bad Version",
"Server Error",
"Too Big",
"Refused",
"Timed Out",
"Database Error",
"RPC Error",
"Unavailable",
"Search Failed",
"Unsupported",
"Busy",
"Expired",
"In Progress",
"Already Exists",
"Error 18",
"Error 19",
"Error 20",
"Error 21",
"Error 22",
"Error 23",
"Vlan Unavailable",
"Insufficient Bandwidth",
"Insufficient Nodes",
"Insufficient Memory",
"No Mapping Possible",
);
class Instance
{
var $instance;
......
......@@ -13,7 +13,7 @@ function (_, sup, moment, dashboardString)
isadmin = window.ISADMIN;
DashboardLoop();
setInterval(DashboardLoop,5000);
setInterval(DashboardLoop,30000);
setInterval(UpdateTimes,1000);
}
......@@ -35,6 +35,17 @@ function (_, sup, moment, dashboardString)
.format("ddd h:mm A"));
}
});
$('[data-toggle="popover"]').popover({
trigger: 'hover',
placement: 'auto',
html: true,
content: function () {
var uuid = $(this).data("uuid");
var html = "<code style='white-space: pre-wrap'>" +
json.value.error_details[uuid].message + "</code>";
return html;
}
});
UpdateTimes();
}
var xmlthing = sup.CallServerMethod(null, "dashboard",
......
......@@ -2,7 +2,12 @@
.panel-body-dashboard {
padding: 5px;
}
.table-dashboard {
margin: 1px;
}
.popover{
max-width: 800px;
}
</style>
<div class='row'>
<div class='col-sm-6'>
......@@ -11,7 +16,7 @@
<h5><center>Experiment Activity</center></h5>
</div>
<div class='panel-body panel-body-dashboard'>
<table class="table table-condensed table-bordered"
<table class="table table-condensed table-bordered table-dashboard"
id='experiments-started-table'>
<thead>
<tr>
......@@ -34,6 +39,24 @@
<td><%- value %></td>
<% }); %>
</tr>
<tr>
<td class="text-right">Mapping</td>
<% _.each(dashboard.breakdown.mapping, function(value, key) { %>
<td><%- value %></td>
<% }); %>
</tr>
<tr>
<td class="text-right">Timed Out</td>
<% _.each(dashboard.breakdown.timedout, function(value, key) { %>
<td><%- value %></td>
<% }); %>
</tr>
<tr>
<td class="text-right">Other</td>
<% _.each(dashboard.breakdown.other, function(value, key) { %>
<td><%- value %></td>
<% }); %>
</tr>
</tbody>
</table>
</div>
......@@ -45,7 +68,7 @@
<h5><center>Latest Experiments</center></h5>
</div>
<div class='panel-body panel-body-dashboard'>
<table class="table table-condensed table-bordered"
<table class="table table-condensed table-bordered table-dashboard"
id='experiments-latest-table'>
<thead>
<tr>
......@@ -171,6 +194,55 @@
</div>
</div>
</div>
<div class='row'>
<div class='col-sm-8 col-sm-offset-2'>
<div class='panel panel-default' id='error-details-panel'>
<div class="panel-heading">
<h5><center>Recent Errors (Details)</center></h5>
</div>
<div class='panel-body panel-body-dashboard'>
<table class="table table-condensed table-bordered">
<thead>
<tr>
<th>Name</th>
<th>When</th>
<th>Reason (Code)</th>
<th>Log</th>
</tr>
</thead>
<tbody>
<% _.each(dashboard.error_details, function(value, key) { %>
<tr>
<% if (value.active) { %>
<td><a href="status.php?uuid=<%- value.uuid %>">
<%- value.name %></a></td>
<% }else { %>
<td><%- value.name %></td>
<% } %>
<td class="format-date"><%- value.created %></td>
<% if (value.message && value.message != "") { %>
<td><span style="text-decoration: underline;"
data-toggle='popover'
data-delay='{"hide":100, "show":500}'
data-html='true'
data-uuid="<%- value.uuid %>"
data-content=""><%- value.reason %></span></td>
<% } else { %>
<td><%- value.reason %></td>
<% }%>
<% if (value.url) { %>
<td><a href="<%- value.url%>">Log</a></td>
<% }else { %>
<td>N/A</td>
<% } %>
</tr>
<% }); %>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class='row'>
<div class="col-xs-4">
Last updated: <span class="format-date-relative" id="last-refresh"></span>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment