Commit b72a16c4 authored by Leigh B Stoller's avatar Leigh B Stoller

Bring in Rob's aster graph code. Add 5 minute download of the data

from all clusters, and regen the json files. Still need to add live
updating.

NOTE: This is not ready yet, Rob needs to figure out why the labels
are broken.
parent 7564c408
......@@ -60,8 +60,9 @@ install: $(addprefix $(INSTALL_BINDIR)/, $(BIN_SCRIPTS)) \
$(addprefix $(INSTALL_SBINDIR)/, $(SBIN_SCRIPTS)) \
$(addprefix $(INSTALL_LIBDIR)/, $(LIB_SCRIPTS)) \
$(addprefix $(INSTALL_LIBEXECDIR)/, $(LIBEXEC_SCRIPTS)) \
$(addprefix $(INSTALL_DIR)/opsdir/libexec/, $(USERLIBEXEC))
$(addprefix $(INSTALL_DIR)/opsdir/libexec/, $(USERLIBEXEC)) \
$(INSTALL_ETCDIR)/cloudlab-fedonly.json \
$(INSTALL_ETCDIR)/cloudlab-nofed.json
boss-install: install install-subdirs
......
......@@ -30,6 +30,9 @@
use strict;
use English;
use Getopt::Std;
use Data::Dumper;
use JSON;
use File::Basename;
#
# Look for APT things that need to be dealt with.
......@@ -307,6 +310,121 @@ sub ReportLockdownExpired()
}
#
# Gather up aggregate info. At the moment, there is still a lot of
# Utah specific goings on here.
#
sub UpdateAggregateGraphs()
{
my $freeblob = {};
my $query_result =
DBQueryWarn("select * from apt_aggregates where noupdate=0");
return 0
if (!$query_result->numrows);
while (my $row = $query_result->fetchrow_hashref()) {
my $urn = $row->{'urn'};
my $url = $row->{'weburl'} . "/node_usage/freenodes.csv";
my $name = $row->{'nickname'};
system("$WGET --no-check-certificate ".
"--timeout=30 --waitretry=30 --retry-connrefused ".
"-q -O /tmp/freenodes.csv '$url'");
next
if ($?);
if (-z "/tmp/freenodes.csv") {
print STDERR "Zero length csv file for $urn\n";
next;
}
if (!open(CSV, "/tmp/freenodes.csv")) {
print STDERR "Could not open csv file for $urn\n";
next;
}
my $line = <CSV>;
if ($line !~ /^Type,Inuse,Free$/) {
print STDERR "Invalid first line in csv file for $urn\n";
close(CSV);
next;
}
while (<CSV>) {
if ($_ =~ /^([-\w]+),(\d+),(\d+)$/) {
$freeblob->{$name}->{$1} = {"inuse" => $2, "free" => $3};
}
}
close(CSV);
}
#print Dumper($freeblob);
#
# We are currently operating from two master json files. We make a
# a copy of those, and then update them with new info, and write them
# back. This will need to be generalized at some point.
#
my $NOFED = "$TB/etc/cloudlab-nofed.json";
my $FEDONLY = "$TB/etc/cloudlab-fedonly.json";
foreach my $file ($NOFED, $FEDONLY) {
if (-e $file) {
my $data = `/bin/cat $file`;
my $obj = decode_json($data);
if (!defined($obj)) {
print STDERR "Could not decide json in $file\n";
next;
}
foreach my $cluster (@{ $obj->{'children'} }) {
my $name = $cluster->{'name'};
next
if (!exists($freeblob->{$name}));
my $total_size = 0;
my $total_inuse = 0;
foreach my $child (@{ $cluster->{'children'} }) {
my $type = $child->{'name'};
if (!exists($freeblob->{$name}->{$type})) {
print STDERR "$name: Did not get a count for $type\n";
next;
}
my $inuse = $freeblob->{$name}->{$type}->{'inuse'};
my $size = $freeblob->{$name}->{$type}->{'free'} + $inuse;
$child->{'howfull'} = int($inuse);
$child->{'size'} = int($size);
$total_inuse += $inuse;
$total_size += $size;
}
$cluster->{'howfull'} = int($total_inuse);
$cluster->{'size'} = int($total_size);
}
#print Dumper($obj);
#
# Write out new file for web ui.
#
my $tfile = "/tmp/$$.json";
if (open(JS, ">$tfile")) {
my $json_text = to_json($obj, { pretty => 1 });
print JS $json_text . "\n";
close(JS);
}
else {
print STDERR "Could not open temp json file for new data\n";
next;
}
system("/bin/mv -f $tfile $TB/www/apt/" . basename($file));
if ($?) {
print STDERR "Could not copy new json file to ".
"$TB/www/apt/" . basename($file) . "\n";
}
unlink($tfile);
}
}
return 0;
}
my $reportcounter = 0;
# Do this once at startup
......@@ -326,6 +444,7 @@ while (1) {
KillFailedInstances();
ExpireInstances();
UpdateAggregateGraphs();
# Do this once every 24 hours.
if ($reportcounter >= (24 * 60 * 60)) {
......
{
"name": "CloudLab Federates",
"children": [
{
"name": "APT",
"order": 4,
"color": "#2fad2f",
"howfull": 192,
"size": 192,
"url": "http://docs.cloudlab.us/hardware.html#%28part._apt-cluster%29",
"children": [ { "name": "r320", "size": 128, "howfull" : 110, "url": "http://docs.cloudlab.us/hardware.html#%28part._apt-cluster%29" },
{ "name": "c6220", "size": 64, "howfull" : 60, "url": "http://docs.cloudlab.us/hardware.html#%28part._apt-cluster%29" } ]
},
{
"name": "DDC",
"order": 5,
"color": "#2f64ad",
"size" : 33,
"howfull": 33,
"url": "http://docs.cloudlab.us/hardware.html#%28part._ig-ddc%29",
"children": [ { "name": "dl360", "size": 33, "howfull": 5, "url": "http://docs.cloudlab.us/hardware.html#%28part._ig-ddc%29" } ]
},
{
"name": "Utah PG",
"order": 6,
"color": "#2164ad",
"size" : 0,
"howfull": 0,
"url": "http://docs.cloudlab.us/hardware.html#%28part._ig-ddc%29",
"children": [ { "name": "pc3000", "size": 33, "howfull": 5,
"url": "http://docs.cloudlab.us/hardware.html#%28part._ig-ddc%29"
},
{ "name": "d710", "size": 33, "howfull": 5,
"url": "http://docs.cloudlab.us/hardware.html#%28part._ig-ddc%29"
}
]
}
]
}
{
"name": "CloudLab Sites",
"children": [
{
"name": "Utah",
"order": 1,
"color": "#ca3737",
"size": 315,
"howfull": 315,
"children": [ { "name": "m400", "size": 315, "howfull": 200, "url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-utah%29" } ],
"url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-utah%29"
},
{
"name": "Clemson",
"order": 2,
"color": "#ca8f37",
"size": 100,
"howfull": 100,
"url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-clemson%29",
"children": [ { "name": "c8220", "size": 96, "howfull": 30, "url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-clemson%29" },
{ "name": "c8220x", "size": 4, "howfull": 3, "url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-clemson%29" } ]
},
{
"name": "Wisconsin",
"order": 3,
"color": "#caca37",
"size": 100,
"howfull": 100,
"url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-wisconsin%29",
"children": [ { "name": "C220M4", "size": 90, "howfull": 30, "url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-wisconsin%29" },
{ "name": "C240M4", "size": 10, "howfull": 3, "url": "http://docs.cloudlab.us/hardware.html#%28part._cloudlab-wisconsin%29" } ]
}
]
}
/*
* Copyright (c) 2015 University of Utah and the Flux Group.
*
* Based on work from:
* http://bl.ocks.org/mbostock/5944371
*/
/*
* Construct a bi-level Aster graph from the JSON data in <file>, on element
* <element> (eg. "#graph"), which should be a <div>. Give the height and width
* in any SVG-legal unit, or say "auto" to fill width of the <element>. <style>
* should be "large" or "small" - the latter version omits most of the labels.
*
* Expects the following to be loaded:
* d3.js (tested with 3.5.5)
* liquidFillGauge.js
* d3.tip.js (tested with 0.6.3)
* Fira fonts (eg. https://fonts.googleapis.com/css?family=Fira+Sans)
* basic-tooltip.css
*/
function bilevelAsterGraph(file, element, size, style) {
/*
* Set the size (radius) of the chart
*/
if (size == "auto") {
size = d3.select(element).node().getBoundingClientRect().width;
}
var margin = {top: size/2, right: size/2, bottom: size/2, left: size/2};
var radius =
Math.min(margin.top, margin.right, margin.bottom, margin.left) - 10;
/*
* We use this prefix, derived from the element selector, to prefix ids
* and such in case there is more than one of these on a page
*/
var prefix = element.replace(/^[.#]/,"");
/*
* The rings are not equal in size - this array defines the distance from
* the center at which each ring starts and ends (eg. the center ring
* starts at rings[0] and ends at rings[1])
*/
if (style == "small") {
var rings = [0, radius/3, radius/3 + radius/10, radius];
} else {
var rings = [0, radius/3, radius/3 + 25, radius];
}
/*
* Create the SVG object that we'll use for everything
*/
var svg = d3.select(element).append("svg")
.attr("display","block")
.attr("width", margin.left + margin.right)
.attr("height", margin.top + margin.bottom)
.style("font-family", "Fira Sans, sans")
.append("g")
// This re-centers things so that 0,0 is in the middle of the circle
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// Set up the tooltips
var tip = d3.tip()
.attr('class', 'd3-tip')
.html(function(d) {
return "<span>" + d.name + ":&nbsp;" +
d.howfull + "&nbsp;/&nbsp;" + d.size + "</span>"; });
svg.call(tip);
/*
* Main partition object - note that it's sorted by a field that we put in
* the data objects, and the size is double the radius
*/
var partition = d3.layout.partition()
.sort(function(a, b) { return d3.ascending(a.order, b.order); })
.size([2 * Math.PI, radius]);
/*
* Function to generate arc-generating functions - you tell it whether you
* want an arc that is full, or respects the howfull field in the data to
* make a partially-full arc.
*/
var arcGenerator = function (type) {
var outerRadius;
return d3.svg.arc()
.startAngle(function(d) { return d.x; })
.endAngle(function(d) { return d.x + d.dx - .01 / (d.depth + .5); })
// Inner radius of the arc is simple
.innerRadius(function(d) { return rings[d.depth]; })
// Outer radius depends on whether we are doing a full arc (1.0 of the
// way to the next ring) or a partial one (percentage of the way to the
// next ring is set in the data)
.outerRadius(function(d) {
return rings[d.depth] +
(rings[d.depth+1] - rings[d.depth]) *
(type == "full" ? 1.0 : (d.howfull/d.size)) - 1; });
}
/*
* This function generates a simple arc - just one line, rather than the
* full region generated by d3's arc() - this makes it more suitable for
* placing text on
*/
var simpleArc = function(d,i) {
// Starts out the same as arcGenerator
var startAngle = function(d) { return d.x; };
var endAngle = function(d) { return d.x + d.dx - .01 / (d.depth + .5); };
var radius = function(d) { return rings[d.depth]; };
return describeArc(0, 0, rings[d.depth], d.x - Math.PI/2,
d.x + d.dx - .01 / (d.depth + .5) - Math.PI/2);
}
/*
* Get the middle angle of the arc for the specified data - used to decide
* whether the text should be "upside down"
*/
function getMidAngle(d) {
return ((d.x - Math.PI/2) +
(d.x + d.dx - .01 / (d.depth + .5) - Math.PI/2))
/ 2;
}
/*
* Convert polar coordinates to cartesian, for generating arcs
*/
function polarToCartesian(centerX, centerY, radius, angleInRadians) {
return {
x: centerX + (radius * Math.cos(angleInRadians)),
y: centerY + (radius * Math.sin(angleInRadians))
};
}
/*
* Helper function to actually produce an arc (in raw SVG commands) -
* needed becase this one generates a simple arc, rather than the outline
* of one as done by the builtin d3 arc()
*/
function describeArc(x, y, radius, startAngle, endAngle){
// Check to see if this arc is going to be upside down
var midangle = (endAngle + startAngle) / 2;
if ((midangle > 0) && (midangle < 1.0*Math.PI)) {
var end = polarToCartesian(x, y, radius, endAngle);
var start = polarToCartesian(x, y, radius, startAngle);
var direction = 0;
} else {
var start = polarToCartesian(x, y, radius, endAngle);
var end = polarToCartesian(x, y, radius, startAngle);
var direction = 1;
}
// Which way around are we going?
var arcSweep = endAngle - startAngle <= Math.PI ? "0" : "1";
// For reasons I don't quite get, we have to start at the 'end' position
// to get it to go the right way around.
var d = [
"M", end.x, end.y,
"A", radius, radius, 0, arcSweep, direction, start.x, start.y
].join(" ");
return d;
}
/*
* Helper function to (recursively) calculate how full a section of the
* graph should be (eg. to figure out the total utilization)
*/
function calcFull(root) {
var percentages = [];
if (typeof(root.children) == 'undefined') {
return [root.howfull/root.size, root.size];
} else {
root.children.forEach(function (e) {
percentages.push(calcFull(e)); });
}
var total = percentages.reduce(function(a, b) { return a + b[1]; }, 0);
var weightedPct = percentages.map(function (a) {
return a[0] * a[1]/total;
}).reduce(function (a,b) {
return a + b; },
0);
return [weightedPct,total];
}
/*
* Finally, work with the actual data
*/
d3.json(file, function(error, root) {
// Compute the initial layout on the entire tree to sum sizes.
// Also compute the full name and fill color for each node,
// and stash the children so they can be restored as we descend.
partition
.value(function(d) { return d.size; })
.nodes(root)
.forEach(function(d) {
d._children = d.children;
d.sum = d.value;
d.key = key(d);
d.fill = fill(d);
});
// Now redefine the value function to use the previously-computed sum.
partition.value(function(d) { return d.sum; });
// Hairline around the outside
var outer = svg.append("circle")
.attr("r", rings[3])
.style("stroke", "#bbbbbb")
.style("fill", "#f2f2f2")
// We create (empty) SVG groups for every data point, since we are going
// to attach a few things to them
var groups = svg.selectAll("path")
.data(partition.nodes(root).slice(1))
.enter().append("g")
// Light-colored full-radius arcs go behind each datapoint, so that that
// the actual datapoints will look more like "how full" indicator
var backgroundArcs = groups.append("path")
.attr("d", arcGenerator("full"))
.attr("class", "backgroundArc")
.attr("class", function(d) { return "depth-" + d.depth; })
.style("fill", function(d) {
// Convert color to HSL so we can make it 'washed out' more easily
var c = d.fill.hsl();
c.l = 0.9;
c.s = 0.5;
return c;
})
// These are the ones that contain the actual data points
var foregroundArcs = groups.append("path")
.attr("d", arcGenerator("partial"))
.attr("class", "foregroundArc")
.attr("class", function(d) { return "depth-" + d.depth; })
.style("fill", function(d) { return d.fill; })
// These will be used to line up the text on
var textArcs = groups.append("path")
.attr("d", simpleArc)
.attr("class", "textArc")
.attr("id", function(d, i) { return prefix + 'textArc' + i; })
.style("fill", "none")
.style("stroke","none")
// Only bother to show this on the arcs with data, not the top-level
// ones, which are only used for labels
groups.selectAll(".depth-2")
.on('mouseover', tip.show)
.on('mouseout', tip.hide)
// Labels
if (style != "small") {
var texts = groups.append("a")
.attr("xlink:href",function(d) { return d.url;} )
.append("text")
// text-anchor middle, plus startOffset 50% centers the text on
// the arc
.attr("text-anchor","middle")
// Place text "above" the arc, magic number that works nicely
// with current font size and ring sizes
.attr("dy", function(d) {
ma = getMidAngle(d);
if (ma > 0 && ma < Math.PI) {
return "1.10em";
} else {
return "-.40em";
}})
.append("textPath")
.attr("startOffset","50%")
.attr("fill","#ffffff")
// Little opacity to color them the same as their bars
.attr("fill-opacity","0.8")
// This is the magic that places the text on the path
.attr("xlink:href",function(d, i){ return '#' + prefix + 'textArc' + i; })
.text(function(d){ return d.name; });
}
// Circle in the center
var pctFull = Math.round(calcFull(root)[0] * 100);
var center = svg.append("g")
.attr("id",prefix + "centergauge")
if (style == "small") {
// Just put the percentage in some plain text
center.append("text")
.text(pctFull)
.attr("text-anchor", "middle")
.attr("dy",".3em");
} else {
// Fancier fill gauge
var center = svg.append("g")
.attr("id",prefix + "centergauge")
var gaugeConfig = liquidFillGaugeDefaultSettings();
gaugeConfig.circleThickness = 0;
gaugeConfig.waveColor = "#2f8ead";
gaugeConfig.textColor = "#2f8ead";
gaugeConfig.waveTextColor = "#e8eeef";
gaugeConfig.waveCount = 1;
gaugeConfig.waveHeight = 0;
gaugeConfig.waveRise = false;
gaugeConfig.waveAnimate = false;
gaugeConfig.valueCountUp = false;
loadLiquidFillGauge(prefix + "centergauge",rings[1],pctFull,gaugeConfig);
}
});
// Simple function to give unique keys to ever entry in the graph
function key(d) {
var k = [], p = d;
while (p.depth) k.push(p.name), p = p.parent;
return k.reverse().join(".");
}
// Get fill color - we only set it on the roots
function fill(d) {
var p = d;
// Get color from the root, so that we don't have to color every single
// datapoint
while (p.depth > 1) p = p.parent;
var c = d3.rgb(p.color);
return c;
}
// Set the size of the graph
d3.select(self.frameElement).style("height", margin.top + margin.bottom + "px");
}
require(window.APT_OPTIONS.configObject,
['js/quickvm_sup',
'js/lib/text!template/cluster-graphs.html',
'js/bilevel', 'js/liquidFillGauge'],
function (sup, clusterString)
{
'use strict';
function initialize()
{
window.APT_OPTIONS.initialize(sup);
$('#cluster-graphs').html(clusterString);
bilevelAsterGraph("/cloudlab-nofed.json",
"#status-nofed","auto","large");
bilevelAsterGraph("/cloudlab-fedonly.json",
"#status-fedonly","auto","large");
}
$(document).ready(initialize);
});
......@@ -8,7 +8,6 @@ window.APT_OPTIONS.configObject = {
'jquery-steps': 'js/lib/jquery.steps.min',
'formhelpers': 'js/lib/bootstrap-formhelpers',
'dateformat': 'js/lib/date.format',
'd3': 'js/lib/d3.v3',
'filestyle': 'js/lib/filestyle',
'marked': 'js/lib/marked',
'moment': 'js/lib/moment',
......@@ -23,8 +22,8 @@ window.APT_OPTIONS.configObject = {
'jquery-grid': { deps: ['jquery-ui'] },
'jquery-steps': { },
'formhelpers': { },
'jacks': { },
'dateformat': { exports: 'dateFormat' },
'd3': { exports: 'd3' },
'filestyle': { },
'marked' : { exports: 'marked' },
'underscore': { exports: '_' },
......
// d3.tip
// Copyright (c) 2013 Justin Palmer
//
// Tooltips for d3.js SVG visualizations
// Public - contructs a new tooltip
//
// Returns a tip
d3.tip = function() {
var direction = d3_tip_direction,
offset = d3_tip_offset,
html = d3_tip_html,
node = initNode(),
svg = null,
point = null,
target = null
function tip(vis) {
svg = getSVGNode(vis)
point = svg.createSVGPoint()
document.body.appendChild(node)
}