Commit 1515865a authored by Kevin Atkinson's avatar Kevin Atkinson

Add support for create node usage/availability stats from node_history

table.  Right now Utah testbed only.
parent 81621205
SCRIPTS = analy from_ptop mk-plots refresh analy2 gather publish sanity
all:
node_usage-install:
cp index.php /usr/testbed/www/node_usage/
cp -p $(SCRIPTS) /usr/testbed/data/node_usage/scripts
Scripts to gather node usage data.
Only designed to work on Utah's testbed, but could be adopted to other
sites with some effort.
Before install:
mkdir /usr/testbed/www/node_usage
mkdir /usr/testbed/data/node_usage
mkdir /usr/testbed/data/node_usage/data
mkdir /usr/testbed/data/node_usage/scripts
mysqladmin create node_usage
Than:
make node_usage-install
And install a crontab entry in /etc/crontab:
30 5 * * * root /usr/testbed/data/node_usage/scripts/refresh
#!/usr/bin/perl
use strict;
use warnings;
use POSIX qw(strftime ceil);
chdir "/usr/testbed/data/node_usage/data";
my $TOLERANCE_ALL = 0.05;
my $TOLERANCE_PC3000 = 0.05;
my $TOLERANCE_INTERVAL = 1/3;
my $START = 1125727200;
my $interval = 60*60;
my $interval_name = "hourly";
my $start = ceil($START / $interval) * $interval;
my $prev_time = 0;
my @prev_data = (0,0);
my @total_so_far = (0,0);
my $next_cutoff = $start;
my $error_frac = 0;
open F, "node_usage.raw";
open O, ">node_usage-$interval_name.dat";
while (<F>) {
chop;
s/^(\d+) (\d+) // or die;
my $time = $2;
my %d;
while (s/^ ([\w\d\-\?]+): (\d+) (\d+) (\d+) //) {
$d{$1} = [$2, $3, $4];
}
no warnings 'uninitialized';
my @num = ($d{pc850}[0] + $d{pc600}[0] + $d{pc2000}[0] + $d{pc3000}[0],
$d{pc3000}[0]);
die unless $time <= $start || $num[0] == 160 + 128 + 40 + 8;
die unless $time <= $start || $num[1] == 160;
my @data = ($d{pc850}[1] + $d{pc600}[1] + $d{pc2000}[1] + $d{pc3000}[1],
$d{pc3000}[1]);
my @error = ($d{pc850}[2] + $d{pc600}[2] + $d{pc2000}[2] + $d{pc3000}[2],
$d{pc3000}[2]);
$data[0] = $data[0] + $error[0]/2;
$data[1] = $data[1] + $error[1]/2;
if ($error[0] > $num[0]*$TOLERANCE_ALL || $error[1] > $num[1]*$TOLERANCE_PC3000) {
#print STDERR "ERROR $time: $error\n";
@data = ('NaN', 'NaN');
}
use warnings;
my $combine = sub {
my ($t) = (@_);
my $frac = ($t - $prev_time)/$interval;
if ($prev_data[0] != $prev_data[0]) { # ie NaN
$error_frac += $frac;
} else {
$total_so_far[0] += $prev_data[0] * $frac;
$total_so_far[1] += $prev_data[1] * $frac;
}
};
while ($time >= $next_cutoff) {
&$combine($next_cutoff);
my @free = @total_so_far;
if ($error_frac > $TOLERANCE_INTERVAL) {
$free[0] = 'NaN';
$free[1] = 'Nan';
} else {
$free[0] /= (1 - $error_frac);
$free[1] /= (1 - $error_frac);
}
my @alloc = ($num[0] - $free[0], $num[1] - $free[1]);
my $dtime = $next_cutoff - $interval;
printf O ("%d %.1f %.1f %.1f %.1f\n", $dtime, @free, @alloc) if $dtime >= $start;
$error_frac = 0;
@total_so_far = (0,0);
$prev_time = $next_cutoff;
$next_cutoff += $interval;
}
&$combine($time);
$prev_time = $time;
@prev_data = @data;
}
#!/usr/bin/perl
use Data::Dumper;
use POSIX 'strftime','mktime';
use strict;
use warnings;
no warnings 'uninitialized';
chdir "/usr/testbed/data/node_usage/data";
my $TOLERANCE_INTERVAL = 1/3;
my %res;
sub tally ($$@) {
my ($str, $what, @d) = @_;
if ($d[0] != $d[0]) { # ie NaN
$res{$what}{data}{$str}{invalid}++;
} else {
$res{$what}{data}{$str}{count}++;
foreach my $i (0 .. 3) {
$res{$what}{data}{$str}{data}[$i] += $d[$i];
}
}
}
sub tally_mod ($$@) {
my ($bin, $what, @d) = @_;
return if $d[0] != $d[0]; # ie NaN
$res{$what}{data}[$bin]{count}++;
foreach my $i (0 .. 3) {
$res{$what}{data}[$bin]{data}[$i] += $d[$i];
}
}
open F, "node_usage-hourly.dat" or die;
$res{hourly_last2weeks} = {type=>'normal'};
$res{daily_last2months} = {type=>'normal'};
$res{daily} = {type=>'normal'};
$res{weekly} = {type=>'normal'};
$res{monthly} = {type=>'normal'};
$res{yearly} = {type=>'normal'};
$res{by_hour} = {type=>'mod'};
$res{by_dayofweek} = {type=>'mod'};
$res{by_hourofweek} = {type=>'mod',div=>24};
$res{by_month} = {type=>'mod'};
my @now = localtime();
my $hourly_start = mktime(0, 0, 0, $now[3]-14, $now[4], $now[5]);
my $daily_start = mktime(0, 0, 0, $now[3], $now[4]-2, $now[5]);
while (<F>) {
chop;
my @d = split / /;
my $time = shift @d;
my @time = localtime($time);
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = @time;
#next unless $year + 1900 == 2009;
my $wday_m = ($wday - 1) % 7;
tally(strftime("%Y-%m-%d_%R", @time), 'hourly_last2weeks', @d)
if ($time >= $hourly_start);
tally(strftime("%Y-%m-%d", @time), 'daily_last2months', @d)
if ($time >= $daily_start);
tally(strftime("%Y-%m-%d", @time), 'daily', @d);
tally(strftime("%Y-%m-%d", $sec,$min,$hour,$mday-$wday_m,$mon,$year), 'weekly', @d);
tally(strftime("%Y-%m", @time), 'monthly', @d);
tally(strftime("%Y", @time), 'yearly', @d);
tally_mod($hour,'by_hour', @d);
tally_mod($wday_m,'by_dayofweek', @d);
tally_mod($wday_m*24+$hour,'by_hourofweek', @d);
tally_mod($mon, 'by_month', @d);
}
foreach my $k (keys %res) {
open F, ">node_usage-$k.dat";
if ($res{$k}{type} eq 'normal') {
foreach my $i (sort keys %{$res{$k}{data}}) {
my @r;
my $d = $res{$k}{data}{$i};
my $invalid = $d->{invalid};
my $count = $d->{count};
if ($invalid / ($invalid + $count) > $TOLERANCE_INTERVAL) {
@r = ('NaN', 'NaN', 'NaN', 'NaN');
} else {
foreach my $j (0 .. 3) {
$r[$j] = $d->{data}[$j]/$count;
}
}
printf F ("%s %5.1f %5.1f %5.1f %5.1f\n", $i, @r);
}
} else {
my $div = $res{$k}{div};
$div = 1 unless defined $res{$k}{div};
foreach my $i (0 .. $#{$res{$k}{data}}) {
my @r;
my $d = $res{$k}{data}[$i];
my $count = $d->{count};
foreach my $j (0 .. 3) {
$r[$j] = $d->{data}[$j]/$count;
}
printf F ("%6.3f %5.1f %5.1f %5.1f %5.1f\n", $i/$div, @r);
}
}
}
#!/usr/bin/perl
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
use POSIX qw(strftime floor ceil);
use Data::Dumper;
use Carp;
use strict;
use warnings;
#
# Configure variables
#
my $TB = "/usr/testbed/";
#
# Turn off line buffering on output
#
$| = 1;
# Load the Testbed support stuff.
use lib "/usr/testbed/lib";
use libdb;
use libtestbed;
chdir "/usr/testbed/data/node_usage/data";
my $qr = DBQueryFatal("select t.idx,action,t.exptidx,eid,pid,UNIX_TIMESTAMP(start_time), UNIX_TIMESTAMP(end_time) from testbed_stats as t left join experiment_stats as e on t.exptidx = e.exptidx where (t.action='swapin' or t.action='start') and exitcode=0 and start_time >= '2005-09-03' order by t.idx");
while (my ($idx,$action,$exptidx,$eid,$pid,$start,$stop) = $qr->fetchrow()) {
unless (defined $pid) {
print "xxx $exptidx\n";
next;
}
my $dir = "/usr/testbed/expinfo/$pid-$eid.$exptidx";
my @res;
foreach my $fn (<$dir/*.ptop>) {
next if $fn =~ /-empty.ptop$/;
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks) = stat("$fn") or die;
next unless $start <= $mtime && $mtime <= $stop;
push @res, [$fn,$mtime];
}
next unless @res == 1;
my %tally;
my $fn = $res[0][0];
my $mtime = $res[0][1];
open F, $fn or die;
while (<F>) {
next unless /^node pc\d+/;
if (/^node (pc\d+) (pc[\d\w]+):1/) {
$tally{$2}++;
} else {
print STDERR "SKIPPING: $idx $fn $.: $_";
}
}
print "$mtime $idx $fn ::";
foreach my $k (sort keys %tally) {
print " $k: $tally{$k}";
}
print "\n";
}
#!/usr/bin/perl
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
use POSIX qw(strftime floor ceil);
use Data::Dumper;
use Carp;
$Data::Dumper::Indent = 1;
use strict;
use warnings;
my $prep = 1;
my $results = 1;
my $fresh = 0;
my $TB = "/usr/testbed/";
# Turn off line buffering on output
$| = 1;
# Load the Testbed support stuff.
use lib "/usr/testbed/lib";
use libdb;
use libtestbed;
chdir "/usr/testbed/data/node_usage/data";
my %node_type;
my %node_class;
my $qr = DBQueryFatal("select node_id,type from nodes");
while (my ($node_id, $type) = $qr->fetchrow()) {
$node_type{$node_id} = $type;
}
$qr = DBQueryFatal("select class,type from node_types");
while (my ($class,$type) = $qr->fetchrow()) {
$node_class{$type} = $class;
}
$node_class{'?'} = '?';
our %last_trans;
our %node_state;
our $prev;
our $prev_line;
our $last_history_id = -1;
if ($prep) {
DBQueryFatal("insert into node_usage.node_history_copy select * from node_history where history_id > (select max(history_id) from node_usage.node_history_copy)");
if ($fresh) {
DBQueryFatal("drop table if exists node_usage.node_trans");
DBQueryFatal("create table node_usage.node_trans (".
" history_id int unsigned not null primary key,".
" stamp int unsigned not null, ".
" node_id char(8) not null, ".
" op enum('alloc','free','invalid') not null".
")");
}
local %last_trans;
local %node_state;
if (!$fresh && -e "gather.state.1") {
do "gather.state.1";
}
$qr = DBQueryFatal("select history_id,node_id,op,stamp from node_usage.node_history_copy where history_id > $last_history_id order by history_id");
local $last_history_id;
while (my ($history_id,$node_id,$op,$stamp) = $qr->fetchrow()) {
my $type = $node_type{$node_id};
$type = '?' unless defined $type;
next unless $node_class{$type} eq 'pc';
my $prev_state = $node_state{$node_id};
$node_state{$node_id} = $op eq 'free' ? 'free' : 'alloc';
my $invalid;
if (defined $prev_state) {
$invalid = "alloc non-free node" if ($op eq 'alloc' && $prev_state ne 'free');
$invalid = "free already free node" if ($op eq 'free' && $prev_state eq 'free');
$invalid = "move free node" if ($op eq 'move' && $prev_state eq 'free');
}
if ($invalid) {
#print STDERR "WARNING: $history_id ($stamp) $last_trans{$node_id}: $invalid\n";
DBQueryFatal("update node_usage.node_trans set op = 'invalid' where history_id=$last_trans{$node_id}");
} elsif (!defined $prev_state || $prev_state ne $node_state{$node_id}) {
DBQueryFatal("insert into node_usage.node_trans values ($history_id, $stamp, '$node_id', '$node_state{$node_id}')");
$last_trans{$node_id} = $history_id;
}
$last_history_id = $history_id;
}
open F, ">gather.state.1";
print F Data::Dumper->Dump([\%last_trans], ['*last_trans']);
print F Data::Dumper->Dump([\%node_state], ['*node_state']);
print F Data::Dumper->Dump([$last_history_id], ['*last_history_id']);
close F;
}
if ($results) {
local %node_state;
local $prev = 0;
local $prev_line = '';
if ($fresh || !-e "gather.state.2") {
unlink "node_usage.raw";
} else {
do "gather.state.2";
}
open F, ">>node_usage.raw";
$qr = DBQueryFatal("select history_id,stamp,node_id,op from node_usage.node_trans where history_id > $last_history_id order by history_id");
while (my ($history_id,$stamp,$node_id,$op) = $qr->fetchrow()) {
my $type = $node_type{$node_id};
$type = '?' unless defined $type;
$node_state{$node_id} = $op;
my %tally;
while (my ($n, $s) = each %node_state) {
my $t = $node_type{$n};
$t = '?' unless defined $t;
$tally{$t}[0]++;
$tally{$t}[1]++ if $s eq 'free';
$tally{$t}[2]++ if $s eq 'invalid';
}
my $line = "$history_id $stamp ";
foreach my $t (sort keys %tally) {
$tally{$t}[1] = 0 unless defined $tally{$t}[1];
$tally{$t}[2] = 0 unless defined $tally{$t}[2];
$line .= " $t: $tally{$t}[0] $tally{$t}[1] $tally{$t}[2] ";
}
$line .= "\n";
print F $prev_line if $stamp != $prev;
$prev = $stamp;
$prev_line = $line;
}
open F, ">gather.state.2";
print F Data::Dumper->Dump([\%node_state], ['*node_state']);
print F Data::Dumper->Dump([$prev, $prev_line], ['*prev', '*prev_line']);
close F;
}
<?php
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2009 University of Utah and the Flux Group.
# All rights reserved.
#
require("../defs.php3");
PAGEHEADER("Testbed Node Usage Stats");
?>
<p>
These graphs show the average number of free nodes over various time
periods. Hourly graphs show the average number of free nodes in the
given hour. Daily graphs show the average number of free nodes in the
given day. Etc.
<p>
<p align=center>
<b>Last 2 Weeks (hourly)</b>
<br>
<img src="node_avail-hourly_last2weeks.png">
<br>
<p align=center>
<b>Last 2 Months (daily)</b>
<br>
<img src="node_avail-daily_last2months.png">
<p align=center>
<hr>
<p align=center>
<b>A Typical Week (hourly)</b>
<br>
<img src="node_avail-by_hourofweek.png">
<p align=center>
<hr>
<p align=center>
<b>Since Sep 2005 (yearly)</b>
<br>
<img src="node_avail-yearly.png">
<p align=center>
<b>Since Sep 2005 (monthly)</b>
<br>
<img src="node_avail-monthly.png">
<p align=center>
<b>Since Sep 2005 (weekly)</b>
<br>
<img src="node_avail-weekly.png">
<p align=center>
Note: The gap in the last two graphs represents periods when no data
was availabe due to bugs in our system.
<?php
PAGEFOOTER();
?>
#!/usr/bin/perl
chdir "/usr/testbed/data/node_usage/data";
sub true {1}
sub false {0}
#my $TERMINAL = "postscript eps enhanced color";
#my $EXT = "eps";
my $TERMINAL = "png";
my $EXT = "png";
my $LW = 2;
sub plot ( $@ ) {
my ($what, %ops) = @_;
open F, ">node_usage-$what.gpl";
print F "set terminal $TERMINAL\n";
print F "set grid\n" unless defined $ops{grid} and !$ops{grid};
print F "\n";
if ($ops{timefmt}) {
print F "set xdata time\n";
print F "set timefmt '$ops{timefmt}'\n";
}
print F "set format x \"$ops{format_x}\"\n" if defined $ops{format_x};
foreach my $o (qw(size xrange xtics)) {
print F "set $o $ops{$o}\n" if defined $ops{$o};
}
print F "set xlabel '$ops{xlabel}'\n" if defined $ops{xlabel};
print F "\n";
my $lw = $ops{lw};
$lw = $LW unless defined $lw;
print F "set output 'node_avail-$what.$EXT'\n";
print F "set ylabel 'Available Nodes'\n";
print F "plot 'node_usage-$what.dat' using 1:2 title 'pc600 pc850 pc2000 pc3000s' with lines lw $lw,\\\n";
print F " 'node_usage-$what.dat' using 1:3 title 'pc3000s' with lines lw $lw\n\n";
print F "set output 'node_usage-$what.$EXT'\n";
print F "set ylabel 'Node Usage'\n";
print F "plot 'node_usage-$what.dat' using 1:4 title 'pc600 pc850 pc2000 pc3000s' with lines lw $lw,\\\n";
print F " 'node_usage-$what.dat' using 1:5 title 'pc3000s' with lines lw $lw\n\n";
close F;
system("gnuplot node_usage-$what.gpl");
}
plot("hourly_last2weeks",timefmt=>'%Y-%m-%d_%H:$M', format_x=>'%a\n%m/%d');
plot("weekly", timefmt=>'%Y-%m-%d');
plot("daily_last2months", timefmt=>'%Y-%m-%d');
plot("daily", timefmt=>'%Y-%m-%d', size=>'2,1');
plot("monthly", timefmt=>'%Y-%m', format_x=>'%m/%y');
plot("yearly", grid=>false, xlabel => 'Year', xtics=>1);
plot("by_hour", xrange=>"[0:23]",
xtics=>"('Midnight' 0,'4 am' 4, '8 am' 8, 'Noon' 12, '4 pm' 16, '8 pm' 20, '11 pm' 23)");
#plot("by_hour", xrange=>"[0:23]",
# xtics=>("('Midnight' 0,'' 1 1, '' 2 1, '' 3 1, ".
# "'4 am' 4, '' 5 1, '' 6 1, ''7 1, ".
# "'8 am' 8, '' 9 1, '' 10 1, '' 11 1, ".
# "'Noon' 12, '' 13 1, '' 14 1, '' 15 1, ".
# "'4 pm' 16, '' 17 1, '' 18 1, '' 19 1, ".
# "'8 pm' 20, '' 21 1, '' 22 1, '11 pm' 23)"));
plot("by_dayofweek", xrange=>"[0:6]",
xtics=>"('Mon' 0, 'Tues' 1, 'Wed' 2, 'Thurs' 3, 'Fri' 4, 'Sat' 5, 'Sun' 6)");
plot("by_hourofweek", xrange=>"[0:7]",
xtics=>"('Midnight Mon' 0, 'Tues' 1, 'Wed' 2, 'Thurs' 3, 'Fri' 4, 'Sat' 5, 'Sun' 6)");
plot("by_month", xrange=>"[0:11]",
xtics=>"('Jan' 0, 'Feb' 1, 'Mar' 2, 'Apr' 3, 'May' 4, 'June' 5, 'July' 6, 'Aug' 7, 'Sep' 8, 'Oct' 9, 'Nov' 10, 'Dec' 11)");
#!/bin/sh
cp /usr/testbed/data/node_usage/data/*.png \
/usr/testbed/devel/kevina/www/node_usage/
#!/bin/sh
cd /usr/testbed/data/node_usage/scripts
./gather
./analy
./analy2
./mk-plots
./publish
#/usr/bin/perl
use Data::Dumper;
# Turn off line buffering on output
$| = 1;
chdir "/usr/testbed/data/node_usage/data";
open A, "node_usage.raw";
my $keep = 100;
my @data;
$data[0] = {time => -1};
$data[$keep-1] = undef;
sub fetch_until ($) {
my ($until) = @_;
while ($data[0]{time}[0] <= $until && ($_ = <A>)) {
pop @data;
chop;
s/^(\d+) (\d+) // or die;
my $time = $2;
my %d;
while (s/^ ([\w\d\-\?]+): (\d+) (\d+) (\d+) //) {
my $type = $1;
my @d = ($2, $3, $4);
next unless $type =~ /^pc(600|850|2000|3000)$/;
$d{$type} = \@d;
}
$data[0]{time}[1] = $time - 1;
unshift @data, {time => [$time, $time], data => \%d};
}
}
open B, "from_ptop.raw";
while (<B>) {
chop;
s/^(\d+).+ ::// or die;
my $time = $1;
my %d;
while (s/^ ([\w\d\-\?]+): (\d+)//) {
my $type = $1;
my @d = ($2);
next unless $type =~ /^pc(600|850|2000|3000)$/;
$d{$type} = $d[0];
}
my %r;
fetch_until $time + 5;
my $c = 0;
foreach my $d (@data) {
last unless $d->{time}[1] >= $time - 10;
next unless $d->{time}[0] <= $time + 1;
$c++;
#print ">>$d->{time}[0] $d->{time}[1]\n";
while (my ($k,$v) = each %{$d->{data}}) {
if (exists $r{$k}) {
foreach my $i (0 .. 2) {
$r{$k}[$i][0] = $v->[$i] if $v->[$i] < $r{$k}[$i][0];
$r{$k}[$i][1] = $v->[$i] if $v->[$i] > $r{$k}[$i][1];
}
} else {
$r{$k} = [map {[$_, $_]} @$v];
}
}
}
#print "XXX ";
#foreach my $k (sort keys %r) {
# print "$k: [$r{$k}[0][0] $r{$k}[0][1]] [$r{$k}[1][0] $r{$k}[1][1]] [$r{$k}[2][0] $r{$k}[2][1]] ";
#}
#print "\nYYY ";
#foreach my $k (sort keys %d) {
# print "$k: $d{$k} ";
#}
my @errors;
foreach my $k (sort keys %r) {
my ($min,$max) = @{$r{$k}[1]};
my $errmax = $r{$k}[2][1];
push @errors, sprintf("%s -%d [%d %d]",$k,$min - $d{$k},$min,$max) if $d{$k} < $min && $min - $d{$k} > $errmax;
push @errors, sprintf("%s +%d [%d %d]",$k,$d{$k} - $max,$min,$max) if $d{$k} > $max && $d{$k} - $max > $errmax;
}
print "$time ";
foreach (@errors) {
print " $_ ";
}
print "\n";
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment