Add support for create node usage/availability stats from node_history

table.  Right now Utah testbed only.

node_usage/GNUmakefile

+
+SCRIPTS = analy from_ptop mk-plots refresh analy2 gather publish sanity
+
+all:
+
+node_usage-install:
+	cp index.php /usr/testbed/www/node_usage/
+	cp -p $(SCRIPTS) /usr/testbed/data/node_usage/scripts

node_usage/README

+
+Scripts to gather node usage data.
+
+Only designed to work on Utah's testbed, but could be adopted to other
+sites with some effort.
+
+Before install:
+
+  mkdir /usr/testbed/www/node_usage
+  mkdir /usr/testbed/data/node_usage
+  mkdir /usr/testbed/data/node_usage/data
+  mkdir /usr/testbed/data/node_usage/scripts
+  mysqladmin create node_usage
+
+Than:
+  
+  make node_usage-install
+
+And install a crontab entry in /etc/crontab:
+
+  30	5	*	*	*	root	/usr/testbed/data/node_usage/scripts/refresh

node_usage/analy

+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use POSIX qw(strftime ceil);
+
+chdir "/usr/testbed/data/node_usage/data";
+
+my $TOLERANCE_ALL = 0.05;
+my $TOLERANCE_PC3000 = 0.05;
+my $TOLERANCE_INTERVAL = 1/3;
+my $START = 1125727200;
+
+my $interval = 60*60;
+my $interval_name = "hourly";
+
+my $start = ceil($START / $interval) * $interval;
+
+my $prev_time = 0;
+my @prev_data = (0,0);
+my @total_so_far = (0,0);
+my $next_cutoff = $start;
+
+my $error_frac = 0;
+
+open F, "node_usage.raw";
+open O, ">node_usage-$interval_name.dat";
+
+while (<F>) {
+    chop;
+    s/^(\d+) (\d+) // or die;
+    my $time = $2;
+    my %d;
+    while (s/^ ([\w\d\-\?]+): (\d+) (\d+) (\d+) //) {
+	$d{$1} = [$2, $3, $4];
+    }
+    no warnings 'uninitialized';
+    my @num = ($d{pc850}[0] + $d{pc600}[0] + $d{pc2000}[0] + $d{pc3000}[0],
+	       $d{pc3000}[0]);
+    die unless $time <= $start || $num[0] == 160 + 128 + 40 + 8;
+    die unless $time <= $start || $num[1] == 160;
+    my @data = ($d{pc850}[1] + $d{pc600}[1] + $d{pc2000}[1] + $d{pc3000}[1],
+		$d{pc3000}[1]);
+    my @error = ($d{pc850}[2] + $d{pc600}[2] + $d{pc2000}[2] + $d{pc3000}[2],
+		 $d{pc3000}[2]);
+    $data[0] = $data[0] + $error[0]/2;
+    $data[1] = $data[1] + $error[1]/2;
+    if ($error[0] > $num[0]*$TOLERANCE_ALL || $error[1] > $num[1]*$TOLERANCE_PC3000) {
+	#print STDERR "ERROR $time: $error\n";
+	@data = ('NaN', 'NaN');
+    }
+    use warnings;
+    
+    my $combine = sub {
+	my ($t) = (@_);
+	my $frac = ($t - $prev_time)/$interval;
+	if ($prev_data[0] != $prev_data[0]) { # ie NaN
+	    $error_frac += $frac;
+	} else {
+	    $total_so_far[0] += $prev_data[0] * $frac;
+	    $total_so_far[1] += $prev_data[1] * $frac;
+            }
+    };
+    
+    while ($time >= $next_cutoff) {
+	&$combine($next_cutoff);
+	my @free = @total_so_far;
+	if ($error_frac > $TOLERANCE_INTERVAL) {
+	    $free[0] = 'NaN';
+	    $free[1] = 'Nan';
+	} else {
+	    $free[0] /= (1 - $error_frac);
+	    $free[1] /= (1 - $error_frac);
+	}
+	my @alloc = ($num[0] - $free[0], $num[1] - $free[1]);
+	my $dtime = $next_cutoff - $interval;
+	printf O ("%d %.1f %.1f %.1f %.1f\n", $dtime, @free, @alloc) if $dtime >= $start;
+	$error_frac = 0;
+	@total_so_far = (0,0);
+	$prev_time = $next_cutoff;
+	$next_cutoff += $interval;
+	
+    }
+    &$combine($time);
+    
+    $prev_time = $time;
+    @prev_data = @data;
+}
+
+

node_usage/analy2

+#!/usr/bin/perl
+
+use Data::Dumper;
+use POSIX 'strftime','mktime';
+
+use strict;
+use warnings;
+no warnings 'uninitialized';
+
+chdir "/usr/testbed/data/node_usage/data";
+
+my $TOLERANCE_INTERVAL = 1/3;
+
+my %res;
+
+sub tally ($$@) {
+    my ($str, $what, @d) = @_;
+    if ($d[0] != $d[0]) { # ie NaN
+	$res{$what}{data}{$str}{invalid}++;
+    } else {
+	$res{$what}{data}{$str}{count}++;
+	foreach my $i (0 .. 3) {
+	    $res{$what}{data}{$str}{data}[$i] += $d[$i];
+	}
+    }
+}
+
+sub tally_mod ($$@) {
+    my ($bin, $what, @d) = @_;
+    return if $d[0] != $d[0]; # ie NaN
+    $res{$what}{data}[$bin]{count}++;
+    foreach my $i (0 .. 3) {
+	$res{$what}{data}[$bin]{data}[$i] += $d[$i];
+    }
+}
+
+open F, "node_usage-hourly.dat" or die;
+
+$res{hourly_last2weeks} = {type=>'normal'};
+$res{daily_last2months} = {type=>'normal'};
+$res{daily} = {type=>'normal'};
+$res{weekly} = {type=>'normal'};
+$res{monthly} = {type=>'normal'};
+$res{yearly} = {type=>'normal'};
+$res{by_hour} = {type=>'mod'};
+$res{by_dayofweek} = {type=>'mod'};
+$res{by_hourofweek} = {type=>'mod',div=>24};
+$res{by_month} = {type=>'mod'};
+
+my @now = localtime();
+my $hourly_start = mktime(0, 0, 0, $now[3]-14, $now[4], $now[5]);
+my $daily_start = mktime(0, 0, 0, $now[3], $now[4]-2, $now[5]);
+
+while (<F>) {
+    chop;
+    my @d = split / /;
+    my $time = shift @d;
+    my @time = localtime($time);
+    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = @time;
+    #next unless $year + 1900 == 2009;
+    my $wday_m = ($wday - 1) % 7;
+    tally(strftime("%Y-%m-%d_%R", @time), 'hourly_last2weeks', @d)
+	if ($time >= $hourly_start);
+    tally(strftime("%Y-%m-%d", @time), 'daily_last2months', @d)
+	if ($time >= $daily_start);
+    tally(strftime("%Y-%m-%d", @time), 'daily', @d);
+    tally(strftime("%Y-%m-%d", $sec,$min,$hour,$mday-$wday_m,$mon,$year), 'weekly', @d);
+    tally(strftime("%Y-%m", @time), 'monthly', @d);
+    tally(strftime("%Y", @time), 'yearly', @d);
+    tally_mod($hour,'by_hour', @d);
+    tally_mod($wday_m,'by_dayofweek', @d);
+    tally_mod($wday_m*24+$hour,'by_hourofweek', @d);
+    tally_mod($mon, 'by_month', @d);
+}
+
+foreach my $k (keys %res) {
+
+    open F, ">node_usage-$k.dat";
+
+    if ($res{$k}{type} eq 'normal') {
+	foreach my $i (sort keys %{$res{$k}{data}}) {
+	    my @r;
+	    my $d = $res{$k}{data}{$i};
+	    my $invalid = $d->{invalid};
+	    my $count = $d->{count};
+	    if ($invalid / ($invalid + $count) > $TOLERANCE_INTERVAL) {
+		@r = ('NaN', 'NaN', 'NaN', 'NaN');
+	    } else {
+		foreach my $j (0 .. 3) {
+		    $r[$j] = $d->{data}[$j]/$count;
+		}
+	    }
+	    printf F ("%s  %5.1f %5.1f %5.1f %5.1f\n", $i, @r);
+	}
+    } else {
+	my $div = $res{$k}{div};
+	$div = 1 unless defined $res{$k}{div};
+	foreach my $i (0 .. $#{$res{$k}{data}}) {
+	    my @r;
+	    my $d = $res{$k}{data}[$i];
+	    my $count = $d->{count};
+	    foreach my $j (0 .. 3) {
+		$r[$j] = $d->{data}[$j]/$count;
+	    }
+	    printf F ("%6.3f  %5.1f %5.1f %5.1f %5.1f\n", $i/$div, @r);
+	}
+    }
+}

node_usage/from_ptop

+#!/usr/bin/perl 
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2007 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+use POSIX qw(strftime floor ceil);
+use Data::Dumper;
+use Carp;
+
+use strict;
+use warnings;
+
+#
+# Configure variables
+#
+my $TB		= "/usr/testbed/";
+
+#
+# Turn off line buffering on output
+#
+$| = 1; 
+
+# Load the Testbed support stuff.
+use lib "/usr/testbed/lib";
+use libdb;
+use libtestbed;
+
+chdir "/usr/testbed/data/node_usage/data";
+
+my $qr = DBQueryFatal("select t.idx,action,t.exptidx,eid,pid,UNIX_TIMESTAMP(start_time), UNIX_TIMESTAMP(end_time) from testbed_stats as t left join experiment_stats as e on t.exptidx = e.exptidx where (t.action='swapin' or t.action='start') and exitcode=0 and start_time >= '2005-09-03' order by t.idx");
+
+
+while (my ($idx,$action,$exptidx,$eid,$pid,$start,$stop) = $qr->fetchrow()) {
+
+    unless (defined $pid) {
+	print "xxx $exptidx\n";
+	next;
+    }
+
+    my $dir = "/usr/testbed/expinfo/$pid-$eid.$exptidx";
+
+    my @res;
+    foreach my $fn (<$dir/*.ptop>) {
+	next if $fn =~ /-empty.ptop$/;
+	my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
+	    $atime,$mtime,$ctime,$blksize,$blocks) = stat("$fn") or die;
+	next unless $start <= $mtime && $mtime <= $stop;
+	push @res, [$fn,$mtime];
+    }
+    
+    next unless @res == 1;
+
+    my %tally;
+
+    my $fn = $res[0][0];
+    my $mtime = $res[0][1];
+
+    open F, $fn or die;
+    while (<F>) {
+	next unless /^node pc\d+/;
+	if (/^node (pc\d+) (pc[\d\w]+):1/) {
+	    $tally{$2}++;
+	} else {
+	    print STDERR "SKIPPING: $idx $fn $.: $_";
+	}
+    }
+
+    print "$mtime $idx $fn ::";
+    foreach my $k (sort keys %tally) {
+	print " $k: $tally{$k}";
+    }
+    print "\n";
+    
+}

node_usage/gather

+#!/usr/bin/perl 
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2007 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+use POSIX qw(strftime floor ceil);
+use Data::Dumper;
+use Carp;
+
+$Data::Dumper::Indent = 1;
+
+use strict;
+use warnings;
+
+my $prep    = 1;
+my $results = 1;
+my $fresh = 0;
+
+my $TB		= "/usr/testbed/";
+
+# Turn off line buffering on output
+$| = 1; 
+
+# Load the Testbed support stuff.
+use lib "/usr/testbed/lib";
+use libdb;
+use libtestbed;
+
+chdir "/usr/testbed/data/node_usage/data";
+
+my %node_type;
+my %node_class;
+
+my $qr = DBQueryFatal("select node_id,type from nodes");
+while (my ($node_id, $type) = $qr->fetchrow()) {
+    $node_type{$node_id} = $type;
+}
+
+$qr = DBQueryFatal("select class,type from node_types");
+while (my ($class,$type) = $qr->fetchrow()) {
+    $node_class{$type} = $class;
+}
+$node_class{'?'} = '?';
+
+our %last_trans;
+our %node_state;
+our $prev;
+our $prev_line;
+our $last_history_id = -1;
+
+if ($prep) {
+
+    DBQueryFatal("insert into node_usage.node_history_copy select * from node_history where history_id > (select max(history_id) from node_usage.node_history_copy)");
+
+    if ($fresh) {
+	DBQueryFatal("drop table if exists node_usage.node_trans");
+	DBQueryFatal("create table node_usage.node_trans (".
+		     "  history_id int unsigned not null primary key,".
+		     "  stamp int unsigned not null, ".
+		     "  node_id char(8) not null, ".
+		     "  op enum('alloc','free','invalid') not null".
+		     ")");
+    }
+    
+    local %last_trans;
+    local %node_state;
+
+    if (!$fresh && -e "gather.state.1") {
+	do "gather.state.1";
+    }
+    
+    $qr = DBQueryFatal("select history_id,node_id,op,stamp from node_usage.node_history_copy where history_id > $last_history_id order by history_id");
+
+    local $last_history_id;
+    
+    while (my ($history_id,$node_id,$op,$stamp) = $qr->fetchrow()) {
+	my $type = $node_type{$node_id};
+	$type = '?' unless defined $type;
+	next unless $node_class{$type} eq 'pc';
+	my $prev_state = $node_state{$node_id};
+	$node_state{$node_id} = $op eq 'free' ? 'free' : 'alloc';
+	my $invalid;
+	if (defined $prev_state) {
+	    $invalid = "alloc non-free node" if ($op eq 'alloc' && $prev_state ne 'free');
+	    $invalid = "free already free node" if ($op eq 'free' && $prev_state eq 'free');
+	    $invalid = "move free node" if ($op eq 'move' && $prev_state eq 'free');
+	}
+	if ($invalid) {
+	    #print STDERR "WARNING: $history_id ($stamp) $last_trans{$node_id}: $invalid\n";
+	    DBQueryFatal("update node_usage.node_trans set op = 'invalid' where history_id=$last_trans{$node_id}");
+	} elsif (!defined $prev_state || $prev_state ne $node_state{$node_id}) {
+	    DBQueryFatal("insert into node_usage.node_trans values ($history_id, $stamp, '$node_id', '$node_state{$node_id}')");
+	    $last_trans{$node_id} = $history_id;
+	}
+	$last_history_id = $history_id;
+    }
+
+    open F, ">gather.state.1";
+    print F Data::Dumper->Dump([\%last_trans], ['*last_trans']);
+    print F Data::Dumper->Dump([\%node_state], ['*node_state']);
+    print F Data::Dumper->Dump([$last_history_id], ['*last_history_id']);
+    close F;
+}
+
+if ($results) {
+
+    local %node_state;
+    local $prev = 0;
+    local $prev_line = '';
+    
+    if ($fresh || !-e "gather.state.2") {
+	unlink "node_usage.raw";
+    } else {
+	do "gather.state.2";
+    }
+
+    open F, ">>node_usage.raw";
+
+    $qr = DBQueryFatal("select history_id,stamp,node_id,op from node_usage.node_trans where history_id > $last_history_id order by history_id");
+    
+    while (my ($history_id,$stamp,$node_id,$op) = $qr->fetchrow()) {
+	my $type = $node_type{$node_id};
+	$type = '?' unless defined $type;
+	$node_state{$node_id} = $op;
+	my %tally;
+	while (my ($n, $s) = each %node_state) {
+	    my $t = $node_type{$n};
+	    $t = '?' unless defined $t;
+	    $tally{$t}[0]++;
+	    $tally{$t}[1]++ if $s eq 'free';
+	    $tally{$t}[2]++ if $s eq 'invalid';
+	}
+	my $line = "$history_id $stamp ";
+	foreach my $t (sort keys %tally) {
+	    $tally{$t}[1] = 0 unless defined $tally{$t}[1];
+	    $tally{$t}[2] = 0 unless defined $tally{$t}[2];
+	    $line .= " $t: $tally{$t}[0] $tally{$t}[1] $tally{$t}[2] ";
+	}
+	$line .= "\n";
+	print F $prev_line if $stamp != $prev;
+	$prev = $stamp;
+	$prev_line = $line;
+    }
+
+    open F, ">gather.state.2";
+    print F Data::Dumper->Dump([\%node_state], ['*node_state']);
+    print F Data::Dumper->Dump([$prev, $prev_line], ['*prev', '*prev_line']);
+    close F;
+}
+
+
+

node_usage/index.php

+<?php
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# All rights reserved.
+#
+require("../defs.php3");
+
+PAGEHEADER("Testbed Node Usage Stats");
+
+?>
+
+<p>
+
+These graphs show the average number of free nodes over various time
+periods. Hourly graphs show the average number of free nodes in the
+given hour. Daily graphs show the average number of free nodes in the
+given day. Etc.
+
+<p>
+
+<p align=center>
+
+<b>Last 2 Weeks (hourly)</b>
+<br>
+<img src="node_avail-hourly_last2weeks.png">
+<br> 
+<p align=center>
+
+<b>Last 2 Months (daily)</b>
+<br>
+<img src="node_avail-daily_last2months.png">
+
+<p align=center>
+<hr>
+<p align=center>
+
+<b>A Typical Week (hourly)</b>
+<br>
+<img src="node_avail-by_hourofweek.png">
+
+<p align=center>
+<hr>
+<p align=center>
+
+<b>Since Sep 2005 (yearly)</b>
+
+<br>
+<img src="node_avail-yearly.png">
+
+<p align=center>
+
+<b>Since Sep 2005 (monthly)</b>
+
+<br>
+<img src="node_avail-monthly.png">
+
+<p align=center>
+
+<b>Since Sep 2005 (weekly)</b>
+
+<br>
+<img src="node_avail-weekly.png">
+
+<p align=center>
+
+Note: The gap in the last two graphs represents periods when no data
+was availabe due to bugs in our system.
+
+<?php
+
+PAGEFOOTER();
+
+?>

node_usage/mk-plots

+#!/usr/bin/perl
+
+chdir "/usr/testbed/data/node_usage/data";
+
+sub true {1}
+sub false {0}
+
+#my $TERMINAL = "postscript eps enhanced color";
+#my $EXT = "eps";
+my $TERMINAL = "png";
+my $EXT = "png";
+my $LW = 2;
+
+sub plot ( $@ ) {
+    my ($what, %ops) = @_;
+    open F, ">node_usage-$what.gpl";
+
+    print F "set terminal $TERMINAL\n";
+    print F "set grid\n" unless defined $ops{grid} and !$ops{grid};
+    print F "\n"; 
+
+    if ($ops{timefmt}) {
+        print F "set xdata time\n";
+        print F "set timefmt '$ops{timefmt}'\n";
+    }
+    print F "set format x \"$ops{format_x}\"\n" if defined $ops{format_x};
+    foreach my $o (qw(size xrange xtics)) {
+        print F "set $o $ops{$o}\n" if defined $ops{$o};
+    }
+    print F "set xlabel '$ops{xlabel}'\n" if defined $ops{xlabel};
+    print F "\n";
+
+    my $lw = $ops{lw};
+    $lw = $LW unless defined $lw;
+
+    print F "set output 'node_avail-$what.$EXT'\n";
+    print F "set ylabel 'Available Nodes'\n";
+    print F "plot 'node_usage-$what.dat' using 1:2 title 'pc600 pc850 pc2000 pc3000s' with lines lw $lw,\\\n";
+    print F "     'node_usage-$what.dat' using 1:3 title 'pc3000s' with lines lw $lw\n\n";
+
+    print F "set output 'node_usage-$what.$EXT'\n";
+    print F "set ylabel 'Node Usage'\n";
+    print F "plot 'node_usage-$what.dat' using 1:4 title 'pc600 pc850 pc2000 pc3000s' with lines lw $lw,\\\n";
+    print F "     'node_usage-$what.dat' using 1:5 title 'pc3000s' with lines lw $lw\n\n";
+    close F;
+
+    system("gnuplot node_usage-$what.gpl");
+}
+
+plot("hourly_last2weeks",timefmt=>'%Y-%m-%d_%H:$M', format_x=>'%a\n%m/%d');
+plot("weekly", timefmt=>'%Y-%m-%d');
+plot("daily_last2months", timefmt=>'%Y-%m-%d');
+plot("daily", timefmt=>'%Y-%m-%d', size=>'2,1');
+plot("monthly", timefmt=>'%Y-%m', format_x=>'%m/%y');
+plot("yearly", grid=>false, xlabel => 'Year', xtics=>1);
+plot("by_hour", xrange=>"[0:23]",
+     xtics=>"('Midnight' 0,'4 am' 4, '8 am' 8, 'Noon' 12, '4 pm' 16, '8 pm' 20, '11 pm' 23)");
+#plot("by_hour", xrange=>"[0:23]",
+#     xtics=>("('Midnight' 0,'' 1 1, '' 2 1, '' 3 1, ".
+#             "'4 am' 4, '' 5 1, '' 6 1, ''7 1, ".
+#             "'8 am' 8, '' 9 1, '' 10 1, '' 11 1, ".
+#             "'Noon' 12, '' 13 1, '' 14 1, '' 15 1, ".
+#             "'4 pm' 16, '' 17 1, '' 18 1, '' 19 1, ".
+#             "'8 pm' 20, '' 21 1, '' 22 1, '11 pm' 23)"));
+plot("by_dayofweek", xrange=>"[0:6]",
+     xtics=>"('Mon' 0, 'Tues' 1, 'Wed' 2, 'Thurs' 3, 'Fri' 4, 'Sat' 5, 'Sun' 6)");
+plot("by_hourofweek", xrange=>"[0:7]",
+     xtics=>"('Midnight Mon' 0, 'Tues' 1, 'Wed' 2, 'Thurs' 3, 'Fri' 4, 'Sat' 5, 'Sun' 6)");
+plot("by_month", xrange=>"[0:11]",
+     xtics=>"('Jan' 0, 'Feb' 1, 'Mar' 2, 'Apr' 3, 'May' 4, 'June' 5, 'July' 6, 'Aug' 7, 'Sep' 8, 'Oct' 9, 'Nov' 10, 'Dec' 11)");

node_usage/publish

+#!/bin/sh
+
+cp /usr/testbed/data/node_usage/data/*.png \
+   /usr/testbed/devel/kevina/www/node_usage/

node_usage/refresh

+#!/bin/sh
+
+cd /usr/testbed/data/node_usage/scripts
+
+./gather
+./analy
+./analy2
+./mk-plots
+./publish

node_usage/sanity

+#/usr/bin/perl
+
+use Data::Dumper;
+
+# Turn off line buffering on output
+$| = 1; 
+
+chdir "/usr/testbed/data/node_usage/data";
+
+open A, "node_usage.raw";
+
+my $keep = 100;
+my @data;
+$data[0] = {time => -1};