idlestats.in 10.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#!/usr/bin/perl -wT
#
# Copyright (c) 2016 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use strict;
use English;
use Getopt::Std;
use Date::Parse;
28
use POSIX qw/ceil floor/;
29
use RRDs;
30
use JSON;
31

32
use lib "@prefix@/lib";
33 34 35 36
use libdb;
use libtestbed;
use EmulabConstants;
use Experiment;
37
use Interface;
38 39 40
use Node;
use User;

41 42 43
# Protos
sub get_stats($$$;$);

44
# Constants
45
my $TB = "@prefix@";
46
my $STEP = 3600; # 1 hour (in seconds).  This should be an RRA epoch.
47
my $DEFWINDOW = 86400 * 14; # two weeks (in seconds).
48 49 50
my $MINTIMESPECLEN = 6;
my $MAXTIMESPECLEN = 100;
my $SD_STATSDIR = "$TB/data/slothd_rrd";
51
my $ALLZEROMAC = "000000000000";
52 53

# Globals
54
my $g_doboth = 0;
55
my $g_doraw = 0;
56 57
my $g_valtype = "MAX";
my $g_now = time();
Kirk Webb's avatar
Kirk Webb committed
58
my $g_end = floor($g_now/$STEP)*$STEP + $STEP; # Now, normalized to STEP.
59
my $g_start = $g_end - floor($DEFWINDOW/$STEP)*$STEP + 2*$STEP;  # Default window, normalized.
60 61
my $g_experiment;
my @g_nodelist = ();
62
my $g_silent = 0;
63 64 65

sub usage() {
    print STDERR
66
	"Return JSON-encoded node activity stastics.\n\n".
67 68
	"Usage: $0 [-d] [-A|-B] [-R] [-S <start_time>] [-E <end_time>] node [node ...]\n" .
	"       $0 [-d] [-A|-B] [-R] [-S <start_time>] [-E <end_time>] -e <pid>,<eid>\n".
69
	"-d:              turn on debugging.\n" .
70
	"-s:              silent mode, no warnings\n" .
71
	"-A:              return averages instead of maximums.\n".
72
	"-B:              return both average and maximum data points.\n".
73
	"-R:              include the latest day's raw 5 minute samples.\n".
74 75 76 77
	"-e <pid>,<eid>:  request data for nodes in an experiment.\n".
	"-S <start_time>: bound the start of the returned data.\n".
	"                 Default is beginning of available data for a list of nodes,\n".
	"                 or the beginning of the specified experiment.\n".
78
	"-E <end_time>:   bound the end of the returned data. Default is 'now'.\n".
79
	"\n".
80 81 82 83 84 85 86
	"Start/end times can be specified as anything recognized by the\n".
	"Date::Parse module. When requesting experiment data, start times\n".
	"prior to the start of the experiment will be truncated to the beginning\n". 
	"of the experiment (with a warning). The start time must be less than\n".
	"the end time. Returned data is reported at a fixed 1 hour granularity.\n".
	"Data series with no data points are indicated as such with stub\n".
	"entries in the output.\n";
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
    exit 1;
}

# un-taint path
$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

# Turn off line buffering on output
$| = 1;

#
# Verify user and get his DB uid and other info for later.
#
my $user;
if ($UID) {
    $user = User->ThisUser();
    if (! defined($user)) {
	die("*** $0:\n".
	    "    You ($UID) do not exist!\n");
    }
}

my %opts = ();

111
if (!getopts("dhABRS:E:e:s", \%opts) || $opts{'h'}) {
112 113 114
    usage();
}

115
if ($opts{'A'}) {
116 117 118
    $g_valtype = "AVERAGE";
}

119 120 121 122
if ($opts{'s'}) {
    $g_silent = 1;
}

123 124
if ($opts{'B'}) {
    $g_doboth = 1;
125 126
}

127 128 129 130
if ($opts{'R'}) {
    $g_doraw = 1;
}

131 132
if ($opts{'e'}) {
    # Lookup will untaint the argument.
133 134
    $g_experiment = Experiment->Lookup($opts{'e'});
    if (!$g_experiment) {
135 136 137 138
	warn "No such experiment: $opts{'e'}\n";
	exit 1;
    }
    if ($UID &&
139 140
	!$g_experiment->AccessCheck($user, TB_EXPT_READINFO)) {
	warn "You ($user) do not have access to experiment $g_experiment\n";
141 142
	exit 1;
    }
143 144
    if ($g_experiment->state() ne EXPTSTATE_ACTIVE) {
	warn "Experiment $g_experiment is not active!\n";
145 146
	exit 1;
    }
147
    @g_nodelist = $g_experiment->NodeList(0,1);
148 149 150
    # Bump start time to the beginning of this experiment.  Note that the
    # first data point may include data from prior to the start of the
    # experiment!
151
    $g_start = ceil($g_experiment->swapin_time()/$STEP)*$STEP;
152 153 154
}

if (@ARGV) {
155
    if ($g_experiment) {
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
	warn "You may request stats for an experiment, or a list of nodes, but not both!\n";
	exit 1;
    }

    foreach my $node_id (@ARGV) {
	# Lookup will untaint arguments
	my $node = Node->Lookup($node_id);
	if (!$node) {
	    warn "Unknown node: $node_id\n";
	    exit 1;
	}
	if ($UID &&
	    !$node->AccessCheck($user, TB_NODEACCESS_READINFO)) {
	    warn "You ($user) do not have access to $node\n";
	    exit 1;
	}
172
	push @g_nodelist, $node;
173 174 175
    }
}

176
if (!@g_nodelist) {
177 178 179 180
    warn "No nodes to operate on (no nodes in experiment, or no nodes listed on command line)!\n";
    exit 1;
}

181 182 183 184 185 186 187 188 189 190 191
if ($opts{'S'}) {
    if ($opts{'S'} !~ /^([-.:\/,\w\s]{$MINTIMESPECLEN,$MAXTIMESPECLEN})$/) {
	warn "Illegal start time spec!\n";
	exit 1;
    }
    my $stime = str2time($1);
    if (!defined($stime)) {
	warn "Start time could not be parsed!\n";
	exit 1;
    }
    $stime = floor($stime/$STEP)*$STEP;
192
    if ($g_experiment && $stime < $g_start) {
193
	warn "Specified start time is prior to start of experiment!\n".
194 195
	     "Truncating to: $g_start\n"
	     if (!$g_silent);
196
    } else {
197
	$g_start = $stime;
198 199 200
    }
}

201 202 203 204 205 206 207 208 209 210 211
if ($opts{'E'}) {
    if ($opts{'E'} !~ /^([-.:\/,\w\s]{$MINTIMESPECLEN,$MAXTIMESPECLEN})$/) {
	warn "Illegal end time spec!\n";
	exit 1;
    }
    my $etime = str2time($1);
    if (!defined($etime)) {
	warn "End time could not be parsed!\n";
	exit 1;
    }
    $etime = floor($etime/$STEP)*$STEP;
212
    if ($etime > $g_now) {
213 214
	warn "End time is in the future! Truncated to: $g_now\n"
	    if (!$g_silent);
215
    }
216
    else {
217
	$g_end = $etime;
218 219 220
    }
}

221
if ($g_start > $g_end) {
222 223 224 225
    warn "Start time must be less than or equal to end time!\n";
    exit 1;
}

226 227 228 229
sub get_stats($$$;$) {
    my ($rrdfile, $dtype, $header, $filter) = @_;
    my ($start, $end, $step) = ($g_start, $g_end, $STEP);
    my $rawvals;
230

231 232 233 234 235 236 237 238 239 240
    if ($dtype eq "RAW") {
	$step = 300; # 5 minutes;
	$end = floor($g_now/$step)*$step; # now, normalized to step.
	$start = $end - 86400; # a day's worth of samples, but...
	# Snap to the start time if it is less than a day prior to now.
	# It should already be aligned to five minutes.
	if ($g_start > $start) {
	    $start = $g_start;
	}
	$dtype = "AVERAGE";
241
    }
242 243 244 245 246 247 248 249
    elsif ($g_doraw) {
	$rawvals = get_stats($rrdfile, "RAW", $header, $filter);
	my $rawstart = floor(($g_now-86400)/$STEP)*$STEP - $STEP;
	if ($start <= $rawstart) {
	    $end = $rawstart;
	} else {
	    return $rawvals;
	}
250 251 252
    }

    my ($rrd_stamp,$rrd_step,$rrd_names,$rrd_data) = 
253 254
	RRDs::fetch($rrdfile, $dtype, "--start=$start", "--end=$end", 
		    "--resolution=$step");
255
    if (RRDs::error) {
256 257
	warn "Could not get interface data from $rrdfile: ". RRDs::error ."\n"
	    if (!$g_silent);
258 259
	next;
    }
260 261
    my $hasvalues = 0; # track whether or not any data exists.
    my @tmpvals = ($header,);
262
    foreach my $rrd_line (@$rrd_data) {
263 264 265 266 267 268 269
	$rrd_line = $filter->($rrd_stamp, $rrd_line)
	    if $filter;
	foreach my $val (@$rrd_line) {
	    $hasvalues = 1
		if (defined($val));
	}
	push @tmpvals, [$rrd_stamp, @$rrd_line];
270 271 272
	$rrd_stamp += $rrd_step;
    }
    if ($hasvalues) {
273 274 275 276 277 278
	# Tack on raw values if they were requested and retrieved.
	if ($rawvals && @$rawvals) {
	    shift @$rawvals;  # Get rid of header.
	    return [@tmpvals, @$rawvals];
	}
	return \@tmpvals;
279 280 281 282
    }
    return [];
}

283
# Do all the things!
284
my @results = ();
285
foreach my $node (@g_nodelist) {
286
    my $node_id = $node->node_id();
287 288
    my $nobj = {};
    $nobj->{'node_id'} = $node_id;
289

Kirk Webb's avatar
Kirk Webb committed
290
    #
291
    # Process top-level node stats.
Kirk Webb's avatar
Kirk Webb committed
292 293 294 295
    #
    # Track whether or not there are data points in the time query range.
    # If not, return an empty array instead of a list of undefined values.
    #
296
    my $mainrrd = "$SD_STATSDIR/${node_id}.rrd";
297 298 299 300 301 302 303
    my $mheader = ["timestamp","load_1min","load_5min","load_15min"];
    my $f_main = sub {
	my ($tstamp, $vals) = @_;
	shift @$vals; # remove the 'last_tty' timestamp.
	return $vals;
    };

304
    if (!-f $mainrrd) {
305 306
	warn "Could not find main rrd file ($mainrrd) for $node_id\n"
	    if (!$g_silent);
307
	$nobj->{'main'} = []; # Indicate no data found.
308 309
    }
    else {
310
	if ($g_doboth) {
311 312 313 314 315 316 317 318
	    my $avg = get_stats($mainrrd, "AVERAGE", $mheader, $f_main);
	    my $max = get_stats($mainrrd, "MAX", $mheader, $f_main);
	    if (@$avg || @$max) {
		$nobj->{'main'}->{'AVG'} = $avg;
		$nobj->{'main'}->{'MAX'} = $max;
	    } else {
		$nobj->{'main'} = [];
	    }
Kirk Webb's avatar
Kirk Webb committed
319
	} else {
320
	    $nobj->{'main'}->{($g_valtype eq "MAX" ? "MAX" : "AVG")} = 
321
		get_stats($mainrrd, $g_valtype, $mheader, $f_main);
Kirk Webb's avatar
Kirk Webb committed
322
	}
323
    }
324

Kirk Webb's avatar
Kirk Webb committed
325
    #
326
    # Process interface statistics.
Kirk Webb's avatar
Kirk Webb committed
327 328 329 330 331 332 333
    #
    # Get the set of known interfaces for this node.  We only consider
    # control and experimental interfaces.  We elide oddball interfaces
    # with an all-zero MAC address.  Track whether or not we find statistics
    # for each interface.  We will mark interfaces with no stats by returning
    # an empty array for them.
    #
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
    my @interfaces = ();
    my %ifmap = ();
    my $ctrlmac = "*unknown*";
    Interface->LookupAll($node, \@interfaces);
    foreach my $intf (@interfaces) {
	next if ($intf->mac() eq $ALLZEROMAC);
	if ($intf->IsControl()) {
	    $ctrlmac = uc($intf->mac());
	    $ifmap{$ctrlmac} = $intf;
	    $intf->{'SEEN'} = 0;
	}
	elsif ($intf->IsExperimental()) {
	    $ifmap{uc($intf->mac())} = $intf;
	    $intf->{'SEEN'} = 0;
	}
    }
Kirk Webb's avatar
Kirk Webb committed
350 351
    $nobj->{'interfaces'}->{'ctrl_iface'} = $ctrlmac; # communicate ctrl iface.
    my @intfrrds = glob "$SD_STATSDIR/${node_id}-*.rrd"; # iface stats files.
352
    my $iheader = ["timestamp","ipkt_rate","opkt_rate"];
353
    foreach my $intfrrd (@intfrrds) {
354
	$intfrrd =~ /${node_id}-([0-9a-f]{12}).rrd$/i;	
Kirk Webb's avatar
Kirk Webb committed
355
	next if (!$1); # skip if mac addr in filename is malformed.
356
	my $mac = uc($1);
Kirk Webb's avatar
Kirk Webb committed
357 358
	next if (!exists($ifmap{$mac})); # skip if iface is not in DB.
	$ifmap{$mac}->{'SEEN'} = 1; # mark.
359
	if ($g_doboth) {
360 361 362 363 364 365 366 367
	     my $avg = get_stats($intfrrd, "AVERAGE", $iheader);
	     my $max = get_stats($intfrrd, "MAX", $iheader);
	     if (@$avg || @$max) {
		 $nobj->{'interfaces'}->{$mac}->{"AVG"} = $avg;
		 $nobj->{'interfaces'}->{$mac}->{"MAX"} = $max;
	     } else {
		 $nobj->{'interfaces'}->{$mac} = [];
	     }
368
	} else {
369 370
	    $nobj->{'interfaces'}->{$mac}->{($g_valtype eq
					     "MAX" ? "MAX" : "AVG")} =
371
		get_stats($intfrrd, $g_valtype, $iheader);
372 373
	}
    }
Kirk Webb's avatar
Kirk Webb committed
374 375
    # Indicate no data found for interfaces where there is no
    # RRD stats file.
376 377
    foreach my $mac (keys %ifmap) {
	if (!$ifmap{$mac}->{'SEEN'}) {
378
	    $nobj->{'interfaces'}->{$mac} = [];
379
	}
380
    }
Kirk Webb's avatar
Kirk Webb committed
381 382

    # Add node data structure to results set.
383 384 385
    push @results, $nobj;
}

386
print to_json(\@results);
387
exit 0;