From 872a5af1f2f26bff20a3353ed1d3cbb3d0ac9560 Mon Sep 17 00:00:00 2001 From: Leigh B Stoller Date: Mon, 4 Jun 2018 07:44:15 -0600 Subject: [PATCH] New script to compute reservation timelines and utilization number. Initially intended for debugging, but now its more useful. :-) --- utils/GNUmakefile.in | 2 +- utils/resutil.in | 302 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 utils/resutil.in diff --git a/utils/GNUmakefile.in b/utils/GNUmakefile.in index 8b8b4ae15..9a416cf30 100644 --- a/utils/GNUmakefile.in +++ b/utils/GNUmakefile.in @@ -59,7 +59,7 @@ SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \ addrfdevice addrfpath reserve announce createimagealias \ predict test-reserve notify-reservations \ deprecate_image pushrootkey addinterface addwire cnetwatch \ - addswitch addtypetoimages + addswitch addtypetoimages resutil WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \ webspewimage webdumpdescriptor webemulabfeature \ diff --git a/utils/resutil.in b/utils/resutil.in new file mode 100644 index 000000000..5564f3397 --- /dev/null +++ b/utils/resutil.in @@ -0,0 +1,302 @@ +#!/usr/bin/perl -w +# +# Copyright (c) 2016-2018 University of Utah and the Flux Group. +# +# {{{EMULAB-LICENSE +# +# This file is part of the Emulab network testbed software. +# +# This file is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This file is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +# License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this file. If not, see . +# +# }}} +# +use strict; +use English; +use Getopt::Std; +use Date::Parse; +use Data::Dumper; +use POSIX; + +# +# Configure variables +# +my $TB = "@prefix@"; +my $TBOPS = "@TBOPSEMAIL@"; + +# +# Turn off line buffering on output +# +$| = 1; + +# +# Untaint the path +# +$ENV{'PATH'} = "/bin:/sbin:/usr/bin:"; + +# +# Testbed Support libraries +# +use lib "@prefix@/lib"; +use emdb; +use libtestbed; +use User; +use Project; +use Reservation; +use WebTask; +use emutil; +use libEmulab; +use ResUtil; + +sub usage() +{ + print STDERR "Usage: resutil ...\n"; + exit(-1); +} +my $optlist = "dt:au:r"; +my $debug = 0; +my $allres = 0; +my $useronly; +my $webtask_id; +my $webtask; +my $project; +my @reservations; +my @tmp = (); + +sub fatal($) +{ + my ($mesg) = $_[0]; + + if (defined($webtask)) { + $webtask->Exited(-1); + $webtask->output($mesg); + } + die("*** $0:\n". + " $mesg\n"); +} + +# +# Parse command arguments. Once we return from getopts, all that should be +# left are the required arguments. +# +my %options = (); +if (! getopts($optlist, \%options)) { + usage(); +} +if (defined($options{d})) { + $debug = 1; + ResUtil::DebugOn(); +} +if (defined($options{"a"})) { + $allres = 1; +} +if (defined($options{"u"})) { + $useronly = User->Lookup($options{"u"}); + if (!defined($useronly)) { + fatal("No such user"); + } +} +if (defined($options{"t"})) { + $webtask_id = $options{"t"}; + $webtask = WebTask->Lookup($webtask_id); + if (!defined($webtask)) { + fatal("Could not lookup webtask $webtask_id"); + } + # Convenient. + $webtask->AutoStore(1); +} + +if (defined($options{"r"})) { + my $res = Reservation->Lookup($ARGV[0]); + if (!defined($res)) { + $res = Reservation->LookupHistorical($ARGV[0]); + if (!defined($res)) { + fatal("Could not lookup reservation"); + } + } + $project = Project->Lookup($res->pid()); + if (!defined($project)) { + fatal("Could not lookup project for reservation"); + } + @reservations = ($res); +} +elsif (@ARGV) { + $project = Project->Lookup($ARGV[0]); + if (!defined($project)) { + fatal("No such project"); + } + + @reservations = CollectReservations($project, $useronly, $allres); + if (!@reservations) { + print STDERR "No reservations to process\n"; + if (defined($webtask)) { + $webtask->Exited(1); + $webtask->output("No reservations to process"); + } + exit(1); + } +} + +# +# Given a reservation details hash, calculate a utilization number +# from the history array. +# +sub ReservationUtilization($@) +{ + my ($res, @records) = @_; + my $count = $res->nodes(); + my $active = defined($res->idx()) ? 1 : 0; + my $resstart = $res->start(); + my $resend = ($active ? time() : + (defined($res->{'deleted'}) ? + $res->{'deleted'} : $res->{'end'})); + my $reshours = (($resend - $resstart) / 3600) * $count; + my $usedhours = 0; + my $inuse = 0; + my $laststamp = $resstart; + my $type = $res->type(); + my $uid = $res->uid(); + my @tmp = @records; + + # Init for caller. + $res->{'reshours'} = $reshours; + $res->{'usedhours'} = undef; + $res->{'utilization'} = undef; + + # + # Scan past timeline entries that are *before* the start of the + # reservation; these are experiments that were running when the + # reservation started, and provide the number of nodes allocated + # at the time the reservation starts. + # + while (@tmp) { + my $ref = $tmp[0]; + my $stamp = $ref->{'t'}; + my $allocated = $ref->{'allocated'}; + $ref->{'tt'} = TBDateStringGMT($ref->{'t'}); + + last + if ($stamp >= $resstart); + + # Watch for nothing allocated by the user at this time stamp + my $using = 0; + if (exists($allocated->{$uid})) { + $using = $allocated->{$uid}->{$type}; + } + $inuse = $using; + $inuse = $count if ($inuse > $count); + shift(@tmp); + } + foreach my $ref (@tmp) { + my $stamp = $ref->{'t'}; + my $reserved = $ref->{'reserved'}; + my $allocated = $ref->{'allocated'}; + $ref->{'tt'} = TBDateStringGMT($ref->{'t'}); + + # If this stamp is after the reservation, we can stop. The + # last entry will be the current number of nodes used till + # the end of the reservation. This entry is typically for the + # end of an experiment start before the end of the reservation. + last + if ($stamp > $resend); + + # Watch for nothing allocated by the user at this time stamp + my $using = 0; + if (exists($allocated->{$uid})) { + $using = $allocated->{$uid}->{$type}; + } + $usedhours += (($stamp - $laststamp) / 3600) * $inuse; + $laststamp = $stamp; + $inuse = $using; + $inuse = $count if ($inuse > $count); + } + # And then a final entry for usage until the end of the reservation. + if ($laststamp) { + $usedhours += (($resend - $laststamp) / 3600) * $inuse; + } + $res->{'reshours'} = $reshours; + $res->{'usedhours'} = $usedhours; + $res->{'utilization'} = POSIX::ceil(($usedhours/$reshours) * 100.0); + $res->{'utilization'} = 100 if ($res->{'utilization'} > 100); + + return 0; +} + +sub Process($$) +{ + my ($project, $reservation) = @_; + + # + # Timeline of alloc/free operations. + # + my @records = CreateTimeline($project, $reservation); + + ReservationUtilization($reservation, @records); + + # + # OK, we got lots of data! For each entry we have counts for all the node + # types allocated by the project, all the node types allocated by each user + # in the project. We also have (from the list of active reservations at + # the time), counts of node types reserved to the project, and by each user + # who created those reservations. + # + foreach my $record (@records) { + my $stamp = $record->{'t'}; + my $reserved = $record->{'reserved'}; + my $allocated = $record->{'allocated'}; + + # The timeline includes stamps when nothing is allocated, but we + # need them for computing the counts. + next + if (!keys(%$reserved)); + + print POSIX::strftime("%m/%d/20%y %H:%M:%S", localtime($stamp)) . " \n"; + + # + # Show the project info first, for each type. + # + my $pid = $project->pid(); + my $res = $reserved->{$pid}; + my $alloc = $allocated->{$pid}; + foreach my $type (keys(%$res)) { + my $rescount = $res->{$type}; + my $using = $alloc->{$type}; + printf(" %-15s %-10s %s\n", "Project", $type, "$rescount/$using"); + } + foreach my $who (sort(keys(%$reserved))) { + next + if ($who eq $pid); + next + if ($useronly && $who ne $useronly->uid()); + + my $res = $reserved->{$who}; + my $alloc = $allocated->{$who}; + next + if (!defined($alloc)); + + foreach my $type (keys(%$res)) { + my $rescount = $res->{$type}; + my $using = $alloc->{$type}; + printf(" %-15s %-10s %s\n", $who, $type, "$rescount/$using"); + } + } + } +} + +foreach my $res (@reservations) { + my $project = Project->Lookup($res->pid()); + Process($project, $res); + print "Utilization: " . $res->{'utilization'} . "%\n"; +} +exit(0); + -- GitLab