Commit 931930ae authored by Leigh B. Stoller's avatar Leigh B. Stoller

A number of changes:

* Put in debugging timestamps to figure out why so slow.

* Cleanup, fixing bad identing, bad style, bad comments, etc.

* Restructure so that instead of locking three tables (nodes,
  reserved, and node_activity), we just need to lock the reserved
  table, by moving the node existence check to before the main loop,
  and the node_activity update to after we unlock tables. Not locking
  the nodes table results in less serialization and less waiting for
  the lock to be granted.

* XXX: Check each node for an entry in tiplines table; Do not call
  into console_setup.in if none of the nodes have tiplines. This is
  not a very clean thing to do of course, but it saves up to half the
  time it takes to do an nalloc cause of the min startup time for our
  perl scripts (almost 1/2 a second). When allocating virtnodes on a
  physnode, no reason to even call console_setup.in.

Bottom line? Well, not as good as one would like ... time to allocate
10 vnodes on a pnode goes from .8 seconds to .4 seconds ...
parent fe523384
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -10,59 +9,82 @@ use Getopt::Std;
#
# nalloc - allocate nodes to an experiment. Takes a pid, and eid, and
# a list of nodes. Only nodes that are free can be allocated, and a
# special hack for sharks is included - allocating 'sh*' expands to
# allocation of the entire rack.
# a list of nodes. Only nodes that are free can be allocated.
#
# Exit status is important! Exit with -1 if an error, else the number
# of nodes that could not be allocated. Otherwise zero.
#
sub usage()
{
print("Usage: nalloc [-v] [-p] <pid> <eid> <node> <node> <...>\n".
" -p enables partial allocation mode\n".
" -v enables debugging output\n");
exit(-1);
}
my $optlist = "dp";
my $debug = 0;
my $partial = 0;
#
# Configure variables
#
my $TB = "@prefix@";
my $TB = "@prefix@";
my $consetup = "$TB/libexec/console_setup";
my $exportsetup = "$TB/sbin/exports_setup";
#
# Testbed Support libraries
#
use lib '@prefix@/lib';
use libdb;
use libtestbed;
my %opt = ();
getopts('vp', \%opt);
#
# Turn off line buffering on output
#
$| = 1;
my $debug = 0;
my $partial = 0;
# For perf analysis.
#TBDebugTimeStampsOn();
if ($opt{v}) {
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if ($opt{p}) {
if (defined($options{"p"})) {
$partial = 1;
}
if (@ARGV < 2) {
die("Usage: nalloc [-v] [-p] <pid> <eid> <node> <node> <...>\n".
" -p enables partial allocation mode\n".
" -v enables debugging output\n");
usage();
}
my $pid = shift;
my $eid = shift;
my @node_names = @ARGV;
my $consetup="$TB/libexec/console_setup";
my $exportsetup="$TB/sbin/exports_setup";
my $error = 0;
my $noalloc = 0;
my $pid = shift;
my $eid = shift;
my @node_names=@ARGV;
my @vals = ();
my @nodes= ();
my $oldreserved_pid = OLDRESERVED_PID;
my $oldreserved_eid = OLDRESERVED_EID;
if ($debug) { print "Expt '$eid', proj '$pid'\n"; }
#
# Locals
#
my $error = 0;
my $noalloc = 0;
my @vals = ();
my @nodes = ();
my @need_consetup = 0;
TBDebugTimeStamp("nalloc started");
#
# Experiment must exist.
#
if (!ExpState($pid,$eid)) {
die "There is no experiment $eid in project $pid\n";
}
TBDebugTimeStamp("nalloc checked exp state");
#
# User must have permission to modify the experiment.
......@@ -72,115 +94,112 @@ if ($UID) {
die("*** You not have permission to allocate nodes in $pid/$eid!\n");
}
}
TBDebugTimeStamp("nalloc checked exp permission");
#
# Before locking any tables, do a quick check to make sure the project
# is allowed to allocate the nodes, by type/class.
# is allowed to allocate the nodes, by type/class, plus other checks.
#
foreach my $n (@node_names) {
# Make sure a valid node name first!
if (! TBValidNodeName($n)) {
die("*** $0:\n".
" No such node $n!\n");
}
# Project allowed to allocated this node type/class?
if (! TBNodeAllocCheck($pid, $n)) {
die("*** $0:\n".
" You are not allowed to allocate $n to project $pid!\n");
}
}
######################################################################
# Step 1 - Make a list of nodes to reserve
#
# We prune nodes that are already reserved, etc., from the list, and
# do expansion of shark shelves
######################################################################
# XXX
# If the node has a tipline, then indicate that we want to call
# console_setup. We want to avoid calling console_setup if all we
# got is a zillion virtual nodes on the command line; wasted work.
#
my $tiplines_result =
DBQueryFatal("select tipname,server from tiplines ".
"where node_id='$n'");
if ($tiplines_result->numrows) {
$need_consetup++;
}
}
TBDebugTimeStamp("nalloc checked node permission ");
DBQueryFatal("lock tables nodes read, reserved write, node_activity write");
# Must lock this table!
DBQueryFatal("lock tables reserved write");
TBDebugTimeStamp("nalloc locked tables");
# Make a list of nodes to reserve.
foreach my $n (@node_names) {
# Shark hack
if ($n =~ /(sh\d+)/ ) { $n = $1."-1"; }
# End shark hack
my ($r_pid, $r_eid);
if (NodeidToExp($n,\$r_pid,\$r_eid)) {
if (NodeidToExp($n, \$r_pid, \$r_eid)) {
# Someone has already reserved this node
if (($r_pid eq $pid) && ($r_eid eq $eid)) {
print "You have already reserved node '$n'.\n";
# Do not increment error code since that throws off tbprerun.
print "Already reserved: '$n'\n";
# Do not increment error code since that throws off caller.
next;
} elsif(NodeidToExpOldReserved($n, \$r_pid, \$r_eid) &&
(($r_pid eq $pid) && ($r_eid eq $eid)) ) {
print "You have already reserved node '$n' which was moved to ".
"a holding reservation: $oldreserved_pid/$oldreserved_eid.\n";
} else {
}
elsif (NodeidToExpOldReserved($n, \$r_pid, \$r_eid) &&
(($r_pid eq $pid) && ($r_eid eq $eid))) {
print "Already reserved in holding reservation: '$n'\n";
}
else {
print "Someone else has already reserved node '$n'.\n";
$noalloc++;
next;
}
}
my $result = DBQueryFatal("select * from nodes where node_id='$n'");
if ($result->numrows() < 1) {
print "Node '$n' does not exist.\n";
$error++;
next;
} else {
# No one has reserved it, and it exists, so add it to my list
# Shark hack
if ($n =~ /(sh\d+)/ ) {
# its a shark - do the whole shelf if its not done already.
my $shelf = $1;
if ( ! (join(",",@nodes) =~ /,$shelf-\d,/)) {
# Shelf hasn't been done yet...
foreach my $n ( 1 .. 8 ) {
push(@vals,"('$shelf-$n','$pid','$eid')");
push(@nodes,"$shelf-$n");
}
}
# End shark hack
} else {
# its not a shark - just add it in...
push(@vals,"('$n','$pid','$eid','$n','','')");
push(@nodes,"$n");
}
}
#
# Add info the list of nodes to reserve; done in a single query below.
#
push(@vals, "('$n','$pid','$eid','$n','','')");
push(@nodes, "$n");
}
TBDebugTimeStamp("nalloc checked all nodes");
if ($debug) { print "List Ready: @vals\nError=$error\n"; }
######################################################################
# Step 2 - Make the reservations in the database
#
# Uses the list built in step 1
######################################################################
if ($debug) {
print "List Ready: @vals\nError=$error\n";
}
if (!$error && (!$noalloc || $partial) && @vals) {
if ($debug) {
print "Resetting node activity...\n";
}
my $cmd = "update node_activity set last_ext_act = now() where ".
join(" or ",map("node_id='$_'",@nodes));
DBQueryWarn($cmd);
# It isn't an error if this fails...
# Now make the reservations in the DB.
if ((!$noalloc || $partial) && @vals) {
print "Reserving nodes...\n"
if ($debug);
if ($debug) {
print "Reserving nodes...\n";
}
$cmd = "replace into reserved (node_id,pid,eid,vname,old_pid,old_eid) values ".
join(",",@vals);
if (!DBQueryWarn($cmd)) {
if (! DBQueryWarn("replace into reserved ".
" (node_id,pid,eid,vname,old_pid,old_eid) ".
"values ". join(",",@vals))) {
$error++;
}
}
TBDebugTimeStamp("nalloc allocated all nodes");
# Unlock tables.
DBQueryFatal("unlock tables");
TBDebugTimeStamp("nalloc unlocked tables");
######################################################################
# Step 3 - Setup consoles
#
# Uses the list built in step 1
######################################################################
# Update node_activity table; no need to do this with tables locked.
if (!$error && (!$noalloc || $partial) && @nodes) {
print "Resetting node activity\n"
if ($debug);
# It isn't an error if this fails...
DBQueryWarn("update node_activity set last_ext_act=now() ".
"where " . join(" or ",map("node_id='$_'", @nodes)));
TBDebugTimeStamp("nalloc updated node_activity table");
}
if (!$error && @nodes) {
# Now setup consoles if needed.
if ($need_consetup && !$error && @nodes) {
system("$consetup @nodes") == 0 or
print STDERR "WARNING: $consetup @nodes failed!\n";
TBDebugTimeStamp("nalloc finished console setup");
}
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment