Commit 103e0385 authored by Leigh Stoller's avatar Leigh Stoller

Add support for multiple pre-reservations per project:

When creating a pre-reserve, new -n option to specify a name for the
reservation, defaults to "default". All other operations require an
-n option to avoid messing with the wrong reservation. You are not allowed
to reuse a reservation name in a project, of course. Priorities are
probably more important now, we might want to change the default from 0 to
some thing higher, and change all the current priorities.

For bookkeeping, the nodes table now has a reservation_name slot that is
set with the reserved_pid. This allows us to revoke the nodes associated
with a specific reservation. Bonus feature is that when setting the
reserved_pid via the web interface, we leave the reservation_name null, so
those won't ever be revoked by the prereserve command line tool.

New feature; when revoking a pre-reserve, we now look to see if nodes being
revoked are free and can be assigned to other pre-reserves. We used to not
do anything, and so had to wait until that node was allocated and released
later, to see if it could move into a pre-reserve.

Also a change required by node specific reservations; when we free a node,
need to make sure we actually use that node, so have to cycle through all
reservations in priority order until it can used. We did not need to do
this before.
parent 70c94bca
......@@ -3654,9 +3654,9 @@ sub SetJailIPFromVnode($$$)
#
# Check for, and update a node pre reservation.
#
sub CheckPreReserve($$)
sub CheckPreReserve($$$)
{
my ($self, $quiet) = @_;
my ($self, $isfree, $quiet) = @_;
my $result = undef;
#
......@@ -3666,17 +3666,33 @@ sub CheckPreReserve($$)
my $node_id = $self->node_id();
DBQueryWarn("lock tables project_reservations write, nodes write, ".
" node_reservations write")
" node_reservations write, reserved read")
or return undef;
#
# isfree is a flag that says we are coming from nfree or stated,
# and the node is going into the free pool. When not set (as from
# prereserve) we have to check the reserved table to see if it is
# free; we do not mess with an allocated node. We do this here with
# the rest of the locked tables.
#
if (!$isfree) {
my $query_result =
DBQueryWarn("select pid,eid from reserved ".
"where node_id='$node_id'");
goto done
if (!$query_result || $query_result->numrows);
}
#
# Need to check for existing reserved_pid, but have to go to the DB,
# not look in the object ($self) since it might be stale.
#
my $query_result =
DBQueryWarn("select reserved_pid,count,active from nodes ".
DBQueryWarn("select reserved_pid,count,active,terminal from nodes ".
"left join project_reservations on ".
" project_reservations.pid=nodes.reserved_pid ".
" project_reservations.pid=nodes.reserved_pid and ".
" project_reservations.name=nodes.reservation_name ".
"where nodes.node_id='$node_id'");
# numrows would be zero if the node was suddenly deleted.
......@@ -3688,12 +3704,13 @@ sub CheckPreReserve($$)
# if the reservation request is still active. If not, we can clear it,
# which will allow it to be set again below, if needed.
#
my ($pid,$count,$active) = $query_result->fetchrow_array();
my ($pid,$count,$active,$terminal) = $query_result->fetchrow_array();
if (defined($pid)) {
goto done
if (defined($count) && $active);
if (defined($count) && $active && !$terminal);
DBQueryWarn("update nodes set reserved_pid=null ".
DBQueryWarn("update nodes set reserved_pid=null, ".
" reservation_name=null ".
"where node_id='$node_id'");
if (!$quiet) {
......@@ -3703,25 +3720,26 @@ sub CheckPreReserve($$)
# Find only active unfilled reservations.
$query_result =
DBQueryWarn("select pid,count from project_reservations ".
"where active=1 and count>0 and ".
DBQueryWarn("select pid,count,name from project_reservations ".
"where active=1 and terminal=0 and count>0 and ".
" (types is null or ".
" FIND_IN_SET('$type', types)) ".
"order by priority desc, created asc ".
"limit 1");
"order by priority desc, created asc ");
goto done
if (!$query_result);
if ($query_result->numrows) {
my ($pid,$count) = $query_result->fetchrow_array();
while ($query_result->numrows) {
my ($pid,$count,$resname) = $query_result->fetchrow_array();
#
# See if this is fullfulling a specific node reservation request.
# Need to delete that row if so.
# Need to delete that row if so.
#
my $noderes_result =
DBQueryWarn("select node_id from node_reservations ".
"where pid='$pid'");
"where pid='$pid' and ".
" reservation_name='$resname'");
goto done
if (!$noderes_result);
......@@ -3740,34 +3758,39 @@ sub CheckPreReserve($$)
}
}
#
# XXX We should look for another reservation to fulfill.
# We cannot use this node for this pre-reserve, but maybe the
# next one.
#
goto done
next
if (!$okay);
}
if (DBQueryWarn("update nodes set reserved_pid='$pid' ".
if (DBQueryWarn("update nodes set reserved_pid='$pid', ".
" reservation_name='$resname' ".
"where node_id='$node_id'")) {
DBQueryWarn("update project_reservations set count=count-1 ".
"where pid='$pid'");
"where pid='$pid' and name='$resname'");
if ($noderes_result->numrows) {
DBQueryWarn("delete from node_reservations ".
"where node_id='$node_id'");
"where node_id='$node_id' and pid='$pid' and ".
" reservation_name='$resname'");
}
$result = $pid;
if ($count == 1) {
SENDMAIL($TBOPS, "Pre Reservation for $pid has completed",
"The pre reservation request for project $pid, ".
"has been fullfilled\n", $TBOPS);
SENDMAIL($TBOPS,
"Pre Reservation $pid,$resname has completed",
"The pre reservation request for project ".
"$pid,$resname has been fullfilled\n", $TBOPS);
}
last;
}
}
done:
DBQueryWarn("unlock tables");
if (defined($result) && !$quiet) {
print "Setting pre reserve for $node_id to $result\n";
print "Setting pre-reserve for $node_id to $result\n";
}
return $result;
}
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -660,7 +660,7 @@ foreach my $node (@freed_nodes) {
or $error++;
# Handle pre-reserve.
my $rpid = $node->CheckPreReserve($quiet);
my $rpid = $node->CheckPreReserve(1, $quiet);
}
next;
}
......@@ -677,7 +677,7 @@ foreach my $node (@freed_nodes) {
}
# Handle pre-reserve.
my $rpid = $node->CheckPreReserve($quiet);
my $rpid = $node->CheckPreReserve(1, $quiet);
print "Releasing node '$node_id' ...\n"
if (!$quiet);
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -1175,7 +1175,7 @@ sub handleCtrlEvent($$) {
$experiment->eid() eq NODERELOADING_EID) {
$nodeobj->RemoveTaintState();
$nodeobj->ClearSchedReload();
my $reserved_pid = $nodeobj->CheckPreReserve(1);
my $reserved_pid = $nodeobj->CheckPreReserve(1,1);
if (defined($reserved_pid)) {
info("$node: Setting pre reserve to $reserved_pid\n");
}
......
......@@ -31,15 +31,16 @@ use Date::Parse;
#
sub usage()
{
print STDERR "Usage: prereserve [-t typelist] [-n priority] ".
"[-s start] [-e end [-r]] pid [count | node_id node_id ...\n";
print STDERR " prereserve -c [-r] pid\n";
print STDERR " prereserve -i pid\n";
print STDERR " prereserve -a pid\n";
print STDERR "Usage: prereserve [-t typelist] [-p priority] ".
"[-s start] [-e end [-r]] [-n resname] pid [count | node_id ...\n";
print STDERR " prereserve -c [-r] -n resname pid\n";
print STDERR " prereserve -i -n resname pid\n";
print STDERR " prereserve -a -n resname pid\n";
print STDERR " prereserve -l\n";
print STDERR " -h This message\n";
print STDERR " -t Comma separated list of node types\n";
print STDERR " -n Priority. Defaults to zero (least priority)\n";
print STDERR " -p Priority. Defaults to zero (least priority)\n";
print STDERR " -n Reservation name; defaults to 'default'\n";
print STDERR " -c Clear pending prereserve for project\n";
print STDERR " -r Revoke current prereserve for project (use with -c)\n";
print STDERR " -i Show pending prereserve for project\n";
......@@ -50,7 +51,7 @@ sub usage()
print STDERR " -a Activate a pending reservation (internal option)\n";
exit(-1);
}
my $optlist = "hdct:n:ilre:s:ma";
my $optlist = "hdct:n:ilre:s:map:";
my $priority = 0;
my $debug = 0;
my $info = 0;
......@@ -60,6 +61,7 @@ my $revoke = 0;
my $sendmail = 0;
my $activate = 0;
my @nodelist = ();
my $resname;
my $starttime;
my $endtime;
my $typelist;
......@@ -69,7 +71,7 @@ my $project;
# Protos
sub fatal($);
sub StartReservation($);
sub StartReservation($$);
#
# Configure variables
......@@ -110,8 +112,11 @@ if (! getopts($optlist, \%options)) {
if (defined($options{h})) {
usage();
}
if (defined($options{p})) {
$priority = $options{p};
}
if (defined($options{n})) {
$priority = $options{n};
$resname = $options{n};
}
if (defined($options{c})) {
$clear = 1;
......@@ -154,6 +159,8 @@ if (defined($options{"s"})) {
if ($info || $clear || ($revoke && !$endtime)) {
usage()
if (@ARGV != 1 || ($revoke && !$clear));
usage()
if (!defined($resname));
$pid = $ARGV[0];
}
......@@ -163,16 +170,17 @@ elsif ($list) {
}
elsif ($activate) {
usage()
if (@ARGV != 1);
if (@ARGV != 1 || !defined($resname));
exit(StartReservation($ARGV[0]));
exit(StartReservation($ARGV[0], $resname));
}
else {
usage()
if (@ARGV < 2);
$pid = shift(@ARGV);
$count = shift(@ARGV);
$pid = shift(@ARGV);
$count = shift(@ARGV);
$resname = "default" if (!defined($resname));
if ($count !~ /^\d+$/) {
if (defined($typelist)) {
......@@ -203,13 +211,14 @@ if ($list) {
"order by needed desc, priority desc, created asc");
if ($query_result->numrows) {
printf("%-18s %-4s %-3s %-10s %-18s %-3s %s\n",
"Project", "Need", "Got", "Creator", "Created", "Pri", "Types");
printf("%-18s %-12s %-4s %-3s %-10s %-18s %-3s %s\n",
"Project", "ResName", "Need", "Got", "Creator", "Created", "Pri", "Types");
print "---------------------------------------------------------------------\n";
}
while (my $row = $query_result->fetchrow_hashref()) {
my $pid = $row->{'pid'};
my $name = $row->{'name'};
my $count = $row->{'count'};
my $created = $row->{'created'};
my $creator = $row->{'creator'};
......@@ -226,8 +235,8 @@ if ($list) {
($current) = $current_result->fetchrow_array()
if ($current_result && $current_result->numrows);
printf("%-20s %-4d %-3d %-10s %-18s %-3d %s\n",
$pid, $count, $current, $creator, $created,
printf("%-20s %-12s %-4d %-3d %-10s %-18s %-3d %s\n",
$pid, $name, $count, $current, $creator, $created,
$priority, $types);
if (defined($starttime)) {
......@@ -272,14 +281,16 @@ if ($info) {
my $pending = 0;
my $nodes_result =
DBQueryFatal("select node_id from nodes where reserved_pid='$pid'");
DBQueryFatal("select node_id from nodes ".
"where reserved_pid='$pid' and ".
" reservation_name='$resname'");
($current) = $nodes_result->numrows
if ($nodes_result && $nodes_result->numrows);
my $query_result =
DBQueryFatal("select * from project_reservations ".
"where pid_idx='$pid_idx'");
"where pid_idx='$pid_idx' and name='$resname'");
if ($query_result->numrows) {
my $row = $query_result->fetchrow_hashref();
......@@ -369,12 +380,52 @@ if ($UID) {
# Clear and exit.
#
if ($clear) {
DBQueryFatal("delete from node_reservations where pid_idx='$pid_idx'");
DBQueryFatal("delete from project_reservations where pid_idx='$pid_idx'");
my $query_result =
DBQueryFatal("select name from project_reservations ".
"where pid_idx='$pid_idx' and name='$resname'");
if (!$query_result->numrows) {
fatal("No such prereserve $pid,$resname");
}
#
# Mark reservation as terminal to prevent it from being used.
# Turning off active is no good, it will just get turned on.
#
DBQueryFatal("update project_reservations set terminal=1 ".
"where pid_idx='$pid_idx' and name='$resname'");
#
# If we are revoking the reservation, lets see if any nodes can be
# moved to a another prereserve instead of going into the free pool.
#
# If NOT revoking the reservation, then we need to clear the reservation
# name in the nodes table, since the prereserve setting is no longer
# associated with a pre-reserve.
#
if ($revoke) {
DBQueryFatal("update nodes set reserved_pid=null ".
"where reserved_pid='$pid'");
my $query_result =
DBQueryFatal("select node_id from nodes ".
"where reserved_pid='$pid' and ".
" reservation_name='$resname'");
while (my ($node_id) = $query_result->fetchrow_array()) {
my $node = Node->Lookup($node_id);
$node->CheckPreReserve(0, 0);
}
DBQueryFatal("update nodes set reserved_pid=null, ".
" reservation_name=null ".
"where reserved_pid='$pid' and ".
" reservation_name='$resname'");
}
else {
DBQueryFatal("update nodes set reservation_name=null ".
"where reserved_pid='$pid' and ".
" reservation_name='$resname'");
}
DBQueryFatal("delete from node_reservations ".
"where pid_idx='$pid_idx' and reservation_name='$resname'");
DBQueryFatal("delete from project_reservations ".
"where pid_idx='$pid_idx' and name='$resname'");
exit(0);
}
......@@ -401,11 +452,36 @@ if (defined($typelist)) {
#
# Lets say that a current request is an error. delete and recreate.
#
DBQueryFatal("lock tables project_reservations write, ".
" node_reservations write");
my $query_result =
DBQueryFatal("select * from project_reservations where pid_idx='$pid_idx'");
DBQueryFatal("select * from project_reservations ".
"where pid_idx='$pid_idx' and name='$resname'");
if ($query_result->numrows) {
fatal("Already have a reservation request for $pid, please clear it first");
fatal("Already have a reservation request $pid,$resname; ".
"please clear it first");
}
#
# It would be odd to allow the same node to be in more then one
# reservation in the same project. Well, I can imagine a scenario; we
# have a bunch of pcXXXs in a pre-reserve, some already allocated. We
# want to revoke that pre-reserve, but keep some subset of the pcXXXs
# in that project. If there was another pre-reserve in the same
# project, we would never allow that subset to get back into the
# wild. It might be useful to support this at some point, I think it
# can be done. Lets see if its needed.
#
if (@nodelist) {
foreach my $node_id (@nodelist) {
my $query_result =
DBQueryFatal("select node_id from node_reservations ".
"where node_id='$node_id' and pid_idx='$pid_idx'");
if ($query_result->numrows) {
fatal("There is already a pre-reservation for $node_id in\n".
"this project; this is not allowed");
}
}
}
#
......@@ -423,21 +499,21 @@ if ($endtime) {
my $tmp = str2time($endtime);
$endarg = ",end=FROM_UNIXTIME($tmp)";
}
DBQueryFatal("lock tables project_reservations write, ".
" node_reservations write");
DBQueryFatal("insert into project_reservations set ".
" pid='$pid', pid_idx='$pid_idx', count='$count', ".
" creator='$uid', creator_idx='$uid_idx', ".
" pid='$pid', pid_idx='$pid_idx', name='$resname',".
" count='$count', creator='$uid', creator_idx='$uid_idx', ".
" created=now(),active=0 $typearg $startarg $endarg");
if (@nodelist) {
foreach my $node_id (@nodelist) {
if (!DBQueryWarn("insert into node_reservations set ".
" pid='$pid', pid_idx='$pid_idx', ".
" reservation_name='$resname', ".
" node_id='$node_id'")) {
DBQueryWarn("delete from project_reservations ".
"where pid_idx='$pid_idx'");
DBQueryWarn("delete from node_reservations ".
"where pid_idx='$pid_idx'");
"where pid_idx='$pid_idx' and ".
" reservation_name='$resname'");
DBQueryWarn("delete from project_reservations ".
"where pid_idx='$pid_idx' and name='$resname'");
exit(-1);
}
}
......@@ -450,7 +526,7 @@ print "Node reservation request for $count nodes has been created.\n";
# have beat us to it already. We check the active bit below.
#
if (!defined($starttime) || str2time($starttime) <= time()) {
exit(StartReservation($pid));
exit(StartReservation($pid, $resname));
}
exit(0);
......@@ -458,21 +534,23 @@ exit(0);
# Activate a reservation request; find as many nodes as possible,
# and then mark it as active.
#
sub StartReservation($)
sub StartReservation($$)
{
my ($pid) = @_;
my ($pid, $resname) = @_;
DBQueryFatal("lock tables nodes write, node_types read, ".
" project_reservations write, ".
" node_reservations write, reserved write");
my $noderes_result =
DBQueryFatal("select node_id from node_reservations where pid='$pid'");
DBQueryFatal("select node_id from node_reservations ".
"where pid='$pid' and reservation_name='$resname'");
my $query_result =
DBQueryFatal("select * from project_reservations where pid='$pid'");
DBQueryFatal("select * from project_reservations ".
"where pid='$pid' and name='$resname'");
if (!$query_result->numrows) {
fatal("No reservation defined for project");
fatal("No reservation $resname defined for project");
}
my $row = $query_result->fetchrow_hashref();
my $active = $row->{'active'};
......@@ -491,7 +569,9 @@ sub StartReservation($)
# prereserve, from nodes already prereserved.
#
$query_result =
DBQueryFatal("select node_id from nodes where reserved_pid='$pid'");
DBQueryFatal("select node_id from nodes ".
"where reserved_pid='$pid' and ".
" reservation_name='$resname'");
my $current = $query_result->numrows;
......@@ -505,30 +585,42 @@ sub StartReservation($)
#
# If we have a node list, then we do not do any type stuff, we
# operate on specific nodes.
# operate on specific nodes only.
#
if ($noderes_result->numrows) {
@types = ();
while (my ($node_id) = $noderes_result->fetchrow_array()) {
my $query_result =
DBQueryFatal("select pid,eid from reserved ".
"where node_id='$node_id'");
if ($query_result->numrows) {
my ($curpid) = $query_result->fetchrow_array();
if ($curpid ne $pid) {
#
# Someone else still has it, skip.
#
next;
}
DBQueryFatal("select pid,reserved_pid from nodes ".
"left join reserved on ".
" reserved.node_id=nodes.node_id ".
"where nodes.node_id='$node_id'");
next
if (!$query_result->numrows);
my ($curpid,$rpid) = $query_result->fetchrow_array();
if ((defined($curpid) && $curpid ne $pid) ||
(defined($rpid) && $rpid ne $pid)) {
#
# Someone else still has it, skip.
#
print "$node_id is still reserved to project $curpid\n"
if (defined($curpid));
print "$node_id is still pre-reserved to project $rpid\n"
if (defined($rpid));
next;
}
# Free or we already have it.
# Free or we already have it. Note that we do not allow a
# specific node to be in more then one reservation in the
# same project/ See note above.
$current++;
DBQueryFatal("delete from node_reservations ".
"where node_id='$node_id'");
DBQueryFatal("update nodes set reserved_pid='$pid' ".
DBQueryFatal("update nodes set reserved_pid='$pid', ".
" reservation_name='$resname' ".
"where node_id='$node_id'");
}
}
......@@ -547,24 +639,30 @@ sub StartReservation($)
}
#
# check for nodes already reserved to the project, but without
# reserved_pid set; they count against a prereserve request and
# also need to be updated.
# check for nodes already reserved to the project; they count
# against the prereserve request. If the reserved_pid is not
# set, we set it to indicate that they are attached to this
# pre-reserve.
#
$query_result =
DBQueryFatal("select reserved.node_id from reserved ".
DBQueryFatal("select reserved.node_id,nodes.reserved_pid, ".
" nodes.reservation_name ".
" from reserved ".
"left join nodes on nodes.node_id=reserved.node_id ".
"left join node_types on node_types.type=nodes.type ".
"where nodes.role='testnode' and ".
" node_types.class='pc' and ".
" nodes.reserved_pid is null and ".
" reserved.pid='$pid' $tclause");
while (my ($node_id) = $query_result->fetchrow_array()) {
DBQueryFatal("update nodes set reserved_pid='$pid' ".
"where node_id='$node_id'");
while (my ($node_id,$reserved_pid,
$reservation_name) = $query_result->fetchrow_array()) {
$current++;
$tcount++;
if (!defined($reserved_pid)) {
DBQueryFatal("update nodes set reserved_pid='$pid', ".
" reservation_name='$resname' ".
"where node_id='$node_id'");
$tcount++;
}
last
if ($current >= $count);
}
......@@ -588,7 +686,8 @@ sub StartReservation($)
$tcount = 0;
while (my ($node_id) = $query_result->fetchrow_array()) {
DBQueryFatal("update nodes set reserved_pid='$pid' ".
DBQueryFatal("update nodes set reserved_pid='$pid', ".
" reservation_name='$resname' ".
"where node_id='$node_id'");
$current++;
$tcount++;
......
......@@ -91,23 +91,27 @@ my $query_result =
" UNIX_TIMESTAMP(now()) > UNIX_TIMESTAMP(end)");
while (my $row = $query_result->fetchrow_hashref()) {
my $pid = $row->{'pid'};
my $pid = $row->{'pid'};
my $resname = $row->{'name'};
if ($debug) {
print "Terminating pre-reserve for project $pid\n";
print "Terminating pre-reserve $pid,$resname\n";
}
if (!$impotent) {
my $output = emutil::ExecQuiet("$PRERESERVE -c -r $pid");
my $output = emutil::ExecQuiet("$PRERESERVE -c -r -n $resname $pid");
if ($?) {
print STDERR "Error terminating pre reservation for $pid!\n";
print STDERR
"Error terminating pre reservation $pid,$resname!\n";
SENDMAIL($TBOPS, "Error Terminating pre reservation for $pid",
"The pre reservation request for project $pid, ".
SENDMAIL($TBOPS,
"Error Terminating pre reservation $pid,$resname",
"The pre reservation request $pid,$resname ".
"could not be terminated!\n\n" . $output . "\n", $TBOPS);
}
else {
SENDMAIL($TBOPS, "Pre Reservation for $pid has been terminated",
"The pre reservation request for project $pid, ".
SENDMAIL($TBOPS,
"Pre Reservation $pid,$resname has been terminated",
"The pre reservation $pid,$resname ".
"has been terminated\n", $TBOPS);
}
}
......@@ -118,23 +122,24 @@ while (my $row = $query_result->fetchrow_hashref()) {
#
$query_result =
DBQueryFatal("select * from project_reservations ".
"where active=0 and start is not null and ".
"where active=0 and terminal=0 and start is not null and ".
" UNIX_TIMESTAMP(now()) >= UNIX_TIMESTAMP(start) ".
"order by priority desc, created asc");
while (my $row = $query_result->fetchrow_hashref()) {
my $pid = $row->{'pid'};
my $pid = $row->{'pid'};
my $resname = $row->{'name'};
if ($debug) {
print "Activating pre-reserve for project $pid\n";
print "Activating pre-reserve $pid,$resname\n";
}
if (!$impotent) {
my $output = emutil::ExecQuiet("$PRERESERVE -a $pid");
my $output = emutil::ExecQuiet("$PRERESERVE -a -n $resname $pid");
if ($?) {
print STDERR "Error activating pre reservation for $pid!\n";
print STDERR "Error activating pre reservation $pid,$resname!\n";
SENDMAIL($TBOPS, "Error activating pre reservation for $pid",
"The pre reservation request for project $pid, ".
SENDMAIL($TBOPS, "Error activating pre reservation $pid,$resname",
"The pre reservation request $pid,$resname ".
"could not be activated!\n\n" . $output . "\n", $TBOPS);
}
}
......
......@@ -605,6 +605,11 @@ class InstanceSliver
# Constructor by lookup on unique index.
#
function InstanceSliver($instance, $urn) {
if (!$instance) {
TBMAIL("stoller", "undefined instance", $urn);
$this->sliver = null;
return;
}
$uuid = $instance->uuid();
$query_result =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment