Commit 103e0385 authored by Leigh Stoller's avatar Leigh Stoller

Add support for multiple pre-reservations per project:

When creating a pre-reserve, new -n option to specify a name for the
reservation, defaults to "default". All other operations require an
-n option to avoid messing with the wrong reservation. You are not allowed
to reuse a reservation name in a project, of course. Priorities are
probably more important now, we might want to change the default from 0 to
some thing higher, and change all the current priorities.

For bookkeeping, the nodes table now has a reservation_name slot that is
set with the reserved_pid. This allows us to revoke the nodes associated
with a specific reservation. Bonus feature is that when setting the
reserved_pid via the web interface, we leave the reservation_name null, so
those won't ever be revoked by the prereserve command line tool.

New feature; when revoking a pre-reserve, we now look to see if nodes being
revoked are free and can be assigned to other pre-reserves. We used to not
do anything, and so had to wait until that node was allocated and released
later, to see if it could move into a pre-reserve.

Also a change required by node specific reservations; when we free a node,
need to make sure we actually use that node, so have to cycle through all
reservations in priority order until it can used. We did not need to do
this before.
parent 70c94bca
...@@ -3654,9 +3654,9 @@ sub SetJailIPFromVnode($$$) ...@@ -3654,9 +3654,9 @@ sub SetJailIPFromVnode($$$)
# #
# Check for, and update a node pre reservation. # Check for, and update a node pre reservation.
# #
sub CheckPreReserve($$) sub CheckPreReserve($$$)
{ {
my ($self, $quiet) = @_; my ($self, $isfree, $quiet) = @_;
my $result = undef; my $result = undef;
# #
...@@ -3666,17 +3666,33 @@ sub CheckPreReserve($$) ...@@ -3666,17 +3666,33 @@ sub CheckPreReserve($$)
my $node_id = $self->node_id(); my $node_id = $self->node_id();
DBQueryWarn("lock tables project_reservations write, nodes write, ". DBQueryWarn("lock tables project_reservations write, nodes write, ".
" node_reservations write") " node_reservations write, reserved read")
or return undef; or return undef;
#
# isfree is a flag that says we are coming from nfree or stated,
# and the node is going into the free pool. When not set (as from
# prereserve) we have to check the reserved table to see if it is
# free; we do not mess with an allocated node. We do this here with
# the rest of the locked tables.
#
if (!$isfree) {
my $query_result =
DBQueryWarn("select pid,eid from reserved ".
"where node_id='$node_id'");
goto done
if (!$query_result || $query_result->numrows);
}
# #
# Need to check for existing reserved_pid, but have to go to the DB, # Need to check for existing reserved_pid, but have to go to the DB,
# not look in the object ($self) since it might be stale. # not look in the object ($self) since it might be stale.
# #
my $query_result = my $query_result =
DBQueryWarn("select reserved_pid,count,active from nodes ". DBQueryWarn("select reserved_pid,count,active,terminal from nodes ".
"left join project_reservations on ". "left join project_reservations on ".
" project_reservations.pid=nodes.reserved_pid ". " project_reservations.pid=nodes.reserved_pid and ".
" project_reservations.name=nodes.reservation_name ".
"where nodes.node_id='$node_id'"); "where nodes.node_id='$node_id'");
# numrows would be zero if the node was suddenly deleted. # numrows would be zero if the node was suddenly deleted.
...@@ -3688,12 +3704,13 @@ sub CheckPreReserve($$) ...@@ -3688,12 +3704,13 @@ sub CheckPreReserve($$)
# if the reservation request is still active. If not, we can clear it, # if the reservation request is still active. If not, we can clear it,
# which will allow it to be set again below, if needed. # which will allow it to be set again below, if needed.
# #
my ($pid,$count,$active) = $query_result->fetchrow_array(); my ($pid,$count,$active,$terminal) = $query_result->fetchrow_array();
if (defined($pid)) { if (defined($pid)) {
goto done goto done
if (defined($count) && $active); if (defined($count) && $active && !$terminal);
DBQueryWarn("update nodes set reserved_pid=null ". DBQueryWarn("update nodes set reserved_pid=null, ".
" reservation_name=null ".
"where node_id='$node_id'"); "where node_id='$node_id'");
if (!$quiet) { if (!$quiet) {
...@@ -3703,25 +3720,26 @@ sub CheckPreReserve($$) ...@@ -3703,25 +3720,26 @@ sub CheckPreReserve($$)
# Find only active unfilled reservations. # Find only active unfilled reservations.
$query_result = $query_result =
DBQueryWarn("select pid,count from project_reservations ". DBQueryWarn("select pid,count,name from project_reservations ".
"where active=1 and count>0 and ". "where active=1 and terminal=0 and count>0 and ".
" (types is null or ". " (types is null or ".
" FIND_IN_SET('$type', types)) ". " FIND_IN_SET('$type', types)) ".
"order by priority desc, created asc ". "order by priority desc, created asc ");
"limit 1");
goto done goto done
if (!$query_result); if (!$query_result);
if ($query_result->numrows) { while ($query_result->numrows) {
my ($pid,$count) = $query_result->fetchrow_array(); my ($pid,$count,$resname) = $query_result->fetchrow_array();
# #
# See if this is fullfulling a specific node reservation request. # See if this is fullfulling a specific node reservation request.
# Need to delete that row if so. # Need to delete that row if so.
# #
my $noderes_result = my $noderes_result =
DBQueryWarn("select node_id from node_reservations ". DBQueryWarn("select node_id from node_reservations ".
"where pid='$pid'"); "where pid='$pid' and ".
" reservation_name='$resname'");
goto done goto done
if (!$noderes_result); if (!$noderes_result);
...@@ -3740,34 +3758,39 @@ sub CheckPreReserve($$) ...@@ -3740,34 +3758,39 @@ sub CheckPreReserve($$)
} }
} }
# #
# XXX We should look for another reservation to fulfill. # We cannot use this node for this pre-reserve, but maybe the
# next one.
# #
goto done next
if (!$okay); if (!$okay);
} }
if (DBQueryWarn("update nodes set reserved_pid='$pid' ". if (DBQueryWarn("update nodes set reserved_pid='$pid', ".
" reservation_name='$resname' ".
"where node_id='$node_id'")) { "where node_id='$node_id'")) {
DBQueryWarn("update project_reservations set count=count-1 ". DBQueryWarn("update project_reservations set count=count-1 ".
"where pid='$pid'"); "where pid='$pid' and name='$resname'");
if ($noderes_result->numrows) { if ($noderes_result->numrows) {
DBQueryWarn("delete from node_reservations ". DBQueryWarn("delete from node_reservations ".
"where node_id='$node_id'"); "where node_id='$node_id' and pid='$pid' and ".
" reservation_name='$resname'");
} }
$result = $pid; $result = $pid;
if ($count == 1) { if ($count == 1) {
SENDMAIL($TBOPS, "Pre Reservation for $pid has completed", SENDMAIL($TBOPS,
"The pre reservation request for project $pid, ". "Pre Reservation $pid,$resname has completed",
"has been fullfilled\n", $TBOPS); "The pre reservation request for project ".
"$pid,$resname has been fullfilled\n", $TBOPS);
} }
last;
} }
} }
done: done:
DBQueryWarn("unlock tables"); DBQueryWarn("unlock tables");
if (defined($result) && !$quiet) { if (defined($result) && !$quiet) {
print "Setting pre reserve for $node_id to $result\n"; print "Setting pre-reserve for $node_id to $result\n";
} }
return $result; return $result;
} }
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2014 University of Utah and the Flux Group. # Copyright (c) 2000-2016 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -660,7 +660,7 @@ foreach my $node (@freed_nodes) { ...@@ -660,7 +660,7 @@ foreach my $node (@freed_nodes) {
or $error++; or $error++;
# Handle pre-reserve. # Handle pre-reserve.
my $rpid = $node->CheckPreReserve($quiet); my $rpid = $node->CheckPreReserve(1, $quiet);
} }
next; next;
} }
...@@ -677,7 +677,7 @@ foreach my $node (@freed_nodes) { ...@@ -677,7 +677,7 @@ foreach my $node (@freed_nodes) {
} }
# Handle pre-reserve. # Handle pre-reserve.
my $rpid = $node->CheckPreReserve($quiet); my $rpid = $node->CheckPreReserve(1, $quiet);
print "Releasing node '$node_id' ...\n" print "Releasing node '$node_id' ...\n"
if (!$quiet); if (!$quiet);
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2014 University of Utah and the Flux Group. # Copyright (c) 2000-2016 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -1175,7 +1175,7 @@ sub handleCtrlEvent($$) { ...@@ -1175,7 +1175,7 @@ sub handleCtrlEvent($$) {
$experiment->eid() eq NODERELOADING_EID) { $experiment->eid() eq NODERELOADING_EID) {
$nodeobj->RemoveTaintState(); $nodeobj->RemoveTaintState();
$nodeobj->ClearSchedReload(); $nodeobj->ClearSchedReload();
my $reserved_pid = $nodeobj->CheckPreReserve(1); my $reserved_pid = $nodeobj->CheckPreReserve(1,1);
if (defined($reserved_pid)) { if (defined($reserved_pid)) {
info("$node: Setting pre reserve to $reserved_pid\n"); info("$node: Setting pre reserve to $reserved_pid\n");
} }
......
This diff is collapsed.
...@@ -91,23 +91,27 @@ my $query_result = ...@@ -91,23 +91,27 @@ my $query_result =
" UNIX_TIMESTAMP(now()) > UNIX_TIMESTAMP(end)"); " UNIX_TIMESTAMP(now()) > UNIX_TIMESTAMP(end)");
while (my $row = $query_result->fetchrow_hashref()) { while (my $row = $query_result->fetchrow_hashref()) {
my $pid = $row->{'pid'}; my $pid = $row->{'pid'};
my $resname = $row->{'name'};
if ($debug) { if ($debug) {
print "Terminating pre-reserve for project $pid\n"; print "Terminating pre-reserve $pid,$resname\n";
} }
if (!$impotent) { if (!$impotent) {
my $output = emutil::ExecQuiet("$PRERESERVE -c -r $pid"); my $output = emutil::ExecQuiet("$PRERESERVE -c -r -n $resname $pid");
if ($?) { if ($?) {
print STDERR "Error terminating pre reservation for $pid!\n"; print STDERR
"Error terminating pre reservation $pid,$resname!\n";
SENDMAIL($TBOPS, "Error Terminating pre reservation for $pid", SENDMAIL($TBOPS,
"The pre reservation request for project $pid, ". "Error Terminating pre reservation $pid,$resname",
"The pre reservation request $pid,$resname ".
"could not be terminated!\n\n" . $output . "\n", $TBOPS); "could not be terminated!\n\n" . $output . "\n", $TBOPS);
} }
else { else {
SENDMAIL($TBOPS, "Pre Reservation for $pid has been terminated", SENDMAIL($TBOPS,
"The pre reservation request for project $pid, ". "Pre Reservation $pid,$resname has been terminated",
"The pre reservation $pid,$resname ".
"has been terminated\n", $TBOPS); "has been terminated\n", $TBOPS);
} }
} }
...@@ -118,23 +122,24 @@ while (my $row = $query_result->fetchrow_hashref()) { ...@@ -118,23 +122,24 @@ while (my $row = $query_result->fetchrow_hashref()) {
# #
$query_result = $query_result =
DBQueryFatal("select * from project_reservations ". DBQueryFatal("select * from project_reservations ".
"where active=0 and start is not null and ". "where active=0 and terminal=0 and start is not null and ".
" UNIX_TIMESTAMP(now()) >= UNIX_TIMESTAMP(start) ". " UNIX_TIMESTAMP(now()) >= UNIX_TIMESTAMP(start) ".
"order by priority desc, created asc"); "order by priority desc, created asc");
while (my $row = $query_result->fetchrow_hashref()) { while (my $row = $query_result->fetchrow_hashref()) {
my $pid = $row->{'pid'}; my $pid = $row->{'pid'};
my $resname = $row->{'name'};
if ($debug) { if ($debug) {
print "Activating pre-reserve for project $pid\n"; print "Activating pre-reserve $pid,$resname\n";
} }
if (!$impotent) { if (!$impotent) {
my $output = emutil::ExecQuiet("$PRERESERVE -a $pid"); my $output = emutil::ExecQuiet("$PRERESERVE -a -n $resname $pid");
if ($?) { if ($?) {
print STDERR "Error activating pre reservation for $pid!\n"; print STDERR "Error activating pre reservation $pid,$resname!\n";
SENDMAIL($TBOPS, "Error activating pre reservation for $pid", SENDMAIL($TBOPS, "Error activating pre reservation $pid,$resname",
"The pre reservation request for project $pid, ". "The pre reservation request $pid,$resname ".
"could not be activated!\n\n" . $output . "\n", $TBOPS); "could not be activated!\n\n" . $output . "\n", $TBOPS);
} }
} }
......
...@@ -605,6 +605,11 @@ class InstanceSliver ...@@ -605,6 +605,11 @@ class InstanceSliver
# Constructor by lookup on unique index. # Constructor by lookup on unique index.
# #
function InstanceSliver($instance, $urn) { function InstanceSliver($instance, $urn) {
if (!$instance) {
TBMAIL("stoller", "undefined instance", $urn);
$this->sliver = null;
return;
}
$uuid = $instance->uuid(); $uuid = $instance->uuid();
$query_result = $query_result =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment