nfree.in 20.3 KB
Newer Older
Leigh Stoller's avatar
Leigh Stoller committed
1
#!/usr/bin/perl -w
Leigh Stoller's avatar
Leigh Stoller committed
2
#
3
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh Stoller's avatar
Leigh Stoller committed
23
#
Leigh Stoller's avatar
Leigh Stoller committed
24
use strict;
25
use English;
Chad Barb's avatar
Chad Barb committed
26
use Getopt::Std;
27 28 29

#
# nfree - Takes pysical node names, and frees them from the experiment they
Mike Hibler's avatar
Mike Hibler committed
30
# are allocated to. If nodes are omitted, frees up all nodes in the given
31 32 33 34
# experiment. Looks in the scheduled_reloads and next_reserve tables to see
# if this node should be re-reserved into another experiment and/or reloaded,
# rather than being put back into the pool of free nodes
#
35 36
sub usage ()
{
37 38
    die("Usage: nfree [-x] [-o] <pid> <eid> (-a | <node> <node> <...>)\n".
	"Releases all nodes in the specified experiment if -a is given.\n".
Chad Barb's avatar
Chad Barb committed
39
	"If nodes are listed, nfree releases only those nodes.\n".
Leigh Stoller's avatar
Leigh Stoller committed
40
	" '-x' frees all virtual nodes on any physical node that is freed.\n".
41
	" '-o' Moves nodes into a oldreserved holding experiment.\n"
Chad Barb's avatar
Chad Barb committed
42
	);
43
}
44
my $optlist = "xoaq";
Leigh Stoller's avatar
Leigh Stoller committed
45 46
my $freeDependantVirtuals = 0;
my $moveToOldReserved     = 0;
47
my $freeAllNodes          = 0;
48
my $quiet                 = 0;
Mac Newbold's avatar
Mac Newbold committed
49

50
# Configure variables
51
my $TB       = "@prefix@";
52
my $TESTMODE = @TESTMODE@;
53

Leigh Stoller's avatar
Leigh Stoller committed
54 55 56 57 58 59 60
# Turn off line buffering on output
$| = 1;

# Untaint the path
$ENV{'PATH'} = "/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

61 62
# Testbed Support libraries
use lib "@prefix@/lib";
63
use libdb;
64
use libtestbed;
65 66
use User;
use Experiment;
67
use Node;
Leigh Stoller's avatar
Leigh Stoller committed
68
use OSinfo;
69
use Lan;
Leigh Stoller's avatar
Leigh Stoller committed
70 71 72 73 74 75

# Local stuff
my $consetup	    = "$TB/libexec/console_setup";
my $osselect        = "$TB/bin/os_select";
my $nodereboot      = "$TB/bin/node_reboot";
my $makeconf        = "$TB/sbin/dhcpd_makeconf";
76
my $snmpit          = "$TB/bin/snmpit";
Leigh Stoller's avatar
Leigh Stoller committed
77 78 79
my $reloadpid	    = "emulab-ops";
my $pendingeid      = "reloadpending";
my $rppendingeid    = "repositionpending";
80 81
my $oldreserved_pid = OLDRESERVED_PID;
my $oldreserved_eid = OLDRESERVED_EID;
Leigh Stoller's avatar
Leigh Stoller committed
82 83
my $lockedpid       = NFREELOCKED_PID();
my $lockedeid       = NFREELOCKED_EID();
84
my @nodes;
Leigh Stoller's avatar
Leigh Stoller committed
85 86 87 88 89
my @freed_nodes     = ();
my @dynanodes       = ();
my $error           = 0;
my %mustzero        = ();
my $mustmakeconf    = 0;
90

Chad Barb's avatar
Chad Barb committed
91 92 93 94
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
Leigh Stoller's avatar
Leigh Stoller committed
95
my %options = ();
Chad Barb's avatar
Chad Barb committed
96 97 98 99 100 101
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"x"})) {
    $freeDependantVirtuals = 1;
}
102 103 104
if (defined($options{"q"})) {
    $quiet = 1;
}
105 106 107
if (defined($options{"o"})) {
    $moveToOldReserved = 1;
}
108 109 110 111
if (defined($options{"a"})) {
    $freeAllNodes = 1;
}
if (@ARGV < 2 || @ARGV == 2 && !$freeAllNodes) {
112 113
    usage();
}
114 115 116
my $pid = shift;
my $eid = shift;

117 118 119 120 121 122 123 124
if (@ARGV > 0 && $ARGV[0] eq '-a') {
    $freeAllNodes = 1;
    shift;
}
if ($freeAllNodes && @ARGV > 0) {
    usage();
}

125
# Make sure that the experiment actually exists
126 127
my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
Leigh Stoller's avatar
Leigh Stoller committed
128 129
    die("*** $0:\n".
        "    There is no experiment '$eid' in project '$pid'.\n");
130
}
Leigh Stoller's avatar
Leigh Stoller committed
131
# Need these below.
132 133 134
my $pid_idx = $experiment->pid_idx();
my $exptidx = $experiment->idx();

Leigh Stoller's avatar
Leigh Stoller committed
135 136 137 138 139 140
# Map a bunch of experiments we might need to reserve nodes to.
my $oldreserved_experiment = Experiment->Lookup($oldreserved_pid,
						$oldreserved_eid);
if (!defined($oldreserved_experiment)) {
    die("*** $0:\n".
        "    There is no experiment $oldreserved_pid/$oldreserved_eid!\n");
141
}
Leigh Stoller's avatar
Leigh Stoller committed
142 143 144 145
my $locker_experiment = Experiment->Lookup($lockedpid, $lockedeid);
if (!defined($locker_experiment)) {
    die("*** $0:\n".
        "    There is no experiment $lockedpid/$lockedeid!\n");
146
}
Leigh Stoller's avatar
Leigh Stoller committed
147 148 149 150
my $reloading_experiment = Experiment->Lookup($reloadpid, $pendingeid);
if (!defined($reloading_experiment)) {
    die("*** $0:\n".
        "    There is no experiment $reloadpid/$pendingeid!\n");
151 152
}
# Only in Utah, see below
Leigh Stoller's avatar
Leigh Stoller committed
153
my $repositioning_experiment;
154 155 156 157 158 159

#
# Verify user and get his DB uid for later. 
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
Leigh Stoller's avatar
Leigh Stoller committed
160 161
    die("*** $0:\n".
	"    You ($UID) do not exist!\n");
162 163
}
my $user_uid = $this_user->uid();
164

165
# Make sure the user has the ability to modify this experiment
166
if (!$experiment->AccessCheck($this_user, TB_EXPT_MODIFY)) {
Leigh Stoller's avatar
Leigh Stoller committed
167 168
    die("*** $0:\n".
	"    You do not have permission to modify $experiment.\n");
169 170
}

171
# Make a list of nodes given on the command line, or get the whole list from
172
# the DB if none provided.
173 174
if (@ARGV) {
    foreach my $n (@ARGV) {
Leigh Stoller's avatar
Leigh Stoller committed
175 176 177 178 179 180
	my $node = Node->Lookup($n);
	if (!defined($node)) {
	    die("*** $0:\n".
		"    No such node $n!\n");
	}
	push(@nodes, $node);
181 182 183 184

	# if -x was specified, remove any 
	# mapping to a node which has a phys_nodeid of $n.
	if ($freeDependantVirtuals) {
Leigh Stoller's avatar
Leigh Stoller committed
185 186 187 188
	    my @virtuals = ();
	    if ($node->VirtualNodes(\@virtuals) != 0) {
		die("*** $0:\n".
		    "    Could not get virtual node list for $node\n");
Chad Barb's avatar
Chad Barb committed
189
	    }
Leigh Stoller's avatar
Leigh Stoller committed
190 191
	    push(@nodes, @virtuals)
		if (@virtuals);
192
	}
193
    }
194
}
Leigh Stoller's avatar
Leigh Stoller committed
195
else {
196 197
    print "Releasing all nodes from experiment $experiment.\n"
	if (!$quiet);
Leigh Stoller's avatar
Leigh Stoller committed
198 199 200 201 202 203 204 205 206 207
    @nodes = $experiment->NodeList(0, 1);
    if (! $moveToOldReserved) {
	my @oldnodes = ();
	if ($experiment->OldReservedNodeList(\@oldnodes) != 0) {
	    die("*** $0:\n".
		"    Could not get oldreserved node list for $experiment\n");
	}
	push(@nodes, @oldnodes);
    }
}
Mike Hibler's avatar
Mike Hibler committed
208

209 210 211 212 213 214
#
# Sanity check. Do not want to release nodes if they have ports that
# are trunked or disabled. Means something went wrong elsewhere and
# we should clean up the mess first. 
#
if (! $moveToOldReserved) {
215 216
    my @untag = ();

217 218 219 220 221 222
    foreach my $node (@nodes) {
	my @interfaces;

	next
	    if ($node->isvirtnode() || $node->isremotenode());

223
	if (0 && VLan->IsNodeInAVlan($node)) {
224 225 226 227 228 229 230 231
	    print STDERR "$node is still in a vlan!\n";
	    $error++;
	}

	if ($node->AllInterfaces(\@interfaces) != 0) {
	    die("*** $0:\n".
		"    Could not get get interfaces for $node\n");
	}
232
	
233 234
	foreach my $interface (@interfaces) {
	    if ($interface->tagged()) {
235 236 237
		print STDERR
		    "$interface is still tagged! But we will fix that.\n";
		push(@untag, $interface);
238 239 240 241 242 243 244 245 246 247 248
	    }
	    if (!$interface->enabled()) {
		print STDERR "$interface is not enabled!\n";
		$error++;
	    }
	}
    }
    if ($error) {
	die("*** $0:\n".
	    "    Please cleanup the previous errors.\n");
    }
249 250 251 252 253 254 255 256 257 258 259
    # Else if no errors, untag the interfaces left in tagged mode.
    if (@untag) {
	my @ports = map($_->node_id() . ":" . $_->iface(), @untag);

	print "*** Turning off tagging for: @ports\n";
	system("$snmpit -U @ports");
	if ($?) {
	    die("*** $0:\n".
		"    Some ports would not untag!\n");
	}
    }
260 261
}

262 263
######################################################################
# Step 1
264
#
265 266
# See what nodes need to be freed, and then lock them down my moving
# them to a holding reservation.
267
#
268 269 270 271 272
# We lock just the reserved table. The prevents races between multiple
# invocations of nfree trying to free the same node. Rather than lock
# a zillion tables, move the reservation into a holding pattern. This
# effectively prevents someone else from freeing the same nodes, and from
# someone else allocating the nodes until we are done cleaning things up.
273
#
274 275 276 277
# NOTE: My reason for not wanting to lock all those tables (9 in the
# original version) is that library calls will fail since mysql locking
# requires that every table used within the locked area, be locked.
# Of course, who knows what tables the library uses, so thats a silly
278
# way to go.
279
#
280 281
######################################################################

282 283
DBQueryFatal("lock tables reserved write, nodes read");

Leigh Stoller's avatar
Leigh Stoller committed
284 285 286 287 288
# Force reload after table lock.
foreach my $node (@nodes) {
    $node->FlushReserved();
}

289 290 291 292
#
# This sanity check for shared nodes; do not want to free a shared
# physical node that still has virtual nodes on it. Bail early.
#
Leigh Stoller's avatar
Leigh Stoller committed
293 294
# It has to be done with tables locked to avoid a race with the mapper
# trying to add a new virtnode to it. See the pool daemon too.
295 296 297 298 299 300
#
if (! $moveToOldReserved) {
    foreach my $node (@nodes) {
	next
	    if ($node->isvirtnode() || $node->isremotenode());

301 302
	if (defined($node->sharing_mode()) && $node->sharing_mode() ne "" &&
	    $node->HasVirtualNodes()) {
303 304 305 306 307 308 309 310 311
	    print STDERR "$node is shared but has virtual nodes on it.\n";
	    $error++;
	}
    }
    if ($error) {
	die("*** $0:\n".
	    "    Please cleanup the previous errors.\n");
    }
}
312

Leigh Stoller's avatar
Leigh Stoller committed
313 314
foreach my $node (@nodes) {
    my $node_id = $node->node_id();
315 316

    #
Leigh Stoller's avatar
Leigh Stoller committed
317 318
    # Check to make sure they have actually reserved the nodes, now that
    # the reserved table is locked.
319
    #
Leigh Stoller's avatar
Leigh Stoller committed
320
    my $rowref = $node->ReservedTableEntry();
321 322
    unless (defined($rowref) &&
	    $rowref->{'pid'} eq $pid && $rowref->{'eid'} eq $eid) {
Leigh Stoller's avatar
Leigh Stoller committed
323
	print STDERR "$node is not reserved by your experiment.\n";
324 325 326 327
	$error++;
	next;
    }

328 329 330 331
    #
    # Remember if the node's disk must be zeroed
    #
    if ($rowref->{'mustwipe'}) {
Leigh Stoller's avatar
Leigh Stoller committed
332 333 334 335
	$mustzero{$node_id} = $rowref->{'mustwipe'};
    }
    else {
	$mustzero{$node_id} = 0;
336 337
    }

Leigh Stoller's avatar
Leigh Stoller committed
338
    if ($moveToOldReserved) {
339 340
	# Move to holding reservation. Node is not free, but is no longer
	# owned by the pid/eid, so cannot be mucked with.
Leigh Stoller's avatar
Leigh Stoller committed
341 342
	if ($node->MoveReservation($oldreserved_experiment) != 0) {
	    print "*** WARNING: Error moving $node to holding experiment!\n";
343 344
	    next;
	}
Leigh Stoller's avatar
Leigh Stoller committed
345 346
    }
    else {
347 348
	# Move to locked reservation. Node is not free, but is no longer
	# owned by the pid/eid, so cannot be mucked with.
Leigh Stoller's avatar
Leigh Stoller committed
349 350
	if ($node->MoveReservation($locker_experiment) != 0) {
	    print "*** WARNING: Error locking down $node!\n";
351 352
	    next;
	}
Mike Hibler's avatar
Mike Hibler committed
353 354 355 356

	# Any node that was part of an elabinelab or plabinelab experiment
	# that is freed requires that we remake the dhcpd.conf file.
	if ($rowref->{'inner_elab_boot'} || $rowref->{'plab_boot'}) {
Leigh Stoller's avatar
Leigh Stoller committed
357 358 359
	    if ($node->ModifyReservation({"inner_elab_boot" => 0,
					  "plab_boot"       => 0}) != 0) {
		print "*** WARNING: Error clearing elab/plab boot for $node\n";
Mike Hibler's avatar
Mike Hibler committed
360 361 362
	    }
	    $mustmakeconf = 1;
	}
Leigh Stoller's avatar
Leigh Stoller committed
363
	push(@freed_nodes, $node);
364 365 366 367 368
    }

}
DBQueryFatal("unlock tables");

369
# We are done if called with a -o
Leigh Stoller's avatar
Leigh Stoller committed
370
if ($moveToOldReserved) {
371 372 373
    exit($error);
}

374 375 376 377 378 379 380 381 382
######################################################################
# Step 1b
#
# Ugh.  If we are resetting the PXE boot program for any of the nodes
# we need to clear them and regenerate the dhcpd.conf file now, before
# we start freeing up nodes in Step 2.  If we delayed HUP'ing til after
# Step 2, then nodes might have already been rebooted by the reload
# daemon.
#
Leigh Stoller's avatar
Leigh Stoller committed
383 384 385
# NOTE: this does not happen very often. elabinelab, plabinelab?
#
foreach my $node (@freed_nodes) {
386 387 388
    if ($node->pxe_boot_path() || $node->next_pxe_boot_path()) {
	$node->Update({"pxe_boot_path" => "NULL",
		       "next_pxe_boot_path" => "NULL"}) == 0 or
Leigh Stoller's avatar
Leigh Stoller committed
389 390
	    die("*** $0:\n".
		"    Could not update pxe_boot_path for $node\n");
Mike Hibler's avatar
Mike Hibler committed
391
	$mustmakeconf = 1;
392 393
    }
}
Mike Hibler's avatar
Mike Hibler committed
394 395
if ($mustmakeconf) {
    system("$makeconf -i -r") == 0 ||
Leigh Stoller's avatar
Leigh Stoller committed
396
	print STDERR "*** WARNING: $makeconf failed!\n";
Mike Hibler's avatar
Mike Hibler committed
397
}
398

399 400 401 402 403 404 405 406 407
######################################################################
# Step 2
#
# Go through the list of nodes we successfully locked down, and clean
# up the node state (nodes, delays, interfaces, etc). Once that is done,
# move them to whatever new reservations are pending, or free the node.
#
######################################################################

Leigh Stoller's avatar
Leigh Stoller committed
408 409 410
foreach my $node (@freed_nodes) {
    my $node_id       = $node->node_id();
    my $mustclean     = 1;
411 412
    my $estate        = $node->eventstate();
    my $isvirt        = $node->isvirtnode();
413
    my $isgeni        = $node->isfednode();
414 415
    my $isdynamic     = $node->isdynamic();
    my $def_boot_osid = $node->def_boot_osid();
Leigh Stoller's avatar
Leigh Stoller committed
416
    my $imageable     = $node->imageable();
417

Leigh Stoller's avatar
Leigh Stoller committed
418 419
    # Clean out all delays
    DBQueryWarn("delete from delays where node_id='$node_id'") || $error++;
420

421
    #
422 423 424 425 426
    # See if the OS it was running was marked as mustclean or not. Basically,
    # this is an OSKit hack to avoid reloading disks that have not been
    # touched by the kernel. If a def_boot_path was set, there is nothing
    # we can figure out, so just reload it. This needs to be more general
    # purpose.
427 428 429
    #
    my $clean;
    if (defined($def_boot_osid)) {
430 431 432 433
	#
	# OSinfo lookup could fail if the OSID was removed at a bad time,
	# so treat a failure as non-fatal and force a clean.
	#
Leigh Stoller's avatar
Leigh Stoller committed
434
	my $osinfo = OSinfo->Lookup($def_boot_osid);
435 436 437 438
	if (defined($osinfo)) {
	    $clean = $osinfo->mustclean();
	} else {
	    $clean = 1;
Leigh Stoller's avatar
Leigh Stoller committed
439
	}
440
    }
441
    if ($isvirt || !$imageable) {
442 443
	# VIRTNODE HACK: Virtual nodes are special. Do not clean or reload.
	$mustclean = 0;
Leigh Stoller's avatar
Leigh Stoller committed
444
	$mustzero{$node_id} = 0;
445
    }
446 447 448 449
    elsif (defined($clean)) {
	# If def_boot_osid set, then $clean is defined. Otherwise not set
	# so default to cleaning node. 
	$mustclean = $clean;
450 451
    }

452
    #
453
    # If the node is virtual, release the shared resources it had
454 455 456 457 458 459 460 461
    # reserved on the physical node.
    #
    if ($isvirt) {
	if ($node->ReleaseSharedBandwidth() == 0) {
	    # Clean the vinterfaces table for this virtual node.
	    DBQueryWarn("delete from vinterfaces where vnode_id='$node_id'")
		or $error++;
	}
462 463
	$node->ReleaseBlockStore() == 0
	    or $error++;
464 465
    }

466 467 468 469
    #
    # If the node is a dynamic virtual node, just save it for later.
    # We will call into the Node library to delete it. 
    #
470
    if ($isvirt && $isdynamic) {
471 472
	$node->SetNodeHistory(TB_NODEHISTORY_OP_DESTROY, $this_user,
			      $experiment);
Leigh Stoller's avatar
Leigh Stoller committed
473
	push(@dynanodes, $node);
474 475
	next;
    }
476

477 478
    if (! $isvirt) {
        # On real nodes, clean out all interfaces except the control net.
Leigh Stoller's avatar
Leigh Stoller committed
479 480 481
	$node->ClearInterfaces() == 0
	    or $error++;
	
482
	# And log phys nodes freed from hwdown
483
	if ($pid eq NODEDEAD_PID() && $eid eq NODEDEAD_EID() && $user_uid) {
Leigh Stoller's avatar
Leigh Stoller committed
484 485
	    $node->InsertNodeLogEntry($this_user, "misc",
				      "Moved from hwdown; nfree");
486
	}
487
    }
488 489
    $node->ClearBootAttributes() == 0
	or $error++;
490

491 492 493
    #
    # If the node is a virtnode, force its state to SHUTDOWN. This is mostly
    # to avoid silly stated warnings for nodes that do not have a physical
494 495
    # representation most of the time! Ditto for geninodes which are really
    # just proxies for a remote physical nodes.
496
    #
Leigh Stoller's avatar
Leigh Stoller committed
497
    $node->SetEventState(TBDB_NODESTATE_SHUTDOWN)
498
	if (($isvirt || $isgeni) && $estate ne TBDB_NODESTATE_SHUTDOWN());
499

500 501 502
    # Clean out the SFS hostid. What about the other keys?
    DBQueryWarn("update node_hostkeys set ".
		"  sfshostid=NULL ".
Leigh Stoller's avatar
Leigh Stoller committed
503 504
		"where node_id='$node_id'")
	or $error++;
505

506
    # Clean out the current_reloads table (a just in case measure).
Leigh Stoller's avatar
Leigh Stoller committed
507 508
    DBQueryWarn("delete from current_reloads where node_id='$node_id'")
	or $error++;
509

510
    # Reset its port allocation stuff.
Leigh Stoller's avatar
Leigh Stoller committed
511 512
    DBQueryWarn("delete from nodeipportnum where node_id='$node_id'")
	or $error++;
513

514
    # Clean the vinterfaces table for this node.
Leigh Stoller's avatar
Leigh Stoller committed
515 516
    DBQueryWarn("delete from vinterfaces where node_id='$node_id'")
	or $error++;
517

518
    # Clean the interface_settings table for this node.
Leigh Stoller's avatar
Leigh Stoller committed
519 520
    DBQueryWarn("delete from interface_settings where node_id='$node_id'")
	or $error++;
521

522
    # If it's a robot, we need to reset its physical location.
Leigh Stoller's avatar
Leigh Stoller committed
523
    my $result =
524
	DBQueryFatal("select building,floor,loc_x,loc_y,orientation ".
Leigh Stoller's avatar
Leigh Stoller committed
525
		     "from node_startloc where node_id='$node_id'");
526 527 528 529 530 531 532 533 534 535 536 537
    if ($result->num_rows()) {
	while (my ($bldg,$floor,$x,$y,$o) = $result->fetchrow_array()) {
	    my $subresult = 
		DBQueryFatal("select pixels_per_meter from floorimages ".
			     "where building='$bldg'");
	    if ($subresult->num_rows()) {
		my ($pixels_per_meter) = $subresult->fetchrow_array();

		$x = int($x * $pixels_per_meter);
		$y = int($y * $pixels_per_meter);
		DBQueryWarn("update location_info set ".
			    "loc_x=$x,loc_y=$y,orientation=$o ".
Leigh Stoller's avatar
Leigh Stoller committed
538
			    "where node_id='$node_id' and building='$bldg' ".
539 540 541 542 543 544 545
			    "and floor=$floor") || $error++;
	    }
	    else {
		warn "No building named $bldg in floorimages";
	    }
	}
    }
546
    # Now its safe to change the reservation.
547

548
    # If the node has a next_reserve entry, change the reservation.
Leigh Stoller's avatar
Leigh Stoller committed
549 550
    my $next_reservation = $node->NextReservation();
    if (defined($next_reservation)) {
551 552
	print "Moving $node to $next_reservation\n"
	    if (!$quiet);
Leigh Stoller's avatar
Leigh Stoller committed
553 554 555 556 557 558 559 560 561 562 563 564 565

	if ($node->MoveReservation($next_reservation) == 0) {
	    $node->SetNodeHistory(TB_NODEHISTORY_OP_MOVE, $this_user,
				  $next_reservation);
	    
	    DBQueryWarn("delete from next_reserve where node_id='$node_id'")
		or $error++;

	    # This little sillyness is for disk reloading.
	    # Kill the last reservation since this path is special.
	    DBQueryWarn("delete from last_reservation ".
			"where node_id='$node_id'")
		or $error++;
566
	}
Leigh Stoller's avatar
Leigh Stoller committed
567
	else {
568
	    $error++;
Leigh Stoller's avatar
Leigh Stoller committed
569
	}
570 571
	next;
    }
572

573 574 575
    # If the node has a reloads entry, change the reservation so that the
    # reload_daemon will pick it up.
    $result =
Leigh Stoller's avatar
Leigh Stoller committed
576 577
	DBQueryFatal("select image_id from scheduled_reloads " .
		     "where node_id='$node_id'");
578 579 580
    my $inreloads = $result->numrows();

    # XXX
581
    if ($inreloads && !$imageable) {
Leigh Stoller's avatar
Leigh Stoller committed
582
	print "WARNING: non-imageable node $node in scheduled_reloads\n";
583 584
	$inreloads = 0;
    }
585

Leigh Stoller's avatar
Leigh Stoller committed
586 587 588 589 590 591 592 593 594 595 596
    #
    # XXX Robots.
    #
    if ($node->type() eq "garcia") {
	if (!defined($repositioning_experiment)) {
	    $repositioning_experiment = Experiment->Lookup($reloadpid,
							   $rppendingeid);
	    if (!defined($repositioning_experiment)) {
		print STDERR "*** WARNING: No repositioning experiment!\n";
		$error++;
		next;
597 598
	    }
	}
599 600
	print "Moving $node to $repositioning_experiment\n"
	    if (!$quiet);
Leigh Stoller's avatar
Leigh Stoller committed
601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616

	if ($node->MoveReservation($repositioning_experiment) == 0) {
	    $node->SetNodeHistory(TB_NODEHISTORY_OP_MOVE, $this_user,
				  $repositioning_experiment);
	    
	    DBQueryWarn("replace into scheduled_reloads ".
			"set node_id='$node_id'")
		or $error++;

	    # This little sillyness is for disk reloading.
	    # Kill the last reservation since this path is special.
	    DBQueryWarn("delete from last_reservation ".
			"where node_id='$node_id'")
		or $error++;
	}
	else {
617
	    $error++;
Leigh Stoller's avatar
Leigh Stoller committed
618
	}
619 620
	next;
    }
621
    elsif (!$TESTMODE && !exists($ENV{'NORELOAD'}) &&
622
	   ((!$isvirt && $imageable) || # XXX force reload hack!
Leigh Stoller's avatar
Leigh Stoller committed
623
	    $inreloads || $mustzero{$node_id})) { # XXX Garcia hack
624 625
	print "Moving $node to $reloading_experiment\n"
	    if (!$quiet);
Leigh Stoller's avatar
Leigh Stoller committed
626 627 628 629 630 631 632 633 634 635

	if ($node->MoveReservation($reloading_experiment) == 0) {
	    $node->SetNodeHistory(TB_NODEHISTORY_OP_MOVE, $this_user,
				  $reloading_experiment);

	    # This little sillyness is for disk reloading.
	    # Kill the last reservation since this path is special.
	    DBQueryWarn("delete from last_reservation ".
			"where node_id='$node_id'")
		or $error++;
636 637 638

	    # Handle pre-reserve.
	    my $rpid = $node->CheckPreReserve($quiet);
Leigh Stoller's avatar
Leigh Stoller committed
639
	}
640 641 642
	next;
    }

643
    # No reloads or reservation changes, so really free the node
644
    #
645 646 647 648 649
    # This little sillyness is for disk reloading. Remember the last
    # project a node was reserved into. At present, there might already
    # be an entry. Eventually, os_setup will look for this and force
    # a reload.
    if ($mustclean) {
650
	DBQueryWarn("replace into last_reservation (pid_idx,node_id,pid)".
Leigh Stoller's avatar
Leigh Stoller committed
651
		    "values ($pid_idx, '$node_id', '$pid')");
652
    }
653

654 655 656
    # Handle pre-reserve.
    my $rpid = $node->CheckPreReserve($quiet);

657 658
    print "Releasing node '$node_id' ...\n"
	if (!$quiet);
Leigh Stoller's avatar
Leigh Stoller committed
659 660 661
    if (DBQueryWarn("delete from reserved where node_id='$node_id'")) {
	$node->SetNodeHistory(TB_NODEHISTORY_OP_FREE, $this_user,
			      $experiment);
662 663
    }
    else {
664
	print STDERR "*** Failed to release node '$node_id'!\n";
665 666
	$error++;
    }
667 668
}

669 670
# Release dynamic nodes.
if (@dynanodes) {
Leigh Stoller's avatar
Leigh Stoller committed
671 672 673
    my @nodeids = map($_->node_id(), @dynanodes);

    Node::DeleteVnodes(@nodeids);
674 675
}

676 677 678 679 680
######################################################################
# Step 3 - Set up console for freed nodes.
#
# Using a list of freed nodes build eariler, run consetup to reset
# their serial consoles.
681 682 683 684 685 686
#
# NOTE: While it may seem like a race to do this after releasing the
# reservation, it really is not. Even if the node is allocated again
# console_setup looks at the current reservation and does the right
# thing, and since nalloc locks the reserved table, ordering will be
# preserved.
687
#
688 689
######################################################################

690
if (@freed_nodes) {
Leigh Stoller's avatar
Leigh Stoller committed
691 692 693 694
    my @nodeids = map($_->node_id(), @freed_nodes);
    
    system("$consetup @nodeids") == 0 ||
	print STDERR "WARNING: $consetup @nodeids failed!\n";
Mac Newbold's avatar
Mac Newbold committed
695 696
}

697
exit($error);
698