os_setup.in

#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2010 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
require 'ctime.pl';
use POSIX ":sys_wait_h";

#
# Reboot the nodes in an experiment. The nodes table will already contain
# all the information. This script deals with possible disk reloading,
# rebooting, and waiting for nodes to come back alive before allowing
# experiment creation to continue.
#
# TODO: Reload disk images.
#
# usage: os_setup <pid> <eid>
#
# errorcode:  0 - all reboots succeeded.
#             1 - some/all reboots failed; retry may help.
#            -1 - failure; retry is inappropriate.
#

sub usage()
{
    print STDERR "Usage: os_setup [-d] <pid> <eid>\n";
    exit(-1);
}
my  $optlist = "d";

#
# Configure variables
#
my $TB		= "@prefix@";
my $DBNAME	= "@TBDBNAME@";
my $TBOPS       = "@TBOPSEMAIL@";
my $TESTMODE    = @TESTMODE@;
my $TFTP	= "/tftpboot";
my $PGENISUPPORT= @PROTOGENI_SUPPORT@;

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libreboot;
use libosload;
use libtestbed;
use libtblog;
use libArchive;
use Template;
use NodeType;
use Experiment;
use Image;
use OSinfo;
use User;
use Node;
if ($PGENISUPPORT) {
    require libGeni;
}

TBDebugTimeStampsOn();

my $vnode_setup = "$TB/sbin/vnode_setup";
my $nodereboot  = "$TB/bin/node_reboot";
my $elab_setup  = "$TB/sbin/elabinelab";
my $dbg		= 0;
my $failed      = 0;
my $noretry     = 0;
my $failedvnodes= 0;
my $failedplab  = 0;
my $failedgeni  = 0;
my $canceled    = 0;
my %nodeobjs    = ();
my %nodes       = ();
my %vnodes      = ();
my %sharednodes = ();
my %vnodephosts = ();
my %vnode2pnode = ();
my %pnodevcount = ();
my %plabvnodes  = ();
my %geninodes   = ();
my %einenodes	= ();
my %osids       = ();
my %osmap       = ();
my %canfail     = ();
my %bios_waittime   = ();	# Indexed by node_type.
my %reboot_waittime = ();	# Indexed by osid.
my %node_types  = ();		# Indexed by node_id.
my %vname = ();                 # Indexed by node_id.
my $plab_setup_pid;		# Run plab setup in parallel.
my $geni_setup_pid;		# Run geni setup in parallel.

#
# This variable keeps track of the failed nodes of all types.
#   values = ['boot'|'osload', 'fatal'|'nonfatal']
my %failed_nodes = ();
sub add_failed_node_fatal($)     {$failed_nodes{$_[0]} = ['boot', 'fatal']}
sub add_failed_node_nonfatal($)  {$failed_nodes{$_[0]} = ['boot', 'nonfatal']}
sub add_failed_node_reload($)    {$failed_nodes{$_[0]} = ['reload', 'fatal']}

my @all_nodes; # list of all nodes before any are deleted from %nodes

#
# Ah, Frisbee works so lets do auto reloading for nodes that do not have
# the proper OS loaded on it. This will be a hash of lists; for each
# imageid, a list of the nodes to pass to os_load for that imageid.
#
my %reloads     = ();
my %reboots	= ();
my %reconfigs	= ();
my %rebooted    = ();
my $doautoload  = 1;
my $dolastload  = 1;

# Protos
sub SetupReload($$$);
sub FirewallSetup($);
sub os_setup_one($$$;$);
sub KillChildren();
				  
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

$| = 1; #Turn off line buffering on output


#
# Used to die with a -1 return code, to indicate to caller (tbswap)
# that the failure is not likely to be fixed with another attempt.
#
sub die_noretry($;$)
{
    my $parms = {};
    $parms = shift if ref $_[0] eq 'HASH';
    my ($mesg) = shift;
    tberror($parms, $mesg);
    KillChildren();
    exit(-1);
}

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 2) {
    usage();
}
if (defined($options{"d"})) {
    $dbg = 1;
}

my $pid = $ARGV[0];
my $eid = $ARGV[1];

#
# Untaint args.
#
if ($pid =~ /^([-\@\w]+)$/) {
    $pid = $1;
}
else {
    die_noretry("Bad data in pid: $pid.");
}
if ($eid =~ /^([-\@\w]+)$/) {
    $eid = $1;
}
else {
    die_noretry("Bad data in eid: $eid.");
}

#
# Verify user and get his DB uid and other info for later.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    die_noretry("You ($UID) do not exist!");
}
my $user_uid      = $this_user->uid();
my $user_name     = $this_user->name();
my $user_email    = $this_user->email();
my $user_email_to = "$user_name <$user_email>";

#
# Check permission.
#
my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
    die_noretry("Could not find experiment object for $pid/$eid!");    
}
if (!$experiment->AccessCheck($this_user, TB_EXPT_MODIFY)) {
    die_noretry("You do not have permission to swap this experiment!");
}
TBDebugTimeStamp("os_setup started");

#
# See if the experiment is firewalled
#
my $firewall;
my $firewalled = $experiment->IsFirewalled(\$firewall);
my $firewallimage;

#
# Ditto ElabinElab.
#
my $elabinelab = $experiment->elabinelab();

#
# Ditto PlabinElab.
#
my $plabinelab = 0;
my $plcnode;
my $plcimage;
if (TBExptPlabInElabPLC($pid, $eid, \$plcnode)) {
    $plabinelab = 1;
}

#
# Get the set of nodes, as well as the nodes table information for them.
#
my $db_result =
    DBQueryFatal("select n.*,l.pid,r.vname,r.sharing_mode,r.inner_elab_role ".
		 "from reserved as r ".
		 "left join nodes as n on n.node_id=r.node_id ".
		 "left join last_reservation as l on n.node_id=l.node_id ".
		 "where r.pid='$pid' and r.eid='$eid'");

if ($db_result->numrows < 1) {
    print "There are no nodes in experiment '$eid' in project '$pid'.\n";
    exit 0;
}

while (my %row = $db_result->fetchhash()) {
    my $node      = $row{'node_id'};
    my $osid      = $row{'def_boot_osid'};
    my $type      = $row{'type'};
    my $jailnode  = $row{'jailflag'};
    my $failmode  = $row{'failureaction'};
    my $vname     = $row{'vname'};
    my $typeinfo  = NodeType->Lookup($type);
    my $class     = $typeinfo->class();
    my $subnode   = $typeinfo->issubnode();
    my $virtnode  = $typeinfo->isvirtnode();
    my $sharednode = defined($row{'sharing_mode'}) 
	&& $row{'sharing_mode'} eq 'using_shared_local';
    my $iseinenode= $elabinelab && defined($row{'inner_elab_role'})
	&& $row{'inner_elab_role'} eq 'node';
    my $isremote  = $typeinfo->isremotenode();
    my $isgeninode= $typeinfo->isfednode();
    my $imageable = $typeinfo->imageable();
    my $plabnode  = $typeinfo->isplabdslice();
    my $bios_wait = $typeinfo->bios_waittime();
    my $bootpath  = 0;
    my $osinfo    = undef;

    my $nodeobj = Node->Lookup($node);
    if (!defined($nodeobj)) {
	die_noretry("Cannot lookup object for $node!");
    }
    $nodeobjs{$node} = $nodeobj;

    if ($isgeninode) {
	#
	# Geni nodes are currently a lot like plab nodes, but that will
	# change later.
	#
	if ($virtnode) {
	    $vnodes{$node} = $virtnode;
	    $sharednodes{$node} = $sharednode;
	}
	else {
	    $nodes{$node}  = $node;
	}
	$geninodes{$node}  = 1;
    }
    elsif ($virtnode) {
	#
	# Virtual nodes are special. Jailed vnodes can do quite a bit,
	# and so run them through the checks below.
	#
	$vnodes{$node} = ($jailnode || $plabnode || $isremote);
	$sharednodes{$node} = $sharednode;
	$plabvnodes{$node} = $plabnode;
	if (! $jailnode && ! $plabnode && !$isremote) {
	    next;
	}
    }
    elsif ($iseinenode) {
	print "Will skip reload/reboot of inner elab node $node.\n";
	$einenodes{$node} = 1;
	next;
    }
    elsif ($subnode && !$imageable) {
	print "Will skip subnode $node ISUP wait.\n";
    }
    else {
	my $nodeAllocState;
	$nodeobj->GetAllocState(\$nodeAllocState);
	$nodes{$node}  = $node;
	$nodeAllocStates{$node} = $nodeAllocState;
	if ($nodeAllocState eq TBDB_ALLOCSTATE_RES_RECONFIG()) {
	    # Terrible use of state machine.
	    $reconfigs{$node} = 1;
	}
	elsif ($nodeAllocState ne TBDB_ALLOCSTATE_RES_READY()) {
	    # only reboot node if assign_wrapper just pulled it into expt.
	    # (e.g. it isnt ALLOCSTATE_RES_READY)
	    $reboots{$node} = 1;
	}
    }

    $osids{$node} = $osid;
    if ($osid) {
	$osinfo = OSinfo->Lookup($osid);

	die_noretry("Could not map $osid to its object!")
	    if (!defined($osinfo));
    }
    $osmap{$node}         = $osinfo;
    $bios_waittime{$type} = (defined($bios_wait) ? $bios_wait : 0);
    $node_types{$node}    = $type;
    $vname{$node}         = $vname;

    #
    # Make sure the files specified in the paths exist. We mount the
    # user tftp directory on boss node, so we can ignore the IP address,
    # and just check the path directly.
    #
    if (defined($row{'def_boot_path'})) {
	my $path = $row{'def_boot_path'};

	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
		die_noretry("File $path for node $node must reside in $TFTP");
	    }

	    if (! -f $path) {
		die_noretry("File $path for node $node does not exist!");
	    }
	    $bootpath = 1;
	}
    }
    if (defined($row{'next_boot_path'})) {
	my $path = $row{'next_boot_path'};

	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
		die_noretry("File $path for node $node must reside in $TFTP");
	    }

	    if (! -f $path) {
		die_noretry("File $path for node $node does not exist!");
	    }
	}
    }

    #
    # XXX - Ditto for RPMs.
    #
    foreach my $rpm (split(":", $row{'rpms'})) {
	if (! -f $rpm) {
	    die_noretry({type => 'primary', severity => SEV_ERROR,
			 error => ['file_not_found', 'rpm', $rpm, $node]},
			"RPM $rpm for node $node does not exist!");
	}
    }

    #
    # XXX - Ditto for tarfiles.
    #
    foreach my $tarspec (split(":", $row{'tarballs'})) {
	my ($dir, $tar) = split(" ", $tarspec);

	if (! -f $tar) {
	    die_noretry({type => 'primary', severity => SEV_ERROR,
			 error => ['file_not_found', 'tar', $tar, $node]},
			"Tarfile $tar for node $node does not exist!");
	}
    }

    #
    # If the virtnode is running a subOS, we set $imageable because it 
    # really is going to be reloaded... even though virtnode types are not
    # typically imageable.
    #
    if ($virtnode && defined($osinfo) && $osinfo->def_parentosid()) {
	$imageable = 1;
    }

    #
    # If there is a path specified, then we don't worry anymore about it.
    # The user must know what is going on. The OSID might have a path
    # associated with it, which means the same thing; we don't worry about
    # it.
    #
    if (!$bootpath && (!$virtnode || ($virtnode && $imageable)) 
	&& !$isgeninode && $imageable) {
	#
	# These checks are not necessary if the front end and web page
	# are doing the right thing, but lets be careful anyway.
	#
	if (! $osinfo) {
	    die_noretry("$node has no bootpath and no def_boot_osid set!");
	}
	#
	# If there is an actual path, its an OSKit kernel not an image.
	#
	if (! defined($osinfo->path()) || $osinfo->path() eq "") {
	    my $nextosinfo;
	    
	    #
	    # Not an OSKit kernel.
	    #
	    if ($osinfo->IsGeneric()) {
		#
		# Map generic OSID to the specific one.
		#
		$nextosinfo = $osinfo->ResolveNextOSID($experiment);
		if (!defined($nextosinfo)) {
		    die_noretry("No next mapping for $osinfo on $node!\n");
		}
		print "Mapping $osinfo on $node to $nextosinfo\n";
		$osinfo = $nextosinfo;
	    }
	    #
	    # Make sure this OSID is actually loaded on the machine.
	    #
	    my $isloaded = $nodeobj->IsOSLoaded($osinfo);
	    if ($isloaded < 0) {
		die_noretry("Error determining if $osinfo ".
			    "is loaded on $node\n");
	    }
	    if ($isloaded) {
		#
		# OSID is loaded, but might need to be cleaned.
		#
		if ($dolastload &&
		    defined($row{'pid'}) && $row{'pid'} ne $pid) {
		    SetupReload($node, $osinfo, $type);
		}
		elsif ($nextosinfo) {
		    #
		    # Seems like a bad place for this; if the OS was
		    # mapped to something else that is already on the
		    # disk, need to reset def_boot_osid.
		    #
		    if ($nodeobj->OSSelect($osinfo, "def_boot_osid", 0)) {
			die_noretry("Could not set boot OS to ".
				    "$osinfo for $node");
		    }
		}
	    }
	    else {
		#
		# OS not loaded.
		#
		SetupReload($node, $osinfo, $type);
	    }
	    $osmap{$node} = $osinfo;
	    $osids{$node} = $osinfo->osid();
	}
    }

    #
    # Set the canfail bit.
    #
    $canfail{$node} = (($failmode eq NODEFAILMODE_FATAL()) ? 0 : 1);

    #
    # Set the reboot waittime from the osid now that we have it
    # finalized.
    #
    $osid = $osids{$node};
    if (!exists($reboot_waittime{$osid})) {
	$reboot_waittime{$osid} = $osmap{$node}->reboot_waittime();
    }

    print STDERR "$node - $osmap{$node} - $canfail{$node}\n"
	if $dbg;
}

#
# XXX Inner elab nodes should never report in to us.
# If they do, make sure they wind up in PXEWAIT.
#
if (keys(%einenodes)) {
    DBQueryFatal("update nodes set ".
		 "  def_boot_osid=NULL,".
		 "  next_boot_osid=NULL,".
		 "  temp_boot_osid=NULL ".
		 "where node_id in (".
		 join(",", map("'$_'", keys %einenodes)). ")");
}    

@all_nodes = (keys %nodes, keys %vnodes);

#
# Perform some prechecks on the images.  This will also have the
# effect of catching the info for the images for latter use
#
# FIXME: WRITEME
#    Maybe this isn't a good idea since it will also attempt to fetch
#    the image from the real boss in an inner-emulab.  This should
#    really be done in parallel.

#
# Collect some info about vnodes.
#
foreach my $vnode (keys(%vnodes)) {
    my $nodeobj = $nodeobjs{$vnode};
    my $jailed  = $vnodes{$vnode};

#    print "$vnode, $jailed\n";

    if (! $jailed) {
	next;
    }
    my $pnode = $nodeobj->phys_nodeid();

#    print "$vnode, $jailed, $pnode\n";

    #
    # Count up the number of jailed nodes on this pnode, and add the
    # mapping. We use this below for determining how long to wait for
    # a particular vnode.
    #
    $pnodevcount{$pnode} = 0
	if (!defined($pnodevcount{$pnode}));
    $pnodevcount{$pnode}++;
    $vnode2pnode{$vnode} = $pnode;
    my $pnodeobj = Node->Lookup($pnode);
    if (!defined($pnodeobj)) {
	die_noretry("Cannot lookup object for $pnode!");
    }
    $nodeobjs{$pnode} = $pnodeobj;
}

#
# Setup the firewall first.  Once it is up we can continue with the
# remaining nodes.
#
# There is very little point in setting up the other nodes at the same time
# as they will not be able to PXE boot until the firewall is up.  We could
# fire them off a little early in hopes of overlapping any BIOS boot time
# with the last stages of the firewall setup, but it probably isn't worth
# the complexity (and would not work with nodes for which "reboot" means
# "fall out of PXEWAIT and boot".
#
# Note that we formerly did just do them all at once and let the nodes
# continually PXE-timeout and reboot until the firewall came up.  But that
# can actually take longer than what we do now, if a node happened to
# timeout and reboot just as the firewall came up (i.e., we would have to
# wait an extra BIOS-reboot cycle, which can be 90 seconds or more.
#
if ($firewalled) {
    my $node = $firewall;
    my $nodeobj = $nodeobjs{$node};

    TBDebugTimeStamp("rebooting/reloading firewall");
    if (!FirewallSetup($node)) {
	tbwarn "Firewall node $node failed to boot.".
	    "This has been reported to testbed-ops.";

	# XXX do we need to set NODEBOOTSTATUS_FAILED here?

	#
	# We assume that firewall node images are "standard" here,
	# and whine to tbops.
	#
	$nodeobj->MarkAsDown();
	$nodeobj->InsertNodeLogEntry($this_user, TB_DEFAULT_NODELOGTYPE(),
			  "'Moved to hwdown by os_setup; ".
			  "failed to boot image for osid " . $osmap{$node} .
			  " in $pid/$eid'");
	SENDMAIL($TBOPS, "1 node is down",
		 "Node:\n".
		 "  $node\n".
		 "in pid/eid $pid/$eid appears to be dead.\n\n".
		 "The node has been taken out of the pool until this matter ".
		 "is resolved.\n",
		 $user_email_to);

	$failed++;
	add_failed_node_fatal($node);
	goto tballdone;
    }

    #
    # Check for cancelation.  Firewall setup may have taken awhile.
    #
    if (!$canceled) {
	$canceled = $experiment->canceled();
	if ($canceled) {
	    tbnotice({cause => 'canceled', severity => SEV_IMMEDIATE,
		      error => ['cancel_flag']},
		     "Swap canceled; will terminate os_setup early!");
	    goto tballdone;
	}
    }

    #
    # remove it from the nodelist
    #
    delete $nodes{$node};
}

#
# Likewise, setup a PLC node before other plabinelab nodes.
# XXX right now, we setup PLC before ANY other node, whether it is
# part of the inner plab or not.
#
if ($plabinelab) {
    my $node = $plcnode;

    TBDebugTimeStamp("rebooting/reloading PLC node");
    if (!os_setup_one($node, $plcimage, "PLC", 10*60)) {
	tbwarn "PLC node $node failed to boot".
	    "This has been reported to testbed-ops.";
	SENDMAIL($TBOPS, "1 node is down",
		 "Node:\n".
		 "  $node\n".
		 "in pid/eid $pid/$eid failed to boot after loading OS.\n\n".
		 "The nodes have been freed.\n",
		 $user_email_to);
	$failed++;
	add_failed_node_fatal($node);
	goto tballdone;
    }

    #
    # Check for cancelation.  PLC setup may have taken awhile.
    #
    if (!$canceled) {
	$canceled = $experiment->canceled();
	if ($canceled) {
	    tbnotice({cause => 'canceled', severity => SEV_IMMEDIATE,
		      error => ['cancel_flag']},
		     "Swap canceled; will terminate os_setup early!");
	    goto tballdone;
	}
    }

    #
    # remove it from the nodelist
    #
    delete $nodes{$node};
}

#
# Start up plab vnode setup now since it doesn't depend on
# physical node readiness.
#
if (grep($_, values(%plabvnodes))) {
    my $plabnumbatch = TBGetSiteVar("plab/setup/vnode_batch_size");
    my $plabwait     = TBGetSiteVar("plab/setup/vnode_wait_time");
    TBDebugTimeStamp("Starting PlanetLab vnode setup.");
    if (!($plab_setup_pid = fork())) { 
        exec("$vnode_setup -p -n $plabnumbatch -w $plabwait $pid $eid") 
            or die_noretry("Exec failed.");
    } elsif ($plab_setup_pid == -1) {
        die_noretry("Plab fork failed.");
    }
}

#
# Ditto for Geni nodes. Parent keeps going.
#
if (keys(%geninodes)) {
    TBDebugTimeStamp("Starting Geni setup.");

    $geni_setup_pid = fork();
    if (! $geni_setup_pid) {
	TBdbfork();	# So we get the event system fork too ...

	if (libGeni::StartSlivers($experiment, $this_user, $dbg)) {
	    print STDERR "*** Could not start Geni slivers\n";
	    exit(-1);
	}
	TBDebugTimeStamp("Geni slivers have been started.");
	exit(0);
    }
    elsif ($geni_setup_pid == -1) {
        die_noretry("Geni fork failed.");
    }
    # Give it a chance to get going.
    sleep(1);
}

#
# We need to issue the reboots and the reloads in parallel.
#
TBDebugTimeStamp("rebooting/reloading nodes started");
if (!$TESTMODE) {
    my @children = ();

    foreach my $imageid ( keys(%reloads) ) {
	my @nodelist = @{ $reloads{$imageid} };

	my %nodeflags = ();

	foreach my $node (@nodelist) {
	    my $nodeobj = $nodeobjs{$node};
	    
	    #
	    # vnodes only get rebooted if this is a modify and we need to
	    # reload them (otherwise they will get rebooted because of presence
	    # in %reboots).
	    #
	    if (defined($vnodes{$node})) {
		my $vstate;
		$nodeobj->GetAllocState(\$vstate);
		if ($vstate eq TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
		    $nodeflags{$node}{'noreboot'} = 1;
		}
	    }

	    #
	    # osload should not wait for shared vnodes.  We need vnode_setup
	    # to boot/reboot them since the underlying pnode won't be booting.
	    # So for them, osload just sets up the reload and finishes.
	    #
	    if (defined($vnodes{$node}) && $sharednodes{$node} == 1) {
		$nodeflags{$node}{'noreboot'} = 1;
		$nodeflags{$node}{'nowait'} = 1;
	    }

	    $nodeobj->SetAllocState(TBDB_ALLOCSTATE_RES_RELOAD());
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_RELOAD();
	    # No point in reboot/reconfig obviously, since node will reboot!
	    delete $reboots{$node};
	    delete $reconfigs{$node};
	    $rebooted{$node} = 1;
	}

	my %reload_args     = ();
	my $reload_failures = {};

	$reload_args{'debug'}     = $dbg;
	$reload_args{'asyncmode'} = 1;
	$reload_args{'imageid'}   = $imageid;
	$reload_args{'nodelist'}  = [ @nodelist ];
	$reload_args{'nodeflags'} = \%nodeflags;

	my $pid = osload(\%reload_args, $reload_failures);
	push(@children, [ $pid, \&osload_wait,
			  [ @nodelist ], $reload_failures ]);
	sleep(5);
    }

    #
    # Fire off the reboots.
    # 
    if (keys(%reboots)) {
	foreach my $node (keys(%reboots)) {
	    my $nodeobj = $nodeobjs{$node};

	    if ($nodeAllocStates{$node} eq TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
		$nodeobj->SetAllocState(TBDB_ALLOCSTATE_RES_REBOOT_CLEAN());
		$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_REBOOT_CLEAN();
	    } else {
		$nodeobj->SetAllocState(TBDB_ALLOCSTATE_RES_REBOOT_DIRTY());
		$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_REBOOT_DIRTY();
	    }
	    # See below, needed for vnode_setup.
	    $rebooted{$node} = 1;
	}

	my @nodelist        = keys(%reboots);
	my %reboot_args     = ();
	my $reboot_failures = {};

	$reboot_args{'debug'}     = $dbg;
	$reboot_args{'waitmode'}  = 0;
	$reboot_args{'asyncmode'} = 1;
	$reboot_args{'nodelist'}  = [ @nodelist ];

	my $pid = nodereboot(\%reboot_args, $reboot_failures);
	push(@children, [ $pid, \&nodereboot_wait,
			  [ @nodelist ], $reboot_failures ]);
	sleep(2);
    }

    #
    # Fire off the reconfigs.
    #
    if (keys(%reconfigs)) {
	my @nodelist        = keys(%reconfigs);
	my %reboot_args     = ();
	my $reboot_failures = {};

	$reboot_args{'debug'}     = $dbg;
	$reboot_args{'waitmode'}  = 0;
	$reboot_args{'asyncmode'} = 1;
	$reboot_args{'reconfig'}  = 1;
	$reboot_args{'nodelist'}  = [ @nodelist ];

	my $pid = nodereboot(\%reboot_args, $reboot_failures);
	push(@children, [ $pid, \&nodereboot_wait,
			  [ @nodelist ], $reboot_failures ]);
    }

    #
    # Wait for all of the children to exit. We look at the $pid to know if
    # command failed/ended immediately; otherwise we need to wait on it.
    # For any failures, record the node failures for later so that we do
    # not wait for them needlessly.
    #
    while (@children) {
	my ($pid, $waitfunc, $listref, $hashref) = @{ pop(@children) };

	# This is not likely to happen.
	next
	    if ($pid == 0);

	if ($pid > 0) {
	    next
		if (! &$waitfunc($pid));
	}
	
	#
	# Failure. Record the failures for later. If the $pid<0 then the
	# entire list failed. Otherwise, have to scan the return hash to
	# find the failures.
	#
	my @nodelist = ();
	
	if ($pid < 0) {
	    @nodelist = @{ $listref };
	}
	else {
	    foreach my $node (keys(%{ $hashref })) {
		push(@nodelist, $node)
		    if ($hashref->{$node});
	    }
	}

	#
	# These errors are unusal enough that we do not want to retry
	# or keep going even if canfail is set. Better to stop and let
	# someone look at what happened.
	#
	$noretry = 1;

	foreach my $node (@nodelist) {
	    tbnotice "Not waiting for $node since its reload/reboot failed!";
	    $failed++;
	    add_failed_node_reload($node);
	    delete($nodes{$node});

	    $nodeobjs{$node}->SetAllocState(TBDB_ALLOCSTATE_DOWN());
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_DOWN();
	}
    }
}
TBDebugTimeStamp("rebooting/reloading finished");

#
# XXX declare the inner plab nodes as UP since we won't be hearing from
# them again (they are talking only to their PLC).
#
if ($plabinelab) {
    my @plabnodes = ();
    TBExptPlabInElabNodes($pid, $eid, \@plabnodes);
    foreach my $node (@plabnodes) {
	if (exists($nodes{$node})) {
	    tbnotice "Not waiting for emulated plab node $node";
	    $nodeobjs{$node}->SetBootStatus(NODEBOOTSTATUS_OKAY);
	    $nodeobjs{$node}->SetAllocState(TBDB_ALLOCSTATE_RES_READY());
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_READY();
	    $nodeobjs{$node}->SetEventState(TBDB_NODESTATE_ISUP());
	    delete($nodes{$node});
	}
    }
}

#
# Remaining nodes we need to wait for. Why do we wait in the face of errors
# above? So that they enter a reasonably known state before we try to tear
# things down. Otherwise we could end up power cycling nodes a lot more often.
# This should probably be handled in other ways, say via stated or the alloc
# state machine.
#
my @nodelist = keys(%nodes);

#
# Now lets wait for them to come back alive. Set up a retry list though
# so that we can give each node at least 1 second chance. Avoids pointless
# experiment failures.
#
if (@nodelist) {
    print "Waiting for local testbed nodes to finish rebooting ...\n";
}

my %retries;
my %waitstart;
foreach my $node ( @nodelist ) {
    $retries{$node} = (exists($geninodes{$node}) ? 0 : 1);
    $waitstart{$node} = time;
}

#
# List of nodes to inform the user and testbed-ops about in the event
# of failures.  We coalesce the nodes here so we only sent one message.
#
my @informuser = ();
my @informtbopswarn = ();
my @informtbopsfatal = ();

TBDebugTimeStamp("Local node waiting started");
while ( @nodelist ) {
    my $node    = shift(@nodelist);
    my $nodeobj = $nodeobjs{$node};
    my $wstart  = $waitstart{$node};
    my $actual_state;
    my $waittime = (60 * 7);	# The default.

    # Compute actual waittime.
    if (defined($bios_waittime{$node_types{$node}}) &&
	defined($reboot_waittime{$osids{$node}})) {
	$waittime = ($bios_waittime{$node_types{$node}} +
		     $reboot_waittime{$osids{$node}}) * 2;
    }