os_setup.in

#!/usr/bin/perl -wT

#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
require 'ctime.pl';

#
# Reboot the nodes in an experiment. The nodes table will already contain
# all the information. This script deals with possible disk reloading,
# rebooting, and waiting for nodes to come back alive before allowing
# experiment creation to continue.
#
# TODO: Reload disk images.
#
# usage: os_setup <pid> <eid>
#
# errorcode:  0 - all reboots succeeded.
#             1 - some/all reboots failed; retry may help.
#            -1 - failure; retry is inappropriate.
#

sub usage()
{
    print STDERR "Usage: os_setup [-d] <pid> <eid>\n";
    exit(-1);
}
my  $optlist = "d";

#
# Configure variables
#
my $TB		= "@prefix@";
my $DBNAME	= "@TBDBNAME@";
my $TBOPS       = "@TBOPSEMAIL@";
my $TESTMODE    = @TESTMODE@;
my $TFTP	= "/tftpboot";

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libreboot;
use libosload;
use libtestbed;
use libtblog;
use libArchive;
use Template;
use NodeType;

TBDebugTimeStampsOn();

my $vnode_setup = "$TB/sbin/vnode_setup";
my $osselect    = "$TB/bin/os_select";
my $nodereboot  = "$TB/bin/node_reboot";
my $elab_setup  = "$TB/sbin/elabinelab";
my $dbg		= 0;
my $failed      = 0;
my $noretry     = 0;
my $failedvnodes= 0;
my $failedplab  = 0;
my $canceled    = 0;
my %nodes       = ();
my %vnodes      = ();
my %vnodephosts = ();
my %vnode2pnode = ();
my %pnodevcount = ();
my %plabvnodes  = ();
my %osids       = ();
my %canfail     = ();
my %bios_waittime   = ();	# Indexed by node_type.
my %reboot_waittime = ();	# Indexed by osid.
my %node_types  = ();		# Indexed by node_id.
my %vname = ();                 # Indexed by node_id.

#
# This variable keeps track of the failed nodes of all types.
#   values = 'UNKNOWN' 'RELOAD', 'BOOT', 'OTHER'
my %failed_nodes = (); 
my %failed_nonfatal_nodes = ();

#
# Ah, Frisbee works so lets do auto reloading for nodes that do not have
# the proper OS loaded on it. This will be a hash of lists; for each
# imageid, a list of the nodes to pass to os_load for that imageid.
#
my %reloads     = ();
my %reboots	= ();
my %reconfigs	= ();
my %rebooted    = ();
my $doautoload  = 1;
my $dolastload  = 1;

# Protos
sub SetupReload($$$);
sub FirewallSetup($);
sub os_setup_one($$$);
				  
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

$| = 1; #Turn off line buffering on output


#
# Used to die with a -1 return code, to indicate to caller (tbswap)
# that the failure is not likely to be fixed with another attempt.
#
sub die_noretry($)
{
    my ($mesg) = shift;
    tberror $mesg;
    exit(-1);
}

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 2) {
    usage();
}
if (defined($options{"d"})) {
    $dbg = 1;
}

my $pid = $ARGV[0];
my $eid = $ARGV[1];

#
# Untaint args.
#
if ($pid =~ /^([-\@\w]+)$/) {
    $pid = $1;
}
else {
    die_noretry("Bad data in pid: $pid.");
}
if ($eid =~ /^([-\@\w]+)$/) {
    $eid = $1;
}
else {
    die_noretry("Bad data in eid: $eid.");
}

#
# Figure out who called us. Only root, people with admin status
# in the DB, or the owner of the experiment can run this script.
#
if ($UID && !TBAdmin($UID) &&
    !TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_MODIFY)) {
    die_noretry("You do not have permission to swap this experiment!");
}

#
# Verify user and get his DB uid.
# XXX - copied from elsewhere: is this translation needed anymore?
#
my $dbuid;
if (! UNIX2DBUID($UID, \$dbuid)) {
    tbdie("You do not exist in the Emulab Database.");
}

#
# Get email info for user, in case we have to alert them about failures
#
my ($user_name,$user_email);
if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
    tbdie("Cannot determine your name and email address.");
}
my $user_email_to = "$user_name <$user_email>";

TBDebugTimeStamp("os_setup started");

#
# See if the experiment is firewalled
#
my $firewall;
my $firewalled = TBExptFirewall($pid, $eid, \$firewall);
my $firewallimageid;

#
# Ditto ElabinElab.
#
my $elabinelab;
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab)) {
    die("*** $0:\n".
	"    Could not get elabinelab status for experiment $pid/$eid\n");
}

#
# Ditto PlabinElab.
#
my $plabinelab = 0;
my $plcnode;
my $plcimageid;
if (TBExptPlabInElabPLC($pid, $eid, \$plcnode)) {
    $plabinelab = 1;
}

#
# Get the set of nodes, as well as the nodes table information for them.
#
my $db_result =
    DBQueryFatal("select n.*,l.pid,r.vname from reserved as r ".
		 "left join nodes as n on n.node_id=r.node_id ".
		 "left join last_reservation as l on n.node_id=l.node_id ".
		 "where r.pid='$pid' and r.eid='$eid'");

if ($db_result->numrows < 1) {
    print "There are no nodes in experiment '$eid' in project '$pid'.\n";
    exit 0;
}

while (my %row = $db_result->fetchhash()) {
    my $node      = $row{'node_id'};
    my $osid      = $row{'def_boot_osid'};
    my $type      = $row{'type'};
    my $jailnode  = $row{'jailflag'};
    my $failmode  = $row{'failureaction'};
    my $vname     = $row{'vname'};
    my $typeinfo  = NodeType->Lookup($type);
    my $class     = $typeinfo->class();
    my $subnode   = $typeinfo->issubnode();
    my $virtnode  = $typeinfo->isvirtnode();
    my $imageable = $typeinfo->imageable();
    my $plabnode  = $typeinfo->isplabdslice();
    my $bios_wait = $typeinfo->bios_waittime();
    my $bootpath  = 0;

    #
    # VIRTNODE HACK: Virtual nodes are special. Jailed vnodes can do quite
    # a bit, and so run them through the checks below.
    #
    if ($virtnode) {
	$vnodes{$node} = ($jailnode || $plabnode);
	$plabvnodes{$node} = $plabnode;
	if (! $jailnode && ! $plabnode) {
	    next;
	}
    }
    elsif ($subnode && !$imageable) {
	print "Will skip subnode $node ISUP wait.\n";
    }
    else {
	my $nodeAllocState;
	TBGetNodeAllocState( $node, \$nodeAllocState );
	$nodes{$node}  = $node;
	$nodeAllocStates{$node} = $nodeAllocState;
	if ($nodeAllocState eq TBDB_ALLOCSTATE_RES_RECONFIG()) {
	    # Terrible use of state machine.
	    $reconfigs{$node} = 1;
	}
	elsif ($nodeAllocState ne TBDB_ALLOCSTATE_RES_READY()) {
	    # only reboot node if assign_wrapper just pulled it into expt.
	    # (e.g. it isnt ALLOCSTATE_RES_READY)
	    $reboots{$node} = 1;
	}
    }
    $osids{$node}         = $osid;
    $bios_waittime{$type} = $bios_wait;
    $node_types{$node}    = $type;
    $vname{$node}         = $vname;

    #
    # Make sure the files specified in the paths exist. We mount the
    # user tftp directory on boss node, so we can ignore the IP address,
    # and just check the path directly.
    #
    if (defined($row{'def_boot_path'})) {
	my $path = $row{'def_boot_path'};

	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
		die_noretry("File $path for node $node must reside in $TFTP");
	    }

	    if (! -f $path) {
		die_noretry("File $path for node $node does not exist!");
	    }
	    $bootpath = 1;
	}
    }
    if (defined($row{'next_boot_path'})) {
	my $path = $row{'next_boot_path'};

	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
		die_noretry("File $path for node $node must reside in $TFTP");
	    }

	    if (! -f $path) {
		die_noretry("File $path for node $node does not exist!");
	    }
	}
    }

    #
    # XXX - Ditto for RPMs.
    #
    foreach my $rpm (split(":", $row{'rpms'})) {
	if (! -f $rpm) {
	    die_noretry("RPM $rpm for node $node does not exist!");
	}
    }

    #
    # XXX - Ditto for tarfiles.
    #
    foreach my $tarspec (split(":", $row{'tarballs'})) {
	my ($dir, $tar) = split(" ", $tarspec);

	if (! -f $tar) {
	    die_noretry("Tarfile $tar for node $node does not exist!");
	}
    }

    #
    # If there is a path specified, then we don't worry anymore about it.
    # The user must know what is going on. The OSID might have a path
    # associated with it, which means the same thing; we don't worry about
    # it.
    #
    if (!$bootpath && !$jailnode && !$plabnode && $imageable) {
	#
	# These checks are not necessary if the front end and web page
	# are doing the right thing, but lets be careful anyway.
	#
	if (! $osid) {
	    die_noretry("$node has no bootpath and no def_boot_osid set!");
	}

	#
	# Grab the info for this OSID. This is part of the image check.
	#
	my $osid_result =
	    DBQueryFatal("select * from os_info where osid='$osid'");

	if ($osid_result->numrows == 0) {
	    die_noretry("No such OSID $osid is defined!");
	}

	my %osid_row   = $osid_result->fetchhash();

	#
	# If there is an actual path, its an OSKit kernel not an image.
	#
	if (! defined($osid_row{'path'}) || $osid_row{'path'} eq "") {
	    #
	    # Not an OSKit kernel.
	    # Make sure this OSID is actually loaded on the machine.
	    #
	    my $p_result =
		DBQueryFatal("select * from partitions ".
			     "where node_id='$node' and osid='$osid'".
			     "order by partition");

	    #
	    # If not loaded, then see if the user was looking for the generic
	    # name of the OS that is loaded.
	    #
	    if ($p_result->numrows == 0) {
		#
		# Check to see if a non specific version specified.
		#
		if (! defined($osid_row{'version'}) ||
		    $osid_row{'version'} eq "") {

		    #
		    # A non-specific version. There needs to be a way to
		    # map it to another osid.
		    #
		    if (!defined($osid_row{'nextosid'})) {
			die_noretry("No mapping can be made for $osid ($node)!");
		    }

		    my $nextosid = TBResolveNextOSID($osid, $pid, $eid);
		    if (!defined($nextosid)) {
			die_noretry("No mapping can be made for $osid ($node)!");
		    }
		
		    #
		    # See if the nextosid is already on the disk. If not,
		    # it needs to be loaded.
		    #
		    my $o_result =
			DBQueryFatal("select osid from partitions as p ".
				     "where p.node_id='$node' and ".
				     "      p.osid='$nextosid'");

		    if (! $o_result->numrows) {
			#
			# User wants a specific version of an OS, but its not
			# loaded on the machine.
			#
			print "Mapping $osid on $node to $nextosid ".
			    "and setting up a reload.\n";

			SetupReload($node, $nextosid, $type);
			$osids{$node} = $nextosid;
		    }
		    else {
			#
			# Already loaded.
			#
			print "Mapping $osid on $node to $nextosid.\n";

			if ($dolastload &&
			    defined($row{'pid'}) && $row{'pid'} ne $pid) {
			    SetupReload($node, $nextosid, $type);
			}
			else {
			    system("$osselect $nextosid $node") and
				die_noretry("Could not set boot OS to ".
					    "$nextosid for $node");
			}
			$osids{$node} = $nextosid;
		    }
		}
		else {
		    #
		    # User wants a specific version of an OS, but its not
		    # loaded on the machine.
		    #
		    SetupReload($node, $osid, $type);
		}
	    }
	    else {
		#
		# OSID is loaded, but might need to be cleaned.
		#
		if ($dolastload &&
		    defined($row{'pid'}) && $row{'pid'} ne $pid) {
		    SetupReload($node, $osid, $type);
		}
	    }
	}
    }

    #
    # Set the canfail bit.
    #
    $canfail{$node} = (($failmode eq NODEFAILMODE_FATAL()) ? 0 : 1);

    #
    # Set the reboot waittime from the osid now that we have it
    # finalized.
    #
    $osid = $osids{$node};
    if (!exists($reboot_waittime{$osid})) {
	$reboot_waittime{$osid} = TBOSIDRebootWaittime($osid);
    }

    print STDERR "$node - $osids{$node} - $canfail{$node}\n"
	if $dbg;
}

#
# Collect some info about vnodes.
#
foreach my $vnode (keys(%vnodes)) {
    my $jailed = $vnodes{$vnode};
    my $pnode;

    if (! $jailed) {
	next;
    }

    if (! TBPhysNodeID($vnode, \$pnode)) {
	die_noretry("Cannot determine phys_nodeid for $vnode!");
    }

    #
    # Count up the number of jailed nodes on this pnode, and add the
    # mapping. We use this below for determining how long to wait for
    # a particular vnode.
    #
    $pnodevcount{$pnode} = 0
	if (!defined($pnodevcount{$pnode}));
    $pnodevcount{$pnode}++;
    $vnode2pnode{$vnode} = $pnode;

    if (!exists($nodes{$pnode})) {
	#
	# Typical on remote nodes; we do not allocate the underlying
	# phys node to the experiment.
	#
	next;
    }

    # Nothing else to do for local jail nodes at this time ...
}

#
# Setup the firewall first.  Once it is up we can continue with the
# remaining nodes.
#
# There is very little point in setting up the other nodes at the same time
# as they will not be able to PXE boot until the firewall is up.  We could
# fire them off a little early in hopes of overlapping any BIOS boot time
# with the last stages of the firewall setup, but it probably isn't worth
# the complexity (and would not work with nodes for which "reboot" means
# "fall out of PXEWAIT and boot".
#
# Note that we formerly did just do them all at once and let the nodes
# continually PXE-timeout and reboot until the firewall came up.  But that
# can actually take longer than what we do now, if a node happened to
# timeout and reboot just as the firewall came up (i.e., we would have to
# wait an extra BIOS-reboot cycle, which can be 90 seconds or more.
#
if ($firewalled) {
    my $node = $firewall;

    TBDebugTimeStamp("rebooting/reloading firewall");
    if (!FirewallSetup($node)) {
	tbwarn "Firewall node $node failed to boot.".
	    "This has been reported to testbed-ops.";

	# XXX do we need to set NODEBOOTSTATUS_FAILED here?

	#
	# We assume that firewall node images are "standard" here,
	# and whine to tbops.
	#
	MarkNodeDown($node);
	TBSetNodeLogEntry($node, $dbuid, TB_DEFAULT_NODELOGTYPE(),
			  "'Moved to hwdown by os_setup; ".
			  "failed to boot image for osid " . $osids{$node} .
			  " in $pid/$eid'");
	SENDMAIL($TBOPS, "1 node is down",
		 "Node:\n".
		 "  $node\n".
		 "in pid/eid $pid/$eid appears to be dead.\n\n".
		 "The node has been taken out of the pool until this matter ".
		 "is resolved.\n");

	$failed++;
	$failed_nodes{$node} = 'UNKNOWN';
	goto tballdone;
    }

    #
    # Check for cancelation.  Firewall setup may have taken awhile.
    #
    if (!$canceled) {
	TBGetCancelFlag($pid, $eid, \$canceled);
	if ($canceled) {
	    tbnotice "Swap canceled; will terminate os_setup early!";
	    goto tballdone;
	}
    }

    #
    # remove it from the nodelist
    #
    delete $nodes{$node};
}

#
# Likewise, setup a PLC node before other plabinelab nodes.
# XXX right now, we setup PLC before ANY other node, whether it is
# part of the inner plab or not.
#
if ($plabinelab) {
    my $node = $plcnode;

    TBDebugTimeStamp("rebooting/reloading PLC node");
    if (!os_setup_one($node, $plcimageid, "PLC")) {
	tbwarn "PLC node $node failed to boot".
	    "This has been reported to testbed-ops.";
	SENDMAIL($TBOPS, "1 node is down",
		 "Node:\n".
		 "  $node\n".
		 "in pid/eid $pid/$eid failed to boot after loading OS.\n\n".
		 "The nodes have been freed.\n");
	$failed++;
	$failed_nodes{$node} = 'UNKNOWN';
	goto tballdone;
    }

    #
    # Check for cancelation.  PLC setup may have taken awhile.
    #
    if (!$canceled) {
	TBGetCancelFlag($pid, $eid, \$canceled);
	if ($canceled) {
	    tbnotice "Swap canceled; will terminate os_setup early!";
	    goto tballdone;
	}
    }

    #
    # remove it from the nodelist
    #
    delete $nodes{$node};
}

#
# We need to issue the reboots and the reloads in parallel.
#
TBDebugTimeStamp("rebooting/reloading nodes started");
if (!$TESTMODE) {
    my @children = ();

    foreach my $imageid ( keys(%reloads) ) {
	my @nodelist = @{ $reloads{$imageid} };

	foreach my $node (@nodelist) {
	    TBSetNodeAllocState( $node, TBDB_ALLOCSTATE_RES_RELOAD() );
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_RELOAD();
	    # No point in reboot/reconfig obviously, since node will reboot!
	    delete $reboots{$node};
	    delete $reconfigs{$node};
	    $rebooted{$node} = 1;
	}

	my %reload_args     = ();
	my $reload_failures = {};

	$reload_args{'debug'}     = $dbg;
	$reload_args{'asyncmode'} = 1;
	$reload_args{'imageid'}   = $imageid;
	$reload_args{'nodelist'}  = [ @nodelist ];

	my $pid = osload(\%reload_args, $reload_failures);
	push(@children, [ $pid, \&osload_wait,
			  [ @nodelist ], $reload_failures ]);
	sleep(5);
    }

    #
    # Fire off the reboots.
    # 
    if (keys(%reboots)) {
	foreach my $node (keys(%reboots)) {
	    if ($nodeAllocStates{$node} eq TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
		TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_REBOOT_CLEAN());
		$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_REBOOT_CLEAN();
	    } else {
		TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_REBOOT_DIRTY());
		$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_REBOOT_DIRTY();
	    }
	    # See below, needed for vnode_setup.
	    $rebooted{$node} = 1;
	}

	my @nodelist        = keys(%reboots);
	my %reboot_args     = ();
	my $reboot_failures = {};

	$reboot_args{'debug'}     = $dbg;
	$reboot_args{'waitmode'}  = 0;
	$reboot_args{'asyncmode'} = 1;
	$reboot_args{'nodelist'}  = [ @nodelist ];

	my $pid = nodereboot(\%reboot_args, $reboot_failures);
	push(@children, [ $pid, \&nodereboot_wait,
			  [ @nodelist ], $reboot_failures ]);
	sleep(2);
    }

    #
    # Fire off the reconfigs.
    #
    if (keys(%reconfigs)) {
	my @nodelist        = keys(%reconfigs);
	my %reboot_args     = ();
	my $reboot_failures = {};

	$reboot_args{'debug'}     = $dbg;
	$reboot_args{'waitmode'}  = 0;
	$reboot_args{'asyncmode'} = 1;
	$reboot_args{'reconfig'}  = 1;
	$reboot_args{'nodelist'}  = [ @nodelist ];

	my $pid = nodereboot(\%reboot_args, $reboot_failures);
	push(@children, [ $pid, \&nodereboot_wait,
			  [ @nodelist ], $reboot_failures ]);
    }

    #
    # Wait for all of the children to exit. We look at the $pid to know if
    # command failed/ended immediately; otherwise we need to wait on it.
    # For any failures, record the node failures for later so that we do
    # not wait for them needlessly.
    #
    while (@children) {
	my ($pid, $waitfunc, $listref, $hashref) = @{ pop(@children) };

	# This is not likely to happen.
	next
	    if ($pid == 0);

	if ($pid > 0) {
	    next
		if (! &$waitfunc($pid));
	}
	
	#
	# Failure. Record the failures for later. If the $pid<0 then the
	# entire list failed. Otherwise, have to scan the return hash to
	# find the failures.
	#
	my @nodelist = ();
	
	if ($pid < 0) {
	    @nodelist = @{ $listref };
	}
	else {
	    foreach my $node (keys(%{ $hashref })) {
		push(@nodelist, $node)
		    if ($hashref->{$node});
	    }
	}

	#
	# These errors are unusal enough that we do not want to retry
	# or keep going even if canfail is set. Better to stop and let
	# someone look at what happened.
	#
	$noretry = 1;

	foreach my $node (@nodelist) {
	    tbnotice "Not waiting for $node since its reload/reboot failed!";
	    $failed++;
	    $failed_nodes{$node} = 'UNKNOWN';
	    delete($nodes{$node});

	    TBSetNodeAllocState($node, TBDB_ALLOCSTATE_DOWN());
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_DOWN();
	}
    }
}
TBDebugTimeStamp("rebooting/reloading finished");

#
# XXX declare the inner plab nodes as UP since we won't be hearing from
# them again (they are talking only to their PLC).
#
if ($plabinelab) {
    my @plabnodes = ();
    TBExptPlabInElabNodes($pid, $eid, \@plabnodes);
    foreach my $node (@plabnodes) {
	if (exists($nodes{$node})) {
	    tbnotice "Not waiting for emulated plab node $node";
	    SetNodeBootStatus($node, NODEBOOTSTATUS_OKAY);
	    TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_READY());
	    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_READY();
	    TBSetNodeEventState($node, TBDB_NODESTATE_ISUP());
	    delete($nodes{$node});
	}
    }
}

#
# Remaining nodes we need to wait for. Why do we wait in the face of errors
# above? So that they enter a reasonably known state before we try to tear
# things down. Otherwise we could end up power cycling nodes a lot more often.
# This should probably be handled in other ways, say via stated or the alloc
# state machine.
#
my @nodelist = keys(%nodes);

#
# Now lets wait for them to come back alive. Set up a retry list though
# so that we can give each node at least 1 second chance. Avoids pointless
# experiment failures.
#
if (@nodelist) {
    print "Waiting for local testbed nodes to finish rebooting ...\n";
}

my %retries;
my %waitstart;
foreach my $node ( @nodelist ) {
    $retries{$node} = 1;
    $waitstart{$node} = time;
}

#
# List of nodes to inform the user and testbed-ops about in the event
# of failures.  We coalesce the nodes here so we only sent one message.
#
my @informuser = ();
my @informtbopswarn = ();
my @informtbopsfatal = ();

TBDebugTimeStamp("Local node waiting started");
while ( @nodelist ) {
    my $node   = shift(@nodelist);
    my $wstart = $waitstart{$node};
    my $actual_state;
    my $waittime = (60 * 7);	# The default.

    # Compute actual waittime.
    if (defined($bios_waittime{$node_types{$node}}) &&
	defined($reboot_waittime{$osids{$node}})) {
	$waittime = ($bios_waittime{$node_types{$node}} +
		     $reboot_waittime{$osids{$node}}) * 2;
    }

    if (!TBNodeStateWait($node, $wstart, $waittime, \$actual_state,
			 (TBDB_NODESTATE_TBFAILED, TBDB_NODESTATE_ISUP))) {
	if ($actual_state eq TBDB_NODESTATE_TBFAILED) {
	    tbwarn "$node reported a TBFAILED event; not retrying";
	    $retries{$node} = 0;
	    goto tbfailed;
	}
	print "$node is alive and well\n";
	SetNodeBootStatus($node, NODEBOOTSTATUS_OKAY);
	TBSetNodeAllocState( $node, TBDB_ALLOCSTATE_RES_READY() );
	$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_READY();
	next;
    }

    #
    # Check for cancelation. Do not want to retry the reboots if the
    # swap was canceled.
    #
    if (!$canceled) {
	TBGetCancelFlag($pid, $eid, \$canceled);

	if ($canceled) {
	    tbnotice "Swap canceled; will terminate os_setup early!";
	}
    }

    if ($retries{$node} && !($canceled || $noretry)) {
	$retries{$node} -= 1;

	tbnotice "Rebooting $node and waiting again ...";

	if (system("$nodereboot $node") == 0) {
	    push(@nodelist, $node);
	    $waitstart{$node} = time;
	    next;
	}
	# Fall through on failure.
    }

    tbwarn "$node may be down. This has been reported to testbed-ops.";

  tbfailed:
    SetNodeBootStatus($node, NODEBOOTSTATUS_FAILED);

    if ($canfail{$node} && !($canceled || $noretry)) {
	push(@informuser, $node);
	$failed_nonfatal_nodes{$node} = 'UNKNOWN';
	tbnotice "Continuing with experiment setup anyway ...";
	next;
    }

    #
    # If the user has picked a standard image and it fails to boot,
    # something is wrong, so reserve it to hwdown experiment. If the
    # image belongs to the user, then we assume its the image at fault,
    # and allow it to be returned to the pool (caller, tbswap will end
    # doing the nfree on nodes with a DOWN allocstate).
    #
    my $pidofosid;
    if (! TBOsidToPid($osids{$node}, \$pidofosid) ||
	$pidofosid eq TBOPSPID()) {
	MarkNodeDown($node);
	TBSetNodeLogEntry($node, $dbuid, TB_DEFAULT_NODELOGTYPE(),
			  "'Moved to hwdown by os_setup; ".
			  "failed to boot image for osid " . $osids{$node} .
			  " in $pid/$eid'");
	push(@informtbopsfatal, $node);
    } else {
	push(@informtbopswarn, $node);
    }
    TBSetNodeAllocState( $node, TBDB_ALLOCSTATE_DOWN() );
    $nodeAllocStates{$node} = TBDB_ALLOCSTATE_DOWN();

    $failed++;
    $failed_nodes{$node} = 'UNKNOWN';
}

#
# Spam time!  Send mail to the user and testbed-ops about failures.
#
my $count = scalar(@informuser);
if ($count > 0) {
    SENDMAIL($user_email_to, "$count nodes are down",
	     "Nodes:\n".
	     "  " . join(" ", @informuser) . "\n".
	     "in pid/eid $pid/$eid appear to be dead.\n\n".
	     "Your experiment will continue to run since these failures\n".
	     "are nonfatal, although you might encounter other problems\n".
	     "if your experiment depends explicitly on these nodes.\n".
	     "You should terminate this experiment if it cannot ".
	     "tolerate these failures.\n\n".
	     "Testbed Operations has also been notified.\n\n".
	     "Thanks\n".
	     "Testbed Operations\n",
	     0,
	     "Cc: $TBOPS");
}
$count = scalar(@informtbopsfatal);
if ($count > 0) {
    SENDMAIL($TBOPS, "$count nodes are down",
	     "Nodes:\n".
	     "  " . join(" ", @informtbopsfatal) . "\n".
	     "in pid/eid $pid/$eid appear to be dead.\n\n".
	     "The nodes have been taken out of the pool until this matter ".
	     "is resolved.\n");
}
$count = scalar(@informtbopswarn);
if ($count > 0) {
    SENDMAIL($TBOPS, "$count nodes are down",
	     "Nodes:\n".
	     "  " . join(" ", @informtbopswarn) . "\n".
	     "in pid/eid $pid/$eid failed to boot after loading OS.\n\n".
	     "The nodes have been freed.\n");
}

TBDebugTimeStamp("Local node waiting finished");

#
# Now deal with virtual nodes.
#
# We do this in a sub script since nodes are not owned by the user
# and so must be setuid root so that ssh will work.
#
my @vnodelist = keys(%vnodes);

#
# Set the allocstate for the local vnodes that were sucessfully rebooted
# and came to ISUP above. These do not need to be setup again! We move
# them to RES_READY, so vnode_setup will ignore them. If they fail to
# hit ISUP, we will move them to DOWN so that vnode_setup will ignore
# them again, in the teardown phase.
#
# Note, we do this even if there were failures above, since the teardown
# phase is going to happen, and we want vnode_setup to know which nodes
# came up with phynodes okay (need to be torndown) and which ones never
# had the chance (no need to teardown). Think swapmod, which does teardown
# in the ACTIVATING state.
#
foreach my $vnode (@vnodelist) {
    my $pnode  = $vnode2pnode{$vnode};

    # Default retry count.
    $retries{$vnode} = 0;

    # Remote node, always does setup.
    next
	if (!exists($nodes{$pnode}));
    
    # Pnode was neither rebooted or reconfiged, so leave allocstate alone
    # for vnode_setup (has to be done).
    next
	if (!exists($rebooted{$pnode}) && !exists($reconfigs{$pnode}));

    if ($nodeAllocStates{$pnode} eq TBDB_ALLOCSTATE_RES_READY()) {
	TBSetNodeAllocState($vnode, TBDB_ALLOCSTATE_RES_READY());
	$nodeAllocStates{$vnode} = TBDB_ALLOCSTATE_RES_READY();
    }
}

#
# Reset the failure lists. See above.
#
@informuser = ();
@informtbopswarn = ();
@informtbopsfatal = ();

#
# XXX - Don't bother if something above failed. A waste of time and
# usually leads to cascading errors.
#
if ($canceled && @vnodelist) {
    tbnotice "Skipping virtual node setup since swapin was canceled!";
}
elsif ($failed && @vnodelist) {
    tbnotice "Skipping virtual node setup since there were previous ".
	"failures!";
}
elsif (@vnodelist) {
    my $vnode_setup_args = ""; # add any generic args here.
    my @retry_list = ();

    TBDebugTimeStamp("Setting up virtual nodes");
    print "Setting up virtual testbed nodes ...\n";

    # If there are any plab vnodes, we have to adjust batching and timeouts
    # accordingly.
    if (grep($_, values(%plabvnodes))) {
        my $plabnumbatch = TBGetSiteVar("plab/setup/vnode_batch_size");
        my $plabwait     = TBGetSiteVar("plab/setup/vnode_wait_time");        
        $vnode_setup_args .= " -n $plabnumbatch -w $plabwait ";
    }

  retry:
    TBDebugTimeStamp("Setting up virtual nodes");
    system("$vnode_setup $vnode_setup_args $pid $eid");
    if ($?) {
	die_noretry("Vnode setup failed!");
    }
    print "Waiting for virtual testbed nodes to finish setting up ...\n";
    TBDebugTimeStamp("Virtual node waiting started");

    foreach my $node (@vnodelist) {
	$waitstart{$node} = time;
    }
    @vnodelist = sort(@vnodelist);

    while ( @vnodelist ) {
	my $node   = shift(@vnodelist);
	my $pnode  = $vnode2pnode{$node};
	my $islocal= exists($nodes{$pnode});
	my $wstart = $waitstart{$node};
	my $curallocstate;
	my $actual_state;

        #
        # Base the maxwait for vnodes on the reboot_waittime field for
        # their respective OSIDs, with some slop time that scales up
        # as a function of the number of vnodes on the parent pnode.
        #
        my $osid        = $osids{$node};
        my $reboot_time = $reboot_waittime{$osid};
	my $maxwait     = $reboot_time + (40 * $pnodevcount{$pnode});

	TBGetNodeAllocState($node, \$curallocstate);

	#
	# See if vnode_setup already determined the node was dead.
	#
	if ($curallocstate ne TBDB_ALLOCSTATE_DOWN() &&
	    $curallocstate ne TBDB_ALLOCSTATE_DEAD()) {

	    if (!TBNodeStateWait($node, $wstart, $maxwait, \$actual_state,
				 (TBDB_NODESTATE_TBFAILED,
				  TBDB_NODESTATE_ISUP))) {

		if ($actual_state eq TBDB_NODESTATE_TBFAILED) {
		    tbwarn "$node reported a TBFAILED event.";
		    goto vtbfailed;
		}
		print "$node is alive and well\n";
		TBDebugTimeStamp("Virtual node $node setup ISUP");
		
		# Might have already been set above.
		TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_READY);
		SetNodeBootStatus($node, NODEBOOTSTATUS_OKAY);
		next;
	    }

	  vtbfailed:
	    TBDebugTimeStamp("Virtual node $node setup FAILED");
	    SetNodeBootStatus($node, NODEBOOTSTATUS_FAILED);
	    TBSetNodeAllocState($node, TBDB_ALLOCSTATE_DOWN());

	    #
	    # If a local node, lets retry since jail setup appears to be
	    # rather flaky.