stated.in

#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#

#
# stated - A daemon to monitor the states of nodes in the testbed. Recives
# state change notification through the event system, and writes the new
# state into the database. Also watches for invalid transitions, timeouts,
# and performs other state-related control functions.
#
# Send it a HUP signal to get it to reload the timeout and transition
# information. Periodically reloads this information regardless, though.
#
# Will restart when sent SIGUSR1, by exec'ing its executable again.
#

# Configure variables
use lib '@prefix@/lib';
my $TB = "@prefix@";
my $REALTB = "/usr/testbed"; # So we know if we're the "real" stated or not
my $BOSSNODE = "@BOSSNODE@";
my $TBOPS = "@TBSTATEDEMAIL@";
my $REALTBOPS = "@TBOPSEMAIL@";
my $TBDBNAME = "@TBDBNAME@";
my $REALTBDBNAME = "tbdb"; # So we know if we're using the "real" db
my $osselect = "$TB/bin/os_select";
my $nodereboot = "$TB/bin/node_reboot";
my $rebootlog  = "$TB/log/nodereboot.log";
my $power = "$TB/bin/power";
my $TBLOG = "@TBLOGFACIL@";

$| = 1;

use event;
use libdb;
use libtestbed;
use TimeoutQueue;
use Getopt::Std;
#use strict;
use English;
use POSIX;			# for strftime, and sigprocmask and friends
use Fcntl;			# file constants for pidfile
use Sys::Syslog;
# Important note about syslog: It defaults to using an inet socket,
# but 'syslogd -s' (the default) doesn't listen for one. So either
# run syslogd without -s, or use setlogsock('unix') before openlog.
# (To get setlocksock: 'use Sys::Syslog qw(:DEFAULT setlogsock);' )

# Do lots of db retries before we fail and die
$libdb::DBQUERY_MAXTRIES = 100;

# Set up some notification throttling
my $mailgap = 15;		# in seconds
my $lastmail = time() - $mailgap + 2; # Send a digest of startup msgs after 2s.
my %msgs = ();

# Number of iterations (roughly, seconds) after which we'll reload
# information from the database. This is so we don't end up with information
# that's _too_ out of sync.
my $reload_time = 600;
my $last_reload = time;

# Command line opts.
my $dbtag = "";
my $debug = 0;
my $nolog = 0;
my $server = "localhost";
my $port   = @BOSSEVENTPORT@;
my $lockfile;
my $pidfile;

# Process command-line arguments

sub usage {
    print << "END";
Usage: $0 [-h] [-d] [-s server] [-p port] [-t dbtag]
-h              This message
-d              Turn on debugging output, and do not go into the background
-l              Do not use syslog; send output to stderr. Use with -d only
-t tag          Use only those nodes with matching tag in nodes table
-s server       Use specified server, instead of this site's bossnode
-p port	        Use specified port
Send SIGHUP to reload database state, or SIGUSR1 to restart completely.
END
    exit(1);
}

# Only root should run this - it won't work when run as a user...
# (Or, let an admin run it if it isn't the real one in /usr/testbed/ )
if ($UID && ( $TB eq $REALTB || ! TBAdmin($UID) ) ) {
    die("Only root can run this script!\n");
}

my @args = @ARGV;    # save a copy for restart before we mess with them.
my %opt = ();
if (!getopts("ds:p:ht:l",\%opt)) { usage(); }

if ($opt{h}) {
    exit &usage;
}
if (@ARGV) {
    exit &usage;
}

if ($opt{s}) {
    $server = $opt{s};
}
if ($opt{p}) {
    $port = $opt{p};
}
if ($opt{l}) {
    usage()
	if (! $opt{d});
    $nolog = 1;
}
if ($opt{t}) {
    $dbtag = $opt{t};
}
if ($opt{d}) {
    $debug = 1;
}

# Grab some constants into variables
my $TBANYMODE    = TBDB_NODEOPMODE_ANY;
my $TBRESET      = TBDB_TBCONTROL_RESET;
my $TBRELOADDONE = TBDB_TBCONTROL_RELOADDONE;
my $TBTIMEOUT    = TBDB_TBCONTROL_TIMEOUT;
my $PXEBOOT      = TBDB_TBCONTROL_PXEBOOT;
my $BOOTING      = TBDB_TBCONTROL_BOOTING;
my $CHECKGENISUP = TBDB_TBCONTROL_CHECKGENISUP;
my $TBNOTIMEOUT  = TBDB_NO_STATE_TIMEOUT;
my $TBNODESTATE  = TBDB_TBEVENT_NODESTATE;
my $TBNODEOPMODE = TBDB_TBEVENT_NODEOPMODE;
my $TBCONTROL    = TBDB_TBEVENT_CONTROL;
my $TBCOMMAND    = TBDB_TBEVENT_COMMAND;
my $TBREBOOT     = TBDB_COMMAND_REBOOT;
my $TBPOWEROFF   = TBDB_COMMAND_POWEROFF;
my $TBPOWERON    = TBDB_COMMAND_POWERON;
my $TBPOWERCYCLE = TBDB_COMMAND_POWERCYCLE;
my $TBISUP       = TBDB_NODESTATE_ISUP;
my $PXEWAIT      = TBDB_NODESTATE_PXEWAIT;
my $PXEWAKEUP    = TBDB_NODESTATE_PXEWAKEUP;
my $PXEBOOTING   = TBDB_NODESTATE_PXEBOOTING;
my $TBTIMEOUTREBOOT   = TBDB_STATED_TIMEOUT_REBOOT;
my $TBTIMEOUTNOTIFY   = TBDB_STATED_TIMEOUT_NOTIFY;
my $TBTIMEOUTCMDRETRY = TBDB_STATED_TIMEOUT_CMDRETRY;
my $TB_OSID_MBKERNEL  = TB_OSID_MBKERNEL;

# Special PXEBOOT state machine that all local nodes use.
my $PXEKERNEL	 = "PXEKERNEL";

if (!$debug) {
    if ( $TB eq $REALTB ) {
	$pidfile = "/var/run/stated.pid";
    } else {
	$pidfile = "$TB/locks/stated.pid";
    }
    debug("Using pidfile $pidfile\n");

    if (-e $pidfile) {
	my $otherpid = `cat $pidfile`;
	my $running = `ps -auxww | grep $otherpid | grep -v grep`;
	if ($running ne "") {
	    fatal("Lockfile $pidfile exists, and process $otherpid appears ".
		  "to be running.\n");
	} else {
	    notify("Lockfile exists, but process $otherpid appears to be dead".
		   "\n".
		   "Removing lock file...\n");
	}
	system("rm $pidfile") &&
	    fatal("Couldn't remove $pidfile: $? $!\n");
    }
    # Background
    # We use syslog, so redirect the output to nothing
    if (TBBackGround("/dev/null")) {
	exit(0);
    }
}
# set up syslog
if (! $nolog) {
    openlog("stated","pid",$TBLOG);
}

if (defined($pidfile)) {
    sysopen(PIDFILE, $pidfile, O_WRONLY | O_EXCL | O_CREAT) ||
	fatal("Couldn't create '$pidfile': $? $!\n");
    print PIDFILE "$$";
    close PIDFILE;
    # If I make it to here, I'll need to clean up the lock file
    $lockfile = $pidfile;
}

# Change my $0 so that it is easier to see in a ps/top
$0 = "$0";

my $URL = "elvin://$server";
if ($port) {
    $URL .= ":$port";
}

# Connect to the event system, and subscribe the the events we want
my $handle = event_register($URL,0);
if (!$handle) {
    fatal("Unable to register with event system\n");
}

my $tuple = address_tuple_alloc();
if (!$tuple) {
    fatal("Could not allocate an address tuple\n");
}

%$tuple = ( objtype => join(",",
			    $TBNODESTATE, $TBNODEOPMODE,
			    $TBCONTROL, $TBCOMMAND) );

if (!event_subscribe($handle,\&handleEvent,$tuple)) {
    fatal("Could not subscribe to events\n");
}

# Read in the pre-existing node states, and timeout and valid transition
# information from the database
my %timeouts  = getTimeouts();
my %valid     = getValid();
my %modeTrans = getModeTrans();
my %triggers  = getTriggers();
my %nodes     = readStates();
my %timeouttag= ();
if ($debug) { qshow(); }

# Gets set if a reload of state from the database should happen.
my $do_reload = 0;
my $sigrestart= 0;
my $sigcleanup= 0;

# Make the daemon reload database state on a sighup - but I'm worried
# about what would happen if we tried to do this mid-loop. So, we'll
# just set a flag and do it when we're done with our current pass.
$SIG{HUP}  = sub { info("SIGHUP - Reloading DB state\n"); $do_reload = 1; };

# Set up other signals.
$SIG{USR1} = \&restart_wrap;
$SIG{USR2} = \&cleanup_wrap;
$SIG{INT}  = \&cleanup_wrap;
$SIG{QUIT} = \&cleanup_wrap;
$SIG{ABRT} = \&cleanup_wrap;
$SIG{TERM} = \&cleanup_wrap;
$SIG{KILL} = \&cleanup_wrap;

# Track if I handled an event or not
my $event_count = 0;

# Control how long I block while waiting for events
my $blockwait=0;
my $nextdeadline=0;
my $mailqueue=0;

notify("Stated starting up\n");

sub process_event_queue() {
    $event_count=0;
    my $lastcount=-1;
    my $wait;
    my $now = time();
    debug("Polling - mq=$mailqueue bw=$blockwait\n");
    if ( $mailqueue == 0) {
	# no messages waiting...
	if ($blockwait) {
	    # we can wait a long time - nothing else will happen
	    # until we get an event, or get woken up by a signal
	    $wait = 600;
	} else {
	    # only wait until the next deadline...
	    if ($nextdeadline > 0) {
		$wait = $nextdeadline - $now;
	    } else {
		$wait = 0;
	    }
	}
    } else {
	# mail is waiting. Only block until it is time to send it.
	$wait = $lastmail + $mailgap - $now;
	debug("Now $now, mailgap $mailgap, last $lastmail ==> wait $wait\n");
    }
    if ($wait < 0) { debug("Wait was $wait!\n"); $wait=0; }
    my $finish = $now + $wait;
    while (($event_count != $lastcount || $wait > 0) &&
	   !($sigrestart || $sigcleanup || $do_reload)) {
	$lastcount = $event_count;
	# Don't block if we got a signal!
	if ($wait<=0 || $sigrestart || $sigcleanup || $do_reload) {
	    event_poll($handle);
	} else {
	    #debug("Using blocking event poll - $wait seconds\n");
	    # timeout param is in milliseconds, so multiply
	    event_poll_blocking($handle, $wait*1000);
	    $now = time();
	    # subtract seconds elapsed from my wait time
	    $wait = $finish - $now;
	    #debug("Finished blocking event poll - $wait seconds remain\n");
	    if ($event_count > 0 &&
		(qsize() > 0 || $mailqueue ||
		 $sigrestart || $sigcleanup || $do_reload)) {
		$blockwait=0;
		$wait=0;
		#debug("Cancelling wait - timeouts/msgs waiting, or HUP'd\n");
		#debug("---End Blocking Wait ---\n");
	    }
	}
	#debug("Wait is $wait\n");
    }
    if ($event_count > 0) {
	debug("Handled $event_count event(s).\n");
    }
}

# Now, we just poll for events, and watch for timeouts
while (1) {
    my $now = time();
    my ($deadline,$node);

    # Check for nodes that have passed their timeout
    if (!qhead($deadline,$node)) {
	info("HEAD: $node in ".($deadline-$now).", queue=".qsize()."\n");
	while ($now >= $deadline && $node ne "") {
	    qpop($deadline,$node);
	    info("POP: $node in ".($deadline-$now).", queue=".qsize()."\n");
	    handleCtrlEvent($node,$TBTIMEOUT);
	    if (0) { qshow(); }
	    if (qhead($deadline,$node)) {
		$deadline=0; $node="";
	    }
	}
    } else {
	$deadline=0;
    }
    $nextdeadline = $deadline;

    if (qsize()==0) {
	$blockwait=1;
	debug("---Blocking wait okay---\n");
    }

    if ($do_reload || ($now - $last_reload > $reload_time)) {
	reload();
	$do_reload = 0;
    }

    # Send any messages in the queue if it is time
    notify("",1);

    if ($sigrestart) { restart(); }
    if ($sigcleanup) { cleanup(); }

    process_event_queue;
}

exit(0);

# Read the current states of nodes from the database
sub readStates(;@) {
    my %oldnodes = @_;

    # Guard against undefined variable warnings
    if (! defined(%oldnodes)) {
	%oldnodes = ();
    }

    #debug("readStates called\n");
    my $result = DBQueryFatal("SELECT node_id, eventstate, " .
			      "state_timestamp, op_mode, " .
			      "op_mode_timestamp, stated_tag FROM nodes ".
			      "where node_id not like 'sh%'");

    my %nodes;
    while (my ($node_id, $state, $timestamp, $mode, $mode_timestamp, $tag)
	   = $result->fetchrow()) {
	$nodes{$node_id}{"tag"} = (defined($tag) ? $tag : "");

	if ($dbtag ne "" && $dbtag eq $nodes{$node_id}{"tag"}) {
	    info("This stated will work on $node_id\n");
	}
	if ($dbtag eq "" && $dbtag ne $nodes{$node_id}{"tag"}) {
	    info("This stated will *NOT* work on $node_id\n");
	}
	if ($dbtag ne $nodes{$node_id}{"tag"}) {
	    remTimeout($node_id);
	}

	#
	# If there's an entry in oldnodes for this node, and it
	# hasn't changed state or time, use the old entry (so that
	# we don't lose information about which nodes we've already
	# notified the ops about, etc.)
	#
	if ($oldnodes{$node_id} && $state && $timestamp &&
	    ($oldnodes{$node_id}{state} eq $state) &&
	    ($oldnodes{$node_id}{mode} eq $mode) &&
	    ($oldnodes{$node_id}{timestamp} == $timestamp)) {
	    $nodes{$node_id} = $oldnodes{$node_id};
	} else {
	    $nodes{$node_id}{state}          = $state;
	    $nodes{$node_id}{timestamp}      = $timestamp;
	    $nodes{$node_id}{mode}           = $mode;
	    $nodes{$node_id}{mode_timestamp} = $mode_timestamp;
	    $nodes{$node_id}{notified}       = 0;
	    $nodes{$node_id}{timedout}       = 0;
	    $nodes{$node_id}{noretry}        = 0;
	    # Is there a timeout? If so, set it up!
	    if ($dbtag eq $nodes{$node_id}{"tag"}) {
		setTimeout($mode,$state,$node_id,$timestamp);
	    }
	}
    }
    return %nodes;
}

#
# Read timeouts for various states from the database
#
sub getTimeouts() {
    #debug("getTimeouts called\n");
    my $result = DBQueryFatal("SELECT op_mode, state, timeout, action " .
			      "FROM state_timeouts");

    my %timeouts;
    while (my ($op_mode, $state, $timeout, $action) = $result->fetchrow()) {
	$timeouts{$op_mode}{$state} = [ $timeout, $action ];
    }
    return %timeouts;
}

#
# Read the list of valid state transitions from the database
#
sub getValid() {
    #debug("getValid called\n");
    my $result = DBQueryFatal("SELECT op_mode, state1, state2 " .
			      "FROM state_transitions");

    my %valid;
    while (my ($mode,$state1, $state2) = $result->fetchrow()) {
	$valid{$mode}{$state1}{$state2} = 1;
    }
    return %valid;
}

#
# Read the list of valid mode transitions from the database
#
sub getModeTrans() {
    #debug("getModeTrans called\n");
    my $result =
      DBQueryFatal("SELECT op_mode1, state1, op_mode2, state2 " .
		   "FROM mode_transitions order by op_mode1,state1");

    my %modeTrans;
    while (my ($mode1,$state1, $mode2, $state2) = $result->fetchrow()) {
	if (!defined($modeTrans{"$mode1:$state1"})) {
	    $modeTrans{"$mode1:$state1"}= ["$mode2:$state2"];
	} else {
	    my @l = @{$modeTrans{"$mode1:$state1"}};
	    push(@l, "$mode2:$state2");
	    $modeTrans{"$mode1:$state1"}= \@l;
	}
    }
    return %modeTrans;
}

#
# Read the list of states which trigger an action
#
sub getTriggers() {
    debug("getTriggers called\n");

    debug("anymode ==> '$TBANYMODE'\n");

    # A note about triggers:
    #
    # "per-node" triggers only affect their specific node in a
    # particular mode/state, and are run first of all. "global"
    # triggers are triggers for a given mode/state that affect all
    # nodes, and are run after any per-node triggers. "Any-mode"
    # triggers are tied to a state, and occur in that state in any
    # mode. The any-mode triggers are over-ridden by global triggers,
    # and if an "Any-mode" trigger for state XYZ exists as well as a
    # global trigger for mode FOOBAR state XYZ, then when I arrive in
    # XYZ any per-node triggers will be run. Then, if I'm in mode
    # FOOBAR, only the global trigger will run. If I'm in any other
    # mode, only the any-mode trigger will run.

    # (our "*" is stored as $TBANYMODE)
    # Per-node triggers have a specific node_id
    # Global triggers have "*" as the node_id
    # Any-mode triggers have "*" as the mode, and can be global or per-node

    # Grab global triggers (including any-mode)
    my $result =
      DBQueryFatal("SELECT op_mode, state, trigger " .
		   "FROM state_triggers where node_id='$TBANYMODE' ".
		   "order by op_mode,state");
    my %t;
    while (my ($mode, $state, $trig) = $result->fetchrow()) {
	my @trigs = split(/\s*,\s*/,$trig);
	$t{"$mode:$state"} = \@trigs;
	debug("trig($mode:$state)\t => ".join(',',@trigs)."\n");
    }

    # Grab per-node triggers (including any-mode)
    $result =
      DBQueryFatal("SELECT node_id, op_mode, state, trigger " .
		   "FROM state_triggers where node_id!='$TBANYMODE' ".
		   "order by op_mode,state");
    while (my ($n, $mode, $state, $trig) = $result->fetchrow()) {
	my @trigs = split(/\s*,\s*/,$trig);
	$t{"$n:$mode:$state"} = \@trigs;
	debug("trig($n:$mode:$state)\t => ".join(',',@trigs)."\n");
    }

    debug(hash_recurse(%t));

    return %t;
}

#
# Gets called for every event that we recieve
#
sub handleEvent($$$) {
    my ($handle,$notification,$data) = @_;
    my $objtype = event_notification_get_objtype($handle,$notification);
    my $objname = event_notification_get_objname($handle,$notification);
    my $eventtype = event_notification_get_eventtype($handle,$notification);

    $event_count++;

    #
    # For readability, only do this on the main stated. This will print all
    # events, which gets cumbersome with debugging versions, so we'll print
    # only applicable events in debug versions, after we decide they apply.
    #
    if ($dbtag eq "") {
	debug("Got an event: ($objtype,$objname,$eventtype)\n");
    }

    #
    # Check to see if another instance is supposed to be handling this node
    #
    if ($objtype ne $TBCOMMAND) {
	my $node = $objname;

	#
	# If we have never seen this node, reload.
	#
	if (! defined($nodes{$node})) {
	    reload();

	    # Still not defined, someone screwed up! This could end up
	    # churning via reload(). Bad.
	    if (! defined($nodes{$node})) {
		notify("Got $objtype/$eventtype for nonexistent $node!\n");
		return;
	    }
	}

	#
	# If a stated_tag was specified on the command line, ignore those
	# nodes that do not match.
	#
	#debug("dbtag='$dbtag', node $node='".$nodes{$node}{"tag"}."'\n");
	if ($dbtag ne $nodes{$node}{"tag"}) {
	    # Record when main stated ignores a node.
	    info("Got $objtype/$eventtype for $node, which is not mine\n")
		if ($dbtag eq "");
	    return;
	}
	if (!checkDBRedirect($node)) {
	    info("Got $objtype/$eventtype for $node, which is not mine\n");
	    return;
	}
    }

    #
    # If this is a debugging version, then this event is for one of my
    # nodes, so I can print out the event now. (Main version prints earlier.)
    #
    if ($dbtag ne "") {
	debug("Got an event: ($objtype,$objname,$eventtype)\n");
    }

 SWITCH: for ($objtype) {

	(/$TBNODESTATE/) && do {
	    stateTransition($objname,$eventtype);
	    last;
	};
	(/$TBNODEOPMODE/) && do {
	    opModeTransition($objname,$eventtype);
	    notify("Use of deprecated event TBNODEOPMODE:\n".
		   "$objname->$eventtype\n");
	    last;
	};
	(/$TBCONTROL/) && do {
	    handleCtrlEvent($objname,$eventtype);
	    last;
	};
	(/$TBCOMMAND/) && do {
	    handleCommand($objname,$eventtype);
	    last;
	};

    }

}

sub stateTransition($$) {

    my ($node,$newstate) = @_;

    # Check for invalid transitions
    my ($oldstate, $mode);
    $oldstate = $nodes{$node}{state};
    $mode     = $nodes{$node}{mode};

    if ($oldstate && $mode && $valid{$mode} && $valid{$mode}{$oldstate} &&
	!$valid{$mode}{$oldstate}{$newstate}) {
	notify("Invalid transition for node $node from $mode/$oldstate " .
	       "to $newstate\n");
    }

    my $now = time();
    $nodes{$node}{state}     = $newstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{notified}  = 0;

    info("$node: $mode/$oldstate => $mode/$newstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$newstate', " .
		 "state_timestamp='$now' WHERE node_id='$node'");

    # Before we set the timeout (overwriting any current ones), we need
    # to check if we had a pending command
    if (qfind($node) &&
	$timeout_tag{$node} =~ /^$TBCOMMAND:/) {
        debug("TimeoutTag = '$timeout_tag{$node}'\n");
	my ($str,$cmd) = split(":",$timeout_tag{$node});
	debug("str=$str\tcmd=$cmd\tTBREBOOT=$TBREBOOT\tstate=$newstate\n");
	if ($cmd eq $TBREBOOT) {
	    if ($newstate eq TBDB_NODESTATE_SHUTDOWN ) {
		info("$node: $TBREBOOT success\n");
		# Timeout will get cleared below by setTimeout call
	    } else {
		notify("$node: $TBREBOOT in progress, but got state ".
		       "$newstate instead of ".TBDB_NODESTATE_SHUTDOWN."!\n");
	    }
	#} elsif ($cmd eq $FOO ) {
	    # Add more here...
	} else {
	    notify("$node: Unknown command timeout '$timeout_tag{$node}' ".
		   "found at $mode/$newstate\n");
	}
    }

    #
    # Check if this state has a timeout, and if so, put it in the queue.
    # Note that any opmode transition below will replace (or remove) this
    # timeout if appropriate.
    #
    setTimeout($mode,$newstate,$node,$now);

    # Check if this state has any triggers
    my @nodetrigs = GetNodeTriggerList($node,$mode,$newstate,1);
    my @trigs = GetNodeTriggerList($node,$mode,$newstate);
    if (@trigs > 0) {
	debug("Running triggers: ".join("/",@trigs)."\n");
	foreach ( @trigs) {
	    my $trig = $_;
	    /^SCRIPT:([-\w\/]+)$/ && do {
		#
		# Run a script. No arguments at the moment.
		#
		my $script = $1;

		#
		# If the path is absolute, run it. Otherwise it has to
		# come from the sbin directory.
		#
		if (! ($script =~ /^\//)) {
		    $script = "$TB/sbin/$script";
		}
		info("$node: Running $script.\n");
		system("$script &");
		next;
	    };
	    /^$PXEBOOT$/ && do {
		#
		# See if we jumped into the PXEBOOT kernel. Bootinfo
		# will send PXEBOOTING every time a node contacts it,
		# which is our indicator that the node is in the first
		# phase of booting. At this point we want to switch
		# state machines since the entire boot process is
		# governed by a single state machine that is
		# independent of the OS that the node will eventually
		# boot.  Rather then encode that in each state
		# machine, we use a special machine with a defined
		# entrypoint (PXEBOOTING) and a defined exitpoint
		# (BOOTING). See below for where we jump back out of
		# this state machine.
		#
		# Jumped in. We need to change the opmode so that
		# the state transitions are legal. We do not
		# bother to save the old opmode since we can
		# figure it out later when we leave.
		#
		debug("Running $PXEBOOT trigger\n");
		if ($mode ne $PXEKERNEL) {
		    info("$node: Forcing mode transition into $PXEKERNEL!\n");
		    opModeTransition($node, $PXEKERNEL, 1);
		    $mode=$PXEKERNEL;
		}
		next;
	    };
	    /^$BOOTING$/ && do {
		#
		# See if we are in the right mode/osid.
		#
		my ($bootosid,$bootopmode) = TBBootWhat($node, $debug);

		info("$node: BootWhat says $bootosid (mode $bootopmode).\n");
		DBQueryFatal("update nodes set osid='$bootosid' ".
			     "where node_id='$node'");

		if ($bootopmode ne $mode) {
		    if ($mode eq $PXEKERNEL) {
			#
			# If we came from PXE boot, then we have to
			# jump out of the PXEKERNEL state machine into
			# whatever state machine is current for the
			# node. Since we came through bootinfo, we
			# know that the node is doing what it is
			# supposed to, and that this change matches
			# what the node is booting.
			#
			info("$node: Forcing mode transition out of $PXEKERNEL!\n");
			opModeTransition($node, $bootopmode, 1);
			$mode=$bootopmode;
		    }
		    elsif ($oldstate eq TBDB_NODESTATE_ISUP) {
			#
			# Skipped SHUTDOWN, which could result in a
			# missed opmode transition. Can this really
			# happen anymore?
			#
			info("$node: Came from ISUP! ".
			     "Checking for mode transition\n");
			my $query_result =
			  DBQueryWarn("select next_op_mode from nodes ".
				      "where node_id='$node'");
			my ($nextmode) = $query_result->fetchrow();
			if ($nextmode) {
			    info("$node: Forcing mode transition!\n");
			    opModeTransition($node, $nextmode, 1);
			    $mode=$nextmode;
			}
		    }
		    else {
			my $str = "$node is running $bootosid, but in ".
			  "mode $mode\ninstead of mode $bootopmode!\n";
			
			if ($bootopmode eq "RELOAD") {
			    #
			    # For now, only force if we're going into
			    # reload mode, so we don't get stuck
			    # looping in reloading.  Can this happen
			    # anymore?
			    #
			    DBQueryFatal("UPDATE nodes SET ".
					 "op_mode='$bootopmode', ".
					 "op_mode_timestamp=".
					 "unix_timestamp(now()) ".
					 "WHERE node_id='$node'");
			    $nodes{$node}{mode} = $bootopmode;
			    $nodes{$node}{mode_timestamp} = $now;
			    $str .= "Forced op_mode to $bootopmode.\n";
			}
			notify($str);
		    }
		}
		next;
	    };
	    /^$CHECKGENISUP$/ && do {
		checkGenISUP($node);
		next;
	    };
	    /^$TBRESET$/ && do {
		# We successfully booted, so clear some flags
		$nodes{$node}{noretry}   = 0;
		$nodes{$node}{timedout}  = 0;
		# Check if we really need to do a reset
		my $r = DBQueryWarn("select osid,def_boot_osid from nodes ".
				    "where node_id='$node'");
		my ($osid,$defosid) = $r->fetchrow();
		if ($osid ne $defosid) {
		    handleCtrlEvent($node,$trig);
		}
		next;
	    };
	    /^$TBRELOADDONE$/ && do {
		handleCtrlEvent($node,$trig);
		next;
	    };
	    /^$TBISUP$/ && do {
		info("$node: Triggered $TBISUP\n");
		EventSendWarn(host      => $BOSSNODE ,
			      objtype   => TBDB_TBEVENT_NODESTATE ,
			      eventtype => TBDB_NODESTATE_ISUP ,
			      objname   => $node);
		next;
	    };
	    notify("Unknown trigger '$trig' for $node in $mode/$newstate!\n");
	}
	# Clear any of the node triggers that we ran.
	# (Don't clear all of them, because some of the triggers we ran
	# may have caused others to be set, and we don't want to nuke them.)
	if (@nodetrigs > 0) {
	    debug("Clearing node triggers: ".join("/",@nodetrigs)."\n");
	    ClearNodeTrigger($node,$mode,$newstate,@nodetrigs);
	}
    }

    # Check if this state can trigger a mode transition
    if (defined($modeTrans{"$mode:$newstate"})) {
	info("$node: Checking for mode transition\n");
	my $r = DBQueryWarn("select next_op_mode from nodes ".
			    "where node_id='$node'");
	my ($nextmode) = $r->fetchrow();
	if ($nextmode) {
	    opModeTransition($node,$nextmode);
	} else {
	    debug("No next mode.\n");
	}
    }
}

sub opModeTransition($$;$) {

    my ($node,$newmode,$force) = @_;
    if (!defined($force)) { $force = 0; }

    info("$node: Mode change to $newmode requested ($force)\n");

    # Check for invalid transitions
    my ($oldstate, $mode, $nextstate);
    $oldstate = $nodes{$node}{state};
    $mode     = $nodes{$node}{mode};

    if (defined($modeTrans{"$mode:$oldstate"}) || $force) {
	if (!$force) {
	    debug("Mode Transition check:\n");
	    my $translist = join(",",@{$modeTrans{"$mode:$oldstate"}});
	    #debug("translist=$translist\n");
	    #debug("splitlist=".join(", ",split(/[:,]/,$translist))."\n");
	    my %trans = split(/[:,]/,$translist);
	    debug("Valid transitions from $mode/$oldstate are:\n");
	    foreach my $k (sort keys %trans) {
		debug("$k => $trans{$k}\n");
	    }
	    if (defined($trans{$newmode})) {
		$nextstate=$trans{$newmode};
	    } else {
		notify("Invalid mode transition for $node from ".
		       "$mode/$oldstate to $newmode!\n");
	    }
	}
    } else {
	notify("Invalid mode transition for $node from $mode/$oldstate: ".
	       "Not a valid mode transition state!\n");
    }
    if (!$nextstate) {
	$nextstate=$oldstate;
    }

    my $now = time();
    $nodes{$node}{state}     = $nextstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{mode}           = $newmode;
    $nodes{$node}{mode_timestamp} = $now;
    $nodes{$node}{notified}       = 0;

    info("$node: $mode/$oldstate => $newmode/$nextstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$nextstate', ".
		 "next_op_mode='', op_mode='$newmode', ".
		 "state_timestamp='$now', ".
		 "op_mode_timestamp='$now' WHERE node_id='$node'");

    # Check if this state has a timeout, and if so, put it in the queue
    setTimeout($newmode,$nextstate,$node,$now);

}

sub handleCtrlEvent($$) {
    my ($node,$event) = @_;

    info("CtrlEvent: $node, $event\n");

    foreach ($event) {
	/^$TBRESET$/ && do {
	    #
	    # Clear next_boot_path with os_select.
	    #
	    $cmd = "$osselect -d -c -1 $node";
	    system($cmd) and
		notify("$node/$event: Could not clear next_boot_path!\n");

	    info("Performed $TBRESET for $node\n");
	    next;
	};
	/^$TBRELOADDONE$/ && do {
	    info("Clearing reload info for $node\n");
	    DBQueryFatal("delete from current_reloads where node_id='$node'");
	    my ($pid,$eid);
	    NodeidToExp($node,\$pid,\$eid);
	    if (($pid eq NODERELOADING_PID) && ($eid eq NODERELOADING_EID)) {
		DBQueryFatal("delete from scheduled_reloads ".
			     "where node_id='$node'");
		DBQueryFatal("delete from reserved where node_id='$node'");
		info("Released $node from $pid/$eid\n");
	    }
	    next;
	};
	/^$TBTIMEOUT$/ && do {
	    my ($mode,$state) = split(":",$timeout_tag{$node});
	    delete($timeout_tag{$node});
	    my $curstate = $nodes{$node}{state};
	    my $curmode = $nodes{$node}{mode};
	    my ($timeout,$action);
	    if (!defined($nodes{$node}{notified})) {
		$nodes{$node}{notified}=0;
	    }
	    $nodes{$node}{notified}++;
	    my $notified = $nodes{$node}{notified};
	    $nodes{$node}{timedout}++;
	    my $timedout = $nodes{$node}{timedout};
	    if ($mode && $state && $timeouts{$mode} &&
		$timeouts{$mode}{$state}) {
		($timeout, $action) = @{$timeouts{$mode}{$state}};
	    }
	    if ($mode eq $TBCOMMAND) {
		# It is a command, not a true state
		if ($action eq $TBTIMEOUTCMDRETRY) {
		    # Retry the command
		    notify("$node: Command $state, retry #$timedout\n");
		    # notify in case we get in a retry loop...
		    handleCommand($node,$state,$timedout,1);
		} else {
		    notify("$node: Unknown timeout action for ".
			   "$mode/$state: '$action'\n");
		}
		next;
	    }

	    #
	    # Trash. This stuff should not be encoded this way, but I have
	    # no idea how timeouts, TBCOMMAND, and actions interact.
	    #
	    if ($curstate eq $PXEWAKEUP) {
		my $optarg = ($debug ? "-d " : "");

		if ($timedout < 3) {
		    #
		    # Try again.
		    #
		    info("Node $node has timed out $timedout times in ".
			 "$PXEWAKEUP!\n".
			 "Sending it a another wakeup command\n");
		}
		else {
		    #
		    # Failed too many times, power cycle instead.
		    #
		    notify("Node $node has timed out $timedout times in ".
			   "$PXEWAKEUP!\n".
			   "Sending it a reboot command\n");
		    $optarg .= "-k";
		}
		my $cmd = "$nodereboot -r $optarg $node";
		debug("$cmd\n");
		system("date 2>&1 >> $rebootlog");
		system("$cmd 2>&1 >> $rebootlog &") and
		    notify("PXEWAKEUP retry: ".
			   "Command '$cmd' failed, error $?: $!\n");

		next;
	    }

	    info("Node $node has timed out in state $mode/$state".
		 ($action ne "" ? "\n\tRequested action $action." : "").
		 "\n");