stated.in

#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# All rights reserved.
#

#
# stated - A daemon to monitor the states of nodes in the testbed. Recives
# state change notification through the event system, and writes the new
# state into the database. Also watches for invalid transitions, timeouts, 
# and performs other state-related control functions.
#
# Send it a HUP signal to get it to reload the timeout and transition
# information. Periodically reloads this information regardless, though.
#
# Will restart when sent SIGUSR1, by exec'ing its executable again.
#

# Configure variables
use lib '@prefix@/lib';
my $TB = "@prefix@";
my $BOSSNODE = "@BOSSNODE@";
my $TBOPS = '@TBSTATEDEMAIL@';
my $REALTBOPS = "@TBOPSEMAIL@";
my $TBDBNAME = "@TBDBNAME@";
my $osselect = "$TB/bin/os_select";

$| = 1;

use event;
use libdb;
use libtestbed;
use Getopt::Std;
#use strict;
use English;
use POSIX; # for strftime, and sigprocmask and friends
use Fcntl; # file constants for pidfile
use Sys::Syslog;
# Important note about syslog: It defaults to using an inet socket,
# but 'syslogd -s' (the default) doesn't listen for one. So either
# run syslogd without -s, or use setlogsock('unix') before openlog.
# (To get setlocksock: 'use Sys::Syslog qw(:DEFAULT setlogsock);' )

# Do lots of db retries before we fail and die
$libdb::DBQUERY_MAXTRIES = 5;

# Number of iterations (roughly, seconds) after which we'll reload 
# information from the database. This is so we don't end up with information
# that's _too_ out of sync.
my $reload_time = 600;
my $last_reload = 0;

# Process command-line arguments

sub usage {
	print << "END";
Usage: $0 [-h] [-d] [-s server] [-p port]
-h              This message
-d              Turn on debugging output, and don't go into the background
-s server       Use specified server, instead of this site's bossnode
-p port	        Use specified port
Send SIGHUP to reload database state, or SIGUSR1 to restart completely.
END
}

my @args = @ARGV; # save a copy for restart before we mess with them.
my %opt = ();
getopts("ds:p:h",\%opt);

if ($opt{h}) { exit &usage; }
if (@ARGV) { exit &usage; }

my ($server,$port,$debug);
if ($opt{s}) { $server = $opt{s}; } else { $server = $BOSSNODE; }
if ($opt{p}) { $port = $opt{p}; }
if ($opt{d}) { $debug = 1; } else { $debug = 0; }

# Grab some constants into variables
my $TBRESET      = TBDB_TBCONTROL_RESET;
my $TBRELOADDONE = TBDB_TBCONTROL_RELOADDONE;
my $TBTIMEOUT    = TBDB_TBCONTROL_TIMEOUT;
my $TBNOTIMEOUT  = TBDB_NO_STATE_TIMEOUT;
my $TBNODESTATE  = TBDB_TBEVENT_NODESTATE;
my $TBNODEOPMODE = TBDB_TBEVENT_NODEOPMODE;
my $TBCONTROL    = TBDB_TBEVENT_TBCONTROL;
my $TB_OSID_MBKERNEL = TB_OSID_MBKERNEL;

my $pidfile;
$pidfile = "$TB/locks/stated.pid";
debug("Using pidfile $pidfile\n");
if (-e $pidfile) {
  my $otherpid = `cat $pidfile`;
  my $running = `ps -auxww | grep $otherpid | grep -v grep`;
  if ($running ne "") {
    fatal("Lockfile $pidfile exists, and process $otherpid appears to be ".
	  "running.\n");
  } else {
    notify("Lockfile exists, but process $otherpid appears to be dead.\n".
	   "Removing lock file...\n");
  }
  system("rm $pidfile") &&
    fatal("Couldn't remove $pidfile: $? $!\n");
}
# Background
if (!$debug) {
  # We use syslog, so redirect the output to nothing
  if (TBBackGround("/dev/null")) { exit(0); }
}
# set up syslog
openlog("stated","pid","user");
sysopen(PIDFILE, $pidfile, O_WRONLY | O_EXCL | O_CREAT) ||
  fatal("Couldn't create '$pidfile': $? $!\n");
print PIDFILE "$$";
close PIDFILE;
# If I make it to here, I'll need to clean up the lock file
my $lockfile=$pidfile;

# Change my $0 so that it is easier to see in a ps/top
$0 = "$0";

my $URL = "elvin://$server";
if ($port) { $URL .= ":$port"; }

# Connect to the event system, and subscribe the the events we want 
my $handle = event_register($URL,0);
if (!$handle) { fatal("Unable to register with event system\n"); }

my $tuple = address_tuple_alloc();
if (!$tuple) { fatal("Could not allocate an address tuple\n"); }

%$tuple = ( objtype => join(",",$TBNODESTATE,$TBNODEOPMODE,$TBCONTROL) );

if (!event_subscribe($handle,\&handleEvent,$tuple)) {
	fatal("Could not subscribe to events\n");
}

# Read in the pre-existing node states, and timeout and valid transition
# information from the database
my %nodes     = readStates();
my %timeouts  = getTimeouts();
my %valid     = getValid();
my %modeTrans = getModeTrans();
my %triggers  = getTriggers();

# Gets set if a reload of state from the database should happen.
my $do_reload = 0;

# Set when I've got a child pinging a node for me.
my $have_children = 0;

# Keep track of which pid was pinging which node.
my %children= (); # $children($child} = "pcXXX";

# Make the daemon reload database state on a sighup - but I'm worried
# about what would happen if we tried to do this mid-loop. So, we'll
# just set a flag and do it when we're done with our current pass.
$SIG{HUP}  = sub { info("SIGHUP - Reloading DB state\n"); $do_reload = 1; };

# Set up other signals.
$SIG{USR1} = \&restart;
$SIG{USR2} = \&cleanup;
$SIG{INT}  = \&cleanup;
$SIG{QUIT} = \&cleanup;
$SIG{ABRT} = \&cleanup;
$SIG{TERM} = \&cleanup;
$SIG{KILL} = \&cleanup;

# Set up some notification throttling
my $mailgap = 15; # in seconds
my $lastmail = time - $mailgap + 2; # Send a digest of startup msgs after 2s.
my %msgs = ();

# Track if I handled an event or not
my $event_count = 0;

notify("Stated starting up\n");

sub process_event_queue() {
    $event_count=0;
    my $lastcount=-1;
    while ($event_count != $lastcount) {
	$lastcount = $event_count;
	event_poll($handle);
    }
    if ($event_count > 0) {
      debug("Handled $event_count event(s).\n");
    }
}

# Now, we just poll for events, and watch for timeouts
while (1) {
	process_event_queue;
	my $now = time();
	#
	# Look for nodes that have passed their timeout
	#
	while (my ($node, $value) = each %nodes) {
	    my $state    = $value->{state};
	    my $mode     = $value->{mode};
	    my $time     = $value->{timestamp};
	    my $notified = $value->{notified};
	    my ($timeout,$action);
	    if ($mode && $state && $timeouts{$mode} &&
		$timeouts{$mode}{$state}) {
		($timeout, $action) = @{$timeouts{$mode}{$state}};
	    }
	    if ((!$notified) && $time && $timeout && 
		$timeout!= $TBNOTIMEOUT && (($time + $timeout) < $now)) {
		handleCtrlEvent($node,$TBTIMEOUT);
		$value->{notified} = 1;
		
	    }
	}

	if ($do_reload || ($now - $last_reload > $reload_time)) {
		reload();
		$do_reload = 0;
	}

	if ($have_children) {
	  # Check for kids that have finished
	  handleChild();
	}

	# Send any messages in the queue if it is time
	notify("",1);

	sleep(1);
}

exit(0);

# Read the current states of nodes from the database
sub readStates(;@) {
    	my %oldnodes = @_;

	#
	# Guard against undefined variable warnings
	#
	if (! defined(%oldnodes)) {
		%oldnodes = ();
	}

	#debug("readStates called\n");
	my $result = DBQueryFatal("SELECT node_id, eventstate, " .
				  "state_timestamp, op_mode, " .
				  "op_mode_timestamp FROM nodes");

	my %nodes;
	while (my ($node_id, $state, $timestamp, $mode, $mode_timestamp)
			= $result->fetchrow()) {
	    	#
		# If there's an entry in oldnodes for this node, and it
		# hasn't changed state or time, use the old entry (so that
		# we don't lose information about which nodes we've already
		# notified the ops about, etc.)
		#
		if ($oldnodes{$node_id} && $state && $timestamp &&
		   ($oldnodes{$node_id}{state} eq $state) &&
		   ($oldnodes{$node_id}{mode} eq $mode) &&
		   ($oldnodes{$node_id}{timestamp} == $timestamp)) {
		       $nodes{$node_id} = $oldnodes{$node_id};
		} else { 
		    $nodes{$node_id}{state}          = $state;
		    $nodes{$node_id}{timestamp}      = $timestamp;
		    $nodes{$node_id}{mode}           = $mode;
		    $nodes{$node_id}{mode_timestamp} = $mode_timestamp;
		}
	}
	return %nodes;
}

#
# Read timeouts for various states from the database
#
sub getTimeouts() {
	#debug("getTimeouts called\n");
	my $result = DBQueryFatal("SELECT op_mode, state, timeout, action " .
				  "FROM state_timeouts");

	my %timeouts;
	while (my ($op_mode, $state, $timeout, $action) = $result->fetchrow()) {
		$timeouts{$op_mode}{$state} = [ $timeout, $action ];
	}
	return %timeouts;
}

#
# Read the list of valid state transitions from the database
#
sub getValid() {
	#debug("getValid called\n");
	my $result = DBQueryFatal("SELECT op_mode, state1, state2 " .
				  "FROM state_transitions");

	my %valid;
	while (my ($mode,$state1, $state2) = $result->fetchrow()) {
		$valid{$mode}{$state1}{$state2} = 1;
	}
	return %valid;
}

#
# Read the list of valid mode transitions from the database
#
sub getModeTrans() {
	#debug("getModeTrans called\n");
	my $result = 
	    DBQueryFatal("SELECT op_mode1, state1, op_mode2, state2 " .
			 "FROM mode_transitions order by op_mode1,state1");

	my %modeTrans;
	while (my ($mode1,$state1, $mode2, $state2) = $result->fetchrow()) {
	    if (!defined($modeTrans{"$mode1:$state1"})) {
		$modeTrans{"$mode1:$state1"}= ["$mode2:$state2"];
	    } else {
		my @l = @{$modeTrans{"$mode1:$state1"}};
		push(@l, "$mode2:$state2");
		$modeTrans{"$mode1:$state1"}= \@l;
	    }
	}
	return %modeTrans;
}

#
# Read the list of states which trigger an action
#
sub getTriggers() {
	#debug("getTriggers called\n");
	my $result = 
	    DBQueryFatal("SELECT op_mode, state, trigger " .
			 "FROM state_triggers order by op_mode,state");
	my %t;
	while (my ($mode,$state, $trig) = $result->fetchrow()) {
	    $t{"$mode:$state"} = $trig;
	}
	return %t;
}

#
# Gets called for every event that we recieve
#
sub handleEvent($$$) {
	my ($handle,$notification,$data) = @_;
	my $objtype = event_notification_get_objtype($handle,$notification);
	my $objname = event_notification_get_objname($handle,$notification);
	my $eventtype = event_notification_get_eventtype($handle,$notification);

	$event_count++;
	info("Got an event: ($objtype,$objname,$eventtype)\n");

	#
	# Check to see if another instance is supposed to be handling this node
	#
	if (!checkDBRedirect($objname)) {
	    info("Got an event for node $objname, which isn't mine\n");
	    return;
	}

	SWITCH: for ($objtype) {

	    (/$TBNODESTATE/) && do {
		stateTransition($objname,$eventtype);
		last;
	    };
	    (/$TBNODEOPMODE/) && do {
		opModeTransition($objname,$eventtype);
		notify("Use of deprecated event TBNODEOPMODE:\n".
		       "$objname->$eventtype\n");
		last;
	    };
	    (/$TBCONTROL/) && do {
		handleCtrlEvent($objname,$eventtype);
		last;
	    };

	}

}

sub stateTransition($$) {

    my ($node,$newstate) = @_;

    # Check for invalid transitions
    my ($oldstate, $mode);
    if ($nodes{$node}) {
	$oldstate = $nodes{$node}{state};
	$mode = $nodes{$node}{mode};
    } else {
	# Try reloading the cache once before we give up on this node
	reload();
	if ($nodes{$node}) {
	    $oldstate = $nodes{$node}{state};
	    $mode = $nodes{$node}{mode};
	} else {
	    notify("Got an event for a node ($node) I don't know about\n");
	}
    }
    if ($oldstate && $mode && $valid{$mode} && $valid{$mode}{$oldstate} &&
	!$valid{$mode}{$oldstate}{$newstate}) {
	notify("Invalid transition for node $node from $mode/$oldstate " .
	       "to $newstate\n");
    }

    my $now = time();
    $nodes{$node}{state}     = $newstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{notified}  = 0;

    info("$node: $mode/$oldstate => $mode/$newstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$newstate', " .
		 "state_timestamp='$now' WHERE node_id='$node'");

    # Check if this is TBDB_NODESTATE_BOOTING , which has actions
    if ($newstate eq TBDB_NODESTATE_BOOTING) {
	my $osid = TBBootWhat($node,$debug);
	my $os_op_mode = os_opmode($osid);
	info("$node: Current OS is '$osid', OS mode is '$os_op_mode'\n");
	DBQueryFatal("UPDATE nodes SET osid='$osid' WHERE node_id='$node'");
	if ($os_op_mode ne $mode) {
	    my $str = "Node $node is running OS '$osid' but in mode '$mode' ".
	      "instead of mode '$os_op_mode'!\n";
	    if ($os_op_mode ne "") {
		DBQueryFatal("UPDATE nodes SET op_mode='$os_op_mode', ".
			     "op_mode_timestamp=unix_timestamp(now()) ".
			     "WHERE node_id='$node'");
		$nodes{$node}{mode} = $os_op_mode;
		$nodes{$node}{mode_timestamp} = $now;
		$str .= "Forced op_mode to '$os_op_mode'.\n";
	    }
	    notify($str);
	}
	checkGenISUP($node);
    }

    # Check if this state is the "successful boot" state
    if (defined($triggers{"$mode:$newstate"})) {
	my @trigs = split(/\s*,\s*/,$triggers{"$mode:$newstate"});
	foreach ( @trigs ) {
	    my $trig = $_;
	    /^$TBRESET$/ && do {
		# Check if we really need to do a reset
		my $r = DBQueryWarn("select osid,def_boot_osid from nodes ".
				    "where node_id='$node'");
		my ($osid,$defosid) = $r->fetchrow();
		if ($osid ne $defosid) {
		    handleCtrlEvent($node,$trig);
		}
		next;
	    };
	    /^$TBRELOADDONE$/ && do {
		handleCtrlEvent($node,$trig);
		next;
	    };
	    notify("Unknown trigger '$trig' for $node in $mode/$newstate!\n");
	}
    }

    # Check if this state can trigger a mode transition
    if (defined($modeTrans{"$mode:$newstate"})) {
	info("$node: Checking for mode transition\n");
	my $r = DBQueryWarn("select next_op_mode from nodes ".
			    "where node_id='$node'");
	my ($nextmode) = $r->fetchrow();
	if ($nextmode) {
	    opModeTransition($node,$nextmode);
	} else { debug("No next mode.\n"); }
    }
}

sub opModeTransition($$) {

    my ($node,$newmode) = @_;

    info("$node: Mode change to $newmode requested\n");
    # Check for invalid transitions
    my ($oldstate, $mode, $nextstate);
    if ($nodes{$node}) {
	$oldstate = $nodes{$node}{state};
	$mode = $nodes{$node}{mode};
    } else {
	# Try reloading the cache once before we give up on this node
	reload();
	if ($nodes{$node}) {
	    $oldstate = $nodes{$node}{state};
	    $mode = $nodes{$node}{mode};
	} else {
	    notify("Got an event for a node ($node) I don't know about\n");
	}
    }
    if (defined($modeTrans{"$mode:$oldstate"})) {
	debug("Mode Transition check:\n");
	my $translist = join(",",@{$modeTrans{"$mode:$oldstate"}});
	#debug("translist=$translist\n");
	#debug("splitlist=".join(", ",split(/[:,]/,$translist))."\n");
	my %trans = split(/[:,]/,$translist);
	debug("Valid transitions from $mode/$oldstate are:\n");
	foreach my $k (sort keys %trans) { debug("$k => $trans{$k}\n"); }
	if (defined($trans{$newmode})) {
	    $nextstate=$trans{$newmode};
	} else {
	    notify("Invalid mode transition for $node from ".
		   "$mode/$oldstate to $newmode!\n");
	}
    } else {
	notify("Invalid mode transition for $node from $mode/$oldstate: ".
	       "Not a valid mode transition state!\n");
    }
    if (!$nextstate) { $nextstate=$oldstate; }

    my $now = time();
    $nodes{$node}{state}     = $nextstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{mode}           = $newmode;
    $nodes{$node}{mode_timestamp} = $now;
    $nodes{$node}{notified}       = 0;

    info("$node: $mode/$oldstate => $newmode/$nextstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$nextstate', ".
		 "next_op_mode='', op_mode='$newmode', ".
		 "state_timestamp='$now', ".
		 "op_mode_timestamp='$now' WHERE node_id='$node'");
}

sub handleCtrlEvent($$) {
    my ($node,$event) = @_;

    info("CtrlEvent: $node, $event\n");

    foreach ($event) {
	/^$TBRESET$/ && do {
	    my $result = DBQueryFatal("SELECT pxe_boot_path, def_boot_osid ".
				      "FROM nodes where node_id='$node'");
	    my ($pxepath,$osid) = $result->fetchrow();

	    # Important note on ordering here:
	    # Because setting a normal osid resets pxe path to PXEBOOT,
	    # We need to read it out first, then set the osid, then set
	    # the pxepath back to its original value at the end.

	    $cmd = "$osselect $osid $node";
	    system($cmd) and
		notify("$node/$event: Couldn't clear next_boot_*\n".
		       "\tcmd=$cmd\n\t*** $!\n");

	    $pxepath = "-p ".$pxepath;
	    if ($pxepath eq "-p ") { $pxepath="PXEBOOT"; };
	    my $cmd = "$osselect -m $pxepath $node";
	    system($cmd) and
		notify("$node/$event: Couldn't clear next_pxe_boot_path\n".
		       "\tcmd=$cmd\n\t*** $!\n");

	    info("Performed RESET for $node to $osid/$pxepath\n");
	    next;
	};
	/^$TBRELOADDONE$/ && do {
	    info("Clearing reload info for $node\n");
	    DBQueryFatal("delete from current_reloads where node_id='$node'");
	    my ($pid,$eid);
	    NodeidToExp($node,\$pid,\$eid);
	    if (($pid eq NODERELOADING_PID) && ($eid eq NODERELOADING_EID)) {
		DBQueryFatal("delete from scheduled_reloads ".
			     "where node_id='$node'");
		DBQueryFatal("delete from reserved where node_id='$node'");
		info("Released $node from $pid/$eid\n");
	    }
	    next;
	};
	/^$TBTIMEOUT$/ && do {
	    my $state = $nodes{$node}{state};
	    my $mode = $nodes{$node}{mode};
	    my ($timeout,$action);
	    if ($mode && $state && $timeouts{$mode} &&
		$timeouts{$mode}{$state}) {
		($timeout, $action) = @{$timeouts{$mode}{$state}};
	    }
	    notify("Node $node has timed out in state $mode/$state".
		   ($action ne "" ? "\n\tRequested action $action." : "").
		   "\n");
	    next;
	};
	notify("$node: Unknown CtrlEvent: $event\n");
    }
}

#
# Check if we need to generate an ISUP
#
sub checkGenISUP($) {
  my ($node) = @_;
  info("$node: Checking ISUP Generation\n");
  my $r = DBQueryWarn("select osfeatures from os_info as o ".
		      "left join nodes as n on o.osid=n.osid ".
		      "where node_id='$node' and osfeatures is not null");
  my $osfeatures="";
  # If we don't get anything back, assume it has no features.
  if ($r->num_rows() > 0) {
    ($osfeatures) = $r->fetchrow();
  }

  my @features = split(",",$osfeatures);
  # Make sure features I care about are defined
  my %can=("ping"=>0, "isup"=>0);
  foreach my $f (@features) {
    $can{"\L$f"}=1;		# make sure it's all lowercase
  }

  # If os will send ISUP on its own, do nothing here.
  if ($can{"isup"}) {
    debug("$node: Will send own ISUP\n"); return 0;
  }

  # If os doesn't support isup but can ping, fork and ping it every
  # few seconds and send isup when it pings, or timeout after too long.
  if ($can{"ping"}) {
    debug("$node: Needs to be pinged\n");
    my $pid = fork();
    if ($pid) {
      $children{$pid}= $node;
      $have_children = 1;
      # don't wait, return and go on with life
      return 0;
    } else {
      info("Forked process $$ to ping $node\n");
      $lockfile = ""; # Don't clean up my pidfile, since I'm a child

      my $wait=5;		# 5 seconds between ping attempts
      my $maxtime=600;		# Set the timer for 10 minutes

      # XXX : If our maxtime is very different from os_setup's
      #idea of how long a node should take to reboot, then we've
      #got a problem, since it might reboot the nodes.

      # Set an alarm in case it never comes up...
      local $SIG{ALRM} = sub { 
	notify("$node: checkGenISUP timed out waiting ".
	       "for ping responses after $maxtime seconds.\n");
	exit(4);
      };

      alarm $maxtime;

      my $n=0;
      while ( $n <= ($maxtime/$wait)) {
	my $status=system("/sbin/ping -c 1 -t 1 $node > /dev/null");
	my $rv = $status >> 8;
	my $sig = $status & 127;
	debug("checkGenISUP: ping returned $status ($rv / $sig)\n");
	if ($rv==0) {
	  # we got a response
	  info("$node: ping response received... sending ISUP\n");
	  # We want to send an ISUP event for the node. But we
	  # can't just call stateTransition, because that will
	  # only make the change happen in our state, not the real
	  # stated that we forked from. We do this by exit(0)
	  # and the handleChild function will pick it up and
	  # send the ISUP for us.
	  exit(0);
	} elsif ($rv==2) {
	  # no response
	  debug("$node: No ping response, waiting $wait seconds...\n");
	} elsif ($rv==68) {
	  # ping: cannot resolve $node: Unknown host
	  notify("$node: ping couldn't resolve $node!\n");
	  exit(2);
	} else {
	  notify("$node: checkGenISUP ping returned $rv!\n");
	  exit(3);
	}
	sleep $wait;
	$n+=1;
      }
      notify("$node: Sent $n pings in ".($n*$wait)." seconds ".
	     "with no response.\n");
      exit(1);
    }
  }
  # If os doesn't support ping or isup, stated sets it to ISUP at the
  # same time.

  debug("$node: OS doesn't ping - sending ISUP\n");
  stateTransition($node, TBDB_NODESTATE_ISUP);
}

# Figure out if this node belongs to us (ie. if it's using our database.)
# Returns 1 if it does, 0 if not
sub checkDBRedirect($) {

	my ($node) = @_;

	#
	# XXX: I don't want to do this every time, for performance reaons,
	# but we need to make sure that we don't get into an inconsistent 
	# state
	#
	my $result = DBQueryFatal("SELECT testdb FROM nodes as n " .
		"LEFT JOIN reserved as r ON n.node_id = r.node_id " .
		"LEFT JOIN experiments as e ON r.pid = e.pid " .
			"AND r.eid = e.eid " .
		"WHERE n.node_id = '$node'");

	if (!$result->num_rows()) {
		notify("Got an event for a node ($node) I don't know about\n");
		return 0;
	}

	my ($testdb) = $result->fetchrow();

	#
	# XXX: It's hokey to hardcode tbdb here, but....
	#

	#debug("checkDBRedirect: $node => $testdb (I'm $TBDBNAME)\n");
	if ((!$testdb && ($TBDBNAME eq "tbdb")) ||
	    ($testdb && ($testdb eq $TBDBNAME))) {
	  return 1;
	} else {
	  return 0;
	}
}

# Reload state from the database
sub reload() {
  debug("Reloading state from database\n");
  $last_reload = time();
  %nodes     = readStates(%nodes);
  %timeouts  = getTimeouts();
  %valid     = getValid();
  %modeTrans = getModeTrans();
  %triggers  = getTriggers();
}

# This gets called to check for forked pids that have finished.  Right
# now we only fork in checkGenISUP to ping nodes.
sub handleChild() {
  my $child = wait;
  if ($child==-1) { return 0; }
  my $node = $children{$child};
  if (!defined($node)) { $node=""; }
  my $stat = $?;
  my $rv = $stat >> 8;
  my $sig = $stat & 127;
  debug("Child = $child, I am $$, children are: (".
	join(",",sort keys %children).")\n");
  if (($node ne "") && ($child!=-1) && ($child!=$$)) {
    info("handleChild: pid $child (node $node), exited $rv (sig $sig)\n");
    delete $children{$child};
    if (($sig == 0) && ($rv == 0)) {
      # Node is pingable, send isup
      info("handleChild: Sending ISUP for $node)\n");
      stateTransition($node, TBDB_NODESTATE_ISUP);
    } else {
      notify("handleChild: Caught a child that failed!\n".
	     "pid $child (node $node), exited $rv (sig $sig)\n");
    }
  }
  if (0+%children == 0) {
    debug("No more children now.\n");
    $have_children=0;
  }
  return 0;
}

sub os_opmode() {
    my $osid = shift || "";
    if ($osid eq $TB_OSID_MBKERNEL) { return "MINIMAL"; }
    my $cmd = "select op_mode from os_info where osid='$osid';";
    my $q = DBQueryFatal($cmd);
    if ($q->numrows() < 1) { return ""; }
    my @r = $q->fetchrow_array();
    my $opmode=$r[0];
    debug("OpMode for '$osid' is '$opmode'\n");
    if (defined($opmode) && $opmode ne "") { return $opmode; }
    return "";
}

sub debug(@) {
  if ($debug) { print @_; }
}

sub fatal($) {
  my $msg = shift;
  notify($msg);
  die($msg);
}

sub showqueue() {
  if ($debug < 2) { return; }
  if ((keys %msgs) > 0) {
    debug("\nMAILQUEUE:\n");
  }
  foreach $k (sort keys %msgs) {
    my @l = @{$msgs{$k}};
    debug("MSGS:\n$k==> (".(@l+0).",'".join("','",@l)."')\n");
  }
}

sub notify($;$) {
  my $message = shift;
  my $checkonly = shift || 0;
  # Use a timestamp, now that we're throttling mail
  my $tstamp=strftime("%b %e %H:%M:%S",localtime);
  showqueue();
  if (!$checkonly) {
    info($message);
    # Queue up the message
    # (The queue is a hash of lists of timestamps, keyed by message
    if (defined($msgs{$message})) {
      push(@{$msgs{$message}},$tstamp);
    } else {
      $msgs{$message} = [$tstamp];
    }
    showqueue();
  }
  my $now = time;
  if ($now - $lastmail >= $mailgap) {
    if ((keys %msgs)>0) {
      debug("SENDING MAILQUEUE\n"."(now $now, lastmail $lastmail, ".
	    ($now-$lastmail).">=$mailgap)\n");
      my $mailbody="";
      my $sep = '-'x5;
      # We're okay to send. Make a digest of all the queued messages.
      foreach my $msg (sort keys %msgs) {
	my @tlist = @{$msgs{$msg}};
	my $count = 0+@tlist;
	$mailbody .= "\n$msg\n";
	if ($count > 1) {
	  my $first = shift @tlist;
	  my $last = pop @tlist;
	  $mailbody .= "($count copies from $first to $last)\n";
	} else {
	  $mailbody .= "($count copy at $tlist[0])\n";
	}
	$mailbody .= "$sep\n";
      }
      # Now reset the mail queue
      %msgs = ();
      showqueue();
      $lastmail = time;
      if (!$debug) {
	SENDMAIL("Stated List <".$TBOPS.">",
		 "Stated Messsage",$mailbody,
		 "Stated Daemon <".$TBOPS.">");
      } else {
	debug("notify: Not sending mail in debug mode\n");
	debug("MAIL CONTAINS:\n".$mailbody."\n");
      }
    }
  }  # else do nothing, not time yet
}

sub announce($) {
  my $message = shift;
  my $tstamp=strftime("%b %e %H:%M:%S",localtime);
  notify("ANNOUCEMENT: ".$message."\n\n(Sent to $REALTBOPS)\n");
  $mailbody = "\n$message\n\n$tstamp\n";
  if (!$debug) {
    SENDMAIL($REALTBOPS,
	     "Stated Messsage",$mailbody,
	     "Stated Daemon <".$TBOPS.">");
  } else {
    debug("announce: Not sending mail in debug mode\n");
    debug("MAIL CONTAINS:\n".$mailbody."\n");
  }
}

sub info($;$) {
    my $message = shift;
    my $notice = shift || 0;
    # Use syslog
    my $prio="info";
    if ($notice) { $prio = "notice"; }
    if ($debug) {
      # Print out log entries like this:
      # Sep 20 09:36:00 stated[238]: Reloading state from database
      print strftime("%b %e %H:%M:%S",localtime)." stated[$$]: $message";
      $message = "DEBUG: ".$message;
    }
    syslog($prio,$message) || notify("syslog failed: $? $!\n");
}

# This gets called if we catch a signal USR1
sub restart {
  info("SIGUSER1 received: Performing final event poll before restarting\n");
  process_event_queue;
  my $params = join(" ",@args);
  my $prog = "";
  # If we're started from an abosolute path, use that.
  if ($0 =~ /^\//) { $prog = $0; } else { $prog = "$TB/sbin/stated"; }
  info("Restarting from '$prog".($params ne "" ? " $params" : "")."'\n");
  if ($handle && event_unregister($handle) == 0) {
    warn "Unable to unregister with event system\n";
  }
  if (defined($lockfile) && $lockfile ne "") { unlink $lockfile; }
  if (!defined(sigprocmask(SIG_UNBLOCK, POSIX::SigSet->new(SIGUSR1,SIGHUP)))) {
    notify("sigprocmask: sig unblock failed! $?, $!\n");
    die("\n");
  }
  announce("Stated restarted\n");
  exec("$prog $params") or 
    do {
      my $msg = "Couldn't restart stated! cmd='$prog $params'\n".
	"Error: ($?) $!\n";
      announce($msg);
      die($msg);
    };
}

# This gets called if we catch a signal (TERM, etc.)
sub cleanup {
  notify("Signal received, exiting\n");
  # now do the normal exit stuff in END {}
  exit(0);
}

# This gets called if we die of 'natural causes' (exit, die, etc.)
END {
  if (defined($lockfile) && $lockfile ne "") {
    unlink $lockfile;
    announce("Stated exiting, cleaning up\n");
  } else {
    # Must be a child
    info("Stated child exiting\n");
  }
  # clean up Syslog
  closelog();
  if ($handle) {
    if (event_unregister($handle) == 0) {
      die "Unable to unregister with event system\n";
    }
  }
}