stated.in 39.6 KB
Newer Older
Robert Ricci's avatar
Robert Ricci committed
1
#!/usr/bin/perl -w
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
5 6 7
# All rights reserved.
#

Robert Ricci's avatar
Robert Ricci committed
8 9 10
#
# stated - A daemon to monitor the states of nodes in the testbed. Recives
# state change notification through the event system, and writes the new
11 12
# state into the database. Also watches for invalid transitions, timeouts, 
# and performs other state-related control functions.
Robert Ricci's avatar
Robert Ricci committed
13 14 15 16
#
# Send it a HUP signal to get it to reload the timeout and transition
# information. Periodically reloads this information regardless, though.
#
17
# Will restart when sent SIGUSR1, by exec'ing its executable again.
Robert Ricci's avatar
Robert Ricci committed
18
#
19

Robert Ricci's avatar
Robert Ricci committed
20 21
# Configure variables
use lib '@prefix@/lib';
22
my $TB = "@prefix@";
23
my $REALTB = "/usr/testbed"; # So we know if we're the "real" stated or not
Robert Ricci's avatar
Robert Ricci committed
24
my $BOSSNODE = "@BOSSNODE@";
25
my $TBOPS = "@TBSTATEDEMAIL@";
26
my $REALTBOPS = "@TBOPSEMAIL@";
27
my $TBDBNAME = "@TBDBNAME@";
28
my $REALTBDBNAME = "tbdb"; # So we know if we're using the "real" db
29
my $osselect = "$TB/bin/os_select";
30 31
my $nodereboot = "$TB/bin/node_reboot";
my $power = "$TB/bin/power";
Robert Ricci's avatar
Robert Ricci committed
32 33 34 35 36 37

$| = 1;

use event;
use libdb;
use libtestbed;
38
use TimeoutQueue;
Robert Ricci's avatar
Robert Ricci committed
39
use Getopt::Std;
40
#use strict;
Robert Ricci's avatar
Robert Ricci committed
41
use English;
Mac Newbold's avatar
Mac Newbold committed
42 43
use POSIX;			# for strftime, and sigprocmask and friends
use Fcntl;			# file constants for pidfile
Mac Newbold's avatar
Mac Newbold committed
44 45 46 47 48
use Sys::Syslog;
# Important note about syslog: It defaults to using an inet socket,
# but 'syslogd -s' (the default) doesn't listen for one. So either
# run syslogd without -s, or use setlogsock('unix') before openlog.
# (To get setlocksock: 'use Sys::Syslog qw(:DEFAULT setlogsock);' )
Robert Ricci's avatar
Robert Ricci committed
49

Mac Newbold's avatar
Mac Newbold committed
50 51 52
# Do lots of db retries before we fail and die
$libdb::DBQUERY_MAXTRIES = 5;

Robert Ricci's avatar
Robert Ricci committed
53 54 55
# Number of iterations (roughly, seconds) after which we'll reload 
# information from the database. This is so we don't end up with information
# that's _too_ out of sync.
56
my $reload_time = 600;
57
my $last_reload = time;
Robert Ricci's avatar
Robert Ricci committed
58 59 60 61

# Process command-line arguments

sub usage {
Mac Newbold's avatar
Mac Newbold committed
62
    print << "END";
63 64 65
Usage: $0 [-h] [-d] [-s server] [-p port]
-h              This message
-d              Turn on debugging output, and don't go into the background
Robert Ricci's avatar
Robert Ricci committed
66 67
-s server       Use specified server, instead of this site's bossnode
-p port	        Use specified port
68
Send SIGHUP to reload database state, or SIGUSR1 to restart completely.
Robert Ricci's avatar
Robert Ricci committed
69 70 71
END
}

Mac Newbold's avatar
Mac Newbold committed
72
# Only root should run this - it won't work when run as a user...
73
# (Or, let an admin run it if it isn't the real one in /usr/testbed/ )
74
if ($UID && ( $TB eq $REALTB || ! TBAdmin($UID) ) ) {
Mac Newbold's avatar
Mac Newbold committed
75 76 77
    die("Only root can run this script!\n");
}

78
my @args = @ARGV;    # save a copy for restart before we mess with them.
Robert Ricci's avatar
Robert Ricci committed
79
my %opt = ();
80
getopts("ds:p:h",\%opt);
Robert Ricci's avatar
Robert Ricci committed
81

Mac Newbold's avatar
Mac Newbold committed
82 83 84 85 86 87
if ($opt{h}) {
    exit &usage;
}
if (@ARGV) {
    exit &usage;
}
Robert Ricci's avatar
Robert Ricci committed
88

89
my ($server,$port,$debug);
Mac Newbold's avatar
Mac Newbold committed
90 91 92 93 94 95 96 97 98 99 100 101 102
if ($opt{s}) {
    $server = $opt{s};
} else {
    $server = $BOSSNODE;
}
if ($opt{p}) {
    $port = $opt{p};
}
if ($opt{d}) {
    $debug = 1;
} else {
    $debug = 0;
}
Robert Ricci's avatar
Robert Ricci committed
103

104
# Grab some constants into variables
105
my $TBANYMODE    = TBDB_NODEOPMODE_ANY;
106 107 108 109 110 111
my $TBRESET      = TBDB_TBCONTROL_RESET;
my $TBRELOADDONE = TBDB_TBCONTROL_RELOADDONE;
my $TBTIMEOUT    = TBDB_TBCONTROL_TIMEOUT;
my $TBNOTIMEOUT  = TBDB_NO_STATE_TIMEOUT;
my $TBNODESTATE  = TBDB_TBEVENT_NODESTATE;
my $TBNODEOPMODE = TBDB_TBEVENT_NODEOPMODE;
112 113 114 115 116 117
my $TBCONTROL    = TBDB_TBEVENT_CONTROL;
my $TBCOMMAND    = TBDB_TBEVENT_COMMAND;
my $TBREBOOT     = TBDB_COMMAND_REBOOT;
my $TBPOWEROFF   = TBDB_COMMAND_POWEROFF;
my $TBPOWERON    = TBDB_COMMAND_POWERON;
my $TBPOWERCYCLE = TBDB_COMMAND_POWERCYCLE;
118
my $TBISUP       = TBDB_NODESTATE_ISUP;
119 120 121 122
my $TBTIMEOUTREBOOT   = TBDB_STATED_TIMEOUT_REBOOT;
my $TBTIMEOUTNOTIFY   = TBDB_STATED_TIMEOUT_NOTIFY;
my $TBTIMEOUTCMDRETRY = TBDB_STATED_TIMEOUT_CMDRETRY;
my $TB_OSID_MBKERNEL  = TB_OSID_MBKERNEL;
123

124 125 126
# This only gets used here, so it isn't in a lib constant.
my $TBFREENODE = "FREENODE";

127
# Set up some notification throttling
Mac Newbold's avatar
Mac Newbold committed
128
my $mailgap = 15;		# in seconds
129 130 131
my $lastmail = time() - $mailgap + 2; # Send a digest of startup msgs after 2s.
my %msgs = ();

Mac Newbold's avatar
Mac Newbold committed
132
my $pidfile;
133
if ( $TB eq $REALTB ) {
Mac Newbold's avatar
Mac Newbold committed
134 135
    $pidfile = "/var/run/stated.pid";
} else {
136
    $pidfile = "$TB/locks/stated.pid";
Mac Newbold's avatar
Mac Newbold committed
137
}
Mac Newbold's avatar
Mac Newbold committed
138 139
debug("Using pidfile $pidfile\n");
if (-e $pidfile) {
Mac Newbold's avatar
Mac Newbold committed
140 141 142 143 144 145 146 147 148 149 150
    my $otherpid = `cat $pidfile`;
    my $running = `ps -auxww | grep $otherpid | grep -v grep`;
    if ($running ne "") {
	fatal("Lockfile $pidfile exists, and process $otherpid appears to be ".
	      "running.\n");
    } else {
	notify("Lockfile exists, but process $otherpid appears to be dead.\n".
	       "Removing lock file...\n");
    }
    system("rm $pidfile") &&
      fatal("Couldn't remove $pidfile: $? $!\n");
Mac Newbold's avatar
Mac Newbold committed
151
}
Robert Ricci's avatar
Robert Ricci committed
152
# Background
153
if (!$debug) {
Mac Newbold's avatar
Mac Newbold committed
154 155 156 157
    # We use syslog, so redirect the output to nothing
    if (TBBackGround("/dev/null")) {
	exit(0);
    }
Robert Ricci's avatar
Robert Ricci committed
158
}
Mac Newbold's avatar
Mac Newbold committed
159 160
# set up syslog
openlog("stated","pid","user");
Mac Newbold's avatar
Mac Newbold committed
161 162 163 164 165 166
sysopen(PIDFILE, $pidfile, O_WRONLY | O_EXCL | O_CREAT) ||
  fatal("Couldn't create '$pidfile': $? $!\n");
print PIDFILE "$$";
close PIDFILE;
# If I make it to here, I'll need to clean up the lock file
my $lockfile=$pidfile;
Robert Ricci's avatar
Robert Ricci committed
167

168 169 170
# Change my $0 so that it is easier to see in a ps/top
$0 = "$0";

Robert Ricci's avatar
Robert Ricci committed
171
my $URL = "elvin://$server";
Mac Newbold's avatar
Mac Newbold committed
172 173 174
if ($port) {
    $URL .= ":$port";
}
Robert Ricci's avatar
Robert Ricci committed
175 176 177

# Connect to the event system, and subscribe the the events we want 
my $handle = event_register($URL,0);
Mac Newbold's avatar
Mac Newbold committed
178 179 180
if (!$handle) {
    fatal("Unable to register with event system\n");
}
Robert Ricci's avatar
Robert Ricci committed
181 182

my $tuple = address_tuple_alloc();
Mac Newbold's avatar
Mac Newbold committed
183 184 185
if (!$tuple) {
    fatal("Could not allocate an address tuple\n");
}
Robert Ricci's avatar
Robert Ricci committed
186

187 188 189
%$tuple = ( objtype => join(",",
			    $TBNODESTATE, $TBNODEOPMODE,
			    $TBCONTROL, $TBCOMMAND) );
190

Robert Ricci's avatar
Robert Ricci committed
191
if (!event_subscribe($handle,\&handleEvent,$tuple)) {
Mac Newbold's avatar
Mac Newbold committed
192
    fatal("Could not subscribe to events\n");
Robert Ricci's avatar
Robert Ricci committed
193 194 195 196
}

# Read in the pre-existing node states, and timeout and valid transition
# information from the database
197 198 199 200
my %timeouts  = getTimeouts();
my %valid     = getValid();
my %modeTrans = getModeTrans();
my %triggers  = getTriggers();
201
my %nodes     = readStates();
202
my %timeouttag= ();
203
if ($debug) { qshow(); }
Robert Ricci's avatar
Robert Ricci committed
204 205 206

# Gets set if a reload of state from the database should happen.
my $do_reload = 0;
207 208
my $sigrestart= 0;
my $sigcleanup= 0;
Robert Ricci's avatar
Robert Ricci committed
209 210 211 212

# Make the daemon reload database state on a sighup - but I'm worried
# about what would happen if we tried to do this mid-loop. So, we'll
# just set a flag and do it when we're done with our current pass.
213 214
$SIG{HUP}  = sub { info("SIGHUP - Reloading DB state\n"); $do_reload = 1; };

Mac Newbold's avatar
Mac Newbold committed
215
# Set up other signals.
216 217 218 219 220 221 222
$SIG{USR1} = \&restart_wrap;
$SIG{USR2} = \&cleanup_wrap;
$SIG{INT}  = \&cleanup_wrap;
$SIG{QUIT} = \&cleanup_wrap;
$SIG{ABRT} = \&cleanup_wrap;
$SIG{TERM} = \&cleanup_wrap;
$SIG{KILL} = \&cleanup_wrap;
Robert Ricci's avatar
Robert Ricci committed
223

224 225 226
# Track if I handled an event or not
my $event_count = 0;

227 228
# Control how long I block while waiting for events
my $blockwait=0;
229
my $nextdeadline=0;
230 231
my $mailqueue=0;

232 233 234 235 236
notify("Stated starting up\n");

sub process_event_queue() {
    $event_count=0;
    my $lastcount=-1;
237 238 239 240 241 242 243 244 245 246 247 248 249
    my $wait;
    my $now = time();
    debug("Polling - mq=$mailqueue bw=$blockwait\n");
    if ( $mailqueue == 0) {
	# no messages waiting...
	if ($blockwait) {
	    # we can wait a long time - nothing else will happen
	    # until we get an event, or get woken up by a signal
	    $wait = 600;
	} else {
	    # only wait until the next deadline...
	    if ($nextdeadline > 0) {
		$wait = $nextdeadline - $now;
250 251
	    } else {
		$wait = 0;
252 253 254 255 256
	    }
	}
    } else {
	# mail is waiting. Only block until it is time to send it.
	$wait = $lastmail + $mailgap - $now;
Mac Newbold's avatar
Mac Newbold committed
257
	debug("Now $now, mailgap $mailgap, last $lastmail ==> wait $wait\n");
258 259 260
    }
    if ($wait < 0) { debug("Wait was $wait!\n"); $wait=0; }
    my $finish = $now + $wait;
Mac Newbold's avatar
Mac Newbold committed
261 262
    while (($event_count != $lastcount || $wait > 0) &&
	   !($sigrestart || $sigcleanup || $do_reload)) {
263
	$lastcount = $event_count;
Mac Newbold's avatar
Mac Newbold committed
264 265
	# Don't block if we got a signal!
	if ($wait<=0 || $sigrestart || $sigcleanup || $do_reload) {
266 267 268 269 270 271 272 273
	    event_poll($handle);
	} else {
	    debug("Using blocking event poll - $wait seconds\n");
	    # timeout param is in milliseconds, so multiply
	    event_poll_blocking($handle, $wait*1000);
	    $now = time();
	    # subtract seconds elapsed from my wait time
	    $wait = $finish - $now;
Mac Newbold's avatar
Mac Newbold committed
274
	    debug("Finished blocking event poll - $wait seconds remain\n");
275
	    if ($event_count > 0 &&
Mac Newbold's avatar
Mac Newbold committed
276 277
		(qsize() > 0 || $mailqueue ||
		 $sigrestart || $sigcleanup || $do_reload)) {
278 279 280 281 282 283 284
		$blockwait=0;
		$wait=0;
		#debug("Cancelling wait - timeouts/msgs waiting, or HUP'd\n");
		#debug("---End Blocking Wait ---\n");
	    }
	}
	#debug("Wait is $wait\n");
285 286
    }
    if ($event_count > 0) {
Mac Newbold's avatar
Mac Newbold committed
287
	debug("Handled $event_count event(s).\n");
288 289
    }
}
Robert Ricci's avatar
Robert Ricci committed
290

291
# Now, we just poll for events, and watch for timeouts
Robert Ricci's avatar
Robert Ricci committed
292
while (1) {
Mac Newbold's avatar
Mac Newbold committed
293
    my $now = time();
294 295 296 297
    my ($deadline,$node);

    # Check for nodes that have passed their timeout
    if (!qhead($deadline,$node)) {
Mac Newbold's avatar
Mac Newbold committed
298
	info("HEAD: $node in ".($deadline-$now).", queue=".qsize()."\n");
299 300
	while ($now >= $deadline && $node ne "") {
	    qpop($deadline,$node);
Mac Newbold's avatar
Mac Newbold committed
301
	    info("POP: $node in ".($deadline-$now).", queue=".qsize()."\n");
302
	    handleCtrlEvent($node,$TBTIMEOUT);
303 304 305 306
	    if (0) { qshow(); }
	    if (qhead($deadline,$node)) {
		$deadline=0; $node="";
	    }
307
	}
308 309
    } else {
	$deadline=0;
310
    }
311
    $nextdeadline = $deadline;
312

313 314 315 316 317
    if (qsize()==0) {
	$blockwait=1;
	debug("---Blocking wait okay---\n");
    }
		
Mac Newbold's avatar
Mac Newbold committed
318 319 320 321
    if ($do_reload || ($now - $last_reload > $reload_time)) {
	reload();
	$do_reload = 0;
    }
Mac Newbold's avatar
Mac Newbold committed
322

Mac Newbold's avatar
Mac Newbold committed
323 324
    # Send any messages in the queue if it is time
    notify("",1);
Mac Newbold's avatar
Mac Newbold committed
325

326 327 328
    if ($sigrestart) { restart(); }
    if ($sigcleanup) { cleanup(); }

329
    process_event_queue;
Robert Ricci's avatar
Robert Ricci committed
330 331
}

Mac Newbold's avatar
Mac Newbold committed
332 333
exit(0);

Robert Ricci's avatar
Robert Ricci committed
334
# Read the current states of nodes from the database
335
sub readStates(;@) {
Mac Newbold's avatar
Mac Newbold committed
336 337 338 339 340 341
    my %oldnodes = @_;

    # Guard against undefined variable warnings
    if (! defined(%oldnodes)) {
	%oldnodes = ();
    }
342

Mac Newbold's avatar
Mac Newbold committed
343 344 345
    #debug("readStates called\n");
    my $result = DBQueryFatal("SELECT node_id, eventstate, " .
			      "state_timestamp, op_mode, " .
346 347
			      "op_mode_timestamp FROM nodes ".
			      "where node_id not like 'sh%'");
Mac Newbold's avatar
Mac Newbold committed
348 349 350 351

    my %nodes;
    while (my ($node_id, $state, $timestamp, $mode, $mode_timestamp)
	   = $result->fetchrow()) {
352
	#
Mac Newbold's avatar
Mac Newbold committed
353 354 355 356
	# If there's an entry in oldnodes for this node, and it
	# hasn't changed state or time, use the old entry (so that
	# we don't lose information about which nodes we've already
	# notified the ops about, etc.)
357
	#
Mac Newbold's avatar
Mac Newbold committed
358 359 360 361 362
	if ($oldnodes{$node_id} && $state && $timestamp &&
	    ($oldnodes{$node_id}{state} eq $state) &&
	    ($oldnodes{$node_id}{mode} eq $mode) &&
	    ($oldnodes{$node_id}{timestamp} == $timestamp)) {
	    $nodes{$node_id} = $oldnodes{$node_id};
363
	} else {
Mac Newbold's avatar
Mac Newbold committed
364 365 366 367
	    $nodes{$node_id}{state}          = $state;
	    $nodes{$node_id}{timestamp}      = $timestamp;
	    $nodes{$node_id}{mode}           = $mode;
	    $nodes{$node_id}{mode_timestamp} = $mode_timestamp;
368 369 370
	    $nodes{$node_id}{notified}       = 0;
	    $nodes{$node_id}{timedout}       = 0;
	    $nodes{$node_id}{noretry}        = 0;
371 372
	    # Is there a timeout? If so, set it up!
	    setTimeout($mode,$state,$node_id,$timestamp);
373
	}
Mac Newbold's avatar
Mac Newbold committed
374 375
    }
    return %nodes;
Robert Ricci's avatar
Robert Ricci committed
376 377 378 379 380 381
}

#
# Read timeouts for various states from the database
#
sub getTimeouts() {
Mac Newbold's avatar
Mac Newbold committed
382 383 384
    #debug("getTimeouts called\n");
    my $result = DBQueryFatal("SELECT op_mode, state, timeout, action " .
			      "FROM state_timeouts");
Robert Ricci's avatar
Robert Ricci committed
385

Mac Newbold's avatar
Mac Newbold committed
386 387 388 389 390
    my %timeouts;
    while (my ($op_mode, $state, $timeout, $action) = $result->fetchrow()) {
	$timeouts{$op_mode}{$state} = [ $timeout, $action ];
    }
    return %timeouts;
Robert Ricci's avatar
Robert Ricci committed
391 392 393 394 395 396
}

#
# Read the list of valid state transitions from the database
#
sub getValid() {
Mac Newbold's avatar
Mac Newbold committed
397 398 399
    #debug("getValid called\n");
    my $result = DBQueryFatal("SELECT op_mode, state1, state2 " .
			      "FROM state_transitions");
Robert Ricci's avatar
Robert Ricci committed
400

Mac Newbold's avatar
Mac Newbold committed
401 402 403 404 405
    my %valid;
    while (my ($mode,$state1, $state2) = $result->fetchrow()) {
	$valid{$mode}{$state1}{$state2} = 1;
    }
    return %valid;
Robert Ricci's avatar
Robert Ricci committed
406 407
}

408 409 410 411
#
# Read the list of valid mode transitions from the database
#
sub getModeTrans() {
Mac Newbold's avatar
Mac Newbold committed
412 413 414 415 416 417 418 419 420 421 422 423 424
    #debug("getModeTrans called\n");
    my $result = 
      DBQueryFatal("SELECT op_mode1, state1, op_mode2, state2 " .
		   "FROM mode_transitions order by op_mode1,state1");

    my %modeTrans;
    while (my ($mode1,$state1, $mode2, $state2) = $result->fetchrow()) {
	if (!defined($modeTrans{"$mode1:$state1"})) {
	    $modeTrans{"$mode1:$state1"}= ["$mode2:$state2"];
	} else {
	    my @l = @{$modeTrans{"$mode1:$state1"}};
	    push(@l, "$mode2:$state2");
	    $modeTrans{"$mode1:$state1"}= \@l;
425
	}
Mac Newbold's avatar
Mac Newbold committed
426 427
    }
    return %modeTrans;
428 429 430 431 432 433
}

#
# Read the list of states which trigger an action
#
sub getTriggers() {
434 435 436 437 438
    debug("getTriggers called\n");
    
    debug("anymode ==> '$TBANYMODE'\n");

    # Grab global triggers
Mac Newbold's avatar
Mac Newbold committed
439 440
    my $result = 
      DBQueryFatal("SELECT op_mode, state, trigger " .
441 442
		   "FROM state_triggers where node_id='$TBANYMODE' ".
		   "order by op_mode,state");
Mac Newbold's avatar
Mac Newbold committed
443
    my %t;
444
    while (my ($mode, $state, $trig) = $result->fetchrow()) {
Mac Newbold's avatar
Mac Newbold committed
445
	$t{"$mode:$state"} = $trig;
446 447 448 449 450 451 452 453 454 455 456 457
	debug("trig($mode:$state)\t => $trig\n");
    }

    # Grab per-node triggers
    $result = 
      DBQueryFatal("SELECT node_id, op_mode, state, trigger " .
		   "FROM state_triggers where node_id!='$TBANYMODE' ".
		   "order by op_mode,state");
    while (my ($n, $mode, $state, $trig) = $result->fetchrow()) {
	my @trigs = split(/\s*,\s*/,$trig);
	$t{"$n:$mode:$state"} = \@trigs;
	debug("trig($n:$mode:$state)\t => ".join(',',@trigs)."\n");
Mac Newbold's avatar
Mac Newbold committed
458
    }
459

Mac Newbold's avatar
Mac Newbold committed
460
    return %t;
461 462
}

Robert Ricci's avatar
Robert Ricci committed
463 464 465 466
#
# Gets called for every event that we recieve
#
sub handleEvent($$$) {
Mac Newbold's avatar
Mac Newbold committed
467 468 469 470 471 472 473 474 475 476 477
    my ($handle,$notification,$data) = @_;
    my $objtype = event_notification_get_objtype($handle,$notification);
    my $objname = event_notification_get_objname($handle,$notification);
    my $eventtype = event_notification_get_eventtype($handle,$notification);

    $event_count++;
    debug("Got an event: ($objtype,$objname,$eventtype)\n");

    #
    # Check to see if another instance is supposed to be handling this node
    #
478
    if ($objtype ne $TBCOMMAND && !checkDBRedirect($objname)) {
Mac Newbold's avatar
Mac Newbold committed
479 480 481
	info("Got an event for node $objname, which isn't mine\n");
	return;
    }
482

Mac Newbold's avatar
Mac Newbold committed
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
  SWITCH: for ($objtype) {
	
	(/$TBNODESTATE/) && do {
	    stateTransition($objname,$eventtype);
	    last;
	};
	(/$TBNODEOPMODE/) && do {
	    opModeTransition($objname,$eventtype);
	    notify("Use of deprecated event TBNODEOPMODE:\n".
		   "$objname->$eventtype\n");
	    last;
	};
	(/$TBCONTROL/) && do {
	    handleCtrlEvent($objname,$eventtype);
	    last;
	};
499 500 501 502
	(/$TBCOMMAND/) && do {
	    handleCommand($objname,$eventtype);
	    last;
	};
503

Mac Newbold's avatar
Mac Newbold committed
504
    }
505 506 507 508 509

}

sub stateTransition($$) {

510
    my ($node,$newstate) = @_;
Robert Ricci's avatar
Robert Ricci committed
511

512 513 514 515 516 517 518 519
    # Check for invalid transitions
    my ($oldstate, $mode);
    if ($nodes{$node}) {
	$oldstate = $nodes{$node}{state};
	$mode = $nodes{$node}{mode};
    } else {
	# Try reloading the cache once before we give up on this node
	reload();
520
	if ($nodes{$node}) {
521 522
	    $oldstate = $nodes{$node}{state};
	    $mode = $nodes{$node}{mode};
Robert Ricci's avatar
Robert Ricci committed
523
	} else {
524
	    notify("Got an event for a node ($node) I don't know about\n");
Robert Ricci's avatar
Robert Ricci committed
525
	}
526 527 528 529
    }
    if ($oldstate && $mode && $valid{$mode} && $valid{$mode}{$oldstate} &&
	!$valid{$mode}{$oldstate}{$newstate}) {
	notify("Invalid transition for node $node from $mode/$oldstate " .
530
	       "to $newstate\n");
531
    }
Robert Ricci's avatar
Robert Ricci committed
532

533 534 535 536
    my $now = time();
    $nodes{$node}{state}     = $newstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{notified}  = 0;
537

538 539 540
    info("$node: $mode/$oldstate => $mode/$newstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$newstate', " .
		 "state_timestamp='$now' WHERE node_id='$node'");
541

542 543 544 545
    # Before we set the timeout (overwriting any current ones), we need
    # to check if we had a pending command
    if (qfind($node) &&
	$timeout_tag{$node} =~ /^$TBCOMMAND:/) {
546
        debug("TimeoutTag = '$timeout_tag{$node}'\n");
547
	my ($str,$cmd) = split(":",$timeout_tag{$node});
548
	debug("str=$str\tcmd=$cmd\tTBREBOOT=$TBREBOOT\tstate=$newstate\n");
549
	if ($cmd eq $TBREBOOT) {
550
	    if ($newstate eq TBDB_NODESTATE_SHUTDOWN ) {
551 552 553
		info("$node: $TBREBOOT success\n");
		# Timeout will get cleared below by setTimeout call
	    } else {
554 555
		notify("$node: $TBREBOOT in progress, but got state ".
		       "$newstate instead of ".TBDB_NODESTATE_SHUTDOWN."!\n");
556 557 558 559 560
	    }
	#} elsif ($cmd eq $FOO ) {
	    # Add more here...
	} else {
	    notify("$node: Unknown command timeout '$timeout_tag{$node}' ".
561
		   "found at $mode/$newstate\n");
562 563 564
	}
    }

565 566 567
    # Check if this state has a timeout, and if so, put it in the queue
    setTimeout($mode,$newstate,$node,$now);

568 569
    # Check if this is TBDB_NODESTATE_BOOTING , which has actions
    if ($newstate eq TBDB_NODESTATE_BOOTING) {
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	# If I skipped shutdown, and came to booting directly from isup,
	# check for a mode transition so I don't miss one...
	if ($oldstate eq TBDB_NODESTATE_ISUP) {
	    info("$node: Came from ISUP! Checking for mode transition\n");
	    my $r = DBQueryWarn("select next_op_mode from nodes ".
				"where node_id='$node'");
	    my ($nextmode) = $r->fetchrow();
	    if ($nextmode) {
		# Force the transition even though it is illegal
		info("$node: Forcing mode transition!\n");
		opModeTransition($node,$nextmode,1);
		$mode=$nextmode;
	    } else {
		debug("No next mode.\n");
	    }
	}

	# Check if I'm in the right mode
588
	my $osid = TBBootWhat($node,$debug);
589 590
	my $os_op_mode = os_opmode($osid);
	info("$node: Current OS is '$osid', OS mode is '$os_op_mode'\n");
591
	DBQueryFatal("UPDATE nodes SET osid='$osid' WHERE node_id='$node'");
592
	if ($os_op_mode ne $mode) {
593 594
	    my $str = "Node $node is running OS '$osid' but in mode '$mode' ".
	      "instead of mode '$os_op_mode'!\n";
595 596 597
	    # For now, only force if we're going into reload mode, so we
	    # don't get stuck looping in reloading.
	    if ($os_op_mode eq "RELOAD") {
598 599 600 601 602 603 604 605 606
		DBQueryFatal("UPDATE nodes SET op_mode='$os_op_mode', ".
			     "op_mode_timestamp=unix_timestamp(now()) ".
			     "WHERE node_id='$node'");
		$nodes{$node}{mode} = $os_op_mode;
		$nodes{$node}{mode_timestamp} = $now;
		$str .= "Forced op_mode to '$os_op_mode'.\n";
	    }
	    notify($str);
	}
607 608
	checkGenISUP($node);
    }
609

610 611 612 613 614
    # Check if this state has any triggers
    my @nodetrigs = GetNodeTriggerList($node,$mode,$newstate);
    if (defined($triggers{"$mode:$newstate"}) ||
        (@nodetrigs > 0) ) {
	# check for global triggers
615
	my @trigs = split(/\s*,\s*/,$triggers{"$mode:$newstate"});
616 617 618 619
	# Run all the triggers
	debug("Running triggers. Global=".join("/",@trigs).
	      "   node=".join("/",@nodetrigs)."\n");
	foreach ( @trigs , @nodetrigs) {
620 621
	    my $trig = $_;
	    /^$TBRESET$/ && do {
622 623 624
		# We successfully booted, so clear some flags
		$nodes{$node}{noretry}   = 0;
		$nodes{$node}{timedout}  = 0;
625 626 627 628 629 630 631 632 633 634 635 636 637
		# Check if we really need to do a reset
		my $r = DBQueryWarn("select osid,def_boot_osid from nodes ".
				    "where node_id='$node'");
		my ($osid,$defosid) = $r->fetchrow();
		if ($osid ne $defosid) {
		    handleCtrlEvent($node,$trig);
		}
		next;
	    };
	    /^$TBRELOADDONE$/ && do {
		handleCtrlEvent($node,$trig);
		next;
	    };
638 639 640 641
	    /^$TBFREENODE$/ && do {
		handleCtrlEvent($node,$trig);
		next;
	    };
642 643 644 645 646 647 648 649
	    /^$TBISUP$/ && do {
		info("$node: Triggered $TBISUP\n");
		EventSendWarn(host      => $BOSSNODE ,
			      objtype   => TBDB_TBEVENT_NODESTATE ,
			      eventtype => TBDB_NODESTATE_ISUP ,
			      objname   => $node);
		next;
	    };
650
	    notify("Unknown trigger '$trig' for $node in $mode/$newstate!\n");
651
	}
652 653 654
	# Clear any of the node triggers that we ran
	debug("Clearing node triggers: ".join("/",@nodetrigs)."\n");
	ClearNodeTrigger($node,$mode,$newstate,@nodetrigs);
655
    }
656

657 658 659 660 661 662 663 664
    # Check if this state can trigger a mode transition
    if (defined($modeTrans{"$mode:$newstate"})) {
	info("$node: Checking for mode transition\n");
	my $r = DBQueryWarn("select next_op_mode from nodes ".
			    "where node_id='$node'");
	my ($nextmode) = $r->fetchrow();
	if ($nextmode) {
	    opModeTransition($node,$nextmode);
Mac Newbold's avatar
Mac Newbold committed
665 666 667
	} else {
	    debug("No next mode.\n");
	}
668 669
    }
}
670

671
sub opModeTransition($$;$) {
Mac Newbold's avatar
Mac Newbold committed
672

673 674
    my ($node,$newmode,$force) = @_;
    if (!defined($force)) { $force = 0; }
Mac Newbold's avatar
Mac Newbold committed
675

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
    info("$node: Mode change to $newmode requested\n");
    # Check for invalid transitions
    my ($oldstate, $mode, $nextstate);
    if ($nodes{$node}) {
	$oldstate = $nodes{$node}{state};
	$mode = $nodes{$node}{mode};
    } else {
	# Try reloading the cache once before we give up on this node
	reload();
	if ($nodes{$node}) {
	    $oldstate = $nodes{$node}{state};
	    $mode = $nodes{$node}{mode};
	} else {
	    notify("Got an event for a node ($node) I don't know about\n");
	}
    }
692
    if (defined($modeTrans{"$mode:$oldstate"}) || $force) {
693
	if (!$force) {
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
	    debug("Mode Transition check:\n");
	    my $translist = join(",",@{$modeTrans{"$mode:$oldstate"}});
	    #debug("translist=$translist\n");
	    #debug("splitlist=".join(", ",split(/[:,]/,$translist))."\n");
	    my %trans = split(/[:,]/,$translist);
	    debug("Valid transitions from $mode/$oldstate are:\n");
	    foreach my $k (sort keys %trans) {
		debug("$k => $trans{$k}\n");
	    }
	    if (defined($trans{$newmode})) {
		$nextstate=$trans{$newmode};
	    } else {
		notify("Invalid mode transition for $node from ".
		       "$mode/$oldstate to $newmode!\n");
	    }
709 710
	}
    } else {
711
	notify("Invalid mode transition for $node from $mode/$oldstate: ".
712 713
	       "Not a valid mode transition state!\n");
    }
Mac Newbold's avatar
Mac Newbold committed
714 715 716
    if (!$nextstate) {
	$nextstate=$oldstate;
    }
Mac Newbold's avatar
Mac Newbold committed
717

718 719 720 721 722 723
    my $now = time();
    $nodes{$node}{state}     = $nextstate;
    $nodes{$node}{timestamp} = $now;
    $nodes{$node}{mode}           = $newmode;
    $nodes{$node}{mode_timestamp} = $now;
    $nodes{$node}{notified}       = 0;
Mac Newbold's avatar
Mac Newbold committed
724

725 726 727 728 729
    info("$node: $mode/$oldstate => $newmode/$nextstate\n");
    DBQueryFatal("UPDATE nodes SET eventstate='$nextstate', ".
		 "next_op_mode='', op_mode='$newmode', ".
		 "state_timestamp='$now', ".
		 "op_mode_timestamp='$now' WHERE node_id='$node'");
Mac Newbold's avatar
Mac Newbold committed
730 731 732 733

    # Check if this state has a timeout, and if so, put it in the queue
    setTimeout($newmode,$nextstate,$node,$now);

734 735 736 737
}

sub handleCtrlEvent($$) {
    my ($node,$event) = @_;
738

739
    info("CtrlEvent: $node, $event\n");
740

741 742
    foreach ($event) {
	/^$TBRESET$/ && do {
Mac Newbold's avatar
Mac Newbold committed
743 744 745
	    my $result = DBQueryFatal("SELECT pxe_boot_path, def_boot_osid ".
				      "FROM nodes where node_id='$node'");
	    my ($pxepath,$osid) = $result->fetchrow();
746

747 748 749 750
	    # Important note on ordering here:
	    # Because setting a normal osid resets pxe path to PXEBOOT,
	    # We need to read it out first, then set the osid, then set
	    # the pxepath back to its original value at the end.
751

752 753
	    $cmd = "$osselect $osid $node";
	    system($cmd) and
Mac Newbold's avatar
Mac Newbold committed
754 755
	      notify("$node/$event: Couldn't clear next_boot_*\n".
		     "\tcmd=$cmd\n\t*** $!\n");
756

757
	    $pxepath = "-p ".$pxepath;
Mac Newbold's avatar
Mac Newbold committed
758 759 760 761
	    if ($pxepath eq "-p ") {
		$pxepath="PXEBOOT";
	    }
	    ;
Mac Newbold's avatar
Mac Newbold committed
762
	    my $cmd = "$osselect -m $pxepath $node";
763
	    system($cmd) and
Mac Newbold's avatar
Mac Newbold committed
764 765
	      notify("$node/$event: Couldn't clear next_pxe_boot_path\n".
		     "\tcmd=$cmd\n\t*** $!\n");
766

Mac Newbold's avatar
Mac Newbold committed
767
	    info("Performed RESET for $node to $osid/$pxepath\n");
768 769 770 771 772 773 774 775 776 777
	    next;
	};
	/^$TBRELOADDONE$/ && do {
	    info("Clearing reload info for $node\n");
	    DBQueryFatal("delete from current_reloads where node_id='$node'");
	    my ($pid,$eid);
	    NodeidToExp($node,\$pid,\$eid);
	    if (($pid eq NODERELOADING_PID) && ($eid eq NODERELOADING_EID)) {
		DBQueryFatal("delete from scheduled_reloads ".
			     "where node_id='$node'");
778 779 780 781
		AddNodeTrigger($node, $TBANYMODE, TBDB_NODESTATE_ISUP,
			       $TBFREENODE)
		  && notify("$node: Couldn't add trigger $TBFREENODE!\n");
		info("Set up freeing of $node from $pid/$eid\n");
782 783 784
	    }
	    next;
	};
785 786 787 788 789 790 791 792
	/^$TBFREENODE$/ && do {
	    # Don't need pid/eid, but we should put it in the log
	    my ($pid,$eid);
	    NodeidToExp($node,\$pid,\$eid);
	    DBQueryFatal("delete from reserved where node_id='$node'");
	    info("Released $node from $pid/$eid\n");
	    next;
	};
793
	/^$TBTIMEOUT$/ && do {
794 795 796 797
	    my ($mode,$state) = split(":",$timeout_tag{$node});
	    delete($timeout_tag{$node});
	    my $curstate = $nodes{$node}{state};
	    my $curmode = $nodes{$node}{mode};
798
	    my ($timeout,$action);
799 800 801 802 803
	    if (!defined($nodes{$node}{notified})) {
		$nodes{$node}{notified}=0;
	    }
	    $nodes{$node}{notified}++;
	    my $notified = $nodes{$node}{notified};
804 805
	    $nodes{$node}{timedout}++;
	    my $timedout = $nodes{$node}{timedout};
806 807 808 809
	    if ($mode && $state && $timeouts{$mode} &&
		$timeouts{$mode}{$state}) {
		($timeout, $action) = @{$timeouts{$mode}{$state}};
	    }
810 811
	    if ($mode eq $TBCOMMAND) {
		# It is a command, not a true state
812
		if ($action eq $TBTIMEOUTCMDRETRY) {
813
		    # Retry the command
814
		    notify("$node: Command $state, retry #$timedout\n");
815
		    # notify in case we get in a retry loop...
816
		    handleCommand($node,$state,$timedout,1);
817 818 819 820 821 822
		} else {
		    notify("$node: Unknown timeout action for ".
			   "$mode/$state: '$action'\n");
		}
		next;
	    }
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
	    info("Node $node has timed out in state $mode/$state".
		 ($action ne "" ? "\n\tRequested action $action." : "").
		 "\n");

	    foreach ($action) {
		/^$TBTIMEOUTREBOOT/ && do {
		    if ($timedout>3) {
			# We've tried too many times...
			notify("Node $node has timed out too many times!\n".
			       "Giving up until it boots sucessfully.\n");
			$nodes{$node}{noretry} = 1;
		    } else {
			# XXX Temporary!  For now notify instead of
			# really rebooting, until the timeout/retry
			# stuff is gone from os_setup and os_load
			notify("Node $node has timed out in state ".
			       "$mode/$state - REBOOT requested\n");
			#handleCommand($node,$TBREBOOT,$timedout,1);
		    }
		    last; };
		/^$TBTIMEOUTNOTIFY/ && do {
		    notify("Node $node has timed out in state $mode/$state\n");
		    last; };
		notify("$node: Unknown Timeout Action: $action\n");
	    }
848 849
	    next;
	};
850
	notify("$node: Unknown CtrlEvent: $event\n");
851 852
    }
}
Robert Ricci's avatar
Robert Ricci committed
853

854 855
sub handleCommand($$;$$) {
    my ($params,$command,$retry,$force) = @_;
856
    if (!defined($retry)) { $retry=0; }
857
    if (!defined($force)) { $force=0; }
858 859 860 861 862 863 864 865

    info("Command: $params, $command (attempt $retry)\n");

    # XXX - Right now we skip the checkDBRedirect calls for our
    # TBCOMMAND events, since they may have a list of nodes in them.
    # We may need to do it here (while iterating over the list), or
    # make some other fix up in handleEvent.

866
    if ($command eq $TBREBOOT && $retry >=4) {
867 868
	announce("Node $params has tried rebooting $retry times and has \n".
		 "still not been successful. Please look into it soon.\n".
869 870 871
	"" );#	 "In the meantime, $params will be powered off.\n");
	# Just return...
	return 0;
872
	# change my command to poweroff.
873
	#$command = $TBPOWEROFF;
874 875 876 877 878 879
    }

    foreach ($command) {
	/^$TBREBOOT$/ && do {
	    # For reboot, the params is a comma-separated list of nodes
	    my @nodes = split(",",$params);
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902

	    # Check if any of these are in the middle of things and
	    # shouldn't be rebooted.
	    if (!$force) {
		foreach $n ( 0 .. $#nodes ) {
		    $node = $nodes[$n];
		    debug("Checking rebooting: $node, $nodes{$node}, ".
			  "$nodes{$node}{state}, $nodes{$node}{noretry}\n");
		    if (($nodes{$node}{state} ne TBDB_NODESTATE_ISUP) &&
			(!$nodes{$node}{noretry}) ) {
			# This node shouldn't be rebooted now...
			# XXX Send feedback here somehow!

			info("$node: Trying to reboot too soon! Skipping.\n");
			# Cut it out of the list
			debug("Nodelist before ==> ".join(" ",@nodes)."\n");
			splice(@nodes,$n,1);
			debug("Nodelist after  ==> ".join(" ",@nodes)."\n");
		    }
		}
		if (@nodes == 0) { next; }
	    }

903 904 905 906 907
	    my $nodelist=join(" ",@nodes);
	    info("Rebooting nodes: $nodelist\n");
	    # Permissions were checked in order to send the message,
	    # so we don't need to do any fancy stuff here.

908 909 910 911 912
	    my $cmd = "$nodereboot -r $nodelist";
	    my $redir = " 2>&1 >> /usr/testbed/log/nodereboot.log &";
	    debug("$cmd $redir\n");
	    system("date $redir");
	    system($cmd.$redir) and
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
	      notify("$params/$command: ".
		     "Command '$cmd' failed, error $?: $!\n");

	    # Set up a timeout, so we retry if we don't get SHUTDOWN in time
	    foreach $node (@nodes) {
		# Note: This will replace any state timeouts currently in
		# the queue. But here that's okay because we're expecting
		# to see another transition really soon anyway.
		setTimeout($TBCOMMAND,$command,$node,time());
	    }

	    info("Performed $command for $params\n");
	    next;
	};
	(/^$TBPOWEROFF$/ || /^$TBPOWERON$/ || /^$TBPOWERCYCLE$/) && do {
	    # For power, the params is a comma-separated list of nodes
	    my @nodes = split(",",$params);
	    my $nodelist=join(" ",@nodes);
	    my %funcmap = ( $TBPOWERCYCLE => "cycle",
			    $TBPOWERON    => "on",
			    $TBPOWEROFF   => "off");
	    my $func = $funcmap{$command};
935
	    info("Sending power $func for nodes: $nodelist\n");
936 937 938 939
	    # Permissions were checked in order to send the message,
	    # so we don't need to do any fancy stuff here.

	    my $cmd = "$power $func $nodelist &";
940 941
	    debug("$cmd\n") and
	    system($cmd) and
942 943 944 945 946 947 948 949 950 951
	      notify("$params/$command: ".
		     "Command '$cmd' failed, error $?: $!\n");

	    info("Performed $command for $params\n");
	    next;
	};
	notify("$params: Unknown Command: $command\n");
    }
}

952 953 954 955
#
# Check if we need to generate an ISUP
#
sub checkGenISUP($) {
Mac Newbold's avatar
Mac Newbold committed
956 957 958 959 960 961 962 963 964 965
    my ($node) = @_;
    debug("$node: Checking ISUP Generation\n");
    my $r = DBQueryWarn("select osfeatures from nodes as n ".
			"left join os_info as o on o.osid=n.osid ".
			"where node_id='$node' and osfeatures is not null");
    my $osfeatures="";
    # If we don't get anything back, assume it has no features.
    if ($r->num_rows() > 0) {
	($osfeatures) = $r->fetchrow();
    }
Mac Newbold's avatar
Mac Newbold committed
966

Mac Newbold's avatar
Mac Newbold committed
967 968 969 970 971 972
    my @features = split(",",$osfeatures);
    # Make sure features I care about are defined
    my %can=("ping"=>0, "isup"=>0);
    foreach my $f (@features) {
	$can{"\L$f"}=1;	# make sure it's all lowercase
    }
Mac Newbold's avatar
Mac Newbold committed
973

Mac Newbold's avatar
Mac Newbold committed
974 975 976 977 978
    # If os will send ISUP on its own, do nothing here.
    if ($can{"isup"}) {
	debug("$node: Will send own ISUP\n"); 
	return 0;
    }
Mac Newbold's avatar
Mac Newbold committed
979

Mac Newbold's avatar
Mac Newbold committed
980 981 982 983 984 985
    # If os doesn't support isup but can ping, fork and ping it every
    # few seconds and send isup when it pings, or timeout after too long.
    if ($can{"ping"}) {
	debug("$node: Needs to be pinged - calling eventping\n");
	system("$TB/sbin/eventping $node &");
	return 0;
986
    }
Mac Newbold's avatar
Mac Newbold committed
987

Mac Newbold's avatar
Mac Newbold committed
988 989 990 991 992 993 994 995
    # If os doesn't support ping or isup, stated sends ISUP just after 
    # the node gets to BOOTING (a bit early, but the best we can do)

    debug("$node: OS doesn't ping - sending ISUP\n");
    EventSendWarn(host      => $BOSSNODE ,
		  objtype   => TBDB_TBEVENT_NODESTATE ,
		  eventtype => TBDB_NODESTATE_ISUP ,
		  objname   => $node);
Robert Ricci's avatar
Robert Ricci committed
996 997
}

998 999 1000 1001
# Figure out if this node belongs to us (ie. if it's using our database.)
# Returns 1 if it does, 0 if not
sub checkDBRedirect($) {

Mac Newbold's avatar
Mac Newbold committed
1002
    my ($node) = @_;
1003

Mac Newbold's avatar
Mac Newbold committed
1004 1005 1006 1007 1008 1009 1010 1011
    # XXX: I don't want to do this every time, for performance reaons,
    # but we need to make sure that we don't get into an inconsistent 
    # state
    my $result=DBQueryFatal("SELECT testdb FROM nodes as n " .
			    "LEFT JOIN reserved as r ON n.node_id=r.node_id ".
			    "LEFT JOIN experiments as e ON r.pid = e.pid " .
			    "AND r.eid = e.eid " .
			    "WHERE n.node_id = '$node'");
1012

Mac Newbold's avatar
Mac Newbold committed
1013 1014 1015 1016
    if (!$result->num_rows()) {
	notify("Got an event for a node ($node) I don't know about\n");
	return 0;
    }
1017

Mac Newbold's avatar
Mac Newbold committed
1018
    my ($testdb) = $result->fetchrow();
1019

Mac Newbold's avatar
Mac Newbold committed
1020 1021 1022
    # XXX: It's hokey to hardcode tbdb here, but....

    #debug("checkDBRedirect: $node => $testdb (I'm $TBDBNAME)\n");
1023
    if ((!$testdb && ($TBDBNAME eq $REALTBDBNAME)) ||
Mac Newbold's avatar
Mac Newbold committed
1024 1025 1026 1027 1028
	($testdb && ($testdb eq $TBDBNAME))) {
	return 1;
    } else {
	return 0;
    }
1029 1030
}

1031 1032 1033 1034
# Check if this state has a timeout, and if so, put it in the queue
sub setTimeout( $$$$ ) {
    my ($mode,$state,$node,$now) = @_;
    if (0) { print "Original: ($mode,$state,$node,$now)\n"; qshow(); }
1035
    if (defined(qfind($node))) { qdelete($node); delete($timeout_tag{$node}); }
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
    if (0) { print "Deleted:\n"; qshow(); }
    if (defined($mode) && defined($state) &&
	defined($timeouts{$mode}) &&
	defined($timeouts{$mode}{$state})) {
	my $deadline = ${$timeouts{$mode}{$state}}[0];
        if (defined($deadline) &&
	    $deadline != $TBNOTIMEOUT) {
	    my $TO = $deadline + $now;
	    debug("Setting timeout for ($node,$mode,$state) at ".
		  "$deadline + $now ($TO)\n");
	    qinsert($TO,$node);
1047
	    $timeout_tag{$node} = "$mode:$state";
1048 1049 1050 1051 1052 1053
	    if (0) { qshow(); }
	}
    }
    if (0) { print "Done:\n"; qshow(); }
}

Robert Ricci's avatar
Robert Ricci committed
1054 1055
# Reload state from the database
sub reload() {
Mac Newbold's avatar
Mac Newbold committed
1056 1057 1058 1059 1060 1061
    debug("Reloading state from database\n");
    $last_reload = time();
    %timeouts  = getTimeouts();
    %valid     = getValid();
    %modeTrans = getModeTrans();
    %triggers  = getTriggers();
1062
    %nodes     = readStates(%nodes);
Robert Ricci's avatar
Robert Ricci committed
1063 1064
}

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094