Commit 643b4276 authored by Mike Hibler's avatar Mike Hibler
Browse files

Add "hostkeys" reporting command and make it part of the watchdog

parent 6bda508d
......@@ -26,7 +26,7 @@ use Exporter;
TMCCCMD_PLABCONFIG TMCCCMD_SUBCONFIG TMCCCMD_LINKDELAYS
TMCCCMD_PROGRAMS TMCCCMD_SYNCSERVER TMCCCMD_KEYHASH TMCCCMD_NODEID
TMCCCMD_NTPINFO TMCCCMD_NTPDRIFT TMCCCMD_EVENTKEY TMCCCMD_ROUTELIST
TMCCCMD_ROLE TMCCCMD_RUSAGE TMCCCMD_WATCHDOGINFO
TMCCCMD_ROLE TMCCCMD_RUSAGE TMCCCMD_WATCHDOGINFO TMCCCMD_HOSTKEYS
);
# Must come after package declaration!
......@@ -148,6 +148,7 @@ my %commandset =
"role" => {TAG => "role"},
"rusage" => {TAG => "rusage"},
"watchdoginfo" => {TAG => "watchdoginfo"},
"hostkeys" => {TAG => "hostkeys"},
);
#
......@@ -190,6 +191,7 @@ sub TMCCCMD_ROUTELIST() { $commandset{"routelist"}->{TAG}; }
sub TMCCCMD_ROLE() { $commandset{"role"}->{TAG}; }
sub TMCCCMD_RUSAGE() { $commandset{"rusage"}->{TAG}; }
sub TMCCCMD_WATCHDOGINFO(){ $commandset{"watchdoginfo"}->{TAG}; }
sub TMCCCMD_HOSTKEYS() { $commandset{"hostkeys"}->{TAG}; }
#
# Caller uses this routine to set configuration of this library
......
......@@ -48,6 +48,8 @@ my $action = "start";
my $logname = "$LOGDIR/emulab-watchdog.log";
my $pidfile = "/var/run/emulab-watchdog.pid";
my $rusagebin = "$BINDIR/plabrusage";
my $keydir = "/etc/ssh";
my @keylist = ("ssh_host_key", "ssh_host_dsa_key", "ssh_host_rsa_key");
my $debug = 0;
my $verbose = 0;
my $updatefailed= 0;
......@@ -69,12 +71,15 @@ my %faketimes;
# Default interval values in seconds.
# Compatible with old, static watchdog.
#
# yeah yeah, all these hashes should be a hash of records or something.
#
my %iv = (
check => 0,
isalive => ((REMOTE() == 1) ? (PLAB() ? 600 : 60) : (JAILED() ? 600 : 180)),
drift => (60 * 60 * 12),
cvsup => (60 * 60 * 12),
rusage => 0,
hkeys => 0,
);
my %funcs = (
......@@ -83,6 +88,16 @@ my %funcs = (
drift => \&ntpdrift,
cvsup => \&runcvsup,
rusage => \&sendrusage,
hkeys => \&sendhkeys,
);
my %immediate = (
check => 0,
isalive => 1,
drift => 0,
cvsup => 0,
rsuage => 1,
hkeys => 1,
);
#
......@@ -93,6 +108,7 @@ sub sendisalive($);
sub ntpdrift($);
sub runcvsup($);
sub sendrusage($);
sub sendhkeys($);
sub logmsg($);
#
......@@ -220,6 +236,7 @@ $retry{check} = 0;
$retry{isalive} = 0;
$retry{drift} = 0;
$retry{rusage} = 0;
$retry{hkeys} = 0;
$curtime = time();
if ($fakeit) {
......@@ -396,9 +413,10 @@ sub setintervals($)
$oiv{drift} = $iv{drift};
$oiv{cvsup} = $iv{cvsup};
$oiv{rusage} = $iv{rusage};
$oiv{hkeys} = $iv{hkeys};
if ($tmccresults[0] =~
/INTERVAL=(-?\d+) ISALIVE=(-?\d+) NTPDRIFT=(-?\d+) CVSUP=(-?\d+) RUSAGE=(-?\d+)/) {
/INTERVAL=(-?\d+) ISALIVE=(-?\d+) NTPDRIFT=(-?\d+) CVSUP=(-?\d+) RUSAGE=(-?\d+) HOSTKEYS=(-?\d+)/) {
$iv{check} = $1
if ($1 >= 0);
$iv{isalive} = $2
......@@ -409,6 +427,8 @@ sub setintervals($)
if ($4 >= 0);
$iv{rusage} = $5
if ($5 >= 0);
$iv{hkeys} = $6
if ($6 >= 0);
}
#
......@@ -418,6 +438,7 @@ sub setintervals($)
$iv{drift} = 0;
$iv{cvsup} = 0;
$iv{rusage} = 0;
$iv{hkeys} = 0;
}
foreach my $key (keys %iv) {
......@@ -444,11 +465,10 @@ sub setintervals($)
} elsif ($firsttime || $oiv{$key} == 0) {
logmsg("setintervals: scheduling $key\n");
#
# XXX special firsttime hack
# isalive and rusage must report immediately
# Some commands need to be run at boottime,
# schedule their first run immediately.
#
if ($firsttime &&
($key eq "isalive" || $key eq "rusage")) {
if ($firsttime && $immediate{$key}) {
qinsert($curtime, $funcs{$key});
} else {
qinsert($curtime + $iv{$key}, $funcs{$key});
......@@ -480,7 +500,8 @@ sub setintervals($)
if ($report) {
logmsg("setintervals: check=$iv{check}, isalive=$iv{isalive}, ".
"drift=$iv{drift}, cvsup=$iv{cvsup}, rusage=$iv{rusage}\n");
"drift=$iv{drift}, cvsup=$iv{cvsup}, rusage=$iv{rusage}, ".
"hostkeys=$iv{hkeys}\n");
}
#
......@@ -643,6 +664,87 @@ resched:
if ($iv{rusage});
}
sub sendhkeys($)
{
my ($curtime) = @_;
if ($fakeit) {
my $delta = $curtime - $faketimes{hkeys};
$faketimes{hkeys} = $curtime;
logmsg("sendhkeys at +$delta\n");
qinsert($curtime + $iv{hkeys}, \&sendhkeys) if ($iv{hkeys});
return;
}
if ($verbose) {
if ($retry{hkeys} == 0) {
logmsg("hostkeys: sending\n");
} else {
logmsg("hostkeys: resending, retry=$retry{hkeys}\n");
}
}
if (! -d $keydir) {
logmsg("hostkeys: no SSH key directory $keydir\n");
$iv{hkeys} = 0;
return;
}
my $arg="";
foreach my $kname (@keylist) {
my $kpath = "$keydir/$kname.pub";
next
if (! -r $kpath);
my $key = `cat $kpath`;
logmsg("hostkeys: could not read keyfile $kpath\n")
if ($?);
chomp($key);
$kname =~ tr/a-z/A-Z/;
$arg .= "$kname='$key' ";
}
if ($arg ne "") {
print "nodeinfo: keys: $arg\n"
if ($debug);
my %tmccargs = ();
$tmccargs{timeout} = 3;
# send these with TCP for now
#$tmccargs{useudp} = 1 if (!$trytcp || $retry{hkeys} != $maxretries);
if (tmcc(TMCCCMD_HOSTKEYS, $arg, undef, %tmccargs) != 0) {
#
# Failed, schedule a retry using a backoff.
#
if ($retry{hkeys} < $maxretries) {
my $nexttime = time() + (1 << $retry{hkeys});
qinsert($nexttime, \&sendhkeys);
$retry{hkeys}++;
logmsg("hostkeys: failed ($?), retry $retry{hkeys}\n");
return;
}
#
# Failed miserably, just whine and reschedule at the normal time.
#
logmsg("hostkeys: failed ($?) after $maxretries attempts\n");
} else {
logmsg("hostkeys: succeeded after $retry{hkeys} retries\n")
if ($retry{hkeys});
}
}
#
# Set up for another interval.
# Since the tmcc call and update can take awhile, we update curtime
#
$retry{hkeys} = 0;
$curtime = time();
qinsert($curtime + $iv{hkeys}, \&sendhkeys)
if ($iv{hkeys});
}
sub logmsg($)
{
my ($msg) = @_;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment