Add "hostkeys" reporting command and make it part of the watchdog

tmcd/common/libtmcc.pm

@@ -26,7 +26,7 @@ use Exporter;
 	     TMCCCMD_PLABCONFIG TMCCCMD_SUBCONFIG TMCCCMD_LINKDELAYS
 	     TMCCCMD_PROGRAMS TMCCCMD_SYNCSERVER TMCCCMD_KEYHASH TMCCCMD_NODEID
 	     TMCCCMD_NTPINFO TMCCCMD_NTPDRIFT TMCCCMD_EVENTKEY TMCCCMD_ROUTELIST
-	     TMCCCMD_ROLE TMCCCMD_RUSAGE TMCCCMD_WATCHDOGINFO
+	     TMCCCMD_ROLE TMCCCMD_RUSAGE TMCCCMD_WATCHDOGINFO TMCCCMD_HOSTKEYS
 	     );
 
 # Must come after package declaration!
@@ -148,6 +148,7 @@ my %commandset =
       "role"		=> {TAG => "role"},
       "rusage"		=> {TAG => "rusage"},
       "watchdoginfo"	=> {TAG => "watchdoginfo"},
+      "hostkeys"	=> {TAG => "hostkeys"},
     );
 
 #
@@ -190,6 +191,7 @@ sub TMCCCMD_ROUTELIST()	{ $commandset{"routelist"}->{TAG}; }
 sub TMCCCMD_ROLE()	{ $commandset{"role"}->{TAG}; }
 sub TMCCCMD_RUSAGE()	{ $commandset{"rusage"}->{TAG}; }
 sub TMCCCMD_WATCHDOGINFO(){ $commandset{"watchdoginfo"}->{TAG}; }
+sub TMCCCMD_HOSTKEYS()  { $commandset{"hostkeys"}->{TAG}; }
 
 #
 # Caller uses this routine to set configuration of this library

tmcd/common/watchdog

@@ -48,6 +48,8 @@ my $action	= "start";
 my $logname	= "$LOGDIR/emulab-watchdog.log";
 my $pidfile	= "/var/run/emulab-watchdog.pid";
 my $rusagebin	= "$BINDIR/plabrusage";
+my $keydir	= "/etc/ssh";
+my @keylist     = ("ssh_host_key", "ssh_host_dsa_key", "ssh_host_rsa_key");
 my $debug	= 0;
 my $verbose     = 0;
 my $updatefailed= 0;
@@ -69,12 +71,15 @@ my %faketimes;
 # Default interval values in seconds.
 # Compatible with old, static watchdog.
 #
+# yeah yeah, all these hashes should be a hash of records or something.
+#
 my %iv = (
     check   => 0,
     isalive => ((REMOTE() == 1) ? (PLAB() ? 600 : 60) : (JAILED() ? 600 : 180)),
     drift   => (60 * 60 * 12),
     cvsup   => (60 * 60 * 12),
     rusage  => 0,
+    hkeys   => 0,
 );
 
 my %funcs = (
@@ -83,6 +88,16 @@ my %funcs = (
     drift   => \&ntpdrift,
     cvsup   => \&runcvsup,
     rusage  => \&sendrusage,
+    hkeys   => \&sendhkeys,     
+);
+
+my %immediate = (
+    check   => 0,
+    isalive => 1,
+    drift   => 0,
+    cvsup   => 0,
+    rsuage  => 1,
+    hkeys   => 1,	 
 );
 
 #
@@ -93,6 +108,7 @@ sub sendisalive($);
 sub ntpdrift($);
 sub runcvsup($);
 sub sendrusage($);
+sub sendhkeys($);
 sub logmsg($);
 
 #
@@ -220,6 +236,7 @@ $retry{check} = 0;
 $retry{isalive} = 0;
 $retry{drift} = 0;
 $retry{rusage} = 0;
+$retry{hkeys} = 0;
 
 $curtime = time();
 if ($fakeit) {
@@ -396,9 +413,10 @@ sub setintervals($)
 	$oiv{drift} = $iv{drift};
 	$oiv{cvsup} = $iv{cvsup};
 	$oiv{rusage} = $iv{rusage};
+	$oiv{hkeys} = $iv{hkeys};
 
 	if ($tmccresults[0] =~
-	    /INTERVAL=(-?\d+) ISALIVE=(-?\d+) NTPDRIFT=(-?\d+) CVSUP=(-?\d+) RUSAGE=(-?\d+)/) {
+	    /INTERVAL=(-?\d+) ISALIVE=(-?\d+) NTPDRIFT=(-?\d+) CVSUP=(-?\d+) RUSAGE=(-?\d+) HOSTKEYS=(-?\d+)/) {
 	    $iv{check} = $1
 		if ($1 >= 0);
 	    $iv{isalive} = $2
@@ -409,6 +427,8 @@ sub setintervals($)
 		if ($4 >= 0);
 	    $iv{rusage} = $5
 		if ($5 >= 0);
+	    $iv{hkeys} = $6
+		if ($6 >= 0);
 	}
 
 	#
@@ -418,6 +438,7 @@ sub setintervals($)
 	    $iv{drift} = 0;
 	    $iv{cvsup} = 0;
 	    $iv{rusage} = 0;
+	    $iv{hkeys} = 0;
 	}
 
 	foreach my $key (keys %iv) {
@@ -444,11 +465,10 @@ sub setintervals($)
 		} elsif ($firsttime || $oiv{$key} == 0) {
 		    logmsg("setintervals: scheduling $key\n");
 		    #
-		    # XXX special firsttime hack
-		    # isalive and rusage must report immediately
+		    # Some commands need to be run at boottime,
+		    # schedule their first run immediately.
 		    #
-		    if ($firsttime &&
-			($key eq "isalive" || $key eq "rusage")) {
+		    if ($firsttime && $immediate{$key}) {
 			qinsert($curtime, $funcs{$key});
 		    } else {
 			qinsert($curtime + $iv{$key}, $funcs{$key});
@@ -480,7 +500,8 @@ sub setintervals($)
 	
     if ($report) {
 	logmsg("setintervals: check=$iv{check}, isalive=$iv{isalive}, ".
-	       "drift=$iv{drift}, cvsup=$iv{cvsup}, rusage=$iv{rusage}\n");
+	       "drift=$iv{drift}, cvsup=$iv{cvsup}, rusage=$iv{rusage}, ".
+	       "hostkeys=$iv{hkeys}\n");
     }
 
     #
@@ -643,6 +664,87 @@ resched:
 	if ($iv{rusage});
 }
 
+sub sendhkeys($)
+{
+    my ($curtime) = @_;
+    
+    if ($fakeit) {
+	my $delta = $curtime - $faketimes{hkeys};
+	$faketimes{hkeys} = $curtime;
+	logmsg("sendhkeys at +$delta\n");
+	qinsert($curtime + $iv{hkeys}, \&sendhkeys) if ($iv{hkeys});
+	return;
+    }
+
+    if ($verbose) {
+	if ($retry{hkeys} == 0) {
+	    logmsg("hostkeys: sending\n");
+	} else {
+	    logmsg("hostkeys: resending, retry=$retry{hkeys}\n");
+	}
+    }
+
+    if (! -d $keydir) {
+	logmsg("hostkeys: no SSH key directory $keydir\n");
+	$iv{hkeys} = 0;
+	return;
+    }
+
+    my $arg="";
+    foreach my $kname (@keylist) {
+	my $kpath = "$keydir/$kname.pub";
+	next
+	    if (! -r $kpath);
+
+	my $key = `cat $kpath`;
+	logmsg("hostkeys: could not read keyfile $kpath\n")
+	    if ($?);
+	chomp($key);
+	$kname =~ tr/a-z/A-Z/;
+
+	$arg .= "$kname='$key' ";
+    }
+
+    if ($arg ne "") {
+	print "nodeinfo: keys: $arg\n"
+	    if ($debug);
+
+	my %tmccargs = ();
+	$tmccargs{timeout} = 3;
+	# send these with TCP for now
+	#$tmccargs{useudp} = 1 if (!$trytcp || $retry{hkeys} != $maxretries);
+
+	if (tmcc(TMCCCMD_HOSTKEYS, $arg, undef, %tmccargs) != 0) {
+	    #
+	    # Failed, schedule a retry using a backoff.
+	    #
+	    if ($retry{hkeys} < $maxretries) {
+		my $nexttime = time() + (1 << $retry{hkeys});
+		qinsert($nexttime, \&sendhkeys);
+		$retry{hkeys}++;
+		logmsg("hostkeys: failed ($?), retry $retry{hkeys}\n");
+		return;
+	    }
+	    #
+	    # Failed miserably, just whine and reschedule at the normal time.
+	    #
+	    logmsg("hostkeys: failed ($?) after $maxretries attempts\n");
+	} else {
+	    logmsg("hostkeys: succeeded after $retry{hkeys} retries\n")
+		if ($retry{hkeys});
+	}
+    }
+
+    #
+    # Set up for another interval.
+    # Since the tmcc call and update can take awhile, we update curtime
+    #
+    $retry{hkeys} = 0;
+    $curtime = time();
+    qinsert($curtime + $iv{hkeys}, \&sendhkeys)
+	if ($iv{hkeys});
+}
+
 sub logmsg($)
 {
     my ($msg) = @_;