Commit 5d7164b3 authored by Leigh Stoller's avatar Leigh Stoller

Attempt to address the problem described in issue #166; that nodes fail

to go from PXEBOOTING (pxewakeup) to actually booting, but we do not
know that for a really long time cause we send a BOOTING event from
bootinfo right after PXEBOOTING, since that was the only place to hook
it in. Well Mike discovered the "on commit" support in dhcpd, and so
that is what we are going to use now. Note that uboot nodes have been
using on commit, now all nodes will when BOOTINFO_EVENTS=0.

Mike's reportboot program is now a daemon, renamed to report_daemon.
The original reportboot program is a little script that writes the
arguments from dhcpd to a unix socket to be picked up by the daemon,
which does the original work of mapping the IP/Mac to a node id and
sending an event. The code has also been modified to run on a subboss
using the same node mapping given to to dhcpd, reconstituted as DBM
file by subboss_dhcpd_makeconf.

The reason for using a daemon this way is so that we do not hang up
dhcpd in case we cannot get to the event system. The unix domain
socket will give us some amount of buffering, but I suspect that any
event problem will eat that space up quickly, and I will be back to
revisit this (probably want reportboot to not block on its write
to the socket).

pxeboot changed to not send PXEBOOTING or BOOTING when
BOOTINFO_EVENTS=0.
parent fb247a24
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -80,6 +80,8 @@ endif
#
all: $(TARGETS)
subboss: $(SUBBOSS_SBIN_SCRIPTS)
$(SUBBOSS_SBIN_SCRIPTS):
include $(TESTBED_SRCDIR)/GNUmakerules
......
#!/usr/bin/perl -wT
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -48,6 +48,7 @@ my $restart = 0;
my $TBOPS = "@TBOPSEMAIL@";
my $CBINDIR = "@CLIENT_BINDIR@";
my $CLOGDIR = "@CLIENT_VARDIR@/logs";
my $CDBDIR = "@CLIENT_VARDIR@/db";
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/sbin:/usr/local/bin';
......@@ -71,6 +72,7 @@ my $PXELINUXCONFIG = "pxelinux.cfg";
# XXX hack for now to support gpxelinux as well
my $GPXELINUXPREFIX = "$PXEBOOTDIR/gpxelinux";
my $GPXELINUXBOOT = "gpxelinux.0";
my $NODEMAPDB = "$CDBDIR/nodemap";
sub SetupPXEBoot($$$$);
sub ClearPXEBoot($$);
......@@ -175,6 +177,25 @@ sub gendhcpdconf($$)
push @nodes, $node;
}
#
# Generate a new node mapping DBM file, that maps IP to node_id.
# This is used by reportboot.
#
if (!dbmopen(%DB, $NODEMAPDB, 0660)) {
warn("Could not open $NODEMAPDB");
return 1;
}
for my $row (@nodes) {
my $ip = $$row{"IP"};
my $mac = $$row{"MAC"};
my $key = "$ip/$mac";
my $hostname = $$row{"HOSTNAME"};
$DB{$key} = $hostname;
$DB{$hostname} = $key;
}
dbmclose(%DB);
if (!open(OF, ">$tmpfile")) {
warn("Could not open $tmpfile\n");
return 1;
......
......@@ -58,29 +58,19 @@ class "PXE" {
}
on commit {
set clip = binary-to-ascii(10, 8, ".", leased-address);
set clhw = binary-to-ascii(16, 8, ":", substring(hardware, 1, 6));
if (substring (option vendor-class-identifier, 0, 9) = "PXEClient") {
set boot = "PXE";
} elsif (substring (option vendor-class-identifier, 0, 6) = "U-boot") {
set boot = "UBOOT";
} else {
set boot = "OS";
}
## If you want to send an event on both PXE and OS contacts, use this
#execute("@prefix@/sbin/reportboot", "-bD", clip, clhw, boot);
if (@BOOTINFO_EVENTS@ = 0) {
if (substring (option vendor-class-identifier, 0, 9) = "PXEClient") {
set boot = "PXE";
} elsif (substring (option vendor-class-identifier, 0, 6) = "U-boot") {
set boot = "PXE";
} else {
set boot = "OS";
}
set clip = binary-to-ascii(10, 8, ".", leased-address);
set clhw = binary-to-ascii(16, 8, ":", substring(hardware, 1, 6));
## If you want to send both events on the PXE or U-boot contact, use this
#if (boot != "OS") {
# execute("@prefix@/sbin/reportboot", "-bDC", clip, clhw, boot);
#}
# If you want to send both events on just U-boot contact, use this
if (boot = "UBOOT") {
execute("@prefix@/sbin/reportboot", "-bDU", clip, clhw, boot);
execute("@prefix@/sbin/reportboot", clip, clhw, boot);
}
}
@DHCPD_CONTROLNET_DECL@
......
......@@ -58,6 +58,22 @@ class "PXE" {
option PXE.emulab-bootinfo @BOSSNODE_IP@;
}
on commit {
if (@BOOTINFO_EVENTS@ = 0) {
if (substring (option vendor-class-identifier, 0, 9) = "PXEClient") {
set boot = "PXE";
} elsif (substring (option vendor-class-identifier, 0, 6) = "U-boot") {
set boot = "PXE";
} else {
set boot = "OS";
}
set clip = binary-to-ascii(10, 8, ".", leased-address);
set clhw = binary-to-ascii(16, 8, ":", substring(hardware, 1, 6));
execute("@prefix@/sbin/reportboot", clip, clhw, boot);
}
}
shared-network emulab {
subnet @CONTROL_NETWORK@ netmask @CONTROL_NETMASK@ {
......
......@@ -57,6 +57,22 @@ class "PXE" {
option PXE.discovery-control 8;
}
on commit {
if (@BOOTINFO_EVENTS@ = 0) {
if (substring (option vendor-class-identifier, 0, 9) = "PXEClient") {
set boot = "PXE";
} elsif (substring (option vendor-class-identifier, 0, 6) = "U-boot") {
set boot = "PXE";
} else {
set boot = "OS";
}
set clip = binary-to-ascii(10, 8, ".", leased-address);
set clhw = binary-to-ascii(16, 8, ":", substring(hardware, 1, 6));
execute("@prefix@/sbin/reportboot", clip, clhw, boot);
}
}
@DHCPD_CONTROLNET_DECL@
shared-network emulab {
......
......@@ -26,6 +26,8 @@ TESTBED_SRCDIR = @top_srcdir@
OBJDIR = ..
SUBDIR = pxe
EVENTSYS = @EVENTSYS@
# This needs a rename.
BOOTINFO_PXEEVENTS = @BOOTINFO_EVENTS@
include $(OBJDIR)/Makeconf
......@@ -66,6 +68,9 @@ BI_DBOBJ += event-support.o
CFLAGS += -DEVENTSYS
LFLAGS += $(TESTBED_LIBOBJDIR)/event/libevent.a
LFLAGS += -L/usr/local/lib -lpubsub -lcrypto
ifeq ($(BOOTINFO_PXEEVENTS),1)
CFLAGS += -DBOOTINFO_PXEEVENTS
endif
endif
bootinfo: main.o bootinfo.o bootinfo.h bootinfo_version.o \
......
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2016 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -100,7 +100,7 @@ bootinfo(struct in_addr ipaddr, char *node_id, struct boot_info *boot_info,
case BIOPCODE_BOOTWHAT_KEYED_REQUEST:
info("%s: KEYED REQUEST (key=[%s], vers %d)\n",
inet_ntoa(ipaddr), boot_info->data, boot_info->version);
#ifdef EVENTSYS
#if defined(EVENTSYS) && defined(BOOTINFO_PXEEVENTS)
needevent = bicache_needevent(ipaddr);
if (!no_event_send && needevent &&
bievent_send(ipaddr, opaque, TBDB_NODESTATE_PXEBOOTING)) {
......@@ -116,7 +116,7 @@ bootinfo(struct in_addr ipaddr, char *node_id, struct boot_info *boot_info,
case BIOPCODE_BOOTWHAT_INFO:
info("%s: REQUEST (vers %d)\n",
inet_ntoa(ipaddr), boot_info->version);
#ifdef EVENTSYS
#if defined(EVENTSYS) && defined(BOOTINFO_PXEEVENTS)
needevent = bicache_needevent(ipaddr);
if (!no_event_send && needevent &&
bievent_send(ipaddr, opaque, TBDB_NODESTATE_PXEBOOTING)) {
......@@ -156,9 +156,11 @@ bootinfo(struct in_addr ipaddr, char *node_id, struct boot_info *boot_info,
info("%s: retry failed PXEBOOTING event\n",
inet_ntoa(ipaddr));
bicache_needevent(ipaddr);
#if defined(BOOTINFO_PXEEVENTS)
if (bievent_send(ipaddr, opaque,
TBDB_NODESTATE_PXEBOOTING))
bicache_clearevent(ipaddr);
#endif
}
switch (boot_whatp->type) {
case BIBOOTWHAT_TYPE_PART:
......@@ -166,8 +168,10 @@ bootinfo(struct in_addr ipaddr, char *node_id, struct boot_info *boot_info,
case BIBOOTWHAT_TYPE_SYSID:
case BIBOOTWHAT_TYPE_MB:
case BIBOOTWHAT_TYPE_MFS:
#if defined(BOOTINFO_PXEEVENTS)
bievent_send(ipaddr, opaque,
TBDB_NODESTATE_BOOTING);
#endif
break;
case BIBOOTWHAT_TYPE_WAIT:
......
......@@ -34,6 +34,7 @@ SYSTEM := $(shell uname -s)
include $(OBJDIR)/Makeconf
SUBBOSS_SBIN_SCRIPTS = reportboot report_boot_daemon
SUBDIRS = checkpass ns2ir nseparse checkup template_cvsroot \
snmpit
ifeq ($(NSVERIFY),1)
......@@ -65,7 +66,7 @@ SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
nfstrace plabinelab smbpasswd_setup smbpasswd_setup.proxy \
rmproj snmpit.proxynew snmpit.proxyv2 pool_daemon \
checknodes_daemon snmpit.proxyv3 image_setup tcpp \
arplockdown bscontrol reportboot \
arplockdown bscontrol reportboot report_boot_daemon \
nfsmfs_setup nfsmfs_setup.proxy
ifeq ($(ISMAINSITE),1)
......@@ -154,6 +155,10 @@ COMPILED_TARGETS =
#
all: $(TARGETS) $(addprefix compiled/, $(COMPILED_TARGETS))
subboss: $(SUBBOSS_SBIN_SCRIPTS)
$(SUBBOSS_SBIN_SCRIPTS):
include $(TESTBED_SRCDIR)/GNUmakerules
CXXFLAGS += -Wall -O2 -g
......@@ -202,6 +207,10 @@ endif
boss-install: all script-install subdir-install
@echo "Don't forget to do a post-install as root"
subboss-install: $(addprefix $(INSTALL_SBINDIR)/, $(SUBBOSS_SBIN_SCRIPTS))
ln -sf $(INSTALL_SBINDIR)/reportboot \
$(DESTDIR)$(CLIENT_BINDIR)/reportboot
#
# Only install the planetlab support if enabled in the defs file
#
......
......@@ -22,9 +22,7 @@
# }}}
#
use English;
use Getopt::Std;
use POSIX qw(strftime);
use Sys::Syslog;
use Socket;
#
# This script is invoked on a dhcpd "commit" event.
......@@ -34,280 +32,46 @@ use Sys::Syslog;
sub usage()
{
print "Usage: $0 [-bdCD] IP MAC boot-type\n";
print("Options:\n");
print(" -b - Run in the background\n");
print(" -d - Turn on debugging\n");
print(" -C - Combine PXEBOOTING/BOOTING on PXE or U-boot boots;\n");
print(" Send nothing on an OS boot.");
print(" -D - Record event times in a DB to moderate send rate\n");
print(" -U - Combine PXEBOOTING/BOOTING on U-boot boots only;\n");
print(" Send nothing on an PXE or OS boot.");
print "Usage: $0 IP MAC boot-type\n";
print("boot-type is one of OS, PXE, UBOOT.\n");
exit(1);
}
my $optlist = "bdCDU";
my $background = 0;
my $debug = 0;
my $combined = 0;
my $ubootonly = 0;
my $usedb = 0;
#
# Functions
#
sub findnode($$);
sub logit($);
sub sendevent($$);
sub fatal($);
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $EVPORT = @BOSSEVENTPORT@;
my $EVSERVER = "boss";
my $EVDB = "$TB/db/reportboot";
#
# Minimum time between events.
# In bootinfo, this is 10 seconds!
#
my $EVMININT = 5;
# Locals
my $logfile = "$TB/log/reportboot.log";
my $SOCKETFILE = "/var/run/reportboot.sock";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libtestbed;
use libdb;
use libtblog;
use event;
use Interface;
if ($UID != 0) {
die("Must be root to run this script\n");
}
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"C"})) {
$combined = 1;
$ubootonly = 0;
}
if (defined($options{"U"})) {
$combined = 1;
$ubootonly = 1;
}
if (defined($options{"D"})) {
$usedb = 1;
}
if (defined($options{"b"})) {
$background = 1;
}
if (defined($options{"d"})) {
$debug = 1;
fatal("Must be root to run this script");
}
if (@ARGV != 3) {
usage();
}
my ($IP,$MAC,$boottype);
if ($ARGV[0] =~ /^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$/) {
$IP = $1;
} else {
print STDERR "'$ARGV[0]' is not an IP address\n";
usage();
}
$MAC = lc($ARGV[1]);
if ($MAC =~ /^([0-9a-f]+):([0-9a-f]+):([0-9a-f]+):([0-9a-f]+):([0-9a-f]+):([0-9a-f]+)$/) {
# dhcpd will return an octet like "02" as "2", so we have to compensate
$MAC = sprintf("%02x%02x%02x%02x%02x%02x",
hex($1), hex($2), hex($3), hex($4), hex($5), hex($6));
} else {
print STDERR "'$ARGV[1]' is not a MAC address\n";
usage();
}
if ($ARGV[2] =~ /^(PXE|OS|UBOOT)$/) {
$boottype = $1;
} else {
print STDERR "Invalid boot-type '$ARGV[2]'\n";
usage();
}
my ($ip,$mac,$which) = @ARGV;
if ($background) {
my $cpid = fork();
if (!defined($cpid)) {
die "$0: could not fork!";
}
if ($cpid) {
exit(0);
}
open(STDIN, "< /dev/null") or
die("opening /dev/null for STDIN: $!");
POSIX::setsid() or
die("setsid failed: $!");
if (! -e $SOCKETFILE) {
fatal("$SOCKETFILE does not exist!");
}
my $nodeid = findnode($IP, $MAC);
if (!$nodeid) {
logit("Ignoring unknown node $IP");
exit(0);
}
#
# See if we need to send events.
# We won't send more than one event per second.
#
if ($usedb) {
my %DB;
my $lasttime = 0;
my $now = time();
my $key = "$IP/$boottype";
if (!dbmopen(%DB, $EVDB, 0660)) {
logit("$nodeid: could not open $EVDB");
exit(1);
}
my $needone = 1;
if (defined($DB{$key})) {
$lasttime = $DB{$key};
# XXX watch for time going backward
if ($now >= $lasttime &&
$now < ($lasttime + $EVMININT)) {
$needone = 0;
}
}
if ($needone) {
$DB{$key} = $now;
}
dbmclose(%DB);
if (!$needone) {
logit("$nodeid: NOT sending BOOTING event (too soon)");
exit(0);
}
}
#
# Combine reporting of PXEBOOTING and BOOTING on the PXE event, reporting
# nothing on the OS event. Use this if you have problems with later
# OS-generated events happening before the BOOTING gets reported.
# Note that this is essentially the same as what bootinfo does.
#
if ($combined) {
if ($boottype eq "UBOOT" || ($boottype eq "PXE" && !$ubootonly)) {
if (sendevent($nodeid, "PXEBOOTING") ||
sendevent($nodeid, "BOOTING")) {
logit("$nodeid: could not send PXEBOOTING/BOOTING events");
exit(1);
}
logit("$nodeid: $boottype: sent PXEBOOTING and BOOTING events");
} else {
logit("$nodeid: $boottype: NOT sending BOOTING event (combo mode)");
}
} else {
my $event = ($boottype eq "OS") ? "BOOTING" : "PXEBOOTING";
if (sendevent($nodeid, $event)) {
logit("$nodeid: $boottype: could not send $event event");
exit(1);
}
logit("$nodeid: $boottype: sent $event event");
}
socket(SOCK, PF_UNIX, SOCK_STREAM, 0)
or fatal("Could not create socket");
connect(SOCK, sockaddr_un($SOCKETFILE))
or fatal("Could not connect socket");
print SOCK "$ip,$mac,$which\n";
close(SOCK);
exit(0);
sub findnode($$)
{
my ($IP,$mac) = @_;
my $iface = Interface->LookupByIP($IP);
return undef
if (!$iface);
# XXX this should never happen since dhcpd looked up the IP by mac.
if ($iface->mac() ne $mac) {
logit("$IP: came in on $mac but expected " . $iface->mac());
return undef;
}
return $iface->node_id();
}
sub logit($)
{
my $message = shift;
# Time stamp log messages like:
# Sep 20 09:36:00 $message
my $tstamp = strftime("%b %e %H:%M:%S", localtime);
if (open(LOG, ">>$logfile")) {
print LOG "$tstamp: $message\n";
close(LOG);
} else {
print STDERR "Could not append to $logfile\n";
}
print STDERR "$message\n" if ($debug);
}
sub sendevent($$)
sub fatal($)
{
my ($node,$event) = @_;
my $URL = "elvin://$EVSERVER:$EVPORT";
# Connect to the event system, and subscribe the the events we want
my $handle = event_register($URL, 0);
if (!$handle) {
logit("$node: unable to register with event system");
return 1;
}
my $tuple = address_tuple_alloc();
if (!$tuple) {
logit("$node: could not allocate an address tuple");
return 1;
}
%$tuple = ( objtype => "TBNODESTATE",
objname => $node,
eventtype => $event,
host => "boss");
my $notification = event_notification_alloc($handle, $tuple);
if (!$notification) {
logit("$node: could not allocate notification");
return 1;
}
if (!event_notify($handle, $notification)) {
logit("$node: could not send $event notification");
return 1;
}
event_notification_free($handle, $notification);
if (event_unregister($handle) == 0) {
logit("$node: WARNING: could not unregister with event system");
}
return 0;
my ($msg) = @_;
die("*** $0:\n".
" $msg\n");
}
#!/usr/bin/perl -w
#
# Copyright (c) 2014-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use English;
use Getopt::Std;
use IO::Socket::UNIX;
use POSIX qw(strftime);
use Carp;
#
# This script is invoked on a dhcpd "commit" event.
#
# Intended to trigger Emulab node boot related activities.
#
sub usage()
{
print "Usage: $0 [-dUDC]\n";
print("Options:\n");
print(" -d - Turn on debugging\n");
print(" -C - Combine PXEBOOTING/BOOTING on PXE or U-boot boots;\n");
print(" Send nothing on an OS boot.");
print(" -D - Record event times in a DB to moderate send rate\n");
print(" -U - Combine PXEBOOTING/BOOTING on U-boot boots only;\n");
print(" Send nothing on an PXE or OS boot.");
print("boot-type is one of OS, PXE, UBOOT.\n");
exit(1);
}
my $optlist = "dCDUn";
my $debug = 0;
my $impotent = 0;
my $combined = 0;
my $ubootonly = 0;
my $issubboss = 0;
my $usedb = 0;
#
# Functions
#
sub report($$$);
sub findnode($$);
sub logit($);
sub sendevent($$);
sub fatal($);
#
# Configure variables
#
my $TBOPS = "@TBOPSEMAIL@";
my $EVSERVER = "boss";
my $EVDB = "reportboot";
my $NODEMAPDB = "nodemap";
my $LOGFILE = "reportboot.log";
my $CLIENT_VARDIR = "@CLIENT_VARDIR@";
my $SOCKETFILE = "/var/run/reportboot.sock";
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
# Protos
sub fatal($);
#
# Minimum time between events.
# In bootinfo, this is 10 seconds!
#
my $EVMININT = 5;
#
# Turn off line buffering on output
#
$| = 1;
#
# We run this on boss *and* subboss so its config is a little cumbersome.
#
if (-e "/etc/emulab/issubboss") {
require "/etc/emulab/paths.pm";
import emulabpaths;
require event;
import event;
require libtestbed;
import libtestbed;
$issubboss = 1;
$EVDB = "$CLIENT_VARDIR/db/$EVDB";
$NODEMAPDB = "$CLIENT_VARDIR/db/$NODEMAPDB";
$LOGFILE = "$CLIENT_VARDIR/logs/$LOGFILE";
}
else {
unshift(@INC, "@prefix@/lib");
require event;
import event;
require libtestbed;
import libtestbed;
require Interface;
$EVDB = "@prefix@/db/$EVDB";
$NODEMAPDB = "@prefix@/db/$NODEMAPDB";
$LOGFILE = "@prefix@/log/$LOGFILE";
}
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"C"})) {
$combined = 1;
$ubootonly = 0;
}
if (defined($options{"U"})) {
$combined = 1;
$ubootonly = 1;
}
if (defined($options{"D"})) {
$usedb = 1;
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"n"})) {
$impotent = 1;
}
if (!$debug) {
if (CheckDaemonRunning("reportboot_daemon")) {
fatal("Not starting another reportboot daemon!");
}
# Go to ground.
if (TBBackGround($LOGFILE)) {
exit(0);
}
if (MarkDaemonRunning("reportboot_daemon")) {
fatal("Could not mark daemon as running!");
}
}
#
# Setup a signal handler for newsyslog.
#
sub handler()
{
my $SAVEEUID = $EUID;
$EUID = 0;
ReOpenLog($LOGFILE);
$EUID = $SAVEEUID;
}
$SIG{HUP} = \&handler
if (! $debug);
#
# We use a unix domain socket to talk to the client that is invoked
# by dhcpd. This provides the buffering so we do not block dhcpd.
#
unlink($SOCKETFILE)
if (-e $SOCKETFILE);
my $socket = IO::Socket::UNIX->new(Type => SOCK_STREAM(),
Local => $SOCKETFILE,
Listen => 5);
if (!$socket) {
fatal("Could not create socket $SOCKETFILE");
}
#
# Loop reading new messages from dhcpd, written with reportboot.
#
while (1)
{