Commit 49b480c6 authored by Leigh B Stoller's avatar Leigh B Stoller

Remove use of evproxy and clusterd; clustering is now built into pubsubd

via the -c and -E/-F options:

* On normal nodes, we used to use evproxy to capture experiment events
  on ops and forward them to the node local pubsubd (where all local
  agents connect to). Now we restart pubsubd with the -c and a -F option
  that says to subscribe to all events '(EXPT == "pid/eid")' at the
  parent pubsubd on ops (-c event-server).

* On shared hosts, we used to start kill pubsubd and start clusterd. Now
  we restart pubsubd with just "-c event-server" which causes it to
  subscribe to all events at the parent pubsubd on ops. This is how
  clusterd did it as well.

Another change is that instead of just kill pubsubd and starting with
new options, we now write a pubsubd.conf file into /usr/local/etc. The
rc.d startup files look for this file and source it as a typical .conf
file and pass the options to pubsubd.

I added deletion of these file to the prepare script.
parent a492e5e2
#!/usr/bin/perl -w
#
# Copyright (c) 2004-2014 University of Utah and the Flux Group.
# Copyright (c) 2004-2014, 2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -65,6 +65,8 @@ my $RCDIR = "$BINDIR/rc";
my $LOGFILE = "$LOGDIR/bootsetup.debug";
my $BOOTLOG = "/var/log/emulab.bootlog";
my %TMCCTIMEOUT = (REMOTE() ? ("timeout" => 5) : ());
my $PUBSUBCONF = "/usr/local/etc/pubsubd.conf";
my $PUBSUBEXPR = "/usr/local/etc/pubsubd.expr";
# Protos.
sub doboot();
......@@ -396,45 +398,61 @@ sub doboot()
}
#
# Start the event proxy.
# Setup pubsub for clustering and restart. In general, all agents
# on a node talk to the local pubsubd so that the number of subscriptions
# on ops is limited to the number of physical nodes.
#
# For SHAREDHOSTs we now use clusterd (no pubsubd or evproxy).
#
if (SHAREDHOST() || STORAGEHOST()) {
my $clusteringoptions;
if (SHAREDHOST() || STORAGEHOST() || SUBBOSS()) {
#
# We run pubsubd in clustering mode, asking for all notifications
# since we do not know what experiments are resident on this node.
#
my $eventserver = "event-server";
if (REMOTE()) {
$eventserver = TB_EVENTSERVER();
}
$clusteringoptions = "-c $eventserver";
}
elsif (!(REMOTE() && !REMOTEDED()) && !GENVNODE()) {
#
# Kill off pubsubd and start clusterd.
# We run pubsubd in clustering mode, asking for just notifications
# for the current experiment.
#
system("killall pubsubd");
if (-x "/usr/local/libexec/clusterd") {
print("Starting pubsub cluster daemon\n");
system("/usr/local/libexec/clusterd -s $eventserver");
if ($?) {
fatal("Error running clusterd");
}
} else {
print STDERR "WARNING: clusterd not installed, not started\n";
my $eventserver = "event-server";
if (REMOTE()) {
$eventserver = TB_EVENTSERVER();
}
$clusteringoptions = "-c $eventserver -F $PUBSUBEXPR";
open(CONF, "> $PUBSUBEXPR") or
fatal("Error opening $PUBSUBEXPR for writing");
print CONF "(EXPT == \"$pid/$eid\")\n";
close(CONF);
}
elsif (!(REMOTE() && !REMOTEDED()) && !GENVNODE()) {
my $evproxy = "$BINDIR/evproxy";
if (defined($clusteringoptions)) {
#
# We do this so that an invocation of the system startup file will
# see the right clustering options.
#
open(CONF, "> $PUBSUBCONF") or
fatal("Error opening $PUBSUBCONF for writing");
print CONF "pubsubd_clustering_flags=\"$clusteringoptions\"\n";
close(CONF);
if (-x $evproxy) {
my $eventserver = "event-server";
if (REMOTE()) {
$eventserver = TB_EVENTSERVER();
}
print("Starting event proxy daemon\n");
system("$evproxy -s $eventserver -e $pid/$eid");
#
# Kill off pubsubd and restart in cluster mode.
#
if (-x "/usr/local/libexec/pubsubd") {
system("killall pubsubd");
print("Starting pubsubd in cluster mode\n");
system("/usr/local/libexec/pubsubd $clusteringoptions");
if ($?) {
fatal("Error running $evproxy");
fatal("Error running pubsubd");
}
}
else {
print("$evproxy does not exist!\n");
} else {
print STDERR "WARNING: pubsubd not installed, not started\n";
}
}
......
......@@ -47,7 +47,10 @@ my $SFSHOSTKEY = "/etc/sfs/sfs_host_key";
my $MOUNTINFO = "/var/db/mounttab";
my $ENTROPY = "/var/db/entropy/*";
my $LOADERCONF = "/boot/loader.conf";
my $PUBSUBCONF = "/usr/local/etc/pubsub-clusterd.conf";
my $PUBSUBEXPR = "/usr/local/etc/pubsub-clusterd.expr";
#
#
# Dead wood in $BINDIR
# If you remove/move a script, list it here so that it goes away when the
......@@ -323,6 +326,16 @@ if (-x "/usr/libexec/locate.updatedb") {
system("/usr/libexec/locate.updatedb");
}
print "Removing pubsubd clustering config file\n";
if (-e $PUBSUBCONF) {
unlink($PUBSUBCONF) or
warn("*** could not remove $PUBSUBCONF\n");
}
if (-e $PUBSUBEXPR) {
unlink($PUBSUBEXPR) or
warn("*** could not remove $PUBSUBEXPR\n");
}
# Leave this print statement here; create_image depends on it.
print "prepare ran successfully!\n";
exit 0;
......
......@@ -51,6 +51,8 @@ my $VARACCTSDIR = "/var/log/sa";
my $IFTAB = "/etc/iftab";
my $ANACRON = "/usr/sbin/anacron";
my $GRUBDEF = "/etc/default/grub";
my $PUBSUBCONF = "/usr/local/etc/pubsub-clusterd.conf";
my $PUBSUBEXPR = "/usr/local/etc/pubsub-clusterd.expr";
#
# Dead wood in $BINDIR
......@@ -457,6 +459,16 @@ if (-e $SSHDCONFIG) {
if ($?);
}
print "Removing pubsubd clustering config file\n";
if (-e $PUBSUBCONF) {
unlink($PUBSUBCONF) or
warn("*** could not remove $PUBSUBCONF\n");
}
if (-e $PUBSUBEXPR) {
unlink($PUBSUBEXPR) or
warn("*** could not remove $PUBSUBEXPR\n");
}
# Leave this print statement here; create_image depends on it.
print "prepare ran successfully!\n";
exit 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment