Commit 66e07584 authored by Mike Hibler's avatar Mike Hibler

Make frisbee more directly IGMP (v2) aware.

Add "-Q <interval>" option to the master server to allow it to act as an
IGMP V2 querier in environment where there is otherwise not one. It does
essentially what the perl-based querier (code.google.com/p/perl-igmp-querier/)
does, sending out a v2 membership query at the specified interval.

This eliminates the need to run mrouted in some environments (e.g., elabinelab)
just to issue IGMP queries. As a result, all the boss-install and elabinelab
setup related to using mrouted to perform this function has been removed.
The elabinelab CONFIG_MROUTED option has been changed to CONFIG_QUERIER
(the former is still recognized and mapped to the latter). The undocumented
defs-* variable NEEDMROUTED has been changed to NEEDMCQUERIER (the former
still exists in install/installvars.pm.in but is always set to 0) to more
accurately reflect the variable's purpose. If NEEDMCQUERIER is set, then
the mfrisbeed startup script is modified to add the "-Q 30" option.

The implementation of the client and server "-K <interval>" keep-alive option
has been changed to directly send IGMP v2 membership reports containing the
associated MC address.

Note that the -K options have always been a hack to work-around assorted
IGMP-related misconfigurations and incompatibilities, and really should
only be used as a last resort. As implemented, they could cause the host
machine to be pruned out of other MC groups at the nearest switch since
they only report membership in the frisbee MC group. With the master server
acting as an IGMP querier, instances of the frisbee server on that host
should no longer need to do keep alives. We still have one case where it
is needed on the client-side: a FreeBSD 8.x or later host connected to an
IGMPv2-only switch. It appears that the IGMPv3 implementation added in
FreeBSD 8.x always sends v3 reports, even when the default is configured
(via sysctl or even recompiling the kernel) as v2.
parent 9f58a417
......@@ -19,6 +19,11 @@ EXPANDCOPYRIGHT = /usr/site/lib/copyright/expand-copyr
WITH_CRYPTO = 1
WITH_SIGNING = 1
#
# Support direct sending of IGMP query/report messages.
#
WITH_IGMP = 1
SYSTEM := $(shell uname -s)
# FreeBSD specific goop
......@@ -147,6 +152,13 @@ ifeq ($(WITH_JUMBO),1)
CFLAGS += -DJUMBO
endif
#
# Send direct IGMP packets
#
ifeq ($(WITH_IGMP),1)
CFLAGS += -DWITH_IGMP
endif
#
# Define this to run the client and server on the same physical machine
# over the loopback interface. You will also have to setup an alias on
......
......@@ -702,8 +702,6 @@ ClientRecvThread(void *arg)
if (debug)
log("Receive pthread starting up ...");
KACounter = keepalive * TIMEOUT_HZ;
/*
* Use this to control the rate at which we request blocks.
* The IdleCounter is how many ticks we let pass without a
......@@ -714,6 +712,17 @@ ClientRecvThread(void *arg)
*/
IdleCounter = 1;
/*
* KACounter is how often we send a multicast "keep alive",
* aka a V2 IGMP report message. The keep alive mechanism is a
* hack and indicates a compatibility issue between the frisbee
* server, frisbee client, and switch IGMP implementations.
* As with the idle counter, we initialize it to one so that an
* immediate message will be sent in order to get things moving
* quickly.
*/
KACounter = keepalive ? 1 : 0;
/*
* This is another throttling mechanism; avoid making repeated
* requests to a server that is not running. That is, if the server
......
......@@ -293,8 +293,14 @@ set_get_values(struct config_host_authinfo *ai, int ix)
snprintf(str, sizeof str, " -W %u",
isindir(STDIMAGEDIR, ii->path) ?
get_maxrate_std : get_maxrate_usr);
#if 0
/*
* Should not be needed anymore. If the multicast group is getting
* dropped, the mserver should be configured to run as a querier (-Q).
*/
if (INELABINELAB)
strcat(str, " -K 15");
#endif
ii->get_options = mystrdup(str);
/* and whack the put_* fields */
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2011 University of Utah and the Flux Group.
* Copyright (c) 2000-2012 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -437,6 +437,11 @@ int ClientNetFindServer(in_addr_t, in_port_t, in_addr_t, char *,
int MsgSend(int, MasterMsg_t *, size_t, int);
int MsgReceive(int, MasterMsg_t *, size_t, int);
#endif
#ifdef WITH_IGMP
void IGMPInit(struct in_addr *iface, struct in_addr *mcaddr);
int IGMPSendQuery(void);
int IGMPSendReport(void);
#endif
/*
* Globals
......
/*
* Implement a couple of simple IGMP V2 messages.
*
* We can either send a general membership query (suitable for implementing
* a simple IGMP querier) or a membership report.
*/
#include <inttypes.h>
#include <string.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#ifdef WITH_IGMP
#include <netinet/ip.h>
#include <netinet/igmp.h>
#endif
#ifdef WITH_IGMP
static struct igmp qpacket, rpacket;
static struct in_addr mciface;
static struct sockaddr_in allhosts, mcgroup;
static uint16_t
igmp_csum(struct igmp *pkt)
{
char *addr = (char *)pkt;
int cc = sizeof(*pkt);
uint32_t csum = 0;
while (cc >= sizeof(uint16_t)) {
csum += *(uint16_t *)addr;
addr += sizeof(uint16_t);
cc -= sizeof(uint16_t);
}
if (cc > 0)
csum = csum + *(uint8_t *)addr;
while ((csum >> 16) != 0)
csum = (csum >> 16) + (csum & 0xFFFF);
return(~csum);
}
static int
igmp_opensocket(void)
{
char ra[4];
int ttl = 1;
int sock;
sock = socket(AF_INET, SOCK_RAW, IPPROTO_IGMP);
if (sock < 0) {
perror("IGMP socket");
return -1;
}
/* set TTL */
if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_TTL,
&ttl, sizeof(ttl)) < 0) {
perror("setsockopt(MULTICAST_TTL)");
close(sock);
return -1;
}
/* fix interface */
if (mciface.s_addr != 0 &&
setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF,
&mciface, sizeof(mciface)) < 0) {
perror("setsockopt(MULTICAST_IF)");
close(sock);
return -1;
}
/* set router alert option */
ra[0] = IPOPT_RA;
ra[1] = 4;
ra[2] = ra[3] = '\0';
if (setsockopt(sock, IPPROTO_IP, IP_OPTIONS, &ra, sizeof(ra)) < 0) {
perror("setsockopt(RA)");
close(sock);
return -1;
}
return sock;
}
#endif
void
IGMPInit(struct in_addr *iface, struct in_addr *mcaddr)
{
#ifdef WITH_IGMP
/* build a prototype query packet */
qpacket.igmp_type = IGMP_MEMBERSHIP_QUERY;
qpacket.igmp_code = 0x64;
memset(&qpacket.igmp_group, 0, sizeof(qpacket.igmp_group));
qpacket.igmp_cksum = 0;
qpacket.igmp_cksum = igmp_csum(&qpacket);
/* sockaddr for queries */
allhosts.sin_len = sizeof(allhosts);
allhosts.sin_family = AF_INET;
allhosts.sin_port = htons(0);
allhosts.sin_addr.s_addr = htonl(INADDR_ALLHOSTS_GROUP);
if (mcaddr != NULL) {
/* build a prototype report packet */
rpacket.igmp_type = IGMP_V2_MEMBERSHIP_REPORT;
rpacket.igmp_code = 0;
rpacket.igmp_group = *mcaddr;
rpacket.igmp_cksum = 0;
rpacket.igmp_cksum = igmp_csum(&rpacket);
/* sockaddr for reports */
mcgroup.sin_len = sizeof(mcgroup);
mcgroup.sin_family = AF_INET;
mcgroup.sin_port = htons(0);
mcgroup.sin_addr = *mcaddr;
}
/* remember the interface */
if (iface != NULL)
mciface = *iface;
else
mciface.s_addr = 0;
#endif
}
int
IGMPSendQuery(void)
{
#ifdef WITH_IGMP
int rv, sock;
if ((sock = igmp_opensocket()) < 0)
return -1;
rv = sendto(sock, &qpacket, sizeof(qpacket), 0,
(struct sockaddr *)&allhosts, sizeof(allhosts));
if (rv < 0)
perror("query sendto");
close(sock);
return (rv != sizeof(qpacket));
#else
fatal("IGMPSendQuery not implemented");
#endif
}
int
IGMPSendReport(void)
{
#ifdef WITH_IGMP
int rv, sock;
if (mcgroup.sin_addr.s_addr == 0)
return 0;
if ((sock = igmp_opensocket()) < 0)
return -1;
rv = sendto(sock, &rpacket, sizeof(rpacket), 0,
(struct sockaddr *)&mcgroup, sizeof(mcgroup));
if (rv < 0)
perror("report sendto");
close(sock);
return (rv != sizeof(rpacket));
#else
fatal("IGMPSendReport not implemented");
#endif
}
#if 0
#include <stdio.h>
#include <stdlib.h>
main(int argc, char **argv)
{
struct in_addr addr, mcaddr;
int i;
if (argc < 2) {
fprintf(stderr, "usage: %s <cnet-interface-IP>\n", argv[0]);
exit(1);
}
inet_aton(argv[1], &addr);
inet_aton("234.5.6.7", &mcaddr);
printf("Testing querier (using iface %s)...\n", inet_ntoa(addr));
IGMPInit(&addr, &mcaddr);
printf("Sending a report for %s...\n", inet_ntoa(mcaddr));
if (IGMPSendReport()) {
printf("report send failed!\n");
exit(1);
}
printf("Sending queries to %s...\n", inet_ntoa(allhosts.sin_addr));
i = 10;
while (i-- > 0) {
if (IGMPSendQuery()) {
printf("query send failed!\n");
exit(1);
}
sleep(30);
}
exit(0);
}
#endif
......@@ -43,6 +43,9 @@ static void get_options(int argc, char **argv);
static int makesocket(int portnum, struct in_addr *ifip, int *tcpsockp);
static void handle_request(int sock);
static int reapchildren(int apid, int *status);
#ifdef WITH_IGMP
static void handle_igmp(void);
#endif
static char * gidstr(int ngids, gid_t gids[]);
static int daemonize = 1;
......@@ -68,6 +71,7 @@ static int usechildauth = 0;
static int mirrormode = 0;
static char *configstyle = "null";
static struct in_addr ifaceip;
static int igmpqueryinterval = 0;
/* XXX the following just keep network.c happy */
int portnum = MS_PORTNUM;
......@@ -101,6 +105,11 @@ main(int argc, char **argv)
inet_ntoa(parentip), parentport,
mirrormode ? " in mirror mode" : "",
GetMSMethods(parentmethods));
#ifdef WITH_IGMP
if (igmpqueryinterval)
log(" acting as IGMP querier on %s with %d second interval",
inet_ntoa(ifaceip), igmpqueryinterval);
#endif
config_init(configstyle, 1);
/* Just dump the config to stdout in human readable form and exit. */
......@@ -130,6 +139,11 @@ main(int argc, char **argv)
}
}
#endif
#ifdef WITH_IGMP
if (igmpqueryinterval)
IGMPInit(&ifaceip, NULL);
#endif
/* Now become a daemon */
if (daemonize)
daemon(0, 0);
......@@ -158,6 +172,12 @@ main(int argc, char **argv)
socklen_t length;
int newsock, rv, maxsock = 0;
#ifdef WITH_IGMP
/* Do the querier thing */
if (igmpqueryinterval)
handle_igmp();
#endif
FD_SET(tcpsock, &ready);
maxsock = tcpsock + 1;
#ifdef USE_LOCALHOST_PROXY
......@@ -1314,7 +1334,7 @@ get_options(int argc, char **argv)
{
int ch;
while ((ch = getopt(argc, argv, "AC:DI:MRX:x:S:P:p:i:dh")) != -1)
while ((ch = getopt(argc, argv, "AC:DI:MRX:x:S:P:p:i:dhQ:")) != -1)
switch(ch) {
case 'A':
usechildauth = 1;
......@@ -1402,6 +1422,20 @@ get_options(int argc, char **argv)
exit(1);
}
break;
case 'Q':
#ifdef WITH_IGMP
igmpqueryinterval = atoi(optarg);
if (igmpqueryinterval <= 0) {
fprintf(stderr,
"Invalid IGMP querier interval '%s'\n",
optarg);
exit(1);
}
#else
fprintf(stderr, "IGMP querier mode not supported\n");
exit(1);
#endif
break;
case 'h':
case '?':
default:
......@@ -2119,6 +2153,33 @@ reapchildren(int wpid, int *statusp)
return corpses;
}
#ifdef WITH_IGMP
/*
* If we are acting as an IGMP querier, see if we are due to send out a
* general membership query. Maybe we should only send queries while we
* have active MC servers running?
*
* Called from the main loop before returning to select to wait for more.
*/
static void
handle_igmp(void)
{
static struct timeval lastquery;
struct timeval now, delta;
gettimeofday(&now, NULL);
timersub(&now, &lastquery, &delta);
if (delta.tv_sec >= igmpqueryinterval) {
if (debug)
log("sending IGMP membership query after %d seconds",
delta.tv_sec);
if (IGMPSendQuery())
warning("could not send IGMP membership query!");
lastquery = now;
}
}
#endif
/*
* XXX debug
*/
......
......@@ -66,6 +66,12 @@ GetIP(char *str, struct in_addr *in)
return 1;
}
/*
* Return the maximum size of a socket buffer.
* Computes it dynamically on the first call.
*
* XXX assumes send/recv max sizes are the same.
*/
int
GetSockbufSize(void)
{
......@@ -222,6 +228,10 @@ CommonInit(int dobind)
&mcastif, sizeof(mcastif)) < 0) {
pfatal("setsockopt(IPPROTO_IP, IP_MULTICAST_IF)");
}
#ifdef WITH_IGMP
IGMPInit(&mcastif, &mcastaddr);
#endif
}
else if (broadcast) {
log("Setting broadcast mode");
......@@ -338,6 +348,12 @@ NetMCKeepAlive(void)
if (sock == -1)
return 1;
#ifdef WITH_IGMP
/* Send a direct V2 report packet if possible */
if (IGMPSendReport() == 0)
return 0;
#endif
mreq.imr_multiaddr.s_addr = mcastaddr.s_addr;
if (mcastif.s_addr)
mreq.imr_interface.s_addr = mcastif.s_addr;
......@@ -349,6 +365,7 @@ NetMCKeepAlive(void)
setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP,
&mreq, sizeof(mreq)) < 0)
return 1;
return 0;
}
......@@ -568,6 +585,9 @@ PacketValid(Packet_t *p, int nchunks)
return 1;
}
/*
* Functions for communicating with the master server.
*/
#ifdef MASTER_SERVER
int
MsgSend(int msock, MasterMsg_t *msg, size_t size, int timo)
......@@ -789,3 +809,163 @@ ClientNetFindServer(in_addr_t sip, in_port_t sport,
return 1;
}
#endif
/*
* Functions for dealing with IGMP
*/
#ifdef WITH_IGMP
#include <netinet/ip.h>
#include <netinet/igmp.h>
#ifdef IGMP_MEMBERSHIP_QUERY
#define IGMP_QUERY IGMP_MEMBERSHIP_QUERY
#else
#define IGMP_QUERY IGMP_HOST_MEMBERSHIP_QUERY
#endif
#ifdef IGMP_V2_MEMBERSHIP_REPORT
#define IGMP_REPORT IGMP_V2_MEMBERSHIP_REPORT
#else
#define IGMP_REPORT IGMP_v2_HOST_MEMBERSHIP_REPORT
#endif
static struct igmp qpacket, rpacket;
static struct in_addr mciface;
static struct sockaddr_in allhosts, mcgroup;
static uint16_t
igmp_csum(struct igmp *pkt)
{
char *addr = (char *)pkt;
int cc = sizeof(*pkt);
uint32_t csum = 0;
while (cc >= sizeof(uint16_t)) {
csum += *(uint16_t *)addr;
addr += sizeof(uint16_t);
cc -= sizeof(uint16_t);
}
if (cc > 0)
csum = csum + *(uint8_t *)addr;
while ((csum >> 16) != 0)
csum = (csum >> 16) + (csum & 0xFFFF);
return(~csum);
}
static int
igmp_opensocket(void)
{
char ra[4];
int ttl = 1;
int sock;
sock = socket(AF_INET, SOCK_RAW, IPPROTO_IGMP);
if (sock < 0) {
perror("IGMP socket");
return -1;
}
/* set TTL */
if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_TTL,
&ttl, sizeof(ttl)) < 0) {
perror("setsockopt(MULTICAST_TTL)");
close(sock);
return -1;
}
/* fix interface */
if (mciface.s_addr != 0 &&
setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF,
&mciface, sizeof(mciface)) < 0) {
perror("setsockopt(MULTICAST_IF)");
close(sock);
return -1;
}
/* set router alert option */
ra[0] = IPOPT_RA;
ra[1] = 4;
ra[2] = ra[3] = '\0';
if (setsockopt(sock, IPPROTO_IP, IP_OPTIONS, &ra, sizeof(ra)) < 0) {
perror("setsockopt(RA)");
close(sock);
return -1;
}
return sock;
}
void
IGMPInit(struct in_addr *iface, struct in_addr *mcaddr)
{
/* build a prototype query packet */
qpacket.igmp_type = IGMP_QUERY;
qpacket.igmp_code = 0x64;
memset(&qpacket.igmp_group, 0, sizeof(qpacket.igmp_group));
qpacket.igmp_cksum = 0;
qpacket.igmp_cksum = igmp_csum(&qpacket);
/* sockaddr for queries */
allhosts.sin_family = AF_INET;
allhosts.sin_port = htons(0);
allhosts.sin_addr.s_addr = htonl(INADDR_ALLHOSTS_GROUP);
if (mcaddr != NULL) {
/* build a prototype report packet */
rpacket.igmp_type = IGMP_REPORT;
rpacket.igmp_code = 0;
rpacket.igmp_group = *mcaddr;
rpacket.igmp_cksum = 0;
rpacket.igmp_cksum = igmp_csum(&rpacket);
/* sockaddr for reports */
mcgroup.sin_family = AF_INET;
mcgroup.sin_port = htons(0);
mcgroup.sin_addr = *mcaddr;
}
/* remember the interface */
if (iface != NULL)
mciface = *iface;
else
mciface.s_addr = 0;
}
int
IGMPSendQuery(void)
{
int rv, sock;
if ((sock = igmp_opensocket()) < 0)
return -1;
rv = sendto(sock, &qpacket, sizeof(qpacket), 0,
(struct sockaddr *)&allhosts, sizeof(allhosts));
if (rv < 0)
perror("query sendto");
close(sock);
return (rv != sizeof(qpacket));
}
int
IGMPSendReport(void)
{
int rv, sock;
if (mcgroup.sin_addr.s_addr == 0)
return 0;
if ((sock = igmp_opensocket()) < 0)
return -1;
rv = sendto(sock, &rpacket, sizeof(rpacket), 0,
(struct sockaddr *)&mcgroup, sizeof(mcgroup));
if (rv < 0)
perror("report sendto");
close(sock);
return (rv != sizeof(rpacket));
}
#endif
......@@ -161,9 +161,10 @@ my $SINGLE_CONTROLNET = 0;
# Support a shared filesystem between fs server and nodes
my $SHAREDFS = 1;
# Start up mrouted so frisbee works (if there is no other mcast router)
# Start up the frisbee master server to act as an IGMP querier
# so that frisbee works (if there is no other querier running).
# -1 means let the system decide.
my $NEEDMROUTED= -1;
my $NEEDQUERIER = -1;
# Is ops a VM (Jail) on boss.
my $OPSVM = 0;
......@@ -209,7 +210,7 @@ my %emulabconfig = (
# CONFIG_USERDB support integrated MySQL DB (part of COLLAB)
# CONFIG_WIKI support integrated wiki (part of COLLAB)
# CONFIG_SHAREDFS support shared filesystem
# CONFIG_MROUTED install/configure multicast router
# CONFIG_QUERIER configure mfrisbeed to act as IGMP querier
#
"CONFIG_NOSETUP" => $NOSETUP,
"CONFIG_SCRATCHFS" => $SCRATCHFS,
......@@ -224,7 +225,7 @@ my %emulabconfig = (
"CONFIG_USERDB" => !$NOCOLLAB,
"CONFIG_WIKI" => !$NOCOLLAB,
"CONFIG_SHAREDFS" => $SHAREDFS,
"CONFIG_MROUTED" => $NEEDMROUTED,
"CONFIG_QUERIER" => $NEEDQUERIER,
"CONFIG_OPSVM" => $OPSVM,
"CONFIG_NODBINIT" => 0,
"CONFIG_GENIRACK" => 0,
......@@ -446,6 +447,15 @@ sub doboot()
$emulabconfig{"ROLE"} = "opsjail"
if ($opsjail);
#
# XXX backward compat for short-lived NEEDMROUTED option.
# mfrisbeed now serves the role that mrouted did briefly.
#
if (defined($emulabconfig{"CONFIG_MROUTED"}) &&
$emulabconfig{"CONFIG_QUERIER"} == -1) {
$emulabconfig{"CONFIG_QUERIER"} = $emulabconfig{"CONFIG_MROUTED"};
}
#
# Look for a private variable cache.
#
......@@ -2164,15 +2174,15 @@ sub SetupBossNode($)
}
#
# Determine if we need to run a multicast router.
# We will if: 1) using a private control net and 2) kernel support mroute
# Determine if we need to run an IGMP querier.
# Right now this is only necessary if we are using a private control
# network.
#
if ($emulabconfig{"CONFIG_MROUTED"} == -1) {
if (!$emulabconfig{"CONFIG_SINGLECNET"} &&
!system("sysctl -N net.inet.ip.mrtstat 2>&1")) {
$emulabconfig{"CONFIG_MROUTED"} = 1;
if ($emulabconfig{"CONFIG_QUERIER"} == -1) {
if (!$emulabconfig{"CONFIG_SINGLECNET"} {
$emulabconfig{"CONFIG_QUERIER"} = 1;
} else {
$emulabconfig{"CONFIG_MROUTED"} = 0;
$emulabconfig{"CONFIG_QUERIER"} = 0;
}
}
......@@ -2390,23 +2400,6 @@ sub SetupBossNode($)
mysystem("cd $TBDIR/obj/testbed/install; ".
" perl emulab-install $pkg -l -b -w $pswd boss");
#
# Need to tweak mrouted.conf as appropriate for an inner-elab
# (ignore the real control net, otherwise we will get ALL mcast traffic
# in the testbed delivered to our doorstep!)
#
if ($emulabconfig{"CONFIG_MROUTED"}) {
# make sure old "standard" mrouted doesn't interfere with port
if (-r "/usr/local/etc/rc.d/mrouted" && -r "/etc/rc.d/mrouted") {
unlink("/etc/rc.d/mrouted");
}
if (-x "/usr/local/sbin/mrouted" && -x "/usr/sbin/mrouted") {
unlink("/usr/sbin/mrouted");
}
mysystem("echo 'phyint $outer_controlif force_leaf noflood deny 0/0 bidir' >> /usr/local/etc/mrouted.conf");
}
#
# Copy the creators ssl certificate into place. This allows the
# inner boss to invoke the XMLRPC server on the outer boss for
......@@ -3155,11 +3148,11 @@ sub CreateDefsFile($)
}
last SWITCH;
};
/^NEEDMROUTED$/ && do {
if ($emulabconfig{"CONFIG_MROUTED"} == 1) {
print OUTDEFS "NEEDMROUTED=1\n";
/^NEEDMCQUERIER$/ && do {
if ($emulabconfig{"CONFIG_QUERIER"} == 1) {
print OUTDEFS "NEEDMCQUERIER=1\n";
} else {
print OUTDEFS "NEEDMROUTED=0\n";
print OUTDEFS "NEEDMCQUERIER=0\n";