Commit 73102ef8 authored by Leigh Stoller's avatar Leigh Stoller

Add support for dynamic registration of ports on experimental nodes so

that clients and servers can avoid using hardwired ports on those
experimental nodes. I have added the following tmcd operation:

	tmcc portregister <service> [<port>]

where we assume its the control network IP (from the DB), and the pid/eid
of the node the experiment belongs to. The given port is entered into
the port_registration table for the experiment, using the service as the
tag. Supplying port=0 clears the registration from the table.

When called like:

	tmcc portregister <service>

we return the registered port, or nothing.

I hacked up a little C library module in libtb so that there is something
that looks like a C interface to this:

 	int
 	PortRegister(char *service, int port);

 	int
 	PortLookup(char *service, char *hostname, int namelen, int *port);

The above routines call out to tmcc of course.

Lastly, I changed the sync server and client to use the new port
registration, via the library calls above.

There are other emulab services that need to be changed as well, but
they can be done on an as needed basis.
parent c93040ae
......@@ -1112,6 +1112,44 @@ sub ClearBackupState($)
TBExptClearBackupState($pid, $eid);
return 0;
}
sub RemovePhysicalState($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $pid = $self->pid();
my $eid = $self->eid();
return -1
if (TBExptRemovePhysicalState($pid, $eid));
return 0;
}
#
# The port registration table is special, and needs to be cleared only
# at certain times. See tbswap.
#
sub ClearPortRegistration($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $pid = $self->pid();
my $eid = $self->eid();
return -1
if (! DBQueryWarn("delete from port_registration ".
"where pid='$pid' and eid='$eid'"));
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002, 2004, 2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2002, 2004, 2005, 2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -24,7 +24,7 @@ control-install: client
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = log.o tbdefs.o popenf.o systemf.o be_user.o
OBJS = log.o tbdefs.o popenf.o systemf.o be_user.o tmcc.o
CFLAGS += -O -g -Wall -I${OBJDIR} -I/usr/local/include
libtb.a: $(OBJS) tbdb.o
......@@ -41,6 +41,7 @@ tbdefs.o: tbdefs.h
popenf.o: popenf.h
systemf.o: systemf.h
be_user.o: be_user.h
tmcc.o: popenf.h tmcc.h
install: all
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2004, 2006 University of Utah and the Flux Group.
* All rights reserved.
*/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "popenf.h"
#define TMCC "/usr/local/etc/emulab/tmcc"
/*
* Silly little helper function to run tmcc.
*/
int
tmcc(char *fmt, char *ret, int retlen, ...)
{
char buf[2*BUFSIZ], *bp = ret;
va_list ap;
FILE *fp;
int retval = 0;
int count = 0;
/* Need to tack on the path to tmcc */
if (strlen(TMCC) + strlen(fmt) + 1 > sizeof(buf)) {
errno = ENOMEM;
return -1;
}
strcpy(buf, TMCC);
strcat(buf, " ");
strcat(buf, fmt);
va_start(ap, retlen);
fp = vpopenf(buf, "r", ap);
va_end(ap);
if (fp == NULL) {
/* errno set by vpopenf */
return -1;
}
while (1) {
int rc = fread(buf, 1, sizeof(buf), fp);
if (rc <= 0) {
if (rc < 0 || ferror(fp)) {
retval = -1;
break;
}
break;
}
if (retlen < rc) {
/* Not enough room in buffer */
errno = ENOMEM;
retval = -1;
break;
}
memcpy(bp, buf, rc);
bp += rc;
retlen -= rc;
count += rc;
}
if (!retval)
retval = count;
if (pclose(fp)) {
errno = EIO;
retval = -1;
}
return retval;
}
/*
* Port lookup and registration
*/
int
PortLookup(char *service, char *hostname, int namelen, int *port)
{
char buf[BUFSIZ], fmt[32];
int rc;
if (tmcc("-i portregister %s", buf, sizeof(buf), service) <= 0)
return -1;
sprintf(fmt, "PORT=%%d NODEID=%%%ds", namelen);
rc = sscanf(buf, fmt, port, hostname);
if (rc != 2)
return -1;
return 0;
}
int
PortRegister(char *service, int port)
{
char buf[BUFSIZ];
return tmcc("-i portregister %s %d", buf, sizeof(buf), service, port);
}
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2006 University of Utah and the Flux Group.
* All rights reserved.
*/
#ifdef __cplusplus
extern "C" {
#endif
int PortLookup(char *service, char *hostname, int namelen, int *port);
int PortRegister(char *service, int port);
#ifdef __cplusplus
}
#endif
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2003, 2004 University of Utah and the Flux Group.
* Copyright (c) 2002-2006 University of Utah and the Flux Group.
* All rights reserved.
*/
#define SERVER_SERVNAME "emulab_syncd"
#define SERVER_PORTNUM 16534
#define SOCKBUFSIZE (1024 * 128)
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2004 University of Utah and the Flux Group.
* Copyright (c) 2000-2004, 2006 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -27,6 +27,7 @@
#include <setjmp.h>
#include "decls.h"
#include "config.h"
#include "tmcc.h"
#ifndef SYNCSERVER
#define SYNCSERVER "/var/emulab/boot/syncserver"
......@@ -73,7 +74,7 @@ main(int argc, char **argv)
unsigned char **hal;
FILE *fp;
struct in_addr serverip;
int portnum = SERVER_PORTNUM;
int portnum = 0;
int useudp = 0;
char *server = NULL;
int initme = 0;
......@@ -271,6 +272,37 @@ main(int argc, char **argv)
exit(1);
}
/*
* Lookup the port.
*/
if (portnum == 0) {
char nodename[BUFSIZ];
int rc;
while (1) {
rc = PortLookup(SERVER_SERVNAME, nodename,
sizeof(nodename), &portnum);
/* Got the port; we do not care about the nodename */
if (rc == 0 && portnum != 0) {
if (debug) {
printf("Sync Server at %s:%d\n",
nodename, portnum);
}
break;
}
if (rc == 0 && portnum == 0) {
fprintf(stderr,
"Could not lookup service with tmcd!");
exit(1);
}
/* Delay */
sleep(5);
}
}
/*
* Build up the request structure to send to the server.
*/
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -23,16 +23,17 @@
#include <paths.h>
#include "decls.h"
#include "log.h"
#include "tmcc.h"
int debug = 0;
int verbose = 0;
static int portnum = SERVER_PORTNUM;
static int portnum = 0;
static void print_barriers(void);
static void clear_barriers(void);
static int readall(int socket, void *buffer, size_t len);
static int writeall(int socket, void *buffer, size_t len);
static int handle_request(int, struct sockaddr_in *, barrier_req_t *,int);
static int makesockets(int portnum, int *udpsockp, int *tcpsockp);
static int makesockets(int *portnum, int *udpsockp, int *tcpsockp);
static int maxtcpsocket(fd_set *fs, unsigned int current_max);
static void remove_tcpclient(int sock);
static void release_client(int sock, struct in_addr ip, int istcp,
......@@ -191,10 +192,18 @@ main(int argc, char **argv)
/*
* Create TCP/UDP server.
*/
if (makesockets(portnum, &udpsock, &tcpsock) < 0) {
if (makesockets(&portnum, &udpsock, &tcpsock) < 0) {
error("Could not make sockets!");
exit(1);
}
/*
* Register with tmcc
*/
if (PortRegister(SERVER_SERVNAME, portnum) < 0) {
error("Could not register service with tmcd!");
exit(1);
}
/* Now become a daemon */
if (!debug)
daemon(0, 1);
......@@ -487,7 +496,7 @@ writeall(int socket, void *buffer, size_t len)
* Create sockets on specified port.
*/
static int
makesockets(int portnum, int *udpsockp, int *tcpsockp)
makesockets(int *portnum, int *udpsockp, int *tcpsockp)
{
struct sockaddr_in name;
int length, i, udpsock, tcpsock;
......@@ -510,7 +519,7 @@ makesockets(int portnum, int *udpsockp, int *tcpsockp)
/* Create name. */
name.sin_family = AF_INET;
name.sin_addr.s_addr = INADDR_ANY;
name.sin_port = htons((u_short) portnum);
name.sin_port = *portnum;
if (bind(tcpsock, (struct sockaddr *) &name, sizeof(name))) {
pfatal("binding stream socket");
}
......@@ -522,7 +531,9 @@ makesockets(int portnum, int *udpsockp, int *tcpsockp)
if (listen(tcpsock, 128) < 0) {
pfatal("listen");
}
info("listening on TCP port %d\n", ntohs(name.sin_port));
*portnum = ntohs(name.sin_port);
info("listening on TCP port %d\n", *portnum);
/*
* Setup UDP socket
......@@ -552,7 +563,7 @@ makesockets(int portnum, int *udpsockp, int *tcpsockp)
/* Create name. */
name.sin_family = AF_INET;
name.sin_addr.s_addr = INADDR_ANY;
name.sin_port = htons((u_short) portnum);
name.sin_port = htons((u_short) *portnum);
if (bind(udpsock, (struct sockaddr *) &name, sizeof(name))) {
pfatal("binding dgram socket");
}
......
......@@ -55,6 +55,7 @@ use libadminctrl;
use libadminmfs;
use libtblog;
use libArchive;
use Experiment;
#require exitonwarn; # exitonwarn isn't really a module, so just require it
......@@ -181,6 +182,12 @@ if (UNIX2DBUID($UID, \$dbuid)) {
UserDBInfo($dbuid, \$uname, \$umail);
}
# Slowly convert to using Experiment module.
my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
tbdie("Could not lookup experiment object!")
}
#
# Print starting message.
#
......@@ -758,7 +765,9 @@ sub doSwapout($) {
#
if ( $type >= CLEANUP ) {
print STDERR "Resetting DB.\n";
TBExptRemovePhysicalState( $pid, $eid );
$experiment->RemovePhysicalState();
# Special.
$experiment->ClearPortRegistration();
}
tblog_set_cleanup(0);
......@@ -1116,6 +1125,9 @@ sub doSwapin($) {
TBDB_ALLOCSTATE_RES_RECONFIG()));
}
}
# Do this only when nodes are to be rebooted.
$experiment->ClearPortRegistration()
if ($type == UPDATE);
}
#
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -31,7 +31,7 @@ use Exporter;
TMCCCMD_CREATOR TMCCCMD_HOSTINFO TMCCCMD_LOCALIZATION
TMCCCMD_BOOTERRNO TMCCCMD_BOOTLOG TMCCCMD_BATTERY TMCCCMD_USERENV
TMCCCMD_TIPTUNNELS TMCCCMD_TRACEINFO TMCCCMD_ELVINDPORT
TMCCCMD_PLABEVENTKEYS
TMCCCMD_PLABEVENTKEYS TMCCCMD_PORTREGISTER
TMCCCMD_MOTELOG
);
......@@ -175,6 +175,7 @@ my %commandset =
"elvindport" => {TAG => "elvindport"},
"plabeventkeys" => {TAG => "plabeventkeys"},
"motelog" => {TAG => "motelog"},
"portregister" => {TAG => "portregister"},
);
#
......@@ -232,6 +233,7 @@ sub TMCCCMD_TRACEINFO (){ $commandset{"traceinfo"}->{TAG}; }
sub TMCCCMD_ELVINDPORT (){ $commandset{"elvindport"}->{TAG}; }
sub TMCCCMD_PLABEVENTKEYS(){ $commandset{"plabeventkeys"}->{TAG}; }
sub TMCCCMD_MOTELOG() { $commandset{"motelog"}->{TAG}; }
sub TMCCCMD_PORTREGISTER() { $commandset{"portregister"}->{TAG}; }
#
# Caller uses this routine to set configuration of this library
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# Copyright (c) 2000-2004, 2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -32,7 +32,7 @@ sub usage()
print STDERR " -D Force command to use a direct, UDP request\n";
exit(1);
}
my $optlist = "ds:p:v:n:k:ul:t:x:o:bcDi:f:";
my $optlist = "ds:p:v:n:k:ul:t:x:o:bcDif:";
my $debug = 0;
my $CMD;
my $ARGS;
......
......@@ -255,6 +255,7 @@ COMMAND_PROTOTYPE(doelvindport);
COMMAND_PROTOTYPE(doplabeventkeys);
COMMAND_PROTOTYPE(dointfcmap);
COMMAND_PROTOTYPE(domotelog);
COMMAND_PROTOTYPE(doportregister);
/*
* The fullconfig slot determines what routines get called when pushing
......@@ -270,10 +271,11 @@ COMMAND_PROTOTYPE(domotelog);
/*
* Flags encode a few other random properties of commands
*/
#define F_REMUDP 0x1 /* remote nodes can request using UDP */
#define F_MINLOG 0x2 /* record minimal logging info normally */
#define F_MAXLOG 0x4 /* record maximal logging info normally */
#define F_ALLOCATED 0x8 /* node must be allocated to make call */
#define F_REMUDP 0x01 /* remote nodes can request using UDP */
#define F_MINLOG 0x02 /* record minimal logging info normally */
#define F_MAXLOG 0x04 /* record maximal logging info normally */
#define F_ALLOCATED 0x08 /* node must be allocated to make call */
#define F_REMNOSSL 0x10 /* remote nodes can request without SSL */
struct command {
char *cmdname;
......@@ -346,6 +348,7 @@ struct command {
{ "plabeventkeys",FULLCONFIG_NONE, 0, doplabeventkeys},
{ "intfcmap", FULLCONFIG_NONE, 0, dointfcmap},
{ "motelog", FULLCONFIG_ALL, F_ALLOCATED, domotelog},
{ "portregister", FULLCONFIG_NONE, F_REMNOSSL, doportregister},
};
static int numcommands = sizeof(command_array)/sizeof(struct command);
......@@ -1040,14 +1043,6 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
goto skipit;
}
}
else if (!reqp->islocal) {
if (!istcp)
goto execute;
error("%s: Remote node connected without SSL!\n",reqp->nodeid);
if (!insecure)
goto skipit;
}
else if (reqp->iscontrol) {
if (!istcp)
goto execute;
......@@ -1130,6 +1125,16 @@ handle_request(int sock, struct sockaddr_in *client, char *rdata, int istcp)
goto skipit;
}
/*
* Ditto for remote node connection without SSL.
*/
if (!reqp->islocal &&
(command_array[i].flags & F_REMNOSSL) == 0) {
error("%s: %s: Invalid NO-SSL request from remote node\n",
reqp->nodeid, command_array[i].cmdname);
goto skipit;
}
if (!reqp->allocated && (command_array[i].flags & F_ALLOCATED) != 0) {
if (verbose || (command_array[i].flags & F_MINLOG) == 0)
error("%s: %s: Invalid request from free node\n",
......@@ -6794,3 +6799,95 @@ COMMAND_PROTOTYPE(domotelog)
mysql_free_result(res);
return 0;
}
/*
* Return motelog info for this node.
*/
COMMAND_PROTOTYPE(doportregister)
{
MYSQL_RES *res;
MYSQL_ROW row;
char buf[MYBUFSIZE], service[128];
int rc, port;
/*
* Dig out the service and the port number.
* Need to be careful about not overflowing the buffer.
*/
sprintf(buf, "%%%ds %%d", sizeof(service));
rc = sscanf(rdata, buf, service, &port);
if (rc == 0) {
error("doportregister: No service specified!\n");
return 1;
}
if (rc != 1 && rc != 2) {
error("doportregister: Wrong number of arguments!\n");
return 1;
}
/* No special characters means it will fit */
mysql_escape_string(buf, service, strlen(service));
if (strlen(buf) >= sizeof(service)) {
error("doportregister: Illegal chars in service!\n");
return 1;
}
strcpy(service, buf);
/*
* Single argument, lookup up service.
*/
if (rc == 1) {
res = mydb_query("select port,node_id "
" from port_registration "
"where pid='%s' and eid='%s' and "
" service='%s'",
2, reqp->pid, reqp->eid, service);
if (!res) {
error("doportregister: %s: "
"DB Error getting registration for %s\n",
reqp->nodeid, service);
return 1;
}
if ((int)mysql_num_rows(res) == 0) {
mysql_free_result(res);
return 0;
}
row = mysql_fetch_row(res);
OUTPUT(buf, sizeof(buf), "PORT=%s NODEID=%s.%s\n",
row[0], row[1], OURDOMAIN);
client_writeback(sock, buf, strlen(buf), tcp);
if (verbose)
info("PORTREG: %s: %s", reqp->nodeid, buf);
return 0;
}
/*
* If port is zero, clear it from the DB
*/
if (port == 0) {
mydb_update("delete from port_registration "
"where pid='%s' and eid='%s' and "
" service='%s'",
reqp->pid, reqp->eid, service);
}
else {
/*
* Register port for the service.
*/
if (mydb_update("replace into port_registration set "
" pid='%s', eid='%s', "
" service='%s', node_id='%s', port='%d'",
reqp->pid, reqp->eid, service,
reqp->nodeid, port)) {
error("doportregister: %s: DB Error setting %s=%d!\n",
reqp->nodeid, service, port);
return 1;
}
}
if (verbose)
info("PORTREG: %s: %s=%d\n", reqp->nodeid, service, port);
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment