Commit 99943a19 authored by Mike Hibler's avatar Mike Hibler

Support for frisbee direct image upload to fs node.

We have had issues with uploading images to boss where they are then written
across NFS to ops. That seems to be a network hop too far on CloudLab Utah
where we have a 10Gb control network. We get occasional transcient timeouts
from somewhere in the TCP code. With the convoluted path through real and
virtual NICs, some with offloading, some without, packets wind up getting
out of order and someone gets far enough behind to cause problems.

So we work around it.

If IMAGEUPLOADTOFS is defined in the defs-* file, we will run a frisbee
master server on the fs (ops) node and the image creation path directs the
nodes to use that server. There is a new hack configuration for the master
server "upload-only" which is extremely specific to ops: it validates the
upload with the boss master server and, if allowed, fires up an upload
server for the client to talk to. The image will thus be directly uploaded
to the local (ZFS) /proj or /groups filesystems on ops. This seems to be
enough to get around the problem.

Note that we could allow this master server to serve downloads as well to
avoid the analogous problem in that direction, but this to date has not
been a problem.

NOTE: the ops node must be in the nodes table in the DB or else boss will
not validate proxied requests from it. The standard install procedure is
supposed to add ops, but we have a couple of clusters where it is not in
the table!
parent e4485562
......@@ -63,8 +63,11 @@ boss-install:
$(INSTALL) -m 644 $(SRCDIR)/install-tarfile.1 \
$(INSTALL_DIR)/opsdir/man/man1/install-tarfile.1
control: control-subdirs
control-install:
@$(MAKE) -C imagezip install
$(MAKE) -C frisbee.redux control-install
fs-install:
......
......@@ -43,6 +43,8 @@ bootinfo_version.c: bootinfoclient.c
install:
control:
control-install:
subboss:
subboss-install:
......
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -37,6 +37,7 @@ all: boss-all tipserv-all
boss-all: capserver
tipserv-all: capture capture-tty capquery caplogserver caplog caplog.bin
client: capture capture-nossl capquery caplog caplog.bin
control:
subboss: client
include $(TESTBED_SRCDIR)/GNUmakerules
......
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -47,6 +47,7 @@ dijkstra-debug: Compressor.o TreeCompressor.o dijkstra.o \
OptimalIpTree.o bitmath.o $(LIBS) -o $@
boss-install:
control:
control-install:
subboss:
subboss-install:
......
......@@ -88,14 +88,20 @@ WITH_MSERVER_NULL = 1
#ifndef WITH_MSERVER_EMULAB
WITH_MSERVER_EMULAB = 1
#endif
#ifndef WITH_MSERVER_UPLOAD
WITH_MSERVER_UPLOAD = 1
#endif
include $(OBJDIR)/Makeconf
all: frisbee frisbeed mfrisbeed frisupload frisuploadd
subboss:
control:
$(MAKE) WITH_MSERVER_EMULAB=0 all
subboss:
$(MAKE) WITH_MSERVER_EMULAB=0 WITH_MSERVER_UPLOAD=0 all
include $(TESTBED_SRCDIR)/GNUmakerules
IZSRCDIR = $(TESTBED_IMAGEZIPSRCDIR)
......@@ -132,8 +138,8 @@ SERVEROBJS = server.o $(SHAREDOBJS)
# Master server configuration
#
# Default master server config
MSERVERFLAGS = -DUSE_NULL_CONFIG $(CFLAGS) -I$(OBJDIR)
MSERVEROBJS = mserver.o $(SHAREDOBJS) config.o config_null.o
MSERVERFLAGS = -DUSE_NULL_CONFIG -DUSE_UPLOAD_CONFIG $(CFLAGS) -I$(OBJDIR)
MSERVEROBJS = mserver.o $(SHAREDOBJS) config.o config_null.o config_upload.o
ifeq ($(SYSTEM),Linux)
MSERVERLIBS = -lrt
else
......@@ -284,6 +290,8 @@ config_emulab.o: $(SRCDIR)/config_emulab.c configdefs.h log.h
$(CC) -c $(MSERVERFLAGS) $(SRCDIR)/config_emulab.c
config_null.o: $(SRCDIR)/config_null.c configdefs.h log.h
$(CC) -c $(MSERVERFLAGS) $(SRCDIR)/config_null.c
config_upload.o: $(SRCDIR)/config_upload.c configdefs.h log.h
$(CC) -c $(MSERVERFLAGS) $(SRCDIR)/config_upload.c
log.o: $(SRCDIR)/log.c decls.h log.h
$(CC) $(CFLAGS) -DLOG_TESTBED=$(LOG_TESTBED) -c $(SRCDIR)/log.c
......@@ -317,6 +325,10 @@ trace.o: decls.h trace.h log.h
install: $(INSTALL_SBINDIR)/mfrisbeed $(INSTALL_SBINDIR)/frisbeed $(INSTALL_SBINDIR)/frisbee $(INSTALL_SBINDIR)/frisuploadd
control-install: control
$(INSTALL_PROGRAM) mfrisbeed $(INSTALL_SBINDIR)/mfrisbeed
$(INSTALL_PROGRAM) frisupload $(INSTALL_SBINDIR)/frisupload
subboss-install: subboss
$(MAKE) WITH_MSERVER_EMULAB=0 install
$(INSTALL_PROGRAM) frisbee $(DESTDIR)/usr/local/bin/frisbee
......
......@@ -93,6 +93,18 @@ config_init(char *style, int readit, char *opts)
FrisLog("Null config init failed");
#else
FrisLog("Not built with Null configuration");
#endif
}
if (strcmp(style, "upload-only") == 0) {
#ifdef USE_UPLOAD_CONFIG
extern struct config *upload_init(char *);
if (myconfig == NULL) {
if ((myconfig = upload_init(opts)) != NULL)
FrisLog("Using upload-only configuration");
} else
FrisLog("Upload-only config init failed");
#else
FrisLog("Not built with Upload-only configuration");
#endif
}
if (myconfig == NULL) {
......
/*
* Copyright (c) 2010-2017 University of Utah and the Flux Group.
* Copyright (c) 2010-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -945,7 +945,9 @@ allow_stddirs(char *imageid,
ci->ngids = ei->ngids;
set_put_values(put, 0);
ci->extra = NULL;
}
} else if (put != NULL && debug)
FrisInfo("authfail: image '%s' not in acceptable directory for '%s/%s'",
imageid, ei->pid, ei->gid);
if (get != NULL &&
((fdir = isindir(shdir, fpath)) ||
(fdir = isindir(pdir, fpath)) ||
......@@ -981,7 +983,9 @@ allow_stddirs(char *imageid,
ci->ngids = ei->ngids;
set_get_values(get, 0);
ci->extra = NULL;
}
} else if (get != NULL && debug)
FrisInfo("authfail: image '%s' not in acceptable directory for '%s/%s'",
imageid, ei->pid, ei->gid);
done:
free(pdir);
......@@ -1376,6 +1380,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
#endif
proxy = emulab_nodeid(req);
if (proxy == NULL) {
if (debug)
FrisInfo("authfail: node %s not found",
inet_ntoa(*req));
emulab_free_host_authinfo(get);
emulab_free_host_authinfo(put);
return 1;
......@@ -1384,17 +1391,27 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
/* Check the role of the node */
role = emulab_noderole(proxy);
if (role == NULL) {
if (debug)
FrisInfo("authfail: node '%s' has no role", proxy);
free(proxy);
emulab_free_host_authinfo(get);
emulab_free_host_authinfo(put);
return 1;
}
/* Must be inner boss, subboss or virtnode host to proxy */
if (strcmp(role, "innerboss") &&
/* XXX ops appears as a regular node */
if (strcmp(role, "node") == 0 && strcmp(proxy, "ops") == 0)
role = mystrdup("opsnode");
/* Must be ops, inner boss, subboss or virtnode host to proxy */
if (strcmp(role, "opsnode") &&
strcmp(role, "innerboss") &&
strcmp(role, "subboss") &&
strcmp(role, "sharedhost") &&
strcmp(role, "virthost")) {
if (debug)
FrisInfo("authfail: node '%s' role '%s' not a proxy",
proxy, role);
free(proxy);
free(role);
emulab_free_host_authinfo(get);
......@@ -1460,7 +1477,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
res = NULL;
} else
#endif
if (strcmp(role, "subboss") == 0) {
if (strcmp(role, "opsnode") == 0) {
res = NULL;
} else if (strcmp(role, "subboss") == 0) {
res = mydb_query("SELECT s.node_id"
" FROM subbosses as s,"
" nodes as n"
......@@ -1497,6 +1516,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
if (res) {
if (mysql_num_rows(res) == 0) {
mysql_free_result(res);
if (debug)
FrisInfo("authfail: '%s' not a proxy for '%s'",
proxy, node);
free(role);
free(proxy);
free(node);
......@@ -1535,8 +1557,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
row = mysql_fetch_row(res);
if (!row[0] || !row[1] || !row[2] || !row[3] || !row[4] ||
!row[5] || !row[6] || !row[7] || !row[8]) {
FrisError("config_host_authinfo: null pid/gid/eid/uname/uid!?");
mysql_free_result(res);
FrisInfo("authfail: null pid/gid/eid/uname/uid for node '%s'!",
node);
emulab_free_host_authinfo(get);
emulab_free_host_authinfo(put);
free(node);
......@@ -1623,8 +1646,11 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
parse_imageid(imageid,
&wantpid, &wantname, &wantvers, &wantmeta);
imageidx = emulab_imageid(wantpid, wantname);
if (imageidx == 0)
if (imageidx == 0) {
if (debug)
FrisInfo("authfail: no such image '%s'", imageid);
goto done;
}
}
if (put != NULL) {
......@@ -1727,6 +1753,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
put->imageinfo =
mymalloc(nrows *
sizeof(struct config_imageinfo));
else if (debug)
FrisInfo("authfail: PUT access to '%s' denied to '%s/%s'",
imageid, ei->pid, ei->eid);
put->numimages = 0;
for (i = 0; i < nrows; i++) {
struct emulab_ii_extra_info *ii;
......@@ -1958,6 +1987,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
get->imageinfo =
mymalloc(nrows *
sizeof(struct config_imageinfo));
else if (debug)
FrisInfo("authfail: GET access to '%s' denied to '%s/%s'",
imageid, ei->pid, ei->eid);
get->numimages = 0;
for (i = 0; i < nrows; i++) {
struct emulab_ii_extra_info *ii;
......
/*
* Copyright (c) 2010-2017 University of Utah and the Flux Group.
* Copyright (c) 2010-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -984,6 +984,6 @@ mystrdup(const char *str)
struct config *
null_init(void)
{
return NULL;
return 0;
}
#endif
/*
* Copyright (c) 2010-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
/*
* This is a special case null config for the Emulab ops node for uploading images.
*
* - imageid must be a path (start with a '/')
* - paths have been validated by an upcall to our parent (boss)
* - paths must start with /proj/<pid>/images/ or /groups/<pid>/<gid>/images/
* - we do not use a tmp file when uploading, assume we are called by create_image
* - we run as the owner/group of the file if it exists, root otherwise
*/
#ifdef USE_UPLOAD_CONFIG
#include <sys/param.h>
#include <sys/stat.h>
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <assert.h>
#include "log.h"
#include "configdefs.h"
/* Emulab includes */
#include "config.h" /* the defs-* defines */
extern int debug;
static char *DEFAULT_PORTLOW = "0";
static char *DEFAULT_NUMPORT = "0";
static char *PROJDIR = PROJROOT_DIR;
static char *GROUPSDIR = GROUPSROOT_DIR;
static uint64_t put_maxsize = 10000000000ULL; /* zero means no limit */
static uint32_t put_maxwait = 2000; /* zero means no limit */
static uint32_t put_maxiwait = 120; /* zero means no limit */
static int port_lo, port_num;
/* Memory alloc functions that abort when no memory */
static void *mymalloc(size_t size);
static char *mystrdup(const char *str);
static void
upload_deinit(void)
{
}
static int
upload_read(void)
{
/* "Reading" the config file is a no-op. Just echo settings. */
FrisLog(" NO download capability");
FrisLog(" image_put_maxsize = %d GB",
(int)(put_maxsize/(1000*1000*1000)));
FrisLog(" image_put_maxwait = %d min",
(int)(put_maxwait/60));
FrisLog(" image_put_maxiwait = %d min",
(int)(put_maxiwait/60));
return 0;
}
static void *
upload_save(void)
{
static int dummy;
/* Just return non-zero value */
return (void *)&dummy;
}
static int
upload_restore(void *state)
{
FrisLog(" NO download capability");
FrisLog(" image_put_maxsize = %d GB",
(int)(put_maxsize/(1000*1000*1000)));
FrisLog(" image_put_maxwait = %d min",
(int)(put_maxwait/60));
FrisLog(" image_put_maxiwait = %d min",
(int)(put_maxiwait/60));
return 0;
}
static void
upload_free(void *state)
{
}
#if 0
/*
* Set the GET methods and options for a particular node/image.
* XXX should never be used.
*/
static void
set_get_values(struct config_host_authinfo *ai, int ix)
{
/* get_methods */
ai->imageinfo[ix].get_methods = CONFIG_IMAGE_UNKNOWN;
/* get_timeout */
ai->imageinfo[ix].get_timeout = 0;
/* get_options */
ai->imageinfo[ix].get_options = "";
/* and whack the put_* fields */
ai->imageinfo[ix].put_maxsize = 0;
ai->imageinfo[ix].put_timeout = 0;
ai->imageinfo[ix].put_itimeout = 0;
ai->imageinfo[ix].put_oldversion = NULL;
ai->imageinfo[ix].put_options = NULL;
/* and pget fields */
ai->imageinfo[ix].pget_options = NULL;
}
#endif
/*
* Set the PUT maxsize/options for a particular node/image.
* XXX right now these are completely pulled out of our posterior.
*/
static void
set_put_values(struct config_host_authinfo *ai, int ix)
{
struct config_imageinfo *ii = &ai->imageinfo[ix];
/* put_maxsize */
ii->put_maxsize = put_maxsize;
/* put_timeout */
ii->put_timeout = put_maxwait;
ii->put_itimeout = put_maxiwait;
/* put_oldversion -- setting to NULL means no tmpfile during upload */
ii->put_oldversion = NULL;
/* put_options */
ii->put_options = NULL;
/* and whack the get_* fields */
ii->get_methods = 0;
ii->get_timeout = 0;
ii->get_options = NULL;
/* and pget fields */
ii->pget_options = NULL;
}
#define FREE(p) { if (p) free(p); }
/*
* Free the dynamically allocated host_authinfo struct.
*/
static void
upload_free_host_authinfo(struct config_host_authinfo *ai)
{
int i;
if (ai == NULL)
return;
FREE(ai->hostid);
if (ai->imageinfo != NULL) {
for (i = 0; i < ai->numimages; i++) {
FREE(ai->imageinfo[i].imageid);
FREE(ai->imageinfo[i].path);
FREE(ai->imageinfo[i].sig);
FREE(ai->imageinfo[i].get_options);
FREE(ai->imageinfo[i].put_oldversion);
FREE(ai->imageinfo[i].put_options);
FREE(ai->imageinfo[i].pget_options);
FREE(ai->imageinfo[i].extra);
}
free(ai->imageinfo);
}
assert(ai->extra == NULL);
free(ai);
}
/*
* Return the IP address/port-range to be used by the server/clients for
* the image listed in ai->imageinfo[0]. Methods lists one or more transfer
* methods that the client can handle, we return the method chosen.
* If first is non-zero, then we need to return a "new" address and *addrp,
* *loportp, and *hiportp are uninitialized. If non-zero, then our last
* choice failed (probably due to a port conflict) and we need to choose
* a new address to try, possibly based on the existing info in *addrp,
* *loportp and *hiportp.
*
* Here we just keep a static index and increment through it for selecting
* a port number.
*
* Return zero on success, non-zero otherwise.
*/
static int
upload_get_server_address(struct config_imageinfo *ii, int methods, int first,
in_addr_t *addrp, in_port_t *loportp,
in_port_t *hiportp, int *methp)
{
static int myidx = 0;
/*
* Unicast is our ONLY choice.
*/
if ((methods & CONFIG_IMAGE_UCAST) == 0) {
FrisError("get_server_address: only support unicast");
return 1;
}
*methp = CONFIG_IMAGE_UCAST;
/* XXX on retries, we don't mess with the address */
if (first)
*addrp = 0;
/*
* Calculate a port range:
*
* If port_lo == 0, the server always chooses so we just return
* lo == hi == 0.
*
* Otherwise, if this is the first call, we use the index to compute
* a starting value in the [port_lo - port_lo+port_num]
* range. Given an increasing index, these keeps us from starting
* to look at the same place everytime and (hopefully) will reduce
* the number of conflicts that the server encounters.
*
* If this was not the first call, we return the full range on
* every call.
*/
if (port_num == 0) {
*loportp = 0;
*hiportp = 0;
} else if (first) {
*loportp = port_lo + (myidx % port_num);
*hiportp = port_lo + port_num - 1;
} else {
*loportp = port_lo;
*hiportp = port_lo + port_num - 1;
}
if (debug)
fprintf(stderr,
"get_server_address: idx %d, addr 0x%x, port [%d-%d]\n",
myidx, *addrp, *loportp, *hiportp);
myidx++;
return 0;
}
/*
* Find a specific image (imageid!=NULL) that a particular node can access
* for PUT. We do not allow GETs.
*
* PUTs calls must be for a single image, that image path must have been
* validated by an upcall to our parent (boss), and the path must start with
* either "/proj/<pid>/images/" or "/groups/<pid>/<gid>/images/".
*
* Return zero on success, non-zero otherwise.
*/
static int
upload_get_host_authinfo(struct in_addr *req, struct in_addr *host,
char *imageid,
struct config_host_authinfo **getp,
struct config_host_authinfo **putp)
{
struct config_host_authinfo *put;
struct config_imageinfo *ci;
struct stat sb;
int exists;
char rpath[PATH_MAX], *cp, *fdir;
int idlen;
/* Upload only! */
if (getp != NULL || putp == NULL) {
if (debug)
fprintf(stderr, "Only handle PUT requests\n");
return 1;
}
/* Must specify an image */
if (imageid == NULL || imageid[0] != '/') {
if (debug)
fprintf(stderr, "PUT requires a full path\n");
return 1;
}
/*
* We seriously limit what chars can be in an imageid since we
* use it as a file name. This check is based on the Emulab
* regex for image paths: "^[-_\w\.\/:+]+$". We need ',' as
* well to allow download of image signatures.
*/
idlen = strlen(imageid);
if (idlen == 0 || idlen >= PATH_MAX) {
if (debug)
fprintf(stderr, "imageid too short/long\n");
return 1;
}
for (cp = imageid; *cp != '\0'; cp++) {
if (isalnum(*cp) || *cp == '-' || *cp == '_' ||
*cp == '.' || *cp == '/' || *cp == ':' ||
*cp == '+' || *cp == ',')
continue;
if (debug)
fprintf(stderr,
"bogus char (0x%x) in imageid\n", *cp);
return 1;
}
/*
* Run the path through realpath and then verify that it is within
* /proj or /groups.
*
* At this point we cannot do a full path check since the
* full path need not exist and we are possibly running with
* enhanced privilege. So we only weed out obviously bogus
* paths here (possibly checking just the partial path
* returned by realpath) and mark the imageinfo as needed a
* full resolution later.
*/
if (myrealpath(imageid, rpath) == NULL) {
if (errno != ENOENT) {
return 1;
}
exists = 0;
} else
exists = 1;
if (debug)
FrisInfo("%s: exists=%d, resolves to: '%s'",
imageid, exists, rpath);
if ((fdir = isindir(PROJDIR, rpath)) == NULL &&
(fdir = isindir(GROUPSDIR, rpath)) == NULL)
return 1;
if (exists && stat(imageid, &sb) < 0)
exists = 0;
put = mymalloc(sizeof *put);
memset(put, 0, sizeof(*put));
/*
* XXX we don't care about the node identity right now.