Commit f37cd9dc authored by Leigh Stoller's avatar Leigh Stoller

Move rpm/tar download from boss to ops, to avoid wasted network traffic.

To turn this option on, define SPEWFROMOPS=1 in your defs file. This
will result in a redirect message from boss which will send the wget
client over to ops. 

A perl setuid root cgi script is run from the webserver on ops when a
/spewrpmtar request is made. This script sends the key,nodeid,file
over to boss via XMLRPC (as elabman). The return is simple yes or no,
the caller is allowed (not allowed) to have that file. Since the
ops script runs as root, it can spew the file back to the caller.

Note that the elabinelab checks for the elabinelab source code are
gone; we are now open source. Also, we spew that file from /share now,
to be consistent.
parent 8323da43
......@@ -97,8 +97,8 @@ LIBEXEC_STUFF = wanlinksolve wanlinkinfo os_setup mkexpdir console_setup \
assign_wrapper assign_wrapper2 os_setup_old \
assign_prepass ptopgen ptopgen_new \
spewlogfile staticroutes routecalc wanassign \
switchmac \
spewrpmtar webfrisbeekiller gentopofile \
switchmac spewrpmtar spewrpmtar_verify spewrpmtar_cgi \
webfrisbeekiller gentopofile \
$(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS)
LIB_STUFF = libtbsetup.pm exitonwarn.pm libtestbed.pm \
......@@ -121,7 +121,7 @@ SETUID_SBIN_SCRIPTS = mkproj rmgroup mkgroup frisbeehelper \
rmuser idleswap named_setup exports_setup \
sfskey_update setgroups newnode_reboot vnode_setup \
elabinelab nfstrace rmproj
SETUID_LIBX_SCRIPTS = console_setup
SETUID_LIBX_SCRIPTS = console_setup spewrpmtar_verify
SETUID_SUEXEC_SCRIPTS= spewlogfile
ifeq ($(SYSTEM),FreeBSD)
......@@ -237,6 +237,7 @@ script-install: $(addprefix $(INSTALL_BINDIR)/, $(BIN_STUFF)) \
$(addprefix $(INSTALL_LIBEXECDIR)/, $(LIBEXEC_STUFF)) \
$(addprefix $(INSTALL_DIR)/opsdir/lib/, libtestbed.pm) \
$(addprefix $(INSTALL_DIR)/opsdir/lib/, libtestbed.py) \
$(addprefix $(INSTALL_DIR)/opsdir/cgi-bin/, spewrpmtar_cgi) \
$(addprefix $(INSTALL_DIR)/opsdir/bin/, $(CTRLBIN_STUFF)) \
$(addprefix $(INSTALL_DIR)/opsdir/sbin/, $(CTRLSBIN_STUFF))\
$(addprefix $(INSTALL_DIR)/opsdir/sbin/, $(FSBIN_STUFF))
......@@ -387,6 +388,15 @@ $(INSTALL_DIR)/opsdir/sbin/%: %
-mkdir -p $(INSTALL_DIR)/opsdir/sbin
$(INSTALL) $< $@
$(INSTALL_DIR)/opsdir/cgi-bin/spewrpmtar_cgi: spewrpmtar_cgi
echo "Installing (link to wrapper) $<"
mkdir -p $(INSTALL_DIR)/opsdir/cgi-bin
-rm -f $@
ln -s $(INSTALL_LIBEXECDIR)/runsuid $@
echo "Installing (real script) $<"
-mkdir -p $(INSTALL_DIR)/opsdir/suidbin
$(SUDO) $(INSTALL_PROGRAM) $< $(INSTALL_DIR)/opsdir/suidbin/$<
compiled/%: %
@echo "Compiling $< to $@"
-mkdir -p compiled
......
......@@ -113,6 +113,7 @@ sub New($$$@)
# Init some per-node stuff.
$node->_rebooted(0);
$node->_vnodecount(0);
$node->_reloaded(0);
}
bless($self, $class);
......@@ -247,6 +248,7 @@ sub SetupReload($$$$)
$node->_loadimage($image);
}
$node->_setupoperation($RELOAD);
$node->_reloaded(1);
}
else {
die_noretry({type => 'primary', severity => SEV_ERROR,
......@@ -879,6 +881,8 @@ sub WaitForNodes($@)
next;
}
if (int($waittime / 60) > $minutes) {
# Changing minutes is why we get this print for just
# a single node each time.
$minutes = int($waittime / 60);
tbnotice("Still waiting for $node_id ($state) - ".
"it's been $minutes minute(s).\n");
......@@ -1700,6 +1704,11 @@ sub Volunteers($)
# The wait times are totally bogus! Need a better way to do this.
#
$node->_maxwait($reboot_time + (90 * $pnode->_vnodecount()));
# Add some time if the node is getting a reload. Also bogus.
$node->_maxwait($node->_maxwait() + 300)
if ($node->_reloaded());
$node->_seteupstatus($libossetup::SETUP_OKAY);
}
return @nodelist;
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use strict;
use English;
use Getopt::Std;
use POSIX qw(mktime);
use CGI;
#
# This is a CGI script that will return a tar/rpm file to a node.
# It used to be a backend to the web server on boss, but is now a
# CGI hosted on ops to avoid NFS transfer to boss which is wasteful.
#
# We run this setuid root cause we have to be able to see inside
# project and user directories. But we must be run as user "nobody"
# since that is who the web server runs as.
#
sub usage()
{
print STDERR "Usage: spewtarfile_cgi [-e]\n".
exit(-1);
}
my $optlist = "e";
my $debug = 1;
my $elabsrc = 0;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $USERROOT = "@USERSROOT_DIR@";
my $PROTOUSER = "elabman";
my $PROTOPROJ = "emulab-ops";
#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use libtestbed;
use libxmlrpc;
# Protos
sub SpewFile();
sub VerifyFile();
sub fatal($);
sub error($);
sub FlipToUser($$);
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
# Turn off line buffering on output
$| = 1;
#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
fatal("Must be setuid! Maybe its a development version?");
}
if ($UID && getpwuid($UID) ne "nobody") {
error("Only nobody or root can run this script!");
}
my $nodeid;
my $key;
my $file;
my $stamp;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"e"})) {
$elabsrc = 1;
}
if (@ARGV) {
# Allow root only debugging.
if (!$UID) {
$nodeid = shift(@ARGV);
$key = shift(@ARGV);
$file = shift(@ARGV);
$stamp = shift(@ARGV);
}
else {
usage();
}
}
else {
my $cgiquery = new CGI;
fatal("Could not create new CGI")
if (!defined($cgiquery));
$nodeid = $cgiquery->param('nodeid');
$key = $cgiquery->param('key');
$file = $cgiquery->param('file');
$stamp = $cgiquery->param('stamp');
$elabsrc = $cgiquery->param('elabinelab_source');
}
if ($elabsrc) {
#
# We are open source, so no need to check anything.
#
$file = "/share/emulab/emulab-src.tar.gz";
$stamp = undef;
if (! -e $file) {
fatal("elabinelab source tarball does not exist!");
}
SpewFile();
exit(0);
}
elsif (! (defined($nodeid) && defined($key) && defined($file))) {
error("Missing arguments");
}
VerifyFile();
SpewFile();
exit(0);
#
# Spew out a file.
#
sub SpewFile()
{
#
# Stat the file get the length.
#
my (undef,undef,undef,undef,undef,undef,undef,$bytelen) = stat($file);
open(FD, "< $file")
or error("Could not open $file!\n");
print "Content-Type: application/octet-stream\n";
print "Content-Length: $bytelen\n";
print "Expires: Mon, 26 Jul 1997 05:00:00 GMT\n";
print "Cache-Control: no-cache, must-revalidate\n";
print "Pragma: no-cache\n";
print "\n";
#
# Deal with NFS read failures (OPS/FS split).
#
my $foffset = 0;
my $retries = 5;
my $buf;
while ($bytelen) {
my $rlen = sysread(FD, $buf, 8192);
if (! defined($rlen)) {
#
# Retry a few times on error to avoid the
# changing-exports-file server problem.
#
if ($retries > 0 && sysseek(FD, $foffset, 0)) {
$retries--;
sleep(1);
next;
}
fatal("Error reading $file: $!");
}
if ($rlen == 0) {
last;
}
if (! syswrite(STDOUT, $buf, $rlen)) {
fatal("Error writing file to stdout: $!");
}
$foffset += $rlen;
$bytelen -= $rlen;
$retries = 5;
}
if ($bytelen) {
fatal("Did not get the entire file! $bytelen bytes left.");
}
close(FD);
return 0;
}
#
# Verify that we can return this file, return error if not allowed.
# Otherwise return 0 for okay.
#
sub VerifyFile()
{
#
# Confirm with the RPC server. We have to do this as elabman
# since it can talk to the RPC server on boss.
#
my $childpid = fork();
fatal("fork failed")
if (!defined($childpid));
if (! $childpid) {
FlipToUser($PROTOUSER, $PROTOPROJ);
my $response = libxmlrpc::CallMethod0("node", "spewrpmtar_verify",
{"node" => $nodeid,
"key" => $key,
"file" => $file});
if (!defined($response)) {
fatal("No response from RPC server!");
}
if ($response->{"code"}) {
my $msg = "Denied by RPC server: " . $response->{"code"};
if (defined($response->{"output"}) && $response->{"output"} ne "") {
$msg .= "\n" . $response->{"output"};
}
error($msg);
}
# Tell parent life is good.
exit(0);
}
#
# Wait for child. If the child return non-zero status, just exit
# cause it already printed out the response to the web server.
#
waitpid($childpid, 0);
if ($?) {
exit(1);
}
#
# Stat the file get the mtime.
#
my (undef,undef,undef,undef,undef,undef,undef,undef,
undef,$mtime) = stat($file);
#
# Check timestamp if supplied. Remember, we get GM timestamps, so
# must convert the local stamp.
#
if (defined($stamp)) {
$mtime = mktime(gmtime($mtime));
if ($stamp >= $mtime) {
print "Content-Type: text/plain\n";
print "Status: 304 File has not changed\n\n";
exit(0);
}
}
return 0;
}
sub error($)
{
my ($msg) = @_;
print "Content-Type: text/plain\n";
print "Status: 400 Bad Request\n\n";
print "$msg\n";
exit 1;
}
sub fatal($)
{
my ($msg) = @_;
SENDMAIL($TBOPS, "spewrpmtar:$file", $msg);
print "Content-Type: text/plain\n";
print "Status: 400 Bad Request\n\n";
print "$msg\n";
exit 1;
}
#
#
#
sub FlipToUser($$)
{
my ($user, $group) = @_;
my $glist;
my $default_gid;
my $unix_uid = getpwnam("$user");
if (!defined($unix_uid)) {
fatal("*** FlipToUser: No such user $user");
}
my $unix_gid = getgrnam("$group");
if (!defined($unix_gid)) {
fatal("*** FlipToUser: No such group $group");
}
$default_gid = $unix_gid;
$glist = "$unix_gid $unix_gid";
$GID = $default_gid;
$EGID = $glist;
$EUID = $UID = $unix_uid;
$ENV{'USER'} = $user;
$ENV{'LOGNAME'} = $user;
$ENV{'HOME'} = "$USERROOT/$user";
return 0;
}
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use strict;
use English;
use Getopt::Std;
use POSIX qw(mktime);
#
# Spew a tar/rpm file to stdout.
#
# The script is setuid and run from the webserver.
#
sub usage()
{
print STDERR "Usage: spewrpmtar_verify <key> <nodeid> <file>\n";
exit(-1);
}
my $optlist = "";
my $debug = 1;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use Node;
use Experiment;
# Protos
sub VerifyFile();
sub fatal($);
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
# Turn off line buffering on output
$| = 1;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV != 3) {
usage();
}
my $key = $ARGV[0];
my $nodeid = $ARGV[1];
my $file = $ARGV[2];
#
# Untaint the arguments.
#
if ($nodeid =~ /^([-\w]+)$/) {
$nodeid = $1;
}
else {
die("*** Bad data in nodeid: $nodeid\n");
}
# Note different taint check (allow /).
if ($file =~ /^([-\@\w\.\/]+)$/) {
$file = $1;
}
else {
die("*** Bad data in argument: $file");
}
my $node = Node->Lookup($nodeid);
if (!defined($node)) {
die("*** $0:\n".
" $nodeid does not exist!\n");
}
my $experiment = $node->Reservation();
if (!defined($experiment)) {
die("*** $0:\n".
" $nodeid is not allocated to an experiment!\n");
}
my $pid = $experiment->pid();
my $eid = $experiment->eid();
my $gid = $experiment->gid();
my $creator = $experiment->GetCreator()->uid();
my $unix_gid= $experiment->GetGroup()->unix_gid();
#
# We need the secret key to match
#
if (!$experiment->keyhash() || $experiment->keyhash() eq "") {
fatal("No keyhash defined for $experiment!");
}
exit(1)
if ($experiment->keyhash() ne $key);
exit(VerifyFile());
#
# Verify that we can return this file, return error if not allowed.
# Otherwise return 0 for okay.
#
sub VerifyFile()
{
#
# First make sure the file is in the rpm or tar list for the node,
# and that it exists and we can read it.
#
if (!VerifyTar() && !VerifyRPM()) {
if ($debug) {
print STDERR "VerifyFile: Could not verify $file!\n";
}
return 1;
}
#
# Now a few other checks.
#
# Use realpath to remove any symlinks to make sure we are not going
# to hand out a file outside the appropriate files systems.
#
my $translated = `realpath $file`;
if ($translated =~ /^([-\@\w\.\/]+)$/) {
$translated = $1;
}
else {
fatal("Bad data returned by realpath: $translated");
}
#
# The file must reside in /proj/$pid/$eid, /groups/$pid/$gid
# or /scratch/$pid. Don't allow anything from /users!
#
if (! TBValidUserDir($translated, 0, undef, $pid, $gid)) {
if ($debug) {
print STDERR "$translated is not in ",
join(' or ', TBValidUserDirList(undef, $pid, $gid)),
".\n";
}
return 1;
}
#
# Stat the file to confirm that its either owned by the experiment
# creator, or in the gid of the experiment.
#
my (undef,undef,undef,undef,$stat_uid,$stat_gid,undef,$length,
undef,undef) = stat($translated);
my (undef,undef,$unix_uid) = getpwnam($creator) or
fatal("No such user $creator\n");
if ($stat_gid != $unix_gid &&
$stat_uid != $unix_uid) {
if ($debug) {
print STDERR "$translated has wrong uid/gid!\n";
}
return 1;
}
return 0;
}
#
# Check the DB to make sure this is a valid TAR/RPM file for the node.
# Must pass a number of other checks too.
#
sub VerifyTar()
{
#
# Get the tarball list from the DB. The requested path must be
# on the list of tarballs for this node.
#
my $query_result =
DBQueryFatal("select tarballs from nodes where node_id='$nodeid'");
# No rpms/tarballs for the node in question.
return 0
if (!$query_result->numrows);
#
# The format is a colon separated list of "dir filename". We must find
# the filename in the list.
#
my ($tarballs) = $query_result->fetchrow_array();
foreach my $tarspec (split(":", $tarballs)) {
my ($dir, $tar) = split(" ", $tarspec);
return 1
if ($tar eq $file && -r $tar);
}
return 0;
}
sub VerifyRPM()
{
my $query_result =
DBQueryFatal("select rpms from nodes where node_id='$nodeid'");
# No rpms/tarballs for the node in question.
return 0
if (!$query_result->numrows);
#
# The format is a colon separated list of filenames. We must find
# the filename in the list.
#
my ($rpms) = $query_result->fetchrow_array();
foreach my $rpm (split(":", $rpms)) {
return 1
if ($rpm eq $file && -r $rpm);
}
return 0;
}
sub fatal($)
{
my ($msg) = @_;
SENDMAIL($TBOPS, "spewrpmtar_verify:$file", $msg);
die("*** $0:\n".
" $msg\n");
}
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2010 University of Utah and the Flux Group.
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -271,13 +271,20 @@ foreach my $node (@nodes) {
next;
}
elsif ($allocstate eq TBDB_ALLOCSTATE_DOWN) {
if (!$plab) {
if ($plab) {
# Plab nodes need to be cleaned up.
print "$node failed to boot; changing to cleanup.\n";
$mode = "cleanup";
}
elsif (!$nodeobj->IsUp()) {
# Node can fail to boot, but can still wind up
# booting later, say by hand.
print "$node appears to be up; will $mode.\n";
}
else {
print "$node failed to boot; skipping $mode.\n";
next;
}
# Plab nodes need to be cleaned up.
print "$node failed to boot; changing to cleanup.\n";
$mode = "cleanup";
}
}
elsif ($exptstate eq EXPTSTATE_ACTIVATING ||
......@@ -323,13 +330,21 @@ foreach my $node (@nodes) {
next;
}
elsif ($allocstate eq TBDB_ALLOCSTATE_DOWN) {
if (!$plab) {
if ($plab) {
# Plab nodes need to be cleaned up.
print "$node failed to boot; ".
"changing to cleanup.\n";
$mode = "cleanup";
}
elsif (!$nodeobj->IsUp()) {
# Node can fail to boot, but can still wind up
# booting later, say by hand.
print "$node appears to be up; will $mode.\n";
}
else {
print "$node failed to boot; skipping $mode.\n";
next;
}
# Plab nodes need to be cleaned up.
print "$node failed to boot; changing to cleanup.\n";
$mode = "cleanup";
}
elsif ($allocstate eq TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
print "$node never booted; skipping $mode.\n";
......
......@@ -80,6 +80,7 @@ $NONAMEDSETUP = @DISABLE_NAMED_SETUP@;
$OPS_VM = @OPSVM_ENABLE@;
$PORTAL_ENABLE = @PORTAL_ENABLE@;
$PORTAL_ISPRIMARY = @PORTAL_ISPRIMARY@;
$SPEWFROMOPS = @SPEWFROMOPS@;
$TBMAILADDR_OPS = "@TBOPSEMAIL_NOSLASH@";
$TBMAILADDR_WWW = "@TBWWWEMAIL_NOSLASH@";
......
......@@ -58,6 +58,15 @@ $optargs = OptionalPageArguments("elabinelab_source", PAGEARG_STRING,
"stamp", PAGEARG_INTEGER,
"md5", PAGEARG_STRING,
"cvstag", PAGEARG_STRING);
#
# Move this to ops, except for elabinelab source code with cvstag.
#
if ($SPEWFROMOPS && (! (isset($elabinelab_source) && isset($cvstag)))) {
$query_string = $_SERVER['QUERY_STRING'];
header("Location: https://$USERNODE/spewrpmtar?". $query_string);
return;
}
$node_id = $node->node_id();
#
......
......@@ -4259,6 +4259,43 @@ class node:
return EmulabResponse(RESPONSE_SUCCESS,value=retval)
#
# Verify request to download a tar/rpm file.
#
def spewrpmtar_verify(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")