genilib-iocage.in 12.1 KB
Newer Older
1 2 3
#!/usr/bin/perl -w

#
4
# Copyright (c) 2015-2019 University of Utah and the Flux Group.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use strict;
use English;
use Getopt::Std;
use BSD::Resource;
use POSIX qw(:signal_h);
use File::Basename;
    
#
# Fire up an instance of a FreeBSD jail to run a geni-lib script.
# Since we must be invoked as root, the caller is responsible for all
# permission checks.
#
# We use iocage to manage jails. We have a template jail instance that
# has python, geni-lib, etc. in it. We clone that per-call to execute
# the geni-lib script. Pretty fast since ZFS is used to clone the filesystem
# and the jail runs essentially no daemons.
#
# XXX Note that although iocage supports resource limits, we don't use them
# right now because it requires a non-standard option to the kernel (that
# we don't have compiled in). So for the moment, we let our caller apply
# any CPU limits. Here are the limits set on the template (in the event we
# someday start using them):
#	cputime="600:sigkill"
#	maxproc="100:deny"
#	memoryuse="1G:deny"
# see rctl(8) for details.
#
# XXX Note that we don't use the passed-in user either. We just run the
# script as "nobody" in the jail. We can add the user's account to the
# jail if that proves necessary.
#

my $CMD_TIMEOUT = 600;
my $CMD_QUOTA	= "2G";

my $IOCAGE = "/usr/local/sbin/iocage";
Mike Hibler's avatar
Mike Hibler committed
61 62
my $PROG = $0;
my $INTERP = "nice -15 /usr/local/bin/python";
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
#my $INTERP = "/bin/sh";

# this is a ZFS snapshot that we make of the base FS
my $JAILROOT = "/iocage/jails";

my $jailattrs = "compression=off quota=$CMD_QUOTA exec_timeout=$CMD_TIMEOUT exec_clean=0";
# XXX for now
$jailattrs .= " rlimits=off";
# XXX speed things up
$jailattrs .= " vnet=off ip6=disable";
# XXX when cloning, hostname is not set 
#$jailattrs .= " hostname=py-cage.emulab.net host_hostname=py-cage.emulab.net";
# XXX maybe not a good idea
#$jailattrs .= " exec_start=/usr/bin/true exec_stop=/usr/bin/true";

my $starttime = time();

sub usage()
{
    print STDOUT
Mike Hibler's avatar
Mike Hibler committed
83 84 85 86 87 88 89 90 91 92 93
	"Usage: $PROG [-d] [-n jailname] [-u user] [-p paramfile] [-o outfile] script\n".
	"   or: $PROG [-CR] [-n jailname]\n".
        "Execute the given geni-lib script in a jail or just create/remove a jail\n".
	"Options:\n".
	"  -u user      User to run script as.\n".
	"  -p paramfile JSON params to pass to script.\n".
	"  -o outfile   File in which to place script results.\n".
	"  -n jailname  Name of jail; default is 'py-cage-<pid>'.\n".
	"  -d           Turn on debugging.\n".
	"  -C           Just create the jail; use 'jexec' to run commands.\n".
	"  -R           Remove an existing (left-over) jail; must specify a name (-n)\n";
94 95 96
 
    exit(-1);
}
Mike Hibler's avatar
Mike Hibler committed
97
my $optlist = "du:p:o:n:CR";
98 99 100 101 102 103 104 105
my $jailname = "py-cage-$$";
my $user = "nobody";
my $uid;
my $gid;
my $pfile;
my $ofile;
my $ifile;

Mike Hibler's avatar
Mike Hibler committed
106 107 108
# action: 1: create, 2: destroy, 4: run script
my $action = 4;

109 110 111 112 113
sub msg(@);

#
# Configure variables
#
Mike Hibler's avatar
Mike Hibler committed
114 115
my $TBROOT   = "@prefix@";
my $GENILIB  = "$TBROOT/lib/geni-lib/";
116 117
my $debug    = 0;

118 119 120 121 122
# Watch for this being defined in the calling environment and use that.
if (exists($ENV{"PYTHONPATH"})) {
    $GENILIB = $ENV{"PYTHONPATH"};
}

123 124 125 126 127 128 129 130 131 132
#
# Turn off line buffering on output
#
$| = 1;

#
# Untaint the path
# 
$ENV{'PATH'} = "/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
Mike Hibler's avatar
Mike Hibler committed
133 134 135 136

# Where geni-lib should be mounted in the jail
my $GENILIB_MNT = "/usr/testbed/lib/geni-lib/";
$ENV{"PYTHONPATH"} = $GENILIB_MNT;
137 138 139 140 141 142 143 144 145

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libtestbed;

my $jailtag;
my $jailstate = 0;
Mike Hibler's avatar
Mike Hibler committed
146
my $jailuuid;
147 148

END {
Mike Hibler's avatar
Mike Hibler committed
149 150
    if ($jailstate) {
	my $ecode = $?;
151

Mike Hibler's avatar
Mike Hibler committed
152 153
	if ($ecode) {
	    print STDERR "Unexpected exit, cleaning up...\n";
154
	}
Mike Hibler's avatar
Mike Hibler committed
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
	if ($action != 1 || $ecode) {
	    if ($jailstate == 2) {
		msg("Stopping jail");
		if (system("$IOCAGE stop $jailtag")) {
		    print STDERR "*** could not stop jail $jailtag\n";
		}
		msg("Stopping done");
		$jailstate = 1;
	    }
	    if ($jailstate == 1) {
		msg("Destroying jail");

		# XXX make sure special FSes get unmounted 
		system("umount $JAILROOT/$jailuuid/root$GENILIB_MNT >/dev/null 2>&1")
		    if ($jailuuid);

		if (system("$IOCAGE destroy -f $jailtag")) {
		    print STDERR "*** could not destroy jail $jailtag\n";
		}
		msg("Destroying done");
		$jailstate = 0;
176 177
	    }
	}
Mike Hibler's avatar
Mike Hibler committed
178
	$? = $ecode;
179 180 181 182 183 184 185 186 187 188 189
    }
}

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
Mike Hibler's avatar
Mike Hibler committed
190 191 192 193 194 195
if (defined($options{"C"})) {
    $action = 1;
}
if (defined($options{"R"})) {
    $action = 2;
}
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"p"})) {
    $pfile = $options{"p"};
}
if (defined($options{"o"})) {
    $ofile = $options{"o"};
}
if (defined($options{"u"})) {
    $user = $options{"u"};
}
if (defined($options{"n"})) {
    $jailname = $options{"n"};
    if ($jailname !~ /^[-\w]+$/) {
	print STDERR "Come on, keep the name simple...\n";
	usage();
    }
}

Mike Hibler's avatar
Mike Hibler committed
216 217 218 219 220 221 222 223
#
# Extract params from the environment (if invoked via rungenilib.proxy).
#
if (!$ofile && exists($ENV{'GENILIB_PORTAL_REQUEST_PATH'})) {
    $ofile = $ENV{'GENILIB_PORTAL_REQUEST_PATH'};
}
if (!$pfile && exists($ENV{'GENILIB_PORTAL_PARAMS_PATH'})) {
    $pfile = $ENV{'GENILIB_PORTAL_PARAMS_PATH'};
224 225
}

Mike Hibler's avatar
Mike Hibler committed
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
if ($action == 4) {
    if (@ARGV < 1) {
	print STDERR "Must specify a script\n";
	usage();
    }
    $ifile = $ARGV[0];

    if (!$ofile) {
	print STDERR "Must specify an output file (-o)\n";
	usage();
    }
} else {
    if (@ARGV != 0) {
	print STDERR "Too many args for create/destroy\n";
	usage();
    }
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
}

# Must be a legit user
(undef,undef,$uid,$gid) = getpwnam($user) or
    die("*** $0:\n".
	"    Invalid user '$user'!");

# Must run as root
if ($UID != 0) {
    die("*** $0:\n".
	"    Must be root to run this script!");
}

# Make sure jail infrastructure is there
if (! -d "$JAILROOT") {
    die("*** $0:\n".
	"    $JAILROOT does not exist; is iocage installed?");
}

Mike Hibler's avatar
Mike Hibler committed
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
#
# Action checks.
# If only creating (-C) or running normally, then jail should not exist.
# If only removing (-R), then jail should exist.
#
# XXX we have to do an 'iocage list' to determine existance and that is
# expensive, so we just don't do it.
#
#if ($action != 2 && -e "$JAILROOT/$jailname") {
#    die("*** $0:\n    $jailname already exists");
#}

#
# Removing an existing jail, just set the jailstate and exit.
#
if ($action == 2) {
    my $info = `$IOCAGE list | grep $jailname`;
    if ($info =~ /\s(\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s/) {
	$jailuuid = $1;
    } else {
	die("*** $0:\n    $jailname does not exist");
    }
    $jailtag = $jailname;
    $jailstate = 2;
    exit(0);
}

288 289 290 291 292 293 294 295 296 297 298 299
#
# Fire up a jail instance
#
msg("Cloning py-cage");
if (system("$IOCAGE clone py-cage tag=$jailname $jailattrs")) {
    print STDERR "Could not create geni-lib jail\n";
    exit(1);
}
msg("Cloning done");
$jailstate = 1;
$jailtag = $jailname;

Mike Hibler's avatar
Mike Hibler committed
300 301 302
# XXX we have to explicitly set this after cloning!
system("$IOCAGE set hostname=$jailtag.emulab.net $jailtag");

303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
#
# Find the UUID so we can locate the ZFS
# list output:
# -     2b0ecbff-3f6c-11e5-a21b-38eaa71273f9  off   down   py-cage-34420
#
msg("Finding UUID");
my $info = `$IOCAGE list | grep $jailtag`;
msg("Finding done");
if ($info =~ /\s(\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s/) {
    $jailuuid = $1;
} else {
    print STDERR "Could not find UUID for geni-lib jail\n";
    exit(1);
}
my $jailrootdir = "$JAILROOT/$jailuuid/root";
if (! -d $jailrootdir) {
    print STDERR "Could not find root FS for $jailtag ($jailuuid)\n";
    exit(1);
}

Mike Hibler's avatar
Mike Hibler committed
323 324 325 326 327 328 329 330 331 332
#
# XXX mount host geni-lib in jail
# This way we don't have to copy in the geni-lib files and they
# will always be up-to-date. Also make dev trees work.
#
if (system("mount -t nullfs -o ro $GENILIB $jailrootdir$GENILIB_MNT")) {
    print STDERR "Could not mount $GENILIB in $jailtag\n";
    exit(1);
}

333 334 335 336 337
#
# Drop our files into the jail FS
# We create a protected directory owned by the user so that people
# cannot snoop from outside the jail.
#
Mike Hibler's avatar
Mike Hibler committed
338 339 340
my ($j_ifile,$j_ofile,$j_pfile);
if ($action != 1) {
    my $tempdir = "/tmp/genilib/";
341
    if (!mkdir("$jailrootdir$tempdir", 0700)) {
Mike Hibler's avatar
Mike Hibler committed
342 343 344
	print STDERR "Could not create geni-lib jail tempdir\n";
	exit(1);
    }
345

Mike Hibler's avatar
Mike Hibler committed
346 347 348 349
    $j_ifile = $tempdir . basename($ifile);
    $j_ofile = $tempdir . basename($ofile);
    $j_pfile = $tempdir . basename($pfile)
	if ($pfile);
350

Mike Hibler's avatar
Mike Hibler committed
351 352
    msg("Stashing files");
    if (system("cp -p $ifile $jailrootdir$j_ifile") ||
353 354
	($pfile && system("cp -p $pfile $jailrootdir$j_pfile")) ||
	system("chown -R $uid:$gid $jailrootdir$tempdir")) {
Mike Hibler's avatar
Mike Hibler committed
355 356 357
	print STDERR "Could not populate jail\n";
	exit(1);
    }
358

Mike Hibler's avatar
Mike Hibler committed
359 360 361
    #
    # XXX adjust the environment for the portal module to reflect the jail.
    #
362 363 364 365
    $ENV{'GENILIB_PORTAL_REQUEST_PATH'} = $j_ofile;
    if (exists($ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'})) {
	$ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'} = $j_ofile;
    }
Mike Hibler's avatar
Mike Hibler committed
366
    if ($pfile) {
367 368 369
	$ENV{'GENILIB_PORTAL_PARAMS_PATH'} = $j_pfile;
    }

Mike Hibler's avatar
Mike Hibler committed
370 371 372 373 374 375 376 377 378 379 380 381 382
    #
    # Make other aspects of the environment appear a little bit sane.
    # XXX okay, not really THAT sane since the user will not exist in the jail.
    #
    $ENV{'SHELL'} = "/bin/sh";
    $ENV{'USER'} = $user;
    $ENV{'USERNAME'} = $user;
    $ENV{'LOGNAME'} = $user;
    $ENV{'HOME'} = $tempdir;
    delete $ENV{'DISPLAY'};
    delete @ENV{'SUDO_UID', 'SUDO_GID', 'SUDO_USER', 'SUDO_COMMAND'};
    delete @ENV{'SSH_AUTH_SOCK', 'SSH_CLIENT', 'SSH_CONNECTION', 'SSH_TTY'};
}
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402

#
# Fire up the jail
#
msg("Start jail");
if (system("$IOCAGE start $jailtag")) {
    print STDERR "Could not start jail $jailtag\n";
    exit(1);
}
msg("Starting done");
$jailstate = 2;

#
# And execute the command as the indicated jail user.
#
# XXX currently we run the command as the real user (-u), but note that
# they will have no passwd entry inside the jail. May cause problems.
# If so, we can run as nobody in the jail (-U nobody) or add the real
# user to the jail (but that is a lot of work for one command...)
#
Mike Hibler's avatar
Mike Hibler committed
403 404 405 406 407 408 409 410
my $status = -1;
if ($action != 1) {
    msg("Execing command");
    $status = system("$IOCAGE exec -u $user $jailtag $INTERP $j_ifile");
    msg("Execing done");
} else {
    $status = 0;
}
411 412 413 414 415 416 417 418 419 420 421 422

if ($status) {
    if ($status == -1) {
	print STDERR "Could not run jail $jailtag\n";
    } elsif ($status & 127) {
	$status &= 127;
	print STDERR "Jail $jailtag execution died with signal $status\n";
	$status = 2;
    } else {
	$status >>= 8;
	print STDERR "Jail $jailtag execution failed with exit code $status\n";
    }
Mike Hibler's avatar
Mike Hibler committed
423 424 425 426 427 428 429

    # XXX odd semantics: if debug is set, don't remove jail on error
    if ($debug) {
	print STDERR "WARNING: not destroying jail, you will need to do it:\n".
	    "    sudo $PROG -R -n $jailtag\n";
	$jailstate = 0;
    }
430 431
}

Mike Hibler's avatar
Mike Hibler committed
432
if ($action != 1) {
433 434 435 436 437 438 439 440 441 442
    #
    # Oh the joys of running as root. Now we need to take away user
    # permission from the jail directory (recall the user can access
    # it from outside) and then verify that the source file isn't a
    # symlink (a cheap-o realpath check). Our caller is responsible
    # for defending the target file.
    #
    my $tempdir = "/tmp/genilib";
    if (-l "$jailrootdir$tempdir" ||
	chown(0, -1, "$jailrootdir$tempdir") != 1) {
Mike Hibler's avatar
Mike Hibler committed
443 444 445
	print STDERR "Could not copy back results of command\n";
	exit(1);
    }
446 447 448 449 450 451 452
    if (-e "$jailrootdir$j_ofile") {
	if (-l "$jailrootdir$j_ofile" ||
	    system("cp $jailrootdir$j_ofile $ofile")) {
	    print STDERR "Could not copy back results of command\n";
	    exit(1);
	}
    }
Mike Hibler's avatar
Mike Hibler committed
453 454
} else {
    print STDERR "Jail '$jailtag' running. Root FS at '$jailrootdir'.\n";
455 456
}

Mike Hibler's avatar
Mike Hibler committed
457
exit($status);
458 459 460

sub msg(@)
{
Mike Hibler's avatar
Mike Hibler committed
461 462 463 464 465
    if ($debug) {
	my $stamp = time() - $starttime;
	printf STDERR "[%3d] ", $stamp;
	print STDERR @_, "\n";
    }
466
}