genilib-jail.in 14.9 KB
Newer Older
1
2
3
#!/usr/bin/perl -w

#
4
# Copyright (c) 2015-2016 University of Utah and the Flux Group.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use strict;
use English;
use Getopt::Std;
use BSD::Resource;
use POSIX qw(:signal_h);
use File::Basename;
    
#
# Fire up an instance of a FreeBSD jail to run a geni-lib script.
# Since we must be invoked as root, the caller is responsible for all
# permission checks.
#
# This is the simple roll-our-own version. We take advantage of a template
# jail filesystem setup by iocage, but don't use iocage for anything else
# as it is toooo slooooow. We clone this FS per-call to execute the geni-lib
# script and setup a basic jail with no daemons, no network, etc.
#
# XXX This version does not support any resource limits other that a quota
# on disk use (ZFS property) and a per-command time limit (jail config).
#
# XXX Note that we don't use the passed-in user. We just run the script as
# "nobody" in the jail.
#

my $USEJAILRUN = 0;

my $CMD_TIMEOUT = 600;
my $CMD_QUOTA	= "2G";

Mike Hibler's avatar
Mike Hibler committed
54
55
my $PROG = $0;
my $INTERP = "nice -15 /usr/local/bin/python";
56
57
58
59
60
61
62
63
64
65
66
67
68
#my $INTERP = "/bin/sh";

# this is a ZFS snapshot that we make of the base FS
my $JAILROOT = "/iocage/jails";
my $ZFSBASE =  "z/iocage/jails";

my $zfsattrs = "-o compression=off -o quota=$CMD_QUOTA";

my $starttime = time();

sub usage()
{
    print STDOUT
Mike Hibler's avatar
Mike Hibler committed
69
70
71
72
73
74
75
76
77
78
	"Usage: $PROG [-d] [-n jailname] [-u user] [-p paramfile] [-o outfile] script\n".
	"   or: $PROG [-CR] [-n jailname]\n".
        "Execute the given geni-lib script in a jail or just create/remove a jail\n".
	"Options:\n".
	"  -u user      User to run script as.\n".
	"  -p paramfile JSON params to pass to script.\n".
	"  -o outfile   File in which to place script results.\n".
	"  -n jailname  Name of jail; default is 'py-cage-<pid>'.\n".
	"  -d           Turn on debugging.\n".
	"  -C           Just create the jail; use 'jexec' to run commands.\n".
79
80
	"  -R           Remove an existing (left-over) jail; must specify a name (-n)\n".
	"  -B base      Which base to use (see /iocage/tags); default: 'py-cage'.\n";
81
82
83
 
    exit(-1);
}
84
85
my $optlist = "du:p:o:n:CRB:";
my $basename = "py-cage";
86
87
88
89
90
91
92
93
my $jailname = "py-cage-$$";
my $user = "nobody";
my $uid;
my $gid;
my $pfile;
my $ofile;
my $ifile;

Mike Hibler's avatar
Mike Hibler committed
94
95
96
# action: 1: create, 2: destroy, 4: run script
my $action = 4;

97
98
99
sub start_jail($$);
sub run_jail($$$$);
sub msg(@);
100
sub mysystem($);
101
102
103
104

#
# Configure variables
#
Mike Hibler's avatar
Mike Hibler committed
105
106
my $TBROOT   = "@prefix@";
my $GENILIB  = "$TBROOT/lib/geni-lib/";
107
108
109
110
111
112
113
114
115
116
117
118
my $debug    = 0;

#
# Turn off line buffering on output
#
$| = 1;

#
# Untaint the path
# 
$ENV{'PATH'} = "/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
Mike Hibler's avatar
Mike Hibler committed
119
120
121
122

# Where geni-lib should be mounted in the jail
my $GENILIB_MNT = "/usr/testbed/lib/geni-lib/";
$ENV{"PYTHONPATH"} = $GENILIB_MNT;
123
124
125
126
127
128
129
130
131

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libtestbed;

my $jailtag;
my $jailstate = 0;
Mike Hibler's avatar
Mike Hibler committed
132
133
my $jailuuid;
my $snapshot;
134
135

END {
Mike Hibler's avatar
Mike Hibler committed
136
137
    if ($jailstate) {
	my $ecode = $?;
138

139
	if ($debug && $ecode) {
Mike Hibler's avatar
Mike Hibler committed
140
	    print STDERR "Unexpected exit, cleaning up...\n";
141
	}
Mike Hibler's avatar
Mike Hibler committed
142
143
144
	if ($action != 1 || $ecode) {
	    if ($jailstate == 2) {
		msg("Stopping jail");
145
		if (mysystem("jail -qr $jailtag")) {
Mike Hibler's avatar
Mike Hibler committed
146
147
148
149
150
151
152
153
154
		    print STDERR "*** could not stop jail $jailtag\n";
		}
		msg("Stopping done");
		$jailstate = 1;
	    }
	    if ($jailstate == 1) {
		msg("Destroying jail FS");

		# XXX make sure special FSes get unmounted 
155
156
157
		mysystem("umount $JAILROOT/$jailtag/dev/fd >/dev/null 2>&1");
		mysystem("umount $JAILROOT/$jailtag/dev >/dev/null 2>&1");
		mysystem("umount $JAILROOT/$jailtag$GENILIB_MNT >/dev/null 2>&1");
Mike Hibler's avatar
Mike Hibler committed
158

159
		if (mysystem("zfs destroy -f $ZFSBASE/$jailtag")) {
Mike Hibler's avatar
Mike Hibler committed
160
161
162
		    print STDERR
			"*** could not destroy jail FS for $jailtag\n";
		}
163
		if ($snapshot && mysystem("zfs destroy -f $snapshot")) {
Mike Hibler's avatar
Mike Hibler committed
164
165
166
167
168
		    print STDERR
			"*** could not destroy snapshot $snapshot\n";
		}
		msg("Destroying done");
		$jailstate = 0;
169
170
	    }
	}
Mike Hibler's avatar
Mike Hibler committed
171
	$? = $ecode;
172
173
174
175
176
177
178
179
180
181
182
    }
}

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
Mike Hibler's avatar
Mike Hibler committed
183
184
185
186
187
188
if (defined($options{"C"})) {
    $action = 1;
}
if (defined($options{"R"})) {
    $action = 2;
}
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"p"})) {
    $pfile = $options{"p"};
}
if (defined($options{"o"})) {
    $ofile = $options{"o"};
}
if (defined($options{"u"})) {
    $user = $options{"u"};
}
if (defined($options{"n"})) {
    $jailname = $options{"n"};
    if ($jailname !~ /^[-\w]+$/) {
	print STDERR "Come on, keep the name simple...\n";
	usage();
    }
}
208
209
210
211
212
213
214
215
if (defined($options{"B"})) {
    my $base = $options{"B"};
    if (! -e "/iocage/tags/$base") {
	print STDERR "No such base '$base'\n";
	usage();
    }
    $basename = $base;
}
216

Mike Hibler's avatar
Mike Hibler committed
217
218
219
220
221
222
#
# Extract params from the environment (if invoked via rungenilib.proxy).
#
if (!$ofile && exists($ENV{'GENILIB_PORTAL_REQUEST_PATH'})) {
    $ofile = $ENV{'GENILIB_PORTAL_REQUEST_PATH'};
}
223
224
225
226
227
228
229
if (!$pfile) {
    if (exists($ENV{'GENILIB_PORTAL_PARAMS_PATH'})) {
	$pfile = $ENV{'GENILIB_PORTAL_PARAMS_PATH'};
    }
    elsif (exists($ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'})) {
	$pfile = $ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'};
    }
230
231
}

Mike Hibler's avatar
Mike Hibler committed
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
if ($action == 4) {
    if (@ARGV < 1) {
	print STDERR "Must specify a script\n";
	usage();
    }
    $ifile = $ARGV[0];

    if (!$ofile) {
	print STDERR "Must specify an output file (-o)\n";
	usage();
    }
} else {
    if (@ARGV != 0) {
	print STDERR "Too many args for create/destroy\n";
	usage();
    }
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
}

# Must be a legit user
(undef,undef,$uid,$gid) = getpwnam($user) or
    die("*** $0:\n".
	"    Invalid user '$user'!");

# Must run as root
if ($UID != 0) {
    die("*** $0:\n".
	"    Must be root to run this script!");
}

# Make sure jail infrastructure is there
if (! -d "$JAILROOT") {
    die("*** $0:\n".
	"    $JAILROOT does not exist; is iocage installed?");
}

Mike Hibler's avatar
Mike Hibler committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
#
# Action checks.
# If only creating (-C) or running normally, then jail should not exist.
# If only removing (-R), then jail should exist.
#
if ($action != 2 && -e "$JAILROOT/$jailname") {
    die("*** $0:\n".
	"    $jailname already exists");
}

#
# XXX figure out the appropriate snapshot.
# This is marginally better than hardwiring a UUID.
#
281
my $path = readlink("/iocage/tags/$basename");
Mike Hibler's avatar
Mike Hibler committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
if (!$path || $path !~ /(\w{8}-\w{4}-\w{4}-\w{4}-\w{12})$/) {
    die("*** $0:\n".
	"    Cannot find UUID for base FS");
}
$jailuuid = $1;

#
# Removing an existing jail, just set the jailstate and exit.
#
if ($action == 2) {
    if (! -e "$JAILROOT/$jailname") {
	die("*** $0:\n".
	    "    $jailname does not exist");
    }
    $jailtag = $jailname;
    $jailstate = 2;
    $snapshot = "$ZFSBASE/$jailuuid/root\@$jailname";
    exit(0);
}

302
303
304
#
# Create a filesystem
#
Mike Hibler's avatar
Mike Hibler committed
305
msg("Snapshotting base FS");
306
if (mysystem("zfs snapshot $ZFSBASE/$jailuuid/root\@$jailname")) {
Mike Hibler's avatar
Mike Hibler committed
307
    print STDERR "Could not create geni-lib jail snapshot\n";
308
    exit(-1);
Mike Hibler's avatar
Mike Hibler committed
309
310
311
}
msg("Snapshotting done");
$snapshot = "$ZFSBASE/$jailuuid/root\@$jailname";
312
msg("Cloning $basename FS");
313
if (mysystem("zfs clone $zfsattrs $snapshot $ZFSBASE/$jailname")) {
314
    print STDERR "Could not create geni-lib jail FS\n";
315
    exit(-1);
316
317
318
319
320
321
}
msg("Cloning done");
$jailstate = 1;
$jailtag = $jailname;
my $jailrootdir = "$JAILROOT/$jailtag";

Mike Hibler's avatar
Mike Hibler committed
322
323
324
325
326
#
# XXX mount host geni-lib in jail
# This way we don't have to copy in the geni-lib files and they
# will always be up-to-date. Also make dev trees work.
#
327
328
329
330
331
# Note that we avoid gratuitous mountd HUP'ing by invoking mount_nullfs
# directly instead of using mount. Alas, the jail command still does a
# couple of "mount" commands, so we are still HUP happy.
#
if (mysystem("mount_nullfs -o ro $GENILIB $jailrootdir$GENILIB_MNT")) {
Mike Hibler's avatar
Mike Hibler committed
332
    print STDERR "Could not mount $GENILIB in $jailtag\n";
333
    exit(-1);
Mike Hibler's avatar
Mike Hibler committed
334
335
}

336
337
338
339
340
#
# Drop our files into the jail FS
# We create a protected directory owned by the user so that people
# cannot snoop from outside the jail.
#
Mike Hibler's avatar
Mike Hibler committed
341
342
343
my ($j_ifile,$j_ofile,$j_pfile);
if ($action != 1) {
    my $tempdir = "/tmp/genilib/";
344
    if (!mkdir("$jailrootdir$tempdir", 0700)) {
Mike Hibler's avatar
Mike Hibler committed
345
	print STDERR "Could not create geni-lib jail tempdir\n";
346
	exit(-1);
Mike Hibler's avatar
Mike Hibler committed
347
    }
348

Mike Hibler's avatar
Mike Hibler committed
349
350
351
352
    $j_ifile = $tempdir . basename($ifile);
    $j_ofile = $tempdir . basename($ofile);
    $j_pfile = $tempdir . basename($pfile)
	if ($pfile);
353

Mike Hibler's avatar
Mike Hibler committed
354
    msg("Stashing files");
355
    if (mysystem("cp -p $ifile $jailrootdir$j_ifile") ||
356
357
	($pfile && -e $pfile &&
	 mysystem("cp -p $pfile $jailrootdir$j_pfile")) ||
358
	mysystem("chown -R $uid:$gid $jailrootdir$tempdir")) {
Mike Hibler's avatar
Mike Hibler committed
359
	print STDERR "Could not populate jail\n";
360
	exit(-1);
Mike Hibler's avatar
Mike Hibler committed
361
    }
362

Mike Hibler's avatar
Mike Hibler committed
363
364
365
    #
    # XXX adjust the environment for the portal module to reflect the jail.
    #
366
367
    $ENV{'GENILIB_PORTAL_REQUEST_PATH'} = $j_ofile;
    if (exists($ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'})) {
368
	$ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'} = $j_pfile;
369
    }
370
    elsif (exists($ENV{'GENILIB_PORTAL_PARAMS_PATH'})) {
371
372
	$ENV{'GENILIB_PORTAL_PARAMS_PATH'} = $j_pfile;
    }
Mike Hibler's avatar
Mike Hibler committed
373
374
375
376
377
378
379
380
381
382
383
384
385

    #
    # Make other aspects of the environment appear a little bit sane.
    # XXX okay, not really THAT sane since the user will not exist in the jail.
    #
    $ENV{'SHELL'} = "/bin/sh";
    $ENV{'USER'} = $user;
    $ENV{'USERNAME'} = $user;
    $ENV{'LOGNAME'} = $user;
    $ENV{'HOME'} = $tempdir;
    delete $ENV{'DISPLAY'};
    delete @ENV{'SUDO_UID', 'SUDO_GID', 'SUDO_USER', 'SUDO_COMMAND'};
    delete @ENV{'SSH_AUTH_SOCK', 'SSH_CLIENT', 'SSH_CONNECTION', 'SSH_TTY'};
386
387
388
389
390
391
}

#
# Fire up the jail
#
my $status = -1;
Mike Hibler's avatar
Mike Hibler committed
392
if ($USEJAILRUN && $action != 1) {
393
394
395
396
397
398
399
    msg("Run jail");
    $status = run_jail($jailtag, $jailrootdir, $user, "$INTERP $j_ifile");
    msg("Run done");
} else {
    msg("Start jail");
    if (start_jail($jailtag, $jailrootdir)) {
	print STDERR "Could not start geni-lib jail $jailtag\n";
400
	exit(-1);
401
402
403
404
405
406
407
408
409
410
411
412
    }
    msg("Starting done");
    $jailstate = 2;

    #
    # And execute the command as the indicated jail user.
    #
    # XXX currently we run the command as the real user (-u), but note that
    # they will have no passwd entry inside the jail. May cause problems.
    # If so, we can run as nobody in the jail (-U nobody) or add the real
    # user to the jail (but that is a lot of work for one command...)
    #
Mike Hibler's avatar
Mike Hibler committed
413
414
    if ($action != 1) {
	msg("Execing command");
415
	$status = mysystem("jexec -u $user $jailtag $INTERP $j_ifile");
Mike Hibler's avatar
Mike Hibler committed
416
417
418
419
	msg("Execing done");
    } else {
	$status = 0;
    }
420
421
422
423
424
425
426
427
}

if ($status) {
    if ($status == -1) {
	print STDERR "Could not run jail $jailtag\n";
    } elsif ($status & 127) {
	$status &= 127;
	print STDERR "Jail $jailtag execution died with signal $status\n";
428
	$status = -2;
429
430
    } else {
	$status >>= 8;
431
432
433
	if ($status <= 100 || $debug) {
	    print STDERR "Jail $jailtag execution failed with exit code $status\n";
	}
434
    }
Mike Hibler's avatar
Mike Hibler committed
435
436
437
438
439
440
441

    # XXX odd semantics: if debug is set, don't remove jail on error
    if ($debug) {
	print STDERR "WARNING: not destroying jail, you will need to do it:\n".
	    "    sudo $PROG -R -n $jailtag\n";
	$jailstate = 0;
    }
442
443
}

Mike Hibler's avatar
Mike Hibler committed
444
if ($action != 1) {
445
446
447
448
449
450
451
452
453
454
    #
    # Oh the joys of running as root. Now we need to take away user
    # permission from the jail directory (recall the user can access
    # it from outside) and then verify that the source file isn't a
    # symlink (a cheap-o realpath check). Our caller is responsible
    # for defending the target file.
    #
    my $tempdir = "/tmp/genilib";
    if (-l "$jailrootdir$tempdir" ||
	chown(0, -1, "$jailrootdir$tempdir") != 1) {
Mike Hibler's avatar
Mike Hibler committed
455
	print STDERR "Could not copy back results of command\n";
456
	exit(-1);
Mike Hibler's avatar
Mike Hibler committed
457
    }
458
459
    if (-e "$jailrootdir$j_ofile") {
	if (-l "$jailrootdir$j_ofile" ||
460
	    mysystem("cp $jailrootdir$j_ofile $ofile")) {
461
	    print STDERR "Could not copy back results of command\n";
462
	    exit(-1);
463
464
	}
    }
465
466
467
468
469
470
471
472
    if (exists($ENV{'GENILIB_PORTAL_DUMPPARAMS_PATH'}) &&
	-e "$jailrootdir$j_pfile") {
	if (-l "$jailrootdir$j_pfile" ||
	    mysystem("cp $jailrootdir$j_pfile $pfile")) {
	    print STDERR "Could not copy back parameter block\n";
	    exit(-1);
	}
    }
Mike Hibler's avatar
Mike Hibler committed
473
474
} else {
    print STDERR "Jail '$jailtag' running. Root FS at '$jailrootdir'.\n";
475
476
}

Mike Hibler's avatar
Mike Hibler committed
477
exit($status);
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507

sub start_jail($$)
{
    my ($name,$fs) = @_;

    my $args = "";

    # identity (need host.hostuuid?)
    $args .= "name=$name host.hostname=$name.emulab.net path=$fs ";

    # security
    $args .= "securelevel=2 devfs_ruleset=4 enforce_statfs=2 children.max=0 ";
    $args .= "allow.set_hostname=0 allow.sysvipc=0 allow.raw_sockets=0 ";
    $args .= "allow.chflags=0 allow.mount=0 allow.mount.devfs=0 ";
    $args .= "allow.mount.nullfs=0 allow.mount.procfs=0 allow.mount.tmpfs=0 ";
    $args .= "allow.mount.zfs=0 allow.quotas=0 allow.socket_af=0 ";
    $args .= "mount.devfs=1 mount.fdescfs=1 ";

    # no networking
    $args .= "ip4=disable ip6=disable ";

    # execution params
    $args .= "exec.prestart=/usr/bin/true exec.poststart=/usr/bin/true ";
    $args .= "exec.prestop=/usr/bin/true exec.poststop=/usr/bin/true ";
    $args .= "exec.start='/bin/sh /etc/rc' exec.stop='/bin/sh /etc/rc.shutdown' ";
    $args .= "exec.clean=0 exec.timeout=$CMD_TIMEOUT stop.timeout=30 ";

    # other stuff
    $args .= "allow.dying persist";

508
    return mysystem("jail -qc $args >/dev/null 2>&1");
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
}

sub run_jail($$$$)
{
    my ($name,$fs,$user,$cmdstr) = @_;

    my $args = "";

    # identity (need host.hostuuid?)
    $args .= "name=$name host.hostname=$name.emulab.net path=$fs ";

    # security
    $args .= "securelevel=2 devfs_ruleset=4 enforce_statfs=2 children.max=0 ";
    $args .= "allow.set_hostname=0 allow.sysvipc=0 allow.raw_sockets=0 ";
    $args .= "allow.chflags=0 allow.mount=0 allow.mount.devfs=0 ";
    $args .= "allow.mount.nullfs=0 allow.mount.procfs=0 allow.mount.tmpfs=0 ";
    $args .= "allow.mount.zfs=0 allow.quotas=0 allow.socket_af=0 ";
    $args .= "mount.devfs=1 mount.fdescfs=1 ";

    # no networking
    $args .= "ip4=disable ip6=disable ";

    # execution params
    $args .= "exec.prestart=/usr/bin/true exec.poststart=/usr/bin/true ";
    $args .= "exec.prestop=/usr/bin/true exec.poststop=/usr/bin/true ";
    $args .= "exec.start='$cmdstr' exec.stop='/usr/bin/true' ";
    $args .= "exec.clean=0 exec.timeout=$CMD_TIMEOUT stop.timeout=30 ";
    $args .= "exec.jail_user=$user ";

    # other stuff
    $args .= "allow.dying";

541
    return mysystem("jail -qc $args >/dev/null 2>&1");
542
543
544
545
}

sub msg(@)
{
Mike Hibler's avatar
Mike Hibler committed
546
547
548
549
550
    if ($debug) {
	my $stamp = time() - $starttime;
	printf STDERR "[%3d] ", $stamp;
	print STDERR @_, "\n";
    }
551
}
552
553
554
555
556
557
558
559
560
561

sub mysystem($)
{
    my $cmd = shift;
    
    if (0) {
	print STDERR "Doing: '$cmd'\n";
    }
    return system($cmd);
}