Checkpoint the jail stuff. Now operates on a local testbed node as

well (requires new libsetup.pm to be installed too). Still a work in
progress.

tmcd/freebsd/GNUmakefile.in

@@ -24,6 +24,7 @@ include $(TESTBED_SRCDIR)/GNUmakerules
 
 DESTDIR		=
 ETCDIR		= $(DESTDIR)/etc
+JAILDIR		= $(ETCDIR)/jail
 INSTALL_DIR	= $(ETCDIR)/testbed
 RC_DIR		= $(INSTALL_DIR)/rc.d
 RRC_DIR		= $(DESTDIR)/usr/local/etc/rc.d
@@ -38,6 +39,10 @@ mfs-install:	script-install bin-install
 dir-install:
 	$(INSTALL) -m 755 -o root -g wheel -d $(INSTALL_DIR)
 	$(INSTALL) -m 755 -o root -g wheel -d $(RC_DIR)
+	$(INSTALL) -m 755 -o root -g wheel -d /var/emulab
+	$(INSTALL) -m 755 -o root -g wheel -d /var/emulab/db
+	$(INSTALL) -m 755 -o root -g wheel -d /var/emulab/jails
+	$(INSTALL) -m 755 -o root -g wheel -d /var/emulab/logs
 
 misc-install:	dir-install
 	$(INSTALL) -m 755 -o root -g wheel -d $(INSTALL_DIR)/sup
@@ -101,3 +106,14 @@ sfs-install:
 	$(INSTALL) -m 775 -o sfs -g sfs -d $(DESTDIR)/var/sfs/root
 	$(INSTALL) -m 775 -o sfs -g sfs -d $(DESTDIR)/var/sfs/root/var
 	$(INSTALL) -m 775 -o sfs -g sfs -d $(DESTDIR)/var/sfs/root/usr
+
+jail-install:	dir-install
+	$(INSTALL) -m 755 -o root -g wheel -d $(JAILDIR)
+	$(INSTALL) -m 644 $(SRCDIR)/jail/group $(JAILDIR)/group
+	$(INSTALL) -m 640 $(SRCDIR)/jail/master.passwd $(JAILDIR)/master.passwd
+	$(INSTALL) -m 755 $(SRCDIR)/jail/rc.conf $(JAILDIR)/rc.conf
+	$(INSTALL) -m 755 $(SRCDIR)/jail/rc.local $(JAILDIR)/rc.local
+	$(INSTALL) -m 755 $(SRCDIR)/jail/injail.pl $(JAILDIR)/injail.pl
+	$(INSTALL) -m 755 $(SRCDIR)/jail/jaildog.pl $(INSTALL_DIR)/jaildog.pl
+	$(INSTALL) -m 755 $(SRCDIR)/jail/jailctl $(INSTALL_DIR)/jailctl
+	$(INSTALL) -m 755 $(SRCDIR)/jail/mkjail.pl $(INSTALL_DIR)/mkjail.pl

tmcd/freebsd/jail/README

+These are files to support building jails on freebsd machines.
+They go in /etc/jail on the host machine. The rest of the support
+is currently in ../ron, but will be merged at some point.
+
+There is a dup set of passwd/group files here. They can probably
+be merged at some point.

tmcd/freebsd/jail/group

+wheel:*:0:root
+daemon:*:1:daemon
+kmem:*:2:root
+sys:*:3:root
+tty:*:4:root
+operator:*:5:root
+mail:*:6:
+obin:*:7:
+news:*:8:
+man:*:9:
+bin:*:10:
+games:*:13:
+staff:*:20:root
+guest:*:31:root
+uucp:*:66:
+xten:*:67:xten
+dialer:*:68:
+network:*:69:
+nogroup:*:65533:
+nobody:*:65534:

tmcd/freebsd/jail/injail.pl

+#!/usr/bin/perl -w
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+
+#
+# The point of this is to fire up the init code inside the jail,
+# and then wait for a signal from outside the jail. When that happens
+# kill off everything inside the jail and exit. So, like a mini version
+# of /sbin/init cause killing the jail cleanly from outside the jail
+# turns out to be rather difficult, and doing it from inside is very easy!
+#
+my $DEFCONSIX = "/bin/sh /etc/rc";
+
+#
+# Catch TERM.
+# 
+sub handler () {
+    $SIG{TERM} = 'IGNORE';
+    system("kill -TERM -1");
+    sleep(1);
+    system("kill -KILL -1");
+    exit(1);
+}
+$SIG{TERM} = \&handler;
+
+my $childpid = fork();
+if (!$childpid) {
+    if (@ARGV) {
+	exec @ARGV;
+    }
+    else {
+	exec $DEFCONSIX;
+    }
+    die("*** $0:\n".
+	"    exec failed: '@ARGV'\n");
+}
+
+#
+# If a command list was provided, we wait for whatever it was to
+# finish. Otherwise sleep forever. 
+#
+if (@ARGV) {
+    waitpid($childpid, 0);
+    $SIG{TERM} = 'IGNORE';
+    system("kill -TERM -1");
+    sleep(1);
+    system("kill -KILL -1");
+}
+else {
+    while (1) {
+	system("/bin/sleep 10000");
+    }
+}
+exit(0);

tmcd/freebsd/jail/jailctl

+#!/bin/sh
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+#
+# Jail startup. To be run inside of a jail!
+#
+case "$1" in
+start)
+	if [ -f /usr/local/etc/emulab/jaildog.pl ]; then
+	    /usr/local/etc/emulab/jaildog.pl > /dev/null 2>&1
+	    echo -n ' Emulab'
+	elif [ -f /etc/testbed/jaildog.pl ]; then
+	    /etc/testbed/jaildog.pl > /dev/null 2>&1
+	    echo -n ' Emulab'
+	fi
+	;;
+stop)
+	#
+	# Don't bother. Inside of a jail everything gets killed at once.
+	#
+	echo -n ' Emulab'
+	;;
+*)
+	echo "Usage: `basename $0` {start|stop}" >&2
+	;;
+esac
+
+exit 0

tmcd/freebsd/jail/jaildog.pl

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use Getopt::Std;
+use English;
+use Errno;
+use POSIX qw(strftime);
+
+#
+# Watchdog for inside a jail. All this does is look for account updates.
+# Might do more later.
+#
+sub usage()
+{
+    print "Usage: jaildog [-t timeout]\n";
+    exit(1);
+}
+my $optlist = "t:";
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint path
+#
+$ENV{'PATH'} = "/bin:/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Must be root to run this.
+# 
+if ($UID != 0) {
+    die("*** $0:\n".
+	"    Must be root to run this script!\n");
+}
+
+#
+# Load the OS independent support library. It will load the OS dependent
+# library and initialize itself. 
+#
+if (-d "/usr/local/etc/emulab") {
+    use lib "/usr/local/etc/emulab";
+    $ENV{'PATH'} .= ":/usr/local/etc/emulab";
+}
+elsif (-d "/etc/testbed") {
+    use lib "/etc/testbed";
+    $ENV{'PATH'} .= ":/etc/testbed";
+}
+use libsetup;
+
+# Locals
+my $timeout = (60 * 60 * 12);	# In seconds of course. 
+my $logname = "/var/tmp/emulab-jaildog.debug";
+my $pidfile = "/var/run/emulab-jaildog.pid";
+my $vnodeid;
+
+#
+# Parse command arguments. Once we return from getopts, all that should be
+# left are the required arguments.
+#
+%options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"t"})) {
+    $timeout = $options{"t"};
+}
+if (@ARGV) {
+    usage();
+}
+
+#
+# Put this into the background and log its output. We *must* do this cause
+# we do not want to halt the boot if the testbed is down!
+# 
+if (1 && TBBackGround($logname)) {
+    #
+    # Parent exits normally
+    #
+    exit(0);
+}
+
+#
+# Setup a handler to catch TERM, and kill our process group.
+#
+my $pgrp = getpgrp(0);
+
+sub handler () {
+    $SIG{TERM} = 'IGNORE';
+    $SIG{INT} = 'IGNORE';
+    kill('TERM', -$pgrp);
+    sleep(5);
+    exit(0);
+}
+$SIG{TERM} = \&handler;
+$SIG{INT}  = \&handler;
+
+#
+# Write our pid into the pid file so we can be killed later (when the
+# experiment is torn down). We must do this first so that we can be
+# killed before we change the sig handlers
+#
+open(PFILE, "> $pidfile")
+    or die("Could not open $pidfile: $!");
+print PFILE "$PID\n";
+close(PFILE);
+
+print "Getting our Emulab configuration ...\n";
+if (! ($vnodeid = jailedsetup())) {
+    die("*** $0:\n".
+	"    Did not get our jailname!\n");
+}
+
+if (-x TMTARBALLS()) {
+    print "Installing Tarballs ...\n";
+    system(TMTARBALLS());
+}
+
+if (-x TMSTARTUPCMD()) {
+    print "Running startup command ...\n";
+    system("runstartup");
+}
+
+#
+# Inform TMCD that we are up and running.
+#
+print "Informing Emulab Operations that we're up and running ...\n";
+system("tmcc state ISUP");
+
+#
+# Fire off a child that does nothing but tell the boss we are alive.
+#
+my $mypid = fork();
+if (! $mypid) {
+    my $failed  = 0;
+    
+    print "Keep alive starting up ... \n";
+
+    while (1) {
+	#
+	# Run tmcc in UDP mode. The command is ignored at the other end.
+	# Its just the connection that tells tmcd we are alive.
+	# Since its UDP, we try it a couple of times if it fails. 
+	#
+	my $retries = 3;
+
+	while ($retries) {
+#	    my $options = "-p 7778 REDIRECT=192.168.100.1";
+	    my $options = "";
+	    if (REMOTE()) {
+		$options .= " -u -t 3";
+	    }
+	    if (defined($vnodeid)) {
+		$options .= " -n $vnodeid";
+	    }
+	    my $result = `tmcc $options isalive`;
+	    if (! $?) {
+		my $date = POSIX::strftime("20%y/%m/%d %H:%M:%S", localtime());
+		
+		chomp $result;
+		my (undef,$update) = split("=", $result);
+		if ($update || $failed) {
+		    print "Running an update at $date ...\n";
+		    system("update -i");
+		    $failed = $?;
+		}
+		last;
+	    }
+	    $retries--;
+	}
+	if (!$retries) {
+	    print "keep alive returned $?\n";
+	}
+	sleep(60);
+    }
+    exit(0);
+}
+
+#
+# Loop!
+# 
+while (1) {
+    sleep($timeout);
+    
+    my $date = POSIX::strftime("20%y/%m/%d %H:%M:%S", localtime());
+
+    print "Dogging it at $date\n";
+    
+    #
+    # Run account update. Use immediate mode so that it exits right away
+    # if the lock is taken (another update already running).
+    #
+    print "Looking for new Emulab accounts ...\n";
+    system("update -i");
+}
+
+exit(0);

tmcd/freebsd/jail/master.passwd

+root:*:0:0::0:0:Charlie &:/root:/bin/csh
+toor:*:0:0::0:0:Bourne-again Superuser:/root:
+daemon:*:1:1::0:0:Owner of many system processes:/root:/sbin/nologin
+operator:*:2:5::0:0:System &:/usr/guest/operator:/bin/csh
+bin:*:3:7::0:0:Binaries Commands and Source,,,:/:/sbin/nologin
+games:*:7:13::0:0:Games pseudo-user:/usr/games:/sbin/nologin
+news:*:8:8::0:0:News Subsystem:/:/sbin/nologin
+man:*:9:9::0:0:Mister Man Pages:/usr/share/man:/sbin/nologin
+uucp:*:66:66::0:0:UUCP pseudo-user:/var/spool/uucppublic:/usr/libexec/uucp/uucico
+xten:*:67:67::0:0:X-10 daemon:/usr/local/xten:/sbin/nologin
+pop:*:68:6::0:0:Post Office Owner:/nonexistent:/sbin/nologin
+nobody:*:65534:65534::0:0:Unprivileged user:/nonexistent:/sbin/nologin
+ftp:*:99:52::0:0:Anonymous Ftp:/var/spool/ftp:/bin/echo
+smmsp:*:25:25::0:0:Sendmail Submission User:/var/spool/clientmqueue:/sbin/nologin
+mysql:*:88:88::0:0:MySQL Daemon:/var/db/mysql:/sbin/nologin
+sfs:*:77:77::0:0:SFS pseudo-user:/:/bin/nologin
+emulabman:*:65520:10::0:0:Emulab Man:/home/emulabman:/bin/tcsh

tmcd/freebsd/jail/mkjail.pl

+#!/usr/bin/perl -w
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+use Fcntl;
+use IO::Handle;
+use Socket;
+
+#
+# Questions:
+#
+# * Whats the hostname for the jail? Perhaps vnodename.emulab.net
+# * What should /etc/resolv.conf look like?
+# 
+
+#
+# Create a jailed environment. There are some stub files stored in
+# /etc/jail that copied into the jail.
+#
+sub usage()
+{
+    print("Usage: mkjail.pl [-s] [-i <ipaddr>] [-p <pid>] <hostname>\n");
+    exit(-1);
+}
+my  $optlist = "i:p:e:s";
+
+#
+# Only real root can run this script.
+#
+if ($UID) {
+    die("Must be root to run this script!\n");
+}
+system("sysctl jail.set_hostname_allowed=0");
+
+#
+# Catch ^C and exit with error. 
+#
+my $leaveme = 0;
+sub handler ($) {
+    my ($signame) = @_;
+    
+    $SIG{INT}  = 'IGNORE';
+    $SIG{USR1} = 'IGNORE';
+    $SIG{TERM} = 'IGNORE';
+    $SIG{HUP}  = 'IGNORE';
+
+    if ($signame eq 'USR1') {
+	$leaveme = 1;
+    }
+    fatal("Caught a SIG${signame}! Killing the jail ...");
+}
+$SIG{INT}  = \&handler;
+$SIG{USR1} = \&handler;
+$SIG{HUP}  = \&handler;
+$SIG{TERM} = 'IGNORE';
+
+#
+# Turn off line buffering on output
+#
+STDOUT->autoflush(1);
+STDERR->autoflush(1);
+
+#
+# Untaint the environment.
+# 
+$ENV{'PATH'} = "/tmp:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:".
+    "/usr/local/bin:/usr/site/bin:/usr/site/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Deal with the screwy path mess that I created!
+#
+my $EMULABPATH;
+if (-e "/usr/local/etc/emulab/tmcc") {
+    $EMULABPATH = "/usr/local/etc/emulab";
+}
+elsif (-e "/etc/testbed/tmcc") {
+    $EMULABPATH = "/etc/testbed";
+}
+else {
+    die("*** $0:\n".
+	"    Could not locate the testbed directory!\n");
+}
+
+#
+# Locals
+#
+my $JAILPATH	= "/var/emulab/jails";
+my $JAILCONFIG  = "/etc/jail";
+my $LOCALROOTFS = "/local";
+my $TMCC	= "$EMULABPATH/tmcc";
+my @ROOTCPDIRS	= ("etc", "root");
+my @ROOTMKDIRS  = ("dev", "tmp", "var", "usr", "proc", "users", "opt",
+		   "bin", "sbin", "home", $LOCALROOTFS);
+my @ROOTMNTDIRS = ("bin", "sbin", "usr");
+my $VNFILEMBS   = 64;
+my $MAXVNDEVS	= 10;
+my $IP;
+my $PID;
+my $debug	= 1;
+my $cleaning	= 0;
+my $vndevice;
+my @mntpoints   = ();
+my $jailpid;
+my $tmccpid;
+my $interactive = 0;
+
+#
+# Parse command arguments. Once we return from getopts, all that should be
+# left are the required arguments.
+#
+%options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (@ARGV != 1) {
+    usage();
+}
+my $HOST = $ARGV[0];
+
+#
+# Untaint the arguments.
+#
+if ($HOST =~ /^([-\w\/]+)$/) {
+    $HOST = $1;
+}
+else {
+    die("Tainted argument $HOST!\n");
+}
+
+if (defined($options{'s'})) {
+    $interactive = 1;
+}
+
+#
+# If no IP, then it defaults to our hostname's IP.
+# 
+if (defined($options{'i'})) {
+    $IP = $options{'i'};
+
+    if ($IP =~ /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/) {
+	$IP = $1;
+    }
+    else {
+	die("Tainted argument $IP!\n");
+    }
+}
+else {
+    my $hostname = `hostname`;
+
+    # Untaint and strip newline.
+    if ($hostname =~ /^([-\w\.]+)$/) {
+	$hostname = $1;
+
+	my (undef,undef,undef,undef,@ipaddrs) = gethostbyname($hostname);
+	$IP = inet_ntoa($ipaddrs[0]);
+    }
+}
+if (!defined($IP)) {
+    usage();
+}
+
+if (defined($options{'p'})) {
+    $PID = $options{'p'};
+
+    if ($PID =~ /^([-\@\w]+)$/) {
+	$PID = $1;
+    }
+    else {
+	die("Tainted argument $PID.");
+    }
+}
+
+print("Setting up jail for HOST:$HOST using IP:$IP\n")
+    if ($debug);
+
+#
+# First create the directory tree and such.
+# 
+chdir($JAILPATH) or
+    die("Could not chdir to $JAILPATH: $!\n");
+
+if (! -e $HOST) {
+    mkdir($HOST, 0770) or
+	fatal("Could not mkdir $HOST in $JAILPATH: $!");
+}
+
+if (-e "$HOST/root") {
+    #
+    # Try to pick up where we left off.
+    # 
+    restorerootfs("$JAILPATH/$HOST");
+}
+else {
+    #
+    # Create the root filesystem.
+    #
+    mkrootfs("$JAILPATH/$HOST");
+}
+
+#
+# Start the tmcc proxy. This path will be valid in both the outer
+# environment and in the jail!
+#
+startproxy("$JAILPATH/$HOST");
+
+#
+# Start the jail. We do it in a child so we can send a signal to the
+# jailed process to force it to shutdown. The jail has to shut itself
+# down.
+#
+$jailpid = fork();
+if ($jailpid) {
+    # We do not really care about the exit status of the jail.
+    waitpid($jailpid, 0);
+    undef($jailpid);
+}
+else {
+    $SIG{TERM} = 'DEFAULT';
+    $ENV{'TMCCVNODEID'} = $HOST;
+
+    my $cmd = "jail $JAILPATH/$HOST/root $HOST $IP $JAILCONFIG/injail.pl";
+    if ($interactive) {
+	$cmd .= " /bin/csh";
+    }
+    exec($cmd);
+    die("*** $0:\n".
+	"    exec failed to start the jail!\n");
+}
+	 
+#
+# Once we exit, cleanup the mess.
+#
+cleanup();
+exit(0);
+
+#
+# Create a file for a vnode device, vnconfig it, newfs, and then
+# mount it on the "root" directory.
+#
+sub mkrootfs($)
+{
+    my ($path) = @_;
+
+    chdir($path) or
+	fatal("Could not chdir to $path: $!");
+
+    mkdir("root", 0770) or
+	fatal("Could not mkdir 'root' in $path: $!");
+    
+    #
+    # Big file of zeros.
+    # 
+    mysystem("dd if=/dev/zero of=root.vnode bs=1m count=$VNFILEMBS");
+
+    #
+    # Find a free vndevice.
+    #
+    for (my $i = 0; $i < $MAXVNDEVS; $i++) {
+	system("vnconfig -e -s labels vn${i} root.vnode");
+	if (! $?) {
+	    $vndevice = $i;
+	    last;
+	}
+    }
+    fatal("Could not find a free vn device!") 
+	if (!defined($vndevice));
+
+    mysystem("disklabel -r -w vn${vndevice} auto");
+    mysystem("newfs -b 8192 -f 1024 -i 4096 -c 15 /dev/vn${vndevice}c");
+    mysystem("mount /dev/vn${vndevice}c root");
+    push(@mntpoints, "$path/root");