Another checkpoint of the firewall code. At this point, you can swapin

a XEN based ElabInElab with boss and ops running firewall rules based
on the ruleset we use on Utah's firewall. To turn this on, add this to
your NS file:

tb-set-elabinelab-attribute CONFIG_FIREWALL_BOSS "yep"
tb-set-elabinelab-attribute CONFIG_FIREWALL_OPS  "yep"

You do not have define both.

clientside/tmcc/common/config/rc.mkelab

@@ -1704,12 +1704,25 @@ sub SetupOpsNode($)
     foreach my $list (@lists) {
 	if ($emulabconfig{"CONFIG_GENIRACK"}) {
 	    my $where = "genirack-ops\@flux.utah.edu";
+	    if (exists($emulabconfig{"GENIRACK_TBOPS"})) {
+		$where .= "\n" . $emulabconfig{"GENIRACK_TBOPS"};
+	    }
 	    
-	    if ($list eq "testbed-logs") {
-		$where = "genirack-logs\@flux.utah.edu";
+	    if ($list eq "testbed-logs" || $list eq "testbed-audit") {
+		if (exists($emulabconfig{"GENIRACK_TBLOGS"})) {
+		    $where = $emulabconfig{"GENIRACK_TBLOGS"};
+		}
+		else {
+		    $where = "genirack-logs\@flux.utah.edu";
+		}
 	    }
 	    elsif ($list eq "testbed-stated") {
-		$where = "genirack-stated\@flux.utah.edu";
+		if (exists($emulabconfig{"GENIRACK_TBSTATED"})) {
+		    $where = $emulabconfig{"GENIRACK_TBSTATED"};
+		}
+		else {
+		    $where = "genirack-stated\@flux.utah.edu";
+		}
 	    }
 	    mysystem("echo '$where' > /etc/mail/lists/$list");
 	}
@@ -2303,22 +2316,25 @@ sub SetupBossNode($)
     # Use natd so that internal control network can talk to outside world.
     # Maybe make an option?
     #
-    print RC "firewall_enable=\"YES\"\n";
-    print RC "firewall_type=\"open\"\n";
-    print RC "natd_interface=\"${outer_controlif}\"\n";
-    print RC "natd_enable=\"YES\"\n";
-    print RC "natd_flags=\"-use_sockets -unregistered_only -same_ports ".
-	"-dynamic -log_facility local6\"\n";
-    #
-    # XXX Woeful hackage!
-    # There seems to be a race with the ipfw and ipdivert code.
-    # By default, rc.firewall adds a "divert" rule before the ipdivert
-    # module is loaded, and the "ipfw add" command fails. However, if we
-    # put the ipdivert module in /boot/loader.conf so it gets loaded at
-    # boot time, everything is fine. So we do that!
-    #
-    if ($FBSD_VERSION > 8) {
-	mysystem("echo 'ipdivert_load=\"YES\"' >> /boot/loader.conf");
+    if (! $emulabconfig{"CONFIG_SINGLECNET"}) {
+	print RC "firewall_enable=\"YES\"\n";
+	print RC "firewall_type=\"open\"\n";
+	print RC "natd_interface=\"${outer_controlif}\"\n";
+	print RC "natd_enable=\"YES\"\n";
+	print RC "natd_flags=\"-use_sockets -unregistered_only -same_ports ".
+	    "-dynamic -log_facility local6\"\n";
+
+	#
+	# XXX Woeful hackage!
+	# There seems to be a race with the ipfw and ipdivert code.
+	# By default, rc.firewall adds a "divert" rule before the ipdivert
+	# module is loaded, and the "ipfw add" command fails. However, if we
+	# put the ipdivert module in /boot/loader.conf so it gets loaded at
+	# boot time, everything is fine. So we do that!
+	#
+	if ($FBSD_VERSION > 8) {
+	    mysystem("echo 'ipdivert_load=\"YES\"' >> /boot/loader.conf");
+	}
     }
 
     # We act as the router for the inner ops and inner nodes.
@@ -3261,6 +3277,20 @@ sub CreateDefsFile($)
 	print OUTDEFS "FANCYBANNER=1\n";
 	print OUTDEFS "ISOLATEADMINS=0\n";
     }
+    if ($emulabconfig{"CONFIG_FIREWALL_BOSS"}) {
+	print OUTDEFS "FIREWALL_BOSS=1\n";
+	if ($emulabconfig{"CONFIG_FIREWALL_BOSS_LOCALRULETMPL"}) {
+	    printf OUTDEFS "FIREWALL_BOSS_LOCALRULETMPL=\"%s\"\n",
+   	           $emulabconfig{"CONFIG_FIREWALL_BOSS_LOCALRULETMPL"};
+	}
+    }
+    if ($emulabconfig{"CONFIG_FIREWALL_OPS"}) {
+	print OUTDEFS "FIREWALL_OPS=1\n";
+	if ($emulabconfig{"CONFIG_FIREWALL_OPS_LOCALRULETMPL"}) {
+	    printf OUTDEFS "FIREWALL_OPS_LOCALRULETMPL=\"%s\"\n",
+   	           $emulabconfig{"CONFIG_FIREWALL_OPS_LOCALRULETMPL"};
+	}
+    }
     close(INDEFS);
     close(OUTDEFS);
     mysystem("cat /tmp/defs-elabinelab");

clientside/tmcc/linux/xen/libvnode_xen.pm

@@ -88,6 +88,7 @@ use File::Copy;
 BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
 use libvnode;
 use libtestbed;
+use libsetup;
 
 #
 # Turn off line buffering on output
@@ -558,6 +559,9 @@ sub vnodeCreate($$$$)
 	$os = "FreeBSD";
 	if ($imagename =~ /FBSD9/) {
 	    $image{'kernel'} = "/boot/freebsd9/kernel";
+	    if (exists($attributes->{'XEN_IPFWKERNEL'})) {
+		$image{'kernel'} = $image{'kernel'} . ".ipfw";
+	    }
 	} elsif ($imagename =~ /FBSD8/) {
 	    $image{'kernel'} = "/boot/freebsd8/kernel";
 	} else {
@@ -976,20 +980,30 @@ sub vnodePreConfigExpNetwork($$$$)
 
 sub vnodeConfigResources($$$$){
     my ($vnode_id, $vmid, $vnconfig, $private) = @_;
+    my $attributes = $vnconfig->{'attributes'};
+    my $memory;
 
     #
-    # Give the vnode some memory.
-    # XXX no way to specify this right now, so we give each vnode
-    # the same amount based on an arbitrary maximum vnode limit.
+    # Give the vnode some memory. We allow the user to specify this
+    # when not a shared host. 
     #
-    my $memory = memoryPerVnode();
-    if ($memory < 48){
-        die("48MB is the minimum amount of memory for a Xen VM; ".
-	    "could only get {$memory}MB.".
-	    "Adjust libvnode_xen::MAX_VNODES");
+    if (!SHAREDHOST() && exists($attributes->{'XEN_MEMSIZE'})) {
+	# Better be MB.
+	$memory = $attributes->{'XEN_MEMSIZE'};
+    }
+    else  {
+	#
+	# XXX no way to specify this right now, so we give each vnode
+	# the same amount based on an arbitrary maximum vnode limit.
+	#
+	$memory = memoryPerVnode();
+	if ($memory < 48){
+	    die("48MB is the minimum amount of memory for a Xen VM; ".
+		"could only get {$memory}MB.".
+		"Adjust libvnode_xen::MAX_VNODES");
+	}
     }
     addConfig($private, "memory = $memory", 1);
-
     return 0;
 }
 

fwrules/GNUmakefile.in

@@ -30,14 +30,27 @@ ETCDIR          = /etc
 OURDOMAIN       = @OURDOMAIN@
 ISMAINSITE	= @TBMAINSITE@
 
+# Configure
+FIREWALL_BOSS   = @FIREWALL_BOSS@
+FIREWALL_OPS    = @FIREWALL_OPS@
+FIREWALL_BOSS_LOCALRULETMPL = @FIREWALL_BOSS_LOCALRULETMPL@
+FIREWALL_OPS_LOCALRULETMPL  = @FIREWALL_OPS_LOCALRULETMPL@
+
 BOSSTOKEN       = boss
 USERTOKEN       = ops
 
-FWRULES_FILES	= $(BOSSTOKEN).ipfw $(USERTOKEN).ipfw
-ifeq ($(ISMAINSITE),1)
-LOCAL_FWRULES   = $(BOSSTOKEN).local.ipfw $(USERTOKEN).local.ipfw
-else
-LOCAL_FWRULES   = 
+FWRULES_FILES	=
+ifneq ($(FIREWALL_BOSS_LOCALRULETMPL),"")
+FWRULES_FILES  += $(FIREWALL_BOSS_LOCALRULETMPL).ipfw
+endif
+ifeq ($(FIREWALL_BOSS),1)
+FWRULES_FILES  += $(BOSSTOKEN).ipfw
+endif
+ifneq ($(FIREWALL_OPS_LOCALRULETMPL),"")
+FWRULES_FILES  += $(FIREWALL_OPS_LOCALRULETMPL).ipfw
+endif
+ifeq ($(FIREWALL_OPS),1)
+FWRULES_FILES  += $(USERTOKEN).ipfw
 endif
 
 include $(OBJDIR)/Makeconf
@@ -46,7 +59,7 @@ include $(OBJDIR)/Makeconf
 # Force dependencies on the scripts so that they will be rerun through
 # configure if the .in file is changed.
 # 
-all: mkfwrules $(LOCAL_FWRULES) $(FWRULES_FILES)
+all: mkfwrules $(FWRULES_FILES)
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
@@ -56,14 +69,30 @@ include $(TESTBED_SRCDIR)/GNUmakerules
 $(LOCAL_FWRULES) $(FWRULES_FILES): mkfwrules
 
 # This is not a safe install target after initial install!
+ifeq ($(FIREWALL_BOSS),1)
 install:
 	@echo "Are you sure you want to reinstall the fwrules files!"
 	@echo "Use the install-real target if you are sure"
 
-install-real:	
+install-real: $(ETCDIR)/boss.ipfw
+else
+install:
+	@echo "FIREWALL_BOSS is not set in your defs file!"
+endif
+
+ifeq ($(FIREWALL_OPS),1)
+control-install:
+	@echo "Are you sure you want to reinstall the fwrules files!"
+	@echo "Use the control-install-real target if you are sure"
+
+control-install-real: $(ETCDIR)/ops.ipfw
+else
+control-install:
+	@echo "FIREWALL_OPS is not set in your defs file!"
+endif
 
 clean: 
-	rm -f mkfwrules $(FWRULES_FILES) $(LOCAL_FWRULES)
+	rm -f mkfwrules $(FWRULES_FILES)
 
 $(ETCDIR)/%: %
 	@echo "Installing $<"

fwrules/boss.local.tmpl

-#
-# Utah specific rules for boss.
-#
-# For backups
-add pass tcp from nstash to me 13782 setup

fwrules/boss.tmpl

+# Setup lo0
+add pass all from any to any via lo0
+add deny all from any to 127.0.0.0/8
+add deny ip from 127.0.0.0/8 to any
+# We do not do ipv6
+add deny all from any to ::1
+add deny all from ::1 to any
+
+# Natd is optional
+%natdrules%
+
 # Match existing dynamic rules early
 add check-state
 

fwrules/mkfwrules.in

@@ -71,6 +71,9 @@ my $PUBLIC_NETWORK           = "@PUBLIC_NETWORK@";
 my $PUBLIC_ROUTER            = "@PUBLIC_ROUTER@";
 my $PUBLIC_NETMASK           = "@PUBLIC_NETMASK@";
 my $NAMED_FORWARDERS         = "@NAMED_FORWARDERS@";
+my $GREP		     = "/usr/bin/grep";
+my $FIREWALL_BOSS_LOCALRULETMPL = "@FIREWALL_BOSS_LOCALRULETMPL@";
+my $FIREWALL_OPS_LOCALRULETMPL  = "@FIREWALL_OPS_LOCALRULETMPL@";
 
 # Simple is good. I stole this out of a google search.
 my @NETMASKS =
@@ -95,12 +98,17 @@ my %SUB = (
 
 #
 # Ug, for elabinelab, it is easier to just set the multicast rule
-# to any to avoid problems. 
+# to "any" to avoid problems. 
 #
 if ($ELABINELAB) {
     $SUB{"multicast"} = "any";
 }
 
+#
+# Natd is optional. 
+#
+my $NATDRULE = "add divert natd ip4 from any to any";
+
 # Testbed libraries.
 use lib "@prefix@/lib";
 
@@ -109,22 +117,57 @@ usage()
 
 my $infile = $ARGV[0];
 
+
+#
+# Look to see if natd is enabled in /etc/rc.conf and what the
+# natd interface is. Might be easier to just read in the shell
+# variables and process them. 
+#
+
+system("$GREP '^natd_enable' /etc/rc.conf | $GREP -q -i yes");
+if (!$?) {
+    my $natd_interface = `$GREP '^natd_interface' /etc/rc.conf`;
+    if (!$? && $natd_interface =~ /=\"(.*)\"$/) {
+	$NATDRULE = "$NATDRULE via $1";
+    }
+    else {
+	die("natd is enabled in /etc/rc.conf, but no natd_interface!\n");
+    }
+}
+else {
+    $NATDRULE = "";
+}
+
 open(IF,"<$infile") or
     die("Unable to open $infile for reading\n");
 
 while (<IF>) {
-    if (/^(.*)\%([-\w]*)\%(.*)$/) {
+    if (/^(.*)\%([-\w]*)\%\s*(.*)$/) {
 	if ($2 eq "localrules") {
-	    my $filename = $3;
-	    $filename =~ s/\s//g;
-	    $filename .= ".local.ipfw";
+	    my $which = $3;
+	    my $filename;
+
+	    if ($which eq "ops" && $FIREWALL_OPS_LOCALRULETMPL ne "") {
+		$filename .= "${FIREWALL_OPS_LOCALRULETMPL}.ipfw";
+	    }
+	    elsif ($which eq "boss" && $FIREWALL_BOSS_LOCALRULETMPL ne "") {
+		$filename .= "${FIREWALL_BOSS_LOCALRULETMPL}.ipfw";
+	    }
+	    next
+		if (! defined($filename));
 	    
 	    if (-e $filename) {
 		system("cat $filename");
 	    }
+	    else {
+		die("Local template file for $which does not exist\n");
+	    }
+	}
+	elsif ($2 eq "natdrules") {
+	    print "$NATDRULE\n";
 	}
 	elsif (exists($SUB{$2})) {
-	    print "$1" . $SUB{$2} . "$3\n";
+	    print "$1" . $SUB{$2} . " $3\n";
 	}
 	else {
 	    die("No substitution for '$2'\n");

fwrules/ops.local.tmpl

-#
-# Utah specific rules for ops.
-#
-# For backups
-add pass tcp from nstash to me 13782 setup

fwrules/ops.tmpl

+# Setup lo0
+add pass all from any to any via lo0
+add deny all from any to 127.0.0.0/8
+add deny ip from 127.0.0.0/8 to any
+# We do not do ipv6
+add deny all from any to ::1
+add deny all from ::1 to any
+
 # Match existing dynamic rules early
 add check-state
 

fwrules/utahboss.tmpl

+#
+# Utah specific rules for boss. To use this file, or another like it
+# that you create in this directory, add to the defs file:
+#
+# FIREWALL_BOSS=1
+# FIREWALL_BOSS_LOCALRULETMPL="utahboss"
+#
+
+#
+# For backups
+add pass tcp from nstash to me 13782 setup

fwrules/utahops.tmpl

+#
+# Utah specific rules for ops.To use this file, or another like it
+# that you create in this directory, add to the defs file:
+#
+# FIREWALL_OPS=1
+# FIREWALL_OPS_LOCALRULETMPL="utahops"
+#
+# For backups
+add pass tcp from nstash to me 13782 setup

install/emulab-install.in

@@ -243,7 +243,7 @@ elsif ($server eq "boss") {
 	      'boss/ssh', 'boss/hostkeys',
 	      'boss/rndc', 'boss/loaderconf', 'boss/sysctlconf',
 	      'boss/sslcerts', 'boss/mailman', 'boss/pubsub',
-	      'boss/software',
+	      'boss/software', 
               #
 	      # The next few items must be after the software install since 
 	      # they use testbed libraries and such.
@@ -251,7 +251,7 @@ elsif ($server eq "boss") {
 	      'boss/dhcpd', 'boss/named', 'boss/flyspray',
 	      'boss/firstuser', 'boss/checkupuser', 'boss/wikidocs',
 	      'boss/mfs', 'boss/images', 'boss/openvz', 'boss/experiments',
-	      'boss/protogeni'
+	      'boss/protogeni', 'boss/firewall'
 	     );
 }
 elsif ($server eq "fs") {
@@ -266,7 +266,8 @@ elsif ($server eq "ops") {
 	      'ops/sendmail', 'ops/syslog', 'exports', 'nfsmounts',
 	      'ops/crontab', 'sudoers', 'samba', 'ops/ssh', 'capture',
 	      'ops/rcfiles', 'apache', 'ops/database', 'ops/mailman',
-	      'ops/cvsd', 'ops/flyspray', 'ops/twiki');
+	      'ops/cvsd', 'ops/flyspray', 'ops/twiki', 'ops/firewall'
+	     );
 }
 
 #

install/installvars.pm.in

@@ -369,6 +369,12 @@ if ($OUTER_BOSS eq '') {
     $OUTER_BOSS = "www.emulab.net";
 }
 
+# Firewall stuff
+$FIREWALL_BOSS		= @FIREWALL_BOSS@;
+$FIREWALL_OPS		= @FIREWALL_OPS@;
+$FIREWALL_BOSS_RULES    = "/etc/boss.ipfw";
+$FIREWALL_OPS_RULES     = "/etc/ops.ipfw";
+
 sub ISBOSSNODE($)	{ return ($_[0] eq $BOSS_SERVERNAME) ? 1 : 0; }
 sub ISOPSNODE($)	{ return ($_[0] eq $OPS_SERVERNAME) ? 1 : 0; }
 sub ISFSNODE($)		{ return ($_[0] eq $FS_SERVERNAME) ? 1 : 0; }

install/phases/boss/firewall

+#
+# Install the firewall rules if firewalling is on.
+#
+use strict;
+use libinstall;
+use installvars;
+
+sub Install($$$)
+{
+    my ($server, $isupdate, $impotent) = @_;
+
+    # Replace if this script does an update for ip/domain.
+    return 0
+	if (! $FIREWALL_BOSS);
+
+    if ($isupdate) {
+    	Phase "backup", "Backing up $FIREWALL_BOSS_RULES", sub {
+	    BackUpFileFatal($FIREWALL_BOSS_RULES);
+	};
+    }
+    Phase "firewall", "Setting up firewall rules", sub {
+	if ($isupdate) {
+	    Phase "building", "Cleaning old rules", sub {
+		ExecQuietFatal("cd $TOP_OBJDIR/fwrules; $GMAKE clean");
+	    };
+	}
+	Phase "building", "Building firewall rules", sub {
+	    ExecQuietFatal("cd $TOP_OBJDIR/fwrules; $GMAKE");
+	};
+	Phase "installing", "Installing firewall rules", sub {
+	    ExecQuietFatal("cd $TOP_OBJDIR/fwrules; $GMAKE install-real");
+	};
+    };
+    return 0;
+}
+
+# Local Variables:
+# mode:perl
+# End:

install/phases/boss/rcconf

@@ -67,6 +67,13 @@ sub Install($$$)
 	    push(@strings, qq|mrouted_enable="YES"|);
 	}
 
+	# Turn on firewall.
+	if ($FIREWALL_BOSS) {
+	    push(@strings, qq|firewall_enable="YES"|);
+	    push(@strings, qq|firewall_type="/etc/boss.ipfw"|);
+
+	}
+
 	AppendToFileFatal($RCCONF, @strings);
     };
     return 0;

install/phases/ops/firewall

+#
+# Install the firewall rules if firewalling is on.
+#
+use strict;
+use libinstall;
+use installvars;
+
+sub Install($$$)
+{
+    my ($server, $isupdate, $impotent) = @_;
+
+    # Replace if this script does an update for ip/domain.
+    return 0
+	if (! $FIREWALL_OPS);
+
+    if ($isupdate) {
+    	Phase "backup", "Backing up $FIREWALL_OPS_RULES", sub {
+	    BackUpFileFatal($FIREWALL_OPS_RULES);
+	};
+    }
+    Phase "firewall", "Setting up firewall rules", sub {
+	if ($isupdate) {
+	    Phase "building", "Cleaning old rules", sub {
+		ExecQuietFatal("cd $TOP_OBJDIR/fwrules; $GMAKE clean");
+	    };
+	}
+	Phase "building", "Building firewall rules", sub {
+	    ExecQuietFatal("cd $TOP_OBJDIR/fwrules; $GMAKE");
+	};
+	Phase "installing", "Installing firewall rules", sub {
+	    ExecQuietFatal("cd $TOP_OBJDIR/fwrules; ".
+			   "  $GMAKE control-install-real");
+	};
+    };
+    return 0;
+}
+
+# Local Variables:
+# mode:perl
+# End:

install/phases/ops/rcconf

@@ -37,6 +37,12 @@ sub Install($$$)
 		 qq|pubsubd_flags="-T 10"|);
     }
 
+    # Turn on firewall.
+    if ($FIREWALL_OPS) {
+	push(@adds, qq|firewall_enable="YES"|);
+	push(@adds, qq|firewall_type="/etc/ops.ipfw"|);
+    }
+
     Phase "rc.conf", "Adding testbed content to rc.conf", sub {
 	DoneIfEdited($RCCONF);
 	BackUpFileFatal($RCCONF);