Experimental changes for SCSI disks (CAM) to enable the write-cache (WCE).

This gives a factor of four improvement in raw sequential write speed
and ensure, for the moment, that the disk is not the bottleneck in frisbee
loading.

tmcd/freebsd/GNUmakefile.in

@@ -90,6 +90,7 @@ frisbee-mfs-install:	destdircheck
 	$(INSTALL) -m 755 $(SRCDIR)/rc.frisbee $(SYSETCDIR)/testbed
 	$(INSTALL) -m 755 $(SRCDIR)/rc.ipod $(SYSETCDIR)/testbed
 	$(INSTALL) -m 755 $(SRCDIR)/slicefix $(SYSETCDIR)/testbed
+	$(INSTALL) -m 755 $(SRCDIR)/camwce $(SYSETCDIR)/testbed
 	$(INSTALL) -m 755 -s ../tmcc-nossl $(SYSETCDIR)/testbed/tmcc
 	$(INSTALL) -m 755 -s ../findif $(SYSETCDIR)/testbed
 
@@ -100,6 +101,7 @@ cdboot-install:	destdircheck common-install etc-install \
 	$(INSTALL) -m 755 $(SRCDIR)/cdboot/rc.cdroot $(SYSETCDIR)/rc.cdroot
 	$(INSTALL) -m 755 $(SRCDIR)/rc.frisbee $(BINDIR)/rc/rc.frisbee
 	$(INSTALL) -m 755 $(SRCDIR)/slicefix $(BINDIR)/slicefix
+	$(INSTALL) -m 755 $(SRCDIR)/camwce $(BINDIR)/camwce
 	cp $(SRCDIR)/cdboot/fstab $(SYSETCDIR)/fstab
 	cp $(SRCDIR)/cdboot/fstab.lbs $(SYSETCDIR)/fstab.lbs
 	$(INSTALL) -m 755 -o root -g wheel -d $(DESTDIR)/boot

tmcd/freebsd/camwce

+#!/bin/sh
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2005 University of Utah and the Flux Group.
+# All rights reserved.
+#
+# Wicked sleezy script for turning write-cache-enable (WCE) on and off
+# for a SCSI drive.  It is sleezy because it acts both as the user interface
+# and as a back-end editor for camcontrol depending on how it is called.
+#
+# Why is this needed?  We need to make up for the lack of a batch mode
+# style of operation in camcontrol coupled with a desire not to install
+# "ed" (rather the 1MB crypto library it depends on but would never use)
+# in the MFS just to edit the camcontrol tmp file and change 1 bit.
+#
+# Note that we are only changing the "current" state so the previous WCE
+# state will be restored on a machine reset.
+#
+
+if [ -r /etc/emulab/paths.sh ]; then
+	. /etc/emulab/paths.sh
+else
+	BINDIR=/etc/testbed
+fi
+
+if [ $# -lt 1 ]; then
+    echo 'usage: camwce <on|off> <dev>'
+    exit 1
+fi
+cof=$1
+case $cof in
+on|off)
+    if [ $# -lt 2 ]; then
+	echo 'usage: camwce <on|off> <dev>'
+	exit 1
+    fi
+    dev=$2
+    case $dev in
+    da*)
+	;;
+    *)
+        echo "$dev: not a SCSI device"
+        exit 1
+	;;
+    esac
+    wce=`camcontrol modepage $dev -m8 | grep 'WCE:' | sed 's/WCE:[ 	]*\([01]\)/\1/'`
+    if [ $cof = "on" -a "$wce"x = "0x" -o $cof = "off" -a "$wce"x = "1x" ]
+    then
+	echo "${dev}: turning write-cache $cof"
+	EDITOR=$BINDIR/camwce camcontrol modepage $dev -m8 -e
+    else
+	echo "${dev}: write-cache already $cof"
+	exit 1
+    fi
+    ;;
+*)
+    #
+    # sleezy: when invoked with one arg, we are being called as an
+    # editor and need to toggle the WCE bit in the given file using an
+    # obscure sed invocation.
+    #
+    if [ $# -ne 1 ]; then
+	echo 'usage: camwce <on|off> <dev>'
+	exit 1
+    fi
+    sed -i '' -e '/WCE:[ 	]*[01]/y/01/10/' $cof
+    ;;
+esac
+exit $?

tmcd/freebsd/rc.frisbee

@@ -7,7 +7,6 @@
 # Optional flag argument says "do not reboot"
 #
 reboot=1
-
 if [ $# -eq 1 -a "$1" = "-noreboot" ]; then
     reboot=0
 fi
@@ -133,11 +132,36 @@ if [ x"$ADDRESS" != x ]; then
 	    ZFILL=""
 	fi
 
+	#
+	# Make sure the write-cache is enabled on SCSI disks.  It makes a
+	# huge difference.  We don't worry about data corruption in the
+	# case of a crash, because we will just reload the disk again anyway
+	# in that situation.
+	#
+	turncacheoff=0
+	case $DISK in
+	da*)
+	    if [ -x $BINDIR/camwce ] && $BINDIR/camwce on $DISK; then
+		turncacheoff=1;
+	    fi
+	    ;;
+	esac
+
 	echo "Running $BINDIR/frisbee $LOADIP $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK at `date`"
 	$BINDIR/tmcc state RELOADING
 
 	$BINDIR/frisbee $LOADIP $MEMARGS $ZFILL $SLICE $MCASTIF $MCASTADDR /dev/$DISK
-	case $? in
+	fstat=$?
+
+	#
+	# Turn the cache back off if we turned it on.
+	# Is this sufficient to ensure the cache gets flushed?
+	#
+	if [ $turncacheoff -eq 1 ]; then
+	    $BINDIR/camwce off $DISK
+	fi
+
+	case $fstat in
 	0)
 		echo "Frisbee run finished"
 		echo "Resizing final disk partition"