Merge the dbtoir branch!

assign/README

@@ -62,6 +62,9 @@ node <node> <type> [<desires>]
 		A weight >= 1.0 will also result in a violation if
 		not filled.
 
+OR
+
+link <src> <dst> <bw>
 
 features and desires
 --------------------

configure

@@ -770,6 +770,8 @@ fi
 
 
 
+
+
 #
 # Okay, I know this is improper usage of --with. Too bad.
 #
@@ -862,7 +864,7 @@ fi
 # SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
 # ./install, which can be erroneously created by make from ./install.sh.
 echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6
-echo "configure:866: checking for a BSD compatible install" >&5
+echo "configure:868: checking for a BSD compatible install" >&5
 if test -z "$INSTALL"; then
 if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then
   echo $ac_n "(cached) $ac_c" 1>&6
@@ -940,14 +942,12 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/batchexp tbsetup/killbatchexp tbsetup/batch_daemon \
 	tbsetup/webbatchexp tbsetup/webkillbatchexp \
 	tbsetup/startexp tbsetup/endexp tbsetup/webstartexp tbsetup/webendexp \
-	tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
-	tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
-	tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
+	tbsetup/snmpit tbsetup/ns2ir/GNUmakefile \
 	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
-	tbsetup/savevlans tbsetup/ir/extract_tb \
-	tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
-	tbsetup/named_setup tbsetup/exports_setup \
-	tbsetup/checkpass/GNUmakefile \
+	tbsetup/ns2ir/sim.tcl tbsetup/savevlans \
+	tbsetup/tbprerun tbsetup/tbswapin tbsetup/tbswapout tbsetup/tbend \
+	tbsetup/tbreport tbsetup/named_setup tbsetup/exports_setup \
+	tbsetup/checkpass/GNUmakefile tbsetup/assign_wrapper tbsetup/ptopgen \
 	tip/GNUmakefile \
 	tmcd/GNUmakefile tmcd/freebsd/GNUmakefile tmcd/linux/GNUmakefile \
 	tmcd/netbsd/GNUmakefile \
@@ -1125,6 +1125,8 @@ s%@TBDBNAME@%$TBDBNAME%g
 s%@WWWDEFS@%$WWWDEFS%g
 s%@TBOPSEMAIL@%$TBOPSEMAIL%g
 s%@TBLOGSEMAIL@%$TBLOGSEMAIL%g
+s%@DELAYCAPACITY@%$DELAYCAPACITY%g
+s%@IPBASE@%$IPBASE%g
 s%@LEDA@%$LEDA%g
 s%@OPT_CFLAGS@%$OPT_CFLAGS%g
 s%@INSTALL_PROGRAM@%$INSTALL_PROGRAM%g

configure.in

@@ -18,6 +18,8 @@ AC_SUBST(TBDBNAME)
 AC_SUBST(WWWDEFS)
 AC_SUBST(TBOPSEMAIL)
 AC_SUBST(TBLOGSEMAIL)
+AC_SUBST(DELAYCAPACITY)
+AC_SUBST(IPBASE)
 
 #
 # Okay, I know this is improper usage of --with. Too bad.
@@ -113,14 +115,12 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/batchexp tbsetup/killbatchexp tbsetup/batch_daemon \
 	tbsetup/webbatchexp tbsetup/webkillbatchexp \
 	tbsetup/startexp tbsetup/endexp tbsetup/webstartexp tbsetup/webendexp \
-	tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
-	tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
-	tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
+	tbsetup/snmpit tbsetup/ns2ir/GNUmakefile \
 	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
-	tbsetup/savevlans tbsetup/ir/extract_tb \
-	tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
-	tbsetup/named_setup tbsetup/exports_setup \
-	tbsetup/checkpass/GNUmakefile \
+	tbsetup/ns2ir/sim.tcl tbsetup/savevlans \
+	tbsetup/tbprerun tbsetup/tbswapin tbsetup/tbswapout tbsetup/tbend \
+	tbsetup/tbreport tbsetup/named_setup tbsetup/exports_setup \
+	tbsetup/checkpass/GNUmakefile tbsetup/assign_wrapper tbsetup/ptopgen \
 	tip/GNUmakefile \
 	tmcd/GNUmakefile tmcd/freebsd/GNUmakefile tmcd/linux/GNUmakefile \
 	tmcd/netbsd/GNUmakefile \

db/GNUmakefile.in

@@ -11,7 +11,7 @@ include $(OBJDIR)/Makeconf
 BIN_SCRIPTS	= mac2if nalloc nfree nodeip
 SBIN_SCRIPTS	= avail inuse showgraph if2port backup webcontrol node_status \
                   genelists
-LIBEXEC_SCRIPTS = ptopgen
+LIBEXEC_SCRIPTS = 
 LIB_SCRIPTS     = libdb.pm
 
 # Stuff installed on plastic for mere users.

db/ptopgen

-#!/usr/local/bin/perl -w
-#
-# Generates a ptop file from the output of an avail command.
-# Takes input on STDIN and output goes to STDOUT
-#
-
-use Mysql;
-my $dbh = Mysql->connect("localhost","tbdb","script","none");
-
-my $d = 0; #debug
-my %nodes = ();
-my %links = ();
-my $node;
-
-while (<>) {
-  if (!/^\|/ ) {next;}
-  if ( /^\|node_id/) {next;}
-  /^\|([a-zA-Z0-9]*).*\|/;
-  $node=$1;
-  if ($node =~/^((tb){0,1}pc|sh)/) {
-    print STDERR "Got '$node' \t" if $d && !defined ($nodes{$node});
-    $nodes{$node} = 1;
-  }
-}
-
-print STDERR "\n" if $d;
-
-my @list = sort keys %nodes;
-foreach my $item (@list) { 
-  print STDERR "From '$item'\t" if $d > 1;
-  if ($item =~ /^sh/) { # If its a shark
-    $item = "node_id='$item-1'"; 
-  } else {
-    $item = "node_id='$item'"; 
-  }
-  print STDERR "To '$item'\n" if $d > 1;
-}
-my $cond = join (" or ",@list);
-
-my $query = "select node_id,card,MAC from interfaces where $cond";
-
-print STDERR "\nUsing query '$query'\n\n" if $d;
-
-my $sth = $dbh->query($query);
-
-while (my @row = $sth->fetchrow()) {
-  print STDERR "Got '$row[0]'\t'$row[1]'\t'$row[2]'\n" if $d > 1;
-  if ( $row[0] =~ /^(sh\d+)-\d/ ) {
-    $links{$1} = $row[2];
-  } else {
-    $links{$row[0]."-".$row[1]} = $row[2];
-  }
-}
-
-# In every ptop
-print "node cisco switch\n";
-
-# Give the nodes:
-foreach $node (sort keys %nodes) {
-  if ($node =~ /pc/) {
-    print "node $node pc:1 delay:2\n";
-    print "link $node-0 $node:$links{$node.'-0'} cisco 100 1\n";
-    print "link $node-1 $node:$links{$node.'-1'} cisco 100 1\n";
-    print "link $node-2 $node:$links{$node.'-2'} cisco 100 1\n";
-    print "link $node-3 $node:$links{$node.'-3'} cisco 100 1\n";
-  } else {
-    print "node $node shark-shelf:1\n";
-    print "link $node $node:$links{$node} cisco 100 1\n";
-  }
-}
-
-# Add lans to ptopfile
-# lan nodes do not really exist, they represent the ability of the cisco
-# to set up lans via vlans.  We set the maxmimum number of links on a lan
-# node to some very high number.  This should actually be the maximum number
-# of ports in a vlan.  Which, since by default everything is in a VLAN, this
-# maxmimum number will be >= the number of ports on the CISCO(s) so really
-# need only be a number higher then we'll ever need.
-$maxlans = 20;
-for ($i=0;$i<$maxlans;$i++) {
-    print "node lan$i lan:1\n";
-    print "link lan$i-link lan$i cisco 100 1000\n";
-}
-

defs-calfeld-emulab

@@ -8,3 +8,6 @@ TBDBNAME=tbdb
 WWWDEFS=default
 TBOPSEMAIL=calfeld@flux.cs.utah.edu
 TBLOGSEMAIL=calfeld@flux.cs.utah.edu
+IPBASE=192.168
+DELAYCAPACITY=2
+

defs-default

@@ -8,3 +8,5 @@ TBDBNAME=tbdb
 WWWDEFS=default
 TBOPSEMAIL='Testbed Operations <testbed-ops@flux.cs.utah.edu>'
 TBLOGSEMAIL=testbed-logs@flux.cs.utah.edu
+IPBASE=192.168
+DELAYCAPACITY=2
\ No newline at end of file

defs-newbold-emulab

@@ -5,5 +5,7 @@ TBDBNAME=tbdb
 WWWDEFS=newbold-emulab
 TBOPSEMAIL=newbold@cs.utah.edu
 TBLOGSEMAIL=newbold@cs.utah.edu
+IPBASE=192.168
+DELAYCAPACITY=2
 
 

defs-stoller-emulab

@@ -5,5 +5,7 @@ TBDBNAME=tbdb
 WWWDEFS=stoller-emulab
 TBOPSEMAIL='Leigh B Stoller <stoller@fast.cs.utah.edu>'
 TBLOGSEMAIL='Leigh B Stoller <stoller@fast.cs.utah.edu>'
+IPBASE=192.168
+DELAYCAPACITY=2
 
 

defs-stoller-home

@@ -5,4 +5,6 @@ TBDBNAME=tbdb
 WWWDEFS=stoller-home
 TBOPSEMAIL='Leigh B Stoller <stoller@stoller.casco.net>'
 TBLOGSEMAIL='Leigh B Stoller <stoller@stoller.casco.net>'
+IPBASE=192.168
+DELAYCAPACITY=2
 

tbsetup/GNUmakefile.in

@@ -8,9 +8,9 @@ SUBDIR		= tbsetup
 
 include $(OBJDIR)/Makeconf
 
-SUBDIRS		= checkpass ir ns2ir
+SUBDIRS		= checkpass ns2ir
 
-BIN_STUFF	= power snmpit tbend tbrun tbprerun tbreport \
+BIN_STUFF	= power snmpit tbend tbswapin tbswapout tbprerun tbreport \
 		  os_load savevlans startexp endexp batchexp killbatchexp \
 		  node_reboot nscheck
 
@@ -22,7 +22,8 @@ SBIN_STUFF	= resetvlans console_setup.proxy sched_reload named_setup \
 
 LIBEXEC_STUFF	= mkprojdir rmproj mkacct-ctrl rmacct-ctrl \
 		  os_setup mkexpdir console_setup webnscheck \
-		  webstartexp webendexp webbatchexp webkillbatchexp
+		  webstartexp webendexp webbatchexp webkillbatchexp \
+		  assign_wrapper ptopgen
 
 LIB_STUFF       = libtbsetup.pm libtestbed.pm snmpit_intel.pm \
                   snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm
@@ -35,11 +36,10 @@ all:	$(BIN_STUFF) $(SBIN_STUFF) $(LIBEXEC_STUFF) $(LIB_STUFF) $(SUBDIRS)
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
-.PHONY:	checkpass ir ns2ir
+.PHONY:	checkpass ns2ir
 checkpass:
 	@$(MAKE) -C checkpass all
-ir:
-	@$(MAKE) -C ir all
+
 ns2ir:
 	@$(MAKE) -C ns2ir all
 
@@ -51,7 +51,6 @@ install: all script-install subdir-install
 # 
 subdir-install:
 	@$(MAKE) -C checkpass install	
-	@$(MAKE) -C ir install	
 	@$(MAKE) -C ns2ir install
 
 script-install:	$(addprefix $(INSTALL_BINDIR)/, $(BIN_STUFF)) \
@@ -99,14 +98,12 @@ clean:	subdir-clean
 
 subdir-clean:
 	@$(MAKE) -C checkpass clean
-	@$(MAKE) -C ir clean
 	@$(MAKE) -C ns2ir clean
 
 distclean:	subdir-distclean
 
 subdir-distclean:
 	@$(MAKE) -C checkpass distclean
-	@$(MAKE) -C ir distclean
 	@$(MAKE) -C ns2ir distclean
 
 #

tbsetup/README

-tbsetup
+tbsetup README
+
+----------------------------------------------------------------------
+tbprerun
 
 Syntax:
-	tbprerun <pid> <eid> <ns-file>
+	tbprerun [-nologfile] <pid> <eid> <ns-file>
 
-tbprerun does all tasks necessary to setup up for an experiment on the
-testbed.  It generates an IR file that can be used with 'tbrun' to
-actually configure the testbed.
+tbprerun takes a brand new experiment, checks the syntax of the
+ns-file, and then converts the ns-file into a virtual topology stored
+in the database. 
 
-CAVEAT: This does do actual reservation of nodes.  Be sure to free the
-nodes using tbend, even if you never run the experiment.
+-nologfile - Prevents creation of a logfile.
 
+----------------------------------------------------------------------
 
-tbrun
+tbswapin
 
 Syntax:
-	tbrun <pid> <eid> <ir-file>
+	tbswapin [-nologfile] <pid> <eid>
 
-tbrun setups up the testbed for the experiment.  
+tbswapin swaps an experiment in.  Specifically it finds and allocates
+a set of resources and configures these resources to emulate the
+desired experiment.  As a part of this it configures the physical
+state in the DB.
 
+-nologfile - Prevents creation/appending of a logfile.
 
-tbreport
+Exit Codes:
+
+tbswapin exits with a variety of codes to represent different error
+conditions.  Almost all of these codes are actually generated by
+assign_wrapper and just passed through by tbswapin.
+
+Exit 0 : All good
+2: Insufficient resources.
+X: other errors
+1+
+        4 - Bandwidth violation
+        8 - Linkusers violation  
+        16 - Desires violation.  
+
+I.e.
+
+1 - Unspecified error
+5 - Bandwidth error 
+9 - Linkusers error
+17 - Desires error. 
+13 - Bandwidth and linkusers error.
+21 - Bandwidth and desires error.
+25 - Linkusers and desires error.
+29 - Bandwidth, linkusers, and desires.
+
+----------------------------------------------------------------------
+
+tbswapout
 
 Syntax:
-	tbreport [-v] <ir-file>
+	tbswapout [-nologfile] [-force] <pid> <eid>
 
-tbreport parses the IR file and produces a human readable report on
-the contents.  -v will include extra information.
+tbswapout is the inverse of tbswapin.  It tears down the physical
+state on the testbed, freeing the resources for other use.  It clears
+the physical state from the DB but leaves the virtual state intact.  I.e.
+it reverse tbswapin but not tbprerun.
 
+-nologfile - Prevents creation/appending of a logfile.
+-force - Does not check current experiment state.
+
+----------------------------------------------------------------------
 
 tbend
 
 Syntax:
-	tbend <pid> <eid>
+	tbend [-nologfile] [-force] <pid> <eid>
+
+tbend is the inverse of tbprerun.  It clears the virtual state from
+the DB and marks the experiment as ended.
+
+-nologfile - Prevents creation/appending of a logfile.
+-force - Does not check current experiment state.
+
+----------------------------------------------------------------------
+
+tbreport
+
+Syntax:
+	tbreport [-v] <pid> <eid>
+
+tbreport displays the current state of an experiment.  It displays
+both the virtual topology, and, if the experiment is currently
+running, the physical mapping.
+
+-v - Display extra information.
+
+----------------------------------------------------------------------
+
+Basic Operation
+
+The basic operation of the tbsetup tools is:
+
+1. tbprerun
+2. tbswapin
+3. do experiment
+4. tbswapout
+5. Repeat steps 2-4 as necessary.
+6. tbend
+
+It is assumed that an experiment is already entered in the DB before
+tbprerun is called.  Likewise, tbend does not remove the entry.
+
+----------------------------------------------------------------------
+
+Experimental States
+
+The experiment goes through the following states:
+
+new - Before tbprerun. tbprerun requires the experiment to be in this
+state before running.
+
+prerunning - During tbprerun.
+
+swapped - After tbprerun or tbswapout.  tbswapin requires the
+experiment to be in this state before running.  tbend requires this
+state unless -force is used.
+
+activating - During tbswapin.
+
+active - After tbswapin.  tbswapout requires this state unless -force
+is used.
+
+swapping - During tbswapout.
+
+terminating - During tbend.
+
+ended - After tbend.
+
+----------------------------------------------------------------------
+
+Non-symmetries
+
+In general tbprerun and tbend are inverse operations and tbswapin and
+tbswapout are inverse operations.  This is not completely true,
+however.  tbswapout does completely reverse tbswapin, and tbend clears
+some state that is set in tbswapin, rather than tbprerun.
+
+Node state: For performance reasons tbswapout does not touch node
+state.  The idea being that node state only matters for active
+experiments and these experiments will reconfigure the node state when
+they are swapped in.
+
+Portmap: The portmap table in the DB is not cleared by tbswapout.
+This is because the portmap table will be used the next time tbswapin
+is called to try to match the portmapping of the previous swap.  The
+purpose behind this is to preserve the same port mapping across swaps.
+The portmap table is cleared in tbend instead.
+
 
-tbend does tear down. 
-This should be run at the end of every experiment even if nothing was
-done in the experiment.

tbsetup/batch_daemon.in

@@ -25,6 +25,11 @@ my $DBNAME   = "@TBDBNAME@";
 my $TBOPS    = "@TBOPSEMAIL@";
 my $TBLOGS   = "@TBLOGSEMAIL@";
 
+#
+# Ug, exit value from startexp when not enough nodes.
+# 
+my $TOOFEWNODES = 2;
+
 my $tbbindir = "$TB/bin/";
 my $batchdir = "$TB/batch";
 my $startexp = "$TB/bin/startexp";
@@ -333,9 +338,9 @@ sub startexp($)
     $query_result =
 	DBquery("insert into experiments ".
 		"(eid, pid, expt_created, expt_name, ".
-		"expt_head_uid, expt_start, expt_ready, batchmode) ".
+		"expt_head_uid, expt_start, state, batchmode) ".
 		"VALUES ('$eid', '$pid', '$rightnow', '$longname', ".
-		"'$creator', '$rightnow', 0, 1)");
+		"'$creator', '$rightnow', 'new', 1)");
     if (! $query_result) {
 	fatal("DB error inserting experiment record. Quitting ...\n");
     }
@@ -387,8 +392,9 @@ sub startexp($)
 		    "attempts=attempts+1 where eid='$eid' and pid='$pid'");
 	$attempts++;
 
-	if (($exit_status == 77 && $attempts >= 9 && (($attempts % 9) == 0)) ||
-	    (($exit_status != 77) && ($attempts % 5) == 0) ||
+	if (($exit_status == $TOOFEWNODES && $attempts >= 9 &&
+	     (($attempts % 9) == 0)) ||
+	    (($exit_status != $TOOFEWNODES) && ($attempts % 5) == 0) ||
 	    ($attempts == 0)) {
 	    
 	    fatal("Could not configure Batch Mode experiment $pid/$eid\n".

tbsetup/console_setup.in

@@ -25,7 +25,7 @@ require libdb;
 $| = 1; 
 
 my $SSH		= "sshtb -n -q -l root plastic.emulab.net";
-my $PROG	= "$TB/sbin/console_setup.proxy";
+my $PROG	= "/usr/testbed/sbin/console_setup.proxy";
 my $TBPID	= "flux";
 my $cmdargs     = "";
 my @row;

tbsetup/endexp.in

@@ -34,6 +34,7 @@ my $projroot = "/proj";
 my $tbdata   = "tbdata";
 my $logname  = 0;
 my $batch    = 0;
+my $estate   = 0;
 
 #
 # Untaint the path
@@ -89,44 +90,6 @@ my $eiddir  = "$expdir/$eid";
 use Mysql;
 my $DB = Mysql->connect("localhost", $DBNAME, "script", "none");
 
-#
-# We have to protect against trying to end an experiment that is currently
-# in the process of being terminated. We use a timestamp for this purpose.
-# If the timestamp is ever non-null, then something is wrong and we should
-# not proceed.
-#
-# We also have to guard against trying to terminate an experiment that
-# is still in the process of configuring. Its easiest to force the user
-# to wait!
-#
-$query_result =
-    $DB->query("SELECT expt_terminating,expt_ready FROM experiments ".
-	       "WHERE eid='$eid' and pid='$pid'");
-
-if (! $query_result) {
-    fatal("DB Error getting experiment termination date for $pid/$eid\n");
-}
-if ($query_result->numrows < 1) {
-    print STDOUT "No such experiment $pid/$eid exists!\n";
-    exit(1);
-}
-
-@row = $query_result->fetchrow_array();
-if (defined($row[0])) {
-    print STDOUT
-	"It appears that $pid/$eid started terminating at $row[0]\n".
-	"You will be notified via email when the experiment has been ".
-	"torn down\n";
-    exit(1);
-}
-if (! $row[1]) {
-    print STDOUT
-	"It appears that experiment $pid/$eid is still configuring.\n".
-	"The user that created the experiment will be notified via email\n".
-	"when it has been fully configured and is ready for use\n";
-    exit(1);
-}
-
 #
 # Get some user information. 
 #
@@ -168,7 +131,59 @@ if (! $isadmin) {
 }
 
 #
-# Set the timestamp.
+# We have to protect against trying to end an experiment that is currently
+# in the process of being terminated. We use a "wrapper" state (actually
+# a timestamp so we can say when termination was requested) since
+# terminating conists of a couple of different experiment states down inside
+# the tb scripts. 
+#
+$query_result = $DB->query("lock tables experiments write");
+if (! $query_result) {
+    fatal("DB Error locking experiments table");
+}
+
+$query_result =
+    $DB->query("SELECT expt_terminating,state FROM experiments ".
+	       "WHERE eid='$eid' and pid='$pid'");
+
+if (! $query_result) {
+    fatal("DB Error getting experiment termination date for $pid/$eid\n");
+}
+if ($query_result->numrows < 1) {
+    print STDOUT "No such experiment $pid/$eid exists!\n";
+    exit(1);
+}
+
+@row = $query_result->fetchrow_array();
+if (defined($row[0])) {
+    print STDOUT
+	"It appears that $pid/$eid started terminating at $row[0]\n".
+	"You will be notified via email when the experiment has been ".
+	"torn down\n";
+    exit(1);
+}
+
+#
+# Okay, check state. We do not allow termination to start when the
+# experiment is in transition. A future task would be to allow this,
+# but for now the experiment must be in one of a few states to proceed
+#
+# Seems like too many states!
+#
+$estate = $row[1];
+if ($estate eq "prerunning" ||
+    $estate eq "activating" ||
+    $estate eq "swapping" ||
+    $estate eq "terminating") {
+    print STDOUT
+	"It appears that experiment $pid/$eid is in transition.\n".
+	"The user that created the experiment will be notified via email\n".
+        "when the experiment is no longer in transition, and can be killed\n";
+    exit(1);
+}
+
+#
+# Set the timestamp now, and unlock the experiments table.
 #
 $stamp = `date '+20%y-%m-%d %H:%M:%S'`;
 
@@ -178,6 +193,11 @@ if (! $query_result) {
     fatal("DB Error setting expt_terminating for experiment $pid/$eid\n");
 }
 
+$query_result = $DB->query("unlock tables");
+if (! $query_result) {
+    fatal("DB Error unlocking experiments table");
+}
+
 #
 # If not in batch mode, go into the background. Parent exits.
 #
@@ -194,6 +214,16 @@ if (! $batch) {
     }
 }
 
+if ($estate eq "active") { 
+    print STDOUT "Running tbswapout with arguments: -nologfile $pid $eid\n";
+    if (system("$tbdir/tbswapout -nologfile $pid $eid") != 0) {
+	fatal("tbswapout failed!\n");
+    }
+}
+elsif ($estate ne "swapped") {
+    fatal("Experiment is in the wrong state: $estate\n");
+}
+
 print STDOUT "Running tbend with arguments: -nologfile $pid $eid\n";