diff --git a/pxe/GNUmakefile.in b/pxe/GNUmakefile.in
index fc2c375568a8acca82e13aed4a7abc95183c2bd0..b92017033b2ddf1aa62c8754f699d7c47d88ec04 100644
--- a/pxe/GNUmakefile.in
+++ b/pxe/GNUmakefile.in
@@ -1,6 +1,6 @@
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2003 University of Utah and the Flux Group.
+# Copyright (c) 2000-2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -16,7 +16,7 @@ include $(OBJDIR)/Makeconf
 # Force dependencies on the scripts so that they will be rerun through
 # configure if the .in file is changed.
 # 
-all: proxydhcp bootinfo proxydhcp.restart bootinfo.restart
+all: proxydhcp bootinfo bootinfosend proxydhcp.restart bootinfo.restart
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
@@ -28,15 +28,19 @@ BI_DBOBJ   = bootinfo_null.o bootinfo_cfile.o bootinfo_mysql.o
 PR_DBSRC   = proxydhcp_cfile.c proxydhcp_mysql.c 
 PR_DBOBJ   = proxydhcp_cfile.o proxydhcp_mysql.o 
 
-INCS    = -I/usr/local/include
+INCS    = -I${OBJDIR} -I/usr/local/include -I$(TESTBED_SRCDIR)/lib/libtb
 
-CFLAGS	+= $(INCS) $(DBFLAG) -DSOLARIS -DHAVE_SOCKADDR_SA_LEN -DUSE_RECVMSG \
-		-DCONFPATH='"$(INSTALL_ETCDIR)/"' -DTBDBNAME='"$(TBDBNAME)"' \
-		-DFALLBACK_HOST='"$(BOSSNODE)"' -DBOSSNODE='"$(BOSSNODE)"' \
-		-DDEFAULT_PATH='"/tftpboot/pxeboot.newnode"' \
-		-DLOG_TESTBED=$(LOG_TESTBED)
+CFLAGS	+= -Wall \
+	$(INCS) $(DBFLAG) -DSOLARIS -DHAVE_SOCKADDR_SA_LEN -DUSE_RECVMSG \
+	-DCONFPATH='"$(INSTALL_ETCDIR)/"' -DTBDBNAME='"$(TBDBNAME)"' \
+	-DFALLBACK_HOST='"$(BOSSNODE)"' -DBOSSNODE='"$(BOSSNODE)"' \
+	-DDEFAULT_PATH='"/tftpboot/pxeboot.newnode"' \
+	-DLOG_TESTBED=$(LOG_TESTBED)
 
 ifeq ($(EVENTSYS),1)
+BI_DBSRC += event-support.c
+BI_DBOBJ += event-support.o
+
 CFLAGS  += -DEVENTSYS -I$(TESTBED_SRCDIR)/event/lib \
 		`elvin-config --cflags vin4c`
 LFLAGS  += $(OBJDIR)/event/lib/libevent.a ${OBJDIR}/lib/libtb/libtb.a \
@@ -48,11 +52,18 @@ proxydhcp: proxydhcp.o $(PR_DBOBJ)
 		-o proxydhcp proxydhcp.o $(PR_DBOBJ) \
 		$(LFLAGS) -L/usr/local/lib/mysql -lmysqlclient
 
-bootinfo: bootinfo.o $(BI_DBOBJ)
+bootinfo: bootinfo.o bootinfo.h bootinfo_version.o bootwhat.h $(BI_DBOBJ)
+	cc $(CFLAGS) $(DBFLAG) $(INCS) \
+		-o bootinfo bootinfo.o bootinfo_version.o $(BI_DBOBJ) \
+		$(LFLAGS) -L/usr/local/lib/mysql -lmysqlclient
+
+bootinfosend: bootinfosend.o bootinfo.h bootinfo_version.o bootwhat.h $(BI_DBOBJ)
 	cc $(CFLAGS) $(DBFLAG) $(INCS) \
-		-o bootinfo bootinfo.o $(BI_DBOBJ) \
+		-o bootinfosend bootinfosend.o bootinfo_version.o $(BI_DBOBJ) \
 		$(LFLAGS) -L/usr/local/lib/mysql -lmysqlclient
 
+bootinfo_mysql.o:	bootinfo.h bootwhat.h
+
 testbootinfo_mysql: bootinfo_mysql.c
 	cc $(CFLAGS) -DUSE_MYSQL_DB -DTEST $(INCS) \
 		-o testmysql $< \
@@ -69,12 +80,20 @@ testproxydhcp_mysql: proxydhcp_mysql.c
 testproxydhcp_cfile: proxydhcp_cfile.c
 	cc $(CFLAGS) -DUSE_CFILE_DB -DTEST $(INCS) -o testcfile $< $(LFLAGS)
 
+proxydhcp_version.c: proxydhcp.c proxydhcp_mysql.c
+	echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
+
+bootinfo_version.c: bootinfo.c bootinfo_mysql.c
+	echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
+
+
 install:	all
 
 install:	$(INSTALL_SBINDIR)/proxydhcp \
 		$(INSTALL_SBINDIR)/proxydhcp.restart \
 		$(INSTALL_SBINDIR)/bootinfo \
 		$(INSTALL_SBINDIR)/bootinfo.restart \
+		$(INSTALL_SBINDIR)/bootinfosend \
 		$(INSTALL_ETCDIR)/proxydhcp.conf \
 		$(INSTALL_ETCDIR)/bootinfo.conf
 
diff --git a/pxe/bootinfo.c b/pxe/bootinfo.c
index cf5c45c1e7da643ee99e23e3f1073bf717e53668..b1563a4b6052e59c1b691a9d48344716a3b8360f 100644
--- a/pxe/bootinfo.c
+++ b/pxe/bootinfo.c
@@ -1,90 +1,176 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2003 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2004 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <arpa/inet.h>
 #include <stdio.h>
-#include <syslog.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
 #include <signal.h>
+#include "log.h"
+#include "tbdefs.h"
 #include "bootwhat.h"
+#include "bootinfo.h"
 
-static void log_bootwhat(struct in_addr ipaddr, boot_what_t *bootinfo);
-static void onhup(int sig);
-static int version = 1;
+static void	log_bootwhat(struct in_addr ipaddr, boot_what_t *bootinfo);
+static void	onhup(int sig);
+static char	*progname;
+int		debug = 0;
 
-main()
+void
+usage()
 {
-	int			sock, length, data, i, mlen, err;
+	fprintf(stderr,
+		"Usage: %s <options> [-d]\n"
+		"options:\n"
+		"-d         - Turn on debugging\n"
+		"-p port    - Specify port number to listen on\n",
+		progname);
+	exit(-1);
+}
+
+int
+main(int argc, char **argv)
+{
+	int			sock, length, mlen, err, c;
 	struct sockaddr_in	name, client;
 	boot_info_t		boot_info;
 	boot_what_t	       *boot_whatp = (boot_what_t *) &boot_info.data;
+	int		        port = BOOTWHAT_DSTPORT;
+	extern char		build_info[];
 
-	openlog("bootinfo", LOG_PID, LOG_TESTBED);
-	syslog(LOG_NOTICE, "daemon starting (version %d)", version);
+	progname = argv[0];
+
+	while ((c = getopt(argc, argv, "p:dhv")) != -1) {
+		switch (c) {
+		case 'd':
+			debug++;
+			break;
+		case 'p':
+			port = atoi(optarg);
+			break;
+		case 'v':
+		    	fprintf(stderr, "%s\n", build_info);
+			exit(0);
+			break;
+		case 'h':
+		case '?':
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
 
-	(void)daemon(0, 0);
+	if (argc)
+		usage();
+
+	if (debug) 
+		loginit(0, 0);
+	else {
+		/* Become a daemon */
+		daemon(0, 0);
+		loginit(1, "bootinfo");
+	}
+	info("%s\n", build_info);
 
 	/* Initialize data base */
 	err = open_bootinfo_db();
 	if (err) {
-		syslog(LOG_ERR, "could not open database");
+		error("could not open database");
+		exit(1);
+	}
+#ifdef EVENTSYS
+	err = bievent_init();
+	if (err) {
+		error("could not initialize event system");
 		exit(1);
 	}
- 
+#endif
 	/* Create socket from which to read. */
 	sock = socket(AF_INET, SOCK_DGRAM, 0);
 	if (sock < 0) {
-		syslog(LOG_ERR, "opening datagram socket: %m");
+		errorc("opening datagram socket");
 		exit(1);
 	}
 	
 	/* Create name. */
 	name.sin_family = AF_INET;
 	name.sin_addr.s_addr = INADDR_ANY;
-	name.sin_port = htons(BOOTWHAT_DSTPORT);
+	name.sin_port = htons((u_short) port);
 	if (bind(sock, (struct sockaddr *) &name, sizeof(name))) {
-		syslog(LOG_ERR, "binding datagram socket: %m");
+		errorc("binding datagram socket");
 		exit(1);
 	}
 	/* Find assigned port value and print it out. */
 	length = sizeof(name);
 	if (getsockname(sock, (struct sockaddr *) &name, &length)) {
-		syslog(LOG_ERR, "getting socket name: %m");
+		errorc("getting socket name");
 		exit(1);
 	}
-	syslog(LOG_NOTICE, "listening on port %d", ntohs(name.sin_port));
+	info("listening on port %d\n", ntohs(name.sin_port));
 
 	signal(SIGHUP, onhup);
 	while (1) {
 		if ((mlen = recvfrom(sock, &boot_info, sizeof(boot_info),
 				     0, (struct sockaddr *)&client, &length))
 		    < 0) {
-			syslog(LOG_ERR, "receiving datagram packet: %m");
+			errorc("receiving datagram packet");
 			exit(1);
 		}
 
 		switch (boot_info.opcode) {
 		case BIOPCODE_BOOTWHAT_REQUEST:
-			syslog(LOG_INFO, "%s: REQUEST",
-			       inet_ntoa(client.sin_addr));
-			err = query_bootinfo_db(client.sin_addr, boot_whatp);
+		case BIOPCODE_BOOTWHAT_INFO:
+			info("%s: REQUEST (vers %d)\n",
+			     inet_ntoa(client.sin_addr), boot_info.version);
+#ifdef	EVENTSYS
+			bievent_send(client.sin_addr,
+				     TBDB_NODESTATE_PXEBOOTING);
+#endif
+			err = query_bootinfo_db(client.sin_addr,
+						boot_info.version,
+						boot_whatp);
 			break;
 
 		default:
-			syslog(LOG_INFO, "%s: invalid packet",
-			       inet_ntoa(client.sin_addr));
+			info("%s: invalid packet %d\n",
+			     inet_ntoa(client.sin_addr), boot_info.opcode);
 			continue;
 		}
 		
-		if (err) {
+		if (err)
 			boot_info.status = BISTAT_FAIL;
-		} else {
-			log_bootwhat(client.sin_addr, boot_whatp);
+		else {
 			boot_info.status = BISTAT_SUCCESS;
+			log_bootwhat(client.sin_addr, boot_whatp);
+#ifdef	EVENTSYS
+			switch (boot_whatp->type) {
+			case BIBOOTWHAT_TYPE_PART:
+			case BIBOOTWHAT_TYPE_SYSID:
+			case BIBOOTWHAT_TYPE_MB:
+			case BIBOOTWHAT_TYPE_MFS:
+				bievent_send(client.sin_addr,
+					     TBDB_NODESTATE_BOOTING);
+				break;
+				
+			case BIBOOTWHAT_TYPE_WAIT:
+				bievent_send(client.sin_addr,
+					     TBDB_NODESTATE_PXEWAIT);
+				break;
+			default:
+				error("%s: invalid boot directive: %d\n",
+				      inet_ntoa(client.sin_addr),
+				      boot_whatp->type);
+				break;
+			}
+#endif
 		}
 		boot_info.opcode = BIOPCODE_BOOTWHAT_REPLY;
 		
@@ -92,11 +178,14 @@ main()
 		client.sin_port = htons(BOOTWHAT_SRCPORT);
 		if (sendto(sock, (char *)&boot_info, sizeof(boot_info), 0,
 			(struct sockaddr *)&client, sizeof(client)) < 0)
-			syslog(LOG_ERR, "sendto: %m");
+			errorc("sendto");
 	}
 	close(sock);
 	close_bootinfo_db();
-	syslog(LOG_NOTICE, "daemon terminating");
+#ifdef  EVENTSYS
+	bievent_shutdown();
+#endif
+	info("daemon terminating\n");
 	exit(0);
 }
 
@@ -105,11 +194,11 @@ onhup(int sig)
 {
 	int err;
 
-	syslog(LOG_NOTICE, "re-initializing configuration database");
+	info("re-initializing configuration database\n");
 	close_bootinfo_db();
 	err = open_bootinfo_db();
 	if (err) {
-		syslog(LOG_ERR, "Could not open database");
+		error("Could not reopen database");
 		exit(1);
 	}
 }
@@ -122,18 +211,23 @@ log_bootwhat(struct in_addr ipaddr, boot_what_t *bootinfo)
 	strncpy(ipstr, inet_ntoa(ipaddr), sizeof ipstr);
 	switch (bootinfo->type) {
 	case BIBOOTWHAT_TYPE_PART:
-		syslog(LOG_INFO, "%s: REPLY: boot from partition %d",
-		       ipstr, bootinfo->what.partition);
+		info("%s: REPLY: boot from partition %d\n",
+		     ipstr, bootinfo->what.partition);
 		break;
 	case BIBOOTWHAT_TYPE_SYSID:
-		syslog(LOG_INFO, "%s: REPLY: boot from partition with sysid %d",
-		       ipstr, bootinfo->what.sysid);
+		info("%s: REPLY: boot from partition with sysid %d\n",
+		     ipstr, bootinfo->what.sysid);
 		break;
 	case BIBOOTWHAT_TYPE_MB:
-		syslog(LOG_INFO, "%s: REPLY: boot multiboot image %s:%s\n",
-		       ipstr,
-		       inet_ntoa(bootinfo->what.mb.tftp_ip),
+		info("%s: REPLY: boot multiboot image %s:%s\n",
+		       ipstr, inet_ntoa(bootinfo->what.mb.tftp_ip),
 		       bootinfo->what.mb.filename);
 		break;
+	case BIBOOTWHAT_TYPE_WAIT:
+		info("%s: REPLY: wait mode\n", ipstr);
+		break;
+	case BIBOOTWHAT_TYPE_MFS:
+		info("%s: REPLY: boot from mfs %s\n", ipstr, bootinfo->what.mfs);
+		break;
 	}
 }
diff --git a/pxe/bootinfo.h b/pxe/bootinfo.h
new file mode 100644
index 0000000000000000000000000000000000000000..12114eafb9dab6c6d916694508274f56f1d93c3d
--- /dev/null
+++ b/pxe/bootinfo.h
@@ -0,0 +1,20 @@
+/*
+ * EMULAB-COPYRIGHT
+ * Copyright (c) 2000-2003 University of Utah and the Flux Group.
+ * All rights reserved.
+ */
+struct boot_what;
+
+int		open_bootinfo_db(void);
+int		close_bootinfo_db(void);
+int		query_bootinfo_db(struct in_addr ipaddr, int version,
+				  struct boot_what *info);
+
+extern int debug;
+
+#ifdef EVENTSYS
+int		bievent_init(void);
+int		bievent_shutdown(void);
+int		bievent_send(struct in_addr ipaddr, char *event);
+#endif
+
diff --git a/pxe/bootinfo_cfile.c b/pxe/bootinfo_cfile.c
index c794ad359d2e7070d30198a79d888cafc97007b3..36a304cc10dbe9339c80e68c6a483f6b8e8f8585 100644
--- a/pxe/bootinfo_cfile.c
+++ b/pxe/bootinfo_cfile.c
@@ -1,6 +1,6 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2002 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2003 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
@@ -11,8 +11,9 @@
 #include <string.h>
 #include <netdb.h>
 #include <syslog.h>
-
+#include "log.h"
 #include "bootwhat.h"
+#include "bootinfo.h"
 
 /*
  * Trivial config file format.
@@ -53,7 +54,7 @@ open_bootinfo_db(void)
 }
 
 int
-query_bootinfo_db(struct in_addr ipaddr, boot_what_t *info)
+query_bootinfo_db(struct in_addr ipaddr, int version, boot_what_t *info)
 {
 	struct config *configp;
 
@@ -84,7 +85,7 @@ parse_host(char *name)
 	if (!isdigit(name[0])) {
 		he = gethostbyname(name);
 		if (he == 0) {
-			syslog(LOG_ERR, "%s: unknown host", name);
+			error("%s: unknown host", name);
 			return 0;
 		}
 		return *(int *)he->h_addr;
@@ -103,7 +104,7 @@ parse_configs(char *filename)
 	int	ipaddr;
 
 	if ((fp = fopen(filename, "r")) == NULL) {
-		syslog(LOG_ERR, "%s: cannot open", filename);
+		error("%s: cannot open", filename);
 		return 1;
 	}
 
@@ -115,7 +116,7 @@ parse_configs(char *filename)
 			continue;
 
 		if (numconfigs >= MAX_CONFIGS) {
-			syslog(LOG_ERR, "%s: too many lines", filename);
+			error("%s: too many lines", filename);
 			fclose(fp);
 			return 1;
 		}
@@ -125,7 +126,7 @@ parse_configs(char *filename)
 		configp = (struct config *) calloc(sizeof *configp, 1);
 		if (!configp) {
 		bad:
-			syslog(LOG_ERR, "%s: parse error", filename);
+			error("%s: parse error", filename);
 			fclose(fp);
 			close_bootinfo_db();
 			return 1;
@@ -136,6 +137,7 @@ parse_configs(char *filename)
 
 		configp->client.s_addr = ipaddr;
 		configp->bootinfolen = sizeof *configp;
+		configp->bootinfo.flags = 0;
 		if (strncmp(action, "sysid=", 6) == 0) {
 			configp->bootinfo.type = BIBOOTWHAT_TYPE_SYSID;
 			configp->bootinfo.what.sysid = atoi(&action[6]);
@@ -193,17 +195,6 @@ find_config(struct in_addr addr)
 #ifdef TEST
 #include <stdarg.h>
 
-void
-syslog(int prio, const char *msg, ...)
-{
-	va_list ap;
-
-	va_start(ap, msg);
-	vfprintf(stderr, msg, ap);
-	va_end(ap);
-	fprintf(stderr, "\n");
-}
-
 static void
 print_bootwhat(boot_what_t *bootinfo)
 {
diff --git a/pxe/bootinfo_mysql.c b/pxe/bootinfo_mysql.c
index 6ea75f121d7ac621806d6654d47e92d7250e95c3..4b95f0a38004075b86702add54b6171610d2ced9 100644
--- a/pxe/bootinfo_mysql.c
+++ b/pxe/bootinfo_mysql.c
@@ -1,6 +1,6 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2003 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2004 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
@@ -8,20 +8,31 @@
 #include <netinet/in.h>
 #include <netdb.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <syslog.h>
+#include <arpa/inet.h>
+#include <string.h>
 
+#include "log.h"
+#include "tbdb.h"
 #include "bootwhat.h"
+#include "bootinfo.h"
 #include <mysql/mysql.h>
 
+/* XXX Should be configured in */
+#define NEWNODEOSID	"NEWNODE-MFS"
+
 #ifdef USE_MYSQL_DB
 
-static char dbname[] = TBDBNAME;
-static MYSQL db;
 static int parse_multiboot_path(char *path, boot_what_t *info);
+static int boot_newnode_mfs(struct in_addr, int, boot_what_t *);
 
 int
 open_bootinfo_db(void)
 {
+	if (!dbinit())
+		return 1;
+
 	return 0;
 }
 
@@ -34,171 +45,183 @@ open_bootinfo_db(void)
 */
 
 int
-query_bootinfo_db(struct in_addr ipaddr, boot_what_t *info)
+query_bootinfo_db(struct in_addr ipaddr, int version, boot_what_t *info)
 {
-	char querybuf[1024];
-	int n, nrows, ncols, part;
-	MYSQL db;
-	MYSQL_RES *res;
-	MYSQL_ROW row;
-	char dbquery[] =
-		"select n.next_boot_path, n.next_boot_cmd_line, "
-		"  n.def_boot_osid, p.partition, n.def_boot_cmd_line, "
-		"  n.def_boot_path, o1.path, n.next_boot_osid, o2.path "
-		" from interfaces as i "
-		"left join nodes as n on i.node_id=n.node_id "
-		"left join partitions as p on "
-		" n.node_id=p.node_id and n.def_boot_osid=p.osid "
-		"left join os_info as o1 on o1.osid=n.def_boot_osid "
-		"left join os_info as o2 on o2.osid=n.next_boot_osid "
-	        "where i.IP = '%s'";
-
-#define NEXT_BOOT_PATH		0
-#define NEXT_BOOT_CMD_LINE	1
-#define DEF_BOOT_OSID		2
-#define PARTITION		3
-#define DEF_BOOT_CMD_LINE	4
-#define DEF_BOOT_PATH		5
-#define DEF_BOOT_OSID_PATH	6
-#define NEXT_BOOT_OSID		7
-#define NEXT_BOOT_OSID_PATH	8
-
-	n = snprintf(querybuf, sizeof querybuf, dbquery, inet_ntoa(ipaddr));
-	if (n > sizeof querybuf) {
-		syslog(LOG_ERR, "query too long for buffer");
-		return 1;
-	}
-	
-	mysql_init(&db);
-	if (mysql_real_connect(&db, 0, "bootinfo", 0, dbname, 0, 0, 0) == 0) {
-		syslog(LOG_ERR, "%s: connect failed: %s",
-			dbname, mysql_error(&db));
-		return 1;
-	}
-
-	/* Debug message into log:
-	syslog(LOG_ERR, "USING QUERY: %s", querybuf);
-	*/
-	
-	if (mysql_real_query(&db, querybuf, n) != 0) {
-		syslog(LOG_ERR, "%s: query failed: %s",
-			dbname, mysql_error(&db));
-		mysql_close(&db);
-		return 1;
-	}
-
-	res = mysql_store_result(&db);
-	if (res == 0) {
-		syslog(LOG_ERR, "%s: store_result failed: %s",
-			dbname, mysql_error(&db));
-		mysql_close(&db);
-		return 1;
-
+	int		nrows, rval = 0;
+	MYSQL_RES	*res;
+	MYSQL_ROW	row;
+	char		ipstr[32];
+
+	info->cmdline[0] = 0;	/* Must zero first byte! */
+	info->flags      = 0;
+	strcpy(ipstr, inet_ntoa(ipaddr));
+
+#define DEF_BOOT_OSID		0
+#define DEF_BOOT_CMDLINE	1
+#define DEF_BOOT_PATH		2
+#define DEF_BOOT_MFS		3
+#define DEF_BOOT_PARTITION	4
+#define TEMP_BOOT_OSID		5
+#define TEMP_BOOT_PATH		6
+#define TEMP_BOOT_MFS		7
+#define TEMP_BOOT_PARTITION	8
+#define NEXT_BOOT_OSID		9
+#define NEXT_BOOT_CMDLINE	10
+#define NEXT_BOOT_PATH		11
+#define NEXT_BOOT_MFS		12
+#define NEXT_BOOT_PARTITION	13
+#define PID			14
+#define DEFINED(x)		(row[(x)] != NULL && row[(x)][0] != '\0')
+#define TOINT(x)		(atoi(row[(x)]))
+
+	res = mydb_query("select n.def_boot_osid, n.def_boot_cmd_line, "
+			 "        odef.path, odef.mfs, pdef.partition, "
+			 "       n.temp_boot_osid, "
+			 "        otemp.path, otemp.mfs, ptemp.partition, "
+			 "       n.next_boot_osid, n.next_boot_cmd_line, "
+			 "        onext.path, onext.mfs, pnext.partition, "
+			 "       r.pid "
+			 " from interfaces as i "
+			 "left join nodes as n on i.node_id=n.node_id "
+			 "left join reserved as r on i.node_id=r.node_id "
+			 "left join partitions as pdef on "
+			 "     n.node_id=pdef.node_id and "
+			 "     n.def_boot_osid=pdef.osid "
+			 "left join os_info as odef on "
+			 "     odef.osid=n.def_boot_osid "
+			 "left join partitions as ptemp on "
+			 "     n.node_id=ptemp.node_id and "
+			 "     n.temp_boot_osid=ptemp.osid "
+			 "left join os_info as otemp on "
+			 "     otemp.osid=n.temp_boot_osid "
+			 "left join partitions as pnext on "
+			 "     n.node_id=pnext.node_id and "
+			 "     n.next_boot_osid=pnext.osid "
+			 "left join os_info as onext on "
+			 "     onext.osid=n.next_boot_osid "
+			 "where i.IP='%s'", 15, inet_ntoa(ipaddr));
+
+	if (!res) {
+		error("Query failed for host %s\n", ipstr);
+		/* XXX Wrong. Should fail so client can request again later */
+		return 0;
 	}
-	mysql_close(&db);
+	nrows = mysql_num_rows(res);
 
-	nrows = (int)mysql_num_rows(res);
 	switch (nrows) {
 	case 0:
-		syslog(LOG_ERR, "%s: no entry for host %s",
-			dbname, inet_ntoa(ipaddr));
 		mysql_free_result(res);
-		return 1;
+		return boot_newnode_mfs(ipaddr, version, info);
 	case 1:
 		break;
 	default:
-		syslog(LOG_ERR, "%s: %d entries for IP %s, using first",
-			dbname, nrows, inet_ntoa(ipaddr));
+		error("%d entries for host %s\n", nrows, ipstr);
 		break;
 	}
-
-	ncols = (int)mysql_num_fields(res);
-	switch (ncols) {
-	case 9: /* Should have 9 fields */
-		break;
-	default:
-		syslog(LOG_ERR, "%s: %d fields in query for IP %s!",
-			dbname, ncols, inet_ntoa(ipaddr));
-		mysql_free_result(res);
-		return 1;
-	}
-
 	row = mysql_fetch_row(res);
 
 	/*
-	 * Check next_boot_path.  If set, assume it is a multiboot kernel.
+	 * Version >=1 supports wait if not allocated. Client will recontact
+	 * us later.
 	 */
-	if ((row[NEXT_BOOT_PATH] != 0 &&
-	     row[NEXT_BOOT_PATH][0] != '\0') ||
-	    (row[NEXT_BOOT_OSID_PATH] != 0 &&
-	     row[NEXT_BOOT_OSID_PATH][0] != '\0')) {
-		info->type = BIBOOTWHAT_TYPE_MB;
-		
-		if (row[NEXT_BOOT_PATH] != 0 &&
-		    row[NEXT_BOOT_PATH][0] != '\0')
-		    parse_multiboot_path(row[NEXT_BOOT_PATH], info);
-		else
-		    parse_multiboot_path(row[NEXT_BOOT_OSID_PATH], info);
-
-		if (row[NEXT_BOOT_CMD_LINE] != 0 &&
-		    row[NEXT_BOOT_CMD_LINE][0] != '\0')
-			strncpy(info->cmdline, row[NEXT_BOOT_CMD_LINE],
-				MAX_BOOT_CMDLINE-1);
-		else
-			info->cmdline[0] = 0;	/* Must zero first byte! */
-		mysql_free_result(res);
-		return 0;
+	if (version >= 1 && row[PID] == (char *) NULL) {
+		info->type = BIBOOTWHAT_TYPE_WAIT;
+		goto done;
 	}
 
 	/*
-	 * There is either a partition number or default boot path.
-	 * The default boot path overrides the partition.
+	 * Check next_boot_osid. It overrides the others. It should be
+	 * the case that partition and path/mfs are mutually exclusive.
+	 * mfs might be set when path is set.  
 	 */
-	if (row[DEF_BOOT_PATH] != 0 && row[DEF_BOOT_PATH][0] != '\0') {
-		info->type = BIBOOTWHAT_TYPE_MB;
-		parse_multiboot_path(row[DEF_BOOT_PATH], info);
+	if (DEFINED(NEXT_BOOT_OSID)) {
+		if (DEFINED(NEXT_BOOT_PATH)) {
+			if (DEFINED(NEXT_BOOT_MFS) && TOINT(NEXT_BOOT_MFS) == 1){
+				info->type = BIBOOTWHAT_TYPE_MFS;
+				strcpy(info->what.mfs, row[NEXT_BOOT_PATH]);
+			}
+			else {
+				info->type = BIBOOTWHAT_TYPE_MB;
+				parse_multiboot_path(row[NEXT_BOOT_PATH], info);
+			}
+		}
+		else if (DEFINED(NEXT_BOOT_PARTITION)) {
+			info->type = BIBOOTWHAT_TYPE_PART;
+			info->what.partition = TOINT(NEXT_BOOT_PARTITION);
+		}
+		else {
+			error("Invalid NEXT_BOOT entry for host %s\n", ipstr);
+			rval = 1;
+		}
+		if (DEFINED(NEXT_BOOT_CMDLINE)) {
+			strncpy(info->cmdline,
+				row[NEXT_BOOT_CMDLINE], MAX_BOOT_CMDLINE-1);
+		}
+		goto done;
 	}
-	else if (row[DEF_BOOT_OSID_PATH] != 0 &&
-		 row[DEF_BOOT_OSID_PATH][0] != '\0') {
-		info->type = BIBOOTWHAT_TYPE_MB;
-		parse_multiboot_path(row[DEF_BOOT_OSID_PATH], info);
-	}
-	else if (row[PARTITION] != 0 && row[PARTITION][0] != '\0') {
-		info->type = BIBOOTWHAT_TYPE_PART;
-		info->what.partition = atoi(row[PARTITION]);
-	}
-	else {
-		syslog(LOG_ERR, "%s: null query result for IP %s!",
-			dbname, inet_ntoa(ipaddr));
-		mysql_free_result(res);
-		return 1;
+
+	/*
+	 * Check temp_boot_osid. It overrides def_boot but not next_boot.
+	 */
+	if (DEFINED(TEMP_BOOT_OSID)) {
+		if (DEFINED(TEMP_BOOT_PATH)) {
+			if (DEFINED(TEMP_BOOT_MFS) && TOINT(TEMP_BOOT_MFS) == 1){
+				info->type = BIBOOTWHAT_TYPE_MFS;
+				strcpy(info->what.mfs, row[TEMP_BOOT_PATH]);
+			}
+			else {
+				info->type = BIBOOTWHAT_TYPE_MB;
+				parse_multiboot_path(row[TEMP_BOOT_PATH], info);
+			}
+		}
+		else if (DEFINED(TEMP_BOOT_PARTITION)) {
+			info->type = BIBOOTWHAT_TYPE_PART;
+			info->what.partition = TOINT(TEMP_BOOT_PARTITION);
+		}
+		else {
+			error("Invalid TEMP_BOOT entry for host %s\n", ipstr);
+			rval = 1;
+		}
+		goto done;
 	}
-	if (row[DEF_BOOT_CMD_LINE] != 0 && row[DEF_BOOT_CMD_LINE][0] != '\0')
-		strncpy(info->cmdline, row[DEF_BOOT_CMD_LINE],
-			MAX_BOOT_CMDLINE-1);
-	else
-		info->cmdline[0] = 0;	/* Must zero first byte! */
-	
+
 	/*
-	 * Just 45040 lines of code later, we are done. 
+	 * Lastly, def_boot.
 	 */
+	if (DEFINED(DEF_BOOT_OSID)) {
+		if (DEFINED(DEF_BOOT_PATH)) {
+			if (DEFINED(DEF_BOOT_MFS) && TOINT(DEF_BOOT_MFS) == 1) {
+				info->type = BIBOOTWHAT_TYPE_MFS;
+				strcpy(info->what.mfs, row[DEF_BOOT_PATH]);
+			}
+			else {
+				info->type = BIBOOTWHAT_TYPE_MB;
+				parse_multiboot_path(row[DEF_BOOT_PATH], info);
+			}
+		}
+		else if (DEFINED(DEF_BOOT_PARTITION)) {
+			info->type = BIBOOTWHAT_TYPE_PART;
+			info->what.partition = TOINT(DEF_BOOT_PARTITION);
+		}
+		else {
+			error("Invalid DEF_BOOT entry for host %s\n", ipstr);
+			rval = 1;
+		}
+		if (DEFINED(DEF_BOOT_CMDLINE)) {
+			strncpy(info->cmdline,
+				row[DEF_BOOT_CMDLINE], MAX_BOOT_CMDLINE-1);
+		}
+		goto done;
+	}
+ done:
 	mysql_free_result(res);
-	return 0;
-
-#undef NEXT_BOOT_PATH
-#undef NEXT_BOOT_CMD_LINE
-#undef DEF_BOOT_OSID
-#undef PARTITION
-#undef DEF_BOOT_CMD_LINE
-#undef DEF_BOOT_PATH
-#undef DEF_BOOT_OSID_PATH
-#undef NEXT_BOOT_OSID
-#undef NEXT_BOOT_OSID_PATH
+	return rval;
 }
 
 int
 close_bootinfo_db(void)
 {
+	dbclose();
 	return 0;
 }
 
@@ -232,20 +255,59 @@ parse_multiboot_path(char *path, boot_what_t *info)
 	return 0;
 }
 
-#ifdef TEST
-#include <stdarg.h>
-
-void
-syslog(int prio, const char *msg, ...)
+/*
+ * Arrange to boot the special newnode kernel.
+ */
+static int
+boot_newnode_mfs(struct in_addr ipaddr, int version, boot_what_t *info)
 {
-	va_list ap;
+	int		nrows;
+	MYSQL_RES	*res;
+	MYSQL_ROW	row;
+
+	error("%s: nonexistent IP, booting '%s'\n",
+	      inet_ntoa(ipaddr), NEWNODEOSID);
 
-	va_start(ap, msg);
-	vfprintf(stderr, msg, ap);
-	va_end(ap);
-	fprintf(stderr, "\n");
+#define MFS_PATH	0
+
+	res = mydb_query("select path from os_info "
+			 "where osid='%s' and mfs=1", 1, NEWNODEOSID);
+
+	if (!res) {
+		error("Query failed\n");
+		/* XXX Wrong. Should fail so client can request again later */
+		return 0;
+	}
+	nrows = mysql_num_rows(res);
+
+	switch (nrows) {
+	case 0:
+		error("No DB entry for OSID %s\n", NEWNODEOSID);
+		mysql_free_result(res);
+		return 1;
+	case 1:
+		break;
+	default:
+		error("Too many DB entries for OSID %s\n", NEWNODEOSID);
+		return 1;
+	}
+	row = mysql_fetch_row(res);
+
+	if (row[MFS_PATH] != 0 && row[MFS_PATH][0] != '\0') {
+		info->type = BIBOOTWHAT_TYPE_MFS;
+		strcpy(info->what.mfs, row[MFS_PATH]);
+		mysql_free_result(res);
+		return 0;
+	}
+	error("No path info for OSID %s\n", NEWNODEOSID);
+	return 1;
+#undef  MFS_PATH
 }
 
+
+#ifdef TEST
+#include <stdarg.h>
+
 static void
 print_bootwhat(boot_what_t *bootinfo)
 {
@@ -263,12 +325,19 @@ print_bootwhat(boot_what_t *bootinfo)
 		       inet_ntoa(bootinfo->what.mb.tftp_ip),
 		       bootinfo->what.mb.filename);
 		break;
+	case BIBOOTWHAT_TYPE_WAIT:
+		printf("No boot; waiting till allocated\n");
+		break;
+	case BIBOOTWHAT_TYPE_MFS:
+		printf("boot from MFS %s\n", bootinfo->what.mfs);
+		break;
 	}
 	if (bootinfo->cmdline[0])
 		printf("Command line %s\n", bootinfo->cmdline);
 		
 }
 
+int
 main(int argc, char **argv)
 {
 	struct in_addr ipaddr;
@@ -278,7 +347,7 @@ main(int argc, char **argv)
 	open_bootinfo_db();
 	while (--argc > 0) {
 		if (inet_aton(*++argv, &ipaddr))
-			if (query_bootinfo_db(ipaddr, boot_whatp) == 0) {
+			if (query_bootinfo_db(ipaddr, 1, boot_whatp) == 0) {
 				printf("%s: ", *argv);
 				print_bootwhat(boot_whatp);
 			} else
@@ -290,5 +359,4 @@ main(int argc, char **argv)
 	exit(0);
 }
 #endif
-
 #endif
diff --git a/pxe/bootinfo_null.c b/pxe/bootinfo_null.c
index 41fe003aab82da310444e114e9bfc01a73733e7d..bc4d68beee7cac5a47332d001270fff8e10c089f 100644
--- a/pxe/bootinfo_null.c
+++ b/pxe/bootinfo_null.c
@@ -1,6 +1,6 @@
 /*
  * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2002 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2003 University of Utah and the Flux Group.
  * All rights reserved.
  */
 
@@ -9,6 +9,7 @@
 #include <stdio.h>
 
 #include "bootwhat.h"
+#include "bootinfo.h"
 
 #ifdef USE_NULL_DB
 
@@ -24,14 +25,16 @@ open_bootinfo_db(void)
 }
 
 int
-query_bootinfo_db(struct in_addr ipaddr, boot_what_t *info)
+query_bootinfo_db(struct in_addr ipaddr, int version, boot_what_t *info)
 {
 #if 0
-	info->type = BIBOOTWHAT_TYPE_MB;
+	info->type  = BIBOOTWHAT_TYPE_MB;
+	info->flags = 0;
 	info->what.mb.tftp_ip.s_addr = 0;
 	strcpy(info->what.mb.filename, NETBOOT);
 #else
-	info->type = BIBOOTWHAT_TYPE_SYSID;
+	info->type  = BIBOOTWHAT_TYPE_SYSID;
+	info->flags = 0;
 	info->what.sysid = 165; /* BSD */
 #endif
 	return 0;
diff --git a/pxe/bootinfosend.c b/pxe/bootinfosend.c
new file mode 100644
index 0000000000000000000000000000000000000000..7053d920034610e0e98e45d03e279fae8e372b36
--- /dev/null
+++ b/pxe/bootinfosend.c
@@ -0,0 +1,220 @@
+/*
+ * EMULAB-COPYRIGHT
+ * Copyright (c) 2000-2004 University of Utah and the Flux Group.
+ * All rights reserved.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <netdb.h>
+#include "log.h"
+#include "tbdefs.h"
+#include "bootwhat.h"
+#include "bootinfo.h"
+
+static void	log_bootwhat(struct in_addr ipaddr, boot_what_t *bootinfo);
+int		debug = 0;
+static char	*progname;
+
+void
+usage()
+{
+	fprintf(stderr,
+		"Usage: %s <options> [-d] [-r | -q] target\n"
+		"options:\n"
+		"-d         - Turn on debugging\n"
+		"-r         - Tell node to query bootinfo again\n"
+		"-q         - Tell node to reboot\n",
+		progname);
+	exit(-1);
+}
+
+int
+main(int argc, char **argv)
+{
+	int			sock, err, c, reboot = 0, query = 0;
+	struct sockaddr_in	name, target;
+	boot_info_t		boot_info;
+	boot_what_t	       *boot_whatp = (boot_what_t *) &boot_info.data;
+	extern char		build_info[];
+	struct hostent	       *he;
+
+	progname = argv[0];
+
+	while ((c = getopt(argc, argv, "dhvrq")) != -1) {
+		switch (c) {
+		case 'd':
+			debug++;
+			break;
+		case 'r':
+			reboot++;
+			break;
+		case 'q':
+			query++;
+			break;
+		case 'v':
+		    	fprintf(stderr, "%s\n", build_info);
+			exit(0);
+			break;
+		case 'h':
+		case '?':
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (!argc)
+		usage();
+	if (query && reboot)
+		usage();
+
+	if (debug) 
+		loginit(0, 0);
+	else
+		loginit(1, "bootinfo");
+	info("%s\n", build_info);
+
+	/* Make sure we can map target */
+	if ((he = gethostbyname(argv[0])) == NULL) {
+		errorc("gethostbyname(%s)", argv[0]);
+		exit(1);
+	}
+
+	bzero(&target, sizeof(target));
+	memcpy((char *)&target.sin_addr, he->h_addr, he->h_length);
+	target.sin_family = AF_INET;
+	target.sin_port   = htons((u_short) BOOTWHAT_SRCPORT);
+
+	err = open_bootinfo_db();
+	if (err) {
+		error("could not open database");
+		exit(1);
+	}
+#ifdef EVENTSYS
+	err = bievent_init();
+	if (err) {
+		error("could not initialize event system");
+		exit(1);
+	}
+#endif
+	/* Create socket */
+	sock = socket(AF_INET, SOCK_DGRAM, 0);
+	if (sock < 0) {
+		errorc("opening datagram socket");
+		exit(1);
+	}
+	err = 1;
+	if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
+		       (char *)&err, sizeof(err)) < 0)
+		errorc("setsockopt(SO_REUSEADDR)");
+	
+	/* Create name. */
+	name.sin_family = AF_INET;
+	name.sin_addr.s_addr = INADDR_ANY;
+	name.sin_port = htons((u_short) BOOTWHAT_SENDPORT);
+	if (bind(sock, (struct sockaddr *) &name, sizeof(name))) {
+		errorc("binding datagram socket");
+		exit(1);
+	}
+
+	bzero(&boot_info, sizeof(boot_info));
+	boot_info.version = BIVERSION_CURRENT;
+	if (reboot) {
+		boot_whatp->type = BIBOOTWHAT_TYPE_REBOOT;
+#ifdef	EVENTSYS
+		bievent_send(target.sin_addr, TBDB_NODESTATE_SHUTDOWN);
+#endif
+	}
+	else if (query) {
+		boot_whatp->type = BIBOOTWHAT_TYPE_AUTO;
+#ifdef	EVENTSYS
+		bievent_send(target.sin_addr, TBDB_NODESTATE_PXEWAKEUP);
+#endif
+	}
+	else {
+		err = query_bootinfo_db(target.sin_addr,
+					boot_info.version,
+					boot_whatp);
+		if (err) {
+			fatal("Could not send bootinfo packet!");
+		}
+#ifdef	EVENTSYS
+		bievent_send(target.sin_addr, TBDB_NODESTATE_PXEBOOTING);
+		switch (boot_whatp->type) {
+		case BIBOOTWHAT_TYPE_PART:
+		case BIBOOTWHAT_TYPE_SYSID:
+		case BIBOOTWHAT_TYPE_MB:
+		case BIBOOTWHAT_TYPE_MFS:
+			bievent_send(target.sin_addr, TBDB_NODESTATE_BOOTING);
+			break;
+				
+		case BIBOOTWHAT_TYPE_WAIT:
+			bievent_send(target.sin_addr, TBDB_NODESTATE_PXEWAIT);
+			break;
+		default:
+			error("%s: invalid boot directive: %d\n",
+			      inet_ntoa(target.sin_addr), boot_whatp->type);
+			break;
+		}
+#endif
+	}
+
+	log_bootwhat(target.sin_addr, boot_whatp);
+	boot_info.status  = BISTAT_SUCCESS;
+	boot_info.opcode  = BIOPCODE_BOOTWHAT_ORDER;
+	
+	if (sendto(sock, (char *)&boot_info, sizeof(boot_info), 0,
+		   (struct sockaddr *)&target, sizeof(target)) < 0)
+		errorc("sendto");
+
+	close(sock);
+	close_bootinfo_db();
+#ifdef  EVENTSYS
+	bievent_shutdown();
+#endif
+	exit(0);
+}
+
+static void
+log_bootwhat(struct in_addr ipaddr, boot_what_t *bootinfo)
+{
+	char ipstr[32];
+
+	strncpy(ipstr, inet_ntoa(ipaddr), sizeof ipstr);
+	switch (bootinfo->type) {
+	case BIBOOTWHAT_TYPE_PART:
+		info("%s: SEND: boot from partition %d\n",
+		     ipstr, bootinfo->what.partition);
+		break;
+	case BIBOOTWHAT_TYPE_SYSID:
+		info("%s: SEND: boot from partition with sysid %d\n",
+		     ipstr, bootinfo->what.sysid);
+		break;
+	case BIBOOTWHAT_TYPE_MB:
+		info("%s: SEND: boot multiboot image %s:%s\n",
+		       ipstr, inet_ntoa(bootinfo->what.mb.tftp_ip),
+		       bootinfo->what.mb.filename);
+		break;
+	case BIBOOTWHAT_TYPE_WAIT:
+		info("%s: SEND: wait mode\n", ipstr);
+		break;
+	case BIBOOTWHAT_TYPE_REBOOT:
+		info("%s: SEND: reboot\n", ipstr);
+		break;
+	case BIBOOTWHAT_TYPE_AUTO:
+		info("%s: SEND: query bootinfo\n", ipstr);
+		break;
+	case BIBOOTWHAT_TYPE_MFS:
+		info("%s: SEND: boot from mfs %s\n", ipstr, bootinfo->what.mfs);
+		break;
+	}
+}
diff --git a/pxe/bootwhat.h b/pxe/bootwhat.h
index 5746b268216e4a2198f089bd495bd3beda90738f..f9d5c095a929f31db3e33042f1890f105c40c5f6 100644
--- a/pxe/bootwhat.h
+++ b/pxe/bootwhat.h
@@ -1,25 +1,33 @@
 /*
- * EMULAB-COPYRIGHT
- * Copyright (c) 2000-2002 University of Utah and the Flux Group.
+ * Copyright (c) 2000, 2001, 2003, 2004 University of Utah and the Flux Group.
  * All rights reserved.
+ *
+ * boot/bootwhat.h from the OSKit.
  */
 
 #ifndef _OSKIT_BOOT_BOOTWHAT_H_
 #define _OSKIT_BOOT_BOOTWHAT_H_
 
-#define BOOTWHAT_DSTPORT		6969
+#define BOOTWHAT_DSTPORT		6968
 #define BOOTWHAT_SRCPORT		9696
+#define BOOTWHAT_SENDPORT		6970
 
 /*
- * This is the structure we pass back and forth between a oskit kernel
+ * This is the structure we pass back and forth between pxeboot on a node
  * and a server running on some other machine, that tells what to do.
+ *
+ * The structure below was changed, adding the version slot by splitting
+ * the opcode from an int into a short. Old clients conveniently look like a
+ * version zero client. The same was done for the "type" field, splitting 
+ * that into "flags" and "type" shorts.
  */
 #define  MAX_BOOT_DATA		512
 #define  MAX_BOOT_PATH		256
 #define  MAX_BOOT_CMDLINE	((MAX_BOOT_DATA - MAX_BOOT_PATH) - 32)
 
 typedef struct {
-	int	opcode;
+	short   version;
+	short	opcode;
 	int	status;
 	char	data[MAX_BOOT_DATA];
 } boot_info_t;
@@ -27,10 +35,21 @@ typedef struct {
 /* Opcode */
 #define BIOPCODE_BOOTWHAT_REQUEST	1	/* What to boot request */
 #define BIOPCODE_BOOTWHAT_REPLY		2	/* What to boot reply */
+#define BIOPCODE_BOOTWHAT_ACK		3	/* Ack to Reply */
+#define BIOPCODE_BOOTWHAT_ORDER		4	/* Unsolicited command */
+#define BIOPCODE_BOOTWHAT_INFO		5	/* Request for bootinfo */
+
+/* Version */
+#define BIVERSION_CURRENT		1	/* Old version is zero */
+
+/* Status */
+#define BISTAT_SUCCESS			0
+#define BISTAT_FAIL			1
 
 /* BOOTWHAT Reply */
-typedef struct {
-	int	type;
+typedef struct boot_what {
+	short	flags;
+	short	type;
 	union {
 		/*
 		 * Type is BIBOOTWHAT_TYPE_PART
@@ -55,7 +74,18 @@ typedef struct {
 			struct in_addr	tftp_ip;
 			char		filename[MAX_BOOT_PATH];
 		} mb;
+
+		/*
+		 * Type is BIBOOTWHAT_TYPE_MFS
+		 *
+		 * Specifies network path to MFS (boss:/tftpboot/frisbee)
+		 * With no host spec, defaults to bootinfo server IP.
+		 */
+		char			mfs[MAX_BOOT_PATH];
 	} what;
+	/*
+	 * Kernel and command line to pass to boot loader or multiboot kernel.
+	 */
 	char	cmdline[1];
 } boot_what_t;
 
@@ -63,9 +93,12 @@ typedef struct {
 #define BIBOOTWHAT_TYPE_PART	1	/* Boot a partition number */
 #define BIBOOTWHAT_TYPE_SYSID	2	/* Boot a system ID */
 #define BIBOOTWHAT_TYPE_MB	3	/* Boot a multiboot image */
+#define BIBOOTWHAT_TYPE_WAIT    4	/* Wait, no boot until later */
+#define BIBOOTWHAT_TYPE_REBOOT	5	/* Reboot */
+#define BIBOOTWHAT_TYPE_AUTO	6	/* Do a bootinfo query */
+#define BIBOOTWHAT_TYPE_MFS	7	/* Boot an MFS from server:/path */
 
-/* Status */
-#define BISTAT_SUCCESS		0
-#define BISTAT_FAIL		1
+/* Flags */
+#define BIBOOTWHAT FLAGS_CMDLINE	0x01	/* Kernel to boot */ 
 
 #endif /* _OSKIT_BOOT_BOOTWHAT_H_ */
diff --git a/pxe/event-support.c b/pxe/event-support.c
new file mode 100644
index 0000000000000000000000000000000000000000..0026ac620a28328d5a37e33fd63183348530808e
--- /dev/null
+++ b/pxe/event-support.c
@@ -0,0 +1,121 @@
+/*
+ * EMULAB-COPYRIGHT
+ * Copyright (c) 2000-2003 University of Utah and the Flux Group.
+ * All rights reserved.
+ */
+#ifdef EVENTSYS
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <netdb.h>
+#include <string.h>
+#include <syslog.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include "config.h"
+#include "log.h" 
+#include "tbdb.h"
+#include "event.h"
+#include "bootinfo.h"
+
+static event_handle_t	event_handle = NULL;
+static address_tuple_t  tuple = NULL;
+
+/*
+ * Connect to the event system. It's not an error to call this function if
+ * already connected. Returns 1 on failure, 0 on sucess.
+ */
+int
+bievent_init(void)
+{
+	if (!event_handle) {
+		event_handle = event_register("elvin://" BOSSNODE, 0);
+
+		if (!event_handle) {
+			error("Unable to register with event system!\n");
+			return 1;
+		}
+		tuple = address_tuple_alloc();
+		if (! tuple) {
+			error("Unable to allocate event tuple!\n");
+			event_unregister(event_handle);
+			event_handle = NULL;
+			return 1;
+		}
+	}
+	return 0;
+}
+
+int
+bievent_shutdown(void)
+{
+	if (event_handle) {
+		if (tuple) {
+			address_tuple_free(tuple);
+			tuple = NULL;
+		}
+		if (!event_unregister(event_handle)) {
+			error("Unable to unregister from event system!\n");
+			event_handle = NULL;
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/*
+ * Send an event to the event system. Automatically connects (registers)
+ * if not already done. Returns 0 on sucess, 1 on failure.
+ */
+int
+bievent_send(struct in_addr ipaddr, char *event)
+{
+	event_notification_t	notification;
+	char			nodeid[TBDB_FLEN_NODEID];
+	char			ip[TBDB_FLEN_IP], *bp;
+
+	if (bievent_init())
+		return 1;
+
+	/* Convert IP to nodeid */
+	bp = inet_ntoa(ipaddr);
+	strcpy(ip, bp);
+	if (! mydb_iptonodeid(ip, nodeid)) {
+		error("Unable to convert IP to nodeid for %s!\n", ip);
+		return 1;
+	}
+	   
+	tuple->host      = BOSSNODE;
+	tuple->objtype   = "TBNODESTATE";
+	tuple->objname   = nodeid;
+	tuple->eventtype = event;
+
+	notification = event_notification_alloc(event_handle,tuple);
+	if (notification == NULL) {
+		error("Unable to allocate notification!\n");
+		return 1;
+	}
+
+	if (debug >= 2)
+	    info("Sending event %s for node %s\n", event, nodeid);
+
+	if (event_notify(event_handle, notification) == NULL) {
+		error("Unable to send notification!");
+		event_notification_free(event_handle, notification);
+
+		/*
+		 * Let's try to disconnect from the event system, so that
+		 * we'll reconnect next time around.
+		 */
+		bievent_shutdown();
+		return 1;
+	}
+
+	event_notification_free(event_handle,notification);
+	return 0;
+	
+}
+#endif
diff --git a/sql/database-fill-supplemental.sql b/sql/database-fill-supplemental.sql
index e8b84f9b3fc7480da2358fa9fe0c38e701f356f2..7118050143851c5c5ae495c89b832e8f0e623903 100644
--- a/sql/database-fill-supplemental.sql
+++ b/sql/database-fill-supplemental.sql
@@ -5,6 +5,6 @@
 -- these is not idempotent, since a site may have changed them for some reason.
 --
 
-INSERT INTO os_info VALUES ('PXEBOOT','emulab-ops','PXEBOOT','root',NULL,'Default pxeboot kernel for contacting bootinfo.','OSKit','','boss:/tftpboot/pxeboot',NULL,'','',0,1,0,'_BOOTWHAT_',NULL,NULL);
-INSERT INTO os_info VALUES ('PXEFBSD','emulab-ops','PXEFBSD','root',NULL,'MFS FreeBSD over PXE','FreeBSD','4.5','boss:/tftpboot/pxeboot.freebsd',NULL,'','ping,ssh,ipod,isup',0,1,0,'NORMAL',NULL,NULL);
-INSERT INTO os_info VALUES ('PXEFRISBEE','emulab-ops','PXEFRISBEE','root',NULL,'Frisbee MFS FreeBSD over PXE','FreeBSD','4.5','boss:/tftpboot/pxeboot.frisbee',NULL,'','ping,ssh,ipod,isup',0,1,0,'RELOAD',NULL,NULL);
+INSERT INTO os_info VALUES ('FREEBSD-MFS','emulab-ops','FREEBSD-MFS','root',NULL,'FreeBSD in an MFS','FreeBSD','4.5','boss:/tftpboot/freebsd',NULL,'','ping,ssh,ipod,isup',0,1,0,'PXEFBSD',NULL,NULL,1);
+INSERT INTO os_info VALUES ('FRISBEE-MFS','emulab-ops','FRISBEE-MFS','root',NULL,'Frisbee (FreeBSD) in an MFS','FreeBSD','4.5','boss:/tftpboot/frisbee',NULL,'','ping,ssh,ipod,isup',0,1,0,'RELOAD',NULL,NULL,1);
+INSERT INTO os_info VALUES ('NEWNODE-MFS','emulab-ops','NEWNODE-MFS','root',NULL,'NewNode (FreeBSD) in an MFS','FreeBSD','4.5','boss:/tftpboot/freebsd.newnode',NULL,'','ping,ssh,ipod,isup',0,1,0,'PXEFBSD',NULL,NULL,1);
diff --git a/tbsetup/node_control.in b/tbsetup/node_control.in
index 398f2e3ed7546da9f780eef382724e0c9938f2bf..1d0146bf270fe72e3a0cc66eb605fd395d7be06a 100644
--- a/tbsetup/node_control.in
+++ b/tbsetup/node_control.in
@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# Copyright (c) 2000-2002, 2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -38,19 +38,16 @@ my %controlset =
  #
  # Symbolic name => Admin, Multi args, nodes field, virt_nodes field, osselect
  #
- default_boot_osid      => [0, 0, "def_boot_osid",      undef,1],
- default_boot_path      => [0, 0, "def_boot_path",      undef,1],
- default_boot_cmdline	=> [0, 0, "def_boot_cmd_line",  "cmd_line",0],
- startup_command	=> [0, 0, "startupcmd",		"startupcmd",0],
- tarfiles		=> [0, 1, "tarballs",		"tarfiles",0],
- rpms			=> [0, 1, "rpms",		"rpms",0],
- deltas			=> [0, 1, "deltas",		"deltas",0],
- next_boot_osid         => [1, 0, "next_boot_osid",     undef,1],
- next_boot_path         => [1, 0, "next_boot_path",     undef,1],
- next_boot_cmdline	=> [1, 0, "next_boot_cmd_line",	undef,0],
- pxe_boot_path          => [0, 0, "pxe_boot_path",    undef,1],
- next_pxe_boot_path     => [0, 0, "next_pxe_boot_path",       undef,1],
- bios_version		=> [1, 0, "bios_version",	undef,0],
+ default_boot_osid      => [0, 0, "def_boot_osid",      undef,       1, ""],
+ default_boot_cmdline	=> [0, 0, "def_boot_cmd_line",  "cmd_line",  0, ""],
+ startup_command	=> [0, 0, "startupcmd",		"startupcmd",0, ""],
+ tarfiles		=> [0, 1, "tarballs",		"tarfiles",  0, ""],
+ rpms			=> [0, 1, "rpms",		"rpms",      0, ""],
+ deltas			=> [0, 1, "deltas",		"deltas",    0, ""],
+ next_boot_osid         => [1, 0, "next_boot_osid",     undef,       1, "-1"],
+ next_boot_cmdline	=> [1, 0, "next_boot_cmd_line",	undef,       0, ""],
+ temp_boot_osid         => [1, 0, "temp_boot_osid",     undef,       1, "-t"],
+ bios_version		=> [1, 0, "bios_version",	undef,       0, ""],
 );
   
 #
@@ -212,23 +209,27 @@ my $virtnodes_updatestr;
 my @osselect_params=();
 
 foreach my $option (keys(%controls)) {
-    my ($admin, $multi, $physdbkey, $virtdbkey, $needs_osselect) 
-      = @{ $controlset{$option} };
+    my ($admin, $multi, $physdbkey, $virtdbkey,
+	$needs_osselect, $osselect_arg) = @{ $controlset{$option} };
     my $value = $controls{$option};
 
     if ($needs_osselect) {
-	# Don't set osselect fields directly in nodes table
-        if ($value ne "") {
-	  # Make sure we ignore blanks, since osselect needs an OSID
-	  my $str = ( $debug ? "-d " : "").
-	    ( $physdbkey =~ /^next_/ ? "-1 " : "" ).
-	      ( $physdbkey =~ /pxe_boot_/ ? "-m " : "").
-		( $physdbkey =~ /^((def)|(next))_boot_path/ ? "-p " : "").
-		  "$value";
-	  if ($debug) { print "$option=$value ($physdbkey) made osselect str='$str'\n"; }
-	  push(@osselect_params, $str);
+	my $str = ($debug ? "-d " : "");
+	    
+        if ($value eq "") {
+	    # Clearing the field.
+	    $str .= "-c $osselect_arg";
+	}
+	else {
+	    # Setting the field
+	    $str .= "$osselect_arg $value";
 	}
-    } else {
+	if ($debug) {
+	    print "$option=$value ($physdbkey) made osselect str='$str'\n";
+	}
+	push(@osselect_params, $str);
+    }
+    else {
 	if (defined($physnodes_updatestr)) {
 	    $physnodes_updatestr = "$physnodes_updatestr, $physdbkey='$value'";
 	}
diff --git a/tbsetup/node_reboot.in b/tbsetup/node_reboot.in
index d508ff29bd50e1a1acab63e187bc3508eac67ae7..e1fa1dbe81eb7e3ff1f39ae4f4480fb3db7a7d1e 100644
--- a/tbsetup/node_reboot.in
+++ b/tbsetup/node_reboot.in
@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2003 University of Utah and the Flux Group.
+# Copyright (c) 2000-2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -18,18 +18,19 @@ use Getopt::Std;
 #
 sub usage()
 {
-    print STDOUT "Usage: node_reboot [-d] [-f] [-n] [-w] node [node ...]\n" .
-	         "       node_reboot [-d] [-f] [-n] [-w] -e pid,eid\n".
+    print "Usage: node_reboot [-d] [-f] [-n] [-w] [-k] node [node ...]\n" .
+	  "       node_reboot [-d] [-f] [-n] [-w] [-k] -e pid,eid\n".
 	"Use the -d option to turn on debugging\n" .
 	"Use the -e option to reboot all the nodes in an experiment\n" .
 	"Use the -n option to not wait for nodes to go down\n" .
 	"Use the -w option to to wait for nodes is come back up\n" .
+	"Use the -k option to power cycle nodes in PXEWAIT mode\n" .
 	"Use the -f option to power cycle (and not wait for nodes to die)\n";
     exit(-1);
 }
 # The hidden -r option runs this in "realmode", ie don't send an event, but
 # really do the work instead.
-my  $optlist = "dfe:nwr";
+my  $optlist = "dfe:nwrk";
 
 #
 # Configure variables
@@ -51,6 +52,7 @@ my $ssh		= "$TB/bin/sshtb -n";
 my $power	= "$TB/bin/power";
 my $ipod	= "$TB/sbin/apod";
 my $vnodesetup	= "$TB/sbin/vnode_setup";
+my $bisend      = "$TB/sbin/bootinfosend";
 my $logfile	= "$TB/log/power.log";
 my $ping	= "/sbin/ping";
 my %pids	= ();
@@ -63,6 +65,7 @@ my $realmode    = 0;
 my $nowait      = 0;
 my $failed      = 0;
 my $eidmode     = 0;
+my $killmode    = 0;
 my $pid;
 my $eid;
 
@@ -94,6 +97,9 @@ if (defined($options{"d"})) {
 if (defined($options{"f"})) {
     $force = 1;
 }
+if (defined($options{"k"})) {
+    $killmode = 1;
+}
 if (defined($options{"w"})) {
     $waitmode = 1;
 }
@@ -398,7 +404,7 @@ exit $failed;
 #
 sub RebootNode {
     my ($pc) = @_;
-    my ($status, $syspid, $mypid, $didipod);
+    my ($status, $syspid, $mypid, $didipod, $nodestate);
 
     print STDOUT "Rebooting $pc ...\n";
 
@@ -411,6 +417,50 @@ sub RebootNode {
     }
     TBdbfork();
 
+    #
+    # Is the node in PXEWAIT? If so we want to wake it up so that it can
+    # query bootinfo and do what it is supposed to do, without a real reboot.
+    # We send the initial wakeup from here, but let stated deal with it
+    # failing (timeout) and resending it. That means we could be called
+    # with the node in PXEWAKEUP, so send it another wakeup. The point is that
+    # stated is controlling the timeouts. Eventually stated gives up and uses
+    # the -k option to force a power cycle.
+    #
+    if (! TBGetNodeEventState($pc, \$nodestate)) {
+	info("$pc has no event state: power cycle");
+	print STDERR "$pc has no event state Power cycling ...\n" if $debug;
+	# Signal to the parent that the node needs to be power cycled
+	exit(2);
+    }
+    if ($nodestate eq TBDB_NODESTATE_PXEWAIT() ||
+	$nodestate eq TBDB_NODESTATE_PXEWAKEUP()) {
+	#
+	# In killmode, we do not want to bother with sending a wakeup event.
+	# Just do the power cycle. This is used to unstick a machine that
+	# is in waitmode, but not responding to the wakeups. 
+	#
+	if ($killmode) {
+	    info("$pc: in $nodestate: but power cycling in killmode");
+	    print STDERR "$pc: in $nodestate: but power cycling in killmode\n"
+		if $debug;
+	    exit(2);
+	}
+	
+	#
+	# The aux program sends the event to stated ...
+	#
+	my $optarg = ($debug ? "-dd" : "");
+	    
+	print STDERR "$pc: in $nodestate: sending wakeup command.\n" if $debug;
+	system("$bisend $optarg -q $pc");
+	if ($?) {
+	    info("$pc: PXEWAKEUP failed ... power cycle");
+	    print STDERR "$pc: PXEWAKEUP failed ... power cycle.\n" if $debug;
+	    exit(2);
+	}
+	exit(0);
+    }
+
     #
     # See if the machine is pingable. If its not pingable, then we just
     # power cycle the machine rather than wait for ssh to time out.
diff --git a/tbsetup/os_load.in b/tbsetup/os_load.in
index f9a24df0a74742e7bdce75166d8f1839a6726163..94256bc88ee61e24aca62527e42bb9f3457369e3 100755
--- a/tbsetup/os_load.in
+++ b/tbsetup/os_load.in
@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2003 University of Utah and the Flux Group.
+# Copyright (c) 2000-2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -66,6 +66,7 @@ my $usedefault  = 1;
 my $imagename;
 my $imageid;
 my $imagepid    = TB_OPSPID;
+my $FRISBEEOSID = TB_OSID_FRISBEE_MFS();
 my %imageid_row;
 my @nodes       = ();
 my %retries	= ();
@@ -248,7 +249,6 @@ foreach my $node (@nodes) {
 
     print STDOUT "Changing default OS for $node to $defosid\n";
     if (!$TESTMODE) {
-        system("$osselect -m PXEBOOT $node");
         system("$osselect $defosid $node");
     }
 
@@ -424,7 +424,7 @@ sub SetupReload($) {
     DBQueryFatal("replace into current_reloads ".
 		 "(node_id, image_id) values ('$node', '$imageid')");
 
-    system "$osselect -1 -m PXEFRISBEE $node" and
+    system "$osselect -1 $FRISBEEOSID $node" and
       die "*** Unable to select frisbee OS\n";
     system "$FRISBEELAUNCHER ".($dbg? "-d ":"")."$imageid" and
       die "*** Unable to launch frisbee daemon\n";
diff --git a/tbsetup/os_select.in b/tbsetup/os_select.in
index 04430a14ab4a0f2759eceedeaa1e72e1a7b0828c..7dfe9bc649a6f90e59b2e683345fc215eecfaea9 100644
--- a/tbsetup/os_select.in
+++ b/tbsetup/os_select.in
@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2003 University of Utah and the Flux Group.
+# Copyright (c) 2000-2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -11,18 +11,18 @@
 
 sub usage() {
     print <<"EOF";
-Usage: os_select [-h] [-d] [-v] [-1] [-p | -m] <osid> <node> [<node> ...]
+Usage: os_select [-h] [-d] [-c] [-1 | -t] [<osid>] <node> [<node> ...]
  -h    Display this help message
  -d    Debug mode
- -v    Verbose mode
- -p    Path mode: osid is really a path to a kernel
- -1    Set up one-time boot to OS instead of changing default OS
- -m    MFS mode: osid={"PXEFBSD","PXEFRISBEE","PXEBOOT","host:/tftpboot/file"}
+ -c    Clear the specified boot osid for nodes. Do not provide an osid.
+ -1    Apply change to one-time boot field
+ -t    Apply change to temporary boot field
  osid  OS identifier for the selected OS (see web interface for listing)
  node  Node identifiers (ie pcXX)
 EOF
     exit(-1);
 }
+my $optlist = "hdc1t";
 
 # un-taint path
 $ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
@@ -49,235 +49,221 @@ my %osidmap = # Map some magic OSIDs to op_modes
     ( $MBKERNEL => "MINIMAL");
 
 # Global vars
-my $d=0; # debug/verbose
-my $oneshot=0; # is this a one-shot OS boot?
-my $pathmode=0; # is osid really a path instead of an osid?
-my $mfs=0; # is this for a pxe_boot_path change?
+my $debug	= 1; # debug/verbose
+my $oneshot	= 0; # apply change to next_boot_osid.
+my $tempmode	= 0; # apply change to temp_boot_osid.
+my $clear       = 0; # Clear the selected boot (def,temp,next).
+my @nodes       = ();
+my $osid	= "";
+my $opmode;
 
 # Set up syslog
-openlog("osselect","pid",$TBLOG);
-
-# Find default pxe boot path
-my $cmd = "select path from os_info where osid='".TB_OSID_PXEBOOT."'";
-my $q = DBQueryFatal($cmd);
-my @r = $q->fetchrow_array();
-my $pxebootpath = $r[0];
-my $cmdline = join(" ",$0,@ARGV);
-debug("ARGV= ".join(" ",@ARGV)."\n");
+openlog("osselect", "pid", $TBLOG);
 
+#
 # Parse command arguments. Once we return from getopts, all that should be
 # left are the required arguments.
-my  $optlist = "hdvp1m";
+#
 %options = ();
 if (! getopts($optlist, \%options)) { usage(); }
 if (defined($options{"h"})) { usage(); }
-if (defined($options{"d"})) { $d++; }
-if (defined($options{"v"})) { $d++; }
-if (defined($options{"p"})) { $pathmode=1; }
+if (defined($options{"d"})) { $debug++; }
 if (defined($options{"1"})) { $oneshot=1; }
-if (defined($options{"m"})) { $mfs=1; }
-#debug("DEBUG LEVEL $d\n");
-
-if (@ARGV < 2) { warning("An osid and a node list are required.\n"); usage(); }
-
-my $osid = shift;
-my @nodes = @ARGV;
-
-# Untaint args.
-if ($osid =~ /^([-\@\w\/\+\.]+)$/) { $osid = $1; }
-elsif (!$mfs) { fatal("Bad data in osid: '$osid'\n"); }
-
-if ($mfs) {
-    if ($osid =~ /^[-0-9a-z\.]+:\/tftpboot\/[-0-9a-z\.\/]+$/) {
-	debug("MFS osid '$osid' is a PXE path\n");
-	$pxebootpath=$osid;
-	my $cmd = "select osid from os_info where path='$osid';";
-	my $q = DBQueryFatal($cmd);
-	if ($q->numrows() > 0) {
-	    my @r = $q->fetchrow_array();
-	    debug("Path '$osid' => OSID '$r[0]'\n");
-	    $osid=$r[0];
-	    $pathmode=0;
-	} else { $pathmode=1; }
-    } else {
-	my $cmd = "select path from os_info where osid='$osid';";
-	my $q = DBQueryFatal($cmd);
-	if ($q->numrows() < 1) {
-	    fatal("Invalid osid '$osid' does not exist.\n");
-	}
-	my @r = $q->fetchrow_array();
-	my $path=$r[0];
-	if (defined($path) && $path ne "") {
-	    $pxebootpath=$path;
-	} else {
-	    fatal("Invalid osid for use with -m: '$osid'\n");
-	}
+if (defined($options{"t"})) { $tempmode=1; }
+if (defined($options{"c"})) { $clear=1; }
+
+# In clearmode, there is no OSID. Just a list of nodes.
+if (! $clear) {
+    usage()
+	if (@ARGV < 2);
+    $osid = shift();
+
+    # Untaint args.
+    if ($osid =~ /^([-\w\+\.]+)$/) {
+	$osid = $1;
+    }
+    else {
+	fatal("Bad data in osid: '$osid'");
     }
 }
-
-my @temp;
-foreach $n (@nodes) {
-    if ($n =~ /^([-\w]+)$/) { push(@temp,$1); }
-    else { warning("Ignoring bad data in node_id: '$n'\n"); }
+else {
+    usage()
+	if (@ARGV < 1);
 }
-@nodes=@temp;
 
-if (@nodes < 1) { fatal("No valid nodes supplied.\n"); }
+# Untaint the nodes.
+foreach my $node ( @ARGV ) {
+    if ($node =~ /^([-\@\w]+)$/) {
+	$node = $1;
+    }
+    else {
+	fatal("Bad node name: $node");
+    }
+    push(@nodes, $node);
+}
 
+#
 # Figure out who called us. Only root, people with admin status
 # in the DB, or members of the right project can do this.
-my ($me) = getpwuid($UID)
-  or fatal("$UID not in passwd file\n");
-if ($UID && !TBAdmin($UID)) {
-    if (! TBNodeAccessCheck($UID,TB_NODEACCESS_MODIFYINFO,@nodes)) {
-        fatal("os_select: You do not have permission to modify ".
-	    "one or more of the nodes. ($me/$UID)\n");
-    }
-    debug("$me/$UID: Access granted to all nodes requested.\n");
-} else { debug("$me/$UID: Running as an admin.\n"); }
-
-my $pernodeopmode=0;
-my $opmode = os_opmode($osid);
-debug("Found opmode '$opmode' for osid '$osid'.\n");
-if ($opmode eq TBDB_NODEOPMODE_BOOTWHAT ) { $pernodeopmode=1; }
-
-foreach $n (@nodes) {
-    my $curmode = node_opmode($n);
-    if (!$curmode) { next; }
-    debug("Current opmode for node '$n' is '$curmode'.\n");
-    # Always set the pxe_boot_path (to the osid in mfs mode, else PXEBOOT)
-    set_pxe_path($n);
-    # Only do the boot osid if we're not in mfs mode
-    if (!$mfs) { set_boot_osid($n); }
-    my $boot = TBBootWhat($n);
-    if (!$boot) { fatal("***Bootwhat query failed. Contact testbed-ops.\n"); }
-    debug("Bootwhat says: $n => $boot\n");
-    if ($pernodeopmode) { 
-	# in per-node mode, opmode is the mode for the os that we're going 
-	# to boot, whatever that may be on each node
-	$opmode= os_opmode($boot); 
-	debug("Desired opmode for node '$n' is '$opmode'.\n");
+#
+if ($UID && !TBAdmin($UID) &&
+    !TBNodeAccessCheck($UID, TB_NODEACCESS_MODIFYINFO, @nodes)) {
+    fatal("os_select: You do not have permission to modify ".
+	  "one or more of the nodes.\n");
+}
+
+#
+# Grab the info for the OSID. 
+#
+if (! $clear) {
+    my $query_result =
+	DBQueryFatal("select * from os_info where osid='$osid'");
+    if ($query_result->numrows != 1) {
+	fatal("Improper DB entry for OSID: $osid");
     }
-    if (!$curmode || ($curmode ne $opmode)) { set_nextmode($n); }
-    else { set_nextmode($n,""); } # Make sure it is clear
+    my %dbrow  = $query_result->fetchhash();
+    $opmode = $dbrow{"op_mode"};
 }
 
-exit();
+foreach my $node (@nodes) {
+    my $curmode = node_opmode($node);
+
+    # Why? When will this happen?
+    next
+	if (!$curmode);
+    
+    debug("Current opmode for $node is $curmode.\n");
+
+    # Set/Clear the osid.
+    set_boot_osid($node);
+
+    #
+    # Determine what osid the node will now boot. We need to know this so we
+    # can set the next opmode. This call has to return *something* or we are
+    # screwed since we will not be able to figure out the opmode.
+    #
+    my ($bootosid, $bootopmode) = TBBootWhat($node, $debug);
+
+    fatal("Bootwhat query failed for $node!")
+	if (!$bootosid);
+    debug("Bootwhat says: $node => $bootosid\n");
+
+    #
+    # If its different then what the node is currently booting, then
+    # set up a transition in stated. If no change, be sure to clear
+    # is since stated does not like a transition to be specified when
+    # none is actually going to be made.
+    #
+    if ($curmode ne $bootopmode) {
+	set_nextmode($node, $bootopmode);
+    }
+    else {
+	# Must clear it.
+	set_nextmode($node, "");
+    }
+}
+exit(0);
 
 #
 # Subroutines
 #
 
-sub set_nextmode() {
-    my $node = shift || "";
-    my $mode = shift;
-    if (!defined($mode)) { $mode = $opmode; }
-    my $cmd = "update nodes set next_op_mode='$mode' where node_id='$node';";
-    debug("Setting next_op_mode for node '$node' to '$mode'.\n");
-    my $q = DBQueryFatal($cmd);
-    return 0;
-}
+# Set the next_op_mode field for a node.
+sub set_nextmode($;$)
+{
+    my ($node, $opmode) = @_;
 
-sub set_boot_osid() {
-    my $node = shift || "";
-    my $field = ($oneshot?"next":"def")."_boot_".($pathmode?"path":"osid");
-    my $cmd = "update nodes set $field='$osid'";
-    # Clear out the paths and the next_boot_osid except for possibly
-    # one that we're going to set. Never clear def_boot_osid, since it
-    # won't ever get in the way of what we really want to boot, and we
-    # want something to fall back on.
-    foreach $f ("next_boot_path", "def_boot_path", "next_boot_osid") {
-	if ($f ne $field) { $cmd .= ", $f=''"; }
+    if ($opmode eq "") {
+	debug("Clearing next_op_mode for $node.\n");
     }
-    $cmd .= " where node_id='$node';";
-    #debug("cmd=$cmd\n");
-    debug("Setting $field for node '$node' to '$osid'.\n");
-    my $q = DBQueryFatal($cmd);
-    return 0;
-}
-
-sub set_pxe_path() {
-    my $node = shift || "";
-    my $field = ($oneshot?"next_":"")."pxe_boot_path";
-    my $cmd = "update nodes set $field='$pxebootpath'";
-    # Clear out the next_pxe_boot_path if we're not going to set it
-    foreach $f ("next_pxe_boot_path") {
-	if ($f ne $field) { $cmd .= ", $f=''"; }
+    else {
+	debug("Setting next_op_mode for $node to $opmode.\n");    
     }
-    $cmd .= " where node_id='$node';";
-    #debug("cmd=$cmd\n");
-    debug("Setting $field for node '$node' to '$pxebootpath'.\n");
-    my $q = DBQueryFatal($cmd);
+    
+    DBQueryFatal("update nodes set next_op_mode='$opmode' ".
+		 "where node_id='$node'");
     return 0;
 }
 
-sub node_opmode() {
-    my $node = shift || "";
-    my $cmd = "select op_mode from nodes where node_id='$node';";
-    my $q = DBQueryFatal($cmd);
-    if ($q->numrows() < 1) {
-	warning("Ignoring invalid node '$node' (non-existent)\n");
-	return 0;
+# Set (or clear) the boot osid.
+sub set_boot_osid($) {
+    my ($node) = @_;
+    my $field  = "def_boot_osid";
+
+    $field = "next_boot_osid"
+	if ($oneshot);
+    $field = "temp_boot_osid"
+	if ($tempmode);
+
+    if ($clear) {
+	debug("Clearing $field for $node.\n");
     }
-    my @r = $q->fetchrow_array();
-    my $opmode=$r[0];
-    if (defined($opmode) && $opmode) { return $opmode; }
-    warning("Invalid opmode '$opmode' for node '$node'.\n");
-    return 0;
+    else {
+	debug("Setting $field for $node to $osid.\n");
+    }
+    
+    DBQueryFatal("update nodes set ${field}='$osid' where node_id='$node'");
 }
 
-sub os_opmode() {
-    my $osid = shift || "";
-    if ($pathmode) { return $osidmap{$MBKERNEL}; }
-    if (defined($osidmap{$osid})) { return $osidmap{$osid}; }
-    my $cmd = "select op_mode from os_info where osid='$osid';";
-    my $q = DBQueryFatal($cmd);
-    if ($q->numrows() < 1) {
-	fatal("Invalid osid '$osid' is non-existent.\n");
+# Return current opmode that a node is set to.
+sub node_opmode($)
+{
+    my ($node) = @_;
+    my $opmode;
+
+    if (! TBGetNodeOpMode($node, \$opmode)) {
+	fatal("Could not get opmode for node $node!");
+    }
+    if (!defined($opmode) || $opmode eq "" ||
+	$opmode eq TBDB_NODEOPMODE_UNKNOWN) {
+	warning("Invalid opmode $opmode for $node.\n");
+	return 0;
     }
-    my @r = $q->fetchrow_array();
-    my $opmode=$r[0];
-    debug("OpMode for '$osid' is '$opmode'\n");
-    if (defined($opmode) && $opmode ne "") { return $opmode; }
-    fatal("No opmode found for osid '$osid'.\n");
+    return $opmode;
 }
 
-sub debug ( $;$ ) {
-  my $msg = shift;
-  my $notice = shift || 0;
-  my $prio="info";
-  if ($notice) { $prio = "notice"; }
-  syslog($prio,$msg) || notify("syslog failed: $! $?\n");
-  if ($d) { print $msg; }
+sub debug($;$)
+{
+    my $msg = shift;
+    my $notice = shift || 0;
+    my $prio="info";
+    
+    if ($notice) { $prio = "notice"; }
+    
+    syslog($prio, $msg);
+    if ($debug) { print $msg; }
 }
 
-sub notify ( $ ) {
-  my $msg = shift;
-  my $user= getpwuid($UID);
-  $msg .= "\ndate=".`date`."\ncmdline=\n$cmdline\n\npid=$$\n\nuser=$user\n";
-  if (!$d) {
-    SENDMAIL($TBOPS,"os_select error",$msg,$TBOPS);
-  } else {
-    debug("notify: Not sending mail in debug mode\n");
-  }
-  debug($msg,1);
+sub notify($)
+{
+    my $msg = shift;
+    
+    if (!$debug) {
+	SENDMAIL($TBOPS, "os_select error", $msg);
+    } 
+    debug($msg, 1);
 }
 
-sub info ( $;$ ) {
-  my $msg = shift;
-  debug($msg);
+sub info($)
+{
+    my $msg = shift;
+    
+    debug($msg);
 }
 
-sub fatal ( $ ) {
-  my $msg = shift;
-  notify("FATAL: ".$msg);
-  die($msg);
+sub fatal($)
+{
+    my $msg = shift;
+    
+    notify("FATAL: $msg\n");
+    exit(1);
 }
 
-sub warning ( $ ) {
-  my $msg = shift;
-  info("WARNING: ".$msg);
-  warn($msg);
+sub warning($)
+{
+    my $msg = shift;
+    
+    info("WARNING: $msg\n");
 }
 
 # This is called when we exit with exit() or die()
diff --git a/utils/create_image.in b/utils/create_image.in
index 999a8e000c6ca894b840f824bcf35656d47ce708..614b0848f31c3b54b35e10e5966b2856446cc324 100755
--- a/utils/create_image.in
+++ b/utils/create_image.in
@@ -62,6 +62,7 @@ my %imageid_row = ();
 my $debug       = 0;
 my $imageid;
 my $imagepid    = TB_OPSPID;
+my $ADMINOSID   = TB_OSID_FREEBSD_MFS;
 my $logname;
 my @row;
 my $dbuid;
@@ -305,8 +306,8 @@ my $maxslack = (3 * 60) / $sleepwait; # NFS cache slop factor
 $needcleanup = 1;
 
 while ($tries) {
-    system("$osselect -m PXEFBSD $node") and
-	fatal("*** Failed to change OS on $node!\n");
+    system("$osselect -t $ADMINOSID $node") and
+	fatal("*** Failed to set temp boot to $ADMINOSID for $node!");
 
     if (! DBQueryWarn("update nodes set ".
 		      "startupcmd='$command', startstatus='none' ".
@@ -443,8 +444,8 @@ sub cleanup ()
     # Reset node (DB) state
     #
 
-    if (system("$osselect -m $saved_pxebootpath $node")) {
-	print("*** Failed to reset PXE boot path to $saved_pxebootpath!\n");
+    if (system("$osselect -c -t $node")) {
+	print("*** Failed to clear temp boot for $node!\n");
         $retval = 0;
     }
 
diff --git a/utils/newnode.in b/utils/newnode.in
index 796b3b3da5b3ed03cddaa21146783429b8095395..760dd93945e4ddbce95786a6a02a411515256d4d 100644
--- a/utils/newnode.in
+++ b/utils/newnode.in
@@ -44,7 +44,7 @@ my $INITIAL_OPMODE = TBDB_NODEOPMODE_RELOAD;
 #
 # MFS to boot the nodes into initially
 #
-my $INITIAL_MFS = "PXEFBSD";
+my $INITIAL_MFS = TB_OSID_FREEBSD_MFS();
 
 #
 # Number of vnodes to create for each physical node
@@ -317,7 +317,7 @@ if ($exports_rv) {
 # likes better
 #
 print "Instructing nodes to boot into the FreeBSD MFS\n";
-my $select_rv = system "$os_select -m $INITIAL_MFS " .
+my $select_rv = system "$os_select $INITIAL_MFS " .
     join(" ",@succeeded_nodes);
 if ($select_rv) {
     warn "WARNING - failed to select FreeBSD MFS for nodes";
diff --git a/utils/node_admin.in b/utils/node_admin.in
index ba2b5b5cc2d4682c3a2c454d20cec217c8af4fce..5ecfe0459b83b1d3a4271b9dc67f5cc37354e837 100755
--- a/utils/node_admin.in
+++ b/utils/node_admin.in
@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# Copyright (c) 2000-2003 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -51,6 +51,7 @@ delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
 #
 my $nodereboot	= "$TB/bin/node_reboot";
 my $osselect    = "$TB/bin/os_select";
+my $ADMINOSID   = TB_OSID_FREEBSD_MFS;
 my $pxebootpath;
 my $dbuid;
 
@@ -106,21 +107,13 @@ if ($UID && !TBAdmin($UID)) {
 }
 
 if ($onoff eq "on") {
-    $pxebootpath = "PXEFBSD";
+    system("$osselect -t $ADMINOSID $node") and
+	die("*** Failed to set temp boot to $ADMINOSID for $node!\n");
 }
 else {
-    my $query_result =
-	DBQueryFatal("select node_types.pxe_boot_path from node_types " .
-		     "left join nodes on nodes.type=node_types.type " .
-		     "where node_id='$node'");
-
-    my @row = $query_result->fetchrow_array();
-    $pxebootpath = $row[0];
+    system("$osselect -c -t $node") and
+	die("*** Failed to clear temp boot for $node!\n");
 }
-
-system("$osselect -m $pxebootpath $node") and
-    die("*** Failed to set PXE boot path for $node!\n");
-
 DBQueryFatal("update nodes set startupcmd='', startstatus='none' ".
 	     "where node_id='$node'");
 
diff --git a/www/nodecontrol.php3 b/www/nodecontrol.php3
index 59d7c9717d0b41666a4eeddc40c4a70796b528f5..e203d755d96fae0cbb19eee01a9fbe118268eb31 100644
--- a/www/nodecontrol.php3
+++ b/www/nodecontrol.php3
@@ -1,7 +1,7 @@
 <?php
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# Copyright (c) 2000-2002, 2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 include("defs.php3");
@@ -22,10 +22,11 @@ LOGGEDINORDIE($uid);
 # Check to make sure that this is a valid nodeid
 #
 $query_result =
-    DBQueryFatal("SELECT node_id FROM nodes WHERE node_id='$node_id'");
+    DBQueryFatal("SELECT * FROM nodes WHERE node_id='$node_id'");
 if (mysql_num_rows($query_result) == 0) {
     USERERROR("The node $node_id is not a valid nodeid!", 1);
 }
+$row = mysql_fetch_array($query_result);
 
 #
 # Admin users can control any node, but normal users can only control
@@ -38,27 +39,38 @@ if (! $isadmin) {
     }
 }
 
-if (isset($def_boot_osid) && strcmp($def_boot_osid, "None") == 0) {
-    $def_boot_osid = "";
-}
-
 #
-# Create a command string, that is slightly different if an admin; we allow
-# admin people to set next_boot parameters. Ordinary folks cannot.
+# Check each parameter. Also note that when setting/clearing values,
+# send the argument to the backend script *only when changed*
 #
-$command_string =
-	"default_boot_osid='$def_boot_osid'		".
-	"default_boot_path='$def_boot_path'		".
-	"default_boot_cmdline='$def_boot_cmd_line'	".
-	"startup_command='$startupcmd'			".
-	"tarfiles='$tarballs'                           ".
-	"rpms='$rpms'                                   ";
+$command_string = "";
+
+if ($def_boot_osid != $row[def_boot_osid]) {
+    $command_string .= "default_boot_osid='$def_boot_osid' ";
+}
+if ($def_boot_cmd_line != $row[def_boot_cmd_line]) {
+    $command_string .= "default_boot_cmdline='$def_boot_cmd_line' ";
+}
+if ($startupcmd != $row[startupcmd]) {
+    $command_string .= "startup_command='$startupcmd' ";
+}
+if ($tarballs != $row[tarballs]) {
+    $command_string .= "tarfiles='$tarballs' ";
+}
+if ($rpms != $row[rpms]) {
+    $command_string .= "rpms='$rpms' ";
+}
 
 if ($isadmin) {
-    $command_string = "$command_string			".
-	"next_boot_osid='$next_boot_osid'		".
-	"next_boot_path='$next_boot_path'		".
-	"next_boot_cmdline='$next_boot_cmd_line'	";
+    if ($next_boot_osid != $row[next_boot_osid]) {
+	$command_string .= "next_boot_osid='$next_boot_osid' ";
+    }
+    if ($next_boot_cmd_line != $row[next_boot_cmd_line]) {
+	$command_string .= "next_boot_cmdline='$next_boot_cmd_line' ";
+    }
+    if ($temp_boot_osid != $row[temp_boot_osid]) {
+	$command_string .= "temp_boot_osid='$temp_boot_osid' ";
+    }
 }
 
 #
diff --git a/www/nodecontrol_form.php3 b/www/nodecontrol_form.php3
index b2afd4f0d7454195685ed53f3afe3f4d45f94305..7143e8838d96e0f402e14992edfdfc34d443789b 100644
--- a/www/nodecontrol_form.php3
+++ b/www/nodecontrol_form.php3
@@ -1,7 +1,7 @@
 <?php
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2002 University of Utah and the Flux Group.
+# Copyright (c) 2000-2002, 2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 include("defs.php3");
@@ -52,11 +52,10 @@ $node_id            = $row[node_id];
 $type               = $row[type];
 $vname		    = $row[vname];
 $def_boot_osid      = $row[def_boot_osid];
-$def_boot_path      = $row[def_boot_path];
 $def_boot_cmd_line  = $row[def_boot_cmd_line];
 $next_boot_osid     = $row[next_boot_osid];
-$next_boot_path     = $row[next_boot_path];
 $next_boot_cmd_line = $row[next_boot_cmd_line];
+$temp_boot_osid     = $row[temp_boot_osid];
 $rpms               = $row[rpms];
 $tarballs           = $row[tarballs];
 $startupcmd         = $row[startupcmd];
@@ -156,14 +155,6 @@ echo "       </select>";
 echo "    </td>
       </tr>\n";
 
-echo "<tr>
-          <td>Def Boot Path:</td>
-          <td class=\"left\">
-              <input type=\"text\" name=\"def_boot_path\" size=\"40\"
-                     value=\"$def_boot_path\"></td>
-      </tr>\n";
-
-
 echo "<tr>
           <td>Def Boot Command Line:</td>
           <td class=\"left\">
@@ -181,8 +172,17 @@ if ($isadmin) {
     
     while ($row = mysql_fetch_array($osid_result)) {
 	$osname = $row[osname];
-	$osid = $row[osid];
-	$pid  = $row[pid];
+	$oosid = $row[oosid];
+	$posid = $row[posid];
+
+        # Use the osid that came from the partitions table, if there
+	# was one - otherwise, go with the os_info table
+	if ($posid) {
+	    $osid = $posid;
+	}
+	else {
+	    $osid = $oosid;
+	}
 
 	echo "<option ";
 	if ($next_boot_osid == $osid) {
@@ -194,20 +194,43 @@ if ($isadmin) {
     echo "    </td>
            </tr>\n";
 
-    echo "<tr>
-             <td>Next Boot Path:</td>
-             <td class=\"left\">
-                 <input type=\"text\" name=\"next_boot_path\" size=\"40\"
-                        value=\"$next_boot_path\"></td>
-          </tr>\n";
-
-
     echo "<tr>
               <td>Next Boot Command Line:</td>
               <td class=\"left\">
                   <input type=\"text\" name=\"next_boot_cmd_line\" size=\"40\"
                          value=\"$next_boot_cmd_line\"></td>
           </tr>\n";
+
+    mysql_data_seek($osid_result, 0);
+
+    echo "<tr>
+              <td>Temp Boot OS:</td>";
+    echo "    <td><select name=\"temp_boot_osid\">\n";
+    echo "                <option value=\"\">No OS</option>\n";
+    
+    while ($row = mysql_fetch_array($osid_result)) {
+	$osname = $row[osname];
+	$oosid = $row[oosid];
+	$posid = $row[posid];
+
+        # Use the osid that came from the partitions table, if there
+	# was one - otherwise, go with the os_info table
+	if ($posid) {
+	    $osid = $posid;
+	}
+	else {
+	    $osid = $oosid;
+	}
+
+	echo "<option ";
+	if ($temp_boot_osid == $osid) {
+	    echo "selected ";
+	}
+	echo "value=\"$osid\">$pid - $osname</option>\n";
+    }
+    echo "       </select>";
+    echo "    </td>
+           </tr>\n";
 }
 
 echo "<tr>
diff --git a/www/showstuff.php3 b/www/showstuff.php3
index 4aa4c5d909050c0596d917ebe683fa875458f617..5e7b3f4d24f0f9a4db3a7138350f07143719fbd5 100644
--- a/www/showstuff.php3
+++ b/www/showstuff.php3
@@ -1,7 +1,7 @@
 <?php
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2003 University of Utah and the Flux Group.
+# Copyright (c) 2000-2004 University of Utah and the Flux Group.
 # All rights reserved.
 #
 #
@@ -1520,10 +1520,9 @@ function SHOWNODE($node_id, $short = 0) {
     $eid		= $row[eid];
     $bios               = $row[bios_version];
     $def_boot_osid      = $row[def_boot_osid];
-    $def_boot_path      = $row[def_boot_path];
     $def_boot_cmd_line  = $row[def_boot_cmd_line];
     $next_boot_osid     = $row[next_boot_osid];
-    $next_boot_path     = $row[next_boot_path];
+    $temp_boot_osid     = $row[temp_boot_osid];
     $next_boot_cmd_line = $row[next_boot_cmd_line];
     $rpms               = $row[rpms];
     $tarballs           = $row[tarballs];
@@ -1554,10 +1553,6 @@ function SHOWNODE($node_id, $short = 0) {
     
     if (!$def_boot_cmd_line)
 	$def_boot_cmd_line = "&nbsp";
-    if (!$def_boot_path)
-	$def_boot_path = "&nbsp";
-    if (!$next_boot_path)
-	$next_boot_path = "&nbsp";
     if (!$next_boot_cmd_line)
 	$next_boot_cmd_line = "&nbsp";
     if (!$rpms)
@@ -1702,11 +1697,6 @@ function SHOWNODE($node_id, $short = 0) {
 
     if (!$short) {
 	if (!$isvirtnode && !$isremotenode) {
-	    echo "<tr>
-                      <td>Def Boot Path:</td>
-                      <td class=left>$def_boot_path</td>
-                  </tr>\n";
-
 	    echo "<tr>
                       <td>Def Boot Command&nbsp;Line:</td>
                       <td class=left>$def_boot_cmd_line</td>
@@ -1725,13 +1715,20 @@ function SHOWNODE($node_id, $short = 0) {
                   </tr>\n";
 
 	    echo "<tr>
-                     <td>Next Boot Path:</td>
-                     <td class=left>$next_boot_path</td>
+                      <td>Next Boot Command Line:</td>
+                      <td class=left>$next_boot_cmd_line</td>
                   </tr>\n";
 
 	    echo "<tr>
-                      <td>Next Boot Command Line:</td>
-                      <td class=left>$next_boot_cmd_line</td>
+                      <td>Temp Boot OS:</td>
+                      <td class=left>";
+    
+	    if ($temp_boot_osid)
+		SPITOSINFOLINK($temp_boot_osid);
+	    else
+		echo "&nbsp";
+
+	    echo "    </td>
                   </tr>\n";
 	}
 	elseif ($isvirtnode) {