Adjust the set of unix gids used for a download server.

When downloading an image, start the frisbeed process with the minimum set of
gids necessary to access the image. This includes the unix gid of the
project that the image is in and, optionally, the unix gid of the project
subgroup if the image is part of one.

Previously, we just use the gid set of the uid of the swapper of the
experiment. Not only was this excessive, but it might also not include the
gids needed in the case of a "global" image that is not in the world-readable
/usr/testbed/images directory.

os/frisbee.redux/config_emulab.c

@@ -1076,6 +1076,7 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 					 " ORDER BY pid,gid,imagename",
 					 5, ei->pid, ei->gid);
 		}
+		assert(res != NULL);
 
 		/* Construct the list of "pid/imagename" imageids */
 		nrows = mysql_num_rows(res);
@@ -1156,23 +1157,25 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 
 		if (imageid != NULL) {
 			/* Interested in a specific image */
-			res = mydb_query("SELECT pid,gid,imagename,path,imageid"
+			res = mydb_query("SELECT pid,gid,imagename,path,imageid,pid_idx,gid_idx"
 					 " FROM images"
 					 " WHERE (global=1"
 					 "  OR (pid='%s' AND (gid='%s' OR shared=1)))"
 					 "  AND pid='%s' AND imagename='%s'"
 					 " ORDER BY pid,gid,imagename",
-					 5, ei->pid, ei->gid,
+					 7, ei->pid, ei->gid,
 					 wantpid, wantname);
 		} else {
 			/* Find all images that this pid/gid can GET */
-			res = mydb_query("SELECT pid,gid,imagename,path,imageid"
+			res = mydb_query("SELECT pid,gid,imagename,path,imageid,pid_idx,gid_idx"
 					 " FROM images"
 					 " WHERE (global=1"
 					 "  OR (pid='%s' AND (gid='%s' OR shared=1)))"
 					 " ORDER BY pid,gid,imagename",
-					 5, ei->pid, ei->gid);
+					 7, ei->pid, ei->gid);
 		}
+		assert(res != NULL);
+
 		/* Construct the list of "pid/imagename" imageids */
 		nrows = mysql_num_rows(res);
 		if (nrows)
@@ -1185,6 +1188,7 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 			struct config_imageinfo *ci;
 			struct stat sb;
 			char *iid;
+			int igids[2];
 
 			row = mysql_fetch_row(res);
 			/* XXX ignore rows with null or empty info */
@@ -1192,7 +1196,9 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 			    !row[1] || !row[1][0] ||
 			    !row[2] || !row[2][0] ||
 			    !row[3] || !row[3][0] ||
-			    !row[4] || !row[4][0])
+			    !row[4] || !row[4][0] ||
+			    !row[5] || !row[5][0] ||
+			    !row[6] || !row[6][0])
 				continue;
 			iid = mymalloc(strlen(row[0]) + strlen(row[2]) + 2);
 			strcpy(iid, row[0]);
@@ -1211,14 +1217,61 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 			} else
 				ci->sig = NULL;
 			ci->uid = ei->suid;
+
 			/*
-			 * XXX note that we don't really need all of the
-			 * swapper's GIDs here, we could just include the
-			 * project and any group GID for the image.
+			 * Find the unix gids to use for any server process.
+			 * This includes the gid of the image's project and
+			 * any subgroup gid if the image is associated with
+			 * one. If for any reason we don't come up with
+			 * these gids, we use the gids for the swapper of
+			 * the experiment.
+			 *
+			 * XXX we could probably combine the lookup of these
+			 * gids with the query above, but it doesn't really
+			 * seem worth it.
 			 */
-			for (j = 0; j < ei->ngids; j++)
-				ci->gids[j] = ei->sgids[j];
-			ci->ngids = ei->ngids;
+			igids[0] = igids[1] = -1;
+			{
+				MYSQL_RES *res2;
+				MYSQL_ROW row2;
+				int nrows2, pidx, gidx;
+
+				pidx = atoi(row[5]);
+				gidx = atoi(row[6]);
+				res2 = mydb_query("SELECT unix_gid"
+						  " FROM groups"
+						  " WHERE pid_idx=%d AND"
+						  "       (gid_idx=pid_idx OR"
+						  "        gid_idx=%d)",
+						  1, pidx, gidx);
+				assert(res2 != NULL);
+				nrows2 = mysql_num_rows(res2);
+				if (nrows2 > 0) {
+					row2 = mysql_fetch_row(res2);
+					if (row2[0] && row2[0][0])
+						igids[0] = atoi(row2[0]);
+					if (nrows2 > 1) {
+						row2 = mysql_fetch_row(res2);
+						if (row2[0] && row2[0][0])
+							igids[1] =
+								atoi(row2[0]);
+					}
+				}
+				mysql_free_result(res2);
+			}
+			if (igids[0] > 0) {
+				ci->gids[0] = igids[0];
+				ci->ngids = 1;
+				if (igids[1] > 0) {
+					ci->gids[1] = igids[1];
+					ci->ngids++;
+				}
+			} else {
+				for (j = 0; j < ei->ngids; j++)
+					ci->gids[j] = ei->sgids[j];
+				ci->ngids = ei->ngids;
+			}
+
 			set_get_values(get, get->numimages);
 			ii = mymalloc(sizeof *ii);
 			ii->DB_imageid = atoi(row[4]);

os/frisbee.redux/network.c

@@ -6,6 +6,12 @@
 
 /*
  * Network routines.
+ *
+ * WINDOWS specific hacks (according to
+ * http://support.microsoft.com/default.aspx?scid=kb;en-us;131978):
+ *
+ * - cannot disable local echo with IP_MULTICAST_LOOP
+ *
  */
 #include <sys/types.h>
 #include <sys/socket.h>

os/imagezip/GNUmakefile.in

@@ -72,10 +72,10 @@ endif
 #
 WITH_FFS	= 1
 WITH_EXTFS	= 1
+WITH_EXT4FS	= 0
 WITH_NTFS	= @WINSUPPORT@
 WITH_FAT	= @WINSUPPORT@
 WITH_HASH	= 0
-WITH_EXT4FS	= 0
 
 include $(OBJDIR)/Makeconf
 
@@ -153,7 +153,8 @@ endif
 # Once proven, switch to 2.0.0.
 #
 ifndef NTFSVER
-NTFSVER		= 1.7.1
+#NTFSVER		= 1.7.1
+NTFSVER		= 2.0.0
 endif
 export		NTFSVER
 

os/imagezip/ext4fs/GNUmakefile.in

 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2004 University of Utah and the Flux Group.
+# Copyright (c) 2000-2011 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -18,9 +18,9 @@ all:	libext4fs.a
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
-OBJS = ext4fs.o reloc_lilo.o
+OBJS = extfs.o reloc_lilo.o
 
-ext4fs.o: $(MAINDIR)/sliceinfo.h $(MAINDIR)/global.h
+extfs.o: $(MAINDIR)/sliceinfo.h $(MAINDIR)/global.h
 reloc_lilo.o: $(MAINDIR)/sliceinfo.h $(MAINDIR)/global.h $(MAINDIR)/imagehdr.h
 
 libext4fs.a:	$(OBJS)

os/imagezip/imagezip.c

@@ -65,6 +65,7 @@ int	level	  = 4;
 long	dev_bsize = 1;
 uint32_t compat   = 0;
 int	frangesize= 64;	/* 32k */
+int	zerofrange= 0;
 int	forcereads= 0;
 int	badsectors= 0;
 int	retrywrites= 1;
@@ -403,7 +404,7 @@ main(int argc, char *argv[])
 	memset(imageid, UUID_LENGTH, '\0');
 
 	gettimeofday(&sstamp, 0);
-	while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XH:Me:k:u:a:")) != -1)
+	while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XH:Me:k:u:a:Z")) != -1)
 		switch(ch) {
 		case 'v':
 			version++;
@@ -471,6 +472,9 @@ main(int argc, char *argv[])
 			if (frangesize < 0)
 				usage();
 			break;
+		case 'Z':
+			zerofrange = 1;
+			break;
 		case 'X':
 			forcereads++;
 			break;
@@ -1176,6 +1180,20 @@ dumpskips(int verbose)
 
 #undef DOHISTO
 
+/*
+ * Zero the data associated with a free range that is being included
+ * in the image because its size is less than frangesize.
+ */
+static void
+zerofixup(void *bstart, off_t bsize, void *fdata)
+{
+	if (debug > 1)
+		fprintf(stderr, "zerofixup: zeroing %llu@%p\n",
+			(unsigned long long)bsize, bstart);
+
+	memset(bstart, 0, bsize);
+}
+
 /*
  * Sort and merge the list of skip blocks.
  * This code also winnows out the free ranges smaller than frangesize.
@@ -1209,6 +1227,12 @@ mergeskips(int verbose)
 						"dropping range [%u-%u]\n",
 						prange->start,
 						prange->start+prange->size-1);
+				if (zerofrange)
+					addfixupfunc(zerofixup,
+						     sectobytes(prange->start),
+						     0,
+						     sectobytes(prange->size),
+						     NULL, 0, RELOC_NONE);
 				total += prange->size;
 #ifdef DOHISTO
 				if (prange->size < 64)

rspec-geni @ 2fddf099

-Subproject commit 35408bf756041ab0664849993aeef02fd681e8c1
+Subproject commit 2fddf0990a12a0b33b02752fb77b3a79427cabad

tbsetup/libvtop_stable.pm.in

@@ -32,7 +32,6 @@ use Carp;
 use POSIX;
 use XML::LibXML;
 use XML::Simple;
-use GeniHRN;
 
 # Configure variables
 my $TB		  = "@prefix@";
@@ -45,7 +44,12 @@ my $DELAYCAPACITY = @DELAYCAPACITY@;	# Can be overridden by user.
 my $DELAYTHRESH   = @DELAYTHRESH@;
 my $PGENISUPPORT  = @PROTOGENI_SUPPORT@;
 my $OURDOMAIN     = "@OURDOMAIN@";
-my $mycmurn       = GeniHRN::Generate("@OURDOMAIN@", "authority", "cm");
+my $mycmurn;
+
+if ($PGENISUPPORT) {
+    require GeniHRN;
+    $mycmurn = GeniHRN::Generate("@OURDOMAIN@", "authority", "cm");
+}
 
 # Flags.
 $VTOP_FLAGS_VERBOSE	= 0x01;

tbsetup/mkproj.in

@@ -286,7 +286,7 @@ if ($MAILMANSUPPORT && !$isnonlocal) {
 }
 
 # Skip if user was already an active user; lots of work avoided.
-if (!$wasactive) {
+if (1 || !$wasactive) {
     system("$MKACCT $leader_uid") == 0 or
 	fatal("$MKACCT $leader_uid failed!");
 }

tbsetup/ptopgen.in

@@ -371,8 +371,6 @@ if ($PGENISUPPORT) {
   require GeniHRN;
 }
 
-use GeniHRN;
-
 sub usage()
 {
     print("Usage: ptopgen [-v] [-s switch] [-p pid [-e eid]] [-m factor] " .

tools/tspitests/script1.sh

 #!/bin/sh
-
+#
 # EMULAB-COPYRIGHT
-# Copyright (c) 2010 University of Utah and the Flux Group.
+# Copyright (c) 2010-2011 University of Utah and the Flux Group.
 # All rights reserved.
+#
 
 MP=`which modprobe`
 KILLALL=`which killall`
@@ -12,16 +13,35 @@ TMCC=`which tmcc`
 SLEEPTIME=1
 
 # Error check later
+echo "Doing TPM setup ..."
 /etc/testbed/rc/rc.tpmsetup
 
 ${MP} tpm_tis
 ${KILLALL} -9 tcsd; sleep ${SLEEPTIME}
 ${TCSD}
 
-SSCRUFT=`${TMCC} quoteprep RELOADSETUP | ${DOQ} RELOADSETUP`
-${TMCC} securestate ${SSCRUFT}
+echo "Requesting info for RELOADSETUP quote ..."
+QINFO=`${TMCC} quoteprep RELOADSETUP`
+if [ -z "$QINFO" ]; then
+    echo "*** could not get RELOADSETUP quote info"
+    exit 1
+fi
+
+echo "Preparing RELOADSETUP quote ..."
+SSCRUFT=`echo $QINFO | ${DOQ} RELOADSETUP`
+if [ -z "$SSCRUFT" ]; then
+    echo "*** could not produce RELOADSETUP quote"
+    exit 1
+fi
+
+echo "Sending RELOADSETUP quote ..."
+RC=`${TMCC} securestate ${SSCRUFT}`
+if [ $? -ne 0 -o "$RC" = "FAILED" ]; then
+    echo "*** could not transition to RELOADSETUP"
+    exit 1
+fi
 
 ${KILLALL} -9 tcsd; sleep ${SLEEPTIME}
 ${TCSD}
-${TMCC} -T imagekey > /tmp/secureloadinfo.out
 
+exit 0

tools/tspitests/script2.sh

 #!/bin/sh
-
+#
 # EMULAB-COPYRIGHT
-# Copyright (c) 2010 University of Utah and the Flux Group.
+# Copyright (c) 2010-2011 University of Utah and the Flux Group.
 # All rights reserved.
+#
 
 KILLALL=`which killall`
 TCSD=`which tcsd`
@@ -15,6 +16,7 @@ REBOOTPCR=15
 
 # Error check later
 
+echo "Setting sign-off PCR ..."
 ${KILLALL} -9 tcsd; sleep ${SLEEPTIME}
 ${TCSD}
 ${TPMS} ${REBOOTPCR}
@@ -22,6 +24,25 @@ ${TPMS} ${REBOOTPCR}
 ${KILLALL} -9 tcsd; sleep ${SLEEPTIME}
 ${TCSD}
 
-SSCRUFT=`${TMCC} quoteprep TPMSIGNOFF | ${DOQ} TPMSIGNOFF`
-${TMCC} securestate ${SSCRUFT}
-
+echo "Requesting info for TPMSIGNOFF quote ..."
+QINFO=`${TMCC} quoteprep TPMSIGNOFF`
+if [ -z "$QINFO" ]; then
+    echo "*** could not get TPMSIGNOFF quote info"
+    exit 1
+fi
+
+echo "Preparing TPMSIGNOFF quote ..."
+SSCRUFT=`echo $QINFO | ${DOQ} TPMSIGNOFF`
+if [ -z "$SSCRUFT" ]; then
+    echo "*** could not produce TPMSIGNOFF quote"
+    exit 1
+fi
+
+echo "Sending TPMSIGNOFF quote ..."
+RC=`${TMCC} securestate ${SSCRUFT}`
+if [ $? -ne 0 -o "$RC" = "FAILED" ]; then
+    echo "*** could not transition to TPMSIGNOFF"
+    exit 1
+fi
+
+exit 0

utils/create_image.in

@@ -703,6 +703,11 @@ sub check_progress($$)
 {
     my (undef, $statusp) = @_;
 
+    if ($runticks == 0) {
+	print "$node_id: started image capture, ".
+	    "waiting up to " . int($maxwait/60) . " minutes\n";
+    }
+
     #
     # XXX frisbee uploader uploads into a temporary file and then moves
     # it into place. So track that tmp file here.
@@ -717,7 +722,7 @@ sub check_progress($$)
     #
     if (defined($statusp) && $statusp->{$node_id} ne "none") {
 	$result = $statusp->{$node_id};
-	print "$node_id: image capture has completed\n";
+	print "$node_id: image capture has completed: status='$result'\n";
 	return 0;
     }
 
@@ -727,6 +732,7 @@ sub check_progress($$)
     $runticks++;
     if ($runticks >= $maxticks) {
 	$result = "timeout";
+	print "$node_id: image capture has completed: timeout\n";
 	return 0;
     }
 
@@ -749,12 +755,14 @@ sub check_progress($$)
     }
     if ($cursize > $maximagesize) {
 	$result = "toobig";
+	print "$node_id: image capture has completed: image too big\n";
 	return 0;
     }
     if ($cursize == $lastsize) {
 	$idleticks++;
 	if ($idleticks >= $maxidleticks) {
 	    $result = "timeout";
+	    print "$node_id: image capture has completed: idle timeout\n";
 	    return 0;
 	}
     } else {
@@ -764,7 +772,8 @@ sub check_progress($$)
     
     if (($runticks % $reportticks) == 0) {
 	my $curtdiff = int($runticks * $checkwait / 60);
-	print "Still waiting ... its been ". $curtdiff ." minutes.".
+	print "$node_id: still waiting ...".
+	    " it has been ". $curtdiff ." minutes.".
 	    " Current image size: $cursize bytes.\n";
     }