Merge in as-of-yet untestbed delta image code.

clientside/os/frisbee.redux/config_emulab.c

@@ -98,6 +98,26 @@ static int INELABINELAB  = ELABINELAB;
 static int INELABINELAB  = 0;
 #endif
 
+/*
+ * An Emulab image ID string looks like:
+ *    [<pid>]<imagename>[<vers>][<meta>]
+ * where:
+ *    <pid> is the project
+ *    <imagename> is the image identifier string
+ *    <vers> is an image version number (not yet implemented)
+ *    <meta> is a string indicating that this is not the actual image,
+ *           rather it is metadata file associated with the image.
+ *           By convention, the string is the filename extension used
+ *           for the metadata file in question. Currently, the only
+ *           metadata string is 'sig' indicating that this is an image
+ *           signature file.
+ * Each of these fields has a separator character distinguishing the
+ * start of the field. These are defined here.
+ */
+#define IID_SEP_NAME '/'
+#define IID_SEP_VERS ':'
+#define IID_SEP_META ','
+
 static uint64_t	put_maxsize = 10000000000ULL;	/* zero means no limit */
 static uint32_t put_maxwait = 2000;		/* zero means no limit */
 static uint32_t put_maxiwait = 120;		/* zero means no limit */
@@ -1037,6 +1057,7 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 	char		*node, *proxy, *role = NULL;
 	int		i, j, nrows;
 	char		*wantpid = NULL, *wantname = NULL, *wantvers = NULL;
+	char		*wantmeta = NULL;
 	struct config_host_authinfo *get = NULL, *put = NULL;
 	struct emulab_ha_extra_info *ei = NULL;
 	int		imageidx = 0, ngids, igids[2];
@@ -1308,7 +1329,7 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 	 */
 	if (imageid != NULL) {
 		wantpid = mystrdup(imageid);
-		wantname = index(wantpid, '/');
+		wantname = index(wantpid, IID_SEP_NAME);
 		if (wantname == NULL) {
 			wantname = wantpid;
 			wantpid = mystrdup("emulab-ops");
@@ -1316,10 +1337,15 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 			*wantname = '\0';
 			wantname = mystrdup(wantname+1);
 		}
-		wantvers = index(wantname, ':');
+		wantvers = index(wantname, IID_SEP_VERS);
 		if (wantvers) {
 			*wantvers = '\0';
-			wantvers  = wantvers+1;
+			wantvers  = mystrdup(wantvers+1);
+		}
+		wantmeta = index(wantvers ? wantvers : wantname, IID_SEP_META);
+		if (wantmeta != NULL) {
+			*wantmeta = '\0';
+			wantmeta = mystrdup(wantmeta+1);
 		}
 		imageidx = emulab_imageid(wantpid, wantname);
 		if (imageidx == 0)
@@ -1669,6 +1695,10 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
 		free(wantpid);
 	if (wantname)
 		free(wantname);
+	if (wantvers)
+		free(wantvers);
+	if (wantmeta)
+		free(wantmeta);
 	if (getp) *getp = get;
 	if (putp) *putp = put;
 	return 0;
@@ -1843,17 +1873,80 @@ emulab_canonicalize_imageid(char *path)
 {
 	MYSQL_RES *res;
 	MYSQL_ROW row;
-	char *iid = NULL;
+	char *iid = NULL, *ipath = NULL;
+	int len = strlen(path), vers;
 
-	res = mydb_query("SELECT CONCAT(pid,'/',imagename) FROM images"
-			 " WHERE path='%s'", 1, path);
-	if (res != NULL) {
-		if (mysql_num_rows(res) > 0) {
-			row = mysql_fetch_row(res);
-			if (row[0])
-				iid = strdup(row[0]);
-		}
+	/*
+	 * XXX right now we only use this to convert pathnames to imageids.
+	 * At some point maybe it will be necessary to convert between
+	 * imageids.
+	 */
+	if (path[0] != '/')
+		return mystrdup(path);
+
+	/*
+	 * XXX see if it might be a sigfile
+	 */
+	if (len > 4 && strcmp(path+len-4, ".sig") == 0) {
+		ipath = mystrdup(path);
+		ipath[len-4] = '\0';
+	}
+
+	/*
+	 * Try to do everything is one swell foop.
+	 * Look up the path in image_versions, returning the version number
+	 * and a string composed of <pid>/<imagename>:<version>. If the
+	 * version number is zero, we remove the ":<version>" part.
+	 */
+	res = mydb_query("SELECT "
+			 "  version,CONCAT(pid,'/',imagename,':',version) "
+			 "FROM image_versions"
+			 "  WHERE deleted IS NULL AND path='%s'",
+			 2, ipath ? ipath : path);
+	if (res == NULL) {
+		if (ipath)
+			free(ipath);
+		return NULL;
+	}
+	if (mysql_num_rows(res) == 0) {
+		mysql_free_result(res);
+		if (ipath)
+			free(ipath);
+		return NULL;
+	}
+
+	/* XXX if rows > 1, we just return info from the first */
+	row = mysql_fetch_row(res);
+	if (row[0] == NULL || row[1] == NULL) {
 		mysql_free_result(res);
+		if (ipath)
+			free(ipath);
+		return NULL;
+	}
+	vers = atoi(row[0]);
+	iid = mystrdup(row[1]);
+	mysql_free_result(res);
+	if (vers == 0) {
+		char *verstr = rindex(iid, ':');
+		*verstr = '\0';
+	}
+ 
+	/*
+	 * Tack on ",sig" to indicate that we want the signature for the
+	 * indicated image.
+	 */
+	if (ipath != NULL) {
+		if (iid != NULL) {
+			char *niid;
+			len = strlen(iid);
+			niid = mymalloc(len + 4);
+			strcpy(niid, iid);
+			niid[len++] = IID_SEP_META;
+			strcpy(&niid[len], "sig");
+			free(iid);
+			iid = niid;
+		}
+		free(ipath);
 	}
 
 	return iid;

clientside/os/frisbee.redux/configdefs.h

@@ -42,8 +42,9 @@ struct config_imageinfo {
 #define CONFIG_PATH_ISDIR	0x2	/* path is a directory */
 #define CONFIG_PATH_ISGLOB	0x4	/* path is a file glob */
 #define CONFIG_PATH_ISRE	0x8	/* path is a perl RE */
-#define CONFIG_PATH_RESOLVE	0x10	/* path needs resolution at use */
-#define CONFIG_PATH_EXISTS	0x20	/* imaged named by path arg exists */
+#define CONFIG_PATH_ISSIGFILE	0x10	/* path is an image sigfile */
+#define CONFIG_PATH_RESOLVE	0x20	/* path needs resolution at use */
+#define CONFIG_PATH_EXISTS	0x40	/* imaged named by path arg exists */
 #define CONFIG_SIG_ISMTIME	0x1000	/* sig is path mtime */
 #define CONFIG_SIG_ISMD5	0x2000	/* sig is MD5 hash of path */
 #define CONFIG_SIG_ISSHA1	0x4000	/* sig is SHA1 hash of path */

clientside/os/frisbee.redux/mserver.c

@@ -732,7 +732,7 @@ handle_get(int sock, struct sockaddr_in *sip, struct sockaddr_in *cip,
 	}
 	ii = copy_imageinfo(&ai->imageinfo[0]);
 	config_free_host_authinfo(ai);		
-	assert((ii->flags & CONFIG_PATH_ISFILE) != 0);
+	assert((ii->flags & (CONFIG_PATH_ISFILE|CONFIG_PATH_ISSIGFILE)) != 0);
 
 	/*
 	 * If the image is currently being uploaded, return TRYAGAIN.
@@ -1167,7 +1167,7 @@ handle_put(int sock, struct sockaddr_in *sip, struct sockaddr_in *cip,
 	}
 	ii = copy_imageinfo(&ai->imageinfo[0]);
 	config_free_host_authinfo(ai);		
-	assert((ii->flags & CONFIG_PATH_ISFILE) != 0);
+	assert((ii->flags & (CONFIG_PATH_ISFILE|CONFIG_PATH_ISSIGFILE)) != 0);
 
 	/*
 	 * If they gave us a size and it exceeds the maxsize, return an error.

db/Image.pm.in

@@ -1843,5 +1843,15 @@ sub Parent($)
     return Image->Lookup($self->parent_imageid(), $self->parent_version());
 }
 
+sub SetDelta($$)
+{
+    my ($self, $delta) = @_;
+
+    return -1
+	if (! $self->Update({'isdelta' => $delta}));
+
+    return 0;
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

utils/clone_image.in

@@ -235,9 +235,10 @@ if (defined($image)) {
 	my $needclone = 1;
 	
 	#
-	# Does the most version in the table not have its ready bit set? If so
-	# it means something went wrong with a previous image creation. We can
-	# reuse it, but reset the provenance just in case the node got reloaded.
+	# Does the most recent version in the table not have its ready bit set?
+	# If so it means something went wrong with a previous image creation.
+	# We can reuse it, but reset the provenance just in case the node got
+	# reloaded.
 	#
 	if (!$image->ready()) {
 	    my $osinfo = OSinfo->Lookup($image->imageid(), $image->version());
@@ -288,7 +289,7 @@ if (defined($image)) {
 	    # officially "released". We *can* use this version of the image
 	    # by explicitly using its version number, before it is released. 
 	    #
-	    if ($image->path() =~ /^\/usr\/testbed/) {
+	    if ($image->path() =~ /^$TB/) {
 		my $path = $PROJROOT . "/" . $image->pid() . "/images/" .
 		    basename($image->path());
 

utils/create_image.in

@@ -31,24 +31,35 @@ use Cwd qw(realpath);
 #
 # Image Creation Tuneables.
 #
-# $maxwait	max wall clock time to allow, progress or not
+# $maxwait	Max total wall clock time to allow for image collection.
+#		We abort after this length of time even if we are still
+#		actively making progress collecting the image.
 #		Empirically we have observed about 1.6MB/sec on a pc850
 #		for a Windows image (the slowest to create), so figuring
 #		1.5MB/sec for a 6GB max image works out to around 72 minutes.
-# $idlewait	max time to wait between periods of progress
-# $checkwait	time between progress checks (must be int div of $idlewait)
-# $reportwait	time between progress reports (must be multiple of $checkwait)
+#		This value comes from sitevar images/create/maxwait if set.
 #
-# $maximagesize	max size in bytes of an image.  This should really be in the
-#		DB (per-testbed, per-project, per-user, per-something), and
-#		not hardwired here.  In the meantime, we set this big and let
-#		disk quotas do the dirty work of limiting size.
+# $idlewait	Max time to allow between periods of progress.
+#		This value ensures that if we get stuck and stop making
+#		progress, we don't have to wait the potentially very long
+#		time til the $maxwait time expires to notice and abort.
+#		This value comes from sitevar images/create/idlewait if set.
+#
+# $checkwait	Time between progress checks (must be int div of $idlewait)
+#		Hardwired here, does not come from DB.
+#
+# $reportwait	Time between progress reports (must be multiple of $checkwait)
+#		Hardwired here, does not come from DB.
+#
+# $maximagesize	Max size in bytes of an image.  Currently this is site-wide
+#		and comes from sitevar images/create/maxsize if set. It should
+#		probably be finer-grained (per-project? per-user?) than that.
 #
 my $maxwait      = (72 * 60);
 my $idlewait     = ( 8 * 60);
 my $reportwait   = ( 2 * 60);
 my $checkwait    = 15;
-my $maximagesize = (6 * 1024**3); # 20GB
+my $maximagesize = (6 * 1024**3);
 
 #
 # Create a disk image.
@@ -64,17 +75,23 @@ sub usage()
 	  "-w          - wait for image to be fully created\n".
 	  "-s          - use ssh instead of frisbee uploader\n".
 	  "-N          - use NFS (if available) instead of frisbee uploader\n".
+	  "-D          - create a 'delta' image rather than a full image\n".
+	  "-S          - create a signature file for the new image\n".
+	  "-M          - do not boot info MFS, run with ssh from current OS\n".
 	  "-p <pid>    - project ID of the image; defaults to system project\n".
 	  "<imagename> - imagename to use\n".
 	  "<node>      - nodeid to create the image from\n");
     exit(-1);
 }
-my $optlist  = "p:wsNdfe";
+my $optlist  = "p:wsNdfeDSM";
 my $waitmode = 0;
 my $usessh   = 0;
 my $usenfs   = 0;
 my $usefup   = 1;
 my $noemail  = 0;
+my $delta    = 0;
+my $nomfs    = 0;
+my $signature= 0;
 my $webtask;
 
 #
@@ -86,8 +103,8 @@ my $TBLOGS      = "@TBLOGSEMAIL@";
 my $BOSSIP	= "@BOSSNODE_IP@";
 my $CONTROL     = "@USERNODE@";
 my $NONFS	= @NOSHAREDFS@;
-my $DOPROVENANCE= @IMAGEPROVENANCE@;
-my $doprovenance= 0;
+my $WITHPROVENANCE= @IMAGEPROVENANCE@;
+my $WITHDELTAS  = @IMAGEDELTAS@;
 
 #
 # Testbed Support libraries
@@ -132,7 +149,8 @@ sub check_progress($$);
 sub run_with_ssh($$);
 
 my $nodereboot	= "$TB/bin/node_reboot";
-my $createimage = "/usr/local/bin/create-image";
+my $createimage = "/usr/local/bin/create-versioned-image";
+my $ocreateimage= "/usr/local/bin/create-image";
 my $reboot_prep = "@CLIENT_BINDIR@/reboot_prepare";
 my $EC2SNAP     = "$TB/sbin/ec2import.proxy";
 my $friskiller  = "$TB/sbin/frisbeehelper";
@@ -162,6 +180,7 @@ my $didbackup   = 0;
 my $node_id;
 my $node;
 my ($experiment,$pid);
+my $doprovenance = 0;
 
 #
 # Parse command arguments. Once we return from getopts, all that should be
@@ -197,6 +216,24 @@ if (defined($options{"f"})) {
     $foreground = 1;
     $waitmode = 0;
 }
+if (defined($options{"D"})) {
+    if (!$WITHDELTAS) {
+	print STDERR "Delta image support not enabled\n";
+	exit(1);
+    }
+    $delta = 1;
+}
+if (defined($options{"S"})) {
+    if (!$WITHDELTAS) {
+	print STDERR "Delta image support not enabled\n";
+	exit(1);
+    }
+    $signature = 1;
+}
+if (defined($options{"M"})) {
+    $nomfs = 1;
+    $usessh = 1;
+}
 if (@ARGV != 2) {
     usage();
 }
@@ -297,6 +334,34 @@ if ($mereuser &&
 	"    You do not have permission to use imageid $imageid!\n");
 }
 
+#
+# See if per-project/per-user provenance feature is set.
+#
+if ($WITHPROVENANCE) {
+    my $project = Project->Lookup($image->pid());
+    if (!defined($project)) {
+	die("*** $0:\n".
+	    "    Could not lookup project for $image\n");
+    }
+    $doprovenance = EmulabFeatures->FeatureEnabled("ImageProvenance",
+						   $this_user, $project);
+}
+
+#
+# When provenance is enabled and we have delta support, we always collect
+# signatures and we always try to create deltas. Note that we set them to
+# a different non-zero value so we can distinguish this case and not print
+# various warnings below.
+#
+if ($doprovenance && $WITHDELTAS) {
+    $delta = 2
+	if ($delta == 0);
+    $signature = 2
+	if ($signature == 0);
+}
+
+my ($srcsigfile, $srcimage, $dstsigfile);
+
 #
 # Is it a local node or a remote EC2 node (need to generalize). 
 #
@@ -309,6 +374,16 @@ if ($target =~ /^.*@.*$/) {
 	    "    Bad data in $target\n");
     }
 
+    if ($delta || $signature) {
+	# Only warn if they explicitly specified an option
+	if ($delta == 1 || $signature == 1) {
+	    print STDERR
+		"*** WARNING: don't support delta imaging of EC2 images, ".
+		"ignoring delta/signature options.\n";
+	}
+	$delta = $signature = 0;
+    }
+
     $isec2node = 1;
     $usefup = $usessh = 0;
     $pid = $image->pid();
@@ -349,6 +424,67 @@ else {
     }
     $pid = $experiment->pid();
 
+    #
+    # If we are creating a delta image, figure out what image we are
+    # deriving from so that we can grab the signature.
+    #
+    if ($delta) {
+	#
+	# Find the source image we are creating a delta from. When provenance
+	# is enabled, we can use the parent image. If not enabled, we attempt
+	# to determine what is already on the node via the partitions table.
+	#
+	# If we cannot determine the source, we just warn and create a full
+	# image instead.
+	#
+	if ($doprovenance) {
+	    $srcimage = $image->Parent();
+	} else {
+	    (undef, $srcimage) = $node->RunningOsImage();
+	}
+	if (defined($srcimage)) {
+	    $srcsigfile = $srcimage->path() . ".sig";
+	    if (! -e "$srcsigfile") {
+		# XXX user may not have direct access to a shared image
+		my $SAVEUID = $UID;
+		$EUID = $UID = 0;
+		if (! -e "$srcsigfile") {
+		    $srcsigfile = undef;
+		}
+		$EUID = $UID = $SAVEUID;
+	    }
+	    if (!defined($srcsigfile)) {
+		if ($delta == 1) {
+		    print "*** WARNING: no signature file for source image, ".
+			"cannot create delta; creating full image instead.\n";
+		}
+		$delta = 0;
+	    }
+	} else {
+	    if ($delta == 1) {
+		print "*** WARNING: no source for image, ".
+		    "cannot create delta; creating full image instead.\n";
+	    }
+	    $delta = 0;
+	}
+    }
+
+    #
+    # If we are creating a signature file for this image, look up the path
+    # so we derive the signature file name.
+    #
+    if ($signature) {
+	if (defined($image->path())) {
+	    $dstsigfile = $image->path() . ".sig";
+	} else {
+	    if ($signature == 1) {
+		print "*** WARNING: no path for image, ".
+		    "cannot create signature file.\n";
+	    }
+	    $signature = 0;
+	}
+    }
+
     #
     # To avoid blowing a cavernous hole ("allow all TCP ports to boss")
     # in the per-experiment firewall, we don't use the frisbee uploader if
@@ -366,15 +502,6 @@ else {
 	}
     }
 }
-# Need this for feature test below.
-my $project = Project->Lookup($image->pid());
-if (!defined($project)) {
-    die("*** $0:\n".
-	"    Could not lookup project for $image\n");
-}
-# Feature override. 
-$doprovenance = 
-    EmulabFeatures->FeatureEnabled("ImageProvenance", $this_user, $project);
 
 #
 # Make sure that the directory exists and is writeable for the user.
@@ -386,20 +513,50 @@ my $usepath  = 0;
 
 #
 # Redirect pathname for global images. See equiv code in clone_image.
+# XXX if the project of the experiment creating the image is not emulab-ops,
+# we make a feeble attempt to avoid clobbering existing files.
 #
-if ($isglobal && ($filename =~ /^\/usr\/testbed/)) {
-    $filename = PROJROOT() . "/$pid/images/" . basename($filename);
+my $hackprefix;
+if ($pid eq TBOPSPID()) {
+    $hackprefix = PROJROOT() . "/$pid/images/";
+} else {
+    $hackprefix = PROJROOT() . "/$pid/images/E_O_";
+}
+if ($isglobal && ($filename =~ /^$TB/)) {
+    $filename = $hackprefix . basename($filename);
     print "*** WARNING: Writing global descriptor to $filename instead!\n";
+
+    #
+    # Ditto for the signature file
+    #
+    if ($dstsigfile && ($dstsigfile =~ /^$TB/)) {
+	$dstsigfile = $hackprefix . basename($dstsigfile);
+    }
+
     #
     # XXX the Emulab config of the master server doesn't know this trick
     # so when it tries to lookup imageid emulab-ops/<whatever> it would
     # still map to /usr/testbed and fail because it cannot update images
-    # outside of /{users,grouop,proj}. So we skirt the issue by passing
+    # outside of /{users,group,proj}. So we skirt the issue by passing
     # it the full path contructed here rather than the imageid.
     #
     $usepath = 1;
 }
 
+#
+# For the source signature file, we actually have to copy it to
+# somewhere where we can read it as well as just fixup the path.
+#
+if ($srcsigfile && ($srcsigfile =~ /^$TB/)) {
+    my $osrcsigfile = $srcsigfile;
+    $srcsigfile = $hackprefix . basename($srcsigfile);
+    if (system("cp -fp $osrcsigfile $srcsigfile")) {
+	die("*** $0:\n".
+	    "    Could not copy source signature file ".
+	    "$osrcsigfile to $srcsigfile\n");
+    }
+}
+
 #
 # Make sure real path is someplace that makes sense; remember that the
 # image is created on the nodes, and it NFS mounts directories on ops.
@@ -440,7 +597,7 @@ if ($image->Lock()) {
 }
 $needunlock = 1;
 
-if ($DOPROVENANCE && $doprovenance && $image->ready()) {
+if ($doprovenance && $image->ready()) {
     $image->Unlock();
     die("*** $0:\n".
 	"    $image ready flag is set, this is inconsistent!\n");
@@ -506,16 +663,6 @@ if (! ($isvirtnode || $isec2node)) {
     $device = "/dev/${devtype}${devnum}";
 }
 
-#
-# Record when this image was updated, so that we can figure out which
-# revision of the testbed image it was based off.
-#
-# Makes no sense to do this when writing a global image to a different path.
-# We need a better way to make new images live.
-#
-$image->MarkUpdate($this_user) == 0 or
-    fatal("Could not mark the update time in $image");
-    
 #
 # Okay, we want to build up a command line that will run the script on
 # on the client node. We use the imageid description to determine what
@@ -524,18 +671,31 @@ $image->MarkUpdate($this_user) == 0 or
 #
 my $startslice;
 my $loadlength;
-my $command    = "$createimage ";
-
-if ($usefup) {
-    my $id = $usepath ? $filename :
-	$image->pid() . "/$imagename" . ($version > 0 ? ":${version}" : "");
-    $command .= " -S $BOSSIP -F $id";
-}
+my $command;
 
+#
+# EC2 images use a special command which is hardwired below.
+#
 if ($isec2node) {
-    $command = "$EC2SNAP ";
+    ;
 }
+#
+# Virtnode images use a version of the old create-image script on the vhost
+#
 elsif ($isvirtnode) {
+    $command = "$ocreateimage";
+    if ($usefup) {
+	my $id;
+	if ($usepath) {
+	    $id = $filename;
+	} elsif ($version > 0) {
+	    $id = $image->pid() . "/$imagename:$version";
+	} else {
+	    $id = $image->pid() . "/$imagename";
+	}
+	$command .= " -S $BOSSIP -F $id";
+    }
+
     #
     # Need to know this is a xen-host to tailor options.
     #
@@ -557,21 +717,65 @@ elsif ($isvirtnode) {
 	}
     }
     $command .= " $node_id";
+
+    if ($usefup || $usessh) {
+	$command .= " -";
+    } else {
+	$command .= " $filename";
+    }
 }
+#
+# Regular nodes use the new script with different argument syntax.
+#
 else {
+    $command = "$createimage";
+
+    my $id;
+    if ($usefup) {
+	$command .= " METHOD=frisbee SERVER=$BOSSIP";
+	if ($usepath) {