Begining support for off-line image delta creation.

New code for managing ranges.

clientside/os/imagezip/GNUmakefile.in

@@ -127,7 +127,7 @@ endif
 # Necessary sometimes.
 #PTHREADCFLAGS	+= -DCONDVARS_WORK
 
-PROGS		= imagezip imageunzip imagedump
+PROGS		= imagezip imageunzip imagedump imagedelta
 CFLAGS		= $(SUBDIRCFLAGS) -I$(SRCDIR) $(LDSTATIC)
 LIBS 		= -lz
 ZIPCFLAGS	= $(CFLAGS) -Wall
@@ -137,6 +137,7 @@ UNZIPLIBS	= $(LIBS) $(PTHREADLIBS)
 UNZIPDIRS	=
 HASHCFLAGS	= $(CFLAGS) $(PTHREADCFLAGS) -Wall
 HASHLIBS	= $(LIBS) -lcrypto $(PTHREADLIBS)
+NDZLIBS		= $(LIBS) libndz/libndz.a
 
 # Secure images
 ifeq ($(WITH_CRYPTO),1)
@@ -153,6 +154,8 @@ ifeq ($(WITH_V3COMPAT),1)
 CFLAGS		+= -DWITH_V3COMPAT
 endif
 
+SUBDIRS		+= libndz
+
 # MBR
 ifeq ($(WITH_MBR),1)
 CFLAGS		+= -DWITH_MBR
@@ -257,6 +260,12 @@ imagehash: imagehash.o version.o
 imagehash.o:	imagehash.c
 	$(CC) -c $(HASHCFLAGS) -o imagehash.o $<
 
+imagedelta: imagedelta.o version.o $(NDZLIBS)
+	$(CC) $(CFLAGS) imagedelta.o version.o $(NDZLIBS) -o imagedelta
+
+imagedelta.o:	imagedelta.c
+	$(CC) -c $(CFLAGS) -o imagedelta.o $<
+
 sizetest: disksize.c
 	$(CC) -DTEST $< -o sizetest
 
@@ -275,8 +284,9 @@ endif
 imagezip.o: sliceinfo.h imagehdr.h global.h range.h hashmap/hashmap.h
 imageunzip.o: imagehdr.h
 imagehash.o: imagehdr.h imagehash.h
+imagedelta.o: imagehdr.h libndz/libndz.h
 
-version.c: imagezip.c imageunzip.c imagedump.c
+version.c: imagezip.c imageunzip.c imagedump.c imagedelta.c
 	echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
 
 install: $(addprefix $(INSTALL_BINDIR)/, $(PROGS))

clientside/os/imagezip/imagedelta.c

+/*
+ * Copyright (c) 2000-2014 University of Utah and the Flux Group.
+ * 
+ * {{{EMULAB-LICENSE
+ * 
+ * This file is part of the Emulab network testbed software.
+ * 
+ * This file is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ * 
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+ * License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this file.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ * }}}
+ */
+
+#define FIXMAP_DEBUG
+
+/*
+ * imagedelta [ -S ] image1.ndz image2.ndz delta1to2.ndz
+ *
+ * Take two images (image1, image2) and produce a delta (delta1to2)
+ * based on the differences. The -S option says to use the signature
+ * files if possible to determine differences between the images.
+ * Otherwise we compare the corresponding areas of both images to
+ * determine if they are different.
+ *
+ * Note that order matters here! We are generating a delta to get from
+ * "image1" to "image2"; i.e., doing:
+ *
+ *  imageunzip image1.ndz /dev/da0
+ *  imageunzip delta1to2.ndz /dev/da0
+ *
+ * would be identical to:
+ *
+ *  imageunzip image2.ndz /dev/da0
+ *
+ * Approach:
+ *
+ * We scan the chunks headers of both images (image1, image2) to produce
+ * allocated range lists for both (R1, R2). We use these to produce a
+ * range list for the delta (RD) as follows.
+ *
+ * - Anything that is in R1 but not R2 does not go in RD.
+ * - Anything in R2 but not in R1 must go into RD.
+ * - For overlapping areas, we read and hash or compare both and,
+ *   if different, include in RD.
+ * - Using RD, select data from image2 that need to be read, decompressed
+ *   and then recompressed into the new image.
+ *
+ * There is the usual issue of dealing with the difference in granularity
+ * and alignment of ranges (arbitrary multiples of 512 byte) vs. hash
+ * blocks (64K byte), but that logic exists in imagezip today.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <zlib.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <openssl/sha.h>
+#include <openssl/md5.h>
+#ifndef NOTHREADS
+#include <pthread.h>
+#endif
+
+#include "imagehdr.h"
+#include "libndz/libndz.h"
+
+struct ifileinfo {
+    struct ndz_file *ndz;
+    int sigfd;
+    struct ndz_rangemap *map, *sigmap;
+} ndz1, ndz2;
+
+struct mergestate {
+    unsigned int ochunkno;
+    unsigned int nchunkno;
+};
+
+int usesigfiles = 0;
+
+static int fixmap(struct ndz_rangemap *map, struct ndz_range *range, void *arg);
+
+void
+usage(void)
+{
+    fprintf(stderr,
+	    "Usage: imagedelta [-S] image1.ndz image2.ndz delta1to2.ndz\n"
+	    "\n"
+	    "Produce a delta image (delta1to2) containing the changes\n"
+	    "necessary to get from image1 to image2.\n"
+	    "\n"
+	    "  -S   Use signature files when computing differences.\n");
+    exit(1);
+}
+
+/*
+ * File must exist and be readable.
+ * If usesigfiles is set, signature file must exist as well.
+ * Reads in the range map and signature as well.
+ */
+void
+openifile(char *file, struct ifileinfo *info)
+{
+    int rv;
+
+    info->ndz = ndz_open(file, 0);
+    if (info->ndz == NULL) {
+	fprintf(stderr, "%s: could not open as NDZ file\n",
+		ndz_filename(info->ndz));
+	exit(1);
+    }
+
+    /* XXX check sigfile */
+    if (usesigfiles) {
+	;
+    }
+
+    /* read range info from image */
+    rv = ndz_readranges(info->ndz, &info->map);
+    if (rv) {
+	fprintf(stderr, "%s: could not read ranges\n",
+		ndz_filename(info->ndz));
+	exit(1);
+    }
+
+    /* read signature info */
+    if (usesigfiles) {
+	;
+    }
+}
+
+static void
+chunkfunc(void *ptr)
+{
+    unsigned int chunkno = (int)ptr;
+    printf("chunkno=%u", chunkno);
+}
+
+static void
+twochunkfunc(void *ptr)
+{
+    struct mergestate *state = (struct mergestate *)ptr;
+    printf("ochunkno=%u, nchunkno=%u", state->ochunkno, state->nchunkno);
+}
+
+int
+main(int argc, char **argv)
+{
+    int ch, version = 0;
+    extern char build_info[];
+    int delta, rv;
+    struct ndz_rangemap *mmap;
+
+    while ((ch = getopt(argc, argv, "Sv")) != -1)
+	switch(ch) {
+	case 'S':
+	    usesigfiles = 1;
+	    break;
+	case 'v':
+	    version++;
+	    break;
+	case 'h':
+	case '?':
+	default:
+	    usage();
+	}
+    argc -= optind;
+    argv += optind;
+
+    if (argc < 3)
+	usage();
+
+    /*
+     * Make sure we can open all the files
+     */
+    ndz1.ndz = ndz_open(argv[0], 0);
+    if (ndz1.ndz == NULL)
+	usage();
+
+    ndz2.ndz = ndz_open(argv[1], 0);
+    if (ndz2.ndz == NULL)
+	usage();
+
+    delta = open(argv[2], O_RDWR|O_CREAT|O_TRUNC, 0666);
+    if (delta < 0) {
+	perror(argv[2]);
+	usage();
+    }
+
+    /*
+     * Read in the range and signature info.
+     */
+    ndz1.map = ndz_readranges(ndz1.ndz);
+    if (ndz1.map == NULL) {
+	fprintf(stderr, "%s: could not read ranges\n",
+		ndz_filename(ndz1.ndz));
+	exit(1);
+    }
+#if 1
+    printf("==== Old ");
+    ndz_rangemap_dump(ndz1.map, chunkfunc);
+    fflush(stdout);
+#endif
+    ndz2.map = ndz_readranges(ndz2.ndz);
+    if (ndz2.map == NULL) {
+	fprintf(stderr, "%s: could not read ranges\n",
+		ndz_filename(ndz2.ndz));
+	exit(1);
+    }
+#if 1
+    printf("==== New ");
+    ndz_rangemap_dump(ndz2.map, chunkfunc);
+    fflush(stdout);
+#endif
+
+    /*
+     * Roll through the new image map looking for overlaps with
+     * the old image.
+     */
+    mmap = ndz_rangemap_init(ndz2.map->loaddr, ndz2.map->hiaddr);
+    assert(mmap != NULL);
+
+    (void) ndz_rangemap_iterate(ndz2.map, fixmap, mmap);
+#if 1
+    printf("==== Merged ");
+    ndz_rangemap_dump(mmap, twochunkfunc);
+    fflush(stdout);
+#endif
+
+    return 0;
+}
+
+static int
+fixmap(struct ndz_rangemap *map, struct ndz_range *range, void *arg)
+{
+    struct ndz_rangemap *mmap = (struct ndz_rangemap *)arg;
+    ndz_addr_t addr, eaddr, oaddr, oeaddr;
+    struct ndz_range *orange, *oprev;
+    int rv;
+
+    addr = range->start;
+    eaddr = range->end;
+
+#ifdef FIXMAP_DEBUG
+    fprintf(stderr, "fixmap [%lu - %lu]:\n", addr, eaddr);
+#endif
+
+    /* Allocate a range in the merge map */
+    rv = ndz_rangemap_alloc(mmap, addr, eaddr-addr+1, NULL);
+    assert(rv == 0);
+
+    orange = ndz_rangemap_lookup(ndz1.map, addr, &oprev);
+
+    /* Current map address in not in old range */
+    if (orange == NULL) {
+	if (oprev == NULL)
+	    oprev = &ndz1.map->head;
+	/* Nothing more in the old map, we are done */
+	if ((orange = oprev->next) == NULL) {
+#ifdef FIXMAP_DEBUG
+	    fprintf(stderr, "  finished old map\n");
+#endif
+	    return 1;
+	}
+#ifdef FIXMAP_DEBUG
+	fprintf(stderr, "  found oldrange [%lu - %lu]:\n", orange->start, orange->end);
+#endif
+	/* No overlap at all with current map range, move on */
+	if (orange->start > eaddr) {
+#ifdef FIXMAP_DEBUG
+	    fprintf(stderr, "  no overlap in oldrange\n");
+#endif
+	    return 0;
+	}
+    }
+#ifdef FIXMAP_DEBUG
+    else
+	fprintf(stderr, "  found oldrange [%lu - %lu]:\n", orange->start, orange->end);
+#endif
+
+    while ((oaddr = orange->start) <= eaddr) {
+	struct mergestate *state;
+
+	/* Determine the extent of the overlap */
+	if (oaddr < addr)
+	    oaddr = addr;
+	if ((oeaddr = orange->end) > eaddr)
+	    oeaddr = eaddr;
+
+#ifdef FIXMAP_DEBUG
+	fprintf(stderr, "    found overlap [%lu - %lu]:\n", oaddr, oeaddr);
+#endif
+	/* Dealloc this part of the range and realloc with flag set */
+	rv = ndz_rangemap_dealloc(mmap, oaddr, oeaddr-oaddr+1);
+	assert(rv == 0);
+	state = calloc(1, sizeof *state);
+	assert(state);
+	state->ochunkno = (unsigned int)orange->data;
+	state->nchunkno = (unsigned int)range->data;
+	rv = ndz_rangemap_alloc(mmap, oaddr, oeaddr-oaddr+1, state);
+	assert(rv == 0);
+
+	/* on to the next old map entry */
+	orange = orange->next;
+
+	/* end of old map, we are done */
+	if (orange == NULL) {
+#ifdef FIXMAP_DEBUG
+	    fprintf(stderr, "  finished old map\n");
+#endif
+	    return 1;
+	}
+#ifdef FIXMAP_DEBUG
+	fprintf(stderr, "  next oldrange [%lu - %lu]:\n", orange->start, orange->end);
+#endif
+
+    }
+
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * End:
+ */
+

clientside/os/imagezip/imagehdr.h

 /*
- * Copyright (c) 2000-2014 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2015 University of Utah and the Flux Group.
  * 
  * {{{EMULAB-LICENSE
  * 
@@ -231,4 +231,5 @@ struct region {
  * Assumed sector (block) size
  */
 #define SECSIZE			512
+
 #endif /* _IMAGEHDR_H_ */

clientside/os/imagezip/imagezip.c

@@ -1333,6 +1333,16 @@ addskip(uint32_t start, uint32_t size)
 	skip->next  = skips;
 	skips       = skip;
 	numskips++;
+#if 1
+	{
+		static FILE *sd;
+		if (sd == NULL) {
+			sd = fopen("/tmp/skips", "w");
+			fprintf(sd, "%lu %lu\n", inputminsec, inputmaxsec);
+		}
+		fprintf(sd, "d %lu %lu\n", start, start + size - 1);
+	}
+#endif
 }
 
 /*

clientside/os/imagezip/libndz/GNUmakefile.in

+#
+# Copyright (c) 2000-2014 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+SRCDIR		= @srcdir@
+TESTBED_SRCDIR	= @top_srcdir@
+OBJDIR		= @top_builddir@
+SUBDIR		= $(subst $(TESTBED_SRCDIR)/,,$(SRCDIR))
+MAINDIR		= $(SRCDIR)/..
+
+include $(OBJDIR)/Makeconf
+
+CFLAGS		+= -DSTATS $(SUBDIRCFLAGS) -I$(MAINDIR) -I$(SRCDIR)
+
+all:	libndz.a
+
+include $(TESTBED_SRCDIR)/GNUmakerules
+
+OBJS = rangemap.o ndzfile.o ndzdata.o
+
+rangemap.o: rangemap.h
+ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
+ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
+
+libndz.a:	$(OBJS)
+	$(AR) $(ARFLAGS) $@ $?
+	$(RANLIB) $@
+
+install: 
+
+clean:
+	rm -f libndz.a $(OBJS)

clientside/os/imagezip/libndz/libndz.h

+/*
+ * Copyright (c) 2014 University of Utah and the Flux Group.
+ * 
+ * {{{EMULAB-LICENSE
+ * 
+ * This file is part of the Emulab network testbed software.
+ * 
+ * This file is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ * 
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+ * License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this file.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ * }}}
+ */
+
+#ifndef _LIBNDZ_H_
+#define	_LIBNDZ_H_
+
+#include "rangemap.h"
+
+typedef uint32_t ndz_chunk_t;
+
+struct ndz_file {
+    int fd;
+    char *fname;
+    int sectsize;
+    int chunksize;
+    ndz_chunk_t nchunks;
+    struct ndz_rangemap *rangemap;
+    /* per-chunk info to verify */
+    /* readahead cache stuff */
+};
+
+struct chunkmap {
+	ndz_addr_t start;
+	ndz_addr_t end;
+} *chunkmap;
+
+struct ndz_chunkhdr {
+    blockhdr_t *header;
+    struct region *region;
+    struct blockreloc *reloc;
+    char data[DEFAULTREGIONSIZE];
+};
+
+struct ndz_file *ndz_open(const char *name, int flags);
+int ndz_close(struct ndz_file *ndz);
+char *ndz_filename(struct ndz_file *ndz);
+ssize_t ndz_read(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset);
+int ndz_readahead(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset);
+
+int ndz_readchunkheader(struct ndz_file *ndz, ndz_chunk_t chunkno,
+			struct ndz_chunkhdr *chunkhdr);
+
+struct ndz_rangemap *ndz_readranges(struct ndz_file *ndz);
+void ndz_dumpranges(struct ndz_rangemap *map);
+
+#endif /* _LIBNDZ_H_ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * End:
+ */

clientside/os/imagezip/libndz/ndzdata.c

+/*
+ * Copyright (c) 2014 University of Utah and the Flux Group.
+ * 
+ * {{{EMULAB-LICENSE
+ * 
+ * This file is part of the Emulab network testbed software.
+ * 
+ * This file is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ * 
+ * This file is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+ * License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this file.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ * }}}
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <sys/stat.h>
+
+#include "../imagehdr.h"
+#include "libndz.h"
+
+static int init_chunkmap(struct ndz_file *ndz);
+static void dump_chunkmap(struct ndz_file *ndz);
+
+/*
+ * Read uncompessed data from an imagefile.
+ * Returns however many bytes of data we found in the image. 
+ */
+ssize_t
+ndz_readdata(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset)
+{
+    ndz_addr_t ssect, esect;
+    struct ndz_range *range;
+    ssize_t rbytes = 0;
+    ndz_chunk_t chunkno;
+
+    if (ndz->rangemap == NULL && ndz_readranges(ndz) == NULL) {
+	errno = EINVAL;
+	return -1;
+    }
+
+    assert(ndz->sectsize != 0);
+    ssect = offset / ndz->sectsize;
+    esect = (offset + bytes - 1) / ndz->sectsize;
+
+    while (ssect <= esect) {
+	range = ndz_rangemap_lookup(ndz->rangemap, ssect, NULL);
+	if (range == NULL)
+	    return rbytes;
+
+	chunkno = (ndz_chunkno)range->data;
+	assert(chunkno > 0);
+	chunkno--;
+
+	/* read the chunk */
+	/* decompress til we got what we want */
+
+	ssect = range->end + 1;
+    }
+
+    return 0;
+}
+
+#if 0
+static int
+init_chunkmap(struct ndz_file *ndz)
+{
+    struct stat sb;
+    ndz_chunk_t chunkno;
+
+    /*
+     * XXX for now we don't handle streaming an image (fd == stdin).
+     * We could do it, we would just have to construct the chunkmap
+     * on the fly.
+     */
+    if (fstat(ndz->fd, &sb) < 0) {
+	perror(ndz->fname);
+	return 1;
+    }
+
+    ndz->nchunks = (sb.st_size + ndz->chunksize - 1) / ndz->chunksize;
+    if (ndz->nchunks == 0)
+	return 0;
+
+    ndz->chunkmap = calloc(ndz->nchunks, sizeof(struct chunkmap));
+    if (ndz->chunkmap == NULL) {
+	fprintf(stderr, "%s: could not allocate chunkmap\n", ndz->fname);
+	ndz->nchunks = 0;
+	return 1;
+    }
+
+    for (chunkno = 0; chunkno < ndz->nchunks; chunkno++) {
+	struct ndz_chunkhdr head;
+	blockhdr_t *hdr;
+	struct region *reg;
+	ndz_addr_t lo, hi;
+	int i;
+
+	if (ndz_readchunkheader(ndz, chunkno, &head) != 0) {
+	    free(ndz->chunkmap);
+	    ndz->chunkmap = NULL;
+	    ndz->nchunks = 0;
+	    return 1;
+	}
+
+	/* null header pointer indicates EOF */
+	if ((hdr = head.header) == NULL) {
+	    fprintf(stderr, "%s: unexpected EOF!?\n", ndz->fname);
+	    free(ndz->chunkmap);
+	    ndz->chunkmap = NULL;
+	    ndz->nchunks = 0;
+	    return 1;
+	}
+
+	reg = head.region;
+	assert(reg != NULL || hdr->regioncount == 0);
+	lo = NDZ_HIADDR;
+	hi = NDZ_LOADDR;
+	for (i = 0; i < hdr->regioncount; i++) {
+	    if (reg->start < lo)
+		lo = reg->start;
+	    if (reg->start+reg->size-1 > hi)
+		hi = reg->start + reg->size - 1;
+	    reg++;
+	}
+	ndz->chunkmap[chunkno].start = lo;
+	ndz->chunkmap[chunkno].end = hi;
+    }
+
+    r