Deal with relocations.

That was more involved than I thought...or it was worth, considering that
only old FreeBSD and really old Redhat (think: LILO) images have them.
But that is the kinda guy I am...

clientside/os/imagezip/imagedelta.c

@@ -528,6 +528,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 		    cstate->header->firstsect = 0;
 		cstate->header->lastsect = pstart;
 
+		/* include any relocations */
+		if (ndz2.ndz->relocmap) {
+		    void *buf = (cstate->curregion + 1);
+		    delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
+		    if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
+			delta.ndz->relocmap = NULL; /* XXX */
+			fprintf(stderr, "Error writing relocation info\n");
+			return 1;
+		    }
+		    delta.ndz->relocmap = NULL; /* XXX */
+		}
+
 		/* and write it */
 		if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
 		    fprintf(stderr, "Error writing compressed data\n");
@@ -634,10 +646,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 	/* finalize the header */
 	cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
 	cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
-
-	/* XXX */
+	/* XXX should always be zero */
+	if (cstate->chunkno == 0)
+	    cstate->header->firstsect = 0;
 	cstate->header->lastsect = delta.ndz->maphi;
 
+	/* include any relocations */
+	if (ndz2.ndz->relocmap) {
+	    void *buf = (cstate->curregion + 1);
+	    delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
+	    if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
+		delta.ndz->relocmap = NULL; /* XXX */
+		fprintf(stderr, "Error writing relocation info\n");
+		return 1;
+	    }
+	    delta.ndz->relocmap = NULL; /* XXX */
+	}
+
 	/* and write it */
 	if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
 	    fprintf(stderr, "Error writing compressed data\n");
@@ -757,7 +782,7 @@ main(int argc, char **argv)
 		    argv[1], ndz2.ndz->hashtype, ndz2.ndz->hashblksize);
 	    exit(1);
 	}
-	delta.map = ndz_compute_delta_sigmap(ndz1.sigmap, ndz2.sigmap);
+	delta.map = ndz_compute_delta_sigmap(ndz1.ndz, ndz2.ndz);
 	if (delta.map == NULL) {
 	    fprintf(stderr, "Could not compute delta for %s and %s\n",
 		    argv[0], argv[1]);
@@ -785,7 +810,7 @@ main(int argc, char **argv)
      * Same deal, but construct the delta map from the ranges maps.
      */
     else {
-	delta.map = ndz_compute_delta(ndz1.map, ndz2.map);
+	delta.map = ndz_compute_delta(ndz1.ndz, ndz2.ndz);
 	if (delta.map == NULL) {
 	    fprintf(stderr, "Could not compute delta for %s and %s\n",
 		    argv[0], argv[1]);
@@ -862,7 +887,6 @@ main(int argc, char **argv)
 	free(cstate);
 
 	/* readjust to reflect the actual number of hash entries */
-	if (!fullsig)
 	delta.ndz->hashentries = delta.ndz->hashcurentry;
 
 	/* write the new sigfile */
@@ -875,7 +899,8 @@ main(int argc, char **argv)
 	ndz_close(ndz2.ndz);
 	ndz_close(delta.ndz);
     } else {
-	fprintf(stderr, "Images are identical, no delta produced!\n");
+	fprintf(stderr, "Images %s and %s are identical, no delta produced!\n",
+		argv[0], argv[1]);
 	ndz_close(ndz2.ndz);
 	ndz_close(delta.ndz);
 	unlink(argv[2]);

clientside/os/imagezip/imagehdr.h

@@ -133,6 +133,33 @@ struct blockhdr_V4 {
 					/* Unique ID for the whole image */
 };
 
+/*
+ * Coming soon in V5:
+ *
+ * 64-bit support.
+ *   Grow blockindex, firstsect, lastsect, region descriptors, may need to
+ *   grow DEFAULTREGIONSIZE to accomodate the doubling in size of regions.
+ *
+ * Flag field?
+ *   For example, to indicate a delta image. Would probably take over the
+ *   otherwise unused blocktotal field.
+ *
+ * Sectorsize field?
+ *   To make explicit the units of sector fields; e.g., 512 vs 4096.
+ *
+ * Chunksize field?
+ *   To support different chunksizes.
+ *
+ * Mandate little-endian on-disk data.
+ *   Code changes only to use appropriate endian macros when reading/writing
+ *   data. No data struct changes needed.
+ *
+ * Support for SHA256 and SHA512 checksums.
+ *   Just some constants here, as checksum is already 64 bytes.
+ *   Will need to grow the imagehash header to accomodate these checksums
+ *   for signatures, but that is separately versioned.
+ */
+
 /*
  * Checksum types supported
  */

clientside/os/imagezip/imagerezip.c

@@ -21,7 +21,7 @@
  * }}}
  */
 
-#define CHUNKIFY_DEBUG
+//#define CHUNKIFY_DEBUG
 
 /*
  * imagerezip [-S] Oimage.ndz Nimage.ndz
@@ -271,7 +271,7 @@ readifile(struct fileinfo *info)
 			    next->start, next->end);
 		fprintf(stderr, "%s: error while validating range/hash maps\n",
 			ndz_filename(info->ndz));
-#if 1
+#if 0
 		printf("==== Image ");
 		ndz_rangemap_dump(info->map, (debug==0), NULL);
 		printf("==== Hash ");
@@ -503,6 +503,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 		    cstate->header->firstsect = 0;
 		cstate->header->lastsect = pstart;
 
+		/* include any relocations */
+		if (old.ndz->relocmap) {
+		    void *buf = (cstate->curregion + 1);
+		    new.ndz->relocmap = old.ndz->relocmap; /* XXX */
+		    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
+			new.ndz->relocmap = NULL; /* XXX */
+			fprintf(stderr, "Error writing relocation info\n");
+			return 1;
+		    }
+		    new.ndz->relocmap = NULL; /* XXX */
+		}
+
 		/* and write it */
 		if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
 		    fprintf(stderr, "Error writing compressed data\n");
@@ -612,10 +624,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 	/* finalize the header */
 	cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
 	cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
-
-	/* XXX */
+	/* XXX should always be zero */
+	if (cstate->chunkno == 0)
+	    cstate->header->firstsect = 0;
 	cstate->header->lastsect = new.ndz->maphi;
 
+	/* include any relocations */
+	if (old.ndz->relocmap) {
+	    void *buf = (cstate->curregion + 1);
+	    new.ndz->relocmap = old.ndz->relocmap; /* XXX */
+	    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
+		new.ndz->relocmap = NULL; /* XXX */
+		fprintf(stderr, "Error writing relocation info\n");
+		return 1;
+	    }
+	    new.ndz->relocmap = NULL; /* XXX */
+	}
+
 	/* and write it */
 	if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
 	    fprintf(stderr, "Error writing compressed data\n");
@@ -732,11 +757,13 @@ main(int argc, char **argv)
      */
     readifile(&old);
 
-#if 1
+#if 0
     printf("==== Old range ");
     ndz_rangemap_dump(old.map, (debug==0), chunkfunc);
+    if (old.sigmap) {
 	printf("==== Old hash ");
 	ndz_hashmap_dump(old.sigmap, (debug==0));
+    }
     fflush(stdout);
 #endif
 

clientside/os/imagezip/libndz/GNUmakefile.in

@@ -34,13 +34,14 @@ all:	libndz.a
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
-OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o
+OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o reloc.o
 
 rangemap.o: rangemap.h
 ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
 ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
 chunk.o: libndz.h $(MAINDIR)/imagehdr.h
 hash.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
+reloc.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
 
 libndz.a:	$(OBJS)
 	$(AR) $(ARFLAGS) $@ $?

clientside/os/imagezip/libndz/hash.c

@@ -348,6 +348,7 @@ ndz_freehashmap(struct ndz_file *ndz)
 struct deltainfo {
     struct ndz_rangemap *omap;
     struct ndz_rangemap *dmap;
+    struct ndz_rangemap *relocmap;
     int omapdone;
 };
 
@@ -441,6 +442,21 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
 #ifdef COMPDELTA_DEBUG
 	    fprintf(stderr, "  exact overlap with old map, ");
 #endif
+
+	    /*
+	     * XXX if the sector range includes a relocation,
+	     * we always force it into the image.
+	     */
+	    if (dinfo->relocmap &&
+		ndz_rangemap_overlap(dinfo->relocmap, addr, eaddr-addr+1)) {
+#ifdef COMPDELTA_DEBUG
+		fprintf(stderr, "has relocation, adding\n");
+#endif
+		rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
+		assert(rv == 0);
+		return 0;
+	    }
+
 	    odata = orange->data;
 	    ndata = range->data;
 	    assert(odata->hashlen == ndata->hashlen);
@@ -450,11 +466,11 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
 #endif
 		rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
 		assert(rv == 0);
-	    } else {
+		return 0;
+	    }
 #ifdef COMPDELTA_DEBUG
 	    fprintf(stderr, "hash same, skipping\n");
 #endif
-	    }
 	    return 0;
 	}
     }
@@ -475,11 +491,15 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
 }
 
 struct ndz_rangemap *
-ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
+ndz_compute_delta(struct ndz_file *ondz, struct ndz_file *nndz)
 {
-    struct ndz_rangemap *dmap;
+    struct ndz_rangemap *omap, *nmap, *dmap;
     struct deltainfo dinfo;
 
+    if (ondz == NULL || (omap = ondz->rangemap) == NULL ||
+	nndz == NULL || (nmap = nndz->rangemap) == NULL)
+	return NULL;
+
     if (omap == NULL || nmap == NULL)
 	return NULL;
 
@@ -491,6 +511,7 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
 
     dinfo.omap = omap;
     dinfo.dmap = dmap;
+    dinfo.relocmap = nndz->relocmap;
     dinfo.omapdone = 0;
 #if 0
     (void) ndz_rangemap_iterate(nmap, compdelta, &dinfo);
@@ -504,12 +525,13 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
  * computing hash alignments.
  */
 struct ndz_rangemap *
-ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
+ndz_compute_delta_sigmap(struct ndz_file *ondz, struct ndz_file *nndz)
 {
-    struct ndz_rangemap *dmap;
+    struct ndz_rangemap *omap, *nmap, *dmap;
     struct deltainfo dinfo;
 
-    if (omap == NULL || nmap == NULL)
+    if (ondz == NULL || (omap = ondz->hashmap) == NULL ||
+	nndz == NULL || (nmap = nndz->hashmap) == NULL)
 	return NULL;
 
     dmap = ndz_rangemap_init(nmap->loaddr, nmap->hiaddr);
@@ -520,6 +542,7 @@ ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
 
     dinfo.omap = omap;
     dinfo.dmap = dmap;
+    dinfo.relocmap = nndz->relocmap;
     dinfo.omapdone = 0;
     (void) ndz_rangemap_iterate(nmap, compfastdelta, &dinfo);
 

clientside/os/imagezip/libndz/libndz.h

@@ -54,6 +54,10 @@ struct ndz_file {
     unsigned chunkuses;
     unsigned chunkhits;
     unsigned chunkreopens;
+    /* relocation information */
+    unsigned relocentries;
+    void *relocdata;
+    struct ndz_rangemap *relocmap;
     /* hash (signature) info */
     unsigned hashtype;
     unsigned hashblksize;
@@ -103,16 +107,20 @@ int ndz_chunk_flush(ndz_chunk_t chobj, int withheader);
 ssize_t ndz_chunk_left(ndz_chunk_t chobj);
 ssize_t ndz_chunk_append(ndz_chunk_t chobj, void *buf, size_t bytes);
 
+int ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
+int ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
+void ndz_reloc_free(struct ndz_file *ndz);
+
 struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);
 int ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile);
 void ndz_hash_data(struct ndz_file *ndz, unsigned char *data,
 		   unsigned long count, unsigned char *hash);
 char *ndz_hash_dump(unsigned char *h, int hlen);
 void ndz_hashmap_dump(struct ndz_rangemap *map, int summaryonly);
-struct ndz_rangemap *ndz_compute_delta(struct ndz_rangemap *omap,
-				       struct ndz_rangemap *nmap);
-struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_rangemap *omap,
-					      struct ndz_rangemap *nmap);
+struct ndz_rangemap *ndz_compute_delta(struct ndz_file *ondz,
+				       struct ndz_file *nndz);
+struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_file *ondz,
+					      struct ndz_file *nndz);
 
 #endif /* _LIBNDZ_H_ */
 

clientside/os/imagezip/libndz/ndzfile.c

@@ -115,6 +115,7 @@ ndz_close(struct ndz_file *ndz)
 
     rv = close(ndz->fd);
     if (rv == 0) {
+	ndz_reloc_free(ndz);
 	if (ndz->hashmap)
 	    ndz_rangemap_deinit(ndz->hashmap);
 	if (ndz->rangemap)
@@ -232,6 +233,7 @@ ndz_readranges(struct ndz_file *ndz)
 	    last = hdr->lastsect;
 	}
 
+	/* get the regions */
 	reg = head.region;
 	assert(reg != NULL || hdr->regioncount == 0);
 	for (i = 0; i < hdr->regioncount; i++) {
@@ -254,6 +256,26 @@ ndz_readranges(struct ndz_file *ndz)
 	    }
 	    reg++;
 	}
+
+	/* get the relocations */
+	if (head.reloc) {
+	    if (ndz->relocmap == NULL) {
+		ndz->relocmap =
+		    ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
+		if (ndz->relocmap == NULL) {
+		    fprintf(stderr, "%s: could not allocate relocmap\n",
+			    ndz->fname);
+		    ndz_rangemap_deinit(map);
+		    return NULL;
+		}
+	    }
+	    if (ndz_reloc_get(ndz, head.header, head.reloc) != 0) {
+		fprintf(stderr, "%s: could not add relocations\n",
+			ndz->fname);
+		ndz_rangemap_deinit(map);
+		return NULL;
+	    }
+	}
     }
 
     ndz->maplo = first;

clientside/os/imagezip/libndz/rangemap.c

@@ -303,6 +303,8 @@ ndz_rangemap_iterate(struct ndz_rangemap *map,
 /*
  * Return a pointer to the first entry in the map.
  * Returns NULL if map is empty.
+ *
+ * Does not affect the map hint.
  */
 struct ndz_range *
 ndz_rangemap_first(struct ndz_rangemap *map)
@@ -313,6 +315,8 @@ ndz_rangemap_first(struct ndz_rangemap *map)
 /*
  * Return a pointer to the last entry in the map.
  * Returns NULL if map is empty.
+ *
+ * Does not affect the map hint.
  */
 struct ndz_range *
 ndz_rangemap_last(struct ndz_rangemap *map)
@@ -337,6 +341,8 @@ ndz_rangemap_last(struct ndz_rangemap *map)
 
 /*
  * Return the number of entries in the map.
+ *
+ * Does not affect the map hint.
  */
 int
 ndz_rangemap_count(struct ndz_rangemap *map)
@@ -464,6 +470,52 @@ ndz_rangemap_lookup(struct ndz_rangemap *map, ndz_addr_t addr,
     return range;
 }
 
+/*
+ * Determine if the indicated range [addr - addr+size-1] overlaps with
+ * any entry in the given map. Returns a pointer to the first map entry
+ * that overlaps, or NULL if there is no overlap.
+ *
+ * Resulting map hint values:
+ *   if range is found, hint points to it.
+ *   if range is not found, hint is unaffected.
+ */
+struct ndz_range *
+ndz_rangemap_overlap(struct ndz_rangemap *map, ndz_addr_t addr,
+		     ndz_size_t size)
+{
+    struct ndz_range *range, *prev;
+    ndz_addr_t eaddr = addr + size - 1;
+
+    /*
+     * Lookup the first address in the range.
+     * If it falls within a range in the map, we are done.
+     * Otherwise look for overlap with the following map entry.
+     */
+    range = ndz_rangemap_lookup(map, addr, &prev);
+    if (range == NULL) {
+	/* if prev is null we are at the beginning */
+	if (prev == NULL)
+	    range = map->head.next;
+	else
+	    range = prev->next;
+
+	/*
+	 * At the end of the map, there can be no overlap.
+	 */
+	if (range == NULL)
+	    return NULL;
+
+	/*
+	 * If our range ends before the next element in the list,
+	 * there is no overlap.
+	 */
+	if (eaddr < range->start)
+	    return NULL;
+    }
+
+    return range;
+}
+
 void
 ndz_rangemap_dumpstats(struct ndz_rangemap *map)
 {

clientside/os/imagezip/libndz/rangemap.h

@@ -78,6 +78,8 @@ int ndz_rangemap_count(struct ndz_rangemap *map);
 struct ndz_range *ndz_rangemap_lookup(struct ndz_rangemap *map,
 				      ndz_addr_t addr,
 				      struct ndz_range **prev);
+struct ndz_range *ndz_rangemap_overlap(struct ndz_rangemap *map,
+				       ndz_addr_t addr, ndz_size_t);
 void ndz_rangemap_dump(struct ndz_rangemap *map, int summaryonly,
 		       void (*dfunc)(struct ndz_rangemap *, void *));