The light at the end of the tunnel...

Getting close: change delta/undelta to set the mtime of the file they
produce to match that of the source, redo the reloc code so it works,
cleanups, enhancements to the test scripts.

clientside/os/imagezip/imagedelta.c

@@ -247,8 +247,9 @@ openifile(char *file, struct fileinfo *info, int usesig)
 	    exit(1);
 	}
 	if (!forcesig && labs(sb1.st_mtime - sb2.st_mtime) > 2) {
-	    fprintf(stderr, "%s: image and signature disagree (%ld != %ld), "
-		    "use -f to override.\n", file, sb1.st_mtime, sb2.st_mtime);
+	    fprintf(stderr, "%s: image and signature modtime disagree "
+		    "(%ld != %ld), use -f to override.\n",
+		    file, sb1.st_mtime, sb2.st_mtime);
 	    exit(1);
 	}
 	close(sigfd);
@@ -548,15 +549,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 		cstate->header->lastsect = pstart;
 
 		/* include any relocations */
-		if (ndz2.ndz->relocmap) {
+		if (delta.ndz->relocentries > 0) {
 		    void *buf = (cstate->curregion + 1);
-		    delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
 		    if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
-			delta.ndz->relocmap = NULL; /* XXX */
 			fprintf(stderr, "Error writing relocation info\n");
 			return 1;
 		    }
-		    delta.ndz->relocmap = NULL; /* XXX */
 		}
 
 		/* and write it */
@@ -671,15 +669,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 	cstate->header->lastsect = delta.ndz->maphi;
 
 	/* include any relocations */
-	if (ndz2.ndz->relocmap) {
+	if (delta.ndz->relocentries > 0) {
 	    void *buf = (cstate->curregion + 1);
-	    delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
 	    if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
-		delta.ndz->relocmap = NULL; /* XXX */
 		fprintf(stderr, "Error writing relocation info\n");
 		return 1;
 	    }
-	    delta.ndz->relocmap = NULL; /* XXX */
 	}
 
 	/* and write it */
@@ -696,6 +691,36 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 
     return 0;
 }
+ 
+static void
+setfiletime(char *dst, char *src)
+{
+    struct timeval tm[2];
+    struct stat sb;
+    int cc;
+
+    /*
+     * Set the modtime of the hash file to match that of the image.
+     * This is a crude (but fast!) method for matching images with
+     * signatures.
+     */
+    cc = stat(src, &sb);
+    if (cc >= 0) {
+#ifdef linux
+	tm[0].tv_sec = sb.st_atime;
+	tm[0].tv_usec = 0;
+	tm[1].tv_sec = sb.st_mtime;
+	tm[1].tv_usec = 0;
+#else
+	TIMESPEC_TO_TIMEVAL(&tm[0], &sb.st_atimespec);
+	TIMESPEC_TO_TIMEVAL(&tm[1], &sb.st_mtimespec);
+#endif
+	cc = utimes(dst, tm);
+    }
+    if (cc < 0)
+	fprintf(stderr, "%s: WARNING: could not set mtime (%s)\n",
+		dst, strerror(errno));
+}
 
 int
 main(int argc, char **argv)
@@ -892,12 +917,19 @@ main(int argc, char **argv)
     delta.ndz->hashentries = ndz2.ndz->hashentries;
     delta.ndz->hashcurentry = 0;
 
+    if (ndz_reloc_copy(ndz2.ndz, delta.ndz)) {
+	fprintf(stderr, "%s: could not copy reloc info for delta image\n",
+		argv[2]);
+	exit(1);
+    }
+
     /*
      * If there is anything in the resulting delta, produce an image!
      */
     if (ndz_rangemap_first(delta.map) != NULL) {
 	struct chunkstate *cstate = calloc(1, sizeof(*cstate));
 	assert(cstate != NULL);
+	ndz_size_t dsect, fsect;
 
 	if (ndz_rangemap_iterate(delta.map, chunkify, cstate) != 0) {
 	    fprintf(stderr, "%s: error while creating new delta image\n",
@@ -916,8 +948,15 @@ main(int argc, char **argv)
 		    argv[2], delta.sigfile);
 	}
 	
+	dsect = ndz_rangemap_sectors(delta.map);
+	fsect = ndz_rangemap_sectors(ndz2.map);
+	printf("%s: %lu sectors, %3.1f%% of full image (%lu sectors)\n",
+	      argv[2], dsect, (double)dsect * 100 / fsect, fsect);
+	      
 	ndz_close(ndz2.ndz);
 	ndz_close(delta.ndz);
+
+	setfiletime(argv[2], argv[1]);
     } else {
 	fprintf(stderr, "Images %s and %s are identical, no delta produced!\n",
 		argv[0], argv[1]);

clientside/os/imagezip/imagehash.c

@@ -872,13 +872,14 @@ checkhash(char *name, struct hashinfo *hinfo)
 
 	stopreader();
 
-	if (!quiet) {
+	if (!quiet)
 		dump_stats(0);
-		if (badhashes)
-			printf("%s: %u regions (%d chunks) had bad hashes, "
-			       "%llu bytes affected\n",
-			       name, badhashes, badchunks,
-			       (unsigned long long)badhashdata);
+	if (badhashes)
+		printf("%s: %u regions (%d chunks) had bad hashes, "
+		       "%llu bytes affected\n",
+		       name, badhashes, badchunks,
+		       (unsigned long long)badhashdata);
+	if (!quiet) {
 		dump_readbufs();
 #ifdef TIMEIT
 		printf("%llu bytes: read cycles: "

clientside/os/imagezip/imagerezip.c

@@ -504,15 +504,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 		cstate->header->lastsect = pstart;
 
 		/* include any relocations */
-		if (old.ndz->relocmap) {
+		if (new.ndz->relocentries > 0) {
 		    void *buf = (cstate->curregion + 1);
-		    new.ndz->relocmap = old.ndz->relocmap; /* XXX */
 		    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
-			new.ndz->relocmap = NULL; /* XXX */
 			fprintf(stderr, "Error writing relocation info\n");
 			return 1;
 		    }
-		    new.ndz->relocmap = NULL; /* XXX */
 		}
 
 		/* and write it */
@@ -630,15 +627,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 	cstate->header->lastsect = new.ndz->maphi;
 
 	/* include any relocations */
-	if (old.ndz->relocmap) {
+	if (new.ndz->relocentries > 0) {
 	    void *buf = (cstate->curregion + 1);
-	    new.ndz->relocmap = old.ndz->relocmap; /* XXX */
 	    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
-		new.ndz->relocmap = NULL; /* XXX */
 		fprintf(stderr, "Error writing relocation info\n");
 		return 1;
 	    }
-	    new.ndz->relocmap = NULL; /* XXX */
 	}
 
 	/* and write it */
@@ -825,6 +819,12 @@ main(int argc, char **argv)
     new.ndz->hashentries = old.ndz->hashentries;
     new.ndz->hashcurentry = 0;
 
+    if (ndz_reloc_copy(old.ndz, new.ndz)) {
+	fprintf(stderr, "%s: could not copy reloc info for new image\n",
+		argv[2]);
+	exit(1);
+    }
+
     /*
      * If there is anything in the old image, produce a new image!
      */

clientside/os/imagezip/imageundelta.c

@@ -754,15 +754,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 		cstate->header->lastsect = pstart;
 
 		/* include any relocations */
-		if (delta.ndz->relocmap) {
+		if (new.ndz->relocentries > 0) {
 		    void *buf = (cstate->curregion + 1);
-		    new.ndz->relocmap = delta.ndz->relocmap; /* XXX */
 		    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
-			new.ndz->relocmap = NULL; /* XXX */
 			fprintf(stderr, "Error writing relocation info\n");
 			return 1;
 		    }
-		    new.ndz->relocmap = NULL; /* XXX */
 		}
 
 		/* and write it */
@@ -880,15 +877,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
 	cstate->header->lastsect = new.ndz->maphi;
 
 	/* include any relocations */
-	if (delta.ndz->relocmap) {
+	if (new.ndz->relocentries > 0) {
 	    void *buf = (cstate->curregion + 1);
-	    new.ndz->relocmap = delta.ndz->relocmap; /* XXX */
 	    if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
-		new.ndz->relocmap = NULL; /* XXX */
 		fprintf(stderr, "Error writing relocation info\n");
 		return 1;
 	    }
-	    new.ndz->relocmap = NULL; /* XXX */
 	}
 
 	/* and write it */
@@ -913,6 +907,36 @@ chunkfunc(struct ndz_rangemap *map, void *ptr)
     printf("chunkno=%u", chunkno);
 }
 
+static void
+setfiletime(char *dst, char *src)
+{
+    struct timeval tm[2];
+    struct stat sb;
+    int cc;
+
+    /*
+     * Set the modtime of the hash file to match that of the image.
+     * This is a crude (but fast!) method for matching images with
+     * signatures.
+     */
+    cc = stat(src, &sb);
+    if (cc >= 0) {
+#ifdef linux
+	tm[0].tv_sec = sb.st_atime;
+	tm[0].tv_usec = 0;
+	tm[1].tv_sec = sb.st_mtime;
+	tm[1].tv_usec = 0;
+#else
+	TIMESPEC_TO_TIMEVAL(&tm[0], &sb.st_atimespec);
+	TIMESPEC_TO_TIMEVAL(&tm[1], &sb.st_mtimespec);
+#endif
+	cc = utimes(dst, tm);
+    }
+    if (cc < 0)
+	fprintf(stderr, "%s: WARNING: could not set mtime (%s)\n",
+		dst, strerror(errno));
+}
+
 int
 main(int argc, char **argv)
 {
@@ -1087,6 +1111,12 @@ main(int argc, char **argv)
 	new.ndz->hashblksize = hashblksize;
 	new.ndz->hashentries = delta.ndz->hashentries;
 	new.ndz->hashcurentry = 0;
+
+	if (ndz_reloc_copy(delta.ndz, new.ndz)) {
+	    fprintf(stderr, "%s: could not copy reloc info for new image\n",
+		    argv[2]);
+	    exit(1);
+	}
     }
 
     /*
@@ -1136,7 +1166,7 @@ main(int argc, char **argv)
 	    if (usesigfile)
 		cstate.newsigmap = new.sigmap;
 	    if (ndz_rangemap_iterate(new.map, chunkify, &cstate) != 0) {
-		fprintf(stderr, "%s: error while creating new delta image\n",
+		fprintf(stderr, "%s: error while creating new image\n",
 			argv[2]);
 		exit(1);
 	    }
@@ -1145,6 +1175,8 @@ main(int argc, char **argv)
 	ndz_close(base.ndz);
 	ndz_close(delta.ndz);
 	ndz_close(new.ndz);
+
+	setfiletime(argv[2], argv[1]);
     } else {
 	fprintf(stderr, "Images %s and %s are identical, no image produced!\n",
 		argv[0], argv[1]);

clientside/os/imagezip/libndz/hash.c

@@ -347,9 +347,9 @@ ndz_freehashmap(struct ndz_file *ndz)
 }
 
 struct deltainfo {
+    struct ndz_file *ndz;
     struct ndz_rangemap *omap;
     struct ndz_rangemap *dmap;
-    struct ndz_rangemap *relocmap;
     int omapdone;
 };
 
@@ -448,8 +448,8 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
 	     * XXX if the sector range includes a relocation,
 	     * we always force it into the image.
 	     */
-	    if (dinfo->relocmap &&
-		ndz_rangemap_overlap(dinfo->relocmap, addr, eaddr-addr+1)) {
+	    if (dinfo->ndz->relocdata &&
+		ndz_reloc_inrange(dinfo->ndz, addr, eaddr-addr+1)) {
 #ifdef COMPDELTA_DEBUG
 		fprintf(stderr, "has relocation, adding\n");
 #endif
@@ -510,9 +510,9 @@ ndz_compute_delta(struct ndz_file *ondz, struct ndz_file *nndz)
 	return NULL;
     }
 
+    dinfo.ndz = nndz;
     dinfo.omap = omap;
     dinfo.dmap = dmap;
-    dinfo.relocmap = nndz->relocmap;
     dinfo.omapdone = 0;
 #if 0
     (void) ndz_rangemap_iterate(nmap, compdelta, &dinfo);
@@ -541,9 +541,9 @@ ndz_compute_delta_sigmap(struct ndz_file *ondz, struct ndz_file *nndz)
 	return NULL;
     }
 
+    dinfo.ndz = nndz;
     dinfo.omap = omap;
     dinfo.dmap = dmap;
-    dinfo.relocmap = nndz->relocmap;
     dinfo.omapdone = 0;
     (void) ndz_rangemap_iterate(nmap, compfastdelta, &dinfo);
 

clientside/os/imagezip/libndz/libndz.h

@@ -57,7 +57,7 @@ struct ndz_file {
     /* relocation information */
     unsigned relocentries;
     void *relocdata;
-    struct ndz_rangemap *relocmap;
+    ndz_addr_t reloclo, relochi;
     /* hash (signature) info */
     unsigned hashtype;
     unsigned hashblksize;
@@ -107,8 +107,11 @@ int ndz_chunk_flush(ndz_chunk_t chobj, int withheader);
 ssize_t ndz_chunk_left(ndz_chunk_t chobj);
 ssize_t ndz_chunk_append(ndz_chunk_t chobj, void *buf, size_t bytes);
 
+void ndz_reloc_init(struct ndz_file *ndz);
 int ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
 int ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
+int ndz_reloc_inrange(struct ndz_file *ndz, ndz_addr_t addr, ndz_size_t size);
+int ndz_reloc_copy(struct ndz_file *ndzfrom, struct ndz_file *ndzto);
 void ndz_reloc_free(struct ndz_file *ndz);
 
 struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);

clientside/os/imagezip/libndz/ndzfile.c

@@ -93,6 +93,8 @@ ndz_open(const char *name, int forwrite)
     ndz->sectsize = SECSIZE;
     ndz->chunksize = CHUNKSIZE;
 
+    ndz_reloc_init(ndz);
+
     /*
      * XXX someday check checksum if present and stash info to check
      * in every chunk header.
@@ -259,16 +261,6 @@ ndz_readranges(struct ndz_file *ndz)
 
 	/* get the relocations */
 	if (head.reloc) {
-	    if (ndz->relocmap == NULL) {
-		ndz->relocmap =
-		    ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
-		if (ndz->relocmap == NULL) {
-		    fprintf(stderr, "%s: could not allocate relocmap\n",
-			    ndz->fname);
-		    ndz_rangemap_deinit(map);
-		    return NULL;
-		}
-	    }
 	    if (ndz_reloc_get(ndz, head.header, head.reloc) != 0) {
 		fprintf(stderr, "%s: could not add relocations\n",
 			ndz->fname);

clientside/os/imagezip/libndz/reloc.c

@@ -24,9 +24,12 @@
 /*
  * Relocation handling routines.
  *
- * XXX using a range map is overkill, since there are almost never more
- * than a few (one) relocations and they are single sector things.
- * But images with relocations are rare, so don't worry about it.
+ * We just associate an array of blockreloc structs with the NDZ file
+ * to keep track of these. This is good enough since there are not ever
+ * very many relocs and they are almost always in the first chunk.
+ *
+ * Note that I originally used a rangemap, but there can be more than
+ * one reloc per sector so that doesn't work.
  */
 
 #include <unistd.h>
@@ -41,13 +44,29 @@
 
 #define RELOC_DEBUG
 
+void
+ndz_reloc_init(struct ndz_file *ndz)
+{
+    assert(ndz != NULL);
+
+    ndz->relocdata = NULL;
+    ndz->relocentries = 0;
+    ndz->reloclo = NDZ_HIADDR;
+    ndz->relochi = NDZ_LOADDR;
+}
+
+/*
+ * Read relocs out of a chunk header and add them to the array of relocs
+ * for the file, reallocing the buffer as necessary. Not terrible efficient,
+ * but does not have to be.
+ */ 
 int
 ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
 {
-    struct blockreloc *relocdata, *reloc = buf;
+    struct blockreloc *relocdata, *chunkreloc = buf;
     int i;
 
-    if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || reloc == NULL)
+    if (ndz == NULL || hdr == NULL || chunkreloc == NULL)
 	return -1;
 
     if (hdr->magic < COMPRESSED_V2 || hdr->reloccount == 0)
@@ -56,68 +75,130 @@ ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
     /* resize the relocation buffer */
     i = ndz->relocentries + hdr->reloccount;
     if (ndz->relocdata == NULL)
-	relocdata = malloc(i * sizeof(*reloc));
+	relocdata = malloc(i * sizeof(struct blockreloc));
     else
-	relocdata = realloc(ndz->relocdata, i * sizeof(*reloc));
+	relocdata = realloc(ndz->relocdata, i * sizeof(struct blockreloc));
     if (relocdata == NULL) {
 	ndz_reloc_free(ndz);
 	return -1;
     }
     ndz->relocdata = relocdata;
 
-    relocdata = ndz->relocdata + ndz->relocentries;
-    memcpy(relocdata, reloc, hdr->reloccount * sizeof(*reloc));
+    relocdata = (struct blockreloc *)ndz->relocdata + ndz->relocentries;
     for (i = 0; i < hdr->reloccount; i++) {
-	ndz_size_t size = (reloc->size + ndz->sectsize - 1) / ndz->sectsize;
-	if (ndz_rangemap_alloc(ndz->relocmap, (ndz_addr_t)reloc->sector, size,
-			       &relocdata[i])) {
-	    ndz_reloc_free(ndz);
-	    return -1;
-	}
+	if (ndz->reloclo == NDZ_HIADDR)
+	    ndz->reloclo = chunkreloc->sector;
+	/* XXX we should be adding these in order; we assume this elsewhere */
+	assert(ndz->reloclo <= chunkreloc->sector);
+	if (chunkreloc->sector > ndz->relochi)
+	    ndz->relochi = chunkreloc->sector;
+
+	*relocdata++ = *chunkreloc++;
     }
     ndz->relocentries += hdr->reloccount;
 
 #ifdef RELOC_DEBUG
     if (hdr->reloccount > 0) {
-	fprintf(stderr, "got %d relocs, %d total\n",
-		hdr->reloccount, ndz->relocentries);
+	fprintf(stderr, "got %d relocs, %d total, range [%lu-%lu]\n",
+		hdr->reloccount, ndz->relocentries,
+		ndz->reloclo, ndz->relochi);
     }
 #endif
 
     return 0;
 }
 
+/*
+ * Find any relocation entries that apply to the indicated chunk and add them.
+ */
 int
 ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
 {
-    struct blockreloc *reloc;
-    struct ndz_range *rrange;
-    ndz_addr_t addr, eaddr;
+    struct blockreloc *chunkreloc, *relocdata;
+    int i;
 
-    if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || buf == NULL)
+    if (ndz == NULL || hdr == NULL || buf == NULL)
 	return -1;
 
-    reloc = buf;
-    addr = hdr->firstsect;
-    eaddr = hdr->lastsect;
-    rrange = ndz_rangemap_overlap(ndz->relocmap, addr, eaddr - addr + 1);
-    while (rrange && rrange->start <= eaddr) {
+    if (ndz->relocentries == 0 ||
+	hdr->firstsect > ndz->relochi || hdr->lastsect < ndz->reloclo)
+	return 0;
+
+    chunkreloc = buf;
+    relocdata = ndz->relocdata;
+    for (i = 0; i < ndz->relocentries; i++) {
+	assert(relocdata->sectoff + relocdata->size <= ndz->sectsize);
+	if (relocdata->sector >= hdr->firstsect &&
+	    relocdata->sector <= hdr->lastsect) {
 #ifdef RELOC_DEBUG
-	fprintf(stderr, "found reloc [%lu-%lu] in chunk range [%lu-%lu]\n",
-		rrange->start, rrange->end, addr, eaddr);
+	    fprintf(stderr, "found reloc for %u in chunk range [%u-%u]\n",
+		    relocdata->sector, hdr->firstsect, hdr->lastsect);
 #endif
-	assert(rrange->start >= addr && rrange->end <= eaddr);
-	assert(rrange->data != NULL);
+	    *chunkreloc++ = *relocdata;
+	}
+	relocdata++;
+    }
+
+    return 0;
+}
 
-	*reloc = *(struct blockreloc *)rrange->data;
-	reloc++;
-	hdr->reloccount++;
-	rrange = rrange->next;
+/*
+ * Returns 1 if there is a relocation in the indicated range, 0 otherwise
+ */
+int
+ndz_reloc_inrange(struct ndz_file *ndz, ndz_addr_t addr, ndz_size_t size)
+{
+    struct blockreloc *relocdata;
+    ndz_addr_t eaddr;
+    int i;
+
+    assert(ndz != NULL);
+
+    eaddr = addr + size - 1;
+    if (ndz->relocentries == 0 || addr > ndz->relochi || eaddr < ndz->reloclo)
+	return 0;
+
+    relocdata = ndz->relocdata;
+    for (i = 0; i < ndz->relocentries; i++) {
+	assert(relocdata->sectoff + relocdata->size <= ndz->sectsize);
+	if (relocdata->sector >= addr && relocdata->sector <= eaddr) {
+#ifdef RELOC_DEBUG
+	    fprintf(stderr, "found a reloc (%u) in range [%lu-%lu]\n",
+		    relocdata->sector, addr, eaddr);
+#endif
+	    return 1;
+	}
+	relocdata++;
     }
 
     return 0;
 }
 
+/*
+ * Reloc info is small so this is relatively painless
+ */
+int
+ndz_reloc_copy(struct ndz_file *ndzfrom, struct ndz_file *ndzto)
+{
+    size_t size;
+
+    if (ndzfrom == NULL || ndzto == NULL || ndzto->relocentries > 0)
+	return -1;
+
+    if (ndzfrom->relocentries == 0)
+	return 0;
+
+    size = ndzfrom->relocentries * sizeof(struct blockreloc);
+    if ((ndzto->relocdata = malloc(size)) == NULL)
+	return -1;
+
+    memcpy(ndzto->relocdata, ndzfrom->relocdata, size);
+    ndzto->relocentries = ndzfrom->relocentries;
+    ndzto->reloclo = ndzfrom->reloclo;
+    ndzto->relochi = ndzfrom->relochi;
+    return 0;
+}
+
 void
 ndz_reloc_free(struct ndz_file *ndz)
 {
@@ -126,10 +207,6 @@ ndz_reloc_free(struct ndz_file *ndz)
 	    free(ndz->relocdata);
 	    ndz->relocdata = NULL;
 	}
-	if (ndz->relocmap) {
-	    ndz_rangemap_deinit(ndz->relocmap);
-	    ndz->relocmap = NULL;
-	}
 	ndz->relocentries = 0;
     }
 }