Commit 98fc9f60 authored by Mike Hibler's avatar Mike Hibler

The light at the end of the tunnel...

Getting close: change delta/undelta to set the mtime of the file they
produce to match that of the source, redo the reloc code so it works,
cleanups, enhancements to the test scripts.
parent 276d2603
This diff is collapsed.
......@@ -247,8 +247,9 @@ openifile(char *file, struct fileinfo *info, int usesig)
exit(1);
}
if (!forcesig && labs(sb1.st_mtime - sb2.st_mtime) > 2) {
fprintf(stderr, "%s: image and signature disagree (%ld != %ld), "
"use -f to override.\n", file, sb1.st_mtime, sb2.st_mtime);
fprintf(stderr, "%s: image and signature modtime disagree "
"(%ld != %ld), use -f to override.\n",
file, sb1.st_mtime, sb2.st_mtime);
exit(1);
}
close(sigfd);
......@@ -548,15 +549,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = pstart;
/* include any relocations */
if (ndz2.ndz->relocmap) {
if (delta.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -671,15 +669,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = delta.ndz->maphi;
/* include any relocations */
if (ndz2.ndz->relocmap) {
if (delta.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -696,6 +691,36 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
return 0;
}
static void
setfiletime(char *dst, char *src)
{
struct timeval tm[2];
struct stat sb;
int cc;
/*
* Set the modtime of the hash file to match that of the image.
* This is a crude (but fast!) method for matching images with
* signatures.
*/
cc = stat(src, &sb);
if (cc >= 0) {
#ifdef linux
tm[0].tv_sec = sb.st_atime;
tm[0].tv_usec = 0;
tm[1].tv_sec = sb.st_mtime;
tm[1].tv_usec = 0;
#else
TIMESPEC_TO_TIMEVAL(&tm[0], &sb.st_atimespec);
TIMESPEC_TO_TIMEVAL(&tm[1], &sb.st_mtimespec);
#endif
cc = utimes(dst, tm);
}
if (cc < 0)
fprintf(stderr, "%s: WARNING: could not set mtime (%s)\n",
dst, strerror(errno));
}
int
main(int argc, char **argv)
......@@ -892,12 +917,19 @@ main(int argc, char **argv)
delta.ndz->hashentries = ndz2.ndz->hashentries;
delta.ndz->hashcurentry = 0;
if (ndz_reloc_copy(ndz2.ndz, delta.ndz)) {
fprintf(stderr, "%s: could not copy reloc info for delta image\n",
argv[2]);
exit(1);
}
/*
* If there is anything in the resulting delta, produce an image!
*/
if (ndz_rangemap_first(delta.map) != NULL) {
struct chunkstate *cstate = calloc(1, sizeof(*cstate));
assert(cstate != NULL);
ndz_size_t dsect, fsect;
if (ndz_rangemap_iterate(delta.map, chunkify, cstate) != 0) {
fprintf(stderr, "%s: error while creating new delta image\n",
......@@ -916,8 +948,15 @@ main(int argc, char **argv)
argv[2], delta.sigfile);
}
dsect = ndz_rangemap_sectors(delta.map);
fsect = ndz_rangemap_sectors(ndz2.map);
printf("%s: %lu sectors, %3.1f%% of full image (%lu sectors)\n",
argv[2], dsect, (double)dsect * 100 / fsect, fsect);
ndz_close(ndz2.ndz);
ndz_close(delta.ndz);
setfiletime(argv[2], argv[1]);
} else {
fprintf(stderr, "Images %s and %s are identical, no delta produced!\n",
argv[0], argv[1]);
......
......@@ -872,13 +872,14 @@ checkhash(char *name, struct hashinfo *hinfo)
stopreader();
if (!quiet) {
if (!quiet)
dump_stats(0);
if (badhashes)
printf("%s: %u regions (%d chunks) had bad hashes, "
"%llu bytes affected\n",
name, badhashes, badchunks,
(unsigned long long)badhashdata);
if (badhashes)
printf("%s: %u regions (%d chunks) had bad hashes, "
"%llu bytes affected\n",
name, badhashes, badchunks,
(unsigned long long)badhashdata);
if (!quiet) {
dump_readbufs();
#ifdef TIMEIT
printf("%llu bytes: read cycles: "
......
......@@ -504,15 +504,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = pstart;
/* include any relocations */
if (old.ndz->relocmap) {
if (new.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -630,15 +627,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = new.ndz->maphi;
/* include any relocations */
if (old.ndz->relocmap) {
if (new.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -825,6 +819,12 @@ main(int argc, char **argv)
new.ndz->hashentries = old.ndz->hashentries;
new.ndz->hashcurentry = 0;
if (ndz_reloc_copy(old.ndz, new.ndz)) {
fprintf(stderr, "%s: could not copy reloc info for new image\n",
argv[2]);
exit(1);
}
/*
* If there is anything in the old image, produce a new image!
*/
......
......@@ -754,15 +754,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = pstart;
/* include any relocations */
if (delta.ndz->relocmap) {
if (new.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = delta.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -880,15 +877,12 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->lastsect = new.ndz->maphi;
/* include any relocations */
if (delta.ndz->relocmap) {
if (new.ndz->relocentries > 0) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = delta.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
......@@ -913,6 +907,36 @@ chunkfunc(struct ndz_rangemap *map, void *ptr)
printf("chunkno=%u", chunkno);
}
static void
setfiletime(char *dst, char *src)
{
struct timeval tm[2];
struct stat sb;
int cc;
/*
* Set the modtime of the hash file to match that of the image.
* This is a crude (but fast!) method for matching images with
* signatures.
*/
cc = stat(src, &sb);
if (cc >= 0) {
#ifdef linux
tm[0].tv_sec = sb.st_atime;
tm[0].tv_usec = 0;
tm[1].tv_sec = sb.st_mtime;
tm[1].tv_usec = 0;
#else
TIMESPEC_TO_TIMEVAL(&tm[0], &sb.st_atimespec);
TIMESPEC_TO_TIMEVAL(&tm[1], &sb.st_mtimespec);
#endif
cc = utimes(dst, tm);
}
if (cc < 0)
fprintf(stderr, "%s: WARNING: could not set mtime (%s)\n",
dst, strerror(errno));
}
int
main(int argc, char **argv)
{
......@@ -1087,6 +1111,12 @@ main(int argc, char **argv)
new.ndz->hashblksize = hashblksize;
new.ndz->hashentries = delta.ndz->hashentries;
new.ndz->hashcurentry = 0;
if (ndz_reloc_copy(delta.ndz, new.ndz)) {
fprintf(stderr, "%s: could not copy reloc info for new image\n",
argv[2]);
exit(1);
}
}
/*
......@@ -1136,7 +1166,7 @@ main(int argc, char **argv)
if (usesigfile)
cstate.newsigmap = new.sigmap;
if (ndz_rangemap_iterate(new.map, chunkify, &cstate) != 0) {
fprintf(stderr, "%s: error while creating new delta image\n",
fprintf(stderr, "%s: error while creating new image\n",
argv[2]);
exit(1);
}
......@@ -1145,6 +1175,8 @@ main(int argc, char **argv)
ndz_close(base.ndz);
ndz_close(delta.ndz);
ndz_close(new.ndz);
setfiletime(argv[2], argv[1]);
} else {
fprintf(stderr, "Images %s and %s are identical, no image produced!\n",
argv[0], argv[1]);
......
......@@ -347,9 +347,9 @@ ndz_freehashmap(struct ndz_file *ndz)
}
struct deltainfo {
struct ndz_file *ndz;
struct ndz_rangemap *omap;
struct ndz_rangemap *dmap;
struct ndz_rangemap *relocmap;
int omapdone;
};
......@@ -448,8 +448,8 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
* XXX if the sector range includes a relocation,
* we always force it into the image.
*/
if (dinfo->relocmap &&
ndz_rangemap_overlap(dinfo->relocmap, addr, eaddr-addr+1)) {
if (dinfo->ndz->relocdata &&
ndz_reloc_inrange(dinfo->ndz, addr, eaddr-addr+1)) {
#ifdef COMPDELTA_DEBUG
fprintf(stderr, "has relocation, adding\n");
#endif
......@@ -510,9 +510,9 @@ ndz_compute_delta(struct ndz_file *ondz, struct ndz_file *nndz)
return NULL;
}
dinfo.ndz = nndz;
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
#if 0
(void) ndz_rangemap_iterate(nmap, compdelta, &dinfo);
......@@ -541,9 +541,9 @@ ndz_compute_delta_sigmap(struct ndz_file *ondz, struct ndz_file *nndz)
return NULL;
}
dinfo.ndz = nndz;
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
(void) ndz_rangemap_iterate(nmap, compfastdelta, &dinfo);
......
......@@ -57,7 +57,7 @@ struct ndz_file {
/* relocation information */
unsigned relocentries;
void *relocdata;
struct ndz_rangemap *relocmap;
ndz_addr_t reloclo, relochi;
/* hash (signature) info */
unsigned hashtype;
unsigned hashblksize;
......@@ -107,8 +107,11 @@ int ndz_chunk_flush(ndz_chunk_t chobj, int withheader);
ssize_t ndz_chunk_left(ndz_chunk_t chobj);
ssize_t ndz_chunk_append(ndz_chunk_t chobj, void *buf, size_t bytes);
void ndz_reloc_init(struct ndz_file *ndz);
int ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
int ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
int ndz_reloc_inrange(struct ndz_file *ndz, ndz_addr_t addr, ndz_size_t size);
int ndz_reloc_copy(struct ndz_file *ndzfrom, struct ndz_file *ndzto);
void ndz_reloc_free(struct ndz_file *ndz);
struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);
......
......@@ -93,6 +93,8 @@ ndz_open(const char *name, int forwrite)
ndz->sectsize = SECSIZE;
ndz->chunksize = CHUNKSIZE;
ndz_reloc_init(ndz);
/*
* XXX someday check checksum if present and stash info to check
* in every chunk header.
......@@ -259,16 +261,6 @@ ndz_readranges(struct ndz_file *ndz)
/* get the relocations */
if (head.reloc) {
if (ndz->relocmap == NULL) {
ndz->relocmap =
ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
if (ndz->relocmap == NULL) {
fprintf(stderr, "%s: could not allocate relocmap\n",
ndz->fname);
ndz_rangemap_deinit(map);
return NULL;
}
}
if (ndz_reloc_get(ndz, head.header, head.reloc) != 0) {
fprintf(stderr, "%s: could not add relocations\n",
ndz->fname);
......
......@@ -24,9 +24,12 @@
/*
* Relocation handling routines.
*
* XXX using a range map is overkill, since there are almost never more
* than a few (one) relocations and they are single sector things.
* But images with relocations are rare, so don't worry about it.
* We just associate an array of blockreloc structs with the NDZ file
* to keep track of these. This is good enough since there are not ever
* very many relocs and they are almost always in the first chunk.
*
* Note that I originally used a rangemap, but there can be more than
* one reloc per sector so that doesn't work.
*/
#include <unistd.h>
......@@ -41,13 +44,29 @@
#define RELOC_DEBUG
void
ndz_reloc_init(struct ndz_file *ndz)
{
assert(ndz != NULL);
ndz->relocdata = NULL;
ndz->relocentries = 0;
ndz->reloclo = NDZ_HIADDR;
ndz->relochi = NDZ_LOADDR;
}
/*
* Read relocs out of a chunk header and add them to the array of relocs
* for the file, reallocing the buffer as necessary. Not terrible efficient,
* but does not have to be.
*/
int
ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *relocdata, *reloc = buf;
struct blockreloc *relocdata, *chunkreloc = buf;
int i;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || reloc == NULL)
if (ndz == NULL || hdr == NULL || chunkreloc == NULL)
return -1;
if (hdr->magic < COMPRESSED_V2 || hdr->reloccount == 0)
......@@ -56,68 +75,130 @@ ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
/* resize the relocation buffer */
i = ndz->relocentries + hdr->reloccount;
if (ndz->relocdata == NULL)
relocdata = malloc(i * sizeof(*reloc));
relocdata = malloc(i * sizeof(struct blockreloc));
else
relocdata = realloc(ndz->relocdata, i * sizeof(*reloc));
relocdata = realloc(ndz->relocdata, i * sizeof(struct blockreloc));
if (relocdata == NULL) {
ndz_reloc_free(ndz);
return -1;
}
ndz->relocdata = relocdata;
relocdata = ndz->relocdata + ndz->relocentries;
memcpy(relocdata, reloc, hdr->reloccount * sizeof(*reloc));
relocdata = (struct blockreloc *)ndz->relocdata + ndz->relocentries;
for (i = 0; i < hdr->reloccount; i++) {
ndz_size_t size = (reloc->size + ndz->sectsize - 1) / ndz->sectsize;
if (ndz_rangemap_alloc(ndz->relocmap, (ndz_addr_t)reloc->sector, size,
&relocdata[i])) {
ndz_reloc_free(ndz);
return -1;
}
if (ndz->reloclo == NDZ_HIADDR)
ndz->reloclo = chunkreloc->sector;
/* XXX we should be adding these in order; we assume this elsewhere */
assert(ndz->reloclo <= chunkreloc->sector);
if (chunkreloc->sector > ndz->relochi)
ndz->relochi = chunkreloc->sector;
*relocdata++ = *chunkreloc++;
}
ndz->relocentries += hdr->reloccount;
#ifdef RELOC_DEBUG
if (hdr->reloccount > 0) {
fprintf(stderr, "got %d relocs, %d total\n",
hdr->reloccount, ndz->relocentries);
fprintf(stderr, "got %d relocs, %d total, range [%lu-%lu]\n",
hdr->reloccount, ndz->relocentries,
ndz->reloclo, ndz->relochi);
}
#endif
return 0;
}
/*
* Find any relocation entries that apply to the indicated chunk and add them.
*/
int
ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *reloc;
struct ndz_range *rrange;
ndz_addr_t addr, eaddr;
struct blockreloc *chunkreloc, *relocdata;
int i;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || buf == NULL)
if (ndz == NULL || hdr == NULL || buf == NULL)
return -1;
reloc = buf;
addr = hdr->firstsect;
eaddr = hdr->lastsect;
rrange = ndz_rangemap_overlap(ndz->relocmap, addr, eaddr - addr + 1);
while (rrange && rrange->start <= eaddr) {
if (ndz->relocentries == 0 ||
hdr->firstsect > ndz->relochi || hdr->lastsect < ndz->reloclo)
return 0;
chunkreloc = buf;
relocdata = ndz->relocdata;
for (i = 0; i < ndz->relocentries; i++) {
assert(relocdata->sectoff + relocdata->size <= ndz->sectsize);
if (relocdata->sector >= hdr->firstsect &&
relocdata->sector <= hdr->lastsect) {
#ifdef RELOC_DEBUG
fprintf(stderr, "found reloc [%lu-%lu] in chunk range [%lu-%lu]\n",
rrange->start, rrange->end, addr, eaddr);
fprintf(stderr, "found reloc for %u in chunk range [%u-%u]\n",
relocdata->sector, hdr->firstsect, hdr->lastsect);
#endif
assert(rrange->start >= addr && rrange->end <= eaddr);
assert(rrange->data != NULL);
*chunkreloc++ = *relocdata;
}
relocdata++;
}
return 0;
}
*reloc = *(struct blockreloc *)rrange->data;
reloc++;
hdr->reloccount++;
rrange = rrange->next;
/*
* Returns 1 if there is a relocation in the indicated range, 0 otherwise
*/
int
ndz_reloc_inrange(struct ndz_file *ndz, ndz_addr_t addr, ndz_size_t size)
{
struct blockreloc *relocdata;
ndz_addr_t eaddr;
int i;
assert(ndz != NULL);
eaddr = addr + size - 1;
if (ndz->relocentries == 0 || addr > ndz->relochi || eaddr < ndz->reloclo)
return 0;
relocdata = ndz->relocdata;
for (i = 0; i < ndz->relocentries; i++) {
assert(relocdata->sectoff + relocdata->size <= ndz->sectsize);
if (relocdata->sector >= addr && relocdata->sector <= eaddr) {
#ifdef RELOC_DEBUG
fprintf(stderr, "found a reloc (%u) in range [%lu-%lu]\n",
relocdata->sector, addr, eaddr);
#endif
return 1;
}
relocdata++;
}
return 0;
}
/*
* Reloc info is small so this is relatively painless
*/
int
ndz_reloc_copy(struct ndz_file *ndzfrom, struct ndz_file *ndzto)
{
size_t size;
if (ndzfrom == NULL || ndzto == NULL || ndzto->relocentries > 0)
return -1;
if (ndzfrom->relocentries == 0)
return 0;
size = ndzfrom->relocentries * sizeof(struct blockreloc);
if ((ndzto->relocdata = malloc(size)) == NULL)
return -1;
memcpy(ndzto->relocdata, ndzfrom->relocdata, size);
ndzto->relocentries = ndzfrom->relocentries;
ndzto->reloclo = ndzfrom->reloclo;
ndzto->relochi = ndzfrom->relochi;
return 0;
}
void
ndz_reloc_free(struct ndz_file *ndz)
{
......@@ -126,10 +207,6 @@ ndz_reloc_free(struct ndz_file *ndz)
free(ndz->relocdata);
ndz->relocdata = NULL;
}
if (ndz->relocmap) {
ndz_rangemap_deinit(ndz->relocmap);
ndz->relocmap = NULL;
}
ndz->relocentries = 0;
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment