Commit e1613a02 authored by Mike Hibler's avatar Mike Hibler

Deal with relocations.

That was more involved than I thought...or it was worth, considering that
only old FreeBSD and really old Redhat (think: LILO) images have them.
But that is the kinda guy I am...
parent be00c1c1
......@@ -528,6 +528,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->firstsect = 0;
cstate->header->lastsect = pstart;
/* include any relocations */
if (ndz2.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -634,10 +646,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
/* finalize the header */
cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
/* XXX */
/* XXX should always be zero */
if (cstate->chunkno == 0)
cstate->header->firstsect = 0;
cstate->header->lastsect = delta.ndz->maphi;
/* include any relocations */
if (ndz2.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -757,7 +782,7 @@ main(int argc, char **argv)
argv[1], ndz2.ndz->hashtype, ndz2.ndz->hashblksize);
exit(1);
}
delta.map = ndz_compute_delta_sigmap(ndz1.sigmap, ndz2.sigmap);
delta.map = ndz_compute_delta_sigmap(ndz1.ndz, ndz2.ndz);
if (delta.map == NULL) {
fprintf(stderr, "Could not compute delta for %s and %s\n",
argv[0], argv[1]);
......@@ -785,7 +810,7 @@ main(int argc, char **argv)
* Same deal, but construct the delta map from the ranges maps.
*/
else {
delta.map = ndz_compute_delta(ndz1.map, ndz2.map);
delta.map = ndz_compute_delta(ndz1.ndz, ndz2.ndz);
if (delta.map == NULL) {
fprintf(stderr, "Could not compute delta for %s and %s\n",
argv[0], argv[1]);
......@@ -862,8 +887,7 @@ main(int argc, char **argv)
free(cstate);
/* readjust to reflect the actual number of hash entries */
if (!fullsig)
delta.ndz->hashentries = delta.ndz->hashcurentry;
delta.ndz->hashentries = delta.ndz->hashcurentry;
/* write the new sigfile */
if (ndz_writehashinfo(fullsig ? ndz2.ndz : delta.ndz,
......@@ -875,7 +899,8 @@ main(int argc, char **argv)
ndz_close(ndz2.ndz);
ndz_close(delta.ndz);
} else {
fprintf(stderr, "Images are identical, no delta produced!\n");
fprintf(stderr, "Images %s and %s are identical, no delta produced!\n",
argv[0], argv[1]);
ndz_close(ndz2.ndz);
ndz_close(delta.ndz);
unlink(argv[2]);
......
......@@ -133,6 +133,33 @@ struct blockhdr_V4 {
/* Unique ID for the whole image */
};
/*
* Coming soon in V5:
*
* 64-bit support.
* Grow blockindex, firstsect, lastsect, region descriptors, may need to
* grow DEFAULTREGIONSIZE to accomodate the doubling in size of regions.
*
* Flag field?
* For example, to indicate a delta image. Would probably take over the
* otherwise unused blocktotal field.
*
* Sectorsize field?
* To make explicit the units of sector fields; e.g., 512 vs 4096.
*
* Chunksize field?
* To support different chunksizes.
*
* Mandate little-endian on-disk data.
* Code changes only to use appropriate endian macros when reading/writing
* data. No data struct changes needed.
*
* Support for SHA256 and SHA512 checksums.
* Just some constants here, as checksum is already 64 bytes.
* Will need to grow the imagehash header to accomodate these checksums
* for signatures, but that is separately versioned.
*/
/*
* Checksum types supported
*/
......
......@@ -21,7 +21,7 @@
* }}}
*/
#define CHUNKIFY_DEBUG
//#define CHUNKIFY_DEBUG
/*
* imagerezip [-S] Oimage.ndz Nimage.ndz
......@@ -271,7 +271,7 @@ readifile(struct fileinfo *info)
next->start, next->end);
fprintf(stderr, "%s: error while validating range/hash maps\n",
ndz_filename(info->ndz));
#if 1
#if 0
printf("==== Image ");
ndz_rangemap_dump(info->map, (debug==0), NULL);
printf("==== Hash ");
......@@ -503,6 +503,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->firstsect = 0;
cstate->header->lastsect = pstart;
/* include any relocations */
if (old.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -612,10 +624,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
/* finalize the header */
cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
/* XXX */
/* XXX should always be zero */
if (cstate->chunkno == 0)
cstate->header->firstsect = 0;
cstate->header->lastsect = new.ndz->maphi;
/* include any relocations */
if (old.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -732,11 +757,13 @@ main(int argc, char **argv)
*/
readifile(&old);
#if 1
#if 0
printf("==== Old range ");
ndz_rangemap_dump(old.map, (debug==0), chunkfunc);
printf("==== Old hash ");
ndz_hashmap_dump(old.sigmap, (debug==0));
if (old.sigmap) {
printf("==== Old hash ");
ndz_hashmap_dump(old.sigmap, (debug==0));
}
fflush(stdout);
#endif
......
......@@ -34,13 +34,14 @@ all: libndz.a
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o
OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o reloc.o
rangemap.o: rangemap.h
ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
chunk.o: libndz.h $(MAINDIR)/imagehdr.h
hash.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
reloc.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
libndz.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $?
......
......@@ -348,6 +348,7 @@ ndz_freehashmap(struct ndz_file *ndz)
struct deltainfo {
struct ndz_rangemap *omap;
struct ndz_rangemap *dmap;
struct ndz_rangemap *relocmap;
int omapdone;
};
......@@ -441,6 +442,21 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
#ifdef COMPDELTA_DEBUG
fprintf(stderr, " exact overlap with old map, ");
#endif
/*
* XXX if the sector range includes a relocation,
* we always force it into the image.
*/
if (dinfo->relocmap &&
ndz_rangemap_overlap(dinfo->relocmap, addr, eaddr-addr+1)) {
#ifdef COMPDELTA_DEBUG
fprintf(stderr, "has relocation, adding\n");
#endif
rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
assert(rv == 0);
return 0;
}
odata = orange->data;
ndata = range->data;
assert(odata->hashlen == ndata->hashlen);
......@@ -450,11 +466,11 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
#endif
rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
assert(rv == 0);
} else {
return 0;
}
#ifdef COMPDELTA_DEBUG
fprintf(stderr, "hash same, skipping\n");
fprintf(stderr, "hash same, skipping\n");
#endif
}
return 0;
}
}
......@@ -475,11 +491,15 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
}
struct ndz_rangemap *
ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
ndz_compute_delta(struct ndz_file *ondz, struct ndz_file *nndz)
{
struct ndz_rangemap *dmap;
struct ndz_rangemap *omap, *nmap, *dmap;
struct deltainfo dinfo;
if (ondz == NULL || (omap = ondz->rangemap) == NULL ||
nndz == NULL || (nmap = nndz->rangemap) == NULL)
return NULL;
if (omap == NULL || nmap == NULL)
return NULL;
......@@ -491,6 +511,7 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
#if 0
(void) ndz_rangemap_iterate(nmap, compdelta, &dinfo);
......@@ -504,12 +525,13 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
* computing hash alignments.
*/
struct ndz_rangemap *
ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
ndz_compute_delta_sigmap(struct ndz_file *ondz, struct ndz_file *nndz)
{
struct ndz_rangemap *dmap;
struct ndz_rangemap *omap, *nmap, *dmap;
struct deltainfo dinfo;
if (omap == NULL || nmap == NULL)
if (ondz == NULL || (omap = ondz->hashmap) == NULL ||
nndz == NULL || (nmap = nndz->hashmap) == NULL)
return NULL;
dmap = ndz_rangemap_init(nmap->loaddr, nmap->hiaddr);
......@@ -520,6 +542,7 @@ ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
(void) ndz_rangemap_iterate(nmap, compfastdelta, &dinfo);
......
......@@ -54,6 +54,10 @@ struct ndz_file {
unsigned chunkuses;
unsigned chunkhits;
unsigned chunkreopens;
/* relocation information */
unsigned relocentries;
void *relocdata;
struct ndz_rangemap *relocmap;
/* hash (signature) info */
unsigned hashtype;
unsigned hashblksize;
......@@ -103,16 +107,20 @@ int ndz_chunk_flush(ndz_chunk_t chobj, int withheader);
ssize_t ndz_chunk_left(ndz_chunk_t chobj);
ssize_t ndz_chunk_append(ndz_chunk_t chobj, void *buf, size_t bytes);
int ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
int ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
void ndz_reloc_free(struct ndz_file *ndz);
struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);
int ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile);
void ndz_hash_data(struct ndz_file *ndz, unsigned char *data,
unsigned long count, unsigned char *hash);
char *ndz_hash_dump(unsigned char *h, int hlen);
void ndz_hashmap_dump(struct ndz_rangemap *map, int summaryonly);
struct ndz_rangemap *ndz_compute_delta(struct ndz_rangemap *omap,
struct ndz_rangemap *nmap);
struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_rangemap *omap,
struct ndz_rangemap *nmap);
struct ndz_rangemap *ndz_compute_delta(struct ndz_file *ondz,
struct ndz_file *nndz);
struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_file *ondz,
struct ndz_file *nndz);
#endif /* _LIBNDZ_H_ */
......
......@@ -115,6 +115,7 @@ ndz_close(struct ndz_file *ndz)
rv = close(ndz->fd);
if (rv == 0) {
ndz_reloc_free(ndz);
if (ndz->hashmap)
ndz_rangemap_deinit(ndz->hashmap);
if (ndz->rangemap)
......@@ -232,6 +233,7 @@ ndz_readranges(struct ndz_file *ndz)
last = hdr->lastsect;
}
/* get the regions */
reg = head.region;
assert(reg != NULL || hdr->regioncount == 0);
for (i = 0; i < hdr->regioncount; i++) {
......@@ -254,6 +256,26 @@ ndz_readranges(struct ndz_file *ndz)
}
reg++;
}
/* get the relocations */
if (head.reloc) {
if (ndz->relocmap == NULL) {
ndz->relocmap =
ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
if (ndz->relocmap == NULL) {
fprintf(stderr, "%s: could not allocate relocmap\n",
ndz->fname);
ndz_rangemap_deinit(map);
return NULL;
}
}
if (ndz_reloc_get(ndz, head.header, head.reloc) != 0) {
fprintf(stderr, "%s: could not add relocations\n",
ndz->fname);
ndz_rangemap_deinit(map);
return NULL;
}
}
}
ndz->maplo = first;
......
......@@ -303,6 +303,8 @@ ndz_rangemap_iterate(struct ndz_rangemap *map,
/*
* Return a pointer to the first entry in the map.
* Returns NULL if map is empty.
*
* Does not affect the map hint.
*/
struct ndz_range *
ndz_rangemap_first(struct ndz_rangemap *map)
......@@ -313,6 +315,8 @@ ndz_rangemap_first(struct ndz_rangemap *map)
/*
* Return a pointer to the last entry in the map.
* Returns NULL if map is empty.
*
* Does not affect the map hint.
*/
struct ndz_range *
ndz_rangemap_last(struct ndz_rangemap *map)
......@@ -337,6 +341,8 @@ ndz_rangemap_last(struct ndz_rangemap *map)
/*
* Return the number of entries in the map.
*
* Does not affect the map hint.
*/
int
ndz_rangemap_count(struct ndz_rangemap *map)
......@@ -464,6 +470,52 @@ ndz_rangemap_lookup(struct ndz_rangemap *map, ndz_addr_t addr,
return range;
}
/*
* Determine if the indicated range [addr - addr+size-1] overlaps with
* any entry in the given map. Returns a pointer to the first map entry
* that overlaps, or NULL if there is no overlap.
*
* Resulting map hint values:
* if range is found, hint points to it.
* if range is not found, hint is unaffected.
*/
struct ndz_range *
ndz_rangemap_overlap(struct ndz_rangemap *map, ndz_addr_t addr,
ndz_size_t size)
{
struct ndz_range *range, *prev;
ndz_addr_t eaddr = addr + size - 1;
/*
* Lookup the first address in the range.
* If it falls within a range in the map, we are done.
* Otherwise look for overlap with the following map entry.
*/
range = ndz_rangemap_lookup(map, addr, &prev);
if (range == NULL) {
/* if prev is null we are at the beginning */
if (prev == NULL)
range = map->head.next;
else
range = prev->next;
/*
* At the end of the map, there can be no overlap.
*/
if (range == NULL)
return NULL;
/*
* If our range ends before the next element in the list,
* there is no overlap.
*/
if (eaddr < range->start)
return NULL;
}
return range;
}
void
ndz_rangemap_dumpstats(struct ndz_rangemap *map)
{
......
......@@ -78,6 +78,8 @@ int ndz_rangemap_count(struct ndz_rangemap *map);
struct ndz_range *ndz_rangemap_lookup(struct ndz_rangemap *map,
ndz_addr_t addr,
struct ndz_range **prev);
struct ndz_range *ndz_rangemap_overlap(struct ndz_rangemap *map,
ndz_addr_t addr, ndz_size_t);
void ndz_rangemap_dump(struct ndz_rangemap *map, int summaryonly,
void (*dfunc)(struct ndz_rangemap *, void *));
void ndz_rangemap_dumpstats(struct ndz_rangemap *map);
......
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
/*
* Relocation handling routines.
*
* XXX using a range map is overkill, since there are almost never more
* than a few (one) relocations and they are single sector things.
* But images with relocations are rare, so don't worry about it.
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "libndz.h"
#define RELOC_DEBUG
int
ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *relocdata, *reloc = buf;
int i;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || reloc == NULL)
return -1;
if (hdr->magic < COMPRESSED_V2 || hdr->reloccount == 0)
return 0;
/* resize the relocation buffer */
i = ndz->relocentries + hdr->reloccount;
if (ndz->relocdata == NULL)
relocdata = malloc(i * sizeof(*reloc));
else
relocdata = realloc(ndz->relocdata, i * sizeof(*reloc));
if (relocdata == NULL) {
ndz_reloc_free(ndz);
return -1;
}
ndz->relocdata = relocdata;
relocdata = ndz->relocdata + ndz->relocentries;
memcpy(relocdata, reloc, hdr->reloccount * sizeof(*reloc));
for (i = 0; i < hdr->reloccount; i++) {
ndz_size_t size = (reloc->size + ndz->sectsize - 1) / ndz->sectsize;
if (ndz_rangemap_alloc(ndz->relocmap, (ndz_addr_t)reloc->sector, size,
&relocdata[i])) {
ndz_reloc_free(ndz);
return -1;
}
}
ndz->relocentries += hdr->reloccount;
#ifdef RELOC_DEBUG
if (hdr->reloccount > 0) {
fprintf(stderr, "got %d relocs, %d total\n",
hdr->reloccount, ndz->relocentries);
}
#endif
return 0;
}
int
ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *reloc;
struct ndz_range *rrange;
ndz_addr_t addr, eaddr;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || buf == NULL)
return -1;
reloc = buf;
addr = hdr->firstsect;
eaddr = hdr->lastsect;
rrange = ndz_rangemap_overlap(ndz->relocmap, addr, eaddr - addr + 1);
while (rrange && rrange->start <= eaddr) {
#ifdef RELOC_DEBUG
fprintf(stderr, "found reloc [%lu-%lu] in chunk range [%lu-%lu]\n",
rrange->start, rrange->end, addr, eaddr);
#endif
assert(rrange->start >= addr && rrange->end <= eaddr);
assert(rrange->data != NULL);
*reloc = *(struct blockreloc *)rrange->data;
reloc++;
hdr->reloccount++;
rrange = rrange->next;
}
return 0;
}
void
ndz_reloc_free(struct ndz_file *ndz)
{
if (ndz) {
if (ndz->relocdata) {
free(ndz->relocdata);
ndz->relocdata = NULL;
}
if (ndz->relocmap) {
ndz_rangemap_deinit(ndz->relocmap);
ndz->relocmap = NULL;
}
ndz->relocentries = 0;
}
}
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment