All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit e1613a02 authored by Mike Hibler's avatar Mike Hibler

Deal with relocations.

That was more involved than I thought...or it was worth, considering that
only old FreeBSD and really old Redhat (think: LILO) images have them.
But that is the kinda guy I am...
parent be00c1c1
......@@ -528,6 +528,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->firstsect = 0;
cstate->header->lastsect = pstart;
/* include any relocations */
if (ndz2.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -634,10 +646,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
/* finalize the header */
cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
/* XXX */
/* XXX should always be zero */
if (cstate->chunkno == 0)
cstate->header->firstsect = 0;
cstate->header->lastsect = delta.ndz->maphi;
/* include any relocations */
if (ndz2.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
delta.ndz->relocmap = ndz2.ndz->relocmap; /* XXX */
if (ndz_reloc_put(delta.ndz, cstate->header, buf) != 0) {
delta.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
delta.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -757,7 +782,7 @@ main(int argc, char **argv)
argv[1], ndz2.ndz->hashtype, ndz2.ndz->hashblksize);
exit(1);
}
delta.map = ndz_compute_delta_sigmap(ndz1.sigmap, ndz2.sigmap);
delta.map = ndz_compute_delta_sigmap(ndz1.ndz, ndz2.ndz);
if (delta.map == NULL) {
fprintf(stderr, "Could not compute delta for %s and %s\n",
argv[0], argv[1]);
......@@ -785,7 +810,7 @@ main(int argc, char **argv)
* Same deal, but construct the delta map from the ranges maps.
*/
else {
delta.map = ndz_compute_delta(ndz1.map, ndz2.map);
delta.map = ndz_compute_delta(ndz1.ndz, ndz2.ndz);
if (delta.map == NULL) {
fprintf(stderr, "Could not compute delta for %s and %s\n",
argv[0], argv[1]);
......@@ -862,8 +887,7 @@ main(int argc, char **argv)
free(cstate);
/* readjust to reflect the actual number of hash entries */
if (!fullsig)
delta.ndz->hashentries = delta.ndz->hashcurentry;
delta.ndz->hashentries = delta.ndz->hashcurentry;
/* write the new sigfile */
if (ndz_writehashinfo(fullsig ? ndz2.ndz : delta.ndz,
......@@ -875,7 +899,8 @@ main(int argc, char **argv)
ndz_close(ndz2.ndz);
ndz_close(delta.ndz);
} else {
fprintf(stderr, "Images are identical, no delta produced!\n");
fprintf(stderr, "Images %s and %s are identical, no delta produced!\n",
argv[0], argv[1]);
ndz_close(ndz2.ndz);
ndz_close(delta.ndz);
unlink(argv[2]);
......
......@@ -133,6 +133,33 @@ struct blockhdr_V4 {
/* Unique ID for the whole image */
};
/*
* Coming soon in V5:
*
* 64-bit support.
* Grow blockindex, firstsect, lastsect, region descriptors, may need to
* grow DEFAULTREGIONSIZE to accomodate the doubling in size of regions.
*
* Flag field?
* For example, to indicate a delta image. Would probably take over the
* otherwise unused blocktotal field.
*
* Sectorsize field?
* To make explicit the units of sector fields; e.g., 512 vs 4096.
*
* Chunksize field?
* To support different chunksizes.
*
* Mandate little-endian on-disk data.
* Code changes only to use appropriate endian macros when reading/writing
* data. No data struct changes needed.
*
* Support for SHA256 and SHA512 checksums.
* Just some constants here, as checksum is already 64 bytes.
* Will need to grow the imagehash header to accomodate these checksums
* for signatures, but that is separately versioned.
*/
/*
* Checksum types supported
*/
......
......@@ -21,7 +21,7 @@
* }}}
*/
#define CHUNKIFY_DEBUG
//#define CHUNKIFY_DEBUG
/*
* imagerezip [-S] Oimage.ndz Nimage.ndz
......@@ -271,7 +271,7 @@ readifile(struct fileinfo *info)
next->start, next->end);
fprintf(stderr, "%s: error while validating range/hash maps\n",
ndz_filename(info->ndz));
#if 1
#if 0
printf("==== Image ");
ndz_rangemap_dump(info->map, (debug==0), NULL);
printf("==== Hash ");
......@@ -503,6 +503,18 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
cstate->header->firstsect = 0;
cstate->header->lastsect = pstart;
/* include any relocations */
if (old.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -612,10 +624,23 @@ chunkify(struct ndz_rangemap *mmap, struct ndz_range *range, void *arg)
/* finalize the header */
cstate->header->size = ndz_chunk_datasize(cstate->chunkobj);
cstate->header->regioncount = (cstate->curregion - cstate->region + 1);
/* XXX */
/* XXX should always be zero */
if (cstate->chunkno == 0)
cstate->header->firstsect = 0;
cstate->header->lastsect = new.ndz->maphi;
/* include any relocations */
if (old.ndz->relocmap) {
void *buf = (cstate->curregion + 1);
new.ndz->relocmap = old.ndz->relocmap; /* XXX */
if (ndz_reloc_put(new.ndz, cstate->header, buf) != 0) {
new.ndz->relocmap = NULL; /* XXX */
fprintf(stderr, "Error writing relocation info\n");
return 1;
}
new.ndz->relocmap = NULL; /* XXX */
}
/* and write it */
if (ndz_chunk_flush(cstate->chunkobj, 1) != 0) {
fprintf(stderr, "Error writing compressed data\n");
......@@ -732,11 +757,13 @@ main(int argc, char **argv)
*/
readifile(&old);
#if 1
#if 0
printf("==== Old range ");
ndz_rangemap_dump(old.map, (debug==0), chunkfunc);
printf("==== Old hash ");
ndz_hashmap_dump(old.sigmap, (debug==0));
if (old.sigmap) {
printf("==== Old hash ");
ndz_hashmap_dump(old.sigmap, (debug==0));
}
fflush(stdout);
#endif
......
......@@ -34,13 +34,14 @@ all: libndz.a
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o
OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o reloc.o
rangemap.o: rangemap.h
ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
chunk.o: libndz.h $(MAINDIR)/imagehdr.h
hash.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
reloc.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
libndz.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $?
......
......@@ -348,6 +348,7 @@ ndz_freehashmap(struct ndz_file *ndz)
struct deltainfo {
struct ndz_rangemap *omap;
struct ndz_rangemap *dmap;
struct ndz_rangemap *relocmap;
int omapdone;
};
......@@ -441,6 +442,21 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
#ifdef COMPDELTA_DEBUG
fprintf(stderr, " exact overlap with old map, ");
#endif
/*
* XXX if the sector range includes a relocation,
* we always force it into the image.
*/
if (dinfo->relocmap &&
ndz_rangemap_overlap(dinfo->relocmap, addr, eaddr-addr+1)) {
#ifdef COMPDELTA_DEBUG
fprintf(stderr, "has relocation, adding\n");
#endif
rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
assert(rv == 0);
return 0;
}
odata = orange->data;
ndata = range->data;
assert(odata->hashlen == ndata->hashlen);
......@@ -450,11 +466,11 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
#endif
rv = ndz_rangemap_alloc(dmap, addr, eaddr-addr+1, NULL);
assert(rv == 0);
} else {
return 0;
}
#ifdef COMPDELTA_DEBUG
fprintf(stderr, "hash same, skipping\n");
fprintf(stderr, "hash same, skipping\n");
#endif
}
return 0;
}
}
......@@ -475,11 +491,15 @@ compfastdelta(struct ndz_rangemap *nmap, struct ndz_range *range, void *arg)
}
struct ndz_rangemap *
ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
ndz_compute_delta(struct ndz_file *ondz, struct ndz_file *nndz)
{
struct ndz_rangemap *dmap;
struct ndz_rangemap *omap, *nmap, *dmap;
struct deltainfo dinfo;
if (ondz == NULL || (omap = ondz->rangemap) == NULL ||
nndz == NULL || (nmap = nndz->rangemap) == NULL)
return NULL;
if (omap == NULL || nmap == NULL)
return NULL;
......@@ -491,6 +511,7 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
#if 0
(void) ndz_rangemap_iterate(nmap, compdelta, &dinfo);
......@@ -504,12 +525,13 @@ ndz_compute_delta(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
* computing hash alignments.
*/
struct ndz_rangemap *
ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
ndz_compute_delta_sigmap(struct ndz_file *ondz, struct ndz_file *nndz)
{
struct ndz_rangemap *dmap;
struct ndz_rangemap *omap, *nmap, *dmap;
struct deltainfo dinfo;
if (omap == NULL || nmap == NULL)
if (ondz == NULL || (omap = ondz->hashmap) == NULL ||
nndz == NULL || (nmap = nndz->hashmap) == NULL)
return NULL;
dmap = ndz_rangemap_init(nmap->loaddr, nmap->hiaddr);
......@@ -520,6 +542,7 @@ ndz_compute_delta_sigmap(struct ndz_rangemap *omap, struct ndz_rangemap *nmap)
dinfo.omap = omap;
dinfo.dmap = dmap;
dinfo.relocmap = nndz->relocmap;
dinfo.omapdone = 0;
(void) ndz_rangemap_iterate(nmap, compfastdelta, &dinfo);
......
......@@ -54,6 +54,10 @@ struct ndz_file {
unsigned chunkuses;
unsigned chunkhits;
unsigned chunkreopens;
/* relocation information */
unsigned relocentries;
void *relocdata;
struct ndz_rangemap *relocmap;
/* hash (signature) info */
unsigned hashtype;
unsigned hashblksize;
......@@ -103,16 +107,20 @@ int ndz_chunk_flush(ndz_chunk_t chobj, int withheader);
ssize_t ndz_chunk_left(ndz_chunk_t chobj);
ssize_t ndz_chunk_append(ndz_chunk_t chobj, void *buf, size_t bytes);
int ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
int ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf);
void ndz_reloc_free(struct ndz_file *ndz);
struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);
int ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile);
void ndz_hash_data(struct ndz_file *ndz, unsigned char *data,
unsigned long count, unsigned char *hash);
char *ndz_hash_dump(unsigned char *h, int hlen);
void ndz_hashmap_dump(struct ndz_rangemap *map, int summaryonly);
struct ndz_rangemap *ndz_compute_delta(struct ndz_rangemap *omap,
struct ndz_rangemap *nmap);
struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_rangemap *omap,
struct ndz_rangemap *nmap);
struct ndz_rangemap *ndz_compute_delta(struct ndz_file *ondz,
struct ndz_file *nndz);
struct ndz_rangemap *ndz_compute_delta_sigmap(struct ndz_file *ondz,
struct ndz_file *nndz);
#endif /* _LIBNDZ_H_ */
......
......@@ -115,6 +115,7 @@ ndz_close(struct ndz_file *ndz)
rv = close(ndz->fd);
if (rv == 0) {
ndz_reloc_free(ndz);
if (ndz->hashmap)
ndz_rangemap_deinit(ndz->hashmap);
if (ndz->rangemap)
......@@ -232,6 +233,7 @@ ndz_readranges(struct ndz_file *ndz)
last = hdr->lastsect;
}
/* get the regions */
reg = head.region;
assert(reg != NULL || hdr->regioncount == 0);
for (i = 0; i < hdr->regioncount; i++) {
......@@ -254,6 +256,26 @@ ndz_readranges(struct ndz_file *ndz)
}
reg++;
}
/* get the relocations */
if (head.reloc) {
if (ndz->relocmap == NULL) {
ndz->relocmap =
ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
if (ndz->relocmap == NULL) {
fprintf(stderr, "%s: could not allocate relocmap\n",
ndz->fname);
ndz_rangemap_deinit(map);
return NULL;
}
}
if (ndz_reloc_get(ndz, head.header, head.reloc) != 0) {
fprintf(stderr, "%s: could not add relocations\n",
ndz->fname);
ndz_rangemap_deinit(map);
return NULL;
}
}
}
ndz->maplo = first;
......
......@@ -303,6 +303,8 @@ ndz_rangemap_iterate(struct ndz_rangemap *map,
/*
* Return a pointer to the first entry in the map.
* Returns NULL if map is empty.
*
* Does not affect the map hint.
*/
struct ndz_range *
ndz_rangemap_first(struct ndz_rangemap *map)
......@@ -313,6 +315,8 @@ ndz_rangemap_first(struct ndz_rangemap *map)
/*
* Return a pointer to the last entry in the map.
* Returns NULL if map is empty.
*
* Does not affect the map hint.
*/
struct ndz_range *
ndz_rangemap_last(struct ndz_rangemap *map)
......@@ -337,6 +341,8 @@ ndz_rangemap_last(struct ndz_rangemap *map)
/*
* Return the number of entries in the map.
*
* Does not affect the map hint.
*/
int
ndz_rangemap_count(struct ndz_rangemap *map)
......@@ -464,6 +470,52 @@ ndz_rangemap_lookup(struct ndz_rangemap *map, ndz_addr_t addr,
return range;
}
/*
* Determine if the indicated range [addr - addr+size-1] overlaps with
* any entry in the given map. Returns a pointer to the first map entry
* that overlaps, or NULL if there is no overlap.
*
* Resulting map hint values:
* if range is found, hint points to it.
* if range is not found, hint is unaffected.
*/
struct ndz_range *
ndz_rangemap_overlap(struct ndz_rangemap *map, ndz_addr_t addr,
ndz_size_t size)
{
struct ndz_range *range, *prev;
ndz_addr_t eaddr = addr + size - 1;
/*
* Lookup the first address in the range.
* If it falls within a range in the map, we are done.
* Otherwise look for overlap with the following map entry.
*/
range = ndz_rangemap_lookup(map, addr, &prev);
if (range == NULL) {
/* if prev is null we are at the beginning */
if (prev == NULL)
range = map->head.next;
else
range = prev->next;
/*
* At the end of the map, there can be no overlap.
*/
if (range == NULL)
return NULL;
/*
* If our range ends before the next element in the list,
* there is no overlap.
*/
if (eaddr < range->start)
return NULL;
}
return range;
}
void
ndz_rangemap_dumpstats(struct ndz_rangemap *map)
{
......
......@@ -78,6 +78,8 @@ int ndz_rangemap_count(struct ndz_rangemap *map);
struct ndz_range *ndz_rangemap_lookup(struct ndz_rangemap *map,
ndz_addr_t addr,
struct ndz_range **prev);
struct ndz_range *ndz_rangemap_overlap(struct ndz_rangemap *map,
ndz_addr_t addr, ndz_size_t);
void ndz_rangemap_dump(struct ndz_rangemap *map, int summaryonly,
void (*dfunc)(struct ndz_rangemap *, void *));
void ndz_rangemap_dumpstats(struct ndz_rangemap *map);
......
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
/*
* Relocation handling routines.
*
* XXX using a range map is overkill, since there are almost never more
* than a few (one) relocations and they are single sector things.
* But images with relocations are rare, so don't worry about it.
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "libndz.h"
#define RELOC_DEBUG
int
ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *relocdata, *reloc = buf;
int i;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || reloc == NULL)
return -1;
if (hdr->magic < COMPRESSED_V2 || hdr->reloccount == 0)
return 0;
/* resize the relocation buffer */
i = ndz->relocentries + hdr->reloccount;
if (ndz->relocdata == NULL)
relocdata = malloc(i * sizeof(*reloc));
else
relocdata = realloc(ndz->relocdata, i * sizeof(*reloc));
if (relocdata == NULL) {
ndz_reloc_free(ndz);
return -1;
}
ndz->relocdata = relocdata;
relocdata = ndz->relocdata + ndz->relocentries;
memcpy(relocdata, reloc, hdr->reloccount * sizeof(*reloc));
for (i = 0; i < hdr->reloccount; i++) {
ndz_size_t size = (reloc->size + ndz->sectsize - 1) / ndz->sectsize;
if (ndz_rangemap_alloc(ndz->relocmap, (ndz_addr_t)reloc->sector, size,
&relocdata[i])) {
ndz_reloc_free(ndz);
return -1;
}
}
ndz->relocentries += hdr->reloccount;
#ifdef RELOC_DEBUG
if (hdr->reloccount > 0) {
fprintf(stderr, "got %d relocs, %d total\n",
hdr->reloccount, ndz->relocentries);
}
#endif
return 0;
}
int
ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *reloc;
struct ndz_range *rrange;
ndz_addr_t addr, eaddr;
if (ndz == NULL || ndz->relocmap == NULL || hdr == NULL || buf == NULL)
return -1;
reloc = buf;
addr = hdr->firstsect;
eaddr = hdr->lastsect;
rrange = ndz_rangemap_overlap(ndz->relocmap, addr, eaddr - addr + 1);
while (rrange && rrange->start <= eaddr) {
#ifdef RELOC_DEBUG
fprintf(stderr, "found reloc [%lu-%lu] in chunk range [%lu-%lu]\n",
rrange->start, rrange->end, addr, eaddr);
#endif
assert(rrange->start >= addr && rrange->end <= eaddr);
assert(rrange->data != NULL);
*reloc = *(struct blockreloc *)rrange->data;
reloc++;
hdr->reloccount++;
rrange = rrange->next;
}
return 0;
}
void
ndz_reloc_free(struct ndz_file *ndz)
{
if (ndz) {
if (ndz->relocdata) {
free(ndz->relocdata);
ndz->relocdata = NULL;
}
if (ndz->relocmap) {
ndz_rangemap_deinit(ndz->relocmap);
ndz->relocmap = NULL;
}
ndz->relocentries = 0;
}
}
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment