...
 
Commits (6)
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -32,7 +32,7 @@
static int read_ext4group(struct ext4_super_block *super,
struct ext4_group_desc *group, int index,
u_int32_t sliceoffset, int infd);
uint64_t sliceoffset, int infd);
extern int fixup_lilo(int slice, int stype, u_int32_t start, u_int32_t size,
char *sname, int infd, int *found);
......@@ -60,7 +60,7 @@ read_linuxslice(int slice, iz_type stype, iz_lba start, iz_size size,
unsigned char groups[EXT4_MAX_BLOCK_SIZE];
int dosslice = slice + 1; /* DOS Numbering */
off_t soff;
u_int32_t fssect;
iz_lba fssect;
/*
* Check for a LILO boot block and create relocations as necessary
......@@ -122,7 +122,7 @@ read_linuxslice(int slice, iz_type stype, iz_lba start, iz_size size,
(off_t)fs.s_blocks_count_lo);
if (excludenonfs && fssect < size) {
warnx("Linux Slice %d: filesystem smaller than partition, "
"excluding [%u-%u]",
"excluding [%lu-%lu]",
dosslice, start+fssect, start+size-1);
addskip(start + fssect, size - fssect);
}
......@@ -216,7 +216,7 @@ static int
read_ext4group(struct ext4_super_block *super,
struct ext4_group_desc *group,
int index,
u_int32_t sliceoffset /* Sector offset of slice */,
uint64_t sliceoffset /* Sector offset of slice */,
int infd)
{
char *p, bitmap[EXT4_MAX_BLOCK_SIZE];
......@@ -366,7 +366,7 @@ read_linuxswap(int slice, iz_type stype, iz_lba start, iz_size size,
fprintf(stderr,
" P%d (Linux Swap)\n", slice + 1 /* DOS Numbering */);
fprintf(stderr,
" start %12d, size %9d\n",
" start %12ld, size %9ld\n",
start, size);
}
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -54,8 +54,8 @@ static int _read_bsdslice(int slice, iz_type bsdtype, iz_lba start,
iz_size size, char *sname, int infd,
int musthavelabel);
static int read_bsdpartition(int infd, struct disklabel *dlabel, int part);
static int read_bsdsblock(int infd, u_int32_t off, int part, struct fs *fsp);
static int read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t off);
static int read_bsdsblock(int infd, uint64_t off, int part, struct fs *fsp);
static int read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, uint64_t off);
#ifdef CLEAR_FREE_INODES
static void inodefixup(void *buf, off_t buflen, void *fdata);
#endif
......@@ -63,7 +63,7 @@ static void inodefixup(void *buf, off_t buflen, void *fdata);
/* Map partition number to letter */
#define BSDPARTNAME(i) ("abcdefghijklmnop"[(i)])
static int32_t freecount;
static int64_t freecount;
/*
* Operate on a BSD slice
......@@ -298,8 +298,8 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
struct cg cg;
char pad[MAXBSIZE];
} cg;
u_int32_t size, offset, fssect;
int32_t sbfree;
uint64_t size, offset, fssect;
int64_t sbfree;
offset = dlabel->d_partitions[part].p_offset;
size = dlabel->d_partitions[part].p_size;
......@@ -336,7 +336,7 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
fssect = bytestosec(fs.fs_fsize * (off_t)fs.fs_size);
if (excludenonfs && fssect < size) {
warnx("BSD Partition '%c': filesystem smaller than partition, "
"excluding [%u-%u]",
"excluding [%lu-%lu]",
BSDPARTNAME(part), offset+fssect, offset+size-1);
addskip(offset + fssect, size - fssect);
}
......@@ -377,7 +377,7 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
if (rval == 0 && freecount != sbfree) {
warnx("BSD Partition '%c': "
"computed free count (%d) != expected free count (%d)",
"computed free count (%ld) != expected free count (%ld)",
BSDPARTNAME(part), freecount, sbfree);
}
......@@ -388,7 +388,7 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
* Includes code yanked from UFS2 ffs_vfsops.c
*/
static int
read_bsdsblock(int infd, u_int32_t offset, int part, struct fs *fsp)
read_bsdsblock(int infd, uint64_t offset, int part, struct fs *fsp)
{
static int sblock_try[] = SBLOCKSEARCH;
union {
......@@ -489,7 +489,7 @@ read_bsdsblock(int infd, u_int32_t offset, int part, struct fs *fsp)
}
static int
read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t offset)
read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, uint64_t offset)
{
int i, max;
u_int8_t *p;
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -34,15 +34,15 @@ extern int forcerelocs;
extern off_t devlseek(int fd, off_t off, int whence);
extern ssize_t devread(int fd, void *buf, size_t nbytes);
extern void addskip(uint32_t start, uint32_t size);
extern void addvalid(uint32_t start, uint32_t size);
extern void addskip(uint64_t start, uint64_t size);
extern void addvalid(uint64_t start, uint64_t size);
extern void addfixup(off_t offset, off_t poffset, off_t size, void *data,
int reloctype);
extern void addfixupfunc(void (*func)(void *, off_t, void *), off_t offset,
off_t poffset, off_t size, void *data, int dsize,
off_t poffset, off_t size, void *data, off_t dsize,
int reloctype);
extern void applyfixups(off_t offset, off_t size, void *data);
extern int hasfixup(uint32_t soffset, uint32_t ssize);
extern int hasfixup(uint64_t soffset, uint64_t ssize);
extern void savefixups(void);
extern void restorefixups(int isempty);
extern void dumpfixups(int verbose, int count);
......@@ -58,4 +58,4 @@ extern SLICEMAP_PROCESS_PROTO(read_ntfsslice);
extern SLICEMAP_PROCESS_PROTO(read_fatslice);
#define sectobytes(s) ((off_t)(s) * secsize)
#define bytestosec(b) (uint32_t)((b) / secsize)
#define bytestosec(b) ((uint64_t)(b) / secsize)
/*
* Copyright (c) 2000-2016 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -279,13 +279,6 @@ parse_gpt(int fd, struct iz_disk *disk, int dowarn)
parttab[i].offset = start;
parttab[i].size = size;
/* XXX right now imagezip only handles 32-bit off/size */
if ((uint64_t)(parttab[i].offset) != start ||
(uint64_t)(parttab[i].size) != size) {
warnx("P%d: Offset/size too large, ignoring", i+1);
parttab[i].flags |= IZFLAG_IGNORE;
}
if (type != IZTYPE_INVALID) {
if (start < losect)
losect = start;
......
/*
* Copyright (c) 2005-2014 University of Utah and the Flux Group.
* Copyright (c) 2005-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -148,7 +148,7 @@ dumphash(struct hashinfo *hinfo)
* Read a range of the disk and compute the hash value.
*/
static int
hash_range(uint32_t start, uint32_t size, unsigned char *hash)
hash_range(uint64_t start, uint64_t size, unsigned char *hash)
{
unsigned char *bp;
size_t count, byte_size;
......@@ -191,7 +191,7 @@ hash_range(uint32_t start, uint32_t size, unsigned char *hash)
/* XXX apply fixups */
if (hasfixup(start, size)) {
#ifdef FOLLOW
fprintf(stderr, " h: [%u-%u] applying fixups\n",
fprintf(stderr, " h: [%lu-%lu] applying fixups\n",
start, start + size - 1);
#endif
applyfixups(byte_start, byte_size, hashdata);
......@@ -212,7 +212,7 @@ hash_range(uint32_t start, uint32_t size, unsigned char *hash)
static int
hash_and_cmp(struct hashregion *hashreg, unsigned char *hash)
{
struct region hreg = hashreg->region;
struct region_32 hreg = hashreg->region;
int iretval;
assert(imagefd >= 0);
......@@ -284,7 +284,7 @@ static int
readhashinfo(char *hfile, struct hashinfo **hinfop)
{
struct hashinfo hi, *hinfo;
int fd, nregbytes, cc, i;
int fd, nregbytes, cc, i, is32;
assert(poffset != ~0);
......@@ -303,10 +303,12 @@ readhashinfo(char *hfile, struct hashinfo **hinfop)
return -1;
}
if (strcmp((char *)hi.magic, HASH_MAGIC) != 0 ||
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2)) {
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2 ||
hi.version == HASH_VERSION_3)) {
fprintf(stderr, "%s: not a valid signature file\n", hfile);
return -1;
}
is32 = (hi.version < HASH_VERSION_3) ? 1 : 0;
nregbytes = hi.nregions * sizeof(struct hashregion);
hinfo = malloc(sizeof(hi) + nregbytes);
if (hinfo == 0) {
......@@ -314,7 +316,11 @@ readhashinfo(char *hfile, struct hashinfo **hinfop)
return -1;
}
*hinfo = hi;
cc = read(fd, hinfo->regions, nregbytes);
if (is32) {
FIXME
} else {
cc = read(fd, hinfo->regions, nregbytes);
}
if (cc != nregbytes) {
free(hinfo);
return -1;
......@@ -577,7 +583,7 @@ add_to_hashmap(struct hashinfo **hinfop, uint32_t rstart, uint32_t rsize,
* in the range, mark it special.
*/
void
hashmap_update_chunk(uint32_t ssect, uint32_t lsect, int chunkno)
hashmap_update_chunk(uint64_t ssect, uint64_t lsect, int chunkno)
{
int i;
assert(chunkno >= 0);
......@@ -622,10 +628,10 @@ hashmap_update_chunk(uint32_t ssect, uint32_t lsect, int chunkno)
*/
int
hashmap_compute_delta(struct range *curranges, char *hfile, int infd,
uint32_t ssect, char *newhashfile,
uint64_t ssect, char *newhashfile,
struct range **nranges)
{
uint32_t gapstart, gapsize, lastdrangeend = 0;
uint64_t gapstart, gapsize, lastdrangeend = 0;
unsigned char hash[HASH_MAXSIZE];
struct range dummy_head, *range_tail;
struct hashregion *hreg, *ereg;
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -194,11 +194,11 @@ usage(void)
static char chunkbuf[CHUNKSIZE];
static unsigned int magic;
static unsigned long chunkcount;
static uint32_t nextsector, nextcovered;
static uint64_t nextsector, nextcovered;
static uint32_t fmax, fmin, franges, amax, amin, aranges;
static uint32_t adist[8]; /* <4k, <8k, <16k, <32k, <64k, <128k, <256k, >=256k */
static int regmax, regmin;
static uint32_t losect, hisect;
static uint64_t losect, hisect;
static uint8_t imageid[UUID_LENGTH];
static int sigtype, enctype;
......@@ -401,7 +401,7 @@ dumpfile(char *name, int fd)
printf(" %d relocations covering %llu bytes\n",
relocs, relocbytes);
if (hisect != ~0)
printf(" covered sector range: [%u-%u]\n",
printf(" covered sector range: [%lu-%lu]\n",
losect, hisect);
printf(" %llu bytes of compressed data\n",
cbytes);
......@@ -474,22 +474,35 @@ static int
dumpchunk(char *name, char *buf, int chunkno, int checkindex)
{
blockhdr_t *hdr;
struct region *reg;
region_t *reg = NULL;
uint32_t count;
int i;
int i, is32 = 1;
uint64_t first = 0, last = 0;
hdr = (blockhdr_t *)buf;
switch (hdr->magic) {
case COMPRESSED_V1:
reg = (struct region *)((struct blockhdr_V1 *)hdr + 1);
reg = (region_t *)((struct blockhdr_V1 *)hdr + 1);
break;
case COMPRESSED_V2:
case COMPRESSED_V3:
reg = (struct region *)((struct blockhdr_V2 *)hdr + 1);
reg = (region_t *)((struct blockhdr_V2 *)hdr + 1);
first = hdr->firstsect;
last = hdr->lastsect;
break;
case COMPRESSED_V5:
is32 = 0;
reg = (region_t *)((struct blockhdr_V5 *)hdr + 1);
first = hdr->firstsect64;
last = hdr->lastsect64;
/* fall into... */
case COMPRESSED_V4:
reg = (struct region *)((struct blockhdr_V4 *)hdr + 1);
if (reg == NULL) {
reg = (region_t *)((struct blockhdr_V4 *)hdr + 1);
first = hdr->firstsect;
last = hdr->lastsect;
}
if (chunkno > 0) {
if (sigtype != hdr->csum_type) {
printf("%s: wrong checksum type in chunk %d\n",
......@@ -552,17 +565,17 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
printf(" Chunk %d: %u compressed bytes, ",
chunkno, hdr->size);
if (hdr->magic > COMPRESSED_V1) {
if (hdr->firstsect != nextcovered) {
if (first != nextcovered) {
printf(" WARNING: chunk %d %s in covered "
"range, %u/%u last-end/cur-start\n",
"range, %lu/%lu last-end/cur-start\n",
chunkno,
(hdr->firstsect < nextcovered) ?
(first < nextcovered) ?
"overlap" : "gap", nextcovered,
hdr->firstsect);
first);
}
nextcovered = hdr->lastsect;
printf("sector range [%u-%u], ",
hdr->firstsect, hdr->lastsect-1);
nextcovered = last;
printf("sector range [%lu-%lu], ",
first, last-1);
if (hdr->reloccount > 0)
printf("%d relocs, ", hdr->reloccount);
}
......@@ -616,34 +629,37 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
hdr->regionsize, DEFAULTREGIONSIZE, chunkno);
for (i = 0; i < hdr->regioncount; i++) {
uint64_t rstart, rsize;
rstart = REG_START(is32, reg);
rsize = REG_SIZE(is32, reg);
if (detail > 1)
printf(" Region %d: %u sectors [%u-%u]\n",
i, reg->size, reg->start,
reg->start + reg->size - 1);
if (reg->start < nextsector)
printf(" Region %d: %lu sectors [%lu-%lu]\n",
i, rsize, rstart,
rstart + rsize - 1);
if (rstart < nextsector)
printf(" WARNING: chunk %d region %d "
"may overlap others\n", chunkno, i);
if (reg->size == 0)
if (rsize == 0)
printf(" WARNING: chunk %d region %d "
"zero-length region\n", chunkno, i);
count = 0;
if (hdr->magic > COMPRESSED_V1) {
if (i == 0) {
if (hdr->firstsect > reg->start)
if (first > rstart)
printf(" WARNING: chunk %d bad "
"firstsect value (%u>%u)\n",
chunkno, hdr->firstsect,
reg->start);
"firstsect value (%lu>%lu)\n",
chunkno, first, rstart);
else
count = reg->start - hdr->firstsect;
count = rstart - first;
} else
count = reg->start - nextsector;
count = rstart - nextsector;
if (i == hdr->regioncount-1) {
if (hdr->lastsect < reg->start + reg->size)
if (last < rstart + rsize)
printf(" WARNING: chunk %d bad "
"lastsect value (%u<%u)\n",
chunkno, hdr->lastsect,
reg->start + reg->size);
"lastsect value (%lu<%lu)\n",
chunkno, last,
rstart + rsize);
else {
if (count > 0) {
sectfree += count;
......@@ -653,16 +669,15 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
fmax = count;
franges++;
}
count = hdr->lastsect -
(reg->start+reg->size);
count = last - (rstart+rsize);
}
}
if (hdr->firstsect < losect)
losect = hdr->firstsect;
if ((hdr->lastsect-1) > hisect)
hisect = hdr->lastsect - 1;
if (first < losect)
losect = first;
if ((last-1) > hisect)
hisect = last - 1;
} else
count = reg->start - nextsector;
count = rstart - nextsector;
if (count > 0) {
sectfree += count;
if (count < fmin)
......@@ -672,7 +687,7 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
franges++;
}
count = reg->size;
count = rsize;
sectinuse += count;
if (count < amin)
amin = count;
......@@ -699,53 +714,63 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
if (dumpmap) {
switch (hdr->magic) {
case COMPRESSED_V1:
if (reg->start - nextsector != 0)
printf("F: [%08x-%08x]\n",
nextsector, reg->start-1);
printf("A: [%08x-%08x]\n",
reg->start, reg->start + reg->size - 1);
if (rstart - nextsector != 0)
printf("F: [%08lx-%08lx]\n",
nextsector, rstart-1);
printf("A: [%08lx-%08lx]\n",
rstart, rstart + rsize - 1);
break;
case COMPRESSED_V2:
case COMPRESSED_V3:
if (i == 0 && hdr->firstsect < reg->start)
printf("F: [%08x-%08x]\n",
hdr->firstsect, reg->start-1);
if (i != 0 && reg->start - nextsector != 0)
printf("F: [%08x-%08x]\n",
nextsector, reg->start-1);
printf("A: [%08x-%08x]\n",
reg->start, reg->start + reg->size - 1);
case COMPRESSED_V4:
case COMPRESSED_V5:
if (i == 0 && first < rstart)
printf("F: [%08lx-%08lx]\n",
first, rstart-1);
if (i != 0 && rstart - nextsector != 0)
printf("F: [%08lx-%08lx]\n",
nextsector, rstart-1);
printf("A: [%08lx-%08lx]\n",
rstart, rstart + rsize - 1);
if (i == hdr->regioncount-1 &&
reg->start+reg->size < hdr->lastsect)
printf("F: [%08x-%08x]\n",
reg->start+reg->size,
hdr->lastsect-1);
rstart+rsize < last)
printf("F: [%08lx-%08lx]\n",
rstart+rsize, last-1);
break;
}
}
nextsector = reg->start + reg->size;
reg++;
nextsector = rstart + rsize;
reg = REG_NEXT(is32, reg);
}
if (hdr->magic == COMPRESSED_V1)
return 0;
for (i = 0; i < hdr->reloccount; i++) {
struct blockreloc *reloc = &((struct blockreloc *)reg)[i];
blockreloc_t *reloc = RELOC_ADDR(is32, reg, i);
struct blockreloc_64 r;
if (is32) {
r.type = reloc->r32.type;
r.sector = reloc->r32.sector;
r.sectoff = reloc->r32.sectoff;
r.size = reloc->r32.size;
} else {
r = reloc->r64;
}
relocs++;
relocbytes += reloc->size;
relocbytes += r.size;
if (reloc->sector < hdr->firstsect ||
reloc->sector >= hdr->lastsect)
if (r.sector < first || r.sector >= last)
printf(" WARNING: "
"Reloc %d at %u not in chunk [%u-%u]\n", i,
reloc->sector, hdr->firstsect, hdr->lastsect-1);
"Reloc %d at %lu not in chunk [%lu-%lu]\n",
i, r.sector, first, last);
if (detail > 1) {
char *relocstr;
switch (reloc->type) {
switch (r.type) {
case RELOC_FBSDDISKLABEL:
relocstr = "FBSDDISKLABEL";
break;
......@@ -768,9 +793,9 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
relocstr = "??";
break;
}
printf(" Reloc %d: %s sector %d, offset %u-%u\n", i,
relocstr, reloc->sector, reloc->sectoff,
reloc->sectoff + reloc->size);
printf(" Reloc %d: %s sector %lu, offset %lu-%lu\n",
i, relocstr, r.sector, (uint64_t)r.sectoff,
(uint64_t)r.sectoff + r.size);
}
}
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -74,6 +74,7 @@ static int report = 0;
static int withchunkno = 1;
static int regfile = 0;
static int nothreads = 0;
static int hashversion = HASH_VERSION;
static int hashtype = HASH_TYPE_SHA1;
static int hashlen = 20;
static long hashblksize = HASHBLK_SIZE;
......@@ -119,8 +120,8 @@ static void free_readbuf(readbuf_t *rbuf);
static void dump_readbufs(void);
static void dump_stats(int sig);
#define sectobytes(s) ((off_t)(s) * SECSIZE)
#define bytestosec(b) (uint32_t)((b) / SECSIZE)
#define sectobytes(s) ((uint64_t)(s) * SECSIZE)
#define bytestosec(b) (uint64_t)((b) / SECSIZE)
int
main(int argc, char **argv)
......@@ -129,7 +130,7 @@ main(int argc, char **argv)
extern char build_info[];
struct hashinfo *hashinfo = 0;
while ((ch = getopt(argc, argv, "cCb:dvhno:rD:NVRF:SXqT")) != -1)
while ((ch = getopt(argc, argv, "cCb:dvhno:rD:NVRF:SXqT2")) != -1)
switch(ch) {
case 'b':
hashblksize = atol(optarg);
......@@ -204,6 +205,9 @@ main(int argc, char **argv)
case 'T':
terse = 1;
break;
case '2':
hashversion = HASH_VERSION_2;
break;
case 'h':
case '?':
default:
......@@ -275,6 +279,24 @@ main(int argc, char **argv)
signame(argv[0]));
exit(1);
}
/*
* Need to use the same hash params when hashing the image
* as are already recorded in the signature.
*/
hashversion = hashinfo->version;
hashtype = hashinfo->hashtype;
switch (hashtype) {
case HASH_TYPE_MD5:
default:
hashlen = 16;
break;
case HASH_TYPE_SHA1:
hashlen = 20;
break;
}
hashblksizeinsec = hashinfo->blksize;
if (hashimage(argv[0], &ihashinfo) != 0) {
fprintf(stderr, "Could not compute hashinfo for %s\n",
argv[0]);
......@@ -282,7 +304,7 @@ main(int argc, char **argv)
}
assert(ihashinfo != NULL);
strcpy((char *)ihashinfo->magic, HASH_MAGIC);
ihashinfo->version = HASH_VERSION_2;
ihashinfo->version = hashversion;
ihashinfo->hashtype = hashtype;
ihashinfo->blksize = hashblksizeinsec;
......@@ -421,11 +443,20 @@ readhashinfo(char *name, struct hashinfo **hinfop)
return -1;
}
if (strcmp((char *)hi.magic, HASH_MAGIC) != 0 ||
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2)) {
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2 ||
hi.version == HASH_VERSION3)) {
fprintf(stderr, "%s: not a valid signature file\n", hname);
goto bad;
}
nbytes = hi.nregions * sizeof(struct hashregion);
switch (hi.version) {
case HASH_VERSION_1:
case HASH_VERSION_2:
nbytes = hi.nregions * sizeof(struct hashregion_32);
break;
default:
nbytes = hi.nregions * sizeof(struct hashregion);
break;
}
hinfo = malloc(sizeof(hi) + nbytes);
if (hinfo == 0) {
fprintf(stderr, "%s: not enough memory for info\n", hname);
......@@ -472,10 +503,11 @@ readhashinfo(char *name, struct hashinfo **hinfop)
#define REGPERBLK 8192 /* ~256KB -- must be power of 2 */
static void
addhash(struct hashinfo **hinfop, uint32_t chunkno, uint32_t start,
uint32_t size, unsigned char hash[HASH_MAXSIZE])
addhash32(struct hashinfo **hinfop, uint32_t chunkno, uint32_t start,
uint32_t size, unsigned char hash[HASH_MAXSIZE])
{
struct hashinfo *hinfo = *hinfop;
struct hasregion_32 regions;
int nreg;
if (report) {
......@@ -487,6 +519,47 @@ addhash(struct hashinfo **hinfop, uint32_t chunkno, uint32_t start,
return;
}
if (hinfo == 0) {
nreg = 0;
hinfo = calloc(1, sizeof(*hinfo));
} else {
nreg = hinfo->nregions;
}
if ((nreg % REGPERBLK) == 0) {
hinfo = realloc(hinfo, sizeof(*hinfo) +
(nreg+REGPERBLK)*sizeof(struct hashregion_32));
if (hinfo == 0) {
fprintf(stderr, "out of memory for hash map\n");
exit(1);
}
*hinfop = hinfo;
}
regions = (struct hashregion_32 *)&hinfo->regions[0];
regions[nreg].chunkno = chunkno;
regions[nreg].region.start = start;
regions[nreg].region.size = size;
memcpy(regions[nreg].hash, hash, HASH_MAXSIZE);
hinfo->nregions++;
nhregions = hinfo->nregions;
}
static void
addhash(struct hashinfo **hinfop, uint32_t chunkno, uint64_t start,
uint64_t size, unsigned char hash[HASH_MAXSIZE])
{
struct hashinfo *hinfo = *hinfop;
int nreg;
if (report) {
printf("%s\t%lu\t%lu",
spewhash(hash, hashlen), start, size);
if (report > 1)
printf("\t%u\tU\t%s", chunkno, fileid);
putchar('\n');
return;
}
if (hinfo == 0) {
nreg = 0;
hinfo = calloc(1, sizeof(*hinfo));
......@@ -528,6 +601,10 @@ dumphash(char *name, struct hashinfo *hinfo, int withchunk)
printf("sig version 2, blksize=%d sectors:\n",
hinfo->blksize);
break;
case HASH_VERSION_3:
printf("sig version 3, blksize=%d sectors:\n",
hinfo->blksize);
break;
default:
printf("unknown signature version (%x), "
"expect garbage:\n", hinfo->version);
......@@ -1464,7 +1541,7 @@ hashfilechunk(int chunkno, char *chunkbufp, int chunksize,
struct hashinfo **hinfop)
{
int resid;
uint32_t cursect = 0, nbytes;
uint64_t cursect = 0, nbytes;
unsigned char hash[HASH_MAXSIZE];
unsigned char *(*hashfunc)(const unsigned char *, size_t,
unsigned char *);
......@@ -1495,7 +1572,7 @@ hashfilechunk(int chunkno, char *chunkbufp, int chunksize,
*/
resid = chunksize - sectobytes(bytestosec(chunksize));
while (chunksize > 0) {
uint32_t rstart, rsize;
uint64_t rstart, rsize;
if (chunksize > hashblksize)
nbytes = hashblksize;
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -25,7 +25,8 @@
#define HASH_VERSION_1 0x20031107
#define HASH_VERSION_2 0x20140618
#define HASH_VERSION HASH_VERSION_2
#define HASH_VERSION_3 0x20180415
#define HASH_VERSION HASH_VERSION_3
#define HASH_MAGIC ".ndzsig"
#define HASHBLK_SIZE (64*1024)
......@@ -36,7 +37,13 @@
#define HASH_CHUNKSETSPAN(c) ((c) | (1 << 31))
struct hashregion {
struct region region;
struct region_64 region;
uint32_t chunkno;
unsigned char hash[HASH_MAXSIZE];
};
struct hashregion_32 {
struct region_32 region;
uint32_t chunkno;
unsigned char hash[HASH_MAXSIZE];
};
......@@ -48,7 +55,7 @@ struct hashinfo {
uint32_t nregions;
uint32_t blksize; /* V2: make hash blocksize explicit */
uint8_t pad[8];
struct hashregion regions[0];
struct hashregion regions[0]; /* V3: hash regions have 64 byte vals */
};
#define HASH_TYPE_MD5 1
......
/*
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -42,14 +42,17 @@
*
* V4 of the block descriptor adds support for integrety protection
* and encryption.
*
* V5 introduced 64-bit blocknumbers.
*/
#define COMPRESSED_MAGIC_BASE 0x69696969
#define COMPRESSED_V1 (COMPRESSED_MAGIC_BASE+0)
#define COMPRESSED_V2 (COMPRESSED_MAGIC_BASE+1)
#define COMPRESSED_V3 (COMPRESSED_MAGIC_BASE+2)
#define COMPRESSED_V4 (COMPRESSED_MAGIC_BASE+3)
#define COMPRESSED_V5 (COMPRESSED_MAGIC_BASE+4)
#define COMPRESSED_MAGIC_CURRENT COMPRESSED_V4
#define COMPRESSED_MAGIC_CURRENT COMPRESSED_V5
/*
* Each compressed block of the file has this little header on it.
......@@ -134,11 +137,36 @@ struct blockhdr_V4 {
};
/*
* Coming soon in V5:
*
* 64-bit support.
* Grow blockindex, firstsect, lastsect, region descriptors, may need to
* grow DEFAULTREGIONSIZE to accomodate the doubling in size of regions.
* Version 5 of the block descriptor adds support for 64-bit sectors/sizes.
* Images of this format also use 64-bit region and relocation structs.
*/
struct blockhdr_V5 {
uint32_t magic; /* magic/version */
uint32_t size; /* Size of compressed part */
int32_t blockindex; /* which block we are */
int32_t blocktotal; /* V1: total number of blocks */
int32_t regionsize; /* sizeof header + regions */
int32_t regioncount; /* number of regions */
/* V2 follows */
uint32_t firstsect; /* first sector described by block */
uint32_t lastsect; /* first sector past block */
int32_t reloccount; /* number of reloc entries */
/* V4 follows */
uint16_t enc_cipher; /* cipher was used to encrypt */
uint16_t csum_type; /* checksum algortihm used */
uint8_t enc_iv[ENC_MAX_KEYLEN];
/* Initialization vector */
unsigned char checksum[SIG_MAX_KEYLEN];
/* (Signed) checksum */
unsigned char imageid[UUID_LENGTH];
/* Unique ID for the whole image */
/* V5 follows */
uint64_t firstsect64; /* first sector described by block */
uint64_t lastsect64; /* first sector past block */
};
/*
* Coming some day in V6:
*
* Flag field?
* For example, to indicate a delta image. Would probably take over the
......@@ -153,19 +181,18 @@ struct blockhdr_V4 {
* Mandate little-endian on-disk data.
* Code changes only to use appropriate endian macros when reading/writing
* data. No data struct changes needed.
*
* Support for SHA256 and SHA512 checksums.
* Just some constants here, as checksum is already 64 bytes.
* Will need to grow the imagehash header to accomodate these checksums
* for signatures, but that is separately versioned.
*/
/*
* Checksum types supported
*/
#define CSUM_NONE 0 /* must be zero */
#define CSUM_SHA1 1 /* SHA1: default */
#define CSUM_SHA1 1 /* SHA1 */
#define CSUM_SHA1_LEN 20
#define CSUM_SHA256 1 /* SHA256 */
#define CSUM_SHA256_LEN 32
#define CSUM_SHA512 1 /* SHA512: default */
#define CSUM_SHA512_LEN 64
/* type field */
#define CSUM_TYPE 0xFF
......@@ -192,12 +219,57 @@ struct blockhdr_V4 {
*
* Relocation descriptors follow the region descriptors in the header area.
*/
struct blockreloc {
struct blockreloc_32 {
uint32_t type; /* relocation type (below) */
uint32_t sector; /* sector it applies to */
uint32_t sectoff; /* offset within the sector */
uint32_t size; /* size of data affected */
};
/* N.B. sector and sectoff swapped to avoid alignment issues */
struct blockreloc_64 {
uint32_t type; /* relocation type (below) */
uint32_t sectoff; /* offset within the sector */
uint64_t sector; /* sector it applies to */
uint64_t size; /* size of data affected */
};
typedef union blockreloc {
struct blockreloc_32 r32;
struct blockreloc_64 r64;
} blockreloc_t;
#define RELOC_VALID(is32, sec, size) \
(!is32 || (uint32_t)(sec) == (sec) || (uint32_t)(size) == size)
#define RELOC_ADDR(is32, base, ix) \
(is32 ? \
(blockreloc_t *)((struct blockreloc_32 *)(base) + (ix)) : \
(blockreloc_t *)((struct blockreloc_64 *)(base) + (ix)))
#define RELOC_RSIZE(is32, num) \
((num) * (is32 ? \
sizeof(struct blockreloc_32) : \
sizeof(struct blockreloc_64)))
#define RELOC_ADD(is32, ptr, _type, _sec, _secoff, _size) \
if (is32) { \
(ptr)->r32.type = (_type); \
(ptr)->r32.sector = (uint32_t)(_sec); \
(ptr)->r32.sectoff = (_secoff); \
(ptr)->r32.size = (uint32_t)(_size); \
} else { \
(ptr)->r64.type = (_type); \
(ptr)->r64.sector = (_sec); \
(ptr)->r64.sectoff = (_secoff); \
(ptr)->r64.size = (_size); \
}
#define RELOC_NEXT(is32, ptr) \
(is32 ? \
(blockreloc_t *)((struct blockreloc_32 *)(ptr) + 1) : \
(blockreloc_t *)((struct blockreloc_64 *)(ptr) + 1))
#define RELOC_NONE 0
#define RELOC_FBSDDISKLABEL 1 /* FreeBSD disklabel */
#define RELOC_OBSDDISKLABEL 2 /* OpenBSD disklabel */
......@@ -211,7 +283,7 @@ struct blockreloc {
#define RELOC_XOR16CKSUM 101 /* 16-bit XOR checksum */
#define RELOC_CKSUMRANGE 102 /* range of previous checksum */
typedef struct blockhdr_V4 blockhdr_t;
typedef struct blockhdr_V5 blockhdr_t;
/*
* This little struct defines the pair. Each number is in sectors. An array
......@@ -220,11 +292,54 @@ typedef struct blockhdr_V4 blockhdr_t;
* how we skip over parts of the disk that do not need to be written
* (swap, free FS blocks).
*/
struct region {
struct region_32 {
uint32_t start;
uint32_t size;
};
struct region_64 {
uint64_t start;
uint64_t size;
};
typedef union region {
struct region_32 r32;
struct region_64 r64;
} region_t;
#define REG_VALID(is32, start, size) \
(!is32 || (uint32_t)(start) == (start) || (uint32_t)(size) == size)
#define REG_DIFF(is32, last, first) \
(is32 ? \
((struct region_32 *)(last) - (struct region_32 *)(first)) : \
((struct region_64 *)(last) - (struct region_64 *)(first)))
#define REG_ADD(is32, ptr, _start, _size) \
if (is32) { \
(ptr)->r32.start = (uint32_t)(_start); \
(ptr)->r32.size = (uint32_t)(_size); \
} else { \
(ptr)->r64.start = (_start); \
(ptr)->r64.size = (_size); \
}
#define REG_NEXT(is32, ptr) \
(is32 ? \
(region_t *)((struct region_32 *)(ptr) + 1) : \
(region_t *)((struct region_64 *)(ptr) + 1))
#define REG_PREV(is32, ptr) \
(is32 ? \
(region_t *)((struct region_32 *)(ptr) - 1) : \
(region_t *)((struct region_64 *)(ptr) - 1))
#define REG_START(is32, ptr) \
(is32 ? (uint64_t)ptr->r32.start : ptr->r64.start)
#define REG_SIZE(is32, ptr) \
(is32 ? (uint64_t)ptr->r32.size : ptr->r64.size)
/*
* Each block has its own region header info.
*
......@@ -236,8 +351,11 @@ struct region {
* area before filling the region header. Since the latter is more likely
* to be common, we tend to the small-ish side.
*
* At 4K with 8 byte region descriptors, we can fix 512 regions into a
* single chunk.
* At 4K, with a V2 image having a 36 byte header and 8 byte region
* descriptors, we can fix 507 regions into a single chunk.
*
* At 4K, with a V5 image having a 360 byte header and 16 byte region
* descriptors, we can fix 233 regions into a single chunk.
*/
#define DEFAULTREGIONSIZE 4096
......
/*
* Copyright (c) 2000-2016 University of Utah and the Flux Group.
* Copyright (c) 2000-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -291,8 +291,8 @@ struct chunkstate {
unsigned char *chunkdatabuf;
blockhdr_t *header;
uint32_t headerleft;
struct region *region;
struct region *curregion;
region_t *region;
region_t *curregion;
};
static int
......
This diff is collapsed.
This diff is collapsed.
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
* Copyright (c) 2014-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -90,6 +90,7 @@ ndz_hash_data(struct ndz_file *ndz, unsigned char *data, unsigned long count,
MD5(data, count, hash);
}
#define SANITY_CHECK
/*
* Read the hash info from a signature file into a region map associated
......@@ -99,14 +100,17 @@ struct ndz_rangemap *
ndz_readhashinfo(struct ndz_file *ndz, char *sigfile)
{
struct hashinfo hi;
struct hashregion hr;
union {
struct hashregion_32 hr32;
struct hashregion hr64;
} hr;
int fd, cc, rv, i;
unsigned hashlen, blksize, hashtype;
struct ndz_rangemap *map;
struct ndz_hashdata *hashdata = NULL;
unsigned lhblock;
#if 0
unsigned lstart, lsize;
#ifdef SANITY_CHECK
uint64_t lstart, lsize;
#endif
if (ndz == NULL || sigfile == NULL)
......@@ -129,7 +133,8 @@ ndz_readhashinfo(struct ndz_file *ndz, char *sigfile)
return NULL;
}
if (strcmp((char *)hi.magic, HASH_MAGIC) != 0 ||
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2)) {
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2 ||
hi.version == HASH_VERSION_3)) {
fprintf(stderr, "%s: not a valid signature file\n", sigfile);
close(fd);
return NULL;
......@@ -158,47 +163,62 @@ ndz_readhashinfo(struct ndz_file *ndz, char *sigfile)
hashlen = (hashtype == HASH_TYPE_MD5) ? 16 : 20;
blksize = (hi.version == HASH_VERSION_1) ?
(HASHBLK_SIZE / ndz->sectsize) : hi.blksize;
ndz->hash32 = (hi.version < HASH_VERSION_3) ? 1 : 0;
lhblock = -1;
for (i = 0; i < hi.nregions; i++) {
cc = read(fd, &hr, sizeof(hr));
if (cc != sizeof(hr)) {
uint32_t chunkno;
uint64_t hstart, hsize;
uint8_t *hash;
size_t sz = ndz->hash32 ? sizeof(hr.hr32) : sizeof(hr.hr64);
cc = read(fd, &hr, sz);
if (cc != sz) {
fprintf(stderr, "%s: incomplete sig entry\n", sigfile);
free(hashdata);
close(fd);
return NULL;
}
hashdata[i].chunkno = hr.chunkno;
if (ndz->hash32) {
chunkno = hr.hr32.chunkno;
hstart = hr.hr32.region.start;
hsize = hr.hr32.region.size;
hash = &hr.hr32.hash[0];
} else {
chunkno = hr.hr64.chunkno;
hstart = hr.hr64.region.start;
hsize = hr.hr64.region.size;
hash = &hr.hr64.hash[0];
}
hashdata[i].chunkno = chunkno;
hashdata[i].hashlen = hashlen;
memcpy(hashdata[i].hash, hr.hash, HASH_MAXSIZE);
memcpy(hashdata[i].hash, hash, HASH_MAXSIZE);
#if 0
#ifdef SANITY_CHECK
/* Sanity check the ranges */
if (1) {
unsigned sb = hr.region.start / blksize;
unsigned eb = (hr.region.start+hr.region.size-1) / blksize;
uint64_t sb = hstart / blksize;
uint64_t eb = (hstart + hsize-1) / blksize;
if (sb != eb)
fprintf(stderr, "*** [%u-%u]: range spans hash blocks\n",
hr.region.start, hr.region.start+hr.region.size-1);
fprintf(stderr, "*** [%lu-%lu]: range spans hash blocks\n",
hstart, hstart + hsize-1);
if (sb == lhblock)
fprintf(stderr, "*** [%u-%u]: range in same hash block ([%u-%u]) as [%u-%u]\n",
hr.region.start, hr.region.start+hr.region.size-1,
fprintf(stderr, "*** [%lu-%lu]: range in same hash block ([%lu-%lu]) as [%lu-%lu]\n",
hstart, hstart + hsize-1,
sb*blksize, sb*blksize+blksize-1,
lstart, lstart+lsize-1);
lhblock = sb;
lstart = hr.region.start;
lsize = hr.region.size;
lstart = hstart;
lsize = hsize;
}
#endif
rv = ndz_rangemap_alloc(map, (ndz_addr_t)hr.region.start,
(ndz_size_t)hr.region.size,
rv = ndz_rangemap_alloc(map, (ndz_addr_t)hstart, (ndz_size_t)hsize,
(void *)&hashdata[i]);
if (rv) {
fprintf(stderr, "%s: bad hash region [%u-%u]\n",
ndz->fname,
(unsigned)hr.region.start,
(unsigned)hr.region.start+hr.region.size-1);
fprintf(stderr, "%s: bad hash region [%lu-%lu]\n",
ndz->fname, hstart, hstart+hsize-1);
ndz_rangemap_deinit(map);
free(hashdata);
close(fd);
......@@ -217,12 +237,22 @@ ndz_readhashinfo(struct ndz_file *ndz, char *sigfile)
return map;
}
struct hashiterarg {
int ofd;
int is32;
};
static int
writehinfo(struct ndz_rangemap *map, struct ndz_range *range, void *arg)
{
struct hashregion hr;
union {
struct hashregion_32 hr32;
struct hashregion hr64;
} hr;
struct ndz_hashdata *hd = range->data;
int ofd = (int)(uintptr_t)arg;
int ofd = ((struct hashiterarg *)arg)->ofd;
int is32 = ((struct hashiterarg *)arg)->is32;
size_t sz;
if (hd == NULL) {
fprintf(stderr, "no hash info for range [%lu-%lu]\n",
......@@ -231,13 +261,24 @@ writehinfo(struct ndz_rangemap *map, struct ndz_range *range, void *arg)
}
assert(hd->hashlen <= HASH_MAXSIZE);
hr.region.start = range->start;
hr.region.size = range->end - range->start + 1;
hr.chunkno = hd->chunkno;
memcpy(hr.hash, hd->hash, hd->hashlen);
if (hd->hashlen < HASH_MAXSIZE)
memset(&hr.hash[hd->hashlen], 0, HASH_MAXSIZE - hd->hashlen);
if (write(ofd, &hr, sizeof(hr)) != sizeof(hr))
if (is32) {
hr.hr32.region.start = (uint32_t)range->start;
hr.hr32.region.size = (uint32_t)(range->end - range->start + 1);
hr.hr32.chunkno = hd->chunkno;
memcpy(hr.hr32.hash, hd->hash, hd->hashlen);
if (hd->hashlen < HASH_MAXSIZE)
memset(&hr.hr32.hash[hd->hashlen], 0, HASH_MAXSIZE - hd->hashlen);
sz = sizeof(hr.hr32);
} else {
hr.hr64.region.start = range->start;
hr.hr64.region.size = range->end - range->start + 1;
hr.hr64.chunkno = hd->chunkno;
memcpy(hr.hr64.hash, hd->hash, hd->hashlen);
if (hd->hashlen < HASH_MAXSIZE)
memset(&hr.hr64.hash[hd->hashlen], 0, HASH_MAXSIZE - hd->hashlen);
sz = sizeof(hr.hr64);
}
if (write(ofd, &hr, sz) != sz)
return 1;
return 0;
......@@ -251,6 +292,7 @@ ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile)
{
int ofd, cc;
struct hashinfo hi;
struct hashiterarg ha;
if (ndz == NULL || ndz->hashmap == NULL || sigfile == NULL)
return -1;
......@@ -266,7 +308,7 @@ ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile)
memset(&hi, 0, sizeof(hi));
strcpy((char *)hi.magic, HASH_MAGIC);
hi.version = HASH_VERSION_2;
hi.version = ndz->hash32 ? HASH_VERSION_2 : HASH_VERSION_3;
hi.hashtype = ndz->hashtype;
hi.nregions = ndz->hashentries;
hi.blksize = ndz->hashblksize;
......@@ -287,8 +329,9 @@ ndz_writehashinfo(struct ndz_file *ndz, char *sigfile, char *ifile)
/*
* Iterate over the sigmap writing out the entries
*/
if (ndz_rangemap_iterate(ndz->hashmap, writehinfo,
(void *)(uintptr_t)ofd) != 0) {
ha.ofd = ofd;
ha.is32 = ndz->hash32;
if (ndz_rangemap_iterate(ndz->hashmap, writehinfo, (void *)&ha) != 0) {
fprintf(stderr,
"%s: could not write one or more hash entries to sigfile %s\n",
ifile, sigfile);
......
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
* Copyright (c) 2014-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -67,10 +67,12 @@ struct ndz_file {
unsigned chunkhits;
unsigned chunkreopens;
/* relocation information */
int reloc32;
unsigned relocentries;
void *relocdata;
ndz_addr_t reloclo, relochi;
/* hash (signature) info */
int hash32;
unsigned hashtype;
unsigned hashblksize;
unsigned hashentries;
......@@ -87,8 +89,8 @@ struct ndz_file {
struct ndz_chunkhdr {
blockhdr_t *header;
struct region *region;
struct blockreloc *reloc;
region_t *region;
blockreloc_t *reloc;
char data[DEFAULTREGIONSIZE];
};
......
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
* Copyright (c) 2014-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -207,13 +207,14 @@ ndz_readranges(struct ndz_file *ndz)
struct ndz_rangemap *map;
struct ndz_chunkhdr head;
blockhdr_t *hdr;
struct region *reg;
region_t *reg;
int rv, i;
ndz_chunkno_t chunkno;
ndz_addr_t first = 0, last = 0;
#ifdef USE_CHUNKMAP
struct stat sb;
#endif
int is32 = 1;
if (ndz == NULL || (ndz->flags & NDZ_FILE_WRITE) != 0)
return NULL;
......@@ -267,13 +268,18 @@ ndz_readranges(struct ndz_file *ndz)
if ((hdr = head.header) == NULL)
break;
if (hdr->magic >= COMPRESSED_V5)
is32 = 0;
if (hdr->magic != COMPRESSED_V1) {
ndz_addr_t firstsect = is32 ? hdr->firstsect : hdr->firstsect64;
ndz_addr_t lastsect = is32 ? hdr->lastsect : hdr->lastsect64;
if (chunkno == 0)
first = hdr->firstsect;
last = hdr->lastsect - 1;
first = firstsect;
last = lastsect - 1;
#ifdef USE_CHUNKMAP
clo = hdr->firstsect;
chi = hdr->lastsect - 1;
clo = firstsect;
chi = lastsect - 1;
#endif
}
......@@ -281,25 +287,32 @@ ndz_readranges(struct ndz_file *ndz)
reg = head.region;
assert(reg != NULL || hdr->regioncount == 0);
for (i = 0; i < hdr->regioncount; i++) {
ndz_addr_t rstart;
ndz_size_t rsize;
if (is32) {
rstart = (ndz_addr_t)reg->r32.start;
rsize = (ndz_size_t)reg->r32.size;
} else {
rstart = (ndz_addr_t)reg->r64.start;
rsize = (ndz_size_t)reg->r64.size;
}
if (hdr->magic == COMPRESSED_V1) {
if (chunkno == 0)
first = (ndz_addr_t)reg->start;
last = (ndz_addr_t)reg->start + (ndz_size_t)reg->size - 1;
first = rstart;
last = rstart + rsize - 1;
#ifdef USE_CHUNKMAP
if (i == 0)
clo = (ndz_addr_t)reg->start;
clo = rstart;
chi = last;
#endif
}
rv = ndz_rangemap_alloc(map,
(ndz_addr_t)reg->start,
(ndz_size_t)reg->size,
rv = ndz_rangemap_alloc(map, rstart, rsize,
(void *)(uintptr_t)(chunkno+1));
if (rv) {
fprintf(stderr, "%s: bad region [%u-%u]\n",
ndz->fname,
(unsigned)reg->start,
(unsigned)reg->start+reg->size-1);
fprintf(stderr, "%s: bad region [%lu-%lu]\n",
ndz->fname, rstart, rstart+rsize-1);
ndz_rangemap_deinit(map);
return NULL;
}
......@@ -357,8 +370,8 @@ ndz_readchunkheader(struct ndz_file *ndz, ndz_chunkno_t chunkno,
{
ssize_t cc;
blockhdr_t *hdr;
struct region *reg;
struct blockreloc *rel;
region_t *reg;
blockreloc_t *rel;
if (ndz == NULL || (ndz->flags & NDZ_FILE_WRITE) != 0)
return -1;
......@@ -380,17 +393,21 @@ ndz_readchunkheader(struct ndz_file *ndz, ndz_chunkno_t chunkno,
hdr = (blockhdr_t *)chunkhdr->data;
switch (hdr->magic) {
case COMPRESSED_V1:
reg = (struct region *)((struct blockhdr_V1 *)hdr + 1);
reg = (region_t *)((struct blockhdr_V1 *)hdr + 1);
rel = NULL;
break;
case COMPRESSED_V2:
case COMPRESSED_V3:
reg = (struct region *)((struct blockhdr_V2 *)hdr + 1);
rel = (struct blockreloc *)(reg + hdr->regioncount);
reg = (region_t *)((struct blockhdr_V2 *)hdr + 1);
rel = (blockreloc_t *)((struct region_32 *)reg + hdr->regioncount);
break;
case COMPRESSED_V4:
reg = (struct region *)((struct blockhdr_V4 *)hdr + 1);
rel = (struct blockreloc *)(reg + hdr->regioncount);
reg = (region_t *)((struct blockhdr_V4 *)hdr + 1);
rel = (blockreloc_t *)((struct region_32 *)reg + hdr->regioncount);
break;
case COMPRESSED_V5:
reg = (region_t *)((struct blockhdr_V5 *)hdr + 1);
rel = (blockreloc_t *)((struct region_64 *)reg + hdr->regioncount);
break;
default:
fprintf(stderr, "%s: bad version 0x%x, not an ndz file?\n",
......
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
* Copyright (c) 2014-2018 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -49,6 +49,7 @@ ndz_reloc_init(struct ndz_file *ndz)
{
assert(ndz != NULL);
ndz->reloc32 = 1;
ndz->relocdata = NULL;
ndz->relocentries = 0;
ndz->reloclo = NDZ_HIADDR;
......@@ -63,7 +64,7 @@ ndz_reloc_init(struct ndz_file *ndz)
int
ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *relocdata, *chunkreloc = buf;
blockreloc_t *relocdata, *chunkreloc = buf;
int i;
if (ndz == NULL || hdr == NULL || chunkreloc == NULL)
......@@ -72,28 +73,43 @@ ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
if (hdr->magic < COMPRESSED_V2 || hdr->reloccount == 0)
return 0;
if (ndz->relocdata == NULL)
ndz->reloc32 = hdr->magic >= COMPRESSED_V5 ? 0 : 1;
else {
int is32 = hdr->magic >= COMPRESSED_V5 ? 0 : 1;
assert(is32 == ndz->reloc32);
}
/* resize the relocation buffer */
i = ndz->relocentries + hdr->reloccount;
if (ndz->relocdata == NULL)
relocdata = malloc(i * sizeof(struct blockreloc));
relocdata = malloc(RELOC_RSIZE(ndz->reloc32, i));
else
relocdata = realloc(ndz->relocdata, i * sizeof(struct blockreloc));
relocdata = realloc(ndz->relocdata, RELOC_RSIZE(ndz->reloc32, i));
if (relocdata == NULL) {
ndz_reloc_free(ndz);
return -1;
}
ndz->relocdata = relocdata;
relocdata = (struct blockreloc *)ndz->relocdata + ndz->relocentries;
relocdata = RELOC_ADDR(ndz->reloc32, ndz->relocdata, ndz->relocentries);
for (i = 0; i < hdr->reloccount; i++) {
ndz_addr_t rsector = ndz->reloc32 ?
chunkreloc->r32.sector : chunkreloc->r64.sector;
if (ndz->reloclo == NDZ_HIADDR)
ndz->reloclo = chunkreloc->sector;
ndz->reloclo = rsector;
/* XXX we should be adding these in order; we assume this elsewhere */
assert(ndz->reloclo <= chunkreloc->sector);
if (chunkreloc->sector > ndz->relochi)
ndz->relochi = chunkreloc->sector;
assert(ndz->reloclo <= rsector);
if (rsector > ndz->relochi)
ndz->relochi = rsector;
*relocdata++ = *chunkreloc++;
if (ndz->reloc32)
relocdata->r32 = chunkreloc->r32;
else
relocdata->r64 = chunkreloc->r64;
relocdata = RELOC_NEXT(ndz->reloc32, relocdata);
chunkreloc = RELOC_NEXT(ndz->reloc32, chunkreloc);
}
ndz->relocentries += hdr->reloccount;
......@@ -114,29 +130,53 @@ ndz_reloc_get(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
int
ndz_reloc_put(struct ndz_file *ndz, blockhdr_t *hdr, void *buf)
{
struct blockreloc *chunkreloc, *relocdata;
blockreloc_t *chunkreloc, *relocdata;
ndz_addr_t first, last;
int i;
if (ndz == NULL || hdr == NULL || buf == NULL)
return -1;
if (ndz->relocentries == 0 ||
hdr->firstsect > ndz->relochi || hdr->lastsect <= ndz->reloclo)
if (ndz->reloc32) {
first = hdr->firstsect;
last = hdr->lastsect;
} else {
first = hdr->firstsect64;
last = hdr->lastsect64;
}
if (ndz->relocentries == 0 || first > ndz->relochi || last <= ndz->reloclo)
return 0;
chunkreloc = buf;
relocdata = ndz->relocdata;
for (i = 0; i < ndz->relocentries; i++) {
assert(relocdata->sectoff + relocdata->size <= ndz->sectsize);
if (relocdata->sector >= hdr->firstsect &&
relocdata->sector < hdr->lastsect) {
ndz_addr_t rsector, roffset;
ndz_size_t rsize;
if (ndz->reloc32) {
rsector = relocdata->r32.sector;
roffset = relocdata->r32.sectoff;
rsize = relocdata->r32.size;
} else {
rsector = relocdata->r64.sector;
roffset = relocdata->r64.sectoff;
rsize = relocdata->r64.size;
}
assert(roffset + rsize <= ndz->sectsize);
if (rsector >= first && rsector < last) {
#ifdef RELOC_DEBUG
fprintf(stderr, "found reloc for %