Commit d7f4f5ee authored by Mike Hibler's avatar Mike Hibler

Deal with fixups when creating delta images.

Add run time option (-M) to tell it to make imagezip try harder to
optimize metadata saving (aka, the free inode hack).
parent d62812ae
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -27,7 +27,7 @@
* to reduce the compressed size of the data.
*/
#define DO_INODES
#undef CLEAR_FREE_INODES
#define CLEAR_FREE_INODES
#ifndef DO_INODES
#undef CLEAR_FREE_INODES
......@@ -449,11 +449,6 @@ read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t offset)
* Look for free inodes
*/
if (cgp->cg_cs.cs_nifree != 0) {
#ifdef CLEAR_FREE_INODES
static uint32_t ufs1_magic = FS_UFS1_MAGIC;
static uint32_t ufs2_magic = FS_UFS2_MAGIC;
uint32_t *magic;
#endif
int tifree = 0;
unsigned long edboff;
int ino;
......@@ -476,63 +471,71 @@ read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t offset)
ino = cg * fsp->fs_ipg;
#ifdef CLEAR_FREE_INODES
if (debug > 1)
fprintf(stderr,
" \t ifree %9d\n",
cgp->cg_cs.cs_nifree);
if (debug > 2)
fprintf(stderr, " ");
magic = (fsp->fs_magic == FS_UFS2_MAGIC) ?
&ufs2_magic : &ufs1_magic;
for (count = i = 0; i < max; i++) {
if (isset(p, i)) {
continue;
}
if (ino_to_fsbo(fsp, ino+i) == 0) {
j = i;
while ((i+1) < max && !isset(p, i+1))
i++;
if (metaoptimize) {
static uint32_t ufs1_magic = FS_UFS1_MAGIC;
static uint32_t ufs2_magic = FS_UFS2_MAGIC;
uint32_t *magic;
dboff = fsbtodb(fsp, ino_to_fsba(fsp, ino+j));
edboff = fsbtodb(fsp, ino_to_fsba(fsp, ino+i));
if (debug > 1)
fprintf(stderr,
" \t ifree %9d\n",
cgp->cg_cs.cs_nifree);
if (debug > 2)
fprintf(stderr, " ");
magic = (fsp->fs_magic == FS_UFS2_MAGIC) ?
&ufs2_magic : &ufs1_magic;
for (count = i = 0; i < max; i++) {
if (isset(p, i)) {
continue;
}
if (ino_to_fsbo(fsp, ino+i) == 0) {
j = i;
while ((i+1) < max && !isset(p, i+1))
i++;
dboff = fsbtodb(fsp,
ino_to_fsba(fsp, ino+j));
edboff = fsbtodb(fsp,
ino_to_fsba(fsp, ino+i));
#if 0
fprintf(stderr, " found free inodes %d-%d"
" db %lu.%u to %lu.%u\n",
ino+j, ino+i,
dboff+offset, ino_to_fsbo(fsp, ino+j),
edboff+offset, ino_to_fsbo(fsp, ino+i));
fprintf(stderr, " found free inodes %d-%d"
" db %lu.%u to %lu.%u\n",
ino+j, ino+i,
dboff+offset, ino_to_fsbo(fsp, ino+j),
edboff+offset, ino_to_fsbo(fsp, ino+i));
#endif
tifree += (i+1 - j);
dbcount = edboff - dboff;
if ((i+1) == max)
dbcount++;
if (dbcount == 0)
continue;
tifree += (i+1 - j);
dbcount = edboff - dboff;
if ((i+1) == max)
dbcount++;
if (dbcount == 0)
continue;
addfixupfunc(inodefixup,
sectobytes(dboff+offset),
sectobytes(offset),
sectobytes(dbcount),
magic, sizeof(magic),
RELOC_NONE);
if (debug > 2) {
if (count)
fprintf(stderr, ",%s",
count % 4 ?
" " :
"\n ");
fprintf(stderr, "%lu:%ld",
dboff+offset, dbcount);
}
count++;
} else
tifree++;
}
assert(i == max);
addfixupfunc(inodefixup,
sectobytes(dboff+offset),
sectobytes(offset),
sectobytes(dbcount),
magic, sizeof(magic),
RELOC_NONE);
if (debug > 2) {
if (count)
fprintf(stderr, ",%s",
count % 4 ?
" " :
"\n ");
fprintf(stderr, "%lu:%ld",
dboff+offset, dbcount);
}
count++;
} else
tifree++;
if (debug > 2)
fprintf(stderr, "\n");
}
assert(i == max);
if (debug > 2)
fprintf(stderr, "\n");
#endif
/*
......@@ -574,7 +577,7 @@ read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t offset)
}
#ifdef CLEAR_FREE_INODES
if (tifree != cgp->cg_cs.cs_nifree)
if (metaoptimize && tifree != cgp->cg_cs.cs_nifree)
fprintf(stderr, "Uh-oh! found %d free inodes, "
"shoulda found %d\n",
tifree, cgp->cg_cs.cs_nifree);
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -11,6 +11,7 @@ extern int debug;
extern int secsize;
extern int slicemode;
extern int dorelocs;
extern int metaoptimize;
extern off_t devlseek(int fd, off_t off, int whence);
extern ssize_t devread(int fd, void *buf, size_t nbytes);
......@@ -21,6 +22,7 @@ extern void addfixup(off_t offset, off_t poffset, off_t size, void *data,
extern void addfixupfunc(void (*func)(void *, off_t, void *), off_t offset,
off_t poffset, off_t size, void *data, int dsize,
int reloctype);
extern int hasfixup(uint32_t soffset, uint32_t ssize);
extern SLICEMAP_PROCESS_PROTO(read_bsdslice);
extern SLICEMAP_PROCESS_PROTO(read_linuxslice);
......
......@@ -20,6 +20,8 @@
#include <sys/time.h>
#endif
#include "sliceinfo.h"
#include "global.h"
#include "imagehdr.h"
#include "hashmap.h"
#include "imagehash.h"
......@@ -43,10 +45,12 @@ struct hashstats {
uint32_t hash_scompares; /* sectors compared */
uint32_t hash_identical; /* hash blocks identical */
uint32_t hash_sidentical;/* sectors identical */
uint32_t gaps; /* free gaps in hash ranges */
uint32_t gaps; /* hash ranges with free gaps */
uint32_t gapsects; /* free sectors in gaps */
uint32_t unchangedgaps; /* hash ranges with gaps that hash ok */
uint32_t gapunchanged; /* unchanged free sectors in gaps */
uint32_t gapnocompare; /* uncompared sectors in gaps */
uint32_t fixup; /* uncompared due to fixup overlap */
} hashstats;
struct timeval time_orig_read, time_curr_read, time_hash,
......@@ -588,20 +592,40 @@ hashmap_compute_delta(struct range *curranges, char *hfile, int infd,
(drange->start == hreg->region.start &&
drange->size >= hreg->region.size)) {
TIMEOP(
changed = hash_and_cmp(infd, hashfunc, hashlen,
hreg, ereg - hreg),
time_hash_and_cmp);
if (changed < 0)
goto error;
/*
* XXX if there is a fixup, all bets are off
* (e.g., they might compare equal now, but not
* after the fixup). Just force inclusion of all
* data.
*
* XXX we could do this on a drange by drange basis
* below, but I deem it not worth the trouble since
* all this code will be changing anyway.
*/
if (hasfixup(hreg->region.start, hreg->region.size)) {
changed = 3;
#ifdef FOLLOW
fprintf(stderr, " H: [%u-%u] fixup overlap\n",
hreg->region.start,
hreg->region.start + hreg->region.size-1);
#endif
} else {
TIMEOP(
changed = hash_and_cmp(infd, hashfunc,
hashlen, hreg,
ereg - hreg),
time_hash_and_cmp);
if (changed < 0)
goto error;
#ifdef FOLLOW
fprintf(stderr, " H: [%u-%u] hash compare: %s\n",
hreg->region.start,
hreg->region.start + hreg->region.size - 1,
changed ? "differ" : "match");
fprintf(stderr, " H: [%u-%u] hash %s\n",
hreg->region.start,
hreg->region.start + hreg->region.size-1,
changed ? "differs" : "matches");
#endif
}
} else {
/*
* There is a gap in the dranges covered by the hreg.
......@@ -619,8 +643,11 @@ hashmap_compute_delta(struct range *curranges, char *hfile, int infd,
hashstats.shared += hreg->region.size;
if (!changed)
hashstats.unchanged += hreg->region.size;
else if (changed == 2)
else if (changed > 1) {
hashstats.nocompare += hreg->region.size;
if (changed == 3)
hashstats.fixup += hreg->region.size;
}
gapstart = hreg->region.start;
gapsize = gapcount = 0;
#endif
......@@ -734,9 +761,12 @@ hashmap_compute_delta(struct range *curranges, char *hfile, int infd,
hashstats.gapsects += gapsize;
if (!changed) {
hashstats.unchanged -= gapsize;
hashstats.unchangedgaps++;
hashstats.gapunchanged += gapsize;
} else if (changed == 2) {
} else if (changed > 1) {
hashstats.nocompare -= gapsize;
if (changed == 3)
hashstats.fixup -= gapsize;
hashstats.gapnocompare += gapsize;
}
#ifdef FOLLOW
......@@ -901,10 +931,14 @@ report_hash_stats(int pnum)
hashstats.gaps);
fprintf(stderr, " Total free sectors covered: %u\n",
hashstats.gapsects);
fprintf(stderr, " Hash blocks compared identical: %u\n",
hashstats.unchangedgaps);
fprintf(stderr, " Free sectors compared identical: %u\n",
hashstats.gapunchanged);
fprintf(stderr, " Allocated sectors assumed changed: %u\n",
hashstats.nocompare);
fprintf(stderr, " Assumed changed due to fixups: %u\n",
hashstats.fixup);
fprintf(stderr,"\nEND HASH STATS\n");
#endif
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -51,6 +51,7 @@ int forcereads= 0;
int badsectors= 0;
int retrywrites= 1;
int dorelocs = 1;
int metaoptimize = 0;
off_t datawritten;
partmap_t ignore, forceraw;
......@@ -119,7 +120,7 @@ int read_shd(char *shddev, char *infile, int infd, u_int32_t ssect,
#ifdef WITH_HASH
struct range *hashmap_compute_delta(struct range *, char *, int, u_int32_t);
void report_hash_stats(void);
void report_hash_stats(int pnum);
#endif
static SLICEMAP_PROCESS_PROTO(read_slice);
......@@ -369,7 +370,7 @@ main(int argc, char *argv[])
extern char build_info[];
gettimeofday(&sstamp, 0);
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:oI:1F:DR:S:XC:H:")) != -1)
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:oI:1F:DR:S:XC:H:M")) != -1)
switch(ch) {
case 'v':
version++;
......@@ -453,6 +454,9 @@ main(int argc, char *argv[])
usage();
#endif
break;
case 'M':
metaoptimize++;
break;
case 'h':
case '?':
default:
......@@ -600,10 +604,6 @@ main(int argc, char *argv[])
* known allocated range that we have just computed. The result
* is a new list of ranges that are currently allocated and that
* have changed from the signature version.
*
* XXX we need to consider relocations here. If an existing
* range has an associated fixup, we should always include it in
* the image.
*/
if (hashfile != NULL) {
struct range *nranges;
......@@ -625,7 +625,7 @@ main(int argc, char *argv[])
fprintf(stderr, "\nAfter delta computation: ");
dumpranges(debug > 1);
}
report_hash_stats();
report_hash_stats(slice);
}
#endif
......@@ -648,6 +648,7 @@ main(int argc, char *argv[])
retrywrites = 0;
}
compress_image();
assert(fixups == NULL);
if (outcanseek)
close(outfd);
......@@ -1335,6 +1336,45 @@ cmpfixups(struct range *r1, struct range *r2)
return 0;
}
/*
* See if there is a fixup associated with any part of the given addr range.
* Returns 1 if so, 0 otherwise.
*/
int
hasfixup(uint32_t soffset, uint32_t ssize)
{
struct range *rp;
struct fixup *fp;
off_t offset, eoffset;
offset = sectobytes(soffset);
eoffset = offset + sectobytes(ssize);
for (rp = fixups; rp != NULL; rp = rp->next) {
fp = rp->data;
/* range completely before fixup, all done */
if (eoffset <= fp->offset)
break;
/* range completely after fixup, keep looking */
if (offset >= fp->offset + fp->size)
continue;
/* otherwise, there is overlap */
#ifdef FOLLOW
fprintf(stderr, "R: [%u-%u] overlaps with F: [%u/%u-%u/%u]\n",
soffset, soffset+ssize-1,
bytestosec(fp->offset),
(uint32_t)fp->offset % SECSIZE,
bytestosec(fp->offset+fp->size-1),
(uint32_t)(fp->offset+fp->size-1) % SECSIZE);
#endif
return 1;
}
return 0;
}
/*
* Look for fixups which overlap the range [offset - offset+size-1].
* If an overlap is found, we overwrite the data for that range with
......@@ -1360,8 +1400,8 @@ applyfixups(off_t offset, off_t size, void *data)
fp,
bytestosec(fp->offset),
(uint32_t)fp->offset % SECSIZE,
bytestosec(fp->offset+fp->size),
(uint32_t)(fp->offset+fp->size) % SECSIZE);
bytestosec(fp->offset+fp->size-1),
(uint32_t)(fp->offset+fp->size-1) % SECSIZE);
#endif
/*
......@@ -1379,7 +1419,7 @@ applyfixups(off_t offset, off_t size, void *data)
*/
if (offset+size <= fp->offset) {
#ifdef FOLLOW
fprintf(stderr, "after, done\n");
fprintf(stderr, "falls after, done\n");
#endif
break;
}
......@@ -1433,8 +1473,8 @@ applyfixups(off_t offset, off_t size, void *data)
fprintf(stderr, "used, reduced to [%u/%u-%u/%u]\n",
bytestosec(fp->offset),
(uint32_t)fp->offset % SECSIZE,
bytestosec(fp->offset+fp->size),
(uint32_t)(fp->offset+fp->size) % SECSIZE);
bytestosec(fp->offset+fp->size-1),
(uint32_t)(fp->offset+fp->size-1) % SECSIZE);
#endif
prev = &entry->next;
} else {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment