Commit f0885af0 authored by Mike Hibler's avatar Mike Hibler

A variety of stop-gap changes:

 * lint: compile cleanly under newer GCC and on 64-bit OS
 * rename SUBBLOCKSIZE to CHUNKSIZE since that is what it is
 * collect some more stats, primarily for frisbee event collection code
 * add stupid hacky bubble-sort workaround but don't define it by default
   cuz it is stupid...and hacky...but it allows mike to make an image of
   his home machine before he dies of old age
 * tweaks to libntfs version support
parent 5dbdd07f
......@@ -112,6 +112,16 @@ ifeq ($(WITH_NTFS),1)
ifndef HAVE_GCC3
CC = gcc30
endif
#
# Default to the old libntfs for now.
# Once proven, switch to 2.0.0.
#
ifndef NTFSVER
NTFSVER = 1.7.1
endif
export NTFSVER
CFLAGS += -DWITH_NTFS
SUBDIRS += ntfs
ZIPLIBS += ntfs/libntfs.a
......@@ -144,7 +154,7 @@ ZIPLIBS += hashmap/libhashmap.a
LIBS += -lcrypto $(PTHREADLIBS)
endif
all: $(SUBDIRS) imagezip imageunzip imagedump imagehash
all: imagezip imageunzip imagedump imagehash
whoami:
@echo "SYSTEM is $(SYSTEM)"
......@@ -155,7 +165,7 @@ endif
include $(TESTBED_SRCDIR)/GNUmakerules
imagezip: imagezip.o disksize.o version.o $(ZIPLIBS)
imagezip: $(SUBDIRS) imagezip.o disksize.o version.o $(ZIPLIBS)
$(CC) $(CFLAGS) imagezip.o disksize.o version.o $(ZIPLIBS) $(LIBS) -o imagezip
imagezip.o: imagezip.c
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2004 University of Utah and the Flux Group.
* Copyright (c) 2000-2004, 2009 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -31,7 +31,7 @@ static void fixup_sector(u_int32_t poff, sectaddr_t *sect);
static void fixup_map(int fd, u_int32_t startsect);
#define FOFFSET(_b, _s, _f) \
((u_int32_t)&((_s *)0)->_f + (u_int32_t)_b)
((u_int32_t)(long)&((_s *)0)->_f + (u_int32_t)_b)
int
fixup_lilo(int slice, int stype, u_int32_t start, u_int32_t size,
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* Copyright (c) 2000-2009 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -237,7 +237,8 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
if (debug) {
fprintf(stderr, " bfree %9lld, bsize %9d, cgsize %9d\n",
fs.fs_cstotal.cs_nbfree, fs.fs_bsize, fs.fs_cgsize);
(long long)fs.fs_cstotal.cs_nbfree,
fs.fs_bsize, fs.fs_cgsize);
}
assert(fs.fs_cgsize <= MAXBSIZE);
assert((fs.fs_cgsize % secsize) == 0);
......@@ -251,7 +252,8 @@ read_bsdpartition(int infd, struct disklabel *dlabel, int part)
if (devlseek(infd, sectobytes(cgoff), SEEK_SET) < 0) {
warn("BSD Partition '%c': "
"Could not seek to cg %d at %lld",
BSDPARTNAME(part), i, sectobytes(cgoff));
BSDPARTNAME(part), i,
(long long)sectobytes(cgoff));
return 1;
}
if ((cc = devread(infd, &cg, fs.fs_cgsize)) < 0) {
......
......@@ -90,8 +90,8 @@ main(int argc, char **argv)
if (!isstdin) {
if ((infd = open(argv[0], O_RDONLY, 0666)) < 0) {
perror("opening input file");
exit(1);
perror(argv[0]);
continue;
}
} else
infd = fileno(stdin);
......@@ -116,7 +116,7 @@ usage(void)
exit(1);
}
static char chunkbuf[SUBBLOCKSIZE];
static char chunkbuf[CHUNKSIZE];
static unsigned int magic;
static unsigned long chunkcount;
static uint32_t nextsector;
......@@ -142,6 +142,7 @@ dumpfile(char *name, int fd)
fmax = amax = 0;
fmin = amin = ~0;
franges = aranges = 0;
regmin = regmax = 0;
memset(adist, 0, sizeof(adist));
if (!isstdin) {
......@@ -151,7 +152,7 @@ dumpfile(char *name, int fd)
perror(name);
return;
}
if ((st.st_size % SUBBLOCKSIZE) != 0)
if ((st.st_size % CHUNKSIZE) != 0)
printf("%s: WARNING: "
"file size not a multiple of chunk size\n",
name);
......@@ -205,11 +206,11 @@ dumpfile(char *name, int fd)
checkindex = 0;
} else
chunkcount = hdr->blocktotal;
if ((filesize / SUBBLOCKSIZE) != chunkcount) {
if ((filesize / CHUNKSIZE) != chunkcount) {
if (chunkcount != 0) {
if (isstdin)
filesize = (off_t)chunkcount *
SUBBLOCKSIZE;
CHUNKSIZE;
else
printf("%s: WARNING: file size "
"inconsistant with "
......@@ -217,7 +218,7 @@ dumpfile(char *name, int fd)
"(%lu != %lu)\n",
name,
(unsigned long)
(filesize/SUBBLOCKSIZE),
(filesize/CHUNKSIZE),
chunkcount);
} else if (magic == COMPRESSED_V1) {
if (!ignorev1)
......@@ -231,7 +232,7 @@ dumpfile(char *name, int fd)
printf("%s: %llu bytes, %lu chunks, version %d\n",
name, (unsigned long long)filesize,
(unsigned long)(filesize / SUBBLOCKSIZE),
(unsigned long)(filesize / CHUNKSIZE),
hdr->magic - COMPRESSED_MAGIC_BASE + 1);
} else if (chunkno == 1 && !ignorev1) {
blockhdr_t *hdr = (blockhdr_t *)chunkbuf;
......@@ -257,7 +258,7 @@ dumpfile(char *name, int fd)
#endif
if (filesize == 0)
filesize = (off_t)(chunkno + 1) * SUBBLOCKSIZE;
filesize = (off_t)(chunkno + 1) * CHUNKSIZE;
cbytes = (unsigned long long)(filesize - wasted);
dbytes = SECTOBYTES(sectinuse);
......@@ -286,20 +287,29 @@ dumpfile(char *name, int fd)
SECTOBYTES(fmin), SECTOBYTES(fmax));
if (aranges) {
int maxsz, i;
uint32_t adistsum;
printf(" %d allocated ranges: %llu/%llu/%llu ave/min/max size\n",
aranges, SECTOBYTES(sectinuse)/aranges,
SECTOBYTES(amin), SECTOBYTES(amax));
printf(" size distribution:\n");
adistsum = 0;
maxsz = 4*SECSIZE;
for (i = 0; i < 7; i++) {
maxsz *= 2;
if (adist[i])
printf(" < %dk bytes: %d\n",
maxsz/1024, adist[i]);
if (adist[i]) {
adistsum += adist[i];
printf(" < %3dk bytes: %6d %4.1f%% %4.1f%%\n",
maxsz/1024, adist[i],
(double)adist[i]/aranges*100,
(double)adistsum/aranges*100);
}
}
if (adist[i]) {
printf(" >= %3dk bytes: %6d %4.1f%%\n",
maxsz/1024, adist[i],
(double)adist[i]/aranges*100);
}
if (adist[i])
printf(" >= %dk bytes: %d\n", maxsz/1024, adist[i]);
}
}
......@@ -336,16 +346,16 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
name, hdr->blocktotal, chunkcount, chunkno);
return 1;
}
if (hdr->size > (SUBBLOCKSIZE - hdr->regionsize)) {
if (hdr->size > (CHUNKSIZE - hdr->regionsize)) {
printf("%s: bad chunksize (%d > %d) in chunk %d\n",
name, hdr->size, SUBBLOCKSIZE-hdr->regionsize, chunkno);
name, hdr->size, CHUNKSIZE-hdr->regionsize, chunkno);
return 1;
}
#if 1
/* include header overhead */
wasted += SUBBLOCKSIZE - hdr->size;
wasted += CHUNKSIZE - hdr->size;
#else
wasted += ((SUBBLOCKSIZE - hdr->regionsize) - hdr->size);
wasted += ((CHUNKSIZE - hdr->regionsize) - hdr->size);
#endif
if (regmin == 0 || hdr->regioncount < regmin)
regmin = hdr->regioncount;
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2006 University of Utah and the Flux Group.
* Copyright (c) 2000-2009 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -63,7 +63,7 @@ static char *imagename;
static char *fileid = NULL;
static char *sigfile = NULL;
static char chunkbuf[SUBBLOCKSIZE];
static char chunkbuf[CHUNKSIZE];
static void usage(void);
static int gethashinfo(char *name, struct hashinfo **hinfo);
......
......@@ -29,17 +29,17 @@
/*
* Each compressed block of the file has this little header on it.
* Since each subblock is independently compressed, we need to know
* Since each block is independently compressed, we need to know
* its internal size (it will probably be shorter than 1MB) since we
* have to know exactly how much to give the inflator.
*/
struct blockhdr_V1 {
uint32_t magic;
uint32_t magic; /* magic/version */
uint32_t size; /* Size of compressed part */
int32_t blockindex;
int32_t blocktotal;
int32_t regionsize;
int32_t regioncount;
int32_t blockindex; /* netdisk: which block we are */
int32_t blocktotal; /* netdisk: total number of blocks */
int32_t regionsize; /* sizeof header + regions */
int32_t regioncount; /* number of regions */
};
/*
......@@ -69,7 +69,7 @@ struct blockhdr_V2 {
* absolute block numbers. This descriptor tells the unzipper what the
* data structure is and where it is located in the block.
*
* Relocation descriptors follow the region descriptors in the header block.
* Relocation descriptors follow the region descriptors in the header area.
*/
struct blockreloc {
uint32_t type; /* relocation type (below) */
......@@ -105,22 +105,33 @@ struct region {
};
/*
* In the new model, each sub region has its own region header info.
* But there is no easy way to tell how many regions before compressing.
* Just leave a page, and hope that 512 regions is enough!
* Each block has its own region header info.
*
* Since there is no easy way to tell how many regions will fit before
* we have compressed the region data, we just have to pick a size here.
* If this area is too small, it is possible that a highly fragmented image
* will fill this header before filling the data area of a block. If the
* region header area is too large, we will almost always fill up the data
* area before filling the region header. Since the latter is more likely
* to be common, we tend to the small-ish side.
*
* This number must be a multiple of the NFS read size in netdisk.
* At 4K with 8 byte region descriptors, we can fix 512 regions into a
* single chunk.
*/
#define DEFAULTREGIONSIZE (1024 * 4)
#define DEFAULTREGIONSIZE 4096
/*
* Ah, the frisbee protocol. The new world order is to break up the
* file into fixed 1MB chunks, with the region info prepended to each
* file into fixed chunks, with the region info prepended to each
* chunk so that it can be layed down on disk independently of all the
* chunks in the file.
*/
#define SUBBLOCKSIZE (1024 * 1024)
#define SUBBLOCKMAX (SUBBLOCKSIZE - DEFAULTREGIONSIZE)
#define F_BLOCKSIZE 1024
#define F_BLOCKSPERCHUNK 1024
#define CHUNKSIZE (F_BLOCKSIZE * F_BLOCKSPERCHUNK)
#define CHUNKMAX (CHUNKSIZE - DEFAULTREGIONSIZE)
/*
* Assumed sector (block) size
......
......@@ -37,8 +37,13 @@
#define MAXWRITEBUFMEM 0 /* 0 == unlimited */
long long totaledata = 0;
long long totalrdata = 0;
long long totalddata = 0; /* total decompressed data */
long long totaledata = 0; /* total bytes covered by image */
long long totalrdata = 0; /* total data written to disk (image+zeros) */
long long totalzdata = 0; /* total zeros written to disk */
long totalwriteops = 0; /* total write operations */
long totalzeroops = 0; /* total zero write operations */
long totalseekops = 0; /* total non-contiguous writes */
int totalchunks, donechunks;
......@@ -84,13 +89,12 @@ static int infd;
static int version= 0;
static unsigned fillpat= 0;
static int readretries = 0;
static char chunkbuf[SUBBLOCKSIZE];
static int nodecompress = 0;
static char chunkbuf[CHUNKSIZE];
#endif
int readmbr(int slice);
int fixmbr(int slice, int dtype);
#ifdef FRISBEE
static int write_subblock(int, const char *);
#endif
static int inflate_subblock(const char *);
void writezeros(off_t offset, off_t zcount);
void writedata(off_t offset, size_t count, void *buf);
......@@ -215,8 +219,25 @@ dump_stats(int sig)
fprintf(stderr, "%d chunks decompressed\n",
donechunks);
}
fprintf(stderr, "Wrote %lld bytes (%lld actual) in %ld seconds\n",
totaledata, totalrdata, (long)estamp.tv_sec);
fprintf(stderr,
"Decompressed %lld bytes, wrote %lld bytes (%lld actual) in %ld.%03ld seconds\n",
totalddata, totaledata, totalrdata,
(long)estamp.tv_sec, (long)estamp.tv_usec/1000);
fprintf(stderr,
"%lld bytes of data in %ld ops (%lld bytes/op), %ld seeks\n",
totalrdata, totalwriteops,
totalrdata/(totalwriteops?:1), totalseekops);
if (totalzdata > 0) {
fprintf(stderr,
" %lld bytes of disk data in %ld ops (%lld bytes/op)\n",
(totalrdata-totalzdata),
(totalwriteops-totalzeroops),
(totalrdata-totalzdata)/((totalwriteops-totalzeroops)?:1));
fprintf(stderr,
" %lld bytes of zero data in %ld ops (%lld bytes/op)\n",
totalzdata, totalzeroops,
totalzdata/(totalzeroops?:1));
}
}
if (debug)
fprintf(stderr, "decompressor blocked: %lu, "
......@@ -516,12 +537,13 @@ main(int argc, char *argv[])
{
int i, ch;
off_t foff;
int chunkno;
extern char build_info[];
#ifdef NOTHREADS
nothreads = 1;
#endif
while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:Cr:")) != -1)
while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:Cr:N")) != -1)
switch(ch) {
#ifdef FAKEFRISBEE
case 'F':
......@@ -572,6 +594,10 @@ main(int argc, char *argv[])
readretries = atoi(optarg);
break;
case 'N':
nodecompress++;
break;
#ifndef NOTHREADS
case 'W':
maxwritebufmem = atoi(optarg);
......@@ -613,7 +639,7 @@ main(int argc, char *argv[])
exit(1);
}
if (fstat(infd, &st) == 0)
totalchunks = st.st_size / SUBBLOCKSIZE;
totalchunks = st.st_size / CHUNKSIZE;
} else {
infd = fileno(stdin);
if (readretries > 0) {
......@@ -744,6 +770,7 @@ main(int argc, char *argv[])
signal(SIGINFO, dump_stats);
#endif
foff = 0;
chunkno = 0;
while (1) {
int count = sizeof(chunkbuf);
char *bp = chunkbuf;
......@@ -752,7 +779,7 @@ main(int argc, char *argv[])
if (dofrisbee) {
if (*nextchunk == -1)
goto done;
foff = (off_t)*nextchunk * SUBBLOCKSIZE;
foff = (off_t)*nextchunk * CHUNKSIZE;
if (lseek(infd, foff, SEEK_SET) < 0) {
perror("seek failed");
exit(1);
......@@ -778,8 +805,14 @@ main(int argc, char *argv[])
bp += cc;
foff += cc;
}
if (inflate_subblock(chunkbuf))
break;
if (nodecompress) {
if (write_subblock(chunkno, chunkbuf))
break;
} else {
if (inflate_subblock(chunkbuf))
break;
}
chunkno++;
}
done:
close(infd);
......@@ -989,7 +1022,7 @@ void *
DiskWriter(void *arg)
{
writebuf_t *wbuf = 0;
static int gotone;
static int gotone = 0;
while (1) {
pthread_testcancel();
......@@ -1037,7 +1070,6 @@ DiskWriter(void *arg)
}
#endif
#ifdef FRISBEE
/*
* Just write the raw, compressed chunk data to disk
*/
......@@ -1047,8 +1079,8 @@ write_subblock(int chunkno, const char *chunkbufp)
writebuf_t *wbuf;
off_t offset, size, bytesleft;
offset = chunkno * SUBBLOCKSIZE;
bytesleft = SUBBLOCKSIZE;
offset = chunkno * CHUNKSIZE;
bytesleft = CHUNKSIZE;
while (bytesleft > 0) {
size = (bytesleft >= OUTSIZE) ? OUTSIZE : bytesleft;
wbuf = alloc_writebuf(offset, size, 1, 1);
......@@ -1064,7 +1096,6 @@ write_subblock(int chunkno, const char *chunkbufp)
return 0;
}
#endif
static int
inflate_subblock(const char *chunkbufp)
......@@ -1156,7 +1187,7 @@ inflate_subblock(const char *chunkbufp)
if (debug == 1)
fprintf(stderr, "Decompressing chunk %04d: %14lld --> ",
blockhdr->blockindex, offset);
blockhdr->blockindex, (long long)offset);
wbuf = NULL;
......@@ -1250,7 +1281,8 @@ inflate_subblock(const char *chunkbufp)
fprintf(stderr,
"%12lld %8d %8d %12lld %10lld %8d %5d %8d"
"\n",
offset, cc, count, totaledata, size,
(long long)offset, cc, count,
totaledata, (long long)size,
ibsize, ibleft, d_stream.avail_in);
}
......@@ -1298,6 +1330,8 @@ inflate_subblock(const char *chunkbufp)
}
assert(wbuf == NULL);
totalddata += ibsize;
/*
* Exhausted our output buffer but may still have more input in
* the current chunk.
......@@ -1337,7 +1371,7 @@ inflate_subblock(const char *chunkbufp)
donechunks++;
if (debug == 1) {
fprintf(stderr, "%14lld\n", offset);
fprintf(stderr, "%14lld\n", (long long)offset);
}
dodots(DODOTS_CHUNKS, 0);
......@@ -1352,6 +1386,9 @@ writezeros(off_t offset, off_t zcount)
assert((offset & (SECSIZE-1)) == 0);
if (offset != nextwriteoffset)
totalseekops++;
#ifndef FRISBEE
if (docrconly)
nextwriteoffset = offset;
......@@ -1368,8 +1405,8 @@ writezeros(off_t offset, off_t zcount)
}
nextwriteoffset = offset;
} else if (offset != nextwriteoffset) {
fprintf(stderr, "Non-contiguous write @ %llu (should be %llu)\n",
offset, nextwriteoffset);
fprintf(stderr, "Non-contiguous write @ %lld (should be %lld)\n",
(long long)offset, (long long)nextwriteoffset);
exit(1);
}
......@@ -1390,8 +1427,8 @@ writezeros(off_t offset, off_t zcount)
perror("Writing Zeros");
} else if ((wcc & (SECSIZE-1)) != 0) {
fprintf(stderr, "Writing Zeros: "
"partial sector write (%d bytes)\n",
wcc);
"partial sector write (%ld bytes)\n",
(long)wcc);
wcc = -1;
} else if (wcc == 0) {
fprintf(stderr, "Writing Zeros: "
......@@ -1404,6 +1441,9 @@ writezeros(off_t offset, off_t zcount)
}
zcount -= zcc;
totalrdata += zcc;
totalzdata += zcc;
totalwriteops++;
totalzeroops++;
nextwriteoffset += zcc;
dodots(DODOTS_ZERO, ozcount-zcount);
}
......@@ -1418,6 +1458,9 @@ writedata(off_t offset, size_t size, void *buf)
/* fprintf(stderr, "Writing %d bytes at %qd\n", size, offset); */
if (offset != nextwriteoffset)
totalseekops++;
#ifndef FRISBEE
if (docrconly) {
compute_crc((u_char *)buf, size, &crc);
......@@ -1429,8 +1472,8 @@ writedata(off_t offset, size_t size, void *buf)
} else if (offset == nextwriteoffset) {
cc = write(outfd, buf, size);
} else {
fprintf(stderr, "Non-contiguous write @ %llu (should be %llu)\n",
offset, nextwriteoffset);
fprintf(stderr, "Non-contiguous write @ %lld (should be %lld)\n",
(long long)offset, (long long)nextwriteoffset);
exit(1);
}
......@@ -1445,6 +1488,7 @@ writedata(off_t offset, size_t size, void *buf)
if (nextwriteoffset > maxwrittenoffset)
maxwrittenoffset = nextwriteoffset;
totalrdata += cc;
totalwriteops++;
dodots(DODOTS_DATA, cc);
}
......@@ -1469,7 +1513,7 @@ zero_remainder()
outputmaxsec = getdisksize(outfd);
disksize = sectobytes(outputmaxsec);
if (debug)
fprintf(stderr, "\ndisksize = %lld\n", disksize);
fprintf(stderr, "\ndisksize = %lld\n", (long long)disksize);
/* XXX must wait for writer thread to finish to get maxwrittenoffset value */
threadwait();
......@@ -1481,15 +1525,18 @@ zero_remainder()
if (debug)
fprintf(stderr, "zeroing %lld bytes at offset %lld "
"(%u sectors at %u)\n",
remaining, maxwrittenoffset,
bytestosec(remaining), bytestosec(maxwrittenoffset));
(long long)remaining,
(long long)maxwrittenoffset,
bytestosec(remaining),
bytestosec(maxwrittenoffset));
wbuf = alloc_writebuf(maxwrittenoffset, remaining, 0, 1);
dowrite_request(wbuf);
} else {
if (debug)
fprintf(stderr, "not zeroing: disksize = %lld, "
"maxwritten = %lld\n",
disksize, maxwrittenoffset);
(long long)disksize,
(long long)maxwrittenoffset);
}
}
......@@ -1639,10 +1686,12 @@ applyrelocs(off_t offset, size_t size, void *buf)
coff = (u_int32_t)(roffset - offset);
if (debug > 1)
fprintf(stderr,
"Applying reloc type %d [%llu-%llu] "
"to [%llu-%llu]\n", reloc->type,
roffset, roffset+reloc->size,
offset, offset+size);
"Applying reloc type %d [%lld-%lld] "
"to [%lld-%lld]\n", reloc->type,
(long long)roffset,
(long long)roffset+reloc->size,
(long long)offset,
(long long)offset+size);
switch (reloc->type) {
case RELOC_NONE:
break;
......
......@@ -33,6 +33,9 @@
#include "sliceinfo.h"
#include "global.h"
/* XXX this is a hack right now */
#define USE_HACKSORT 0
#define min(a,b) ((a) <= (b) ? (a) : (b))
char *infilename;
......@@ -96,7 +99,7 @@ struct blockreloc *relocs;
int numregions, numrelocs;
void dumpskips(int verbose);
void sortrange(struct range *head, int domerge,
static void sortrange(struct range **head, int domerge,
int (*rangecmp)(struct range *, struct range *));
int mergeskips(int verbose);
int mergeranges(struct range *head);
......@@ -563,7 +566,7 @@ main(int argc, char *argv[])
rval = read_shd(chkpointdev, infilename, infd,
inputminsec, addvalid);
if (rval == 0)
sortrange(ranges, 1, 0);