Commit 8e8e8b1d authored by Leigh B. Stoller's avatar Leigh B. Stoller

Rework for Frisbee support. The image is now compressed in 1MB chunks,

with each chunk an independent compression block that can be layed
down on disk without any other external information.
parent dab40c16
......@@ -15,9 +15,15 @@ LIBS = -lz
imagezip: imagezip.o
$(CC) $(CFLAGS) imagezip.o $(LIBS) -o imagezip
newzip: newzip.o
$(CC) $(CFLAGS) newzip.o $(LIBS) -o newzip
imageunzip: imageunzip.o
$(CC) $(CFLAGS) imageunzip.o $(LIBS) -o imageunzip
newunzip: newunzip.o
$(CC) $(CFLAGS) newunzip.o $(LIBS) -o newunzip
install: $(INSTALL_BINDIR)/imagezip $(INSTALL_BINDIR)/imageunzip
clean:
......
......@@ -2,33 +2,31 @@
#include <sys/types.h>
/*
* This little sucker goes on the beginning of the outputfile. It is blocked
* in the first 1K of the file. It tells the inflator how many 1K blocks
* come after it, which contain the start/size pairs of the good stuff that
* needs to be written to disk.
* Magic number when image is compressed
*/
struct imagehdr {
off_t filesize;
int magic;
int regionsize;
int regioncount;
} *hdr;
/*
* This number is defined to be the same as the NFS read size in netdisk.
* The above structure is padded to this when it is written to the
* output file.
*/
#define IMAGEHDRMINSIZE 1024
#define COMPRESSED_MAGIC 0x69696969
/*
* Magic number when image is compresses
* Each compressed block of the file has this little header on it.
* Since each subblock is independently compressed, we need to know
* its internal size (it will probably be shorter than 1MB) since we
* have to know exactly how much to give the inflator.
*/
#define COMPRESSED_MAGIC 0x69696969
struct blockhdr {
int magic;
unsigned long size; /* Size of compressed part */
int blockindex;
int blocktotal;
int regionsize;
int regioncount;
};
/*
* This little struct defines the pair. Each number is in sectors. An array
* of these come after the header above, and is padded to a 1K boundry.
* The region says where to write the next part of the input file, which is
* how we skip over parts of the disk that do not need to be written
* (swap, free FS blocks).
*/
struct region {
unsigned long start;
......@@ -36,9 +34,19 @@ struct region {
};
/*
* This number is defined to be the same as the NFS read size in
* netdisk. The array of regions is padded to this when it is written
* to the output file.
* In the new model, each sub region has its own region header info.
* But there is no easy way to tell how many regions before compressing.
* Just leave a page, and hope that 512 regions is enough!
*
* This number must be a multiple of the NFS read size in netdisk.
*/
#define REGIONMINSIZE 1024
#define DEFAULTREGIONSIZE (1024 * 4)
/*
* Ah, the frisbee protocol. The new world order is to break up the
* file into fixed 1MB chunks, with the region info prepended to each
* chunk so that it can be layed down on disk independently of all the
* chunks in the file.
*/
#define SUBBLOCKSIZE (1024 * 1024)
#define SUBBLOCKMAX (SUBBLOCKSIZE - DEFAULTREGIONSIZE)
......@@ -24,17 +24,76 @@
#define BSIZE 0x10000
char inbuf[BSIZE], outbuf[2*BSIZE], zeros[BSIZE];
int infd, outfd;
int doseek = 0;
int debug = 1;
long long total = 0;
int inflate_subblock(void);
void writezeros(off_t zcount);
int
main(int argc, char **argv)
{
int cc, err, count, infd;
if (argc < 2 || argc > 3) {
fprintf(stderr, "usage: "
"%s <input filename> [output filename]\n", argv[0]);
exit(1);
}
if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
perror("opening input file");
exit(1);
}
if (argc == 3) {
if ((outfd =
open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
perror("opening output file");
exit(1);
}
doseek = 1;
}
else
outfd = fileno(stdout);
while (1) {
off_t offset, correction;
/*
* Decompress one subblock at a time. After its done
* make sure the input file pointer is at a block
* boundry. Move it up if not.
*/
if (inflate_subblock())
break;
if ((offset = lseek(infd, (off_t) 0, SEEK_CUR)) < 0) {
perror("Getting current seek pointer");
exit(1);
}
if (offset & (SUBBLOCKSIZE - 1)) {
correction = SUBBLOCKSIZE -
(offset & (SUBBLOCKSIZE - 1));
if ((offset = lseek(infd, correction, SEEK_CUR)) < 0) {
perror("correcting seek pointer");
exit(1);
}
}
}
close(infd);
return 0;
}
int
inflate_subblock(void)
{
int cc, err, count;
z_stream d_stream; /* inflation stream */
long long total = 0, blockcount = 0;
char *bp, *prog = argv[0];
struct imagehdr hdr;
struct region *regions, *curregion;
struct timeval stamp, estamp;
char *bp;
struct blockhdr *blockhdr;
struct region *curregion;
off_t offset, size;
char buf[DEFAULTREGIONSIZE];
d_stream.zalloc = (alloc_func)0;
d_stream.zfree = (free_func)0;
......@@ -46,62 +105,55 @@ main(int argc, char **argv)
err = inflateInit(&d_stream);
CHECK_ERR(err, "inflateInit");
if (argc != 2) {
fprintf(stderr, "usage: "
"%s <input filename>\n", argv[0]);
exit(1);
}
if (strcmp(argv[1], "-")) {
if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
perror("opening input file");
exit(1);
}
}
else
infd = fileno(stdin);
/*
* Read the header. It is uncompressed, and holds the real
* image size and the magic number.
*/
if ((cc = read(infd, inbuf, IMAGEHDRMINSIZE)) < 0) {
if ((cc = read(infd, buf, DEFAULTREGIONSIZE)) <= 0) {
if (cc == 0)
return 1;
perror("reading zipped image header goo");
exit(1);
}
assert(cc == IMAGEHDRMINSIZE);
memcpy(&hdr, inbuf, sizeof(hdr));
fprintf(stderr, "Filesize: %qd, Magic: %x, "
"Regionsize %d, RegionCount %d\n",
hdr.filesize, hdr.magic, hdr.regionsize, hdr.regioncount);
assert(cc == DEFAULTREGIONSIZE);
blockhdr = (struct blockhdr *) buf;
/*
* Read in the valid data region information.
*/
regions = (struct region *) calloc(hdr.regionsize, 1);
curregion = regions;
if ((cc = read(infd, regions, hdr.regionsize)) < 0) {
perror("reading region info");
if (blockhdr->magic != COMPRESSED_MAGIC) {
fprintf(stderr, "Bad Magic Number!\n");
exit(1);
}
assert(cc == hdr.regionsize);
curregion = (struct region *) (blockhdr + 1);
/*
* Start with the first region.
*/
offset = curregion->start * (off_t) SECSIZE;
size = curregion->size * (off_t) SECSIZE;
assert(size);
curregion++;
hdr.regioncount--;
blockhdr->regioncount--;
if (debug)
fprintf(stderr, "Decompressing: %14qd --> ", offset);
gettimeofday(&stamp, 0);
while (1) {
if ((cc = read(infd, inbuf, sizeof(inbuf))) <= 0) {
if (cc == 0)
break;
/*
* Read just up to the end of compressed data.
*/
if (blockhdr->size >= sizeof(inbuf))
count = sizeof(inbuf);
else
count = blockhdr->size;
if ((cc = read(infd, inbuf, count)) <= 0) {
if (cc == 0) {
return 1;
}
perror("reading zipped image");
exit(1);
}
assert(cc == count);
blockhdr->size -= cc;
d_stream.next_in = inbuf;
d_stream.avail_in = cc;
......@@ -111,8 +163,7 @@ main(int argc, char **argv)
err = inflate(&d_stream, Z_SYNC_FLUSH);
if (err != Z_OK && err != Z_STREAM_END) {
fprintf(stderr,
"%s: inflate failed, err=%ld\n", prog, err);
fprintf(stderr, "inflate failed, err=%ld\n", err);
exit(1);
}
count = sizeof(outbuf) - d_stream.avail_out;
......@@ -128,11 +179,11 @@ main(int argc, char **argv)
else
cc = size;
if ((cc = write(1, bp, cc)) != cc) {
if ((cc = write(outfd, bp, cc)) != cc) {
if (cc < 0) {
perror("Writing uncompressed data");
}
fprintf(stderr, "%s: inflate failed\n", prog);
fprintf(stderr, "inflate failed\n");
exit(1);
}
......@@ -141,74 +192,80 @@ main(int argc, char **argv)
size -= cc;
offset += cc;
total += cc;
blockcount += cc;
/*
* Hit the end of the region. Need to figure out
* where the next one starts. We write a block of
* zeros in the empty space between this region
* and the next. We could lseek too, but only if
* and the next. We can lseek, but only if
* not writing to stdout.
*/
if (! size) {
off_t newoffset, zcount;
int zcc;
off_t newoffset;
/*
* No more regions. Must be done.
*/
if (!hdr.regioncount)
if (!blockhdr->regioncount)
break;
newoffset = curregion->start * (off_t) SECSIZE;
zcount = newoffset - offset;
while (zcount) {
if (zcount <= BSIZE)
zcc = (int) zcount;
else
zcc = BSIZE;
if ((zcc =
write(1, zeros, zcc)) != zcc) {
if (zcc < 0) {
perror("Writing "
"Zeros");
}
exit(1);
}
zcount -= zcc;
total += zcc;
blockcount += zcc;
}
writezeros(newoffset - offset);
offset = newoffset;
size = curregion->size * (off_t) SECSIZE;
assert(size);
curregion++;
hdr.regioncount--;
blockhdr->regioncount--;
}
}
if (d_stream.avail_in)
goto inflate_again;
if (blockcount > (1024 * 16 * 8192)) {
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
fprintf(stderr, "Wrote %qd bytes ", total);
fprintf(stderr, "in %ld seconds\n", estamp.tv_sec);
blockcount = 0;
}
if (err == Z_STREAM_END)
break;
}
assert(hdr.regioncount == 0);
err = inflateEnd(&d_stream);
CHECK_ERR(err, "inflateEnd");
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
fprintf(stderr, "Finshed! Wrote %qd bytes ", total);
fprintf(stderr, "in %ld seconds\n", estamp.tv_sec);
assert(blockhdr->regioncount == 0);
assert(size == 0);
assert(blockhdr->size == 0);
if (debug)
fprintf(stderr, "%14qd\n", total);
close(infd);
return 0;
}
void
writezeros(off_t zcount)
{
int zcc;
if (doseek) {
if (lseek(outfd, zcount, SEEK_CUR) < 0) {
perror("Skipping ahead");
exit(1);
}
total += zcount;
return;
}
while (zcount) {
if (zcount <= BSIZE)
zcc = (int) zcount;
else
zcc = BSIZE;
if ((zcc = write(outfd, zeros, zcc)) != zcc) {
if (zcc < 0) {
perror("Writing Zeros");
}
exit(1);
}
zcount -= zcc;
total += zcc;
}
}
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment