Commit 8c3c41f4 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Fixes! The new NFS server appears to be more asynchronous, and errors

caused by hupping mountd are not getting propogated back until later
writes. Must do an fsync to catch the error. Too many fsyncs slows
down imagezip by a factor of three though, so I reorged things so that
we create the 1MB chunks all in memory, and then write out each one
out as finished. Much better; only a 10% slowdown.
parent 33b144a7
...@@ -33,7 +33,7 @@ ...@@ -33,7 +33,7 @@
int infd, outfd; int infd, outfd;
int secsize = 512; /* XXX bytes. */ int secsize = 512; /* XXX bytes. */
int debug = 0; int debug = 1;
int info = 0; int info = 0;
int slicemode = 0; int slicemode = 0;
int maxmode = 0; int maxmode = 0;
...@@ -108,31 +108,52 @@ ssize_t ...@@ -108,31 +108,52 @@ ssize_t
mywrite(int fd, const void *buf, size_t nbytes) mywrite(int fd, const void *buf, size_t nbytes)
{ {
int cc, i, count = 0; int cc, i, count = 0;
off_t startoffset, endoffset;
int maxretries = 10;
if ((startoffset = lseek(fd, (off_t) 0, SEEK_CUR)) < 0) {
perror("mywrite: seeking to get output file ptr");
exit(1);
}
while (nbytes) { for (i = 0; i < maxretries; i++) {
int maxretries = 10; while (nbytes) {
for (i = 0; i < maxretries; i++) {
cc = write(fd, buf, nbytes); cc = write(fd, buf, nbytes);
if (cc > 0) { if (cc > 0) {
nbytes -= cc; nbytes -= cc;
buf += cc; buf += cc;
count += cc; count += cc;
goto again; continue;
} }
if (i == 0) if (i == 0)
perror("write error: will retry"); perror("write error: will retry");
sleep(1); sleep(1);
nbytes += count;
buf -= count;
count = 0;
goto again;
} }
perror("write error: busted for too long"); if (fsync(fd) < 0) {
fflush(stderr); perror("fsync error: will retry");
exit(1); sleep(1);
nbytes += count;
buf -= count;
count = 0;
goto again;
}
return count;
again: again:
if (lseek(fd, startoffset, SEEK_SET) < 0) {
perror("mywrite: seeking to set file ptr");
exit(1);
}
} }
return count; perror("write error: busted for too long");
fflush(stderr);
exit(1);
} }
/* Map partition number to letter */ /* Map partition number to letter */
...@@ -224,7 +245,7 @@ main(argc, argv) ...@@ -224,7 +245,7 @@ main(argc, argv)
rval = read_image(); rval = read_image();
#endif #endif
sortskips(); sortskips();
if (debug) if (debug > 2)
dumpskips(); dumpskips();
makeranges(); makeranges();
...@@ -1007,7 +1028,7 @@ makeranges(void) ...@@ -1007,7 +1028,7 @@ makeranges(void)
numranges++; numranges++;
} }
if (debug) { if (debug > 2) {
range = ranges; range = ranges;
while (range) { while (range) {
printf(" %12d %9d\n", range->start, range->size); printf(" %12d %9d\n", range->start, range->size);
...@@ -1019,11 +1040,14 @@ makeranges(void) ...@@ -1019,11 +1040,14 @@ makeranges(void)
/* /*
* Compress the image. * Compress the image.
*/ */
static u_char output_buffer[SUBBLOCKSIZE];
static int buffer_offset;
int int
compress_image(void) compress_image(void)
{ {
int cc, partial, i, count; int cc, partial, i, count;
off_t fileregoff, inputoffset, size, outputoffset; off_t inputoffset, size, outputoffset;
off_t tmpoffset, rangesize, totalinput = 0; off_t tmpoffset, rangesize, totalinput = 0;
struct range *prange = ranges; struct range *prange = ranges;
struct blockhdr *blkhdr; struct blockhdr *blkhdr;
...@@ -1065,18 +1089,10 @@ compress_image(void) ...@@ -1065,18 +1089,10 @@ compress_image(void)
/* /*
* Reserve room for the subblock hdr and the region pairs. * Reserve room for the subblock hdr and the region pairs.
* We go back and fill it it later after the subblock is * We go back and fill it it later after the subblock is
* written and we know much input data was compressed into * done and we know much input data was compressed into
* the block. We remember the offset so we can come back * the block.
* to it later.
*/ */
if ((fileregoff = lseek(outfd, (off_t) 0, SEEK_CUR)) < 0) { buffer_offset = DEFAULTREGIONSIZE;
perror("remembering where block header goes");
exit(1);
}
if (lseek(outfd, DEFAULTREGIONSIZE, SEEK_CUR) < 0) {
perror("lseeking past block header and region pairs");
exit(1);
}
if (debug) { if (debug) {
printf("Compressing range: %14qd --> ", inputoffset); printf("Compressing range: %14qd --> ", inputoffset);
...@@ -1101,6 +1117,8 @@ compress_image(void) ...@@ -1101,6 +1117,8 @@ compress_image(void)
if (debug) if (debug)
printf("%12qd\n", inputoffset + size); printf("%12qd\n", inputoffset + size);
totalinput += size;
/* /*
* The last subblock is special if inputmaxsec is nonzero. * The last subblock is special if inputmaxsec is nonzero.
* In that case, we have to finish off the compression block * In that case, we have to finish off the compression block
...@@ -1122,28 +1140,17 @@ compress_image(void) ...@@ -1122,28 +1140,17 @@ compress_image(void)
curregion++; curregion++;
/* /*
* Remember where in the output file we are. * Stash the region info into the beginning of the block.
*/ */
if ((tmpoffset = lseek(outfd, (off_t) 0, SEEK_CUR)) < 0) { memcpy(output_buffer, buf, sizeof(buf));
perror("seeking to remember output offset");
exit(1);
}
if (lseek(outfd, (off_t) fileregoff, SEEK_SET) < 0) {
perror("seeking back to block header offset");
exit(1);
}
if ((cc = mywrite(outfd, buf, sizeof(buf))) < 0) {
perror("writing subblock header and regions");
exit(1);
}
assert(cc == sizeof(buf));
totalinput += size;
if (partial) { if (partial) {
if (lseek(outfd, (off_t) tmpoffset, SEEK_SET) < 0) { /*
perror("seeking back to where we where!"); * Write out the finished chunk to disk, and
exit(1); * start over from the beginning of the buffer.
} */
mywrite(outfd, output_buffer, sizeof(output_buffer));
buffer_offset = 0;
outputoffset += size; outputoffset += size;
inputoffset += size; inputoffset += size;
...@@ -1172,19 +1179,11 @@ compress_image(void) ...@@ -1172,19 +1179,11 @@ compress_image(void)
/* /*
* Reserve room for the subblock hdr and the region pairs. * Reserve room for the subblock hdr and the region pairs.
* We go back and fill it it later after the subblock is * We go back and fill it it later after the subblock is
* written and we know much input data was compressed into * done and we know much input data was compressed into
* the block. We remember the offset so we can come back * the block.
* to it later.
*/ */
if ((fileregoff = lseek(outfd, (off_t) 0, SEEK_CUR)) < 0) { buffer_offset = DEFAULTREGIONSIZE;
perror("remembering where block header goes");
exit(1);
}
if (lseek(outfd, DEFAULTREGIONSIZE, SEEK_CUR) < 0) {
perror("lseeking past block header and region pairs");
exit(1);
}
while (prange) { while (prange) {
inputoffset = (off_t) prange->start * (off_t) secsize; inputoffset = (off_t) prange->start * (off_t) secsize;
...@@ -1202,7 +1201,7 @@ compress_image(void) ...@@ -1202,7 +1201,7 @@ compress_image(void)
/* /*
* Compress the chunk. * Compress the chunk.
*/ */
if (debug < 3) { if (debug > 0 && debug < 3) {
printf("Compressing range: %14qd --> ", inputoffset); printf("Compressing range: %14qd --> ", inputoffset);
fflush(stdout); fflush(stdout);
} }
...@@ -1273,38 +1272,20 @@ compress_image(void) ...@@ -1273,38 +1272,20 @@ compress_image(void)
blkhdr->regioncount = (curregion - regions); blkhdr->regioncount = (curregion - regions);
/* /*
* Remember where in the output file we are. * Stash the region info into the beginning of the block.
*/ */
if ((tmpoffset = lseek(outfd, (off_t) 0, SEEK_CUR)) < 0) { memcpy(output_buffer, buf, sizeof(buf));
perror("seeking to remember output offset");
exit(1);
}
if (lseek(outfd, (off_t) fileregoff, SEEK_SET) < 0) {
perror("seeking back to block header offset");
exit(1);
}
if ((cc = mywrite(outfd, buf, sizeof(buf))) < 0) {
perror("writing subblock header and regions");
exit(1);
}
assert(cc == sizeof(buf));
if (lseek(outfd, (off_t) tmpoffset, SEEK_SET) < 0) {
perror("seeking back to where we where!");
exit(1);
}
/* /*
* Moving to the next block. Reserve the header area, * Write out the finished chunk to disk.
* remembering where we are now so we can come back
* later and write it.
*/ */
fileregoff = tmpoffset; mywrite(outfd, output_buffer, sizeof(output_buffer));
if (lseek(outfd, DEFAULTREGIONSIZE, SEEK_CUR) < 0) {
perror("lseeking past block header and region pairs"); /*
exit(1); * Moving to the next block. Reserve the header area.
} */
curregion = regions; buffer_offset = DEFAULTREGIONSIZE;
curregion = regions;
/* /*
* Okay, so its possible that we ended the region at the * Okay, so its possible that we ended the region at the
...@@ -1340,15 +1321,17 @@ compress_image(void) ...@@ -1340,15 +1321,17 @@ compress_image(void)
blkhdr->regionsize = DEFAULTREGIONSIZE; blkhdr->regionsize = DEFAULTREGIONSIZE;
blkhdr->regioncount = (curregion - regions); blkhdr->regioncount = (curregion - regions);
if (lseek(outfd, (off_t) fileregoff, SEEK_SET) < 0) { /*
perror("seeking back to block header offset"); * Stash the region info into the beginning of the block.
exit(1); */
} memcpy(output_buffer, buf, sizeof(buf));
if ((cc = mywrite(outfd, buf, sizeof(buf))) < 0) {
perror("writing subblock header and regions"); /*
exit(1); * Write out the finished chunk to disk, and
} * start over from the beginning of the buffer.
assert(cc == sizeof(buf)); */
mywrite(outfd, output_buffer, sizeof(output_buffer));
buffer_offset = 0;
} }
if (debug) { if (debug) {
...@@ -1504,11 +1487,10 @@ compress_chunk(off_t size, int *partial, unsigned long *subblksize) ...@@ -1504,11 +1487,10 @@ compress_chunk(off_t size, int *partial, unsigned long *subblksize)
} }
count = BSIZE - d_stream.avail_out; count = BSIZE - d_stream.avail_out;
if ((cc = mywrite(outfd, outbuf, count)) < 0) { /* Stash into the output buffer */
perror("writing output file"); memcpy(&output_buffer[buffer_offset], outbuf, count);
exit(1); buffer_offset += count;
} assert(buffer_offset <= SUBBLOCKSIZE);
assert(cc == count);
/* /*
* If we have reached the subblock maximum, then need * If we have reached the subblock maximum, then need
...@@ -1567,11 +1549,11 @@ compress_finish(unsigned long *subblksize) ...@@ -1567,11 +1549,11 @@ compress_finish(unsigned long *subblksize)
*/ */
count = BSIZE - d_stream.avail_out; count = BSIZE - d_stream.avail_out;
if (count) { if (count) {
if ((cc = mywrite(outfd, outbuf, count)) < 0) { /* Stash into the output buffer */
perror("writing output file"); memcpy(&output_buffer[buffer_offset], outbuf, count);
exit(1); buffer_offset += count;
} assert(buffer_offset <= SUBBLOCKSIZE);
assert(cc == count);
subblockleft -= count; subblockleft -= count;
assert(subblockleft >= 0); assert(subblockleft >= 0);
} }
...@@ -1592,15 +1574,16 @@ compress_finish(unsigned long *subblksize) ...@@ -1592,15 +1574,16 @@ compress_finish(unsigned long *subblksize)
count = sizeof(outbuf); count = sizeof(outbuf);
else else
count = subblockleft; count = subblockleft;
if ((cc = mywrite(outfd, outbuf, count)) < 0) { /* Stash zeros into the output buffer */
perror("writing output file"); memcpy(&output_buffer[buffer_offset], outbuf, count);
exit(1); buffer_offset += count;
} assert(buffer_offset <= SUBBLOCKSIZE);
assert(cc == count);
subblockleft -= count; subblockleft -= count;
} }
subblockleft = SUBBLOCKMAX; subblockleft = SUBBLOCKMAX;
return 1; return 1;
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment