Commit a8ef625f authored by Mike Hibler's avatar Mike Hibler

Really important stuff:

Unified the 'dot' handling (status printing) of frisbee and imagezip.
They now both report the number of chunks remaining along with the dots.
Also put out a periodic splat for every GB of uncompressed data we write.
This is useful when you are zero-filling, since otherwise it appears that
frisbee has hung when it is really just zeroing the last unused 100GB of
your disk.
parent 4c6cd45a
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2004 University of Utah and the Flux Group.
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -57,7 +57,6 @@ int randomize = 1;
int portnum;
struct in_addr mcastaddr;
struct in_addr mcastif;
static int dotcol;
static struct timeval stamp;
static struct in_addr serverip;
......@@ -68,8 +67,9 @@ static void RequestChunk(int timedout);
static void RequestStamp(int chunk, int block, int count, void *arg);
static int RequestRedoTime(int chunk, unsigned long long curtime);
extern int ImageUnzipInit(char *filename, int slice, int debug, int zero,
int nothreads, int dostype,
int nothreads, int dostype, int dodots,
unsigned long writebufmem);
extern void ImageUnzipSetChunkCount(unsigned long chunkcount);
extern void ImageUnzipSetMemory(unsigned long writebufmem);
extern int ImageWriteChunk(int chunkno, char *chunkdata);
extern int ImageUnzipChunk(char *chunkdata);
......@@ -422,7 +422,7 @@ main(int argc, char **argv)
maxwritebufmem = maxmem/2;
}
ImageUnzipInit(filename, slice, debug, zero, nothreads, dostype,
ImageUnzipInit(filename, slice, debug, zero, nothreads, dostype, 3,
maxwritebufmem*1024*1024);
if (tracing) {
......@@ -707,20 +707,6 @@ ChunkerStartup(void)
ChunkBuffer[i].thischunk, i,
(wasidle*idledelay) / 1000000,
((wasidle*idledelay) % 1000000) / 1000);
else {
struct timeval estamp;
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
printf(".");
fflush(stdout);
if (dotcol++ > 65) {
dotcol = 0;
printf("%4ld %6d\n",
estamp.tv_sec, chunkcount);
}
}
CLEVENT(1, EV_CLIDCSTART,
ChunkBuffer[i].thischunk, wasidle,
......@@ -1247,6 +1233,7 @@ PlayFrisbee(void)
}
gettimeofday(&timeo, 0);
TotalChunkCount = p->msg.join.blockcount / CHUNKSIZE;
ImageUnzipSetChunkCount(TotalChunkCount);
/*
* If we have partitioned up the memory and have allocated
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2004 University of Utah and the Flux Group.
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* All rights reserved.
*/
......@@ -76,14 +76,14 @@ static int dofill = 0;
static int nothreads = 0;
static int rdycount;
static int imageversion = 1;
static int dots = 0;
static int dotcol;
static struct timeval stamp;
#ifndef FRISBEE
static int infd;
static int version= 0;
static unsigned fillpat= 0;
static int dots = 0;
static int dotcol;
static char chunkbuf[SUBBLOCKSIZE];
static struct timeval stamp;
#endif
int readmbr(int slice);
int fixmbr(int slice, int dtype);
......@@ -190,10 +190,11 @@ dump_stats(int sig)
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
if (sig == 0 && debug != 1 && dots) {
while (dotcol++ <= 60)
while (dotcol++ <= 66)
fprintf(stderr, " ");
fprintf(stderr, "%4ld %13lld\n", estamp.tv_sec, totaledata);
fprintf(stderr, "%4ld %6d\n",
estamp.tv_sec, totalchunks - donechunks);
}
else {
if (sig) {
......@@ -216,6 +217,57 @@ dump_stats(int sig)
}
#endif
#define DODOTS_CHUNKS 1
#define DODOTS_DATA 2
#define DODOTS_ZERO 3
#define DODOTS_SKIP 4
void dodots(int dottype, off_t cc)
{
static int lastgb = 0;
int count = 0, newgb;
char chr = 0;
switch (dottype) {
case DODOTS_CHUNKS:
if ((dots & 1) == 0)
return;
count = 1;
chr = '.';
break;
case DODOTS_DATA:
case DODOTS_ZERO:
case DODOTS_SKIP:
if ((dots & 2) == 0)
return;
/*
* totaledata has not been updated yet, so we have to
* figure it out ourselves.
*/
newgb = (totaledata + cc) / 1000000000LL;
if ((count = newgb - lastgb) <= 0)
return;
lastgb = newgb;
chr = (dottype == DODOTS_DATA) ? '*' :
(dottype == DODOTS_ZERO) ? 'o' : 's';
break;
}
while (count-- > 0) {
fputc(chr, stderr);
if (dotcol++ > 65) {
struct timeval estamp;
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
fprintf(stderr, "%4ld %6d\n",
estamp.tv_sec, totalchunks - donechunks);
dotcol = 0;
}
}
}
void
dump_writebufs(void)
{
......@@ -441,7 +493,8 @@ usage(void)
" -z Write zeros to free blocks.\n"
" -p pattern Write 32 bit pattern to free blocks.\n"
" NOTE: Use -z/-p to avoid seeking.\n"
" -o Output 'dots' indicating progress\n"
" -o Output progress indicator (compressed chunks processed)\n"
" -O Output progress indicator (GBs of uncompressed data written)\n"
" -n Single threaded (slow) mode\n"
" -d Turn on progressive levels of debugging\n"
" -W size MB of memory to use for write buffering\n");
......@@ -457,7 +510,7 @@ main(int argc, char *argv[])
#ifdef NOTHREADS
nothreads = 1;
#endif
while ((ch = getopt(argc, argv, "vdhs:zp:onFD:W:C")) != -1)
while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:C")) != -1)
switch(ch) {
#ifdef FAKEFRISBEE
case 'F':
......@@ -477,7 +530,11 @@ main(int argc, char *argv[])
break;
case 'o':
dots++;
dots |= 1;
break;
case 'O':
dots |= 2;
break;
case 's':
......@@ -686,7 +743,6 @@ main(int argc, char *argv[])
}
if (inflate_subblock(chunkbuf))
break;
donechunks++;
}
done:
close(infd);
......@@ -710,8 +766,11 @@ main(int argc, char *argv[])
*/
int
ImageUnzipInit(char *filename, int _slice, int _debug, int _fill,
int _nothreads, int _dostype, unsigned long _writebufmem)
int _nothreads, int _dostype, int _dodots,
unsigned long _writebufmem)
{
gettimeofday(&stamp, 0);
if (outfd >= 0)
close(outfd);
......@@ -724,6 +783,7 @@ ImageUnzipInit(char *filename, int _slice, int _debug, int _fill,
dofill = _fill;
nothreads = _nothreads;
dostype = _dostype;
dots = _dodots;
#ifndef NOTHREADS
maxwritebufmem = _writebufmem;
#endif
......@@ -766,6 +826,12 @@ ImageUnzipInit(char *filename, int _slice, int _debug, int _fill,
return 0;
}
void
ImageUnzipSetChunkCount(unsigned long _chunkcount)
{
totalchunks = _chunkcount;
}
void
ImageUnzipSetMemory(unsigned long _writebufmem)
{
......@@ -949,6 +1015,10 @@ write_subblock(int chunkno, char *chunkbufp)
offset += size;
bytesleft -= size;
}
donechunks++;
dodots(DODOTS_CHUNKS, 0);
return 0;
}
#endif
......@@ -1024,8 +1094,10 @@ inflate_subblock(char *chunkbufp)
if (dofill) {
wbuf = alloc_writebuf(offset, size, 0, 1);
dowrite_request(wbuf);
} else
} else {
dodots(DODOTS_SKIP, size);
totaledata += size;
}
}
/*
......@@ -1172,8 +1244,10 @@ inflate_subblock(char *chunkbufp)
newoffset-offset,
0, 1);
dowrite_request(wbzero);
} else
} else {
dodots(DODOTS_SKIP, newoffset-offset);
totaledata += newoffset-offset;
}
offset = newoffset;
if (wbuf)
wbuf->offset = newoffset;
......@@ -1219,29 +1293,18 @@ inflate_subblock(char *chunkbufp)
if (dofill) {
wbuf = alloc_writebuf(offset, size, 0, 1);
dowrite_request(wbuf);
} else
} else {
dodots(DODOTS_SKIP, size);
totaledata += size;
}
offset += size;
}
donechunks++;
if (debug == 1) {
fprintf(stderr, "%14lld\n", offset);
}
#ifndef FRISBEE
else if (dots) {
fprintf(stderr, ".");
if (dotcol++ > 59) {
struct timeval estamp;
gettimeofday(&estamp, 0);
estamp.tv_sec -= stamp.tv_sec;
fprintf(stderr, "%4ld %13lld\n",
estamp.tv_sec, totaledata);
dotcol = 0;
}
}
#endif
dodots(DODOTS_CHUNKS, 0);
return 0;
}
......@@ -1250,6 +1313,7 @@ void
writezeros(off_t offset, off_t zcount)
{
size_t zcc;
off_t ozcount;
assert((offset & (SECSIZE-1)) == 0);
......@@ -1274,6 +1338,7 @@ writezeros(off_t offset, off_t zcount)
exit(1);
}
ozcount = zcount;
while (zcount) {
if (zcount <= OUTSIZE)
zcc = zcount;
......@@ -1294,6 +1359,7 @@ writezeros(off_t offset, off_t zcount)
zcount -= zcc;
totalrdata += zcc;
nextwriteoffset += zcc;
dodots(DODOTS_ZERO, ozcount-zcount);
}
}
......@@ -1329,6 +1395,7 @@ writedata(off_t offset, size_t size, void *buf)
}
nextwriteoffset = offset + cc;
totalrdata += cc;
dodots(DODOTS_DATA, cc);
}
#include "sliceinfo.h"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment