client.c 32 KB
Newer Older
Leigh B. Stoller's avatar
Leigh B. Stoller committed
1
2
/*
 * EMULAB-COPYRIGHT
Mike Hibler's avatar
Mike Hibler committed
3
 * Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
4
5
6
 * All rights reserved.
 */

7
8
/*
 * Frisbee client.
9
10
11
12
 *
 * TODO: Deal with a dead server. Its possible that too many clients
 * could swamp the boss with unanswerable requests. Might need some 
 * backoff code.
13
14
15
16
17
18
19
20
21
22
23
24
25
 */
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <syslog.h>
#include <signal.h>
#include <stdarg.h>
#include <pthread.h>
Mike Hibler's avatar
Mike Hibler committed
26
#include <assert.h>
27
#include "decls.h"
Mike Hibler's avatar
Mike Hibler committed
28
#include "utils.h"
29
#include "trace.h"
30

31
32
33
34
35
36
37
38
39
40
#ifdef DOEVENTS
#include "event.h"

static char *eventserver;
static Event_t event;
static int exitstatus;
#endif

/* Tunable constants */
int		maxchunkbufs = MAXCHUNKBUFS;
Mike Hibler's avatar
Mike Hibler committed
41
42
int		maxwritebufmem = MAXWRITEBUFMEM;
int		maxmem = 0;
43
44
45
46
47
48
int		pkttimeout = PKTRCV_TIMEOUT;
int		idletimer = CLIENT_IDLETIMER_COUNT;
int		maxreadahead = MAXREADAHEAD;
int		maxinprogress = MAXINPROGRESS;
int		redodelay = CLIENT_REQUEST_REDO_DELAY;
int		idledelay = CLIENT_WRITER_IDLE_DELAY;
Mike Hibler's avatar
Mike Hibler committed
49
int		startdelay = 0, startat = 0;
50

51
int		debug = 0;
52
int		tracing = 0;
53
54
char		traceprefix[64];
int		randomize = 1;
55
56
57
int		portnum;
struct in_addr	mcastaddr;
struct in_addr	mcastif;
58
static int	dotcol;
Mike Hibler's avatar
Mike Hibler committed
59
60
static struct timeval stamp;
static struct in_addr serverip;
61
62
63
64
65

/* Forward Decls */
static void	PlayFrisbee(void);
static void	GotBlock(Packet_t *p);
static void	RequestChunk(int timedout);
Mike Hibler's avatar
Mike Hibler committed
66
static void	RequestStamp(int chunk, int block, int count, void *arg);
67
static int	RequestRedoTime(int chunk, unsigned long long curtime);
Mike Hibler's avatar
Mike Hibler committed
68
69
70
71
extern int	ImageUnzipInit(char *filename, int slice, int debug, int zero,
			       int nothreads, int dostype,
			       unsigned long writebufmem);
extern void	ImageUnzipSetMemory(unsigned long writebufmem);
72
extern int	ImageUnzipChunk(char *chunkdata);
73
74
75
76
77
78
extern void	ImageUnzipFlush(void);
extern int	ImageUnzipQuit(void);

/*
 * Chunk descriptor, one for each CHUNKSIZE*BLOCKSIZE bytes of an image file.
 * For each chunk, record its state and the time at which it was last
Mike Hibler's avatar
Mike Hibler committed
79
 * requested by someone.  Ours indicates a previous request was made by us.
80
81
 */
typedef struct {
Mike Hibler's avatar
Mike Hibler committed
82
83
	unsigned long long lastreq:62;
	unsigned long long ours:1;
84
85
	unsigned long long done:1;
} Chunk_t;
86
87
88
89
90
91
92
93
94
95
96

/*
 * The chunker data structure. For each chunk in progress, we maintain this
 * array of blocks (plus meta info). This serves as a cache to receive
 * blocks from the server while we write completed chunks to disk. The child
 * thread reads packets and updates this cache, while the parent thread
 * simply looks for completed blocks and writes them. The "inprogress" slot
 * serves a free/allocated flag, while the ready bit indicates that a chunk
 * is complete and ready to write to disk.
 */
typedef struct {
Mike Hibler's avatar
Mike Hibler committed
97
	int	   thischunk;		/* Which chunk in progress */
98
	int	   state;		/* State of chunk */
Mike Hibler's avatar
Mike Hibler committed
99
100
	int	   blockcount;		/* Number of blocks not received yet */
	BlockMap_t blockmap;		/* Which blocks have been received */
101
102
103
104
	struct {
		char	data[BLOCKSIZE];
	} blocks[CHUNKSIZE];		/* Actual block data */
} ChunkBuffer_t;
105
106
107
#define CHUNK_EMPTY	0
#define CHUNK_FILLING	1
#define CHUNK_FULL	2
108
109

Chunk_t		*Chunks;		/* Chunk descriptors */
110
111
112
113
114
ChunkBuffer_t   *ChunkBuffer;		/* The cache */
int		*ChunkRequestList;	/* Randomized chunk request order */
int		TotalChunkCount;	/* Total number of chunks in file */
int		IdleCounter;		/* Countdown to request more data */

115
#ifdef STATS
Mike Hibler's avatar
Mike Hibler committed
116
extern unsigned long decompblocks, writeridles;	/* XXX imageunzip.c */
117
118
ClientStats_t	Stats;
#define DOSTAT(x)	(Stats.u.v1.x)
119
120
121
122
#else
#define DOSTAT(x)
#endif

123
char *usagestr = 
124
 "usage: frisbee [-drzbn] [-s #] <-p #> <-m ipaddr> <output filename>\n"
125
 " -d              Turn on debugging. Multiple -d options increase output.\n"
126
127
128
129
 " -r              Randomly delay first request by up to one second.\n"
 " -z              Zero fill unused block ranges (default is to seek past).\n"
 " -b              Use broadcast instead of multicast\n"
 " -n              Do not use extra threads in diskwriter\n"
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 " -p portnum      Specify a port number.\n"
 " -m mcastaddr    Specify a multicast address in dotted notation.\n"
 " -i mcastif      Specify a multicast interface in dotted notation.\n"
 " -s slice        Output to DOS slice (DOS numbering 1-4)\n"
 "                 NOTE: Must specify a raw disk device for output filename.\n"
 "\n";

void
usage()
{
	fprintf(stderr, usagestr);
	exit(1);
}

int
main(int argc, char **argv)
{
Mike Hibler's avatar
Mike Hibler committed
147
	int	ch, mem;
148
	char   *filename;
149
150
151
152
	int	zero = 0;
	int	nothreads = 0;
	int	dostype = -1;
	int	slice = 0;
153

154
	while ((ch = getopt(argc, argv, "dhp:m:s:i:tbznT:r:E:D:C:W:S:M:R:")) != -1)
155
156
157
158
159
		switch(ch) {
		case 'd':
			debug++;
			break;
			
160
161
162
163
		case 'b':
			broadcast++;
			break;
			
164
165
166
167
168
169
#ifdef DOEVENTS
		case 'E':
			eventserver = optarg;
			break;
#endif

170
171
172
173
174
175
176
177
		case 'p':
			portnum = atoi(optarg);
			break;
			
		case 'm':
			inet_aton(optarg, &mcastaddr);
			break;

178
179
180
181
		case 'n':
			nothreads++;
			break;

182
183
184
185
		case 'i':
			inet_aton(optarg, &mcastif);
			break;

186
187
188
189
		case 'r':
			startdelay = atoi(optarg);
			break;

190
191
192
193
		case 's':
			slice = atoi(optarg);
			break;

Mike Hibler's avatar
Mike Hibler committed
194
195
196
197
198
199
200
201
		case 'S':
			if (!inet_aton(optarg, &serverip)) {
				fprintf(stderr, "Invalid server IP `%s'\n",
					optarg);
				exit(1);
			}
			break;

202
203
204
205
		case 't':
			tracing++;
			break;

206
207
208
209
		case 'T':
			strncpy(traceprefix, optarg, sizeof(traceprefix));
			break;

210
211
212
213
		case 'z':
			zero++;
			break;

214
215
216
217
		case 'D':
			dostype = atoi(optarg);
			break;

Mike Hibler's avatar
Mike Hibler committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
		case 'C':
			mem = atoi(optarg);
			if (mem < 1)
				mem = 1;
			else if (mem > 1024)
				mem = 1024;
			maxchunkbufs = (mem * 1024 * 1024) /
				sizeof(ChunkBuffer_t);
			break;

		case 'W':
			mem = atoi(optarg);
			if (mem < 1)
				mem = 1;
			else if (mem > 1024)
				mem = 1024;
			maxwritebufmem = mem;
			break;

		case 'M':
			mem = atoi(optarg);
			if (mem < 2)
				mem = 2;
			else if (mem > 2048)
				mem = 2048;
			maxmem = mem;
			break;

246
247
248
249
250
251
252
253
254
		case 'R':
			maxreadahead = atoi(optarg);
			if (maxinprogress < maxreadahead * 4) {
				maxinprogress = maxreadahead * 4;
				if (maxinprogress > maxchunkbufs)
					maxinprogress = maxchunkbufs;
			}
			break;

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
		case 'h':
		case '?':
		default:
			usage();
		}
	argc -= optind;
	argv += optind;

	if (argc != 1)
		usage();
	filename = argv[0];

	if (!portnum || ! mcastaddr.s_addr)
		usage();

	ClientLogInit();
	ClientNetInit();
272

273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
#ifdef DOEVENTS
	if (eventserver != NULL && EventInit(eventserver) != 0) {
		log("Failed to initialize event system, events ignored");
		eventserver = NULL;
	}
	if (eventserver != NULL) {
		log("Waiting for START event...");
		EventWait(EV_ANY, &event);
		if (event.type != EV_START)
			goto done;

	again:
		if (event.data.start.startdelay > 0)
			startdelay = event.data.start.startdelay;
		else
			startdelay = 0;
Mike Hibler's avatar
Mike Hibler committed
289
290
291
292
		if (event.data.start.startat > 0)
			startat = event.data.start.startat;
		else
			startat = 0;
293
294
295
296
297
298
299
300
301
302
303
304
305
		if (event.data.start.pkttimeout >= 0)
			pkttimeout = event.data.start.pkttimeout;
		else
			pkttimeout = PKTRCV_TIMEOUT;
		if (event.data.start.idletimer >= 0)
			idletimer = event.data.start.idletimer;
		else
			idletimer = CLIENT_IDLETIMER_COUNT;
		if (event.data.start.chunkbufs >= 0 &&
		    event.data.start.chunkbufs <= 1024)
			maxchunkbufs = event.data.start.chunkbufs;
		else
			maxchunkbufs = MAXCHUNKBUFS;
Mike Hibler's avatar
Mike Hibler committed
306
307
308
309
310
311
312
313
314
315
		if (event.data.start.writebufmem >= 0 &&
		    event.data.start.writebufmem < 4096)
			maxwritebufmem = event.data.start.writebufmem;
		else
			maxwritebufmem = MAXWRITEBUFMEM;
		if (event.data.start.maxmem >= 0 &&
		    event.data.start.maxmem < 4096)
			maxmem = event.data.start.maxmem;
		else
			maxmem = 0;
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
		if (event.data.start.readahead >= 0 &&
		    event.data.start.readahead <= maxchunkbufs)
			maxreadahead = event.data.start.readahead;
		else
			maxreadahead = MAXREADAHEAD;
		if (event.data.start.inprogress >= 0 &&
		    event.data.start.inprogress <= maxchunkbufs)
			maxinprogress = event.data.start.inprogress;
		else
			maxinprogress = MAXINPROGRESS;
		if (event.data.start.redodelay >= 0)
			redodelay = event.data.start.redodelay;
		else
			redodelay = CLIENT_REQUEST_REDO_DELAY;
		if (event.data.start.idledelay >= 0)
			idledelay = event.data.start.idledelay;
		else
			idledelay = CLIENT_WRITER_IDLE_DELAY;

		if (event.data.start.slice >= 0)
			slice = event.data.start.slice;
		else
			slice = 0;
		if (event.data.start.zerofill >= 0)
			zero = event.data.start.zerofill;
		else
			zero = 0;
		if (event.data.start.randomize >= 0)
			randomize = event.data.start.randomize;
		else
			randomize = 1;
		if (event.data.start.nothreads >= 0)
			nothreads = event.data.start.nothreads;
		else
			nothreads = 0;
351
352
353
354
		if (event.data.start.dostype >= 0)
			dostype = event.data.start.dostype;
		else
			dostype = -1;
355
356
357
358
		if (event.data.start.debug >= 0)
			debug = event.data.start.debug;
		else
			debug = 0;
Mike Hibler's avatar
Mike Hibler committed
359
		if (event.data.start.trace >= 0)
360
			tracing = event.data.start.trace;
Mike Hibler's avatar
Mike Hibler committed
361
		else
362
			tracing = 0;
Mike Hibler's avatar
Mike Hibler committed
363
364
365
366
		if (event.data.start.traceprefix[0] > 0)
			strncpy(traceprefix, event.data.start.traceprefix, 64);
		else
			traceprefix[0] = 0;
367

Mike Hibler's avatar
Mike Hibler committed
368
		log("Starting: slice=%d, startat=%d, startdelay=%d, zero=%d, "
369
		    "randomize=%d, nothreads=%d, debug=%d, tracing=%d, "
Mike Hibler's avatar
Mike Hibler committed
370
371
372
373
		    "pkttimeout=%d, idletimer=%d, idledelay=%d, redodelay=%d, "
		    "maxmem=%d, chunkbufs=%d, maxwritebumfem=%d, "
		    "maxreadahead=%d, maxinprogress=%d",
		    slice, startat, startdelay, zero, randomize, nothreads,
374
		    debug, tracing, pkttimeout, idletimer, idledelay, redodelay,
Mike Hibler's avatar
Mike Hibler committed
375
376
		    maxmem, maxchunkbufs, maxwritebufmem,
		    maxreadahead, maxinprogress);
377
378
379
	}
#endif

Mike Hibler's avatar
Mike Hibler committed
380
381
	redodelay = sleeptime(redodelay, "request retry delay", 0);
	idledelay = sleeptime(idledelay, "writer idle delay", 0);
382

Mike Hibler's avatar
Mike Hibler committed
383
384
385
386
387
388
389
390
391
392
393
394
	/*
	 * Set initial memory limits.  These may be adjusted when we
	 * find out how big the image is.
	 */
	if (maxmem != 0) {
		/* XXX divide it up 50/50 */
		maxchunkbufs = (maxmem/2 * 1024*1024) / sizeof(ChunkBuffer_t);
		maxwritebufmem = maxmem/2;
	}

	ImageUnzipInit(filename, slice, debug, zero, nothreads, dostype,
		       maxwritebufmem*1024*1024);
395

396
	if (tracing) {
397
		ClientTraceInit(traceprefix);
398
399
400
		TraceStart(tracing);
	}

401
	PlayFrisbee();
402
403
404
405
406
407

	if (tracing) {
		TraceStop();
		TraceDump();
	}

408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
	ImageUnzipQuit();

#ifdef DOEVENTS
	if (eventserver != NULL) {
		log("Waiting for START/STOP event...");
		EventWait(EV_ANY, &event);
		if (event.type == EV_START) {
#ifdef STATS
			memset(&Stats, 0, sizeof(Stats));
#endif
			goto again;
		}
	done:
		if (event.type == EV_STOP && event.data.stop.exitstatus >= 0)
			exitstatus = event.data.stop.exitstatus;
		exit(exitstatus);
	}
#endif

427
428
429
430
431
432
433
434
435
436
	exit(0);
}

/*
 * The client receive thread. This thread takes in packets from the server.
 */
void *
ClientRecvThread(void *arg)
{
	Packet_t	packet, *p = &packet;
437
	int		BackOff;
Mike Hibler's avatar
Mike Hibler committed
438
	static int	gotone;
439
440
441
442

	if (debug)
		log("Receive pthread starting up ...");

443
444
445
446
447
448
449
	/*
	 * Use this to control the rate at which we request blocks.
	 * The IdleCounter is how many ticks we let pass without a
	 * useful block, before we make another request. We want that to
	 * be short, but not too short; we do not want to pummel the
	 * server. 
	 */
450
	IdleCounter = idletimer;
451

452
453
454
455
456
457
458
459
460
461
462
	/*
	 * This is another throttling mechanism; avoid making repeated
	 * requests to a server that is not running. That is, if the server
	 * is not responding, slowly back off our request rate (to about
	 * one a second) until the server starts responding.  This will
	 * prevent a large group of clients from pummeling the server
	 * machine, when there is no server running to respond (say, if the
	 * server process died).
	 */
	BackOff = 0;

463
	while (1) {
464
465
466
467
468
469
470
471
472
#ifdef NEVENTS
		static int needstamp = 1;
		struct timeval pstamp;
		if (needstamp) {
			gettimeofday(&pstamp, 0);
			needstamp = 0;
		}
#endif

473
474
475
		/*
		 * If we go too long without getting a block, we want
		 * to make another chunk request.
476
477
478
479
480
481
482
		 *
		 * XXX fixme: should probably be if it hasn't received
		 * a block that it is able to make use of.  But that has
		 * problems in that any new request we make will wind up
		 * at the end of the server work list, and we might not
		 * see that block for longer than our timeout period,
		 * leading us to issue another request, etc.
483
		 */
Mike Hibler's avatar
Mike Hibler committed
484
		if (PacketReceive(p) != 0) {
485
486
			pthread_testcancel();
			if (--IdleCounter <= 0) {
Mike Hibler's avatar
Mike Hibler committed
487
488
				if (gotone)
					DOSTAT(recvidles++);
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
				CLEVENT(2, EV_CLIRTIMO,
					pstamp.tv_sec, pstamp.tv_usec, 0, 0);
#ifdef NEVENTS
				needstamp = 1;
#endif
				RequestChunk(1);
				IdleCounter = idletimer;

				if (BackOff++) {
					IdleCounter += BackOff;
					if (BackOff > TIMEOUT_HZ)
						BackOff = TIMEOUT_HZ;
				}
			}
			continue;
		}
		pthread_testcancel();
Mike Hibler's avatar
Mike Hibler committed
506
		gotone = 1;
507
508
509
510
511
512
513
514
515

		if (! PacketValid(p, TotalChunkCount)) {
			log("received bad packet %d/%d, ignored",
			    p->hdr.type, p->hdr.subtype);
			continue;
		}

		switch (p->hdr.subtype) {
		case PKTSUBTYPE_BLOCK:
Mike Hibler's avatar
Mike Hibler committed
516
517
518
519
520
521
522
523
524
525
526
527
528
			/*
			 * Ensure blocks comes from where we expect.
			 * The validity of hdr.srcip has already been checked.
			 */
			if (serverip.s_addr != 0 &&
			    serverip.s_addr != p->hdr.srcip) {
				struct in_addr tmp = { p->hdr.srcip };
				log("received BLOCK from non-server %s",
				    inet_ntoa(tmp));
				continue;
			}

			CLEVENT(3, EV_CLIGOTPKT,
529
530
531
532
533
534
				pstamp.tv_sec, pstamp.tv_usec, 0, 0);
#ifdef NEVENTS
			needstamp = 1;
#endif
			BackOff = 0;
			GotBlock(p);
Mike Hibler's avatar
Mike Hibler committed
535
536
537
538
539
540
541
			/*
			 * We may have missed the request for this chunk/block
			 * so treat the arrival of a block as an indication
			 * that someone requested it.
			 */
			RequestStamp(p->msg.block.chunk, p->msg.block.block,
				     1, 0);
542
543
544
545
546
547
			break;

		case PKTSUBTYPE_REQUEST:
			CLEVENT(4, EV_CLIREQMSG,
				p->hdr.srcip, p->msg.request.chunk,
				p->msg.request.block, p->msg.request.count);
Mike Hibler's avatar
Mike Hibler committed
548
549
550
551
552
553
554
555
556
			RequestStamp(p->msg.request.chunk, p->msg.request.block,
				     p->msg.request.count, 0);
			break;

		case PKTSUBTYPE_PREQUEST:
			CLEVENT(4, EV_CLIPREQMSG,
				p->hdr.srcip, p->msg.request.chunk, 0, 0);
			BlockMapApply(&p->msg.prequest.blockmap,
				      p->msg.prequest.chunk, RequestStamp, 0);
557
558
559
560
561
562
563
564
565
			break;

		case PKTSUBTYPE_JOIN:
		case PKTSUBTYPE_LEAVE:
			/* Ignore these. They are from other clients. */
			CLEVENT(4, EV_OCLIMSG,
				p->hdr.srcip, p->hdr.subtype, 0, 0);
			break;
		}
566
567
568
569
570
571
572
573
574
	}
}

/*
 * The heart of the game.
 */
static void
ChunkerStartup(void)
{
Mike Hibler's avatar
Mike Hibler committed
575
576
577
578
579
	pthread_t	child_pid;
	void		*ignored;
	int		chunkcount = TotalChunkCount;
	int		i, wasidle = 0;
	static int	gotone;
580
581

	/*
582
	 * Allocate the chunk descriptors, request list and cache buffers.
583
	 */
Mike Hibler's avatar
Mike Hibler committed
584
585
	Chunks = calloc(chunkcount, sizeof(*Chunks));
	if (Chunks == NULL)
586
		fatal("Chunks: No more memory");
587

Mike Hibler's avatar
Mike Hibler committed
588
589
	ChunkRequestList = calloc(chunkcount, sizeof(*ChunkRequestList));
	if (ChunkRequestList == NULL)
590
591
		fatal("ChunkRequestList: No more memory");

Mike Hibler's avatar
Mike Hibler committed
592
593
	ChunkBuffer = malloc(maxchunkbufs * sizeof(ChunkBuffer_t));
	if (ChunkBuffer == NULL)
594
595
596
597
598
		fatal("ChunkBuffer: No more memory");

	/*
	 * Set all the buffers to "free"
	 */
599
600
	for (i = 0; i < maxchunkbufs; i++)
		ChunkBuffer[i].state = CHUNK_EMPTY;
601

602
603
604
	for (i = 0; i < TotalChunkCount; i++)
		ChunkRequestList[i] = i;
	
605
606
607
608
609
610
611
612
	/*
	 * We randomize the block selection so that multiple clients
	 * do not end up getting stalled by each other. That is, if
	 * all the clients were requesting blocks in order, then all
	 * the clients would end up waiting until the last client was
	 * done (since the server processes client requests in FIFO
	 * order).
	 */
613
614
615
616
617
618
619
620
621
622
	if (randomize) {
		for (i = 0; i < 50 * TotalChunkCount; i++) {
			int c1 = random() % TotalChunkCount;
			int c2 = random() % TotalChunkCount;
			int t1 = ChunkRequestList[c1];
			int t2 = ChunkRequestList[c2];

			ChunkRequestList[c2] = t1;
			ChunkRequestList[c1] = t2;
		}
623
624
625
626
627
628
629
630
631
632
633
634
635
636
	}

	if (pthread_create(&child_pid, NULL,
			   ClientRecvThread, (void *)0)) {
		fatal("Failed to create pthread!");
	}

	/*
	 * Loop until all chunks have been received and written to disk.
	 */
	while (chunkcount) {
		/*
		 * Search the chunk cache for a chunk that is ready to write.
		 */
637
		for (i = 0; i < maxchunkbufs; i++)
638
			if (ChunkBuffer[i].state == CHUNK_FULL)
639
640
641
642
				break;

		/*
		 * If nothing to do, then get out of the way for a while.
643
		 * XXX should be a condition variable.
644
		 */
645
		if (i == maxchunkbufs) {
Mike Hibler's avatar
Mike Hibler committed
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
#ifdef DOEVENTS
			Event_t event;
			if (eventserver != NULL &&
			    EventCheck(&event) && event.type == EV_STOP) {
				log("Aborted after %d chunks",
				    TotalChunkCount-chunkcount);
				break;
			}
#endif
			if (!wasidle) {
				CLEVENT(1, EV_CLIWRIDLE, 0, 0, 0, 0);
				if (debug)
					log("No chunks ready to write!");
			}
			if (gotone)
				DOSTAT(nochunksready++);
662
			fsleep(idledelay);
Mike Hibler's avatar
Mike Hibler committed
663
			wasidle++;
664
665
			continue;
		}
Mike Hibler's avatar
Mike Hibler committed
666
		gotone = 1;
667
668
669
670

		/*
		 * We have a completed chunk. Write it to disk.
		 */
671
		if (debug)
Mike Hibler's avatar
Mike Hibler committed
672
673
674
675
			log("Writing chunk %d (buffer %d) after idle=%d.%03d",
			    ChunkBuffer[i].thischunk, i,
			    (wasidle*idledelay) / 1000000,
			    ((wasidle*idledelay) % 1000000) / 1000);
676
677
678
679
680
681
682
683
684
685
686
687
688
689
		else {
			struct timeval estamp;

			gettimeofday(&estamp, 0);
			estamp.tv_sec -= stamp.tv_sec;
		
			printf(".");
			fflush(stdout);
			if (dotcol++ > 65) {
				dotcol = 0;
				printf("%4ld %6d\n",
				       estamp.tv_sec, chunkcount);
			}
		}
690

Mike Hibler's avatar
Mike Hibler committed
691
692
693
694
695
		CLEVENT(1, EV_CLIWRSTART,
			ChunkBuffer[i].thischunk, wasidle,
			decompblocks, writeridles);
		wasidle = 0;

696
697
		if (ImageUnzipChunk(ChunkBuffer[i].blocks[0].data))
			pfatal("ImageUnzipChunk failed");
698
699
700
701

		/*
		 * Okay, free the slot up for another chunk.
		 */
702
		ChunkBuffer[i].state = CHUNK_EMPTY;
703
		chunkcount--;
704
		CLEVENT(1, EV_CLIWRDONE,
Mike Hibler's avatar
Mike Hibler committed
705
706
			ChunkBuffer[i].thischunk, chunkcount,
			decompblocks, writeridles);
707
708
709
710
711
712
713
714
	}
	/*
	 * Kill the child and wait for it before returning. We do not
	 * want the child absorbing any more packets, cause that would
	 * mess up the termination handshake with the server. 
	 */
	pthread_cancel(child_pid);
	pthread_join(child_pid, &ignored);
715
716
717
718
719
720
721
722
723
724

	/*
	 * Make sure any asynchronous writes are done
	 * and collect stats from the unzipper.
	 */
	ImageUnzipFlush();
#ifdef STATS
	{
		extern long long totaledata, totalrdata;
		
Mike Hibler's avatar
Mike Hibler committed
725
		Stats.u.v1.decompblocks = decompblocks;
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
		Stats.u.v1.writeridles = writeridles;
		Stats.u.v1.ebyteswritten = totaledata;
		Stats.u.v1.rbyteswritten = totalrdata;
	}
#endif

	free(ChunkBuffer);
	free(ChunkRequestList);
	free(Chunks);
}

/*
 * Note that someone has made a request from the server right now.
 * This is either a request by us or one we snooped.
 *
 * We use the time stamp to determine when we should repeat a request to
 * the server.  If we update the stamp here, we are further delaying
 * a re-request.  The general strategy is: if a chunk request contains
 * any blocks that we will be able to use, we update the stamp to delay
 * what would otherwise be a redundant request.
 */
static void
Mike Hibler's avatar
Mike Hibler committed
748
RequestStamp(int chunk, int block, int count, void *arg)
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
{
	int stampme = 0;

	/*
	 * If not doing delays, don't bother with the stamp
	 */
	if (redodelay == 0)
		return;

	/*
	 * Common case of a complete chunk request, always stamp.
	 * This will include chunks we have already written and wouldn't
	 * be re-requesting, but updating the stamp doesn't hurt anything.
	 */
	if (block == 0 && count == CHUNKSIZE)
		stampme = 1;
	/*
	 * Else, request is for a partial chunk. If we are not currently
	 * processing this chunk, then the chunk data will be of use to
	 * us so we update the stamp.  Again, this includes chunks we
	 * are already finished with, but no harm.
	 */
	else if (! Chunks[chunk].done)
		stampme = 1;
	/*
	 * Otherwise, this is a partial chunk request for which we have
	 * already received some blocks.  We need to determine if the
	 * request contains any blocks that we need to complete our copy
	 * of the chunk.  If so, we conservatively update the stamp as it
	 * implies there is at least some chunk data coming that we will
	 * be able to use.  If the request contains only blocks that we
	 * already have, then the returned data will be of no use to us
	 * for completing our copy and we will still have to make a
	 * further request (i.e., we don't stamp).
	 */
	else {
Mike Hibler's avatar
Mike Hibler committed
785
		int i;
786
787
788

		for (i = 0; i < maxchunkbufs; i++)
			if (ChunkBuffer[i].thischunk == chunk &&
789
			    ChunkBuffer[i].state == CHUNK_FILLING)
790
				break;
Mike Hibler's avatar
Mike Hibler committed
791
792
793
794
		if (i < maxchunkbufs &&
		    BlockMapIsAlloc(&ChunkBuffer[i].blockmap, block, count)
		    != count)
				stampme = 1;
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
	}

	if (stampme) {
		struct timeval tv;

		gettimeofday(&tv, 0);
		Chunks[chunk].lastreq =
			(unsigned long long)tv.tv_sec * 1000000 + tv.tv_usec;
		CLEVENT(5, EV_CLISTAMP, chunk, tv.tv_sec, tv.tv_usec, 0);
	}
}

/*
 * Returns 1 if we have not made (or seen) a request for the given chunk
 * "for awhile", 0 otherwise.
 */
static int
RequestRedoTime(int chunk, unsigned long long curtime)
{
	if (Chunks[chunk].lastreq == 0 || redodelay == 0 ||
	    (int)(curtime - Chunks[chunk].lastreq) >= redodelay)
		return 1;
	return 0;
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
}

/*
 * Receive a single data block. If the block is for a chunk in progress, then
 * insert the data and check for a completed chunk. It will be up to the main
 * thread to process that chunk.
 *
 * If the block is the first of some chunk, then try to allocate a new chunk.
 * If the chunk buffer is full, then drop the block. If this happens, it
 * indicates the chunk buffer is not big enough, and should be increased.
 */
static void
GotBlock(Packet_t *p)
{
	int	chunk = p->msg.block.chunk;
	int	block = p->msg.block.block;
	int	i, free = -1;
Mike Hibler's avatar
Mike Hibler committed
835
	static int lastnoroomchunk = -1, lastnoroomblocks, inprogress;
836
837
838
839

	/*
	 * Search the chunk buffer for a match (or a free one).
	 */
840
	for (i = 0; i < maxchunkbufs; i++) {
841
		if (ChunkBuffer[i].state == CHUNK_EMPTY) {
842
843
			if (free == -1)
				free = i;
844
845
846
			continue;
		}
		
847
		if (ChunkBuffer[i].state == CHUNK_FILLING &&
848
849
850
		    ChunkBuffer[i].thischunk == chunk)
			break;
	}
851
	if (i == maxchunkbufs) {
852
853
854
855
856
		/*
		 * Did not find it. Allocate the free one, or drop the
		 * packet if there is no free chunk.
		 */
		if (free == -1) {
Mike Hibler's avatar
Mike Hibler committed
857
858
859
860
861
862
863
864
865
866
			if (chunk != lastnoroomchunk) {
				CLEVENT(1, EV_CLINOROOM, chunk, block,
					lastnoroomblocks, 0);
				lastnoroomchunk = chunk;
				lastnoroomblocks = 0;
				if (debug)
					log("No free buffer for chunk %d!",
					    chunk);
			}
			lastnoroomblocks++;
867
			DOSTAT(nofreechunks++);
868
869
			return;
		}
Mike Hibler's avatar
Mike Hibler committed
870
871
		lastnoroomchunk = -1;
		lastnoroomblocks = 0;
872
873
874
875

		/*
		 * Was this chunk already processed? 
		 */
876
877
		if (Chunks[chunk].done) {
			CLEVENT(3, EV_CLIDUPCHUNK, chunk, block, 0, 0);
878
			DOSTAT(dupchunk++);
Mike Hibler's avatar
Mike Hibler committed
879
			if (debug > 2)
880
881
882
				log("Duplicate chunk %d ignored!", chunk);
			return;
		}
883
		Chunks[chunk].done = 1;
884
885

		if (debug)
886
			log("Starting chunk %d (buffer %d)", chunk, free);
887
888

		i = free;
889
		ChunkBuffer[i].state      = CHUNK_FILLING;
890
891
		ChunkBuffer[i].thischunk  = chunk;
		ChunkBuffer[i].blockcount = CHUNKSIZE;
Mike Hibler's avatar
Mike Hibler committed
892
893
894
895
		bzero(&ChunkBuffer[i].blockmap,
		      sizeof(ChunkBuffer[i].blockmap));
		inprogress++;
		CLEVENT(1, EV_CLISCHUNK, chunk, block, inprogress, 0);
896
897
898
899
900
901
902
903
	}

	/*
	 * Insert the block and update the metainfo. We have to watch for
	 * duplicate blocks in the same chunk since another client may
	 * issue a request for a lost block, and we will see that even if
	 * we do not need it (cause of broadcast/multicast).
	 */
Mike Hibler's avatar
Mike Hibler committed
904
	if (BlockMapAlloc(&ChunkBuffer[i].blockmap, block)) {
905
		CLEVENT(3, EV_CLIDUPBLOCK, chunk, block, 0, 0);
906
		DOSTAT(dupblock++);
Mike Hibler's avatar
Mike Hibler committed
907
		if (debug > 2)
908
909
910
911
912
			log("Duplicate block %d in chunk %d", block, chunk);
		return;
	}
	ChunkBuffer[i].blockcount--;
	memcpy(ChunkBuffer[i].blocks[block].data, p->msg.block.buf, BLOCKSIZE);
Mike Hibler's avatar
Mike Hibler committed
913
914
915
916
917
918
919
920
921
922

#ifdef NEVENTS
	/*
	 * If we switched chunks before completing the previous, make a note.
	 */
	{
		static int lastchunk = -1, lastblock, lastchunkbuf;

		if (lastchunk != -1 && chunk != lastchunk &&
		    lastchunk == ChunkBuffer[lastchunkbuf].thischunk &&
923
		    ChunkBuffer[lastchunkbuf].state == CHUNK_FILLING)
Mike Hibler's avatar
Mike Hibler committed
924
925
926
927
928
929
930
931
			CLEVENT(1, EV_CLILCHUNK, lastchunk, lastblock, 0, 0);
		lastchunkbuf = i;
		lastchunk = chunk;
		lastblock = block;
		CLEVENT(3, EV_CLIBLOCK, chunk, block,
			ChunkBuffer[i].blockcount, 0);
	}
#endif
932
933
934
935
936

	/*
	 * Anytime we receive a packet thats needed, reset the idle counter.
	 * This will prevent us from sending too many requests.
	 */
937
	IdleCounter = idletimer;
938
939
940
941
942

	/*
	 * Is the chunk complete? If so, then release it to the main thread.
	 */
	if (ChunkBuffer[i].blockcount == 0) {
Mike Hibler's avatar
Mike Hibler committed
943
944
		inprogress--;
		CLEVENT(1, EV_CLIECHUNK, chunk, block, inprogress, 0);
945
946
		if (debug)
			log("Releasing chunk %d to main thread", chunk);
947
		ChunkBuffer[i].state = CHUNK_FULL;
948
949
950
951
952
953
954
955
956
957
958

		/*
		 * Send off a request for a chunk we do not have yet. This
		 * should be enough to ensure that there is more work to do
		 * by the time the main thread finishes the chunk we just
		 * released.
		 */
		RequestChunk(0);
	}
}

Mike Hibler's avatar
Mike Hibler committed
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
/*
 * Request a chunk/block/range we do not have.
 */
static void
RequestMissing(int chunk, BlockMap_t *map, int count)
{
	Packet_t	packet, *p = &packet;

	if (debug)
		log("Requesting missing blocks of chunk:%d", chunk);
	
	p->hdr.type       = PKTTYPE_REQUEST;
	p->hdr.subtype    = PKTSUBTYPE_PREQUEST;
	p->hdr.datalen    = sizeof(p->msg.prequest);
	p->msg.prequest.chunk = chunk;
	p->msg.prequest.retries = Chunks[chunk].ours;
	BlockMapInvert(map, &p->msg.prequest.blockmap);
	PacketSend(p, 0);
#ifdef STATS
978
	assert(count == BlockMapIsAlloc(&p->msg.prequest.blockmap,0,CHUNKSIZE));
Mike Hibler's avatar
Mike Hibler committed
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
	if (count == 0)
		log("Request 0 blocks from chunk %d", chunk);
	Stats.u.v1.lostblocks += count;
	Stats.u.v1.requests++;
	if (Chunks[chunk].ours)
		Stats.u.v1.rerequests++;
#endif
	CLEVENT(1, EV_CLIPREQ, chunk, count, 0, 0);

	/*
	 * Since stamps are per-chunk and we wouldn't be here
	 * unless we were requesting something we are missing
	 * we can just unconditionally stamp the chunk.
	 */
	RequestStamp(chunk, 0, CHUNKSIZE, (void *)1);
	Chunks[chunk].ours = 1;
}

997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
/*
 * Request a chunk/block/range we do not have.
 */
static void
RequestRange(int chunk, int block, int count)
{
	Packet_t	packet, *p = &packet;

	if (debug)
		log("Requesting chunk:%d block:%d count:%d",
		    chunk, block, count);
	
	p->hdr.type       = PKTTYPE_REQUEST;
	p->hdr.subtype    = PKTSUBTYPE_REQUEST;
	p->hdr.datalen    = sizeof(p->msg.request);
	p->msg.request.chunk = chunk;
	p->msg.request.block = block;
	p->msg.request.count = count;
1015
	PacketSend(p, 0);
1016
	CLEVENT(1, EV_CLIREQ, chunk, block, count, 0);
1017
1018
	DOSTAT(requests++);

Mike Hibler's avatar
Mike Hibler committed
1019
1020
	RequestStamp(chunk, block, count, (void *)1);
	Chunks[chunk].ours = 1;
1021
1022
1023
1024
1025
}

static void
RequestChunk(int timedout)
{
1026
1027
1028
	int		   i, j, k;
	int		   emptybufs, fillingbufs;
	unsigned long long stamp = 0;
1029

1030
1031
	CLEVENT(1, EV_CLIREQCHUNK, timedout, 0, 0, 0);

1032
1033
1034
1035
1036
1037
1038
	if (! timedout) {
		struct timeval tv;

		gettimeofday(&tv, 0);
		stamp = (unsigned long long)tv.tv_sec * 1000000 + tv.tv_usec;
	}

1039
1040
1041
	/*
	 * Look for unfinished chunks.
	 */
1042
1043
1044
1045
1046
	emptybufs = fillingbufs = 0;
	for (i = 0; i < maxchunkbufs; i++) {
		/*
		 * Skip empty and full buffers
		 */
1047
		if (ChunkBuffer[i].state == CHUNK_EMPTY) {
1048
1049
1050
1051
			/*
			 * Keep track of empty chunk buffers while we are here
			 */
			emptybufs++;
1052
			continue;
1053
		}
1054
		if (ChunkBuffer[i].state == CHUNK_FULL)
1055
1056
1057
			continue;

		fillingbufs++;
1058
1059

		/*
1060
		 * Make sure this chunk is eligible for re-request.
1061
		 */
1062
1063
		if (! timedout &&
		    ! RequestRedoTime(ChunkBuffer[i].thischunk, stamp))
1064
1065
1066
			continue;

		/*
Mike Hibler's avatar
Mike Hibler committed
1067
		 * Request all the missing blocks
1068
		 */
Mike Hibler's avatar
Mike Hibler committed
1069
1070
1071
1072
		DOSTAT(prequests++);
		RequestMissing(ChunkBuffer[i].thischunk,
			       &ChunkBuffer[i].blockmap,
			       ChunkBuffer[i].blockcount);
1073
1074
	}

Mike Hibler's avatar
Mike Hibler committed
1075
1076
	CLEVENT(2, EV_CLIREQRA, emptybufs, fillingbufs, 0, 0);

1077
	/*
1078
1079
1080
1081
	 * Issue read-ahead requests.
	 *
	 * If we already have enough unfinished chunks on our plate
	 * or we have no room for read-ahead, don't do it.
1082
	 */
Mike Hibler's avatar
Mike Hibler committed
1083
	if (emptybufs == 0 || fillingbufs >= maxinprogress)
1084
1085
1086
		return;

	/*
1087
	 * Scan our request list looking for candidates.
1088
	 */
Mike Hibler's avatar
Mike Hibler committed
1089
1090
	k = (maxreadahead > emptybufs) ? emptybufs : maxreadahead;
	for (i = 0, j = 0; i < TotalChunkCount && j < k; i++) {
1091
		int chunk = ChunkRequestList[i];
1092
1093
1094
1095
1096
1097
		
		/*
		 * If already working on this chunk, skip it.
		 */
		if (Chunks[chunk].done)
			continue;
1098

1099
1100
1101
1102
1103
1104
		/*
		 * Issue a request for the chunk if it isn't already
		 * on the way.  This chunk, whether requested or not
		 * is considered a read-ahead to us.
		 */
		if (timedout || RequestRedoTime(chunk, stamp))
1105
			RequestRange(chunk, 0, CHUNKSIZE);
1106
1107

		j++;
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
	}
}

/*
 * Join the Frisbee team, and then go into the main loop above.
 */
static void
PlayFrisbee(void)
{
	Packet_t	packet, *p = &packet;
1118
	struct timeval  estamp, timeo;
1119
	unsigned int	myid;
Mike Hibler's avatar
Mike Hibler committed
1120
	int		delay;
1121
1122

	gettimeofday(&stamp, 0);
1123
	CLEVENT(1, EV_CLISTART, 0, 0, 0, 0);
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138

	/*
	 * Init the random number generator. We randomize the block request
	 * sequence above, and its important that each client have a different
	 * sequence!
	 */
	srandomdev();

	/*
	 * A random number ID. I do not think this is really necessary,
	 * but perhaps might be useful for determining when a client has
	 * crashed and returned.
	 */
	myid = random();
	
1139
1140
	/*
	 * To avoid a blast of messages from a large number of clients,
Mike Hibler's avatar
Mike Hibler committed
1141
1142
1143
1144
	 * we can delay a small amount before startup.  If startat is
	 * non-zero we delay for that number of seconds.  Otherwise, if
	 * startdelay is non-zero, the delay value is uniformly distributed
	 * between 0 and startdelay seconds, with ms granularity.
1145
	 */
Mike Hibler's avatar
Mike Hibler committed
1146
1147
1148
1149
1150
1151
1152
	if (startat > 0)
		delay = startat * 1000;
	else if (startdelay > 0)
		delay = random() % (startdelay * 1000);
	else
		delay = 0;
	if (delay) {
1153
1154
1155
1156
1157
1158
1159
		if (debug)
			log("Starup delay: %d.%03d seconds",
			    delay/1000, delay%1000);
		DOSTAT(delayms = delay);
		fsleep(delay * 1000);
	}

1160
1161
1162
1163
1164
	/*
	 * Send a join the team message. We block waiting for a reply
	 * since we need to know the total block size. We resend the
	 * message (dups are harmless) if we do not get a reply back.
	 */
1165
	gettimeofday(&timeo, 0);
1166
	while (1) {
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
		struct timeval now;

		gettimeofday(&now, 0);
		if (timercmp(&timeo, &now, <=)) {
#ifdef DOEVENTS
			Event_t event;
			if (eventserver != NULL &&
			    EventCheck(&event) && event.type == EV_STOP) {
				log("Aborted during JOIN");
				return;
			}
#endif
Mike Hibler's avatar
Mike Hibler committed
1179
			CLEVENT(1, EV_CLIJOINREQ, myid, 0, 0, 0);
1180
			DOSTAT(joinattempts++);
1181
1182
1183
1184
			p->hdr.type       = PKTTYPE_REQUEST;
			p->hdr.subtype    = PKTSUBTYPE_JOIN;
			p->hdr.datalen    = sizeof(p->msg.join);
			p->msg.join.clientid = myid;
1185
1186
1187
1188
			PacketSend(p, 0);
			timeo.tv_sec = 0;
			timeo.tv_usec = 500000;
			timeradd(&timeo, &now, &timeo);
1189
1190
1191
1192
		}

		/*
		 * Throw away any data packets. We cannot start until
1193
		 * we get a reply back.
1194
		 */
Mike Hibler's avatar
Mike Hibler committed
1195
		if (PacketReceive(p) == 0 &&
1196
		    p->hdr.subtype == PKTSUBTYPE_JOIN &&
1197
		    p->hdr.type == PKTTYPE_REPLY) {
Mike Hibler's avatar
Mike Hibler committed
1198
1199
			CLEVENT(1, EV_CLIJOINREP,
				p->msg.join.blockcount, 0, 0, 0);
1200
1201
1202
			break;
		}
	}
1203
1204
	gettimeofday(&timeo, 0);
	TotalChunkCount = p->msg.join.blockcount / CHUNKSIZE;
1205
	
Mike Hibler's avatar
Mike Hibler committed
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
	/*
	 * If we have partitioned up the memory and have allocated
	 * more chunkbufs than chunks in the file, reallocate the
	 * excess to disk buffering.  If the user has explicitly
	 * partitioned the memory, we leave everything as is.
	 */
	if (maxmem != 0 && maxchunkbufs > TotalChunkCount) {
		int excessmb;

		excessmb = ((maxchunkbufs - TotalChunkCount) *
			    sizeof(ChunkBuffer_t)) / (1024 * 1024);
		maxchunkbufs = TotalChunkCount;
		if (excessmb > 0) {
			maxwritebufmem += excessmb;
			ImageUnzipSetMemory(maxwritebufmem*1024*1024);
		}
	}
 
1224
1225
1226
	log("Joined the team after %d sec. ID is %u. "
	    "File is %d chunks (%d blocks)",
	    timeo.tv_sec - stamp.tv_sec,
1227
1228
1229
1230
	    myid, TotalChunkCount, p->msg.join.blockcount);

	ChunkerStartup();

1231
	gettimeofday(&estamp, 0);
1232
	timersub(&estamp, &stamp, &estamp);
1233
	
1234
	/*
1235
1236
1237
	 * Done! Send off a leave message, but do not worry about whether
	 * the server gets it. All the server does with it is print a
	 * timestamp, and that is not critical to operation.
1238
	 */
Mike Hibler's avatar
Mike Hibler committed
1239
	CLEVENT(1, EV_CLILEAVE, myid, estamp.tv_sec, 0, 0);
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
#ifdef STATS
	p->hdr.type       = PKTTYPE_REQUEST;
	p->hdr.subtype    = PKTSUBTYPE_LEAVE2;
	p->hdr.datalen    = sizeof(p->msg.leave2);
	p->msg.leave2.clientid = myid;
	p->msg.leave2.elapsed  = estamp.tv_sec;
	Stats.version            = CLIENT_STATS_VERSION;
	Stats.u.v1.runsec        = estamp.tv_sec;
	Stats.u.v1.runmsec       = estamp.tv_usec / 1000;
	Stats.u.v1.chunkbufs     = maxchunkbufs;
Mike Hibler's avatar
Mike Hibler committed
1250
	Stats.u.v1.writebufmem   = maxwritebufmem;
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
	Stats.u.v1.maxreadahead  = maxreadahead;
	Stats.u.v1.maxinprogress = maxinprogress;
	Stats.u.v1.pkttimeout    = pkttimeout;
	Stats.u.v1.startdelay    = startdelay;
	Stats.u.v1.idletimer     = idletimer;
	Stats.u.v1.idledelay     = idledelay;
	Stats.u.v1.redodelay     = redodelay;
	Stats.u.v1.randomize     = randomize;
	p->msg.leave2.stats      = Stats;
	PacketSend(p, 0);

	log("");
	ClientStatsDump(myid, &Stats);
#else
1265
1266
1267
1268
1269
	p->hdr.type       = PKTTYPE_REQUEST;
	p->hdr.subtype    = PKTSUBTYPE_LEAVE;
	p->hdr.datalen    = sizeof(p->msg.leave);
	p->msg.leave.clientid = myid;
	p->msg.leave.elapsed  = estamp.tv_sec;
1270
	PacketSend(p, 0);
1271
#endif
1272
	log("\nLeft the team after %ld seconds on the field!", estamp.tv_sec);
1273
}