imageunzip.c 52.1 KB
Newer Older
Leigh B. Stoller's avatar
Leigh B. Stoller committed
1
/*
2
 * Copyright (c) 2000-2015 University of Utah and the Flux Group.
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 * 
 * {{{EMULAB-LICENSE
 * 
 * This file is part of the Emulab network testbed software.
 * 
 * This file is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 * 
 * This file is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
 * License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this file.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * }}}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
22
23
 */

24
25
26
27
28
29
/*
 * Usage: imageunzip <input file>
 *
 * Writes the uncompressed data to stdout.
 */

30
31
32
33
34
35
#ifdef WITH_CRYPTO
#include <openssl/evp.h>
#include <openssl/sha.h>
#include <openssl/rand.h>
#endif
#include <ctype.h>
36
#include <stdio.h>
37
#include <stdlib.h>
38
39
40
#include <fcntl.h>
#include <assert.h>
#include <unistd.h>
Mike Hibler's avatar
all:    
Mike Hibler committed
41
#include <string.h>
42
#include <errno.h>
43
#include <zlib.h>
44
45
#include <sys/types.h>
#include <sys/time.h>
46
#include "imagehdr.h"
47
#include "queue.h"
48
#include "checksum.h"
49
50
51
#ifndef NOTHREADS
#include <pthread.h>
#endif
52

Mike Hibler's avatar
all:    
Mike Hibler committed
53
54
55
56
57
58
59
/*
 * Define this if you want to test frisbee's random presentation of chunks
 */
#ifndef FRISBEE
#define FAKEFRISBEE
#endif

60
#define MAXWRITEBUFMEM	0	/* 0 == unlimited */
Mike Hibler's avatar
Mike Hibler committed
61

62
63
64
65
66
67
68
long long totalddata = 0;	/* total decompressed data */
long long totaledata = 0;	/* total bytes covered by image */
long long totalrdata = 0;	/* total data written to disk (image+zeros) */
long long totalzdata = 0;	/* total zeros written to disk */
long totalwriteops   = 0;	/* total write operations */
long totalzeroops    = 0;	/* total zero write operations */
long totalseekops    = 0;	/* total non-contiguous writes */
69

70
71
int totalchunks, donechunks;

72
73
74
/*
 * In slice mode, we read the DOS MBR to find out where the slice is on
 * the raw disk, and then seek to that spot. This avoids sillyness in
75
 * the BSD kernel having to do with disklabels.
76
77
78
 *
 * These numbers are in sectors.
 */
79
80
static long		outputminsec	= 0;
static long		outputmaxsec	= 0;
81
82

/* Why is this not defined in a public header file? */
83
#define BOOT_MAGIC	0xAA55
84

85
86
#define CHECK_ERR(err, msg) { \
    if (err != Z_OK) { \
87
88
	fprintf(stderr, "%s error: %d\n", msg, err); \
	exit(1); \
89
90
91
    } \
}

Mike Hibler's avatar
Mike Hibler committed
92
93
94
95
/* XXX keep macros in sync with global.h */
int secsize = 512;
#define sectobytes(s)	((off_t)(s) * secsize)
#define bytestosec(b)	(uint32_t)((b) / secsize)
Mike Hibler's avatar
all:    
Mike Hibler committed
96

97
98
99
100
101
102
103
104
105
106
/*
 * Sector alignment is required for buffers used with read/write on
 * files opened with O_DIRECT (at least under Linux). So in several places
 * we allocate +SECSIZE bytes to data buffers so that we can ensure this
 * alignment. We could do this just when the -f (directio) command line
 * is specified, but we just do it all the time right now as the space
 * cost is generally small.
 */
#define SECALIGN(p)	(void *)(((uintptr_t)(p) + (SECSIZE-1)) & ~(SECSIZE-1))
#define ISSECALIGNED(p)	(((uintptr_t)(p) & (SECSIZE-1)) == 0)
107
#define ISSECMULT(s)	(((size_t)(s) & (SECSIZE-1)) == 0)
108

Mike Hibler's avatar
all:    
Mike Hibler committed
109
#define OUTSIZE (256 * 1024)
110
static char	 zeros[OUTSIZE+SECSIZE];
111
112
113
114
115
116
117
118
119
120
121

static int	 dostype = -1;
static int	 slice = 0;
static int	 debug = 0;
static int	 outfd;
static int	 dofill = 0;
static int	 nothreads = 0;
static int	 rdycount;
static int	 imageversion = 1;
static int	 dots   = 0;
static int	 dotcol;
122
static int	 directio = 0;
123
static int	 ignoreskips = 0;
Mike Hibler's avatar
Mike Hibler committed
124
static struct timeval stamp;
125
#ifndef FRISBEE
126
127
128
129
130
131
static int	 infd;
static int	 version= 0;
static unsigned	 fillpat= 0;
static int	 readretries = 0;
static int	 nodecompress = 0;
static char	 chunkbuf[CHUNKSIZE];
132
#endif
Mike Hibler's avatar
Mike Hibler committed
133
134
int		 getslicebounds(int slice);
void		 setslicetype(int slice, int dostype);
135
static int	 write_subblock(int, const char *, int);
136
137
138
static int	 inflate_subblock(const char *);
void		 writezeros(off_t offset, off_t zcount);
void		 writedata(off_t offset, size_t count, void *buf);
139

140
141
142
static void	zero_remainder(void);
static void	getrelocinfo(const blockhdr_t *hdr);
static size_t	applyrelocs(off_t offset, size_t cc, void *buf);
Mike Hibler's avatar
all:    
Mike Hibler committed
143

144
145
146
static int	 seekable;
static off_t	 nextwriteoffset;
static off_t	 maxwrittenoffset;
Mike Hibler's avatar
all:    
Mike Hibler committed
147

148
static int	 imagetoobigwarned;
Mike Hibler's avatar
Mike Hibler committed
149
150

#ifndef FRISBEE
151
static int	 docrconly = 0;
Mike Hibler's avatar
Mike Hibler committed
152
static u_int32_t crc;
153
154
extern void	 compute_crc(u_char *buf, int blen, u_int32_t *crcp);
ssize_t		 read_withretry(int fd, void *buf, size_t nbytes, off_t foff);
Mike Hibler's avatar
Mike Hibler committed
155
#endif
156

Mike Hibler's avatar
all:    
Mike Hibler committed
157
158
159
#ifdef FAKEFRISBEE
#include <sys/stat.h>

160
161
static int	dofrisbee;
static int	*chunklist, *nextchunk;
Mike Hibler's avatar
all:    
Mike Hibler committed
162
163
#endif

164
/* UUID for the current image, all chunks must have the same ID */
165
static unsigned char uuid[UUID_LENGTH];
166
167
static int has_id = 0;

168
169
170
/*
 * Some stats
 */
Mike Hibler's avatar
Mike Hibler committed
171
unsigned long decompblocks;
172
173
unsigned long writeridles;

174
#ifdef NOTHREADS
175
176
177
#define		threadinit()
#define		threadwait()
#define		threadquit()
Mike Hibler's avatar
Mike Hibler committed
178
179
180
181
182
183

/* XXX keep the code simple */
#define pthread_mutex_lock(l)
#define pthread_mutex_unlock(l)
#define pthread_testcancel()
#undef CONDVARS_WORK
184
#else
185
186
187
188
static void	 threadinit(void);
static void	 threadwait(void);
static void	 threadquit(void);
static void	*DiskWriter(void *arg);
189
190
191
#ifdef FRISBEE
extern void (*DiskStatusCallback)(int);
#endif
192

193
static int	writeinprogress; /* XXX */
194
static pthread_t child_pid;
195
static pthread_mutex_t	writequeue_mutex;
Mike Hibler's avatar
Mike Hibler committed
196
#ifdef CONDVARS_WORK
197
static pthread_cond_t	writequeue_cond;
Mike Hibler's avatar
Mike Hibler committed
198
#endif
Mike Hibler's avatar
Mike Hibler committed
199
#endif
200

201
202
203
/*
 * A queue of ready to write data blocks.
 */
Mike Hibler's avatar
Mike Hibler committed
204
typedef struct {
205
206
207
	int		refs;
	size_t		size;
	char		data[0];
Mike Hibler's avatar
Mike Hibler committed
208
209
} buffer_t;

210
typedef struct {
211
212
213
214
215
	queue_chain_t	chain;
	off_t		offset;
	off_t		size;
	buffer_t	*buf;
	char		*data;
Mike Hibler's avatar
Mike Hibler committed
216
217
} writebuf_t;

218
219
static unsigned long long	   maxwritebufmem = MAXWRITEBUFMEM;
static volatile unsigned long long curwritebufmem, curwritebufs;
Mike Hibler's avatar
Mike Hibler committed
220
#ifndef NOTHREADS
221
222
static queue_head_t	writequeue;
static pthread_mutex_t	writebuf_mutex;
Mike Hibler's avatar
Mike Hibler committed
223
#ifdef CONDVARS_WORK
224
static pthread_cond_t	writebuf_cond;
Mike Hibler's avatar
Mike Hibler committed
225
#endif
Mike Hibler's avatar
Mike Hibler committed
226
#endif
227
static volatile int	writebufwanted;
228

Mike Hibler's avatar
Mike Hibler committed
229
/* stats */
230
231
unsigned long		maxbufsalloced;
unsigned long long	maxmemalloced;
232
unsigned long		memsplits;
Mike Hibler's avatar
Mike Hibler committed
233

234
#ifdef WITH_CRYPTO
235
/* security */
236
static int do_checksum = 0;
237
static int do_decrypt = 0;
238
static int cipher = ENC_NONE;
239
240
static unsigned char encryption_key[ENC_MAX_KEYLEN];
#endif
241

Mike Hibler's avatar
Mike Hibler committed
242
243
244
245
#ifndef CONDVARS_WORK
int fsleep(unsigned int usecs);
#endif

246
#ifndef FRISBEE
247
248
249
void
dump_stats(int sig)
{
250
251
252
253
254
255
256
257
258
259
260
261
262
	struct timeval estamp;

	gettimeofday(&estamp, 0);
	estamp.tv_sec -= stamp.tv_sec;
	if (sig == 0 && debug != 1 && dots) {
		while (dotcol++ <= 66)
			fprintf(stderr, " ");

		fprintf(stderr, "%4ld %6d\n",
			(long)estamp.tv_sec, totalchunks - donechunks);
	}
	else {
		if (sig) {
Mike Hibler's avatar
Mike Hibler committed
263
#ifndef NOTHREADS
264
265
			if (pthread_self() == child_pid)
				return;
Mike Hibler's avatar
Mike Hibler committed
266
#endif
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
			if (dots && dotcol)
				fputc('\n', stderr);
			if (totalchunks)
				fprintf(stderr, "%d of %d chunks decompressed\n",
					donechunks, totalchunks);
			else
				fprintf(stderr, "%d chunks decompressed\n",
					donechunks);
		}
		fprintf(stderr,
			"Decompressed %lld bytes, wrote %lld bytes (%lld actual) in %ld.%03ld seconds\n",
			totalddata, totaledata, totalrdata,
			(long)estamp.tv_sec, (long)estamp.tv_usec/1000);
		fprintf(stderr,
			"%lld bytes of data in %ld ops (%lld bytes/op), %ld seeks\n",
			totalrdata, totalwriteops,
			totalrdata/(totalwriteops?:1), totalseekops);
		if (totalzdata > 0) {
			fprintf(stderr,
				"  %lld bytes of disk data in %ld ops (%lld bytes/op)\n",
				(totalrdata-totalzdata),
				(totalwriteops-totalzeroops),
				(totalrdata-totalzdata)/((totalwriteops-totalzeroops)?:1));
			fprintf(stderr,
				"  %lld bytes of zero data in %ld ops (%lld bytes/op)\n",
				totalzdata, totalzeroops,
				totalzdata/(totalzeroops?:1));
		}
	}
	if (debug)
		fprintf(stderr, "decompressor blocked: %lu, "
			"writer idle: %lu, writes performed: %d\n",
			decompblocks, writeridles, rdycount);
300
}
301
#endif
302

303
304
305
306
#define DODOTS_CHUNKS	1
#define DODOTS_DATA	2
#define DODOTS_ZERO	3
#define DODOTS_SKIP	4
Mike Hibler's avatar
Mike Hibler committed
307
308
309

void dodots(int dottype, off_t cc)
{
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
	static int lastgb = 0;
	int count = 0, newgb;
	char chr = 0;


	switch (dottype) {
	case DODOTS_CHUNKS:
		if ((dots & 1) == 0)
			return;
		count = 1;
		chr = '.';
		break;
	case DODOTS_DATA:
	case DODOTS_ZERO:
	case DODOTS_SKIP:
		if ((dots & 2) == 0)
			return;
		/*
		 * totaledata has not been updated yet, so we have to
		 * figure it out ourselves.
		 */
		newgb = (totaledata + cc) / 1000000000LL;
		if ((count = newgb - lastgb) <= 0)
			return;
		lastgb = newgb;
		chr = (dottype == DODOTS_DATA) ? '.' :
			(dottype == DODOTS_ZERO) ? 'z' : 's';
		break;
	}

	while (count-- > 0) {
		fputc(chr, stderr);
		if (dotcol++ > 65) {
			struct timeval estamp;

			gettimeofday(&estamp, 0);
			estamp.tv_sec -= stamp.tv_sec;
			fprintf(stderr, "%4ld %6d\n",
				(long)estamp.tv_sec, totalchunks - donechunks);
			dotcol = 0;
		}
	}
Mike Hibler's avatar
Mike Hibler committed
352
353
}

Mike Hibler's avatar
Mike Hibler committed
354
355
356
void
dump_writebufs(void)
{
357
	fprintf(stderr, "%lu max bufs, %llu max memory\n",
358
		maxbufsalloced, maxmemalloced);
359
360
	fprintf(stderr, "%lu buffers split, %lu blocked on buf memory\n",
		memsplits, decompblocks);
Mike Hibler's avatar
Mike Hibler committed
361
362
363
364
365
}

static writebuf_t *
alloc_writebuf(off_t offset, off_t size, int allocbuf, int dowait)
{
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
	writebuf_t *wbuf;
	buffer_t *buf = NULL;
	size_t bufsize;

	pthread_mutex_lock(&writebuf_mutex);
	wbuf = malloc(sizeof(*wbuf));
	if (wbuf == NULL) {
		fprintf(stderr, "could not alloc writebuf header\n");
		exit(1);
	}
	bufsize = allocbuf ? size : 0;
	if (bufsize) {
		do {
			if (maxwritebufmem &&
			    curwritebufmem + bufsize > maxwritebufmem)
				buf = NULL;
			else
383
384
385
				/* +SECSIZE is for O_DIRECT alignement */
				buf = malloc(sizeof(buffer_t) + bufsize
					     + SECSIZE);
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401

			if (buf == NULL) {
				if (!dowait) {
					free(wbuf);
					pthread_mutex_unlock(&writebuf_mutex);
					return NULL;
				}

				decompblocks++;
				writebufwanted = 1;
				/*
				 * Once again it appears that linuxthreads
				 * condition variables don't work well.
				 * We seem to sleep longer than necessary.
				 */
				do {
Mike Hibler's avatar
Mike Hibler committed
402
#ifdef CONDVARS_WORK
403
404
					pthread_cond_wait(&writebuf_cond,
							  &writebuf_mutex);
Mike Hibler's avatar
Mike Hibler committed
405
#else
406
407
408
					pthread_mutex_unlock(&writebuf_mutex);
					fsleep(1000);
					pthread_mutex_lock(&writebuf_mutex);
409
#endif
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
					pthread_testcancel();
				} while (writebufwanted);
			}
		} while (buf == NULL);
		buf->refs = 1;
		buf->size = bufsize;
	}
	curwritebufs++;
	curwritebufmem += bufsize;
	if (curwritebufs > maxbufsalloced)
		maxbufsalloced = curwritebufs;
	if (curwritebufmem > maxmemalloced)
		maxmemalloced = curwritebufmem;
	pthread_mutex_unlock(&writebuf_mutex);

	queue_init(&wbuf->chain);
	wbuf->offset = offset;
	wbuf->size = size;
	wbuf->buf = buf;
429
	wbuf->data = buf ? SECALIGN(buf->data) : NULL;
430
431

	return wbuf;
Mike Hibler's avatar
Mike Hibler committed
432
433
434
435
436
}

static writebuf_t *
split_writebuf(writebuf_t *wbuf, off_t doff, int dowait)
{
437
438
439
440
441
	writebuf_t *nwbuf;
	off_t size;

	assert(wbuf->buf != NULL);

442
	memsplits++;
443
444
445
446
447
448
449
450
451
452
453
454
	assert(doff < wbuf->size);
	size = wbuf->size - doff;
	nwbuf = alloc_writebuf(wbuf->offset+doff, size, 0, dowait);
	if (nwbuf) {
		wbuf->size -= size;
		pthread_mutex_lock(&writebuf_mutex);
		wbuf->buf->refs++;
		pthread_mutex_unlock(&writebuf_mutex);
		nwbuf->buf = wbuf->buf;
		nwbuf->data = wbuf->data + doff;
	}
	return nwbuf;
Mike Hibler's avatar
Mike Hibler committed
455
}
456

457
static void
Mike Hibler's avatar
Mike Hibler committed
458
free_writebuf(writebuf_t *wbuf)
459
{
460
461
462
463
464
465
466
467
468
469
	assert(wbuf != NULL);

	pthread_mutex_lock(&writebuf_mutex);
	if (wbuf->buf && --wbuf->buf->refs == 0) {
		curwritebufs--;
		curwritebufmem -= wbuf->buf->size;
		assert(curwritebufmem >= 0);
		free(wbuf->buf);
		if (writebufwanted) {
			writebufwanted = 0;
Mike Hibler's avatar
Mike Hibler committed
470
#ifdef CONDVARS_WORK
471
			pthread_cond_signal(&writebuf_cond);
472
#endif
473
474
475
476
		}
	}
	free(wbuf);
	pthread_mutex_unlock(&writebuf_mutex);
Mike Hibler's avatar
Mike Hibler committed
477
478
479
480
481
}

static void
dowrite_request(writebuf_t *wbuf)
{
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
	off_t offset, size;
	void *buf;

	offset = wbuf->offset;
	size = wbuf->size;
	buf = wbuf->data;
	assert(offset >= 0);
	assert(size > 0);

	/*
	 * Adjust for partition start and ensure data fits
	 * within partition boundaries.
	 */
	offset += sectobytes(outputminsec);
	assert((offset & (SECSIZE-1)) == 0);
	if (outputmaxsec > 0 && offset + size > sectobytes(outputmaxsec)) {
		if (!imagetoobigwarned) {
			fprintf(stderr, "WARNING: image too large "
				"for target slice, truncating\n");
			imagetoobigwarned = 1;
		}
		if (offset >= sectobytes(outputmaxsec)) {
			free_writebuf(wbuf);
			return;
		}
		size = sectobytes(outputmaxsec) - offset;
		wbuf->size = size;
	}
	wbuf->offset = offset;

	totaledata += size;

	if (nothreads) {
		/*
		 * Null buf means its a request to zero.
		 * If we are not filling, just return.
		 */
		if (buf == NULL) {
			if (dofill)
				writezeros(offset, size);
		} else {
			assert(size <= OUTSIZE);

			/*
			 * Handle any relocations
			 */
			size = (off_t)applyrelocs(offset, (size_t)size, buf);
			writedata(offset, (size_t)size, buf);
		}
		free_writebuf(wbuf);
		return;
	}
534
535

#ifndef NOTHREADS
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
	if (buf == NULL) {
		if (!dofill) {
			free_writebuf(wbuf);
			return;
		}
	} else {
		assert(size <= OUTSIZE);

		/*
		 * Handle any relocations
		 */
		size = (off_t)applyrelocs(offset, (size_t)size, buf);
		wbuf->size = size;
	}

	/*
	 * Queue it up for the writer thread
	 */
	pthread_mutex_lock(&writequeue_mutex);
	queue_enter(&writequeue, wbuf, writebuf_t *, chain);
Mike Hibler's avatar
Mike Hibler committed
556
#ifdef CONDVARS_WORK
557
	pthread_cond_signal(&writequeue_cond);
Mike Hibler's avatar
Mike Hibler committed
558
#endif
559
	pthread_mutex_unlock(&writequeue_mutex);
560
#endif
561
}
562

563
#ifndef FRISBEE
564
565
566
static void
usage(void)
{
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
	fprintf(stderr, "usage: "
		"imageunzip options <input filename> [output filename]\n"
		" -v              Print version info and exit\n"
		" -s slice        Output to DOS slice (DOS numbering 1-4)\n"
		"                 NOTE: Must specify a raw disk device.\n"
		" -D DOS-ptype    Set the DOS partition type in slice mode.\n"
		" -z              Write zeros to free blocks.\n"
		" -p pattern      Write 32 bit pattern to free blocks.\n"
		"                 NOTE: Use -z/-p to avoid seeking.\n"
		" -o              Output progress indicator (compressed chunks processed)\n"
		" -O              Output progress indicator (GBs of uncompressed data written)\n"
		" -n              Single threaded (slow) mode\n"
		" -d              Turn on progressive levels of debugging\n"
		" -r retries      Number of image read retries to attempt\n"
		" -W size         MB of memory to use for write buffering\n"
582
		" -f              Force use of direct IO (O_DIRECT) to reduce system cache effects.\n"
583
584
585
586
587
		" -c              Check per-chunk checksum\n"
		" -a keyfile      File containing pubkey used to sign checksum (implies -c)\n"
		" -e cipher       Decrypt the image with the given cipher\n"
		" -k keyfile      File containing key used for encryption/decryption\n"
		" -u uuid         UUID for image\n");
588
	exit(1);
589
}
590

591
int
592
main(int argc, char *argv[])
593
{
594
595
596
597
	int		i, ch;
	off_t		foff;
	int		chunkno;
	extern char	build_info[];
598

599
#ifdef NOTHREADS
600
	nothreads = 1;
601
#endif
602
	while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:Cr:Na:ck:eu:fIM:")) != -1)
603
		switch(ch) {
Mike Hibler's avatar
all:    
Mike Hibler committed
604
#ifdef FAKEFRISBEE
605
606
607
		case 'F':
			dofrisbee++;
			break;
Mike Hibler's avatar
all:    
Mike Hibler committed
608
#endif
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
		case 'd':
			debug++;
			break;

		case 'n':
			nothreads++;
			break;

		case 'v':
			version++;
			break;

		case 'o':
			dots |= 1;
			break;

		case 'O':
			dots |= 2;
			break;

		case 's':
			slice = atoi(optarg);
			break;

		case 'D':
634
			dostype = (int)strtoul(optarg, NULL, 0);
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
			break;

		case 'p':
			fillpat = strtoul(optarg, NULL, 0);
		case 'z':
			dofill++;
			break;

		case 'C':
			docrconly++;
			dofill++;
			seekable = 0;
			break;

		case 'r':
			readretries = atoi(optarg);
			break;

		case 'N':
			nodecompress++;
			break;
Mike Hibler's avatar
Mike Hibler committed
656

Mike Hibler's avatar
Mike Hibler committed
657
#ifndef NOTHREADS
658
		case 'W':
659
			maxwritebufmem = (unsigned long long)atoi(optarg);
660
661
			if (MAXWRITEBUFMEM > 0 &&
			    maxwritebufmem >= MAXWRITEBUFMEM)
662
663
664
				maxwritebufmem = MAXWRITEBUFMEM;
			maxwritebufmem *= (1024 * 1024);
			break;
Mike Hibler's avatar
Mike Hibler committed
665
#endif
666
667
668
669
670
671
672
673
674

		case 'f':
#ifdef O_DIRECT
			directio++;
#else
			fprintf(stderr, "WARNING: O_DIRECT not supported\n");
#endif
			break;

675
#ifdef WITH_CRYPTO
676
		case 'c':
677
#ifndef SIGN_CHECKSUM
678
679
			do_checksum = 1;
			break;
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
#else
			fprintf(stderr, "Unsigned checksums not supported\n");
			exit(1);
#endif

		case 'a':
#ifdef SIGN_CHECKSUM
			if (!init_checksum(optarg))
				exit(1);
			do_checksum = 1;
			break;
#else
			fprintf(stderr, "Signed checksums not supported\n");
			exit(1);
#endif

		case 'e':
			/* Encryption cipher */
698
699
700
701
			if (strcmp(optarg, "bf_cbc") == 0) {
				cipher = ENC_BLOWFISH_CBC;
			}
			else {
702
703
				fprintf(stderr,
					"Only know \"bf_cbc\" (blowfish CBC)\n");
704
705
				usage();
			}
706
707
708
709
710
711
712
			do_decrypt = 1;
			break;

		case 'k':
			if (!encrypt_readkey(optarg, encryption_key,
					     ENC_MAX_KEYLEN))
				exit(1);
713
714
715
			/* XXX can you intuit the cipher from the key? */
			if (cipher == ENC_NONE)
				cipher = ENC_BLOWFISH_CBC;
716
			do_decrypt = 1;
717
			break;
718
#endif
Mike Hibler's avatar
Mike Hibler committed
719
720
		case 'u':
			/* UUID for image id. */
721
			if (!hexstr_to_mem(uuid, optarg, UUID_LENGTH))
Mike Hibler's avatar
Mike Hibler committed
722
723
724
				usage();
			has_id = 1;
			break;
725

726
727
728
729
		/* ignore skipped ranges on output (which must be stdout) */
		case 'I':
			ignoreskips++;
			break;
730
731
732
733
		/* set maxsector for error checking */
		case 'M':
			outputmaxsec = strtoul(optarg, NULL, 0);
			break;
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
		case 'h':
		case '?':
		default:
			usage();
		}
	argc -= optind;
	argv += optind;

	if (version || debug) {
		fprintf(stderr, "%s\n", build_info);
		if (version)
			exit(0);
	}

	if (argc < 1 || argc > 2)
		usage();

	if (fillpat) {
		unsigned	*bp = (unsigned *) &zeros;

		for (i = 0; i < sizeof(zeros)/sizeof(unsigned); i++)
			*bp++ = fillpat;
	}

	if (strcmp(argv[0], "-")) {
		struct stat st;

		if ((infd = open(argv[0], O_RDONLY, 0666)) < 0) {
			perror("opening input file");
			exit(1);
		}
		if (fstat(infd, &st) == 0)
			totalchunks = st.st_size / CHUNKSIZE;
	} else {
		infd = fileno(stdin);
		if (readretries > 0) {
			fprintf(stderr,
				"WARNING: cannot retry read operations\n");
			readretries = 0;
		}
	}
775

776
777
778
	if (docrconly)
		outfd = -1;
	else if (argc == 2 && strcmp(argv[1], "-")) {
779
780
		int flags;

781
782
783
784
785
786
787
788
789
790
791
792
793
		/*
		 * This option is really a hack so we can compare the
		 * content of two images regardless of how they got split
		 * into chunks; i.e., for cases where we cannot just compare
		 * the image files directly. So we force them to use stdout
		 * to make it painfully obvious that this is not a useful
		 * option!
		 */
		if (ignoreskips) {
			fprintf(stderr, "Must output to stdout with -I\n");
			exit(1);
		}

794
795
796
797
798
799
800
801
802
803
804
805
		/*
		 * XXX perform seek and MBR checks before we truncate
		 * the output file.  If they have their input/output
		 * arguments reversed, we don't want to truncate their
		 * image file!  (I speak from bitter experience).
		 */
		if (slice && (outfd = open(argv[1], O_RDONLY, 0666)) >= 0) {
			if (lseek(outfd, (off_t)0, SEEK_SET) < 0) {
				fprintf(stderr, "Output file is not seekable, "
					"cannot specify a slice\n");
				exit(1);
			}
Mike Hibler's avatar
Mike Hibler committed
806
807
			if (getslicebounds(slice)) {
				fprintf(stderr, "Failed to read slice bounds\n");
808
809
810
811
812
813
814
815
				exit(1);
			}
			close(outfd);
		}

		/*
		 * Open the output file for writing.
		 */
816
817
818
819
820
821
		flags = O_RDWR|O_CREAT|O_TRUNC;
#ifdef O_DIRECT
		if (directio)
		    flags |= O_DIRECT;
#endif

822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
	open_again:
		outfd = open(argv[1], flags, 0666);
		if (outfd < 0) {
			if (directio && errno == EINVAL) {
				/*
				 * XXX possibly directio is not supported,
				 * try again without.
				 */
				directio = 0;
				flags &= ~O_DIRECT;
				fprintf(stderr,
					"WARNING: open with O_DIRECT failed, "
					"trying again without ...\n");
				goto open_again;
			}
837
838
839
840
841
842
843
844
845
846
847
848
			perror("opening output file");
			exit(1);
		}
	}
	else
		outfd = fileno(stdout);

	/*
	 * If the output device isn't seekable we must modify our behavior:
	 * we cannot really handle slice mode, we must always zero fill
	 * (cannot skip free space) and we cannot use pwrite.
	 */
849
850
851
852
	if (ignoreskips) {
		dofill = 0;
		seekable = 0;
	} else if (lseek(outfd, (off_t)0, SEEK_SET) < 0) {
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
		if (slice) {
			fprintf(stderr, "Output file is not seekable, "
				"cannot specify a slice\n");
			exit(1);
		}
		if (!dofill && !docrconly)
			fprintf(stderr,
				"WARNING: output file is not seekable, "
				"must zero-fill free space\n");
		dofill = 1;
		seekable = 0;
	} else
		seekable = 1;

	if (slice) {
		off_t	minseek;

Mike Hibler's avatar
Mike Hibler committed
870
871
		if (getslicebounds(slice)) {
			fprintf(stderr, "Failed to read slice bounds\n");
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
			exit(1);
		}
		minseek = sectobytes(outputminsec);

		if (lseek(outfd, minseek, SEEK_SET) < 0) {
			perror("Setting seek pointer to slice");
			exit(1);
		}

		/*
		 * XXX zeroing and slice mode don't mix.
		 *
		 * To do so properly, we would have to zero all space before
		 * and after the slice in question.  But zeroing before would
		 * clobber the MBR and boot info, probably not what was
		 * intended.  So we just don't do it til we figure out what
		 * the "right" behavior is.
		 */
		if (dofill) {
			fprintf(stderr,
				"WARNING: requested zeroing in slice mode, "
				"will NOT zero outside of slice!\n");
		}
	}

	threadinit();
	gettimeofday(&stamp, 0);
899

Mike Hibler's avatar
all:    
Mike Hibler committed
900
#ifdef FAKEFRISBEE
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
	if (dofrisbee) {
		int i;

		if (totalchunks == 0) {
			fprintf(stderr, "Must have statable input file\n");
			exit(1);
		}

		chunklist = (int *) calloc(totalchunks+1, sizeof(*chunklist));
		assert(chunklist != NULL);

		for (i = 0; i < totalchunks; i++)
			chunklist[i] = i;
		chunklist[i] = -1;

		srandom((long)(stamp.tv_usec^stamp.tv_sec));
		for (i = 0; i < 50 * totalchunks; i++) {
			int c1 = random() % totalchunks;
			int c2 = random() % totalchunks;
			int t1 = chunklist[c1];
			int t2 = chunklist[c2];

			chunklist[c2] = t1;
			chunklist[c1] = t2;
		}
		nextchunk = chunklist;
	}
Mike Hibler's avatar
all:    
Mike Hibler committed
928
929
#endif

930
#ifdef SIGINFO
931
	signal(SIGINFO, dump_stats);
932
#endif
933
934
935
936
937
	foff = 0;
	chunkno = 0;
	while (1) {
		int	count = sizeof(chunkbuf);
		char	*bp   = chunkbuf;
938

Mike Hibler's avatar
all:    
Mike Hibler committed
939
#ifdef FAKEFRISBEE
940
941
942
943
944
945
946
947
948
949
		if (dofrisbee) {
			if (*nextchunk == -1)
				goto done;
			foff = (off_t)*nextchunk * CHUNKSIZE;
			if (lseek(infd, foff, SEEK_SET) < 0) {
				perror("seek failed");
				exit(1);
			}
			nextchunk++;
		}
Mike Hibler's avatar
all:    
Mike Hibler committed
950
#endif
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
		/*
		 * Decompress one subblock at a time. We read the entire
		 * chunk and hand it off. Since we might be reading from
		 * stdin, we have to make sure we get the entire amount.
		 */
		while (count) {
			int	cc;

			if ((cc = read_withretry(infd, bp, count, foff)) <= 0) {
				if (cc == 0)
					goto done;
				perror("reading zipped image");
				exit(1);
			}
			count -= cc;
			bp    += cc;
			foff  += cc;
		}
		if (nodecompress) {
970
			if (write_subblock(chunkno, chunkbuf, CHUNKSIZE))
971
972
973
974
975
976
977
				break;
		} else {
			if (inflate_subblock(chunkbuf))
				break;
		}
		chunkno++;
	}
978
 done:
979
980
981
982
983
984
985
986
987
988
	close(infd);

	/* When zeroing, may need to zero the rest of the disk */
	zero_remainder();

	/* This causes the output queue to drain */
	threadquit();

	/* Set the MBR type if necesary */
	if (slice && dostype >= 0)
Mike Hibler's avatar
Mike Hibler committed
989
		setslicetype(slice, dostype);
990

991
992
	/* Flush any cached data and close the output device */
	if (outfd >= 0) {
993
994
		/* can return EINVAL if device doesn't support fsync */
		if (fsync(outfd) < 0 && errno != EINVAL) {
995
996
997
998
999
1000
			perror("flushing output data");
			exit(1);
		}
		close(outfd);
	}

For faster browsing, not all history is shown. View entire blame