The New Netdisk! Faster than a speeding bullet, able to skip free FS

blocks in a single lseek, more powerful than gzip on steriods!

configure

@@ -924,7 +924,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
 	discvr/GNUmakefile \
 	lib/GNUmakefile \
-	os/GNUmakefile os/imagezip/GNUmakefile \
+	os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
 	pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
 	security/GNUmakefile security/paperbag \
 	tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
@@ -933,8 +933,10 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \
 	tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
 	tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
+	tbsetup/ir/handle_virt \
 	tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
-	tbsetup/ns2ir/parse.tcl tbsetup/savevlans \
+	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
+	tbsetup/savevlans tbsetup/ir/extract_tb \
 	tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
 	tbsetup/named_setup \
 	tbsetup/checkpass/GNUmakefile \

configure.in

@@ -90,7 +90,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
 	discvr/GNUmakefile \
 	lib/GNUmakefile \
-	os/GNUmakefile os/imagezip/GNUmakefile \
+	os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
 	pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
 	security/GNUmakefile security/paperbag \
 	tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
@@ -101,8 +101,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
 	tbsetup/ir/handle_virt \
 	tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
-	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl.in \
-	tbsetup/savevlans tbsetup/ir/extract_tb.in \
+	tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
+	tbsetup/savevlans tbsetup/ir/extract_tb \
 	tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
 	tbsetup/named_setup \
 	tbsetup/checkpass/GNUmakefile \

os/GNUmakefile.in

@@ -10,14 +10,14 @@ include $(OBJDIR)/Makeconf
 
 SUBDIRS	= imagezip
 
-all:	$(SUBDIRS)
+all:	$(SUBDIRS) split-image.sh
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
 imagezip:
 	@$(MAKE) -C imagezip all
 
-install: 
+install: $(INSTALL_SBINDIR)/split-image.sh
 	@$(MAKE) -C imagezip install
 
 control-install: 

os/imagezip/GNUmakefile.in

@@ -9,7 +9,7 @@ all:	imagezip imageunzip
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
-CFLAGS = -O2 -g
+CFLAGS = -O -g -static
 LIBS = -lz
 
 imagezip: imagezip.o

os/imagezip/imagehdr.h

+
+#include <sys/types.h>
+
+/*
+ * This little sucker goes on the beginning of the outputfile. It is blocked
+ * in the first 1K of the file. It tells the inflator how many 1K blocks
+ * come after it, which contain the start/size pairs of the good stuff that
+ * needs to be written to disk.
+ */
+struct imagehdr {
+	off_t	filesize;
+	int	magic;
+	int	regionsize;
+	int	regioncount;
+} *hdr;
+
+/*
+ * This number is defined to be the same as the NFS read size in netdisk.
+ * The above structure is padded to this when it is written to the
+ * output file.
+ */
+#define IMAGEHDRMINSIZE		1024
+
+/*
+ * Magic number when image is compresses
+ */
+#define COMPRESSED_MAGIC	0x69696969
+
+/*
+ * This little struct defines the pair. Each number is in sectors. An array
+ * of these come after the header above, and is padded to a 1K boundry.
+ */
+struct region {
+	unsigned long	start;
+	unsigned long	size;
+};
+
+/*
+ * This number is defined to be the same as the NFS read size in
+ * netdisk.  The array of regions is padded to this when it is written
+ * to the output file.
+ */
+#define REGIONMINSIZE		1024
+

os/imagezip/imageunzip.c

@@ -11,6 +11,7 @@
 #include <zlib.h>
 #include <sys/types.h>
 #include <sys/time.h>
+#include "imagehdr.h"
 
 #define CHECK_ERR(err, msg) { \
     if (err != Z_OK) { \
@@ -19,18 +20,21 @@
     } \
 }
 
+#define SECSIZE 512
 #define BSIZE	0x10000
-char		inbuf[BSIZE], outbuf[2*BSIZE];
+char		inbuf[BSIZE], outbuf[2*BSIZE], zeros[BSIZE];
 
 int
 main(int argc, char **argv)
 {
-	int		cc, err, infd;
+	int		cc, err, count, infd;
 	z_stream	d_stream; /* inflation stream */
-	off_t		offset;
-	long long	filesize, magic, total = 0, blockcount = 0;
-	char		*prog = argv[0];
+	long long	total = 0, blockcount = 0;
+	char		*bp, *prog = argv[0];
+	struct imagehdr hdr;
+	struct region	*regions, *curregion;
 	struct timeval  stamp, estamp;
+	off_t		offset, size;
 
 	d_stream.zalloc   = (alloc_func)0;
 	d_stream.zfree    = (free_func)0;
@@ -48,23 +52,47 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
-	if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
-		perror("opening input file");
-		exit(1);
+	if (strcmp(argv[1], "-")) {
+		if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
+			perror("opening input file");
+			exit(1);
+		}
 	}
+	else
+		infd = fileno(stdin);
 
 	/*
-	 * Read the 1024 byte header. It is uncompressed, and holds
-	 * the real image size and the magic number.
+	 * Read the header. It is uncompressed, and holds the real
+	 * image size and the magic number.
 	 */
-	if ((cc = read(infd, inbuf, 1024)) < 0) {
+	if ((cc = read(infd, inbuf, IMAGEHDRMINSIZE)) < 0) {
 		perror("reading zipped image header goo");
 		exit(1);
 	}
-	assert(cc == 1024);
-	filesize = *((long long *) inbuf);
-	magic    = *((long long *) &inbuf[sizeof(filesize)]);
-	fprintf(stderr, "Filesize = %qd, Magic = %qx\n", filesize, magic);
+	assert(cc == IMAGEHDRMINSIZE);
+	memcpy(&hdr, inbuf, sizeof(hdr));
+	fprintf(stderr, "Filesize: %qd, Magic: %x, "
+		        "Regionsize %d, RegionCount %d\n",
+		hdr.filesize, hdr.magic, hdr.regionsize, hdr.regioncount);
+
+	/*
+	 * Read in the valid data region information.
+	 */
+	regions   = (struct region *) calloc(hdr.regionsize, 1);
+	curregion = regions;
+	if ((cc = read(infd, regions, hdr.regionsize)) < 0) {
+		perror("reading region info");
+		exit(1);
+	}
+	assert(cc == hdr.regionsize);
+
+	/*
+	 * Start with the first region. 
+	 */
+	offset = curregion->start * (off_t) SECSIZE;
+	size   = curregion->size  * (off_t) SECSIZE;
+	curregion++;
+	hdr.regioncount--;
 
 	gettimeofday(&stamp, 0);
 	while (1) {
@@ -87,17 +115,79 @@ main(int argc, char **argv)
 				"%s: inflate failed, err=%ld\n", prog, err);
 			exit(1);
 		}
-		cc = sizeof(outbuf) - d_stream.avail_out;
+		count = sizeof(outbuf) - d_stream.avail_out;
+		bp = outbuf;
+
+		while (count) {
+			/*
+			 * Write data only as far as the end of the current
+			 * region.
+			 */
+			if (count < size)
+				cc = count;
+			else
+				cc = size;
+
+			if ((cc = write(1, bp, cc)) != cc) {
+				if (cc < 0) {
+					perror("Writing uncompressed data");
+				}
+				fprintf(stderr, "%s: inflate failed\n", prog);
+				exit(1);
+			}
 
-		if ((cc = write(1, outbuf, cc)) != cc) {
-			if (cc < 0) {
-				perror("Writing uncompressed data");
+			count  -= cc;
+			bp     += cc;
+			size   -= cc;
+			offset += cc;
+			total  += cc;
+			blockcount += cc;
+
+			/*
+			 * Hit the end of the region. Need to figure out
+			 * where the next one starts. We write a block of
+			 * zeros in the empty space between this region
+			 * and the next. We could lseek too, but only if
+			 * not writing to stdout. 
+			 */
+			if (! size) {
+				off_t	newoffset, zcount;
+				int	zcc;
+
+				/*
+				 * No more regions. Must be done.
+				 */
+				if (!hdr.regioncount)
+					break;
+				
+				newoffset = curregion->start * (off_t) SECSIZE;
+				zcount    = newoffset - offset;
+
+				while (zcount) {
+					if (zcount <= BSIZE)
+						zcc = (int) zcount;
+					else
+						zcc = BSIZE;
+
+					if ((zcc =
+					     write(1, zeros, zcc)) != zcc) {
+						if (zcc < 0) {
+							perror("Writing "
+							       "Zeros");
+						}
+						exit(1);
+					}
+					zcount -= zcc;
+					total  += zcc;
+					blockcount += zcc;
+				}
+
+				offset = newoffset;
+				size   = curregion->size * (off_t) SECSIZE;
+				curregion++;
+				hdr.regioncount--;
 			}
-			fprintf(stderr, "%s: inflate failed\n", prog);
-			exit(1);
 		}
-		total += cc;
-		blockcount += cc;
 		if (d_stream.avail_in)
 			goto inflate_again;
 
@@ -109,6 +199,8 @@ main(int argc, char **argv)
 			blockcount = 0;
 		}
 	}
+	assert(hdr.regioncount == 0);
+	
 	err = inflateEnd(&d_stream);
 	CHECK_ERR(err, "inflateEnd");
 

os/imagezip/linux_types.h

+typedef unsigned char		__u8;
+typedef signed   char		__s8;
+
+typedef signed   long long	__s64;
+typedef unsigned long long	__u64;
+
+typedef	signed   short		__s16;
+typedef	unsigned short		__u16;
+
+typedef	signed   long		__s32;
+typedef	unsigned long		__u32;

os/split-image.sh

-#!/bin/sh
-
-#
-# Split up an image and imagezip it. Beware, the sector offsets and sizes
-# are mhardcoded in here.
-#
-
-image=wd0
-zip=/q/testbed/image/imagezip
-
-#
-# Boot block: start 0, size 63
-# 
-dd if=$image of=${image}-mbr bs=1b count=63
-$zip ${image}-mbr ${image}-mbr.ndz
-
-#
-# So, who picked these sizes? Da ya think maybe we could pick sizes
-# that are nicely divisible by 32k or 64k, instead of an odd number?
-# 
-
-#
-# FreeBSD: start 63, size 6281352
-# 
-dd if=$image of=${image}-fbsd ibs=1b skip=63 obs=1b count=6281352
-$zip ${image}-fbsd ${image}-fbsd.ndz
-
-#
-# Linux: start 6281415, size 6281415
-# 
-dd if=$image of=${image}-rhat ibs=1b skip=6281415 obs=1b count=6281415
-$zip ${image}-rhat ${image}-rhat.ndz
-
-$zip $image $image-all.ndz

os/split-image.sh.in

+#!/bin/sh
+
+#
+# Split up an image and imagezip it. Beware, some sector offsets and sizes
+# are hardcoded in here.
+#
+
+#
+# Configure variables
+#
+TB="@prefix@"
+
+#
+# My zipper.
+# 
+zipper="$TB/bin/imagezip"
+
+#
+# The input image
+# 
+image=wd0
+
+#
+# Boot block: start 0, size 63
+# 
+$zipper -d -r -c 62 $image ${image}-mbr.ndz
+
+#
+# FreeBSD:
+#
+$zipper -d -s 1 $image ${image}-fbsd.ndz
+
+#
+# Linux:
+#
+$zipper -d -s 2 $image ${image}-rhat.ndz
+
+#
+# All of it.
+#
+$zipper -d $image ${image}-all.ndz