Commit 1ca45147 authored by Leigh B. Stoller's avatar Leigh B. Stoller
Browse files

The New Netdisk! Faster than a speeding bullet, able to skip free FS

blocks in a single lseek, more powerful than gzip on steriods!
parent b38191a4
...@@ -924,7 +924,7 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -924,7 +924,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \ db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
discvr/GNUmakefile \ discvr/GNUmakefile \
lib/GNUmakefile \ lib/GNUmakefile \
os/GNUmakefile os/imagezip/GNUmakefile \ os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \ pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
security/GNUmakefile security/paperbag \ security/GNUmakefile security/paperbag \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \ tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
...@@ -933,8 +933,10 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -933,8 +933,10 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \ tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \
tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \ tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \ tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \
tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \ tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
tbsetup/ns2ir/parse.tcl tbsetup/savevlans \ tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
tbsetup/savevlans tbsetup/ir/extract_tb \
tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \ tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
tbsetup/named_setup \ tbsetup/named_setup \
tbsetup/checkpass/GNUmakefile \ tbsetup/checkpass/GNUmakefile \
......
...@@ -90,7 +90,7 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -90,7 +90,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \ db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
discvr/GNUmakefile \ discvr/GNUmakefile \
lib/GNUmakefile \ lib/GNUmakefile \
os/GNUmakefile os/imagezip/GNUmakefile \ os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \ pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
security/GNUmakefile security/paperbag \ security/GNUmakefile security/paperbag \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \ tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
...@@ -101,8 +101,8 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -101,8 +101,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \ tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \ tbsetup/ir/handle_virt \
tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \ tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl.in \ tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
tbsetup/savevlans tbsetup/ir/extract_tb.in \ tbsetup/savevlans tbsetup/ir/extract_tb \
tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \ tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
tbsetup/named_setup \ tbsetup/named_setup \
tbsetup/checkpass/GNUmakefile \ tbsetup/checkpass/GNUmakefile \
......
...@@ -10,14 +10,14 @@ include $(OBJDIR)/Makeconf ...@@ -10,14 +10,14 @@ include $(OBJDIR)/Makeconf
SUBDIRS = imagezip SUBDIRS = imagezip
all: $(SUBDIRS) all: $(SUBDIRS) split-image.sh
include $(TESTBED_SRCDIR)/GNUmakerules include $(TESTBED_SRCDIR)/GNUmakerules
imagezip: imagezip:
@$(MAKE) -C imagezip all @$(MAKE) -C imagezip all
install: install: $(INSTALL_SBINDIR)/split-image.sh
@$(MAKE) -C imagezip install @$(MAKE) -C imagezip install
control-install: control-install:
......
...@@ -9,7 +9,7 @@ all: imagezip imageunzip ...@@ -9,7 +9,7 @@ all: imagezip imageunzip
include $(TESTBED_SRCDIR)/GNUmakerules include $(TESTBED_SRCDIR)/GNUmakerules
CFLAGS = -O2 -g CFLAGS = -O -g -static
LIBS = -lz LIBS = -lz
imagezip: imagezip.o imagezip: imagezip.o
......
This diff is collapsed.
#include <sys/types.h>
/*
* This little sucker goes on the beginning of the outputfile. It is blocked
* in the first 1K of the file. It tells the inflator how many 1K blocks
* come after it, which contain the start/size pairs of the good stuff that
* needs to be written to disk.
*/
struct imagehdr {
off_t filesize;
int magic;
int regionsize;
int regioncount;
} *hdr;
/*
* This number is defined to be the same as the NFS read size in netdisk.
* The above structure is padded to this when it is written to the
* output file.
*/
#define IMAGEHDRMINSIZE 1024
/*
* Magic number when image is compresses
*/
#define COMPRESSED_MAGIC 0x69696969
/*
* This little struct defines the pair. Each number is in sectors. An array
* of these come after the header above, and is padded to a 1K boundry.
*/
struct region {
unsigned long start;
unsigned long size;
};
/*
* This number is defined to be the same as the NFS read size in
* netdisk. The array of regions is padded to this when it is written
* to the output file.
*/
#define REGIONMINSIZE 1024
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include <zlib.h> #include <zlib.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/time.h> #include <sys/time.h>
#include "imagehdr.h"
#define CHECK_ERR(err, msg) { \ #define CHECK_ERR(err, msg) { \
if (err != Z_OK) { \ if (err != Z_OK) { \
...@@ -19,18 +20,21 @@ ...@@ -19,18 +20,21 @@
} \ } \
} }
#define SECSIZE 512
#define BSIZE 0x10000 #define BSIZE 0x10000
char inbuf[BSIZE], outbuf[2*BSIZE]; char inbuf[BSIZE], outbuf[2*BSIZE], zeros[BSIZE];
int int
main(int argc, char **argv) main(int argc, char **argv)
{ {
int cc, err, infd; int cc, err, count, infd;
z_stream d_stream; /* inflation stream */ z_stream d_stream; /* inflation stream */
off_t offset; long long total = 0, blockcount = 0;
long long filesize, magic, total = 0, blockcount = 0; char *bp, *prog = argv[0];
char *prog = argv[0]; struct imagehdr hdr;
struct region *regions, *curregion;
struct timeval stamp, estamp; struct timeval stamp, estamp;
off_t offset, size;
d_stream.zalloc = (alloc_func)0; d_stream.zalloc = (alloc_func)0;
d_stream.zfree = (free_func)0; d_stream.zfree = (free_func)0;
...@@ -48,23 +52,47 @@ main(int argc, char **argv) ...@@ -48,23 +52,47 @@ main(int argc, char **argv)
exit(1); exit(1);
} }
if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) { if (strcmp(argv[1], "-")) {
perror("opening input file"); if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
exit(1); perror("opening input file");
exit(1);
}
} }
else
infd = fileno(stdin);
/* /*
* Read the 1024 byte header. It is uncompressed, and holds * Read the header. It is uncompressed, and holds the real
* the real image size and the magic number. * image size and the magic number.
*/ */
if ((cc = read(infd, inbuf, 1024)) < 0) { if ((cc = read(infd, inbuf, IMAGEHDRMINSIZE)) < 0) {
perror("reading zipped image header goo"); perror("reading zipped image header goo");
exit(1); exit(1);
} }
assert(cc == 1024); assert(cc == IMAGEHDRMINSIZE);
filesize = *((long long *) inbuf); memcpy(&hdr, inbuf, sizeof(hdr));
magic = *((long long *) &inbuf[sizeof(filesize)]); fprintf(stderr, "Filesize: %qd, Magic: %x, "
fprintf(stderr, "Filesize = %qd, Magic = %qx\n", filesize, magic); "Regionsize %d, RegionCount %d\n",
hdr.filesize, hdr.magic, hdr.regionsize, hdr.regioncount);
/*
* Read in the valid data region information.
*/
regions = (struct region *) calloc(hdr.regionsize, 1);
curregion = regions;
if ((cc = read(infd, regions, hdr.regionsize)) < 0) {
perror("reading region info");
exit(1);
}
assert(cc == hdr.regionsize);
/*
* Start with the first region.
*/
offset = curregion->start * (off_t) SECSIZE;
size = curregion->size * (off_t) SECSIZE;
curregion++;
hdr.regioncount--;
gettimeofday(&stamp, 0); gettimeofday(&stamp, 0);
while (1) { while (1) {
...@@ -87,17 +115,79 @@ main(int argc, char **argv) ...@@ -87,17 +115,79 @@ main(int argc, char **argv)
"%s: inflate failed, err=%ld\n", prog, err); "%s: inflate failed, err=%ld\n", prog, err);
exit(1); exit(1);
} }
cc = sizeof(outbuf) - d_stream.avail_out; count = sizeof(outbuf) - d_stream.avail_out;
bp = outbuf;
while (count) {
/*
* Write data only as far as the end of the current
* region.
*/
if (count < size)
cc = count;
else
cc = size;
if ((cc = write(1, bp, cc)) != cc) {
if (cc < 0) {
perror("Writing uncompressed data");
}
fprintf(stderr, "%s: inflate failed\n", prog);
exit(1);
}
if ((cc = write(1, outbuf, cc)) != cc) { count -= cc;
if (cc < 0) { bp += cc;
perror("Writing uncompressed data"); size -= cc;
offset += cc;
total += cc;
blockcount += cc;
/*
* Hit the end of the region. Need to figure out
* where the next one starts. We write a block of
* zeros in the empty space between this region
* and the next. We could lseek too, but only if
* not writing to stdout.
*/
if (! size) {
off_t newoffset, zcount;
int zcc;
/*
* No more regions. Must be done.
*/
if (!hdr.regioncount)
break;
newoffset = curregion->start * (off_t) SECSIZE;
zcount = newoffset - offset;
while (zcount) {
if (zcount <= BSIZE)
zcc = (int) zcount;
else
zcc = BSIZE;
if ((zcc =
write(1, zeros, zcc)) != zcc) {
if (zcc < 0) {
perror("Writing "
"Zeros");
}
exit(1);
}
zcount -= zcc;
total += zcc;
blockcount += zcc;
}
offset = newoffset;
size = curregion->size * (off_t) SECSIZE;
curregion++;
hdr.regioncount--;
} }
fprintf(stderr, "%s: inflate failed\n", prog);
exit(1);
} }
total += cc;
blockcount += cc;
if (d_stream.avail_in) if (d_stream.avail_in)
goto inflate_again; goto inflate_again;
...@@ -109,6 +199,8 @@ main(int argc, char **argv) ...@@ -109,6 +199,8 @@ main(int argc, char **argv)
blockcount = 0; blockcount = 0;
} }
} }
assert(hdr.regioncount == 0);
err = inflateEnd(&d_stream); err = inflateEnd(&d_stream);
CHECK_ERR(err, "inflateEnd"); CHECK_ERR(err, "inflateEnd");
......
This diff is collapsed.
typedef unsigned char __u8;
typedef signed char __s8;
typedef signed long long __s64;
typedef unsigned long long __u64;
typedef signed short __s16;
typedef unsigned short __u16;
typedef signed long __s32;
typedef unsigned long __u32;
#!/bin/sh
#
# Split up an image and imagezip it. Beware, the sector offsets and sizes
# are mhardcoded in here.
#
image=wd0
zip=/q/testbed/image/imagezip
#
# Boot block: start 0, size 63
#
dd if=$image of=${image}-mbr bs=1b count=63
$zip ${image}-mbr ${image}-mbr.ndz
#
# So, who picked these sizes? Da ya think maybe we could pick sizes
# that are nicely divisible by 32k or 64k, instead of an odd number?
#
#
# FreeBSD: start 63, size 6281352
#
dd if=$image of=${image}-fbsd ibs=1b skip=63 obs=1b count=6281352
$zip ${image}-fbsd ${image}-fbsd.ndz
#
# Linux: start 6281415, size 6281415
#
dd if=$image of=${image}-rhat ibs=1b skip=6281415 obs=1b count=6281415
$zip ${image}-rhat ${image}-rhat.ndz
$zip $image $image-all.ndz
#!/bin/sh
#
# Split up an image and imagezip it. Beware, some sector offsets and sizes
# are hardcoded in here.
#
#
# Configure variables
#
TB="@prefix@"
#
# My zipper.
#
zipper="$TB/bin/imagezip"
#
# The input image
#
image=wd0
#
# Boot block: start 0, size 63
#
$zipper -d -r -c 62 $image ${image}-mbr.ndz
#
# FreeBSD:
#
$zipper -d -s 1 $image ${image}-fbsd.ndz
#
# Linux:
#
$zipper -d -s 2 $image ${image}-rhat.ndz
#
# All of it.
#
$zipper -d $image ${image}-all.ndz
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment