Commit 1ca45147 authored by Leigh Stoller's avatar Leigh Stoller

The New Netdisk! Faster than a speeding bullet, able to skip free FS

blocks in a single lseek, more powerful than gzip on steriods!
parent b38191a4
......@@ -924,7 +924,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
discvr/GNUmakefile \
lib/GNUmakefile \
os/GNUmakefile os/imagezip/GNUmakefile \
os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
security/GNUmakefile security/paperbag \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
......@@ -933,8 +933,10 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \
tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \
tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
tbsetup/ns2ir/parse.tcl tbsetup/savevlans \
tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
tbsetup/savevlans tbsetup/ir/extract_tb \
tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
tbsetup/named_setup \
tbsetup/checkpass/GNUmakefile \
......
......@@ -90,7 +90,7 @@ outfiles="$outfiles Makeconf GNUmakefile \
db/GNUmakefile db/nalloc db/nfree db/if2port db/backup \
discvr/GNUmakefile \
lib/GNUmakefile \
os/GNUmakefile os/imagezip/GNUmakefile \
os/GNUmakefile os/split-image.sh os/imagezip/GNUmakefile \
pxe/GNUmakefile pxe/proxydhcp.restart pxe/bootinfo.restart \
security/GNUmakefile security/paperbag \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
......@@ -101,8 +101,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \
tbsetup/ns2ir/postparse tbsetup/ir/handle_os tbsetup/ir/handle_ip \
tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl.in \
tbsetup/savevlans tbsetup/ir/extract_tb.in \
tbsetup/ns2ir/parse.tcl tbsetup/ns2ir/tb_compat.tcl \
tbsetup/savevlans tbsetup/ir/extract_tb \
tbsetup/tbprerun tbsetup/tbrun tbsetup/tbend tbsetup/tbreport \
tbsetup/named_setup \
tbsetup/checkpass/GNUmakefile \
......
......@@ -10,14 +10,14 @@ include $(OBJDIR)/Makeconf
SUBDIRS = imagezip
all: $(SUBDIRS)
all: $(SUBDIRS) split-image.sh
include $(TESTBED_SRCDIR)/GNUmakerules
imagezip:
@$(MAKE) -C imagezip all
install:
install: $(INSTALL_SBINDIR)/split-image.sh
@$(MAKE) -C imagezip install
control-install:
......
......@@ -9,7 +9,7 @@ all: imagezip imageunzip
include $(TESTBED_SRCDIR)/GNUmakerules
CFLAGS = -O2 -g
CFLAGS = -O -g -static
LIBS = -lz
imagezip: imagezip.o
......
This diff is collapsed.
#include <sys/types.h>
/*
* This little sucker goes on the beginning of the outputfile. It is blocked
* in the first 1K of the file. It tells the inflator how many 1K blocks
* come after it, which contain the start/size pairs of the good stuff that
* needs to be written to disk.
*/
struct imagehdr {
off_t filesize;
int magic;
int regionsize;
int regioncount;
} *hdr;
/*
* This number is defined to be the same as the NFS read size in netdisk.
* The above structure is padded to this when it is written to the
* output file.
*/
#define IMAGEHDRMINSIZE 1024
/*
* Magic number when image is compresses
*/
#define COMPRESSED_MAGIC 0x69696969
/*
* This little struct defines the pair. Each number is in sectors. An array
* of these come after the header above, and is padded to a 1K boundry.
*/
struct region {
unsigned long start;
unsigned long size;
};
/*
* This number is defined to be the same as the NFS read size in
* netdisk. The array of regions is padded to this when it is written
* to the output file.
*/
#define REGIONMINSIZE 1024
......@@ -11,6 +11,7 @@
#include <zlib.h>
#include <sys/types.h>
#include <sys/time.h>
#include "imagehdr.h"
#define CHECK_ERR(err, msg) { \
if (err != Z_OK) { \
......@@ -19,18 +20,21 @@
} \
}
#define SECSIZE 512
#define BSIZE 0x10000
char inbuf[BSIZE], outbuf[2*BSIZE];
char inbuf[BSIZE], outbuf[2*BSIZE], zeros[BSIZE];
int
main(int argc, char **argv)
{
int cc, err, infd;
int cc, err, count, infd;
z_stream d_stream; /* inflation stream */
off_t offset;
long long filesize, magic, total = 0, blockcount = 0;
char *prog = argv[0];
long long total = 0, blockcount = 0;
char *bp, *prog = argv[0];
struct imagehdr hdr;
struct region *regions, *curregion;
struct timeval stamp, estamp;
off_t offset, size;
d_stream.zalloc = (alloc_func)0;
d_stream.zfree = (free_func)0;
......@@ -48,23 +52,47 @@ main(int argc, char **argv)
exit(1);
}
if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
perror("opening input file");
exit(1);
if (strcmp(argv[1], "-")) {
if ((infd = open(argv[1], O_RDONLY, 0666)) < 0) {
perror("opening input file");
exit(1);
}
}
else
infd = fileno(stdin);
/*
* Read the 1024 byte header. It is uncompressed, and holds
* the real image size and the magic number.
* Read the header. It is uncompressed, and holds the real
* image size and the magic number.
*/
if ((cc = read(infd, inbuf, 1024)) < 0) {
if ((cc = read(infd, inbuf, IMAGEHDRMINSIZE)) < 0) {
perror("reading zipped image header goo");
exit(1);
}
assert(cc == 1024);
filesize = *((long long *) inbuf);
magic = *((long long *) &inbuf[sizeof(filesize)]);
fprintf(stderr, "Filesize = %qd, Magic = %qx\n", filesize, magic);
assert(cc == IMAGEHDRMINSIZE);
memcpy(&hdr, inbuf, sizeof(hdr));
fprintf(stderr, "Filesize: %qd, Magic: %x, "
"Regionsize %d, RegionCount %d\n",
hdr.filesize, hdr.magic, hdr.regionsize, hdr.regioncount);
/*
* Read in the valid data region information.
*/
regions = (struct region *) calloc(hdr.regionsize, 1);
curregion = regions;
if ((cc = read(infd, regions, hdr.regionsize)) < 0) {
perror("reading region info");
exit(1);
}
assert(cc == hdr.regionsize);
/*
* Start with the first region.
*/
offset = curregion->start * (off_t) SECSIZE;
size = curregion->size * (off_t) SECSIZE;
curregion++;
hdr.regioncount--;
gettimeofday(&stamp, 0);
while (1) {
......@@ -87,17 +115,79 @@ main(int argc, char **argv)
"%s: inflate failed, err=%ld\n", prog, err);
exit(1);
}
cc = sizeof(outbuf) - d_stream.avail_out;
count = sizeof(outbuf) - d_stream.avail_out;
bp = outbuf;
while (count) {
/*
* Write data only as far as the end of the current
* region.
*/
if (count < size)
cc = count;
else
cc = size;
if ((cc = write(1, bp, cc)) != cc) {
if (cc < 0) {
perror("Writing uncompressed data");
}
fprintf(stderr, "%s: inflate failed\n", prog);
exit(1);
}
if ((cc = write(1, outbuf, cc)) != cc) {
if (cc < 0) {
perror("Writing uncompressed data");
count -= cc;
bp += cc;
size -= cc;
offset += cc;
total += cc;
blockcount += cc;
/*
* Hit the end of the region. Need to figure out
* where the next one starts. We write a block of
* zeros in the empty space between this region
* and the next. We could lseek too, but only if
* not writing to stdout.
*/
if (! size) {
off_t newoffset, zcount;
int zcc;
/*
* No more regions. Must be done.
*/
if (!hdr.regioncount)
break;
newoffset = curregion->start * (off_t) SECSIZE;
zcount = newoffset - offset;
while (zcount) {
if (zcount <= BSIZE)
zcc = (int) zcount;
else
zcc = BSIZE;
if ((zcc =
write(1, zeros, zcc)) != zcc) {
if (zcc < 0) {
perror("Writing "
"Zeros");
}
exit(1);
}
zcount -= zcc;
total += zcc;
blockcount += zcc;
}
offset = newoffset;
size = curregion->size * (off_t) SECSIZE;
curregion++;
hdr.regioncount--;
}
fprintf(stderr, "%s: inflate failed\n", prog);
exit(1);
}
total += cc;
blockcount += cc;
if (d_stream.avail_in)
goto inflate_again;
......@@ -109,6 +199,8 @@ main(int argc, char **argv)
blockcount = 0;
}
}
assert(hdr.regioncount == 0);
err = inflateEnd(&d_stream);
CHECK_ERR(err, "inflateEnd");
......
This diff is collapsed.
typedef unsigned char __u8;
typedef signed char __s8;
typedef signed long long __s64;
typedef unsigned long long __u64;
typedef signed short __s16;
typedef unsigned short __u16;
typedef signed long __s32;
typedef unsigned long __u32;
#!/bin/sh
#
# Split up an image and imagezip it. Beware, the sector offsets and sizes
# are mhardcoded in here.
#
image=wd0
zip=/q/testbed/image/imagezip
#
# Boot block: start 0, size 63
#
dd if=$image of=${image}-mbr bs=1b count=63
$zip ${image}-mbr ${image}-mbr.ndz
#
# So, who picked these sizes? Da ya think maybe we could pick sizes
# that are nicely divisible by 32k or 64k, instead of an odd number?
#
#
# FreeBSD: start 63, size 6281352
#
dd if=$image of=${image}-fbsd ibs=1b skip=63 obs=1b count=6281352
$zip ${image}-fbsd ${image}-fbsd.ndz
#
# Linux: start 6281415, size 6281415
#
dd if=$image of=${image}-rhat ibs=1b skip=6281415 obs=1b count=6281415
$zip ${image}-rhat ${image}-rhat.ndz
$zip $image $image-all.ndz
#!/bin/sh
#
# Split up an image and imagezip it. Beware, some sector offsets and sizes
# are hardcoded in here.
#
#
# Configure variables
#
TB="@prefix@"
#
# My zipper.
#
zipper="$TB/bin/imagezip"
#
# The input image
#
image=wd0
#
# Boot block: start 0, size 63
#
$zipper -d -r -c 62 $image ${image}-mbr.ndz
#
# FreeBSD:
#
$zipper -d -s 1 $image ${image}-fbsd.ndz
#
# Linux:
#
$zipper -d -s 2 $image ${image}-rhat.ndz
#
# All of it.
#
$zipper -d $image ${image}-all.ndz
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment