Commit aa5491b1 authored by Mike Hibler's avatar Mike Hibler

Add support for creating a "delta" image from a hash signature file

(as generated by imagehash).  This is Mike's piss-all-over-it version of
Prashanth's implementation of Mike's algorithm.  Delta images are created
by specifying "-H <sigfile>" on the command line.

This is still experimental.
parent 652f3592
......@@ -10,8 +10,9 @@ OBJDIR = ../..
SUBDIR = os/imagezip
DISTFILES = global.h imagehdr.h queue.h sliceinfo.h \
imagedump.c imagedump.8 imagezip.c imagezip.8 \
imageunzip.c crc.c imageunzip.8 \
extfs/*.[hc] fat/*.[hc] ffs/*.[hc] ntfs/*.[hc] shd/*.[hc]
imageunzip.c crc.c disksize.c imageunzip.8 \
extfs/*.[hc] fat/*.[hc] ffs/*.[hc] ntfs/*.[hc] shd/*.[hc] \
hashmap/*.[hc]
EXPANDCOPYRIGHT = /usr/site/lib/copyright/expand-copyr
......@@ -55,6 +56,7 @@ WITH_FFS = 1
WITH_EXTFS = 1
WITH_NTFS = @WINSUPPORT@
WITH_FAT = @WINSUPPORT@
WITH_HASH = 1
include $(OBJDIR)/Makeconf
......@@ -79,6 +81,8 @@ endif
CFLAGS = $(SUBDIRCFLAGS) -I$(SRCDIR) -static
LIBS = -lz
ZIPCFLAGS = $(CFLAGS) -Wall
ZIPLIBS =
UNZIPCFLAGS = $(CFLAGS) $(PTHREADCFLAGS) -Wall
UNZIPLIBS = $(LIBS) $(PTHREADLIBS)
HASHCFLAGS = $(CFLAGS) $(PTHREADCFLAGS) -Wall
......@@ -88,14 +92,14 @@ HASHLIBS = $(LIBS) -lcrypto $(PTHREADLIBS)
ifeq ($(WITH_FFS),1)
CFLAGS += -DWITH_FFS
SUBDIRS += ffs
FSLIBS += ffs/libffs.a
ZIPLIBS += ffs/libffs.a
endif
# EXT2/EXT3
ifeq ($(WITH_EXTFS),1)
CFLAGS += -DWITH_EXTFS
SUBDIRS += extfs
FSLIBS += extfs/libextfs.a
ZIPLIBS += extfs/libextfs.a
endif
# with NTFS
......@@ -105,9 +109,9 @@ CC = gcc30
endif
CFLAGS += -DWITH_NTFS
SUBDIRS += ntfs
FSLIBS += ntfs/libntfs.a
ZIPLIBS += ntfs/libntfs.a
ifndef HAVE_LOCALE
FSLIBS += ntfs/liblocale.a
ZIPLIBS += ntfs/liblocale.a
endif
endif
......@@ -115,17 +119,26 @@ endif
ifeq ($(WITH_FAT),1)
CFLAGS += -DWITH_FAT
SUBDIRS += fat
FSLIBS += fat/libfat.a
ZIPLIBS += fat/libfat.a
endif
# with SHD
ifeq ($(WITH_SHD),1)
CFLAGS += -DWITH_SHD
SUBDIRS += shd
FSLIBS += shd/libshd.a
ZIPLIBS += shd/libshd.a
SHDLIBS += shd/libshd.a
endif
# with HASH
ifeq ($(WITH_HASH),1)
CFLAGS += -DWITH_HASH -DHASHSTATS
SUBDIRCFLAGS += -DHASHSTATS
SUBDIRS += hashmap
ZIPLIBS += hashmap/libhashmap.a
LIBS += -lcrypto $(PTHREADLIBS)
endif
all: $(SUBDIRS) imagezip imageunzip imagedump imagehash
whoami:
......@@ -137,8 +150,11 @@ endif
include $(TESTBED_SRCDIR)/GNUmakerules
imagezip: imagezip.o version.o $(FSLIBS)
$(CC) $(CFLAGS) imagezip.o version.o $(LIBS) $(FSLIBS) -o imagezip
imagezip: imagezip.o disksize.o version.o $(ZIPLIBS)
$(CC) $(CFLAGS) imagezip.o disksize.o version.o $(ZIPLIBS) $(LIBS) -o imagezip
imagezip.o: imagezip.c
$(CC) -c $(ZIPCFLAGS) -o imagezip.o $<
imageunzip: imageunzip.o crc.o version.o
$(CC) $(CFLAGS) imageunzip.o crc.o version.o $(UNZIPLIBS) -o imageunzip
......@@ -155,12 +171,12 @@ imagehash: imagehash.o version.o
imagehash.o: imagehash.c
$(CC) -c $(HASHCFLAGS) -o imagehash.o $<
ffs extfs ntfs fat shd:
ffs extfs ntfs fat shd hashmap:
@$(MAKE) SUBDIRCFLAGS="$(SUBDIRCFLAGS)" -C $@ all
imagezip.o: sliceinfo.h imagehdr.h global.h
imageunzip.o: imagehdr.h
imagehash.o: imagehdr.h
imagehash.o: imagehdr.h imagehash.h
version.c: imagezip.c imageunzip.c imagedump.c
echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
......
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2005 University of Utah and the Flux Group.
* All rights reserved.
*/
/*
* Determine the size of the target disk in sectors.
*/
#include <unistd.h>
#include <stdio.h>
#ifdef __FreeBSD__
#if __FreeBSD__ >= 5
#include <sys/disk.h>
#else
#include <sys/disklabel.h>
#endif
#else
#ifdef __linux__
#include <sys/ioctl.h>
#include <linux/fs.h>
#endif
#endif
unsigned long
getdisksize(int fd)
{
unsigned long disksize = 0;
unsigned int ssize = 512;
#ifdef linux
if (disksize == 0) {
int rv;
rv = ioctl(fd, BLKGETSIZE, &disksize);
if (rv < 0)
disksize = 0;
}
#else
#ifdef DIOCGMEDIASIZE
if (disksize == 0) {
int rv;
off_t dsize;
if (ioctl(fd, DIOCGSECTORSIZE, &ssize) < 0)
ssize = 512;
rv = ioctl(fd, DIOCGMEDIASIZE, &dsize);
if (rv >= 0)
disksize = (unsigned long)(dsize / ssize);
}
#else
#ifdef DIOCGDINFO
if (disksize == 0) {
int rv;
struct disklabel label;
rv = ioctl(fd, DIOCGDINFO, &label);
if (rv >= 0)
disksize = label.d_secperunit;
}
#endif
#endif
#endif
/*
* OS wouldn't tell us anything directly, try a seek to the
* end of the device.
*/
if (disksize == 0) {
off_t lastoff;
lastoff = lseek(fd, (off_t)0, SEEK_END);
if (lastoff > 0)
disksize = (unsigned long)(lastoff / ssize);
}
/*
* Make sure we can seek to that sector
*/
if (lseek(fd, (off_t)(disksize-1) * ssize, SEEK_SET) < 0)
fprintf(stderr, "WARNING: "
"could not seek to final sector (%lu) of disk\n",
disksize - 1);
return disksize;
}
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# All rights reserved.
#
SRCDIR = @srcdir@
TESTBED_SRCDIR = @top_srcdir@
OBJDIR = ../../..
SUBDIR = os/imagezip/hashmap
MAINDIR = $(SRCDIR)/..
include $(OBJDIR)/Makeconf
CFLAGS += $(SUBDIRCFLAGS) -I$(MAINDIR) -I$(SRCDIR)
all: libhashmap.a
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = hashmap.o
hashmap.o: hashmap.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
libhashmap.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $?
$(RANLIB) $@
install:
clean:
rm -f hashmap.o
This diff is collapsed.
/*
* EMULAB-COPYRIGHT
* Copyright (c) 2000-2004 University of Utah and the Flux Group.
* All rights reserved.
*/
/* XXX from global.h */
extern int secsize;
#define sectobytes(s) ((off_t)(s) * secsize)
#define bytestosec(b) (uint32_t)((b) / secsize)
/* XXX from imagezip.c */
struct range {
uint32_t start; /* In sectors */
uint32_t size; /* In sectors */
void *data;
struct range *next;
};
void free_ranges(struct range **);
#define MAX(x, y) (((x) > (y)) ? (x) : (y))
#define MIN(x, y) (((x) > (y)) ? (y) : (x))
......@@ -53,7 +53,14 @@ int retrywrites= 1;
int dorelocs = 1;
off_t datawritten;
partmap_t ignore, forceraw;
#ifdef WITH_SHD
char *chkpointdev;
#endif
#ifdef WITH_HASH
char *hashfile;
#endif
#define HDRUSED(reg, rel) \
(sizeof(blockhdr_t) + \
......@@ -66,6 +73,7 @@ char *chkpointdev;
*
* These numbers are in sectors.
*/
extern unsigned long getdisksize(int fd);
unsigned long inputminsec = 0;
unsigned long inputmaxsec = 0; /* 0 means the entire input image */
......@@ -89,6 +97,7 @@ void sortrange(struct range *head, int domerge,
int mergeskips(int verbose);
int mergeranges(struct range *head);
void makeranges(void);
void freeranges(struct range *);
void dumpranges(int verbose);
void addvalid(uint32_t start, uint32_t size);
void addreloc(off_t offset, off_t size, int reloctype);
......@@ -107,6 +116,11 @@ int read_shd(char *shddev, char *infile, int infd, u_int32_t ssect,
void (*add)(uint32_t, uint32_t));
#endif
#ifdef WITH_HASH
struct range *hashmap_compute_delta(struct range *, char *, int, u_int32_t);
void report_hash_stats(void);
#endif
static SLICEMAP_PROCESS_PROTO(read_slice);
struct slicemap fsmap[] = {
......@@ -351,15 +365,18 @@ main(int argc, char *argv[])
char *outfilename = 0;
int rawmode = 0;
int slicetype = 0;
struct timeval sstamp;
extern char build_info[];
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:oI:1F:DR:S:XC:")) != -1)
gettimeofday(&sstamp, 0);
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:oI:1F:DR:S:XC:H:")) != -1)
switch(ch) {
case 'v':
version++;
break;
case 'i':
info++;
debug++;
break;
case 'D':
retrywrites = 0;
......@@ -421,7 +438,20 @@ main(int argc, char *argv[])
forcereads++;
break;
case 'C':
#ifdef WITH_SHD
chkpointdev = optarg;
#else
fprintf(stderr, "'C' option not supported\n");
usage();
#endif
break;
case 'H':
#ifdef WITH_HASH
hashfile = optarg;
#else
fprintf(stderr, "'H' option not supported\n");
usage();
#endif
break;
case 'h':
case '?':
......@@ -431,7 +461,7 @@ main(int argc, char *argv[])
argc -= optind;
argv += optind;
if (version || info || debug) {
if (version || debug) {
fprintf(stderr, "%s\n", build_info);
if (version) {
fprintf(stderr, "Supports");
......@@ -442,6 +472,9 @@ main(int argc, char *argv[])
fsmap[ch].desc);
#ifdef WITH_SHD
fprintf(stderr, ", SHD device");
#endif
#ifdef WITH_HASH
fprintf(stderr, ", hash-signature comparison");
#endif
fprintf(stderr, "\n");
exit(0);
......@@ -468,9 +501,6 @@ main(int argc, char *argv[])
else
outfilename = argv[1];
if (info && !debug)
debug++;
if (!slicemode && dorelocs)
dorelocs = 0;
......@@ -480,8 +510,19 @@ main(int argc, char *argv[])
exit(1);
}
if (chkpointdev) {
#if 0
/*
* Use OS-specific techniques to discover the size of the disk.
* Note that this could produce an image that will not fit on a
* smaller disk, as imagezip will consider any space beyond the
* final partition as allocated and will record the ranges.
*/
if (!slicemode && !maxmode)
inputmaxsec = getdisksize(infd);
#endif
#ifdef WITH_SHD
if (chkpointdev) {
rval = 0;
if (dorelocs) {
fprintf(stderr, "WARNING: no relocation info "
......@@ -515,11 +556,9 @@ main(int argc, char *argv[])
fprintf(stderr, "* * * Aborting * * *\n");
exit(1);
}
#else
fprintf(stderr, "Checkpoint device not supported\n\n");
usage();
} else
#endif
} else {
{
/*
* Create the skip list by scanning the filesystems on
* the disk or indicated partition.
......@@ -541,40 +580,95 @@ main(int argc, char *argv[])
/*
* Create a valid range list from the skip list
*/
(void) mergeskips(info || debug > 2);
(void) mergeskips(debug > 1);
if (debug)
dumpskips(info || debug > 2);
dumpskips(debug > 1);
makeranges();
}
if (debug)
dumpranges(info || debug > 2);
dumpranges(debug > 1);
sortrange(fixups, 0, cmpfixups);
fflush(stderr);
if (info) {
close(infd);
exit(0);
#ifdef WITH_HASH
/*
* If we are creating a "delta" image from a hash signature,
* we read in the signature info and reconcile that with the
* known allocated range that we have just computed. The result
* is a new list of ranges that are currently allocated and that
* have changed from the signature version.
*
* XXX we need to consider relocations here. If an existing
* range has an associated fixup, we should always include it in
* the image.
*/
if (hashfile != NULL) {
struct range *nranges;
/*
* next compare allocated 'ranges' and 'hinfo' to find out the
* changed blocks -- computing the hashes for some 'ranges'
* in the process
*/
nranges = hashmap_compute_delta(ranges, hashfile, infd,
inputminsec);
if (nranges == NULL)
fprintf(stderr, "NO differences !!!!\n");
freeranges(ranges);
ranges = nranges;
if (debug) {
fprintf(stderr, "\nAfter delta computation: ");
dumpranges(debug > 1);
}
report_hash_stats();
}
#endif
if (strcmp(outfilename, "-")) {
if ((outfd = open(outfilename, O_RDWR|O_CREAT|O_TRUNC, 0666))
< 0) {
perror("opening output file");
exit(1);
/*
* Now we have all the allocated information, create the image
* (unless we just want an info report).
*/
if (!info) {
if (strcmp(outfilename, "-")) {
if ((outfd = open(outfilename, O_RDWR|O_CREAT|O_TRUNC,
0666)) < 0) {
perror("opening output file");
exit(1);
}
outcanseek = 1;
}
else {
outfd = fileno(stdout);
outcanseek = 0;
retrywrites = 0;
}
outcanseek = 1;
compress_image();
if (outcanseek)
close(outfd);
}
else {
outfd = fileno(stdout);
outcanseek = 0;
retrywrites = 0;
close(infd);
{
struct timeval stamp;
unsigned int ms;
gettimeofday(&stamp, 0);
if (stamp.tv_usec < sstamp.tv_usec) {
stamp.tv_usec += 1000000;
stamp.tv_sec--;
}
ms = (stamp.tv_sec - sstamp.tv_sec) * 1000 +
(stamp.tv_usec - sstamp.tv_usec) / 1000;
fprintf(stderr,
"\nFinished in %u.%03u seconds\n",
ms / 1000, ms % 1000);
}
compress_image();
fflush(stderr);
close(infd);
if (outcanseek)
close(outfd);
exit(0);
}
......@@ -799,6 +893,7 @@ char *usagestr =
" -c count Compress <count> number of sectors (not with slice mode)\n"
" -D Do `dangerous' writes (don't check for async errors)\n"
" -1 Output a version one image file\n"
" -H hashfile Use the specified imagehash-generated signature to produce a delta image\n"
"\n"
" Debugging options (not to be used by mere mortals!)\n"
" -d Turn on debugging. Multiple -d options increase output\n"
......@@ -924,7 +1019,7 @@ mergeskips(int verbose)
while (*prevp) {
prange = *prevp;
if (prange->size < (uint32_t)frangesize) {
if (debug > 1)
if (debug > 2)
fprintf(stderr,
"dropping range [%u-%u]\n",
prange->start,
......@@ -1029,7 +1124,7 @@ mergeranges(struct range *head)
break;
if (prange->start + prange->size == prange->next->start) {
if (debug > 1)
if (debug > 2)
fprintf(stderr,
"merging ranges [%u-%u] and [%u-%u]\n",
prange->start,
......@@ -1091,6 +1186,18 @@ makeranges(void)
}
}
void
freeranges(struct range *head)
{
struct range *next;
while (head != NULL) {
next = head->next;
free(head);
head = next;
}
}
void
dumpranges(int verbose)
{
......@@ -1196,7 +1303,7 @@ applyfixups(off_t offset, off_t size, void *data)
coff = (u_int32_t)(fp->offset - offset);
clen = (u_int32_t)fp->size;
if (debug > 1)
if (debug > 2)
fprintf(stderr,
"Applying fixup [%llu-%llu] "
"to [%llu-%llu]\n",
......@@ -1325,7 +1432,7 @@ compress_image(void)
/*
* Compress the chunk.
*/
if (debug > 0 && debug < 3) {
if (debug > 1 && debug < 3) {
fprintf(stderr,
"Compressing range: %14lld --> ", inputoffset);
fflush(stderr);
......@@ -1334,7 +1441,7 @@ compress_image(void)
size = compress_chunk(inputoffset, rangesize,
&full, &blkhdr->size);
if (debug >= 3) {
if (debug > 2) {
fprintf(stderr, "%14lld -> %12lld %10ld %10u %10d %d\n",
inputoffset, inputoffset + size,
prange->start - inputminsec,
......@@ -1576,7 +1683,7 @@ compress_image(void)
}
inputoffset += size;
if (debug || dots)
if (debug > 1 || dots)
fprintf(stderr, "\n");
compress_status(0);
fflush(stderr);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment