Commit 5f31d498 authored by Mike Hibler's avatar Mike Hibler

Grinding away on imagedelta/undelta...

parent 6738c988
......@@ -4557,6 +4557,7 @@ outfiles="Makeconf GNUmakefile setversion \
os/growdisk/GNUmakefile \
os/zapdisk/GNUmakefile \
os/imagezip/GNUmakefile \
os/imagezip/libndz/GNUmakefile \
os/imagezip/mbr/GNUmakefile \
os/imagezip/gpt/GNUmakefile \
os/imagezip/ffs/GNUmakefile \
......
......@@ -294,6 +294,7 @@ outfiles="Makeconf GNUmakefile setversion \
os/growdisk/GNUmakefile \
os/zapdisk/GNUmakefile \
os/imagezip/GNUmakefile \
os/imagezip/libndz/GNUmakefile \
os/imagezip/mbr/GNUmakefile \
os/imagezip/gpt/GNUmakefile \
os/imagezip/ffs/GNUmakefile \
......
......@@ -37,9 +37,9 @@
#ifndef lint
#if 0
static char sccsid[] = "@(#)crc.c 8.1 (Berkeley) 6/17/93";
#endif
static const char rcsid[] =
"$FreeBSD: src/usr.bin/cksum/crc.c,v 1.4 1999/12/05 20:03:21 charnier Exp $";
#endif
#endif /* not lint */
#include <sys/types.h>
......
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -24,13 +24,18 @@
#define FIXMAP_DEBUG
/*
* imagedelta [ -S ] image1.ndz image2.ndz delta1to2.ndz
* imagedelta [ -S -f ] image1.ndz image2.ndz delta1to2.ndz
*
* Take two images (image1, image2) and produce a delta (delta1to2)
* based on the differences. The -S option says to use the signature
* files if possible to determine differences between the images.
* Otherwise we compare the corresponding areas of both images to
* determine if they are different.
* files: image1.ndz.sig and image2.ndz.sig, if possible to determine
* differences between the images. Signature files will be rejected
* unless they can be positively matched with the image (right now,
* via the modtime!) Using -f will force it to use a questionable
* signature file.
*
* Without signature files, we compare the corresponding areas of both
* images to determine if they are different.
*
* Note that order matters here! We are generating a delta to get from
* "image1" to "image2"; i.e., doing:
......@@ -80,7 +85,7 @@
struct ifileinfo {
struct ndz_file *ndz;
int sigfd;
char *sigfile;
struct ndz_rangemap *map, *sigmap;
} ndz1, ndz2;
......@@ -90,6 +95,7 @@ struct mergestate {
};
int usesigfiles = 0;
int forcesig = 0;
static int fixmap(struct ndz_rangemap *map, struct ndz_range *range, void *arg);
......@@ -97,12 +103,13 @@ void
usage(void)
{
fprintf(stderr,
"Usage: imagedelta [-S] image1.ndz image2.ndz delta1to2.ndz\n"
"Usage: imagedelta [-Sf] image1.ndz image2.ndz delta1to2.ndz\n"
"\n"
"Produce a delta image (delta1to2) containing the changes\n"
"necessary to get from image1 to image2.\n"
"\n"
" -S Use signature files when computing differences.\n");
" -S Use signature files when computing differences.\n"
" -f With -S, force imagedelta to use a questionable sigfile.\n");
exit(1);
}
......@@ -114,7 +121,7 @@ usage(void)
void
openifile(char *file, struct ifileinfo *info)
{
int rv;
int sigfd;
info->ndz = ndz_open(file, 0);
if (info->ndz == NULL) {
......@@ -123,14 +130,39 @@ openifile(char *file, struct ifileinfo *info)
exit(1);
}
/* XXX check sigfile */
if (usesigfiles) {
;
struct stat sb1, sb2;
info->sigfile = malloc(strlen(file) + 5);
assert(info->sigfile != NULL);
strcpy(info->sigfile, file);
strcat(info->sigfile, ".sig");
sigfd = open(info->sigfile, 0);
if (sigfd < 0) {
fprintf(stderr, "%s: could not find signature file %s\n",
file, info->sigfile);
exit(1);
}
if (fstat(info->ndz->fd, &sb1) < 0 || fstat(sigfd, &sb2) < 0) {
fprintf(stderr, "%s: could stat image or signature file\n", file);
exit(1);
}
if (!forcesig && sb1.st_mtime != sb2.st_mtime) {
fprintf(stderr, "%s: image and signature disagree, "
"use -f to override.\n", file);
exit(1);
}
close(sigfd);
}
}
void
readifile(struct ifileinfo *info)
{
/* read range info from image */
rv = ndz_readranges(info->ndz, &info->map);
if (rv) {
info->map = ndz_readranges(info->ndz);
if (info->map == NULL) {
fprintf(stderr, "%s: could not read ranges\n",
ndz_filename(info->ndz));
exit(1);
......@@ -138,8 +170,14 @@ openifile(char *file, struct ifileinfo *info)
/* read signature info */
if (usesigfiles) {
;
}
info->sigmap = ndz_readhashinfo(info->ndz, info->sigfile);
if (info->sigmap == NULL) {
fprintf(stderr, "%s: could not read signature info\n",
ndz_filename(info->ndz));
exit(1);
}
} else
info->sigmap = NULL;
}
static void
......@@ -161,7 +199,7 @@ main(int argc, char **argv)
{
int ch, version = 0;
extern char build_info[];
int delta, rv;
int delta;
struct ndz_rangemap *mmap;
while ((ch = getopt(argc, argv, "Sv")) != -1)
......@@ -169,6 +207,9 @@ main(int argc, char **argv)
case 'S':
usesigfiles = 1;
break;
case 'f':
forcesig = 1;
break;
case 'v':
version++;
break;
......@@ -240,6 +281,12 @@ main(int argc, char **argv)
fflush(stdout);
#endif
/*
* Iterate through the produced map either copying the data
* directly into the new image (data == NULL) or hashing the
* two versions to determine if it should be copied.
*/
return 0;
}
......
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -28,23 +28,35 @@ MAINDIR = $(SRCDIR)/..
include $(OBJDIR)/Makeconf
CFLAGS += -DSTATS $(SUBDIRCFLAGS) -I$(MAINDIR) -I$(SRCDIR)
CFLAGS += -DSTATS $(SUBDIRCFLAGS) -I$(MAINDIR) -I$(SRCDIR) #-DDEBUG
all: libndz.a
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = rangemap.o ndzfile.o ndzdata.o
OBJS = rangemap.o ndzfile.o ndzdata.o chunk.o hash.o
rangemap.o: rangemap.h
ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
chunk.o: libndz.h $(MAINDIR)/imagehdr.h
hash.o: libndz.h $(MAINDIR)/imagehdr.h $(MAINDIR)/imagehash.h
libndz.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $?
$(RANLIB) $@
tests: ndzfiletest ndzdatatest rangemaptest
ndzfiletest: libndz.a
$(CC) -DNDZFILE_TEST $(CFLAGS) -o ndzfiletest $(SRCDIR)/ndzfile.c libndz.a
ndzdatatest: libndz.a
$(CC) -DNDZDATA_TEST $(CFLAGS) -o ndzdatatest $(SRCDIR)/ndzdata.c libndz.a -lz
rangemaptest:
$(CC) -DRANGEMAP_TEST $(CFLAGS) -o rangemaptest $(SRCDIR)/rangemap.c
install:
clean:
rm -f libndz.a $(OBJS)
rm -f libndz.a $(OBJS) ndzfiletest ndzdatatest rangemaptest
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
/*
* Chunk-oriented IO routines.
*
* Since chunks are independently compressed, we can manipulate them
* independently.
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <zlib.h>
#include <sys/stat.h>
#include "libndz.h"
#define CDATASIZE (128*1024)
struct ndz_chunk {
struct ndz_file *ndz;
ndz_chunkno_t chunkno;
off_t foff;
z_stream z;
char *cdatabuf;
};
ndz_chunk_t
ndz_chunk_open(struct ndz_file *ndz, ndz_chunkno_t chunkno)
{
struct ndz_chunk *chunk = malloc(sizeof *chunk);
if (chunk == NULL)
return NULL;
chunk->cdatabuf = malloc(CDATASIZE);
if (chunk->cdatabuf == NULL) {
free(chunk);
return NULL;
}
chunk->ndz = ndz;
chunk->chunkno = chunkno;
chunk->z.zalloc = Z_NULL;
chunk->z.zfree = Z_NULL;
chunk->z.opaque = Z_NULL;
chunk->z.next_in = Z_NULL;
chunk->z.avail_in = 0;
chunk->z.next_out = Z_NULL;
if (inflateInit(&chunk->z) != Z_OK) {
free(chunk);
return NULL;
}
chunk->foff = (off_t)chunkno * ndz->chunksize + DEFAULTREGIONSIZE;
return (ndz_chunk_t)chunk;
}
void
ndz_chunk_close(ndz_chunk_t chobj)
{
struct ndz_chunk *chunk = (struct ndz_chunk *)chobj;
if (chunk == NULL)
return;
/* release any cache resources */
inflateEnd(&chunk->z);
free(chunk);
}
ndz_chunkno_t
ndz_chunk_chunkno(ndz_chunk_t chobj)
{
struct ndz_chunk *chunk = (struct ndz_chunk *)chobj;
if (chunk == NULL)
return ~0;
return chunk->chunkno;
}
/*
* Sequentially read data from a chunk til there is no more to be read
*/
ssize_t
ndz_chunk_read(ndz_chunk_t chobj, void *buf, size_t bytes)
{
int rv;
ssize_t cc;
struct ndz_chunk *chunk = (struct ndz_chunk *)chobj;
if (chunk == NULL)
return -1;
chunk->z.next_out = (Bytef *)buf;
chunk->z.avail_out = bytes;
while (chunk->z.avail_out > 0) {
/* read more compressed data from file if necessary */
if (chunk->z.avail_in == 0) {
cc = ndz_read(chunk->ndz, chunk->cdatabuf, CDATASIZE, chunk->foff);
if (cc <= 0)
return cc;
chunk->z.next_in = (Bytef *)chunk->cdatabuf;
chunk->z.avail_in = cc;
chunk->foff += cc;
}
assert(chunk->z.next_in != Z_NULL);
assert(chunk->z.avail_in > 0);
rv = inflate(&chunk->z, Z_SYNC_FLUSH);
if (rv == Z_STREAM_END) {
#ifdef DEBUG
fprintf(stderr, "chunk_read hit STREAM_END at foff=%ld, avail_out=%d\n",
(unsigned long)chunk->foff, chunk->z.avail_out);
#endif
break;
}
if (rv != Z_OK) {
fprintf(stderr, "%s: inflate failed, rv=%d\n",
chunk->ndz->fname, rv);
return -1;
}
}
return (bytes - chunk->z.avail_out);
}
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
/*
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
/*
* Hashing-related functions.
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <sys/stat.h>
#include "libndz.h"
#include "imagehash.h"
struct hashdata {
ndz_chunkno_t chunkno;
unsigned char hash[HASH_MAXSIZE];
};
/*
* Read the hash info from a signature file into a region map associated
* with the ndz file.
*/
struct ndz_rangemap *
ndz_readhashinfo(struct ndz_file *ndz, char *sigfile)
{
struct hashinfo hi;
struct hashregion hr;
int fd, cc, rv, i;
struct ndz_rangemap *map;
struct hashdata *hashdata = NULL;
if (ndz == NULL)
return NULL;
if (ndz->hashmap)
return ndz->hashmap;
fd = open(sigfile, O_RDONLY);
if (fd < 0) {
perror(sigfile);
return NULL;
}
cc = read(fd, &hi, sizeof(hi));
if (cc != sizeof(hi)) {
if (cc < 0)
perror(sigfile);
else
fprintf(stderr, "%s: too short\n", sigfile);
close(fd);
return NULL;
}
if (strcmp((char *)hi.magic, HASH_MAGIC) != 0 ||
!(hi.version == HASH_VERSION_1 || hi.version == HASH_VERSION_2)) {
fprintf(stderr, "%s: not a valid signature file\n", sigfile);
close(fd);
return NULL;
}
map = ndz_rangemap_init(NDZ_LOADDR, NDZ_HIADDR-NDZ_LOADDR);
if (map == NULL) {
fprintf(stderr, "%s: could not allocate rangemap\n",
ndz->fname);
close(fd);
return NULL;
}
/* allocate the hash data elements all in one piece for convienience */
if (hi.nregions) {
hashdata = malloc(hi.nregions * sizeof(struct hashdata));
if (hashdata == NULL) {
fprintf(stderr, "%s: could not allocate hashmap data\n",
ndz->fname);
close(fd);
return NULL;
}
}
for (i = 0; i < hi.nregions; i++) {
cc = read(fd, &hr, sizeof(hr));
if (cc != sizeof(hr)) {
fprintf(stderr, "%s: incomplete sig entry\n", sigfile);
free(hashdata);
close(fd);
return NULL;
}
hashdata[i].chunkno = hr.chunkno;
memcpy(hashdata[i].hash, hr.hash, HASH_MAXSIZE);
rv = ndz_rangemap_alloc(map, (ndz_addr_t)hr.region.start,
(ndz_size_t)hr.region.size,
(void *)&hashdata[i]);
if (rv) {
fprintf(stderr, "%s: bad hash region [%u-%u]\n",
ndz->fname,
(unsigned)hr.region.start,
(unsigned)hr.region.start+hr.region.size-1);
ndz_rangemap_deinit(map);
free(hashdata);
close(fd);
return NULL;
}
}
close(fd);
ndz->hashmap = map;
ndz->hashdata = hashdata;
ndz->hashblksize = (hi.version == HASH_VERSION_1) ?
(HASHBLK_SIZE / ndz->sectsize) : hi.blksize;
#if 0
/* Compensate for partition offset */
for (i = 0; i < hinfo->nregions; i++) {
struct hashregion *hreg = &hinfo->regions[i];
assert(hreg->region.size <= hashblksize);
hreg->region.start += poffset;
}
#endif
return map;
}
void
ndz_freehashmap(struct ndz_file *ndz)
{
if (ndz->hashmap) {
ndz_rangemap_deinit(ndz->hashmap);
ndz->hashmap = NULL;
}
if (ndz->hashdata) {
free(ndz->hashdata);
ndz->hashdata = NULL;
}
ndz->hashblksize = 0;
}
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
/*
* Copyright (c) 2014 University of Utah and the Flux Group.
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -24,26 +24,45 @@
#ifndef _LIBNDZ_H_
#define _LIBNDZ_H_
#include "imagehdr.h"
#include "rangemap.h"
typedef uint32_t ndz_chunk_t;
typedef uint32_t ndz_chunkno_t;
/* XXX keep this opaque so we don't create dependencies on zlib */
typedef void * ndz_chunk_t;
#ifdef maybenotneeded
struct ndz_chunkmap {
ndz_addr_t start;
ndz_addr_t end;
};
#endif
struct ndz_file {
int fd;
int seekable;
off_t curoff;
char *fname;
int sectsize;
int chunksize;
ndz_chunk_t nchunks;
ndz_chunkno_t nchunks;
ndz_chunk_t chunkobj;
ndz_addr_t chunksect;
#ifdef STATS
unsigned chunkuses;
unsigned chunkhits;
#endif
#ifdef maybenotneeded
struct ndz_chunkmap *chunkmap;
#endif
struct ndz_rangemap *rangemap;
unsigned hashblksize;
void *hashdata;
struct ndz_rangemap *hashmap;
/* per-chunk info to verify */
/* readahead cache stuff */
};
struct chunkmap {
ndz_addr_t start;
ndz_addr_t end;
} *chunkmap;
struct ndz_chunkhdr {
blockhdr_t *header;
struct region *region;
......@@ -57,12 +76,19 @@ char *ndz_filename(struct ndz_file *ndz);
ssize_t ndz_read(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset);
int ndz_readahead(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset);
int ndz_readchunkheader(struct ndz_file *ndz, ndz_chunk_t chunkno,
int ndz_readchunkheader(struct ndz_file *ndz, ndz_chunkno_t chunkno,
struct ndz_chunkhdr *chunkhdr);
ssize_t ndz_readdata(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset);
struct ndz_rangemap *ndz_readranges(struct ndz_file *ndz);
void ndz_dumpranges(struct ndz_rangemap *map);
ndz_chunk_t ndz_chunk_open(struct ndz_file *ndz, ndz_chunkno_t chunkno);
void ndz_chunk_close(ndz_chunk_t chobj);
ssize_t ndz_chunk_read(ndz_chunk_t chobj, void *buf, size_t bytes);
ndz_chunkno_t ndz_chunk_chunkno(ndz_chunk_t chobj);
struct ndz_rangemap *ndz_readhashinfo(struct ndz_file *ndz, char *sigfile);
#endif /* _LIBNDZ_H_ */
/*
......
This diff is collapsed.
/*
* Copyright (c) 2014 University of Utah and the Flux Group.
* Copyright (c) 2014-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -28,7 +28,6 @@
#include <errno.h>
#include <assert.h>
#include "../imagehdr.h"
#include "libndz.h"
struct ndz_file *
......@@ -41,10 +40,26 @@ ndz_open(const char *name, int flags)
blockhdr_t *hdr;
unsigned int magic;
fd = open(name, 0);
if (fd < 0)
/* XXX only do read right now */
if (flags != 0) {
fprintf(stderr, "%s: ndz_open can only read right now.\n", name);
goto fail;
}
if (strcmp(name, "-") == 0) {
fd = fileno(stdin);
name = "<STDIN>";
ndz->seekable = 0;
} else {
fd = open(name, 0);
if (fd < 0) {
perror(name);
goto fail;
}
ndz->seekable = 1;
}
ndz->fd = fd;
ndz->curoff = 0;
/*
* It should have at least one chunk header. Read that and verify
......@@ -115,8 +130,17 @@ ndz_read(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset)
size_t count = bytes;
char *bp = buf;
if (lseek(ndz->fd, offset, SEEK_SET) < 0)
return -1;
if (ndz->seekable) {
if (lseek(ndz->fd, offset, SEEK_SET) < 0)
return -1;
} else {
if (offset != ndz->curoff) {
fprintf(stderr, "%s: non-contiguous read on unseekable input.\n",
ndz->fname);
errno = ESPIPE;
return -1;
}
}
/*
* We might be reading from stdin or a pipe, so we may not get the
......@@ -136,6 +160,7 @@ ndz_read(struct ndz_file *ndz, void *buf, size_t bytes, off_t offset)
bp += cc;
}
ndz->curoff += (bytes - count);
return bytes - count;
}
......@@ -157,7 +182,7 @@ ndz_readranges(struct ndz_file *ndz)
blockhdr_t *hdr;
struct region *reg;
int rv, i;
ndz_chunk_t chunkno;
ndz_chunkno_t chunkno;
if (ndz == NULL)
return NULL;
......@@ -177,7 +202,7 @@ ndz_readranges(struct ndz_file *ndz)
for (chunkno = 0; ; chunkno++) {
rv = ndz_readchunkheader(ndz, chunkno, &head);
if (rv)
return rv;
return NULL;
/* null header pointer indicates EOF */
if ((hdr = head.header) == NULL)
......@@ -207,7 +232,7 @@ ndz_readranges(struct ndz_file *ndz)
}
int
ndz_readchunkheader(struct ndz_file *ndz, ndz_chunk_t chunkno,
ndz_readchunkheader(struct ndz_file *ndz, ndz_chunkno_t chunkno,
struct ndz_chunkhdr *chunkhdr)
{
ssize_t cc;
......@@ -216,7 +241,7 @@ ndz_readchunkheader(struct ndz_file *ndz, ndz_chunk_t chunkno,
struct blockreloc *rel;
cc = ndz_read(ndz, chunkhdr->data, sizeof chunkhdr->data,