Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
emulab-devel
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
143
Issues
143
List
Boards
Labels
Service Desk
Milestones
Merge Requests
6
Merge Requests
6
Operations
Operations
Incidents
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
emulab
emulab-devel
Commits
6738c988
Commit
6738c988
authored
Dec 16, 2014
by
Mike Hibler
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Begining support for off-line image delta creation.
New code for managing ranges.
parent
9781359e
Changes
15
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
4076 additions
and
3 deletions
+4076
-3
clientside/os/imagezip/GNUmakefile.in
clientside/os/imagezip/GNUmakefile.in
+12
-2
clientside/os/imagezip/imagedelta.c
clientside/os/imagezip/imagedelta.c
+342
-0
clientside/os/imagezip/imagehdr.h
clientside/os/imagezip/imagehdr.h
+2
-1
clientside/os/imagezip/imagezip.c
clientside/os/imagezip/imagezip.c
+10
-0
clientside/os/imagezip/libndz/GNUmakefile.in
clientside/os/imagezip/libndz/GNUmakefile.in
+50
-0
clientside/os/imagezip/libndz/imageunzip.c
clientside/os/imagezip/libndz/imageunzip.c
+2342
-0
clientside/os/imagezip/libndz/libndz.h
clientside/os/imagezip/libndz/libndz.h
+74
-0
clientside/os/imagezip/libndz/ndzdata.c
clientside/os/imagezip/libndz/ndzdata.c
+192
-0
clientside/os/imagezip/libndz/ndzfile.c
clientside/os/imagezip/libndz/ndzfile.c
+308
-0
clientside/os/imagezip/libndz/nfile
clientside/os/imagezip/libndz/nfile
+0
-0
clientside/os/imagezip/libndz/rangemap.c
clientside/os/imagezip/libndz/rangemap.c
+648
-0
clientside/os/imagezip/libndz/rangemap.h
clientside/os/imagezip/libndz/rangemap.h
+94
-0
clientside/os/imagezip/libndz/rtest
clientside/os/imagezip/libndz/rtest
+0
-0
configure
configure
+1
-0
configure.ac
configure.ac
+1
-0
No files found.
clientside/os/imagezip/GNUmakefile.in
View file @
6738c988
...
...
@@ -127,7 +127,7 @@ endif
# Necessary sometimes.
#PTHREADCFLAGS += -DCONDVARS_WORK
PROGS = imagezip imageunzip imagedump
PROGS = imagezip imageunzip imagedump
imagedelta
CFLAGS = $(SUBDIRCFLAGS) -I$(SRCDIR) $(LDSTATIC)
LIBS = -lz
ZIPCFLAGS = $(CFLAGS) -Wall
...
...
@@ -137,6 +137,7 @@ UNZIPLIBS = $(LIBS) $(PTHREADLIBS)
UNZIPDIRS =
HASHCFLAGS = $(CFLAGS) $(PTHREADCFLAGS) -Wall
HASHLIBS = $(LIBS) -lcrypto $(PTHREADLIBS)
NDZLIBS = $(LIBS) libndz/libndz.a
# Secure images
ifeq ($(WITH_CRYPTO),1)
...
...
@@ -153,6 +154,8 @@ ifeq ($(WITH_V3COMPAT),1)
CFLAGS += -DWITH_V3COMPAT
endif
SUBDIRS += libndz
# MBR
ifeq ($(WITH_MBR),1)
CFLAGS += -DWITH_MBR
...
...
@@ -257,6 +260,12 @@ imagehash: imagehash.o version.o
imagehash.o: imagehash.c
$(CC) -c $(HASHCFLAGS) -o imagehash.o $<
imagedelta: imagedelta.o version.o $(NDZLIBS)
$(CC) $(CFLAGS) imagedelta.o version.o $(NDZLIBS) -o imagedelta
imagedelta.o: imagedelta.c
$(CC) -c $(CFLAGS) -o imagedelta.o $<
sizetest: disksize.c
$(CC) -DTEST $< -o sizetest
...
...
@@ -275,8 +284,9 @@ endif
imagezip.o: sliceinfo.h imagehdr.h global.h range.h hashmap/hashmap.h
imageunzip.o: imagehdr.h
imagehash.o: imagehdr.h imagehash.h
imagedelta.o: imagehdr.h libndz/libndz.h
version.c: imagezip.c imageunzip.c imagedump.c
version.c: imagezip.c imageunzip.c imagedump.c
imagedelta.c
echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
install: $(addprefix $(INSTALL_BINDIR)/, $(PROGS))
...
...
clientside/os/imagezip/imagedelta.c
0 → 100644
View file @
6738c988
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
#define FIXMAP_DEBUG
/*
* imagedelta [ -S ] image1.ndz image2.ndz delta1to2.ndz
*
* Take two images (image1, image2) and produce a delta (delta1to2)
* based on the differences. The -S option says to use the signature
* files if possible to determine differences between the images.
* Otherwise we compare the corresponding areas of both images to
* determine if they are different.
*
* Note that order matters here! We are generating a delta to get from
* "image1" to "image2"; i.e., doing:
*
* imageunzip image1.ndz /dev/da0
* imageunzip delta1to2.ndz /dev/da0
*
* would be identical to:
*
* imageunzip image2.ndz /dev/da0
*
* Approach:
*
* We scan the chunks headers of both images (image1, image2) to produce
* allocated range lists for both (R1, R2). We use these to produce a
* range list for the delta (RD) as follows.
*
* - Anything that is in R1 but not R2 does not go in RD.
* - Anything in R2 but not in R1 must go into RD.
* - For overlapping areas, we read and hash or compare both and,
* if different, include in RD.
* - Using RD, select data from image2 that need to be read, decompressed
* and then recompressed into the new image.
*
* There is the usual issue of dealing with the difference in granularity
* and alignment of ranges (arbitrary multiples of 512 byte) vs. hash
* blocks (64K byte), but that logic exists in imagezip today.
*/
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <assert.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <errno.h>
#include <openssl/sha.h>
#include <openssl/md5.h>
#ifndef NOTHREADS
#include <pthread.h>
#endif
#include "imagehdr.h"
#include "libndz/libndz.h"
struct
ifileinfo
{
struct
ndz_file
*
ndz
;
int
sigfd
;
struct
ndz_rangemap
*
map
,
*
sigmap
;
}
ndz1
,
ndz2
;
struct
mergestate
{
unsigned
int
ochunkno
;
unsigned
int
nchunkno
;
};
int
usesigfiles
=
0
;
static
int
fixmap
(
struct
ndz_rangemap
*
map
,
struct
ndz_range
*
range
,
void
*
arg
);
void
usage
(
void
)
{
fprintf
(
stderr
,
"Usage: imagedelta [-S] image1.ndz image2.ndz delta1to2.ndz
\n
"
"
\n
"
"Produce a delta image (delta1to2) containing the changes
\n
"
"necessary to get from image1 to image2.
\n
"
"
\n
"
" -S Use signature files when computing differences.
\n
"
);
exit
(
1
);
}
/*
* File must exist and be readable.
* If usesigfiles is set, signature file must exist as well.
* Reads in the range map and signature as well.
*/
void
openifile
(
char
*
file
,
struct
ifileinfo
*
info
)
{
int
rv
;
info
->
ndz
=
ndz_open
(
file
,
0
);
if
(
info
->
ndz
==
NULL
)
{
fprintf
(
stderr
,
"%s: could not open as NDZ file
\n
"
,
ndz_filename
(
info
->
ndz
));
exit
(
1
);
}
/* XXX check sigfile */
if
(
usesigfiles
)
{
;
}
/* read range info from image */
rv
=
ndz_readranges
(
info
->
ndz
,
&
info
->
map
);
if
(
rv
)
{
fprintf
(
stderr
,
"%s: could not read ranges
\n
"
,
ndz_filename
(
info
->
ndz
));
exit
(
1
);
}
/* read signature info */
if
(
usesigfiles
)
{
;
}
}
static
void
chunkfunc
(
void
*
ptr
)
{
unsigned
int
chunkno
=
(
int
)
ptr
;
printf
(
"chunkno=%u"
,
chunkno
);
}
static
void
twochunkfunc
(
void
*
ptr
)
{
struct
mergestate
*
state
=
(
struct
mergestate
*
)
ptr
;
printf
(
"ochunkno=%u, nchunkno=%u"
,
state
->
ochunkno
,
state
->
nchunkno
);
}
int
main
(
int
argc
,
char
**
argv
)
{
int
ch
,
version
=
0
;
extern
char
build_info
[];
int
delta
,
rv
;
struct
ndz_rangemap
*
mmap
;
while
((
ch
=
getopt
(
argc
,
argv
,
"Sv"
))
!=
-
1
)
switch
(
ch
)
{
case
'S'
:
usesigfiles
=
1
;
break
;
case
'v'
:
version
++
;
break
;
case
'h'
:
case
'?'
:
default:
usage
();
}
argc
-=
optind
;
argv
+=
optind
;
if
(
argc
<
3
)
usage
();
/*
* Make sure we can open all the files
*/
ndz1
.
ndz
=
ndz_open
(
argv
[
0
],
0
);
if
(
ndz1
.
ndz
==
NULL
)
usage
();
ndz2
.
ndz
=
ndz_open
(
argv
[
1
],
0
);
if
(
ndz2
.
ndz
==
NULL
)
usage
();
delta
=
open
(
argv
[
2
],
O_RDWR
|
O_CREAT
|
O_TRUNC
,
0666
);
if
(
delta
<
0
)
{
perror
(
argv
[
2
]);
usage
();
}
/*
* Read in the range and signature info.
*/
ndz1
.
map
=
ndz_readranges
(
ndz1
.
ndz
);
if
(
ndz1
.
map
==
NULL
)
{
fprintf
(
stderr
,
"%s: could not read ranges
\n
"
,
ndz_filename
(
ndz1
.
ndz
));
exit
(
1
);
}
#if 1
printf
(
"==== Old "
);
ndz_rangemap_dump
(
ndz1
.
map
,
chunkfunc
);
fflush
(
stdout
);
#endif
ndz2
.
map
=
ndz_readranges
(
ndz2
.
ndz
);
if
(
ndz2
.
map
==
NULL
)
{
fprintf
(
stderr
,
"%s: could not read ranges
\n
"
,
ndz_filename
(
ndz2
.
ndz
));
exit
(
1
);
}
#if 1
printf
(
"==== New "
);
ndz_rangemap_dump
(
ndz2
.
map
,
chunkfunc
);
fflush
(
stdout
);
#endif
/*
* Roll through the new image map looking for overlaps with
* the old image.
*/
mmap
=
ndz_rangemap_init
(
ndz2
.
map
->
loaddr
,
ndz2
.
map
->
hiaddr
);
assert
(
mmap
!=
NULL
);
(
void
)
ndz_rangemap_iterate
(
ndz2
.
map
,
fixmap
,
mmap
);
#if 1
printf
(
"==== Merged "
);
ndz_rangemap_dump
(
mmap
,
twochunkfunc
);
fflush
(
stdout
);
#endif
return
0
;
}
static
int
fixmap
(
struct
ndz_rangemap
*
map
,
struct
ndz_range
*
range
,
void
*
arg
)
{
struct
ndz_rangemap
*
mmap
=
(
struct
ndz_rangemap
*
)
arg
;
ndz_addr_t
addr
,
eaddr
,
oaddr
,
oeaddr
;
struct
ndz_range
*
orange
,
*
oprev
;
int
rv
;
addr
=
range
->
start
;
eaddr
=
range
->
end
;
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
"fixmap [%lu - %lu]:
\n
"
,
addr
,
eaddr
);
#endif
/* Allocate a range in the merge map */
rv
=
ndz_rangemap_alloc
(
mmap
,
addr
,
eaddr
-
addr
+
1
,
NULL
);
assert
(
rv
==
0
);
orange
=
ndz_rangemap_lookup
(
ndz1
.
map
,
addr
,
&
oprev
);
/* Current map address in not in old range */
if
(
orange
==
NULL
)
{
if
(
oprev
==
NULL
)
oprev
=
&
ndz1
.
map
->
head
;
/* Nothing more in the old map, we are done */
if
((
orange
=
oprev
->
next
)
==
NULL
)
{
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" finished old map
\n
"
);
#endif
return
1
;
}
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" found oldrange [%lu - %lu]:
\n
"
,
orange
->
start
,
orange
->
end
);
#endif
/* No overlap at all with current map range, move on */
if
(
orange
->
start
>
eaddr
)
{
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" no overlap in oldrange
\n
"
);
#endif
return
0
;
}
}
#ifdef FIXMAP_DEBUG
else
fprintf
(
stderr
,
" found oldrange [%lu - %lu]:
\n
"
,
orange
->
start
,
orange
->
end
);
#endif
while
((
oaddr
=
orange
->
start
)
<=
eaddr
)
{
struct
mergestate
*
state
;
/* Determine the extent of the overlap */
if
(
oaddr
<
addr
)
oaddr
=
addr
;
if
((
oeaddr
=
orange
->
end
)
>
eaddr
)
oeaddr
=
eaddr
;
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" found overlap [%lu - %lu]:
\n
"
,
oaddr
,
oeaddr
);
#endif
/* Dealloc this part of the range and realloc with flag set */
rv
=
ndz_rangemap_dealloc
(
mmap
,
oaddr
,
oeaddr
-
oaddr
+
1
);
assert
(
rv
==
0
);
state
=
calloc
(
1
,
sizeof
*
state
);
assert
(
state
);
state
->
ochunkno
=
(
unsigned
int
)
orange
->
data
;
state
->
nchunkno
=
(
unsigned
int
)
range
->
data
;
rv
=
ndz_rangemap_alloc
(
mmap
,
oaddr
,
oeaddr
-
oaddr
+
1
,
state
);
assert
(
rv
==
0
);
/* on to the next old map entry */
orange
=
orange
->
next
;
/* end of old map, we are done */
if
(
orange
==
NULL
)
{
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" finished old map
\n
"
);
#endif
return
1
;
}
#ifdef FIXMAP_DEBUG
fprintf
(
stderr
,
" next oldrange [%lu - %lu]:
\n
"
,
orange
->
start
,
orange
->
end
);
#endif
}
return
0
;
}
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
clientside/os/imagezip/imagehdr.h
View file @
6738c988
/*
* Copyright (c) 2000-201
4
University of Utah and the Flux Group.
* Copyright (c) 2000-201
5
University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
...
...
@@ -231,4 +231,5 @@ struct region {
* Assumed sector (block) size
*/
#define SECSIZE 512
#endif
/* _IMAGEHDR_H_ */
clientside/os/imagezip/imagezip.c
View file @
6738c988
...
...
@@ -1333,6 +1333,16 @@ addskip(uint32_t start, uint32_t size)
skip
->
next
=
skips
;
skips
=
skip
;
numskips
++
;
#if 1
{
static
FILE
*
sd
;
if
(
sd
==
NULL
)
{
sd
=
fopen
(
"/tmp/skips"
,
"w"
);
fprintf
(
sd
,
"%lu %lu
\n
"
,
inputminsec
,
inputmaxsec
);
}
fprintf
(
sd
,
"d %lu %lu
\n
"
,
start
,
start
+
size
-
1
);
}
#endif
}
/*
...
...
clientside/os/imagezip/libndz/GNUmakefile.in
0 → 100644
View file @
6738c988
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
SRCDIR = @srcdir@
TESTBED_SRCDIR = @top_srcdir@
OBJDIR = @top_builddir@
SUBDIR = $(subst $(TESTBED_SRCDIR)/,,$(SRCDIR))
MAINDIR = $(SRCDIR)/..
include $(OBJDIR)/Makeconf
CFLAGS += -DSTATS $(SUBDIRCFLAGS) -I$(MAINDIR) -I$(SRCDIR)
all: libndz.a
include $(TESTBED_SRCDIR)/GNUmakerules
OBJS = rangemap.o ndzfile.o ndzdata.o
rangemap.o: rangemap.h
ndzfile.o: libndz.h $(MAINDIR)/imagehdr.h
ndzdata.o: libndz.h $(MAINDIR)/imagehdr.h
libndz.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $?
$(RANLIB) $@
install:
clean:
rm -f libndz.a $(OBJS)
clientside/os/imagezip/libndz/imageunzip.c
0 → 100644
View file @
6738c988
This diff is collapsed.
Click to expand it.
clientside/os/imagezip/libndz/libndz.h
0 → 100644
View file @
6738c988
/*
* Copyright (c) 2014 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
#ifndef _LIBNDZ_H_
#define _LIBNDZ_H_
#include "rangemap.h"
typedef
uint32_t
ndz_chunk_t
;
struct
ndz_file
{
int
fd
;
char
*
fname
;
int
sectsize
;
int
chunksize
;
ndz_chunk_t
nchunks
;
struct
ndz_rangemap
*
rangemap
;
/* per-chunk info to verify */
/* readahead cache stuff */
};
struct
chunkmap
{
ndz_addr_t
start
;
ndz_addr_t
end
;
}
*
chunkmap
;
struct
ndz_chunkhdr
{
blockhdr_t
*
header
;
struct
region
*
region
;
struct
blockreloc
*
reloc
;
char
data
[
DEFAULTREGIONSIZE
];
};
struct
ndz_file
*
ndz_open
(
const
char
*
name
,
int
flags
);
int
ndz_close
(
struct
ndz_file
*
ndz
);
char
*
ndz_filename
(
struct
ndz_file
*
ndz
);
ssize_t
ndz_read
(
struct
ndz_file
*
ndz
,
void
*
buf
,
size_t
bytes
,
off_t
offset
);
int
ndz_readahead
(
struct
ndz_file
*
ndz
,
void
*
buf
,
size_t
bytes
,
off_t
offset
);
int
ndz_readchunkheader
(
struct
ndz_file
*
ndz
,
ndz_chunk_t
chunkno
,
struct
ndz_chunkhdr
*
chunkhdr
);
struct
ndz_rangemap
*
ndz_readranges
(
struct
ndz_file
*
ndz
);
void
ndz_dumpranges
(
struct
ndz_rangemap
*
map
);
#endif
/* _LIBNDZ_H_ */
/*
* Local variables:
* mode: C
* c-set-style: "BSD"
* c-basic-offset: 4
* End:
*/
clientside/os/imagezip/libndz/ndzdata.c
0 → 100644
View file @
6738c988
/*
* Copyright (c) 2014 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
* This file is part of the Emulab network testbed software.
*
* This file is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* This file is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this file. If not, see <http://www.gnu.org/licenses/>.
*
* }}}
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <sys/stat.h>
#include "../imagehdr.h"
#include "libndz.h"
static
int
init_chunkmap
(
struct
ndz_file
*
ndz
);
static
void
dump_chunkmap
(
struct
ndz_file
*
ndz
);
/*
* Read uncompessed data from an imagefile.
* Returns however many bytes of data we found in the image.
*/
ssize_t
ndz_readdata
(
struct
ndz_file
*
ndz
,
void
*
buf
,
size_t
bytes
,
off_t
offset
)
{
ndz_addr_t
ssect
,
esect
;
struct
ndz_range
*
range
;
ssize_t
rbytes
=
0
;
ndz_chunk_t
chunkno
;
if
(
ndz
->
rangemap
==
NULL
&&
ndz_readranges
(
ndz
)
==
NULL
)
{
errno
=
EINVAL
;
return
-
1
;
}
assert
(
ndz
->
sectsize
!=
0
);
ssect
=
offset
/
ndz
->
sectsize
;
esect
=
(
offset
+
bytes
-
1
)
/
ndz
->
sectsize
;
while
(
ssect
<=
esect
)
{
range
=
ndz_rangemap_lookup
(
ndz
->
rangemap
,
ssect
,
NULL
);
if
(
range
==
NULL
)
return
rbytes
;
chunkno
=
(
ndz_chunkno
)
range
->
data
;
assert
(
chunkno
>
0
);
chunkno
--
;
/* read the chunk */
/* decompress til we got what we want */
ssect
=
range
->
end
+
1
;
}