...
 
Commits (39)
......@@ -4556,6 +4556,7 @@ outfiles="Makeconf GNUmakefile setversion \
os/growdisk/GNUmakefile \
os/zapdisk/GNUmakefile \
os/imagezip/GNUmakefile \
os/imagezip/libndz/GNUmakefile \
os/imagezip/mbr/GNUmakefile \
os/imagezip/gpt/GNUmakefile \
os/imagezip/ffs/GNUmakefile \
......
......@@ -293,6 +293,7 @@ outfiles="Makeconf GNUmakefile setversion \
os/growdisk/GNUmakefile \
os/zapdisk/GNUmakefile \
os/imagezip/GNUmakefile \
os/imagezip/libndz/GNUmakefile \
os/imagezip/mbr/GNUmakefile \
os/imagezip/gpt/GNUmakefile \
os/imagezip/ffs/GNUmakefile \
......
......@@ -34,6 +34,8 @@ DISTFILES = global.h imagehdr.h queue.h sliceinfo.h \
EXPANDCOPYRIGHT = /usr/site/lib/copyright/expand-copyr
WITH_RANGEMAP = 0
#
# At least support MBR partitioning
#
......@@ -127,7 +129,7 @@ endif
# Necessary sometimes.
#PTHREADCFLAGS += -DCONDVARS_WORK
PROGS = imagezip imageunzip imagedump
PROGS = imagezip imageunzip imagedump imagerezip imagedelta imageundelta
CFLAGS = $(SUBDIRCFLAGS) -I$(SRCDIR) $(LDSTATIC)
LIBS = -lz
ZIPCFLAGS = $(CFLAGS) -Wall
......@@ -137,6 +139,7 @@ UNZIPLIBS = $(LIBS) $(PTHREADLIBS)
UNZIPDIRS =
HASHCFLAGS = $(CFLAGS) $(PTHREADCFLAGS) -Wall
HASHLIBS = $(LIBS) -lcrypto $(PTHREADLIBS)
NDZLIBS = libndz/libndz.a $(LIBS)
# Secure images
ifeq ($(WITH_CRYPTO),1)
......@@ -153,6 +156,8 @@ ifeq ($(WITH_V3COMPAT),1)
CFLAGS += -DWITH_V3COMPAT
endif
SUBDIRS += libndz
# MBR
ifeq ($(WITH_MBR),1)
CFLAGS += -DWITH_MBR
......@@ -225,6 +230,11 @@ ZIPLIBS += hashmap/libhashmap.a
LIBS += -lcrypto $(PTHREADLIBS)
endif
ifeq ($(WITH_RANGEMAP),1)
ZIPCFLAGS += -DTEST_RANGEMAP
ZIPLIBS += libndz/libndz.a
endif
all: $(PROGS)
whoami:
......@@ -257,6 +267,24 @@ imagehash: imagehash.o version.o
imagehash.o: imagehash.c
$(CC) -c $(HASHCFLAGS) -o imagehash.o $<
imagedelta: imagedelta.o version.o $(NDZLIBS)
$(CC) $(CFLAGS) imagedelta.o version.o $(NDZLIBS) -o imagedelta
imagedelta.o: imagedelta.c
$(CC) -c $(CFLAGS) -o imagedelta.o $<
imageundelta: imageundelta.o version.o $(NDZLIBS)
$(CC) $(CFLAGS) imageundelta.o version.o $(NDZLIBS) -o imageundelta
imageundelta.o: imageundelta.c
$(CC) -c $(CFLAGS) -o imageundelta.o $<
imagerezip: imagerezip.o version.o $(NDZLIBS)
$(CC) $(CFLAGS) imagerezip.o version.o $(NDZLIBS) -o imagerezip
imagerezip.o: imagerezip.c
$(CC) -c $(CFLAGS) -o imagerezip.o $<
sizetest: disksize.c
$(CC) -DTEST $< -o sizetest
......@@ -275,8 +303,11 @@ endif
imagezip.o: sliceinfo.h imagehdr.h global.h range.h hashmap/hashmap.h
imageunzip.o: imagehdr.h
imagehash.o: imagehdr.h imagehash.h
imagedelta.o: imagehdr.h imagehash.h libndz/libndz.h
imageundelta.o: imagehdr.h imagehash.h libndz/libndz.h
imagerezip.o: imagehdr.h imagehash.h libndz/libndz.h
version.c: imagezip.c imageunzip.c imagedump.c
version.c: imagezip.c imageunzip.c imagedump.c imagedelta.c imageundelta.c imagerezip.c
echo >$@ "char build_info[] = \"Built `date +%d-%b-%Y` by `id -nu`@`hostname | sed 's/\..*//'`:`pwd`\";"
install: $(addprefix $(INSTALL_BINDIR)/, $(PROGS))
......
#
# Check SHA1 hash file against image.
#
my $hashall = 0;
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (! -e "$imagedir/$file.sha1") {
print "[FAIL] no signature!\n";
next;
}
# make sure it is the right format:
# SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
my $sha1 = `cat $imagedir/$file.sha1`;
chomp($sha1);
if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$sha1 = $1;
} else {
print "[FAIL] bogus .sha1 file\n";
next;
}
my $nsha1 = `sha1 $imagedir/$file`;
chomp($nsha1);
if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$nsha1 = $1;
} else {
print "[FAIL] did not correctly compute sha1!\n";
next;
}
if ($sha1 eq $nsha1) {
print "[OK]\n";
} else {
print "[BAD]\n";
}
}
exit(0);
#
# Parse a directory of images looking for those that follow our versioning
# convention. For those, start with the lowest numbered version and create
# deltas for all but the first and last, e.g.:
#
# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
# rm UBUNTU14-64-STD.ndz:1
# ...
# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
# rm UBUNTU14-64-STD.ndz:9
#
my $frombase = 1;
my $removefull = 0;
my $IMAGEDELTA = "/tmp/imagedelta";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "deltafy." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
if (@files == 0) {
print STDERR "$imagedir: no images found\n";
unlink($tstamp);
exit(1);
}
my %filehash = map { ("$_" => 1) } @files;
my %images = ();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the base files
foreach my $file (@files) {
if ($file =~ /^(.*)\.ndz$/) {
my $ibase = $1;
if (-l "$imagedir/$file") {
print STDERR "$imagedir: ignoring symlink '$file'\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
$images{$ibase}{'name'} = $file;
$images{$ibase}{'lastvers'} = 0;
@{$images{$ibase}{'versions'}} = ();
delete $filehash{$file};
}
}
#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
# Find all the versions
foreach my $file (@files) {
next if (!exists($filehash{$file}));
if ($file =~ /^(.*).ndz:(\d+)$/) {
my ($ibase,$vers) = ($1,$2);
if (exists($images{$ibase})) {
push @{$images{$ibase}{'versions'}}, $vers;
if ($vers > $images{$ibase}{'lastvers'}) {
$images{$ibase}{'lastvers'} = $vers;
}
delete $filehash{$file};
next;
}
print STDERR "*** version with no base '$file', ignoring\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all versions and signatures exist
foreach my $ibase (keys %images) {
my $nukeit = 0;
my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
my $fbase = "$ibase.ndz";
my $vstr = "";
if ($vers > 0) {
$vstr = ":$vers";
if (!exists($versions{$vers})) {
if ($frombase) {
print STDERR "WARNING: ";
} else {
print STDERR "*** ";
}
print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
if (!$frombase) {
print STDERR ", ignoring\n";
$nukeit = 1;
} else {
print STDERR "\n";
delete $filehash{"$fbase$vstr.sig"};
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr"};
}
}
# got sig?
if (!exists($filehash{"$fbase$vstr.sig"})) {
if (!$frombase) {
print STDERR "*** no signature for $fbase$vstr, ignoring\n";
$nukeit = 1;
} else {
print STDERR "WARNING: no signature for $fbase$vstr, ".
"ignoring version\n";
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr.sig"};
}
# what about the hash?
delete $filehash{"$fbase$vstr.sha1"};
}
if ($nukeit) {
delete $images{$ibase};
}
}
# warn about unknown files
if (scalar(keys %filehash) > 0) {
print STDERR "WARNING: unknown files:\n";
}
foreach my $file (sort keys %filehash) {
print STDERR " $file\n";
}
# what do we have left
foreach my $ibase (sort keys %images) {
my $lvers = $images{$ibase}{'lastvers'};
print STDERR "$ibase: image and $lvers versions\n";
foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
my $base = "$ibase.ndz";
if ($vers > 1 && !$frombase) {
$base = "$ibase.ndz:" . ($vers-1);
}
my $this = "$ibase.ndz:$vers";
if ($frombase &&
(! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
print STDERR "$ibase: version $vers skipped because of missing files\n";
next;
}
my $delta = "$ibase.ddz:$vers";
if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
}
}
}
exit(0);
#
# Create signature files where missing. Check others.
#
my $hashall = 0;
my $IMAGEHASH = "/tmp/imagehash";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "hashem." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my $clogfile = "$imagedir/check.log";
my $glogfile = "$imagedir/generate.log";
system("echo '' >$clogfile; date >>$clogfile");
system("echo '' >$glogfile; date >>$glogfile");
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# no symlinks
next if (-l "$imagedir/$file");
# no sha1s
next if ($file =~ /\.sha1$/);
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (-e "$imagedir/$file.sig") {
print "found sig...";
if (checksig($file)) {
print "[OK]\n";
next;
}
print "[BAD]...re";
}
print "generating...";
if (gensig($file)) {
print "[OK]\n";
next;
}
print "[FAIL]\n";
}
exit(0);
sub checksig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
return 0;
}
return 1;
}
sub gensig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
return 0;
}
return 1;
}
This diff is collapsed.
#
# Run an image I through its paces:
#
# 1. ensure image has a valid signature I.sig, make one if not
# 2. create a memdisk and load image on it
# 3. compare image on disk with signature I.sig (imagehash)
# 4. recapture image with signature to I.new (imagezip)
# 5. compare old and new signatures (imagehash)
#
my $TMPDIR = "/local/tmp";
my $LOGDIR = "/local/logs";
my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump");
my $MAXSECTORS = (20 * 1024 * 1024 * 2);
my $LVMSTRIPE = 6;
my $checksig = 0;
my $cleanonfail = 0;
my $os = `uname`;
chomp($os);
if ($os !~ /^(Linux|FreeBSD)$/) {
die "Unknown OS '$os'\n";
}
my $arch = `uname -m`;
chomp($arch);
if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
die "Unknown arch '$arch'\n";
}
my $bindir = "/images/bin/${os}_${arch}";
foreach my $bin (@NEEDBINS) {
if (! -x "$bindir/$bin") {
die "Cannot find $bindir/$bin\n";
}
}
if (@ARGV == 0) {
print STDERR "Usage: testimage.pl image1.ndz [ image2.ndz ... ]\n";
exit(1);
}
my $tstamp = time();
print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
my $rv = 0;
foreach my $image (@ARGV) {
$rv += testimage($image);
}
exit($rv);
sub testimage($)
{
my $image = shift;
print "$image: START signature check\n";
if (sigcheck($image)) {
print "$image: ERROR signature check\n";
return 1;
}
print "$image: END signature check\n";
print "$image: START image unzip\n";
# create a memdisk for the image
my $ssize = imagesize($image);
my $dev = makedisk($image, $ssize);
if (!$dev) {
print "$image: ERROR image unzip\n";
return 1;
}
# and load it
if (mysystem("$bindir/imageunzip -f $image $dev")) {
print "$image: ERROR image unzip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image unzip\n";
print "$image: START loaded image verify\n";
if (mysystem("$bindir/imagehash -q $image $dev")) {
print "$image: ERROR image verify\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END loaded image verify\n";
#
# XXX gak! Without an MBR/GPT, imagezip cannot (yet) figure out
# what the filesystem is. We make a wild guess here based on image name.
#
my ($ifile,$itype);
if ($image =~ /([^\/]+)$/) {
$ifile = $1;
} else {
$ifile = $image;
}
if ($ifile =~ /\+/) {
# full image, don't need anything
$itype = "";
} elsif ($image =~ /FBSD/) {
# FreeBSD
$itype = "-S 165 -c $ssize";
} elsif ($ifile =~ /WIN/) {
# WIN XP or 7 are full disk images
$itype = "";
} else {
# assume Linux
$itype = "-S 131 -c $ssize";
}
my $nimage = "$image.new";
print "$image: START image rezip\n";
if (mysystem("$bindir/imagezip $itype -U $nimage.sig $dev $nimage")) {
print "$image: ERROR image rezip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image rezip\n";
print "$image: START image sigfile compare\n";
if (comparesigfiles($image, $nimage)) {
print "$image: ERROR image sigfile compare\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image sigfile compare\n";
unmakedisk($image, $dev);
return 0;
}
sub comparesigfiles($$)
{
my ($image1,$image2) = @_;
if (mysystem("$bindir/imagehash -Rq -o $image1.sig > $image1.sig.txt")) {
print "$image1: could not dump signature\n";
return 1;
}
if (mysystem("$bindir/imagehash -Rq -o $image2.sig > $image2.sig.txt")) {
print "$image2: could not dump signature\n";
return 1;
}
if (mysystem("diff -q $image1.sig.txt $image2.sig.txt")) {
print "*** signatures differ (diff $image1.sig.txt $image2.sig.txt)\n";
return 1;
}
unlink("$image1.sig.txt", "$image2.sig.txt");
return 0;
}
sub sigcheck($)
{
my $image = shift;
if (! -e "$image") {
print STDERR "$image: does not exist\n";
return 1;
}
if (! -e "$image.sig") {
print STDERR "$image: signature does not exist\n";
return 1;
}
if ($checksig) {
if (mysystem("$bindir/imagehash -qSX $image")) {
print "$image: signature did not check\n";
return 1;
}
# gen a new format sig file and compare
if (mysystem("$bindir/imagehash -qcX -o ${image}foo.sig $image")) {
print "$image: could not generate signature\n";
return 1;
}
if (comparesigfiles($image, "${image}foo")) {
print "$image: new signature does not match old\n";
return 1;
}
}
return 0;
}
sub imagesize($)
{
my $image = shift;
my @output = `$bindir/imagedump $image`;
if ($?) {
print "$image: *** could not get size of image\n";
return 0;
}
foreach my $line (@output) {
chomp($line);
if ($line =~ /covered sector range: \[(\d+)-(\d+)\]/) {
if ($1 != 0) {
print "$image: WARNING: image does not start at 0\n";
}
$ssize = $2 + 1;
last;
}
}
return $ssize;
}
sub makedisk($$)
{
my ($image,$ssize) = @_;
my ($istr,$dev);
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
my $mb = int(($ssize + 2047) / 2048);
$mb += 100;
if ($os eq "Linux") {
if ($ssize > $MAXSECTORS) {
print "$image: ERROR: image too large ($ssize) for ramdisk,".
" using LV instead\n";
if (mysystem("lvcreate -i $LVMSTRIPE -L ${mb}m -n $istr emulab")) {
print STDERR "could not create LV\n";
return undef;
}
return "/dev/emulab/$istr";
}
# XXX there has to be a better way!
#
# mount -t tmpfs -o size=20580m tmpfs /mnt/FOO.ndz
# dd if=/dev/zero of=/mnt/FOO.ndz/disk bs=1024k seek=20479 count=1
# losetup -f
# losetup /dev/loop0 /mnt/FOO.ndz/disk
#
my $mountpoint = "/mnt/$istr";
if (!mkdir($mountpoint)) {
print STDERR "could not make mountpoint $mountpoint\n";
return undef;
}
if (mysystem("mount -t tmpfs -o size=${mb}m tmpfs $mountpoint")) {
rmdir($mountpoint);
return undef;
}
my $mbm1 = $mb - 1;
if (mysystem("dd if=/dev/zero of=$mountpoint/disk bs=1024k seek=$mbm1 count=1")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
$dev = `losetup -f`;
chomp($dev);
if (mysystem("losetup $dev $mountpoint/disk")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
} else {
print STDERR "Cannot do this under $os yet\n";
}
return $dev;
}
sub unmakedisk($$)
{
my ($image,$dev) = @_;
my $istr;
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
if ($dev eq "/dev/emulab/$istr") {
if (mysystem("lvremove -f emulab/$istr")) {
print STDERR "$image: could not destroy LV\n";
return -1;
}
} elsif (mysystem("losetup -d $dev") ||
mysystem("umount /mnt/$istr") ||
!rmdir("/mnt/$istr")) {
print STDERR "$image: could not tear down ramdisk\n";
return -1;
}
return 0;
}
sub mysystem($)
{
my ($cmd) = @_;
my $logfile = "$LOGDIR/testimage.$tstamp.log";
my $now = localtime();
my $redir;
if (open(FD, ">>$logfile")) {
print FD "==== $now: $cmd\n";
close(FD);
}
if ($cmd =~ />/) {
$redir = "2>>$logfile";
} else {
$redir = ">>$logfile 2>&1";
}
if (system("$cmd $redir")) {
my $stat = $?;
print STDERR "*** '$cmd' failed, see '$logfile'\n";
return $stat;
}
return 0;
}
#
# Parse a directory of images looking for those that follow our versioning
# convention and for which the intermediate versions are deltas (identified
# here by using the all-new-convention of a .ddz suffix!)
#
# For each of those deltas use the base and delta to produce a new version
# of the full image file that we can compare to the original.
#
my $frombase = 0;
my $removefull = 0;
my $IMAGEDELTA = "/tmp/imageundelta";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "undeltafy." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ddz* 2>/dev/null`;
chomp @files;
if (@files == 0) {
print STDERR "$imagedir: no deltas found\n";
unlink($tstamp);
exit(1);
}
my %filehash = map { ("$_" => 1) } @files;
my %images = ();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the bases and associate versions with them
foreach my $file (@files) {
if ($file =~ /^(.*).ddz:(\d+)$/) {
my ($ibase,$vers) = ($1,$2);
if (!exists($images{$ibase})) {
$images{$ibase}{'versions'} = ();
if (! -e "$imagedir/$ibase.ndz") {
print STDERR
"*** cannot find base image '$ibase.ndz' for '$file'\n";
}
}
push @{$images{$ibase}{'versions'}}, $vers;
if ($vers > $images{$ibase}{'lastvers'}) {
$images{$ibase}{'lastvers'} = $vers;
}
delete $filehash{$file};
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all version deltas and their signatures exist
foreach my $ibase (keys %images) {
my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
my $fbase = "$ibase.ndz";
my $vstr = "";
if ($vers > 0) {
$vstr = ":$vers";
if (!exists($versions{$vers})) {
print STDERR "*** no version $vers of base '$ibase' ('$fbase$vstr'), ignoring\n";
delete $images{$ibase};
last;
}
}
# got sig?
if (!exists($filehash{"$fbase$vstr.sig"})) {
print STDERR "*** no signature for $fbase$vstr, ignoring\n";
delete $images{$ibase};
last;
}
delete $filehash{"$fbase$vstr.sig"};
# what about the hash?
delete $filehash{"$fbase$vstr.sha1"};
}
}
# warn about unknown files
if (scalar(keys %filehash) > 0) {
print STDERR "WARNING: unknown files:\n";
}
foreach my $file (sort keys %filehash) {
print STDERR " $file\n";
}
exit(0);
# what do we have left
foreach my $ibase (sort keys %images) {
my $lvers = $images{$ibase}{'lastvers'};
print STDERR "$ibase: image and $lvers versions\n";
foreach my $vers (1 .. $images{$ibase}{'lastvers'} - 1) {
my $base = "$ibase.ndz";
if ($vers > 1 && !$frombase) {
$base = "$ibase.ndz:" . ($vers-1);
}
my $this = "$ibase.ndz:$vers";
my $delta = "$ibase.ddz:$vers";
if (system("$IMAGEDELTA -S $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
print STDERR "*** '$IMAGEDELTA -S $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
}
}
}
exit(0);
......@@ -37,9 +37,9 @@
#ifndef lint
#if 0
static char sccsid[] = "@(#)crc.c 8.1 (Berkeley) 6/17/93";
#endif
static const char rcsid[] =
"$FreeBSD: src/usr.bin/cksum/crc.c,v 1.4 1999/12/05 20:03:21 charnier Exp $";
#endif
#endif /* not lint */
#include <sys/types.h>
......
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -66,7 +66,7 @@ read_linuxslice(int slice, iz_type stype, iz_lba start, iz_size size,
* Check for a LILO boot block and create relocations as necessary
* (if the partition starts at 0, the values are already relative)
*/
if (dorelocs && start > 0 &&
if (dorelocs && (start > 0 || forcerelocs) &&
fixup_lilo(slice, stype, start, size, sname, infd, &rval) != 0)
return 1;
......
/*
* Copyright (c) 2000-2004, 2009 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -170,7 +170,8 @@ fixup_lilo(int slice, int stype, u_int32_t start, u_int32_t size,
/*
* Check for aliases which produce a map entry pointing
* to the same sectors as the original. We don't want to
* relocate those sectors multiple times!
* relocate those sectors multiple times! But we do need
* to relocate the pointers.
*/
s0 = getsector(&ip->start);
for (a_ip = dtab.idtab.images; a_ip < ip; a_ip++) {
......@@ -180,14 +181,6 @@ fixup_lilo(int slice, int stype, u_int32_t start, u_int32_t size,
if (s0 == a_s0)
break;
}
if (a_ip < ip) {
if (debug > 1)
fprintf(stderr, " LILO parse: "
"skipping %s (alias for %s)\n",
ip->name, a_ip->name);
ip++;
continue;
}
/*
* Fixup initrd (if present) and map sector(s)
......@@ -202,7 +195,14 @@ fixup_lilo(int slice, int stype, u_int32_t start, u_int32_t size,
}
fixup_sector(FOFFSET(poff, struct idtab, images[i].start),
&ip->start);
fixup_map(infd, s0);
if (a_ip < ip) {
if (debug > 1)
fprintf(stderr, " LILO parse: "
"skipping %s (alias for %s)\n",
ip->name, a_ip->name);
} else
fixup_map(infd, s0);
ip++;
}
......
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -254,8 +254,8 @@ _read_bsdslice(int slice, iz_type bsdtype, iz_lba start, iz_size size,
* Note that event if partitions were relative (absoffset == 0) we
* have converted the value in dlabel to absolute by this point.
*/
if (dorelocs &&
start != 0 && dlabel.label.d_partitions[0].p_offset == start) {
if (dorelocs && (start > 0 || forcerelocs) &&
dlabel.label.d_partitions[0].p_offset == start) {
for (i = 0; i < npart; i++) {
if (dlabel.label.d_partitions[i].p_size == 0)
continue;
......@@ -651,7 +651,7 @@ read_bsdcg(struct fs *fsp, struct cg *cgp, int cg, u_int32_t offset)
fprintf(stderr, " ");
max = fsp->fs_ipg;
#if 1
#if 0
/*
* Paranoia!
*/
......
......@@ -30,6 +30,7 @@ extern int slicemode;
extern int dorelocs;
extern int metaoptimize;
extern int excludenonfs;
extern int forcerelocs;
extern off_t devlseek(int fd, off_t off, int whence);
extern ssize_t devread(int fd, void *buf, size_t nbytes);
......
This diff is collapsed.
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -141,6 +141,7 @@ main(int argc, char **argv)
if (!isstdin) {
if ((infd = open(argv[0], O_RDONLY, 0666)) < 0) {
perror(argv[0]);
errors++;
continue;
}
} else
......@@ -155,13 +156,14 @@ main(int argc, char **argv)
fprintf(stderr,
"%s: Cannot validate checksum signing key\n",
argv[0]);
errors++;
continue;
}
}
#endif
#endif
errors = dumpfile(isstdin ? "<stdin>" : argv[0], infd);
errors += dumpfile(isstdin ? "<stdin>" : argv[0], infd);
#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
......@@ -192,7 +194,7 @@ usage(void)
static char chunkbuf[CHUNKSIZE];
static unsigned int magic;
static unsigned long chunkcount;
static uint32_t nextsector;
static uint32_t nextsector, nextcovered;
static uint32_t fmax, fmin, franges, amax, amin, aranges;
static uint32_t adist[8]; /* <4k, <8k, <16k, <32k, <64k, <128k, <256k, >=256k */
static int regmax, regmin;
......@@ -213,6 +215,7 @@ dumpfile(char *name, int fd)
isstdin = (fd == fileno(stdin));
wasted = sectinuse = sectfree = 0;
nextsector = 0;
nextcovered = 0;
relocs = 0;
relocbytes = 0;
hisect = 0;
......@@ -549,6 +552,15 @@ dumpchunk(char *name, char *buf, int chunkno, int checkindex)
printf(" Chunk %d: %u compressed bytes, ",
chunkno, hdr->size);
if (hdr->magic > COMPRESSED_V1) {
if (hdr->firstsect != nextcovered) {
printf(" WARNING: chunk %d %s in covered "
"range, %u/%u last-end/cur-start\n",
chunkno,
(hdr->firstsect < nextcovered) ?
"overlap" : "gap", nextcovered,
hdr->firstsect);
}
nextcovered = hdr->lastsect;
printf("sector range [%u-%u], ",
hdr->firstsect, hdr->lastsect-1);
if (hdr->reloccount > 0)
......
This diff is collapsed.
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -20,6 +20,8 @@
*
* }}}
*/
#ifndef _IMAGEHASH_H_
#define _IMAGEHASH_H_
#define HASH_VERSION_1 0x20031107
#define HASH_VERSION_2 0x20140618
......@@ -52,3 +54,5 @@ struct hashinfo {
#define HASH_TYPE_MD5 1
#define HASH_TYPE_SHA1 2
#define HASH_TYPE_RAW 3
#endif /* _IMAGEHASH_H_ */
/*
* Copyright (c) 2000-2014 University of Utah and the Flux Group.
* Copyright (c) 2000-2015 University of Utah and the Flux Group.
*
* {{{EMULAB-LICENSE
*
......@@ -83,7 +83,7 @@ struct blockhdr_V2 {
int32_t regioncount; /* number of regions */
/* V2 follows */
uint32_t firstsect; /* first sector described by block */
uint32_t lastsect; /* last sector described by block */
uint32_t lastsect; /* first sector past block */
int32_t reloccount; /* number of reloc entries */
};
......@@ -120,7 +120,7 @@ struct blockhdr_V4 {
int32_t regioncount; /* number of regions */
/* V2 follows */
uint32_t firstsect; /* first sector described by block */
uint32_t lastsect; /* last sector described by block */
uint32_t lastsect; /* first sector past block */
int32_t reloccount; /* number of reloc entries */
/* V4 follows */
uint16_t enc_cipher; /* cipher was used to encrypt */
......@@ -133,6 +133,33 @@ struct blockhdr_V4 {
/* Unique ID for the whole image */
};
/*
* Coming soon in V5:
*
* 64-bit support.
* Grow blockindex, firstsect, lastsect, region descriptors, may need to
* grow DEFAULTREGIONSIZE to accomodate the doubling in size of regions.
*
* Flag field?
* For example, to indicate a delta image. Would probably take over the
* otherwise unused blocktotal field.
*
* Sectorsize field?
* To make explicit the units of sector fields; e.g., 512 vs 4096.
*
* Chunksize field?
* To support different chunksizes.
*
* Mandate little-endian on-disk data.
* Code changes only to use appropriate endian macros when reading/writing
* data. No data struct changes needed.
*
* Support for SHA256 and SHA512 checksums.
* Just some constants here, as checksum is already 64 bytes.
* Will need to grow the imagehash header to accomodate these checksums
* for signatures, but that is separately versioned.
*/
/*
* Checksum types supported
*/
......@@ -231,4 +258,5 @@ struct region {
* Assumed sector (block) size
*/
#define SECSIZE 512
#endif /* _IMAGEHDR_H_ */
This diff is collapsed.
This diff is collapsed.
......@@ -120,6 +120,7 @@ static int imageversion = 1;
static int dots = 0;
static int dotcol;
static int directio = 0;
static int ignoreskips = 0;
static struct timeval stamp;
#ifndef FRISBEE
static int infd;
......@@ -598,7 +599,7 @@ main(int argc, char *argv[])
#ifdef NOTHREADS
nothreads = 1;
#endif
while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:Cr:Na:ck:eu:f")) != -1)
while ((ch = getopt(argc, argv, "vdhs:zp:oOnFD:W:Cr:Na:ck:eu:fIM:")) != -1)
switch(ch) {
#ifdef FAKEFRISBEE
case 'F':
......@@ -722,6 +723,14 @@ main(int argc, char *argv[])
has_id = 1;
break;
/* ignore skipped ranges on output (which must be stdout) */
case 'I':
ignoreskips++;
break;
/* set maxsector for error checking */
case 'M':
outputmaxsec = strtoul(optarg, NULL, 0);
break;
case 'h':
case '?':
default:
......@@ -769,6 +778,19 @@ main(int argc, char *argv[])
else if (argc == 2 && strcmp(argv[1], "-")) {
int flags;
/*
* This option is really a hack so we can compare the
* content of two images regardless of how they got split
* into chunks; i.e., for cases where we cannot just compare
* the image files directly. So we force them to use stdout
* to make it painfully obvious that this is not a useful
* option!
*/
if (ignoreskips) {
fprintf(stderr, "Must output to stdout with -I\n");
exit(1);
}
/*
* XXX perform seek and MBR checks before we truncate
* the output file. If they have their input/output
......@@ -824,7 +846,10 @@ main(int argc, char *argv[])
* we cannot really handle slice mode, we must always zero fill
* (cannot skip free space) and we cannot use pwrite.
*/
if (lseek(outfd, (off_t)0, SEEK_SET) < 0) {
if (ignoreskips) {
dofill = 0;
seekable = 0;
} else if (lseek(outfd, (off_t)0, SEEK_SET) < 0) {
if (slice) {
fprintf(stderr, "Output file is not seekable, "
"cannot specify a slice\n");
......@@ -1850,7 +1875,7 @@ writedata(off_t offset, size_t size, void *buf)
#endif
if (seekable) {
cc = pwrite(outfd, buf, size, offset);
} else if (offset == nextwriteoffset) {
} else if (offset == nextwriteoffset || ignoreskips) {
cc = write(outfd, buf, size);
} else {
fprintf(stderr, "Non-contiguous write @ %lld (should be %lld)\n",
......@@ -2227,6 +2252,7 @@ reloc_lilo(void *addr, int reloctype, uint32_t size)
if (sector == 0)
break;
sector += outputminsec;
assert(outputmaxsec == 0 || sector <= outputmaxsec);
putsector(sect, sector, sect->device, sect->nsect);
sect++;
}
......
......@@ -98,6 +98,7 @@ int forcereads= 0;
int badsectors= 0;
int retrywrites= 1;
int dorelocs = 1;
int forcerelocs = 0;
int metaoptimize = 0;
int filemode = 0;
int excludenonfs = 0;
......@@ -404,7 +405,7 @@ main(int argc, char *argv[])
memset(imageid, '\0', UUID_LENGTH);
gettimeofday(&sstamp, 0);
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XxH:U:P:Me:k:u:a:Z")) != -1)
while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XxH:U:P:Me:k:u:a:ZL")) != -1)
switch(ch) {
case 'v':
version++;
......@@ -425,6 +426,9 @@ main(int argc, char *argv[])
case 'b':
slicetype = IZTYPE_FBSDNOLABEL;
break;
case 'L':
forcerelocs = 1;
break;
case 'N':
dorelocs = 0;
break;
......@@ -646,6 +650,9 @@ main(int argc, char *argv[])
if (!slicemode && !filemode && dorelocs)
dorelocs = 0;
if (forcerelocs && !dorelocs)
dorelocs = 1;
infilename = argv[0];
if (strcmp(infilename, "-")) {
if ((infd = open(infilename, O_RDONLY, 0)) < 0) {
......@@ -778,8 +785,12 @@ main(int argc, char *argv[])
if (debug)
dumpskips(debug > 1);
makeranges();
#ifdef TEST_RANGEMAP
dumpranges(debug > 1);
#else
if (debug)
dumpranges(debug > 1);
#endif
if (ranges == NULL) {
/*
* No valid ranges, exit with an error.
......@@ -975,6 +986,25 @@ read_image(int fd)
gotbb = 2;
bbstr = "MBR";
}
/*
* If we are forcing interpretation as a BSD slice, there should
* not be an MBR, however there is sometimes this thing called the
* "historical bogus partition table" (by the geom code) which
* is an MBR with P4 having 50000 sectors. We need to avoid this
* so we force looking for just a disklabel instead.
*/
if (gotbb == 2) {
parttab = disk.slices;
if (parttab[3].type == IZTYPE_386BSD &&
parttab[3].size == 50000 && parttab[3].offset == 0 &&
parttab[2].size == 0 && parttab[2].offset == 0 &&
parttab[1].size == 0 && parttab[1].offset == 0 &&
parttab[0].size == 0 && parttab[0].offset == 0) {
fprintf(stderr,
"WARNING: ignoring historical bogus MBR\n");
gotbb = 0;
}
}
#endif
if (!gotbb) {
......@@ -1072,7 +1102,7 @@ read_image(int fd)
*/
if (!slicemode) {
iz_lba dlow = disk.lodata;
iz_lba dhigh = disk.hidata;
iz_lba dhigh = maxmode ? (inputmaxsec-1) : disk.hidata;
iz_lba losect = disk.losect;
iz_lba hisect = disk.hisect;
......@@ -1190,14 +1220,14 @@ read_image(int fd)
if (!slicemode) {
if (dstart != inputminsec) {
fprintf(stderr,
"partitioner value (%lu) different than "
"partitioner low value (%lu) different than "
"computed value (%lu); using the former\n",
(unsigned long)dstart, inputminsec);
inputminsec = dstart;
}
if (!maxmode && dstart+dsize != inputmaxsec) {
fprintf(stderr,
"partitioner value (%lu) different than "
"partitioner high value (%lu) different than "
"computed value (%lu); using the former\n",
(unsigned long)(dstart+dsize), inputmaxsec);
inputmaxsec = dstart + dsize;
......@@ -1293,6 +1323,102 @@ usage()
exit(1);
}
#ifdef TEST_RANGEMAP
#include <libndz/rangemap.h>
struct ndz_rangemap *rangemap;
static void
initmap(void)
{
ndz_addr_t maplo, maphi;
int rv;
maplo = inputminsec;
maphi = inputmaxsec ? (inputmaxsec-1) : NDZ_HIADDR;
fprintf(stderr, "Initializing range map [%lu-%lu]\n", maplo, maphi);
rangemap = ndz_rangemap_init(maplo, maphi);
assert(rangemap != NULL);
/* consider everything allocated to start */
rv = ndz_rangemap_alloc(rangemap, maplo, maphi-maplo+1, NULL);
assert(rv == 0);
}
static void
freerange(ndz_addr_t start, ndz_size_t size)
{
int rv;
if (rangemap == NULL)
initmap();
rv = ndz_rangemap_dealloc(rangemap, start, size);
assert(rv == 0);
}
static void
allocrange(ndz_addr_t start, ndz_size_t size)
{
int rv;
if (rangemap == NULL)
initmap();
rv = ndz_rangemap_alloc(rangemap, start, size, NULL);
assert(rv == 0);
}
static int
verifyfunc(struct ndz_rangemap *imap, struct ndz_range *range, void *arg)
{
struct range **izrangep = arg;
struct range *izrange = *izrangep;
ndz_addr_t addr, eaddr, izaddr, izeaddr;
addr = range->start;
eaddr = range->end;
if (izrange == NULL) {
fprintf(stderr, "*** [%lu-%lu]: ran out of iz ranges!\n",
addr, eaddr);
return 1;
}
izaddr = izrange->start;
izeaddr = izaddr + izrange->size - 1;
if (addr != izaddr || eaddr != izeaddr) {
fprintf(stderr,
"*** [%lu-%lu]: does not match iz range [%lu-%lu]!\n",
addr, eaddr, izaddr, izeaddr);
return 1;
}
*izrangep = izrange->next;
return 0;
}
static void
comparemap(int verbose)
{
/* XXX adjust max */
if (inputmaxsec != 0)
ndz_rangemap_dealloc(rangemap, inputmaxsec,
NDZ_HIADDR - inputmaxsec + 1);
ndz_rangemap_dump(rangemap, verbose ? 0 : 1, NULL);
if (frangesize == 0) {
struct range *currange = ranges;
/* ranges should be 1-to-1 with map ranges */
if (ndz_rangemap_iterate(rangemap, verifyfunc, &currange))
fprintf(stderr, "*** Maps differ!\n");
if (currange != NULL)
fprintf(stderr, "*** More iz ranges starting at %u!\n",
currange->start);
}
}
#endif
/*
* Add a range of free space to skip
*/
......@@ -1314,6 +1440,10 @@ addskip(uint32_t start, uint32_t size)
skip->next = skips;
skips = skip;
numskips++;
#ifdef TEST_RANGEMAP
freerange((ndz_addr_t)start, (ndz_size_t)size);
#endif
}
/*
......@@ -1707,6 +1837,9 @@ dumpranges(int verbose)
fprintf(stderr,
"Total Number of Valid Sectors: %u (bytes %llu) in %d ranges\n",
(unsigned)total, (unsigned long long)sectobytes(total), nranges);
#ifdef TEST_RANGEMAP
comparemap(verbose);
#endif
}
/*
......
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#