Commit ba3ab203 authored by Mike Hibler's avatar Mike Hibler
Browse files

Scripts for crunching over large collections of images.

Work in progress.
parent a4f6e098
#
# Check SHA1 hash file against image.
#
my $hashall = 0;
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (! -e "$imagedir/$file.sha1") {
print "[FAIL] no signature!\n";
next;
}
# make sure it is the right format:
# SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
my $sha1 = `cat $imagedir/$file.sha1`;
chomp($sha1);
if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$sha1 = $1;
} else {
print "[FAIL] bogus .sha1 file\n";
next;
}
my $nsha1 = `sha1 $imagedir/$file`;
chomp($nsha1);
if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$nsha1 = $1;
} else {
print "[FAIL] did not correctly compute sha1!\n";
next;
}
if ($sha1 eq $nsha1) {
print "[OK]\n";
} else {
print "[BAD]\n";
}
}
exit(0);
#
# Parse a directory of images looking for those that follow our versioning
# convention. For those, start with the lowest numbered version and create
# deltas for all but the first and last, e.g.:
#
# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
# rm UBUNTU14-64-STD.ndz:1
# ...
# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
# rm UBUNTU14-64-STD.ndz:9
#
my $frombase = 1;
my $removefull = 0;
my $IMAGEDELTA = "/tmp/imagedelta";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "deltafy." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
if (@files == 0) {
print STDERR "$imagedir: no images found\n";
unlink($tstamp);
exit(1);
}
my %filehash = map { ("$_" => 1) } @files;
my %images = ();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the base files
foreach my $file (@files) {
if ($file =~ /^(.*)\.ndz$/) {
my $ibase = $1;
if (-l "$imagedir/$file") {
print STDERR "$imagedir: ignoring symlink '$file'\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
$images{$ibase}{'name'} = $file;
$images{$ibase}{'lastvers'} = 0;
@{$images{$ibase}{'versions'}} = ();
delete $filehash{$file};
}
}
#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
# Find all the versions
foreach my $file (@files) {
next if (!exists($filehash{$file}));
if ($file =~ /^(.*).ndz:(\d+)$/) {
my ($ibase,$vers) = ($1,$2);
if (exists($images{$ibase})) {
push @{$images{$ibase}{'versions'}}, $vers;
if ($vers > $images{$ibase}{'lastvers'}) {
$images{$ibase}{'lastvers'} = $vers;
}
delete $filehash{$file};
next;
}
print STDERR "*** version with no base '$file', ignoring\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all versions and signatures exist
foreach my $ibase (keys %images) {
my $nukeit = 0;
my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
my $fbase = "$ibase.ndz";
my $vstr = "";
if ($vers > 0) {
$vstr = ":$vers";
if (!exists($versions{$vers})) {
if ($frombase) {
print STDERR "WARNING: ";
} else {
print STDERR "*** ";
}
print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
if (!$frombase) {
print STDERR ", ignoring\n";
$nukeit = 1;
} else {
print STDERR "\n";
delete $filehash{"$fbase$vstr.sig"};
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr"};
}
}
# got sig?
if (!exists($filehash{"$fbase$vstr.sig"})) {
if (!$frombase) {
print STDERR "*** no signature for $fbase$vstr, ignoring\n";
$nukeit = 1;
} else {
print STDERR "WARNING: no signature for $fbase$vstr, ".
"ignoring version\n";
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr.sig"};
}
# what about the hash?
delete $filehash{"$fbase$vstr.sha1"};
}
if ($nukeit) {
delete $images{$ibase};
}
}
# warn about unknown files
if (scalar(keys %filehash) > 0) {
print STDERR "WARNING: unknown files:\n";
}
foreach my $file (sort keys %filehash) {
print STDERR " $file\n";
}
# what do we have left
foreach my $ibase (sort keys %images) {
my $lvers = $images{$ibase}{'lastvers'};
print STDERR "$ibase: image and $lvers versions\n";
foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
my $base = "$ibase.ndz";
if ($vers > 1 && !$frombase) {
$base = "$ibase.ndz:" . ($vers-1);
}
my $this = "$ibase.ndz:$vers";
if ($frombase &&
(! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
print STDERR "$ibase: version $vers skipped because of missing files\n";
next;
}
my $delta = "$ibase.ddz:$vers";
if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
}
}
}
exit(0);
#
# Create signature files where missing. Check others.
#
my $hashall = 0;
my $IMAGEHASH = "/tmp/imagehash";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "hashem." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my $clogfile = "$imagedir/check.log";
my $glogfile = "$imagedir/generate.log";
system("echo '' >$clogfile; date >>$clogfile");
system("echo '' >$glogfile; date >>$glogfile");
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# no symlinks
next if (-l "$imagedir/$file");
# no sha1s
next if ($file =~ /\.sha1$/);
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (-e "$imagedir/$file.sig") {
print "found sig...";
if (checksig($file)) {
print "[OK]\n";
next;
}
print "[BAD]...re";
}
print "generating...";
if (gensig($file)) {
print "[OK]\n";
next;
}
print "[FAIL]\n";
}
exit(0);
sub checksig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
return 0;
}
return 1;
}
sub gensig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
return 0;
}
return 1;
}
<
#
# Given two full images I1 and I2:
#
# I. Basic:
#
# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
# 3. compare sigs of I2.ndz and I2.ndz.v2
#
# What we learn: imagedelta/undelta don't crash on the image in question,
# delta/undelta are correct at the level of the signature file.
#
# II. More advanced:
#
# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
# 7. compare sigs of I2.ddz I2.ddz.v2
# 7a. if sigs don't compare [*], we compate I2.ddz.v2 against the I2 disk
#
# What we learn: imagezip and imagedelta delta creation are consistent.
#
# III. Over the top:
#
# 8. apply delta I2.ddz.v2 to I1 disk
# 9. compare contents of I1 and I2 disks [2]
# 10. load full I2.ndz.v2 image on I3 disk
# 11. compare contents of I2 and I3 disks [3]
#
# What we learn: new format files can be loaded (imagezip), actual on-disk
# contents are correct.
#
# Footnotes:
#
# [1] Since delta signatures generated by imagezip are imperfect,
# the sigs may not match. To ensure their equivalence, we assume
# the imagezip sig is correct and verify the other by checking it
# against the on-disk image.
#
# [2] We cannot just "cmp" the two disks due to inexactness of the
# delta created by imagedelta. It may include unused blocks that
# will get propogate back to disk. Hence, we create new image files
# from both disks using imagezip--with "-F 0" to ensure no free sects
# are included--and byte compare those image files.
#
# [3] As with imagedelta, imageundelta is inexact and may have included
# garbage sectors from the original image. We perform the same
# comparison as in [2] instead.
#
my $TMPDIR = "/local/tmp";
my $LOGDIR = "/local/logs";
my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump",
"imagedelta", "imageundelta");
my $MAXPHASE = 2;
my $MAXSECTORS = (20 * 1024 * 1024 * 2);
my $LVMSTRIPE = 6;
my $checksig = 0;
my $cleanonfail = 0;
my $os = `uname`;
chomp($os);
if ($os !~ /^(Linux|FreeBSD)$/) {
die "Unknown OS '$os'\n";
}
my $arch = `uname -m`;
chomp($arch);
if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
die "Unknown arch '$arch'\n";
}
my $bindir = "/images/bin/${os}_${arch}";
foreach my $bin (@NEEDBINS) {
if (! -x "$bindir/$bin") {
die "Cannot find $bindir/$bin\n";
}
}
if (@ARGV != 2) {
print STDERR "Usage: testdelta.pl imageV1.ndz imageV2.ndz\n";
exit(1);
}
my $uid = $>;
if ($uid == 0) {
print STDERR "Do not run as root, script will sudo when necessary\n";
exit(1);
}
my $tstamp = time();
print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
my %image1 = ( "name" => $ARGV[0] );
my %image2 = ( "name" => $ARGV[1] );
my $rv = 0;
$rv = phase1(\%image1, \%image2)
if ($MAXPHASE > 0);
$rv = phase2(\%image1, \%image2)
if ($rv == 0 && $MAXPHASE > 1);
$rv = phase3(\%image1, \%image2)
if ($rv == 0 && $MAXPHASE > 2);
if ($rv == 0) {
unmakedisk($image1{'name'}, $image1{'disk'});
unmakedisk($image2{'name'}, $image2{'disk'});
}
exit($rv);
#
# I. Basic:
#
# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
# 3. compare sigs of I2.ndz and I2.ndz.v2
#
sub phase1($$)
{
my ($image1,$image2) = @_;
my $fixedsig = 0;
my $imagev1 = $image1->{'name'};
my $imagev2 = $image2->{'name'};
logit("START signature check of both images");
if (sigcheck($imagev1) || sigcheck($imagev2)) {
logit("FAILED signature check of both images");
return 1;
}
logit("END signature check of both images");
my $delta = $imagev2;
$delta =~ s/ndz/ddz/;
if ($imagev2 eq $delta) {
logit(" $imagev2: malformed name, no .ndz");
return 1;
}
logit("START image2 delta creation");
again:
if (mysystem("$bindir/imagedelta -SVF $imagev1 $imagev2 $delta.new", 0)) {
logit(" WARNING: could not generate delta image");
if (!$fixedsig) {
logit(" recreating signature for $imagev2...");
mysystem("mv $imagev2.sig $imagev2.bak.sig");
if (!mysystem("$bindir/imagehash -qcX $imagev2")) {
logit(" validating new signature for $imagev2...");
if (!comparesigfiles("$imagev2.bak", $imagev2, 0)) {
unlink("$imagev2.bak.sig");
logit("RESTART image2 delta creation");
$fixedsig = 1;
goto again;
}
unlink("$imagev2.sig");
}
mysystem("mv $imagev2.bak.sig $imagev2.sig");
}
logit("FAILED image2 delta creation");
return 1;
}
logit("END image2 delta creation");
logit("START undelta to image2.new");
if (mysystem("$bindir/imageundelta -SV $imagev1 $delta.new $imagev2.new")) {
logit("FAILED undelta to image2.new");
return 1;
}
logit("END undelta to image2.new");
logit("START signature compare of image2 and image2.new");
if (comparesigfiles($imagev2, "$imagev2.new", 1)) {
logit(" $imagev2.new: signature mismatch with $imagev2");
logit("FAILED signature compare of image2 and image2.new");
return 1;
}
logit("END signature compare of image2 and image2.new");
return 0;
}
#
# II. More advanced:
#
# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
# 7. compare sigs of I2.ddz I2.ddz.v2
#
sub phase2($$)
{
my ($image1,$image2) = @_;
my $imagev1 = $image1->{'name'};
my $imagev2 = $image2->{'name'};
my $delta = $imagev2;
$delta =~ s/ndz/ddz/;
if ($imagev2 eq $delta) {
logit(" $imagev2: malformed name, no .ndz");
return 1;
}
logit("START unzip of image1");
my ($devv1,$ssizev1) = unzipimage($imagev1);
if (!$devv1) {
logit("FAILED unzip of image1");
return 1;
}
logit("END unzip of image1");
logit("START signature check of image1 on disk");
if ($checksig) {
if (mysystem("$bindir/imagehash -qX $imagev1 $devv1")) {
logit("FAILED signature check of image1 on disk");
unmakedisk($imagev1, $devv1) if ($cleanonfail);
return 1;
}
}
logit("END signature check of image1 on disk");
logit("START unzip of image2");
my ($devv2,$ssizev2) = unzipimage($imagev2);
if (!$devv2) {
logit("FAILED unzip of image2");
unmakedisk($imagev1, $devv1) if ($cleanonfail);
return 1;
}
logit("END unzip of image2");
logit("START signature check of image2 on disk");
if ($checksig) {
if (mysystem("$bindir/imagehash -qX $imagev2 $devv2")) {
logit("FAILED signature check of image2 on disk");
unmakedisk($imagev2, $devv2) if ($cleanonfail);
unmakedisk($imagev1, $devv1) if ($cleanonfail);
return 1;
}
}
logit("END signature check of image2 on disk");
logit("START create delta of image2 from disk with imagezip");
if (zipimage("$delta.old", "$imagev1.sig", $devv2, $ssizev2)) {
logit("FAILED create delta of image2 from disk with imagezip");
unmakedisk($imagev2, $devv2) if ($cleanonfail);
unmakedisk($imagev1, $devv1) if ($cleanonfail);
return 1;
}
logit("END create delta of image2 from disk with imagezip");
#
# Hashmaps for delta images produced by imagezip are less than
# perfect as they can split a contiguous range in the new image if
# the corresponding range in the old image was not fully populated
# (and probably under other conditions too!)
#
if (-e "$delta.new") {
logit("START image sigfile compare for deltas");
# returns -1 if files are missing
my $rv = comparesigfiles("$delta.old", "$delta.new", 0);
if ($rv > 0) {