Scripts for crunching over large collections of images.

Work in progress.

clientside/os/imagezip/TESTS/checksha.pl

+#
+# Check SHA1 hash file against image.
+#
+
+my $hashall = 0;
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+
+#print "Found: ", join(' ', @files), "\n";
+
+my @images = ();
+foreach my $file (@files) {
+    # straight up ndz
+    if ($file =~ /\.ndz$/) {
+	push @images, $file;
+	next;
+    }
+
+    # versioned ndz
+    if ($file =~ /\.ndz:\d+$/) {
+	push @images, $file;
+	next;
+    }
+}
+
+print STDERR "Found ", int(@images), " images\n";
+foreach my $file (@images) {
+    print "$file: ";
+    if (! -e "$imagedir/$file.sha1") {
+	print "[FAIL] no signature!\n";
+	next;
+    }
+
+    # make sure it is the right format:
+    # SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
+    my $sha1 = `cat $imagedir/$file.sha1`;
+    chomp($sha1);
+    if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
+	$sha1 = $1;
+    } else {
+	print "[FAIL] bogus .sha1 file\n";
+	next;
+    }
+
+    my $nsha1 = `sha1 $imagedir/$file`;
+    chomp($nsha1);
+    if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
+	$nsha1 = $1;
+    } else {
+	print "[FAIL] did not correctly compute sha1!\n";
+	next;
+    }
+    
+    if ($sha1 eq $nsha1) {
+	print "[OK]\n";
+    } else {
+	print "[BAD]\n";
+    }
+}
+
+exit(0);

clientside/os/imagezip/TESTS/deltafy.pl

+#
+# Parse a directory of images looking for those that follow our versioning
+# convention. For those, start with the lowest numbered version and create
+# deltas for all but the first and last, e.g.:
+#
+# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
+# rm UBUNTU14-64-STD.ndz:1
+# ...
+# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
+# rm UBUNTU14-64-STD.ndz:9
+#
+
+my $frombase = 1;
+my $removefull = 0;
+
+my $IMAGEDELTA = "/tmp/imagedelta";
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my $tstamp = "deltafy." . time();
+if (!open(ST, ">$imagedir/$tstamp")) {
+    print STDERR "$imagedir: cannot write to directory\n";
+    exit(1);
+}
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+if (@files == 0) {
+    print STDERR "$imagedir: no images found\n";
+    unlink($tstamp);
+    exit(1);
+}
+
+my %filehash = map { ("$_" => 1) } @files;
+my %images = ();
+
+#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
+
+# Find all the base files
+foreach my $file (@files) {
+    if ($file =~ /^(.*)\.ndz$/) {
+	my $ibase = $1;
+	if (-l "$imagedir/$file") {
+	    print STDERR "$imagedir: ignoring symlink '$file'\n";
+	    delete $filehash{$file};
+	    delete $filehash{"$file.sig"};
+	    delete $filehash{"$file.sha1"};
+	    next;
+	}
+	$images{$ibase}{'name'} = $file;
+	$images{$ibase}{'lastvers'} = 0;
+	@{$images{$ibase}{'versions'}} = ();
+	delete $filehash{$file};
+    }
+}
+
+#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
+
+# Find all the versions
+foreach my $file (@files) {
+    next if (!exists($filehash{$file}));
+    if ($file =~ /^(.*).ndz:(\d+)$/) {
+	my ($ibase,$vers) = ($1,$2);
+	if (exists($images{$ibase})) {
+	    push @{$images{$ibase}{'versions'}}, $vers;
+	    if ($vers > $images{$ibase}{'lastvers'}) {
+		$images{$ibase}{'lastvers'} = $vers;
+	    }
+	    delete $filehash{$file};
+	    next;
+	}
+	print STDERR "*** version with no base '$file', ignoring\n";
+	delete $filehash{$file};
+	delete $filehash{"$file.sig"};
+	delete $filehash{"$file.sha1"};
+	next;
+    }
+}
+
+#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
+
+# Make sure all versions and signatures exist
+foreach my $ibase (keys %images) {
+    my $nukeit = 0;
+    my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
+    foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
+	my $fbase = "$ibase.ndz";
+	my $vstr = "";
+	if ($vers > 0) {
+	    $vstr = ":$vers";
+	    if (!exists($versions{$vers})) {
+		if ($frombase) {
+		    print STDERR "WARNING: ";
+		} else {
+		    print STDERR "*** ";
+		}
+		print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
+		if (!$frombase) {
+		    print STDERR ", ignoring\n";
+		    $nukeit = 1;
+		} else {
+		    print STDERR "\n";
+		    delete $filehash{"$fbase$vstr.sig"};
+		    delete $filehash{"$fbase$vstr.sha1"};
+		    next;
+		}
+	    } else {
+		delete $filehash{"$fbase$vstr"};
+	    }
+	}
+	# got sig?
+	if (!exists($filehash{"$fbase$vstr.sig"})) {
+	    if (!$frombase) {
+		print STDERR "*** no signature for $fbase$vstr, ignoring\n";
+		$nukeit = 1;
+	    } else {
+		print STDERR "WARNING: no signature for $fbase$vstr, ".
+		    "ignoring version\n";
+		delete $filehash{"$fbase$vstr.sha1"};
+		next;
+	    }
+	} else {
+	    delete $filehash{"$fbase$vstr.sig"};
+	}
+
+	# what about the hash?
+	delete $filehash{"$fbase$vstr.sha1"};
+    }
+    if ($nukeit) {
+	delete $images{$ibase};
+    }
+}
+
+# warn about unknown files
+if (scalar(keys %filehash) > 0) {
+    print STDERR "WARNING: unknown files:\n";
+}
+foreach my $file (sort keys %filehash) {
+    print STDERR "  $file\n";
+}
+
+# what do we have left
+foreach my $ibase (sort keys %images) {
+    my $lvers = $images{$ibase}{'lastvers'};
+    print STDERR "$ibase: image and $lvers versions\n";
+
+    foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
+	my $base = "$ibase.ndz";
+	if ($vers > 1 && !$frombase) {
+	    $base = "$ibase.ndz:" . ($vers-1);
+	}
+	my $this = "$ibase.ndz:$vers";
+	if ($frombase &&
+	    (! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
+	    print STDERR "$ibase: version $vers skipped because of missing files\n";
+	    next;
+	}
+	my $delta = "$ibase.ddz:$vers";
+	if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
+	    print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
+	}
+    }
+}
+
+exit(0);

clientside/os/imagezip/TESTS/hashem.pl

+#
+# Create signature files where missing. Check others.
+#
+
+my $hashall = 0;
+
+my $IMAGEHASH = "/tmp/imagehash";
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my $tstamp = "hashem." . time();
+if (!open(ST, ">$imagedir/$tstamp")) {
+    print STDERR "$imagedir: cannot write to directory\n";
+    exit(1);
+}
+
+my $clogfile = "$imagedir/check.log";
+my $glogfile = "$imagedir/generate.log";
+
+system("echo '' >$clogfile; date >>$clogfile");
+system("echo '' >$glogfile; date >>$glogfile");
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+
+#print "Found: ", join(' ', @files), "\n";
+
+my @images = ();
+foreach my $file (@files) {
+    # no symlinks
+    next if (-l "$imagedir/$file");
+
+    # no sha1s
+    next if ($file =~ /\.sha1$/);
+
+    # straight up ndz
+    if ($file =~ /\.ndz$/) {
+	push @images, $file;
+	next;
+    }
+
+    # versioned ndz
+    if ($file =~ /\.ndz:\d+$/) {
+	push @images, $file;
+	next;
+    }
+}
+
+print STDERR "Found ", int(@images), " images\n";
+foreach my $file (@images) {
+    print "$file: ";
+    if (-e "$imagedir/$file.sig") {
+	print "found sig...";
+	if (checksig($file)) {
+	    print "[OK]\n";
+	    next;
+	}
+	print "[BAD]...re";
+    }
+    print "generating...";
+    if (gensig($file)) {
+	print "[OK]\n";
+	next;
+    }
+    print "[FAIL]\n";
+}
+
+exit(0);
+
+sub checksig($)
+{
+    my $file = shift;
+    
+    if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
+	return 0;
+    }
+    return 1;
+}
+
+sub gensig($)
+{
+    my $file = shift;
+    
+    if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
+	return 0;
+    }
+    return 1;
+}

clientside/os/imagezip/TESTS/testdelta.pl

+#
+# Given two full images I1 and I2:
+#
+# I. Basic:
+#
+# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
+# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
+# 3. compare sigs of I2.ndz and I2.ndz.v2
+#
+# What we learn: imagedelta/undelta don't crash on the image in question,
+# delta/undelta are correct at the level of the signature file.
+#
+# II. More advanced:
+#
+# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
+# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
+# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
+# 7. compare sigs of I2.ddz I2.ddz.v2
+# 7a. if sigs don't compare [*], we compate I2.ddz.v2 against the I2 disk
+#
+# What we learn: imagezip and imagedelta delta creation are consistent.
+#
+# III. Over the top:
+#
+# 8.  apply delta I2.ddz.v2 to I1 disk
+# 9.  compare contents of I1 and I2 disks [2]
+# 10. load full I2.ndz.v2 image on I3 disk
+# 11. compare contents of I2 and I3 disks [3]
+#
+# What we learn: new format files can be loaded (imagezip), actual on-disk
+# contents are correct.
+#
+# Footnotes:
+#
+# [1] Since delta signatures generated by imagezip are imperfect,
+#     the sigs may not match. To ensure their equivalence, we assume
+#     the imagezip sig is correct and verify the other by checking it
+#     against the on-disk image.
+#
+# [2] We cannot just "cmp" the two disks due to inexactness of the
+#     delta created by imagedelta. It may include unused blocks that
+#     will get propogate back to disk. Hence, we create new image files
+#     from both disks using imagezip--with "-F 0" to ensure no free sects
+#     are included--and byte compare those image files.
+#
+# [3] As with imagedelta, imageundelta is inexact and may have included
+#     garbage sectors from the original image. We perform the same
+#     comparison as in [2] instead.
+#
+my $TMPDIR = "/local/tmp";
+my $LOGDIR = "/local/logs";
+my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump",
+    "imagedelta", "imageundelta");
+
+my $MAXPHASE = 2;
+
+my $MAXSECTORS = (20 * 1024 * 1024 * 2);
+my $LVMSTRIPE = 6;
+
+my $checksig = 0;
+my $cleanonfail = 0;
+
+my $os = `uname`;
+chomp($os);
+if ($os !~ /^(Linux|FreeBSD)$/) {
+    die "Unknown OS '$os'\n";
+}
+my $arch = `uname -m`;
+chomp($arch);
+if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
+    die "Unknown arch '$arch'\n";
+}
+my $bindir = "/images/bin/${os}_${arch}";
+
+foreach my $bin (@NEEDBINS) {
+    if (! -x "$bindir/$bin") {
+	die "Cannot find $bindir/$bin\n";
+    }
+}
+
+if (@ARGV != 2) {
+    print STDERR "Usage: testdelta.pl imageV1.ndz imageV2.ndz\n";
+    exit(1);
+}
+
+my $uid = $>;
+if ($uid == 0) {
+    print STDERR "Do not run as root, script will sudo when necessary\n";
+    exit(1);
+}
+
+my $tstamp = time();
+print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
+
+my %image1 = ( "name" => $ARGV[0] );
+my %image2 = ( "name" => $ARGV[1] );
+
+my $rv = 0;
+$rv = phase1(\%image1, \%image2)
+    if ($MAXPHASE > 0);
+$rv = phase2(\%image1, \%image2)
+    if ($rv == 0 && $MAXPHASE > 1);
+$rv = phase3(\%image1, \%image2)
+    if ($rv == 0 && $MAXPHASE > 2);
+
+if ($rv == 0) {
+    unmakedisk($image1{'name'}, $image1{'disk'});
+    unmakedisk($image2{'name'}, $image2{'disk'});
+}
+
+exit($rv);
+
+#
+# I. Basic:
+#
+# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
+# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
+# 3. compare sigs of I2.ndz and I2.ndz.v2
+#
+sub phase1($$)
+{
+    my ($image1,$image2) = @_;
+    my $fixedsig = 0;
+
+    my $imagev1 = $image1->{'name'};
+    my $imagev2 = $image2->{'name'};
+
+    logit("START signature check of both images");
+    if (sigcheck($imagev1) || sigcheck($imagev2)) {
+	logit("FAILED signature check of both images");
+	return 1;
+    }
+    logit("END signature check of both images");
+
+    my $delta = $imagev2;
+    $delta =~ s/ndz/ddz/;
+    if ($imagev2 eq $delta) {
+	logit("  $imagev2: malformed name, no .ndz");
+	return 1;
+    }
+
+    logit("START image2 delta creation");
+again:
+    if (mysystem("$bindir/imagedelta -SVF $imagev1 $imagev2 $delta.new", 0)) {
+	logit("  WARNING: could not generate delta image");
+	if (!$fixedsig) {
+	    logit("  recreating signature for $imagev2...");
+	    mysystem("mv $imagev2.sig $imagev2.bak.sig");
+	    if (!mysystem("$bindir/imagehash -qcX $imagev2")) {
+		logit("  validating new signature for $imagev2...");
+		if (!comparesigfiles("$imagev2.bak", $imagev2, 0)) {
+		    unlink("$imagev2.bak.sig");
+		    logit("RESTART image2 delta creation");
+		    $fixedsig = 1;
+		    goto again;
+		}
+		unlink("$imagev2.sig");
+	    }
+	    mysystem("mv $imagev2.bak.sig $imagev2.sig");
+	}
+	logit("FAILED image2 delta creation");
+	return 1;
+    }
+    logit("END image2 delta creation");
+
+    logit("START undelta to image2.new");
+    if (mysystem("$bindir/imageundelta -SV $imagev1 $delta.new $imagev2.new")) {
+	logit("FAILED undelta to image2.new");
+	return 1;
+    }
+    logit("END undelta to image2.new");
+
+    logit("START signature compare of image2 and image2.new");
+    if (comparesigfiles($imagev2, "$imagev2.new", 1)) {
+	logit("  $imagev2.new: signature mismatch with $imagev2");
+	logit("FAILED signature compare of image2 and image2.new");
+	return 1;
+    }
+    logit("END signature compare of image2 and image2.new");
+
+    return 0;
+}
+
+#
+# II. More advanced:
+#
+# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
+# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
+# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
+# 7. compare sigs of I2.ddz I2.ddz.v2
+#
+sub phase2($$)
+{
+    my ($image1,$image2) = @_;
+
+    my $imagev1 = $image1->{'name'};
+    my $imagev2 = $image2->{'name'};
+
+    my $delta = $imagev2;
+    $delta =~ s/ndz/ddz/;
+    if ($imagev2 eq $delta) {
+	logit("  $imagev2: malformed name, no .ndz");
+	return 1;
+    }
+
+    logit("START unzip of image1");
+    my ($devv1,$ssizev1) = unzipimage($imagev1);
+    if (!$devv1) {
+	logit("FAILED unzip of image1");
+	return 1;
+    }
+    logit("END unzip of image1");
+
+    logit("START signature check of image1 on disk");
+    if ($checksig) {
+	if (mysystem("$bindir/imagehash -qX $imagev1 $devv1")) {
+	    logit("FAILED signature check of image1 on disk");
+	    unmakedisk($imagev1, $devv1) if ($cleanonfail);
+	    return 1;
+	}
+    }
+    logit("END signature check of image1 on disk");
+
+    logit("START unzip of image2");
+    my ($devv2,$ssizev2) = unzipimage($imagev2);
+    if (!$devv2) {
+	logit("FAILED unzip of image2");
+	unmakedisk($imagev1, $devv1) if ($cleanonfail);
+	return 1;
+    }
+    logit("END unzip of image2");
+
+    logit("START signature check of image2 on disk");
+    if ($checksig) {
+	if (mysystem("$bindir/imagehash -qX $imagev2 $devv2")) {
+	    logit("FAILED signature check of image2 on disk");
+	    unmakedisk($imagev2, $devv2) if ($cleanonfail);
+	    unmakedisk($imagev1, $devv1) if ($cleanonfail);
+	    return 1;
+	}
+    }
+    logit("END signature check of image2 on disk");
+
+    logit("START create delta of image2 from disk with imagezip");
+    if (zipimage("$delta.old", "$imagev1.sig", $devv2, $ssizev2)) {
+	logit("FAILED create delta of image2 from disk with imagezip");
+	unmakedisk($imagev2, $devv2) if ($cleanonfail);
+	unmakedisk($imagev1, $devv1) if ($cleanonfail);
+	return 1;
+    }
+    logit("END create delta of image2 from disk with imagezip");
+
+    #
+    # Hashmaps for delta images produced by imagezip are less than
+    # perfect as they can split a contiguous range in the new image if
+    # the corresponding range in the old image was not fully populated
+    # (and probably under other conditions too!)
+    #
+    if (-e "$delta.new") {
+	logit("START image sigfile compare for deltas");
+	# returns -1 if files are missing
+	my $rv = comparesigfiles("$delta.old", "$delta.new", 0);
+	if ($rv > 0) {