Scripts for crunching over large collections of images.

Work in progress.

clientside/os/imagezip/TESTS/checksha.pl

+#
+# Check SHA1 hash file against image.
+#
+
+my $hashall = 0;
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+
+#print "Found: ", join(' ', @files), "\n";
+
+my @images = ();
+foreach my $file (@files) {
+    # straight up ndz
+    if ($file =~ /\.ndz$/) {
+	push @images, $file;
+	next;
+    }
+
+    # versioned ndz
+    if ($file =~ /\.ndz:\d+$/) {
+	push @images, $file;
+	next;
+    }
+}
+
+print STDERR "Found ", int(@images), " images\n";
+foreach my $file (@images) {
+    print "$file: ";
+    if (! -e "$imagedir/$file.sha1") {
+	print "[FAIL] no signature!\n";
+	next;
+    }
+
+    # make sure it is the right format:
+    # SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
+    my $sha1 = `cat $imagedir/$file.sha1`;
+    chomp($sha1);
+    if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
+	$sha1 = $1;
+    } else {
+	print "[FAIL] bogus .sha1 file\n";
+	next;
+    }
+
+    my $nsha1 = `sha1 $imagedir/$file`;
+    chomp($nsha1);
+    if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
+	$nsha1 = $1;
+    } else {
+	print "[FAIL] did not correctly compute sha1!\n";
+	next;
+    }
+    
+    if ($sha1 eq $nsha1) {
+	print "[OK]\n";
+    } else {
+	print "[BAD]\n";
+    }
+}
+
+exit(0);

clientside/os/imagezip/TESTS/deltafy.pl

+#
+# Parse a directory of images looking for those that follow our versioning
+# convention. For those, start with the lowest numbered version and create
+# deltas for all but the first and last, e.g.:
+#
+# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
+# rm UBUNTU14-64-STD.ndz:1
+# ...
+# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
+# rm UBUNTU14-64-STD.ndz:9
+#
+
+my $frombase = 1;
+my $removefull = 0;
+
+my $IMAGEDELTA = "/tmp/imagedelta";
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my $tstamp = "deltafy." . time();
+if (!open(ST, ">$imagedir/$tstamp")) {
+    print STDERR "$imagedir: cannot write to directory\n";
+    exit(1);
+}
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+if (@files == 0) {
+    print STDERR "$imagedir: no images found\n";
+    unlink($tstamp);
+    exit(1);
+}
+
+my %filehash = map { ("$_" => 1) } @files;
+my %images = ();
+
+#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
+
+# Find all the base files
+foreach my $file (@files) {
+    if ($file =~ /^(.*)\.ndz$/) {
+	my $ibase = $1;
+	if (-l "$imagedir/$file") {
+	    print STDERR "$imagedir: ignoring symlink '$file'\n";
+	    delete $filehash{$file};
+	    delete $filehash{"$file.sig"};
+	    delete $filehash{"$file.sha1"};
+	    next;
+	}
+	$images{$ibase}{'name'} = $file;
+	$images{$ibase}{'lastvers'} = 0;
+	@{$images{$ibase}{'versions'}} = ();
+	delete $filehash{$file};
+    }
+}
+
+#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
+
+# Find all the versions
+foreach my $file (@files) {
+    next if (!exists($filehash{$file}));
+    if ($file =~ /^(.*).ndz:(\d+)$/) {
+	my ($ibase,$vers) = ($1,$2);
+	if (exists($images{$ibase})) {
+	    push @{$images{$ibase}{'versions'}}, $vers;
+	    if ($vers > $images{$ibase}{'lastvers'}) {
+		$images{$ibase}{'lastvers'} = $vers;
+	    }
+	    delete $filehash{$file};
+	    next;
+	}
+	print STDERR "*** version with no base '$file', ignoring\n";
+	delete $filehash{$file};
+	delete $filehash{"$file.sig"};
+	delete $filehash{"$file.sha1"};
+	next;
+    }
+}
+
+#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
+
+# Make sure all versions and signatures exist
+foreach my $ibase (keys %images) {
+    my $nukeit = 0;
+    my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
+    foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
+	my $fbase = "$ibase.ndz";
+	my $vstr = "";
+	if ($vers > 0) {
+	    $vstr = ":$vers";
+	    if (!exists($versions{$vers})) {
+		if ($frombase) {
+		    print STDERR "WARNING: ";
+		} else {
+		    print STDERR "*** ";
+		}
+		print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
+		if (!$frombase) {
+		    print STDERR ", ignoring\n";
+		    $nukeit = 1;
+		} else {
+		    print STDERR "\n";
+		    delete $filehash{"$fbase$vstr.sig"};
+		    delete $filehash{"$fbase$vstr.sha1"};
+		    next;
+		}
+	    } else {
+		delete $filehash{"$fbase$vstr"};
+	    }
+	}
+	# got sig?
+	if (!exists($filehash{"$fbase$vstr.sig"})) {
+	    if (!$frombase) {
+		print STDERR "*** no signature for $fbase$vstr, ignoring\n";
+		$nukeit = 1;
+	    } else {
+		print STDERR "WARNING: no signature for $fbase$vstr, ".
+		    "ignoring version\n";
+		delete $filehash{"$fbase$vstr.sha1"};
+		next;
+	    }
+	} else {
+	    delete $filehash{"$fbase$vstr.sig"};
+	}
+
+	# what about the hash?
+	delete $filehash{"$fbase$vstr.sha1"};
+    }
+    if ($nukeit) {
+	delete $images{$ibase};
+    }
+}
+
+# warn about unknown files
+if (scalar(keys %filehash) > 0) {
+    print STDERR "WARNING: unknown files:\n";
+}
+foreach my $file (sort keys %filehash) {
+    print STDERR "  $file\n";
+}
+
+# what do we have left
+foreach my $ibase (sort keys %images) {
+    my $lvers = $images{$ibase}{'lastvers'};
+    print STDERR "$ibase: image and $lvers versions\n";
+
+    foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
+	my $base = "$ibase.ndz";
+	if ($vers > 1 && !$frombase) {
+	    $base = "$ibase.ndz:" . ($vers-1);
+	}
+	my $this = "$ibase.ndz:$vers";
+	if ($frombase &&
+	    (! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
+	    print STDERR "$ibase: version $vers skipped because of missing files\n";
+	    next;
+	}
+	my $delta = "$ibase.ddz:$vers";
+	if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
+	    print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
+	}
+    }
+}
+
+exit(0);

clientside/os/imagezip/TESTS/hashem.pl

+#
+# Create signature files where missing. Check others.
+#
+
+my $hashall = 0;
+
+my $IMAGEHASH = "/tmp/imagehash";
+
+if (@ARGV < 1) {
+    print STDERR "Usage: $0 directory-of-images\n";
+    exit(1);
+}
+my $imagedir = $ARGV[0];
+if (! -d $imagedir) {
+    print STDERR "$imagedir: not a directory\n";
+    exit(1);
+}
+
+my $tstamp = "hashem." . time();
+if (!open(ST, ">$imagedir/$tstamp")) {
+    print STDERR "$imagedir: cannot write to directory\n";
+    exit(1);
+}
+
+my $clogfile = "$imagedir/check.log";
+my $glogfile = "$imagedir/generate.log";
+
+system("echo '' >$clogfile; date >>$clogfile");
+system("echo '' >$glogfile; date >>$glogfile");
+
+my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
+chomp @files;
+
+#print "Found: ", join(' ', @files), "\n";
+
+my @images = ();
+foreach my $file (@files) {
+    # no symlinks
+    next if (-l "$imagedir/$file");
+
+    # no sha1s
+    next if ($file =~ /\.sha1$/);
+
+    # straight up ndz
+    if ($file =~ /\.ndz$/) {
+	push @images, $file;
+	next;
+    }
+
+    # versioned ndz
+    if ($file =~ /\.ndz:\d+$/) {
+	push @images, $file;
+	next;
+    }
+}
+
+print STDERR "Found ", int(@images), " images\n";
+foreach my $file (@images) {
+    print "$file: ";
+    if (-e "$imagedir/$file.sig") {
+	print "found sig...";
+	if (checksig($file)) {
+	    print "[OK]\n";
+	    next;
+	}
+	print "[BAD]...re";
+    }
+    print "generating...";
+    if (gensig($file)) {
+	print "[OK]\n";
+	next;
+    }
+    print "[FAIL]\n";
+}
+
+exit(0);
+
+sub checksig($)
+{
+    my $file = shift;
+    
+    if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
+	return 0;
+    }
+    return 1;
+}
+
+sub gensig($)
+{
+    my $file = shift;
+    
+    if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
+	return 0;
+    }
+    return 1;
+}

clientside/os/imagezip/TESTS/testimage.pl

+#
+# Run an image I through its paces:
+#
+# 1. ensure image has a valid signature I.sig, make one if not
+# 2. create a memdisk and load image on it
+# 3. compare image on disk with signature I.sig (imagehash)
+# 4. recapture image with signature to I.new (imagezip)
+# 5. compare old and new signatures (imagehash)
+#
+my $TMPDIR = "/local/tmp";
+my $LOGDIR = "/local/logs";
+my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump");
+
+my $MAXSECTORS = (20 * 1024 * 1024 * 2);
+my $LVMSTRIPE = 6;
+
+my $checksig = 0;
+my $cleanonfail = 0;
+
+my $os = `uname`;
+chomp($os);
+if ($os !~ /^(Linux|FreeBSD)$/) {
+    die "Unknown OS '$os'\n";
+}
+my $arch = `uname -m`;
+chomp($arch);
+if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
+    die "Unknown arch '$arch'\n";
+}
+my $bindir = "/images/bin/${os}_${arch}";
+
+foreach my $bin (@NEEDBINS) {
+    if (! -x "$bindir/$bin") {
+	die "Cannot find $bindir/$bin\n";
+    }
+}
+
+if (@ARGV == 0) {
+    print STDERR "Usage: testimage.pl image1.ndz [ image2.ndz ... ]\n";
+    exit(1);
+}
+
+my $tstamp = time();
+print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
+
+my $rv = 0;
+foreach my $image (@ARGV) {
+    $rv += testimage($image);
+}
+exit($rv);
+
+sub testimage($)
+{
+    my $image = shift;
+
+    print "$image: START signature check\n";
+    if (sigcheck($image)) {
+	print "$image: ERROR signature check\n";
+	return 1;
+    }
+    print "$image: END signature check\n";
+
+    print "$image: START image unzip\n";
+    # create a memdisk for the image
+    my $ssize = imagesize($image);
+    my $dev = makedisk($image, $ssize);
+    if (!$dev) {
+	print "$image: ERROR image unzip\n";
+	return 1;
+    }
+    # and load it
+    if (mysystem("$bindir/imageunzip -f $image $dev")) {
+	print "$image: ERROR image unzip\n";
+	unmakedisk($image, $dev) if ($cleanonfail);
+	return 1;
+    }
+    print "$image: END image unzip\n";
+
+    print "$image: START loaded image verify\n";
+    if (mysystem("$bindir/imagehash -q $image $dev")) {
+	print "$image: ERROR image verify\n";
+	unmakedisk($image, $dev) if ($cleanonfail);
+	return 1;
+    }
+    print "$image: END loaded image verify\n";
+
+    #
+    # XXX gak! Without an MBR/GPT, imagezip cannot (yet) figure out
+    # what the filesystem is. We make a wild guess here based on image name.
+    #
+    my ($ifile,$itype);
+    if ($image =~ /([^\/]+)$/) {
+	$ifile = $1;
+    } else {
+	$ifile = $image;
+    }
+    if ($ifile =~ /\+/) {
+	# full image, don't need anything
+	$itype = "";
+    } elsif ($image =~ /FBSD/) {
+	# FreeBSD
+	$itype = "-S 165 -c $ssize";
+    } elsif ($ifile =~ /WIN/) {
+	# WIN XP or 7 are full disk images
+	$itype = "";
+    } else {
+	# assume Linux
+	$itype = "-S 131 -c $ssize";
+    }
+
+    my $nimage = "$image.new";
+    print "$image: START image rezip\n";
+    if (mysystem("$bindir/imagezip $itype -U $nimage.sig $dev $nimage")) {
+	print "$image: ERROR image rezip\n";
+	unmakedisk($image, $dev) if ($cleanonfail);
+	return 1;
+    }
+    print "$image: END image rezip\n";
+
+    print "$image: START image sigfile compare\n";
+    if (comparesigfiles($image, $nimage)) {
+	print "$image: ERROR image sigfile compare\n";
+	unmakedisk($image, $dev) if ($cleanonfail);
+	return 1;
+    }
+    print "$image: END image sigfile compare\n";
+
+    unmakedisk($image, $dev);
+    return 0;
+}
+
+sub comparesigfiles($$)
+{
+    my ($image1,$image2) = @_;
+
+    if (mysystem("$bindir/imagehash -Rq -o $image1.sig > $image1.sig.txt")) {
+	print "$image1: could not dump signature\n";
+	return 1;
+    }
+    if (mysystem("$bindir/imagehash -Rq -o $image2.sig > $image2.sig.txt")) {
+	print "$image2: could not dump signature\n";
+	return 1;
+    }
+    if (mysystem("diff -q $image1.sig.txt $image2.sig.txt")) {
+	print "*** signatures differ (diff $image1.sig.txt $image2.sig.txt)\n";
+	return 1;
+    }
+
+    unlink("$image1.sig.txt", "$image2.sig.txt");
+    return 0;
+}
+
+sub sigcheck($)
+{
+    my $image = shift;
+
+    if (! -e "$image") {
+	print STDERR "$image: does not exist\n";
+	return 1;
+    }
+    if (! -e "$image.sig") {
+	print STDERR "$image: signature does not exist\n";
+	return 1;
+    }
+
+    if ($checksig) {
+	if (mysystem("$bindir/imagehash -qSX $image")) {
+	    print "$image: signature did not check\n";
+	    return 1;
+	}
+	# gen a new format sig file and compare
+	if (mysystem("$bindir/imagehash -qcX -o ${image}foo.sig $image")) {
+	    print "$image: could not generate signature\n";
+	    return 1;
+	}
+	if (comparesigfiles($image, "${image}foo")) {
+	    print "$image: new signature does not match old\n";
+	    return 1;
+	}
+    }
+
+    return 0;
+}
+
+sub imagesize($)
+{
+    my $image = shift;
+
+    my @output = `$bindir/imagedump $image`;
+    if ($?) {
+	print "$image: *** could not get size of image\n";
+	return 0;
+    }
+    foreach my $line (@output) {
+	chomp($line);
+	if ($line =~ /covered sector range: \[(\d+)-(\d+)\]/) {
+	    if ($1 != 0) {
+		print "$image: WARNING: image does not start at 0\n";
+	    }
+	    $ssize = $2 + 1;
+	    last;
+	}
+    }
+
+    return $ssize;
+}
+
+sub makedisk($$)
+{
+    my ($image,$ssize) = @_;
+    my ($istr,$dev);
+
+    if ($image =~ /([^\/]+)\/([^\/]+)$/) {
+	$istr = "$1-$2";
+    } elsif ($image =~ /([^\/]+)$/) {
+	$istr = $1;
+    } else {
+	print "$image: *** could not parse '$image'\n";
+	return undef;
+    }
+
+    my $mb = int(($ssize + 2047) / 2048);
+    $mb += 100;
+
+    if ($os eq "Linux") {
+	if ($ssize > $MAXSECTORS) {
+	    print "$image: ERROR: image too large ($ssize) for ramdisk,".
+		" using LV instead\n";
+	    if (mysystem("lvcreate -i $LVMSTRIPE -L ${mb}m -n $istr emulab")) {
+		print STDERR "could not create LV\n";
+		return undef;
+	    }
+	    return "/dev/emulab/$istr";
+	}
+
+	# XXX there has to be a better way!
+	#
+	# mount -t tmpfs -o size=20580m tmpfs /mnt/FOO.ndz
+	# dd if=/dev/zero of=/mnt/FOO.ndz/disk bs=1024k seek=20479 count=1
+	# losetup -f
+	# losetup /dev/loop0 /mnt/FOO.ndz/disk
+	#
+	my $mountpoint = "/mnt/$istr";
+	if (!mkdir($mountpoint)) {
+	    print STDERR "could not make mountpoint $mountpoint\n";
+	    return undef;
+	}
+	if (mysystem("mount -t tmpfs -o size=${mb}m tmpfs $mountpoint")) {
+	    rmdir($mountpoint);
+	    return undef;
+	}
+	my $mbm1 = $mb - 1;
+	if (mysystem("dd if=/dev/zero of=$mountpoint/disk bs=1024k seek=$mbm1 count=1")) {
+	    mysystem("umount $mountpoint");
+	    rmdir($mountpoint);
+	    return undef;
+	}
+	$dev = `losetup -f`;
+	chomp($dev);
+	if (mysystem("losetup $dev $mountpoint/disk")) {
+	    mysystem("umount $mountpoint");
+	    rmdir($mountpoint);
+	    return undef;
+	}
+    } else {
+	print STDERR "Cannot do this under $os yet\n";
+    }
+
+    return $dev;
+}
+
+sub unmakedisk($$)
+{
+    my ($image,$dev) = @_;
+    my $istr;
+
+    if ($image =~ /([^\/]+)\/([^\/]+)$/) {
+	$istr = "$1-$2";
+    } elsif ($image =~ /([^\/]+)$/) {
+	$istr = $1;
+    } else {
+	print "$image: *** could not parse '$image'\n";
+	return undef;
+    }
+
+    if ($dev eq "/dev/emulab/$istr") {
+	if (mysystem("lvremove -f emulab/$istr")) {
+	    print STDERR "$image: could not destroy LV\n";
+	    return -1;
+	}
+    } elsif (mysystem("losetup -d $dev") ||
+	     mysystem("umount /mnt/$istr") ||
+	     !rmdir("/mnt/$istr")) {
+	print STDERR "$image: could not tear down ramdisk\n";
+	return -1;
+    }
+    return 0;
+}
+
+sub mysystem($)
+{
+    my ($cmd) = @_;
+    my $logfile = "$LOGDIR/testimage.$tstamp.log";
+    my $now = localtime();
+    my $redir;
+
+    if (open(FD, ">>$logfile")) {
+	print FD "==== $now: $cmd\n";
+	close(FD);
+    }
+
+    if ($cmd =~ />/) {
+	$redir = "2>>$logfile";
+    } else {
+	$redir = ">>$logfile 2>&1";
+    }
+    if (system("$cmd $redir")) {
+	my $stat = $?;
+	print STDERR "*** '$cmd' failed, see '$logfile'\n";
+	return $stat;
+    }
+
+    return 0;
+}

clientside/os/imagezip/TESTS/undeltafy.pl

+#
+# Parse a directory of images looking for those that follow our versioning
+# convention and for which the intermediate versions are deltas (identi