Commit ba3ab203 authored by Mike Hibler's avatar Mike Hibler

Scripts for crunching over large collections of images.

Work in progress.
parent a4f6e098
#
# Check SHA1 hash file against image.
#
my $hashall = 0;
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (! -e "$imagedir/$file.sha1") {
print "[FAIL] no signature!\n";
next;
}
# make sure it is the right format:
# SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
my $sha1 = `cat $imagedir/$file.sha1`;
chomp($sha1);
if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$sha1 = $1;
} else {
print "[FAIL] bogus .sha1 file\n";
next;
}
my $nsha1 = `sha1 $imagedir/$file`;
chomp($nsha1);
if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$nsha1 = $1;
} else {
print "[FAIL] did not correctly compute sha1!\n";
next;
}
if ($sha1 eq $nsha1) {
print "[OK]\n";
} else {
print "[BAD]\n";
}
}
exit(0);
#
# Parse a directory of images looking for those that follow our versioning
# convention. For those, start with the lowest numbered version and create
# deltas for all but the first and last, e.g.:
#
# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
# rm UBUNTU14-64-STD.ndz:1
# ...
# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
# rm UBUNTU14-64-STD.ndz:9
#
my $frombase = 1;
my $removefull = 0;
my $IMAGEDELTA = "/tmp/imagedelta";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "deltafy." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
if (@files == 0) {
print STDERR "$imagedir: no images found\n";
unlink($tstamp);
exit(1);
}
my %filehash = map { ("$_" => 1) } @files;
my %images = ();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the base files
foreach my $file (@files) {
if ($file =~ /^(.*)\.ndz$/) {
my $ibase = $1;
if (-l "$imagedir/$file") {
print STDERR "$imagedir: ignoring symlink '$file'\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
$images{$ibase}{'name'} = $file;
$images{$ibase}{'lastvers'} = 0;
@{$images{$ibase}{'versions'}} = ();
delete $filehash{$file};
}
}
#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
# Find all the versions
foreach my $file (@files) {
next if (!exists($filehash{$file}));
if ($file =~ /^(.*).ndz:(\d+)$/) {
my ($ibase,$vers) = ($1,$2);
if (exists($images{$ibase})) {
push @{$images{$ibase}{'versions'}}, $vers;
if ($vers > $images{$ibase}{'lastvers'}) {
$images{$ibase}{'lastvers'} = $vers;
}
delete $filehash{$file};
next;
}
print STDERR "*** version with no base '$file', ignoring\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all versions and signatures exist
foreach my $ibase (keys %images) {
my $nukeit = 0;
my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
my $fbase = "$ibase.ndz";
my $vstr = "";
if ($vers > 0) {
$vstr = ":$vers";
if (!exists($versions{$vers})) {
if ($frombase) {
print STDERR "WARNING: ";
} else {
print STDERR "*** ";
}
print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
if (!$frombase) {
print STDERR ", ignoring\n";
$nukeit = 1;
} else {
print STDERR "\n";
delete $filehash{"$fbase$vstr.sig"};
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr"};
}
}
# got sig?
if (!exists($filehash{"$fbase$vstr.sig"})) {
if (!$frombase) {
print STDERR "*** no signature for $fbase$vstr, ignoring\n";
$nukeit = 1;
} else {
print STDERR "WARNING: no signature for $fbase$vstr, ".
"ignoring version\n";
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr.sig"};
}
# what about the hash?
delete $filehash{"$fbase$vstr.sha1"};
}
if ($nukeit) {
delete $images{$ibase};
}
}
# warn about unknown files
if (scalar(keys %filehash) > 0) {
print STDERR "WARNING: unknown files:\n";
}
foreach my $file (sort keys %filehash) {
print STDERR " $file\n";
}
# what do we have left
foreach my $ibase (sort keys %images) {
my $lvers = $images{$ibase}{'lastvers'};
print STDERR "$ibase: image and $lvers versions\n";
foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
my $base = "$ibase.ndz";
if ($vers > 1 && !$frombase) {
$base = "$ibase.ndz:" . ($vers-1);
}
my $this = "$ibase.ndz:$vers";
if ($frombase &&
(! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
print STDERR "$ibase: version $vers skipped because of missing files\n";
next;
}
my $delta = "$ibase.ddz:$vers";
if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
}
}
}
exit(0);
#
# Create signature files where missing. Check others.
#
my $hashall = 0;
my $IMAGEHASH = "/tmp/imagehash";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "hashem." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my $clogfile = "$imagedir/check.log";
my $glogfile = "$imagedir/generate.log";
system("echo '' >$clogfile; date >>$clogfile");
system("echo '' >$glogfile; date >>$glogfile");
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# no symlinks
next if (-l "$imagedir/$file");
# no sha1s
next if ($file =~ /\.sha1$/);
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (-e "$imagedir/$file.sig") {
print "found sig...";
if (checksig($file)) {
print "[OK]\n";
next;
}
print "[BAD]...re";
}
print "generating...";
if (gensig($file)) {
print "[OK]\n";
next;
}
print "[FAIL]\n";
}
exit(0);
sub checksig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
return 0;
}
return 1;
}
sub gensig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
return 0;
}
return 1;
}
This diff is collapsed.
#
# Run an image I through its paces:
#
# 1. ensure image has a valid signature I.sig, make one if not
# 2. create a memdisk and load image on it
# 3. compare image on disk with signature I.sig (imagehash)
# 4. recapture image with signature to I.new (imagezip)
# 5. compare old and new signatures (imagehash)
#
my $TMPDIR = "/local/tmp";
my $LOGDIR = "/local/logs";
my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump");
my $MAXSECTORS = (20 * 1024 * 1024 * 2);
my $LVMSTRIPE = 6;
my $checksig = 0;
my $cleanonfail = 0;
my $os = `uname`;
chomp($os);
if ($os !~ /^(Linux|FreeBSD)$/) {
die "Unknown OS '$os'\n";
}
my $arch = `uname -m`;
chomp($arch);
if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
die "Unknown arch '$arch'\n";
}
my $bindir = "/images/bin/${os}_${arch}";
foreach my $bin (@NEEDBINS) {
if (! -x "$bindir/$bin") {
die "Cannot find $bindir/$bin\n";
}
}
if (@ARGV == 0) {
print STDERR "Usage: testimage.pl image1.ndz [ image2.ndz ... ]\n";
exit(1);
}
my $tstamp = time();
print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
my $rv = 0;
foreach my $image (@ARGV) {
$rv += testimage($image);
}
exit($rv);
sub testimage($)
{
my $image = shift;
print "$image: START signature check\n";
if (sigcheck($image)) {
print "$image: ERROR signature check\n";
return 1;
}
print "$image: END signature check\n";
print "$image: START image unzip\n";
# create a memdisk for the image
my $ssize = imagesize($image);
my $dev = makedisk($image, $ssize);
if (!$dev) {
print "$image: ERROR image unzip\n";
return 1;
}
# and load it
if (mysystem("$bindir/imageunzip -f $image $dev")) {
print "$image: ERROR image unzip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image unzip\n";
print "$image: START loaded image verify\n";
if (mysystem("$bindir/imagehash -q $image $dev")) {
print "$image: ERROR image verify\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END loaded image verify\n";
#
# XXX gak! Without an MBR/GPT, imagezip cannot (yet) figure out
# what the filesystem is. We make a wild guess here based on image name.
#
my ($ifile,$itype);
if ($image =~ /([^\/]+)$/) {
$ifile = $1;
} else {
$ifile = $image;
}
if ($ifile =~ /\+/) {
# full image, don't need anything
$itype = "";
} elsif ($image =~ /FBSD/) {
# FreeBSD
$itype = "-S 165 -c $ssize";
} elsif ($ifile =~ /WIN/) {
# WIN XP or 7 are full disk images
$itype = "";
} else {
# assume Linux
$itype = "-S 131 -c $ssize";
}
my $nimage = "$image.new";
print "$image: START image rezip\n";
if (mysystem("$bindir/imagezip $itype -U $nimage.sig $dev $nimage")) {
print "$image: ERROR image rezip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image rezip\n";
print "$image: START image sigfile compare\n";
if (comparesigfiles($image, $nimage)) {
print "$image: ERROR image sigfile compare\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image sigfile compare\n";
unmakedisk($image, $dev);
return 0;
}
sub comparesigfiles($$)
{
my ($image1,$image2) = @_;
if (mysystem("$bindir/imagehash -Rq -o $image1.sig > $image1.sig.txt")) {
print "$image1: could not dump signature\n";
return 1;
}
if (mysystem("$bindir/imagehash -Rq -o $image2.sig > $image2.sig.txt")) {
print "$image2: could not dump signature\n";
return 1;
}
if (mysystem("diff -q $image1.sig.txt $image2.sig.txt")) {
print "*** signatures differ (diff $image1.sig.txt $image2.sig.txt)\n";
return 1;
}
unlink("$image1.sig.txt", "$image2.sig.txt");
return 0;
}
sub sigcheck($)
{
my $image = shift;
if (! -e "$image") {
print STDERR "$image: does not exist\n";
return 1;
}
if (! -e "$image.sig") {
print STDERR "$image: signature does not exist\n";
return 1;
}
if ($checksig) {
if (mysystem("$bindir/imagehash -qSX $image")) {
print "$image: signature did not check\n";
return 1;
}
# gen a new format sig file and compare
if (mysystem("$bindir/imagehash -qcX -o ${image}foo.sig $image")) {
print "$image: could not generate signature\n";
return 1;
}
if (comparesigfiles($image, "${image}foo")) {
print "$image: new signature does not match old\n";
return 1;
}
}
return 0;
}
sub imagesize($)
{
my $image = shift;
my @output = `$bindir/imagedump $image`;
if ($?) {
print "$image: *** could not get size of image\n";
return 0;
}
foreach my $line (@output) {
chomp($line);
if ($line =~ /covered sector range: \[(\d+)-(\d+)\]/) {
if ($1 != 0) {
print "$image: WARNING: image does not start at 0\n";
}
$ssize = $2 + 1;
last;
}
}
return $ssize;
}
sub makedisk($$)
{
my ($image,$ssize) = @_;
my ($istr,$dev);
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
my $mb = int(($ssize + 2047) / 2048);
$mb += 100;
if ($os eq "Linux") {
if ($ssize > $MAXSECTORS) {
print "$image: ERROR: image too large ($ssize) for ramdisk,".
" using LV instead\n";
if (mysystem("lvcreate -i $LVMSTRIPE -L ${mb}m -n $istr emulab")) {
print STDERR "could not create LV\n";
return undef;
}
return "/dev/emulab/$istr";
}
# XXX there has to be a better way!
#
# mount -t tmpfs -o size=20580m tmpfs /mnt/FOO.ndz
# dd if=/dev/zero of=/mnt/FOO.ndz/disk bs=1024k seek=20479 count=1
# losetup -f
# losetup /dev/loop0 /mnt/FOO.ndz/disk
#
my $mountpoint = "/mnt/$istr";
if (!mkdir($mountpoint)) {
print STDERR "could not make mountpoint $mountpoint\n";
return undef;
}
if (mysystem("mount -t tmpfs -o size=${mb}m tmpfs $mountpoint")) {
rmdir($mountpoint);
return undef;
}
my $mbm1 = $mb - 1;
if (mysystem("dd if=/dev/zero of=$mountpoint/disk bs=1024k seek=$mbm1 count=1")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
$dev = `losetup -f`;
chomp($dev);
if (mysystem("losetup $dev $mountpoint/disk")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
} else {
print STDERR "Cannot do this under $os yet\n";
}
return $dev;
}
sub unmakedisk($$)
{
my ($image,$dev) = @_;
my $istr;
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
if ($dev eq "/dev/emulab/$istr") {
if (mysystem("lvremove -f emulab/$istr")) {
print STDERR "$image: could not destroy LV\n";
return -1;
}
} elsif (mysystem("losetup -d $dev") ||
mysystem("umount /mnt/$istr") ||
!rmdir("/mnt/$istr")) {
print STDERR "$image: could not tear down ramdisk\n";
return -1;
}
return 0;
}
sub mysystem($)
{
my ($cmd) = @_;
my $logfile = "$LOGDIR/testimage.$tstamp.log";
my $now = localtime();
my $redir;
if (open(FD, ">>$logfile")) {
print FD "==== $now: $cmd\n";
close(FD);
}
if ($cmd =~ />/) {
$redir = "2>>$logfile";
} else {
$redir = ">>$logfile 2>&1";
}
if (system("$cmd $redir")) {
my $stat = $?;
print STDERR "*** '$cmd' failed, see '$logfile'\n";
return $stat;
}
return 0;
}
#
# Parse a directory of images looking for those that follow our versioning
# convention and for which the intermediate versions are deltas (identi