All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit ba3ab203 authored by Mike Hibler's avatar Mike Hibler

Scripts for crunching over large collections of images.

Work in progress.
parent a4f6e098
#
# Check SHA1 hash file against image.
#
my $hashall = 0;
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (! -e "$imagedir/$file.sha1") {
print "[FAIL] no signature!\n";
next;
}
# make sure it is the right format:
# SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
my $sha1 = `cat $imagedir/$file.sha1`;
chomp($sha1);
if ($sha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$sha1 = $1;
} else {
print "[FAIL] bogus .sha1 file\n";
next;
}
my $nsha1 = `sha1 $imagedir/$file`;
chomp($nsha1);
if ($nsha1 =~ /^SHA1 .* = ([0-9a-f]{40})$/) {
$nsha1 = $1;
} else {
print "[FAIL] did not correctly compute sha1!\n";
next;
}
if ($sha1 eq $nsha1) {
print "[OK]\n";
} else {
print "[BAD]\n";
}
}
exit(0);
#
# Parse a directory of images looking for those that follow our versioning
# convention. For those, start with the lowest numbered version and create
# deltas for all but the first and last, e.g.:
#
# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
# rm UBUNTU14-64-STD.ndz:1
# ...
# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
# rm UBUNTU14-64-STD.ndz:9
#
my $frombase = 1;
my $removefull = 0;
my $IMAGEDELTA = "/tmp/imagedelta";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "deltafy." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
if (@files == 0) {
print STDERR "$imagedir: no images found\n";
unlink($tstamp);
exit(1);
}
my %filehash = map { ("$_" => 1) } @files;
my %images = ();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the base files
foreach my $file (@files) {
if ($file =~ /^(.*)\.ndz$/) {
my $ibase = $1;
if (-l "$imagedir/$file") {
print STDERR "$imagedir: ignoring symlink '$file'\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
$images{$ibase}{'name'} = $file;
$images{$ibase}{'lastvers'} = 0;
@{$images{$ibase}{'versions'}} = ();
delete $filehash{$file};
}
}
#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
# Find all the versions
foreach my $file (@files) {
next if (!exists($filehash{$file}));
if ($file =~ /^(.*).ndz:(\d+)$/) {
my ($ibase,$vers) = ($1,$2);
if (exists($images{$ibase})) {
push @{$images{$ibase}{'versions'}}, $vers;
if ($vers > $images{$ibase}{'lastvers'}) {
$images{$ibase}{'lastvers'} = $vers;
}
delete $filehash{$file};
next;
}
print STDERR "*** version with no base '$file', ignoring\n";
delete $filehash{$file};
delete $filehash{"$file.sig"};
delete $filehash{"$file.sha1"};
next;
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all versions and signatures exist
foreach my $ibase (keys %images) {
my $nukeit = 0;
my %versions = map { ($_ => 1) } @{$images{$ibase}{'versions'}};
foreach my $vers (0 .. $images{$ibase}{'lastvers'}) {
my $fbase = "$ibase.ndz";
my $vstr = "";
if ($vers > 0) {
$vstr = ":$vers";
if (!exists($versions{$vers})) {
if ($frombase) {
print STDERR "WARNING: ";
} else {
print STDERR "*** ";
}
print STDERR "no version $vers of '$ibase' ('$fbase$vstr')";
if (!$frombase) {
print STDERR ", ignoring\n";
$nukeit = 1;
} else {
print STDERR "\n";
delete $filehash{"$fbase$vstr.sig"};
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr"};
}
}
# got sig?
if (!exists($filehash{"$fbase$vstr.sig"})) {
if (!$frombase) {
print STDERR "*** no signature for $fbase$vstr, ignoring\n";
$nukeit = 1;
} else {
print STDERR "WARNING: no signature for $fbase$vstr, ".
"ignoring version\n";
delete $filehash{"$fbase$vstr.sha1"};
next;
}
} else {
delete $filehash{"$fbase$vstr.sig"};
}
# what about the hash?
delete $filehash{"$fbase$vstr.sha1"};
}
if ($nukeit) {
delete $images{$ibase};
}
}
# warn about unknown files
if (scalar(keys %filehash) > 0) {
print STDERR "WARNING: unknown files:\n";
}
foreach my $file (sort keys %filehash) {
print STDERR " $file\n";
}
# what do we have left
foreach my $ibase (sort keys %images) {
my $lvers = $images{$ibase}{'lastvers'};
print STDERR "$ibase: image and $lvers versions\n";
foreach my $vers (1 .. $images{$ibase}{'lastvers'}) {
my $base = "$ibase.ndz";
if ($vers > 1 && !$frombase) {
$base = "$ibase.ndz:" . ($vers-1);
}
my $this = "$ibase.ndz:$vers";
if ($frombase &&
(! -e "$imagedir/$this" || ! -e "$imagedir/$this.sig")) {
print STDERR "$ibase: version $vers skipped because of missing files\n";
next;
}
my $delta = "$ibase.ddz:$vers";
if (system("$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta\n")) {
print STDERR "*** '$IMAGEDELTA -SF $imagedir/$base $imagedir/$this $imagedir/$delta' failed!\n";
}
}
}
exit(0);
#
# Create signature files where missing. Check others.
#
my $hashall = 0;
my $IMAGEHASH = "/tmp/imagehash";
if (@ARGV < 1) {
print STDERR "Usage: $0 directory-of-images\n";
exit(1);
}
my $imagedir = $ARGV[0];
if (! -d $imagedir) {
print STDERR "$imagedir: not a directory\n";
exit(1);
}
my $tstamp = "hashem." . time();
if (!open(ST, ">$imagedir/$tstamp")) {
print STDERR "$imagedir: cannot write to directory\n";
exit(1);
}
my $clogfile = "$imagedir/check.log";
my $glogfile = "$imagedir/generate.log";
system("echo '' >$clogfile; date >>$clogfile");
system("echo '' >$glogfile; date >>$glogfile");
my @files = `cd $imagedir; /bin/ls -1 *.ndz*`;
chomp @files;
#print "Found: ", join(' ', @files), "\n";
my @images = ();
foreach my $file (@files) {
# no symlinks
next if (-l "$imagedir/$file");
# no sha1s
next if ($file =~ /\.sha1$/);
# straight up ndz
if ($file =~ /\.ndz$/) {
push @images, $file;
next;
}
# versioned ndz
if ($file =~ /\.ndz:\d+$/) {
push @images, $file;
next;
}
}
print STDERR "Found ", int(@images), " images\n";
foreach my $file (@images) {
print "$file: ";
if (-e "$imagedir/$file.sig") {
print "found sig...";
if (checksig($file)) {
print "[OK]\n";
next;
}
print "[BAD]...re";
}
print "generating...";
if (gensig($file)) {
print "[OK]\n";
next;
}
print "[FAIL]\n";
}
exit(0);
sub checksig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -SX $imagedir/$file) >>$clogfile 2>&1")) {
return 0;
}
return 1;
}
sub gensig($)
{
my $file = shift;
if (system("(echo $file; $IMAGEHASH -cX $imagedir/$file) >>$glogfile 2>&1")) {
return 0;
}
return 1;
}
This diff is collapsed.
#
# Run an image I through its paces:
#
# 1. ensure image has a valid signature I.sig, make one if not
# 2. create a memdisk and load image on it
# 3. compare image on disk with signature I.sig (imagehash)
# 4. recapture image with signature to I.new (imagezip)
# 5. compare old and new signatures (imagehash)
#
my $TMPDIR = "/local/tmp";
my $LOGDIR = "/local/logs";
my @NEEDBINS = ("imagezip", "imageunzip", "imagehash", "imagedump");
my $MAXSECTORS = (20 * 1024 * 1024 * 2);
my $LVMSTRIPE = 6;
my $checksig = 0;
my $cleanonfail = 0;
my $os = `uname`;
chomp($os);
if ($os !~ /^(Linux|FreeBSD)$/) {
die "Unknown OS '$os'\n";
}
my $arch = `uname -m`;
chomp($arch);
if ($arch !~ /^(x86_32|x86_64|aarch64|i386|amd64)$/) {
die "Unknown arch '$arch'\n";
}
my $bindir = "/images/bin/${os}_${arch}";
foreach my $bin (@NEEDBINS) {
if (! -x "$bindir/$bin") {
die "Cannot find $bindir/$bin\n";
}
}
if (@ARGV == 0) {
print STDERR "Usage: testimage.pl image1.ndz [ image2.ndz ... ]\n";
exit(1);
}
my $tstamp = time();
print "Logs will be $LOGDIR/$tstamp.*.log ...\n";
my $rv = 0;
foreach my $image (@ARGV) {
$rv += testimage($image);
}
exit($rv);
sub testimage($)
{
my $image = shift;
print "$image: START signature check\n";
if (sigcheck($image)) {
print "$image: ERROR signature check\n";
return 1;
}
print "$image: END signature check\n";
print "$image: START image unzip\n";
# create a memdisk for the image
my $ssize = imagesize($image);
my $dev = makedisk($image, $ssize);
if (!$dev) {
print "$image: ERROR image unzip\n";
return 1;
}
# and load it
if (mysystem("$bindir/imageunzip -f $image $dev")) {
print "$image: ERROR image unzip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image unzip\n";
print "$image: START loaded image verify\n";
if (mysystem("$bindir/imagehash -q $image $dev")) {
print "$image: ERROR image verify\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END loaded image verify\n";
#
# XXX gak! Without an MBR/GPT, imagezip cannot (yet) figure out
# what the filesystem is. We make a wild guess here based on image name.
#
my ($ifile,$itype);
if ($image =~ /([^\/]+)$/) {
$ifile = $1;
} else {
$ifile = $image;
}
if ($ifile =~ /\+/) {
# full image, don't need anything
$itype = "";
} elsif ($image =~ /FBSD/) {
# FreeBSD
$itype = "-S 165 -c $ssize";
} elsif ($ifile =~ /WIN/) {
# WIN XP or 7 are full disk images
$itype = "";
} else {
# assume Linux
$itype = "-S 131 -c $ssize";
}
my $nimage = "$image.new";
print "$image: START image rezip\n";
if (mysystem("$bindir/imagezip $itype -U $nimage.sig $dev $nimage")) {
print "$image: ERROR image rezip\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image rezip\n";
print "$image: START image sigfile compare\n";
if (comparesigfiles($image, $nimage)) {
print "$image: ERROR image sigfile compare\n";
unmakedisk($image, $dev) if ($cleanonfail);
return 1;
}
print "$image: END image sigfile compare\n";
unmakedisk($image, $dev);
return 0;
}
sub comparesigfiles($$)
{
my ($image1,$image2) = @_;
if (mysystem("$bindir/imagehash -Rq -o $image1.sig > $image1.sig.txt")) {
print "$image1: could not dump signature\n";
return 1;
}
if (mysystem("$bindir/imagehash -Rq -o $image2.sig > $image2.sig.txt")) {
print "$image2: could not dump signature\n";
return 1;
}
if (mysystem("diff -q $image1.sig.txt $image2.sig.txt")) {
print "*** signatures differ (diff $image1.sig.txt $image2.sig.txt)\n";
return 1;
}
unlink("$image1.sig.txt", "$image2.sig.txt");
return 0;
}
sub sigcheck($)
{
my $image = shift;
if (! -e "$image") {
print STDERR "$image: does not exist\n";
return 1;
}
if (! -e "$image.sig") {
print STDERR "$image: signature does not exist\n";
return 1;
}
if ($checksig) {
if (mysystem("$bindir/imagehash -qSX $image")) {
print "$image: signature did not check\n";
return 1;
}
# gen a new format sig file and compare
if (mysystem("$bindir/imagehash -qcX -o ${image}foo.sig $image")) {
print "$image: could not generate signature\n";
return 1;
}
if (comparesigfiles($image, "${image}foo")) {
print "$image: new signature does not match old\n";
return 1;
}
}
return 0;
}
sub imagesize($)
{
my $image = shift;
my @output = `$bindir/imagedump $image`;
if ($?) {
print "$image: *** could not get size of image\n";
return 0;
}
foreach my $line (@output) {
chomp($line);
if ($line =~ /covered sector range: \[(\d+)-(\d+)\]/) {
if ($1 != 0) {
print "$image: WARNING: image does not start at 0\n";
}
$ssize = $2 + 1;
last;
}
}
return $ssize;
}
sub makedisk($$)
{
my ($image,$ssize) = @_;
my ($istr,$dev);
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
my $mb = int(($ssize + 2047) / 2048);
$mb += 100;
if ($os eq "Linux") {
if ($ssize > $MAXSECTORS) {
print "$image: ERROR: image too large ($ssize) for ramdisk,".
" using LV instead\n";
if (mysystem("lvcreate -i $LVMSTRIPE -L ${mb}m -n $istr emulab")) {
print STDERR "could not create LV\n";
return undef;
}
return "/dev/emulab/$istr";
}
# XXX there has to be a better way!
#
# mount -t tmpfs -o size=20580m tmpfs /mnt/FOO.ndz
# dd if=/dev/zero of=/mnt/FOO.ndz/disk bs=1024k seek=20479 count=1
# losetup -f
# losetup /dev/loop0 /mnt/FOO.ndz/disk
#
my $mountpoint = "/mnt/$istr";
if (!mkdir($mountpoint)) {
print STDERR "could not make mountpoint $mountpoint\n";
return undef;
}
if (mysystem("mount -t tmpfs -o size=${mb}m tmpfs $mountpoint")) {
rmdir($mountpoint);
return undef;
}
my $mbm1 = $mb - 1;
if (mysystem("dd if=/dev/zero of=$mountpoint/disk bs=1024k seek=$mbm1 count=1")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
$dev = `losetup -f`;
chomp($dev);
if (mysystem("losetup $dev $mountpoint/disk")) {
mysystem("umount $mountpoint");
rmdir($mountpoint);
return undef;
}
} else {
print STDERR "Cannot do this under $os yet\n";
}
return $dev;
}
sub unmakedisk($$)
{
my ($image,$dev) = @_;
my $istr;
if ($image =~ /([^\/]+)\/([^\/]+)$/) {
$istr = "$1-$2";
} elsif ($image =~ /([^\/]+)$/) {
$istr = $1;
} else {
print "$image: *** could not parse '$image'\n";
return undef;
}
if ($dev eq "/dev/emulab/$istr") {
if (mysystem<