Commit 0bb906f4 authored by Mike Hibler's avatar Mike Hibler

New imagevalidate tool for printing/checking/updating image metadata.

This should be run whenever an image is created or updated and possibly
periodically over existing images. It makes sure that various image
metadata fields are up to date:

 * hash: the SHA1 hash of the image. This field has been around for
   awhile and was previously maintained by "imagehash".

 * size: the size of the image file.

 * range: the sector range covered by the uncompressed image data.

 * mtime: modification time of the image. This is the "updated"
   datetime field in the DB. Its intent was always to track the update
   time of the image, but it wasn't always exact (create-image would
   update this with the current time at the start of the image capture
   process).

Documentation? Umm...the usage message is comprehensive!
It sports a variety of useful options, but the basics are:

 * imagevalidate -p <image> ...
    Print current DB metadata for indicated images. <image> can either
    be a <pid>/<imagename> string or the numeric imageid.

 * imagevalidate <image> ...
    Check the mtime, size, hash, and image range of the image file and
    compare them to the values in the DB. Whine for ones which are out
    of date.

 * imagevalidate -u <image> ...
    Compare and then update DB metadata fields that are out of date.

Fixed a variety of scripts that either used imagehash or computed the
SHA1 hash directly to now use imagevalidate.
parent 605221af
......@@ -4,12 +4,13 @@
use strict;
use libinstall;
use installvars;
use EmulabConstants;
my $UTAHURL = "http://www.emulab.net/downloads";
my $DESCRIPTORS = "$TOP_SRCDIR/install/descriptors-v3.xml";
my $GENDEV = "$TOP_SRCDIR/install/descriptors-gendev.xml";
my @STDIMAGES = ("FBSD82-STD", "FEDORA15-STD");
my @MBRS = ("emulab-mbr.dd", "emulab-mbr2.dd");
my @MBRS = ("emulab-mbr.dd", "emulab-mbr2.dd", "emulab-mbr3.dd");
my $STDIMAGESURL = "$UTAHURL/images-STD";
sub Install($$$)
......@@ -97,6 +98,16 @@ sub Install($$$)
" $SUDO -u $PROTOUSER $WAP ".
" perl load-descriptors -a $localfile");
};
#
# XXX the metadata file may not contain any or all of the newer
# DB state. So we update the metadata using imagevalidate.
#
Phase "${imagename}_validate", "Validating DB info for image.", sub {
my $iname = TBOPSPID() . "/" . $imagename;
ExecQuietFatal("$SUDO -u $PROTOUSER ".
"$PREFIX/sbin/imagevalidate -uq $iname");
};
}
foreach my $mbr (@MBRS) {
my $localfile = "$PREFIX/images/$mbr";
......
......@@ -52,6 +52,15 @@ sub Install($$$)
" $SUDO -u $PROTOUSER $WAP ".
" perl load-descriptors -a $localfile");
};
#
# XXX the metadata file may not contain any or all of the newer
# DB state. So we update the metadata using imagevalidate.
#
Phase "${imagename}_validate", "Validating DB info for image.", sub {
my $iname = TBOPSPID() . "/" . $imagename;
ExecQuietFatal("$SUDO -u $PROTOUSER ".
"$PREFIX/sbin/imagevalidate -uq $iname");
};
}
#
......
......@@ -58,6 +58,15 @@ sub Install($$$)
" $SUDO -u $PROTOUSER $WAP ".
" perl load-descriptors -a $localfile");
};
#
# XXX the metadata file may not contain any or all of the newer
# DB state. So we update the metadata using imagevalidate.
#
Phase "${imagename}_validate", "Validating DB info for image.", sub {
my $iname = TBOPSPID() . "/" . $imagename;
ExecQuietFatal("$SUDO -u $PROTOUSER ".
"$PREFIX/sbin/imagevalidate -uq $iname");
};
}
#
......
......@@ -49,7 +49,7 @@ SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
prereserve grantimage getimages localize_mfs \
management_iface sharevlan check-shared-bw \
addspecialdevice addspecialiface imagehash clone_image \
addvpubaddr imageinfo ctrladdr image_import \
addvpubaddr imageinfo imagevalidate ctrladdr image_import \
prereserve_check tcppd addexternalnetwork \
update_sitevars delete_image sitecheckin sitecheckin_client \
mktestbedtest fixrootcert addservers poolmonitor \
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -131,6 +131,7 @@ my $friskiller = "$TB/sbin/frisbeehelper";
my $osselect = "$TB/bin/os_select";
my $checkquota = "$TB/sbin/checkquota";
my $imagehash = "$TB/bin/imagehash";
my $imagevalidate = "$TB/sbin/imagevalidate";
my $SHA1 = "/sbin/sha1";
my $SCP = "/usr/bin/scp";
my $def_devtype = "ad";
......@@ -811,34 +812,34 @@ if (! -x $imagehash ||
}
#
# Hash the file itself since we really want an integrity check
# on the image file.
# Update fields in the DB related to the image.
#
my $hashfile = "${filename}.sha1";
my $filehash = `$SHA1 $filename`;
if ($?) {
fatal("Could not generate sha1 hash of $filename");
# Note that we do not do this for "standard" images since they get uploaded
# into /proj/emulab-ops rather than /usr/testbed. We could automatically move
# the image into place here, but that makes us nervous. We prefer an admin do
# that by hand after testing the new image!
#
my $tbopsmsg = "";
if ($isglobal && $usepath) {
$tbopsmsg =
"Did not update DB state for global image $pid/$imagename since\n".
"image was written to '$filename'\n".
"instead of $TB/images. Move image into place and run:\n".
" $imagevalidate -uq $pid/$imagename\n";
}
if ($filehash =~ /^SHA1.*= (\w*)$/) {
if ($isglobal && $usepath) {
print "*** WARNING: Not updating SHA1 in DB record since the ".
"image was written to /proj!\n";
print " See $hashfile instead\n";
}
else {
$image->SetHash($1) == 0
or fatal("Failed to set the hash for $image");
}
elsif (system("$imagevalidate -uq $pid/$imagename") != 0) {
$tbopsmsg =
"DB state update for image $pid/$imagename failed, try again with:\n".
" $imagevalidate -u $pid/$imagename\n";
}
else {
fatal("Could not parse the sha1 hash: '$filehash'")
if ($tbopsmsg) {
SENDMAIL($TBOPS,
"Image DB state update failure for $pid/$imagename",
$tbopsmsg,
$TBOPS,
undef,
());
}
unlink($hashfile)
if (-e $hashfile);
open(HASH, ">$hashfile") or
fatal("Could not open $hashfile for writing: $!");
print HASH $filehash;
close($hashfile);
print "Image creation succeeded.\n";
print "Image written to $filename.\n";
......
#!/usr/bin/perl -w
#
# Copyright (c) 2010-2013 University of Utah and the Flux Group.
# Copyright (c) 2010-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -68,6 +68,7 @@ my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez";
my $IMAGEDUMP = "$TB/bin/imagedump";
my $SHA1 = "/sbin/sha1";
my $SAVEUID = $UID;
my $IMAGEVALIDATE = "$TB/sbin/imagevalidate";
#
# Untaint the path
......@@ -268,8 +269,12 @@ if ($getimage) {
$image->Unlock();
exit(1);
}
# Update the hash in the DB.
$image->SetHash($newhash);
# Update DB info. Use the hash we were given, no need to recalculate.
my $imageid = $image->imageid();
if (system("$IMAGEVALIDATE -uq -H '$newhash' $imageid")) {
# XXX should this be fatal?
print STDERR "Could not update DB info for $image\n";
}
}
$image->Unlock();
}
......
#!/usr/bin/perl -w
#
# Copyright (c) 2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use English;
use strict;
use Getopt::Std;
use Data::Dumper;
use File::stat;
#
# Validate information for an image in the DB.
# Currently we validate:
#
# * that image file exists
# * that file mtime matches DB update time
# * file size is correct
# * SHA1 hash is correct
# * covered sector range is correct
#
# The update option will fix all but the first.
#
sub usage()
{
print("Usage: imagevalidate [-dfupqR] [-H hash] [-V str] <imageid> ...\n" .
" imagevalidate [-dfupqR] [-H hash] [-V str] -a\n" .
"Options:\n".
" -d Turn on debug mode\n".
" -f Only update if DB says an image is out of date\n".
" -u Update incorrect or missing info in the DB\n".
" -p Show current information from the DB\n".
" -q Update quietly, no messages about mismatches\n".
" -R Set the relocatable flag if image file has relocations\n".
" -a Update all images\n".
" -P pid Update all images for a specific pid\n".
" -U Do not modify updater_uid in DB\n".
" -H hash Use the provided hash rather than recalculating\n".
" -V str Comma separated list of fields to validate/update\n".
" valid values: 'hash', 'range', 'size', 'all'\n".
" default is 'all'\n");
exit(-1);
}
my $optlist = "dfnupqRaP:UH:V:";
my $debug = 0;
my $showinfo = 0;
my $update = 0;
my $fastupdate = 0;
my $setreloc = 0;
my $quiet = 0;
my $doall = 0;
my $doallpid;
my $nouser = 0;
my %validate = ();
my @images = ();
my $userperm = TB_IMAGEID_READINFO();
my $newhash;
#
# Configure variables
#
my $TB = "@prefix@";
my $SHA1 = "/sbin/sha1";
my $IMAGEINFO = "$TB/sbin/imageinfo";
# Protos
sub doimage($);
sub makehashfile($$);
sub fatal($);
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use EmulabConstants;
use Image;
use OSinfo;
use User;
use Project;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"f"})) {
$fastupdate = 1;
}
if (defined($options{"u"})) {
$update = 1;
$userperm = TB_IMAGEID_MODIFYINFO();
}
if (defined($options{"p"})) {
$showinfo = 1;
}
if (defined($options{"q"})) {
$quiet = 1;
}
if (defined($options{"R"})) {
fatal("Do not use -R; image relocations are NOT a reliable indicator!");
#$setreloc = 1;
}
if (defined($options{"a"})) {
$doall = 1;
}
if (defined($options{"P"})) {
if ($options{"P"} =~ /^([-\w]+)$/) {
$doallpid = $1;
$doall = 1;
} else {
fatal("Invalid project name for -P");
}
}
if (defined($options{"U"})) {
$nouser = 1;
}
if (defined($options{"H"})) {
if ($options{"H"} =~ /^([\da-fA-F]+)$/) {
$newhash = lc($1);
} else {
fatal("Invalid hash string");
}
}
if (defined($options{"V"})) {
foreach my $f (split(',', $options{"V"})) {
$validate{$f} = 1;
}
} else {
$validate{"all"} = 1;
}
@images = @ARGV;
my ($user,$user_uid);
if ($UID) {
$user = User->ThisUser();
if (!defined($user)) {
fatal("You ($UID) do not exist!");
}
$user_uid = $user->uid();
}
if ($nouser && $UID && !$user->IsAdmin()) {
fatal("Only admin can use -U");
}
if ($doall) {
if ($UID && !$user->IsAdmin()) {
fatal("Only admin can use -a");
}
if ($doallpid) {
if (!Project->Lookup($doallpid)) {
fatal("No such project '$doallpid'");
}
}
@images = Image->ListAll("ndz", $doallpid);
if (@images > 100 && $validate{"hash"}) {
print STDERR "WARNING: processing ", int(@images),
" images, will take a LONG time!\n";
}
}
if (!$doall && @images == 0) {
usage();
}
if (defined($newhash) && @images > 1) {
fatal("-H option can only be used with a single image");
}
my $errs = 0;
foreach my $pidimage (@images) {
$errs += doimage($pidimage);
}
exit($errs);
sub doimage($)
{
my ($pidimage) = @_;
my $image = Image->Lookup($pidimage);
if (!defined($image)) {
print STDERR "$pidimage: no such image\n";
return 1;
}
my $imageid = $image->imageid();
# If the user is not an admin, must have perm on the image.
if ($UID && !$user->IsAdmin() && !$image->AccessCheck($user, $userperm)) {
print STDERR "$pidimage: insufficient privilege\n";
return 1;
}
my $path = $image->path();
$path = ""
if (!defined($path));
my $hash = $image->hash();
$hash = ""
if (!defined($hash));
my $size = $image->size();
my $lbalo = $image->lba_low();
my $lbahi = $image->lba_high();
my $lbasize = $image->lba_size();
my $relocatable = $image->relocatable();
my $stamp;
$image->GetUpdate(\$stamp);
$stamp = 0
if (!defined($stamp));
if ($showinfo) {
print "$pidimage: path: $path\n";
print "$pidimage: mtime: $stamp\n";
if ($validate{"all"} || $validate{"size"}) {
my $chunks = int(($size + (1024*1024-1)) / (1024*1024));
print "$pidimage: size: $size ($chunks chunks)\n";
}
if ($validate{"all"} || $validate{"hash"}) {
print "$pidimage: hash: $hash\n";
}
# XXX do sector range
if ($validate{"all"} || $validate{"range"}) {
print "$pidimage: range: [$lbalo-$lbahi] (ssize: $lbasize), ".
"relocatable=$relocatable\n";
}
return 0;
}
#
# The image file has to exist for us to check hash or sector range.
#
if (!$path) {
print STDERR "$pidimage: path: NULL image path\n";
return 1;
}
if (! -r "$path") {
print STDERR "$pidimage: path: image path '$path' cannot be read\n";
return 1;
}
my $mtime = stat($path)->mtime;
my $fsize = stat($path)->size;
if (!defined($mtime) || !defined($fsize)) {
print STDERR "$pidimage: path: cannot stat '$path'\n";
return 1;
}
my $rv = 0;
my $changed = 0;
#
# Check/fix mtime.
#
if ($stamp == $mtime) {
if ($fastupdate) {
print STDERR "$pidimage: skipping due to time stamp\n"
if ($debug);
return 0;
}
} else {
print("$pidimage: mtime: DB timestamp ($stamp) != mtime ($mtime)\n")
if (!$update || !$quiet);
if ($update) {
$changed = 1;
}
}
#
# Check/fix file size.
#
if ($validate{"all"} || $validate{"size"}) {
if ($fsize != $size) {
print("$pidimage: size: DB size ($size) != file size ($fsize)\n")
if (!$update || !$quiet);
if ($update) {
print("$pidimage: size: ")
if (!$quiet);
if ($image->SetSize($fsize) == 0) {
$changed = 1;
print "[FIXED]\n"
if (!$quiet);
} else {
print "[FAILED]\n"
if (!$quiet);
$rv = 1;
}
} else {
$rv = 1;
}
}
}
#
# Check/fix hash.
#
if ($validate{"all"} || $validate{"hash"}) {
my $filehash = $newhash;
if (!defined($filehash)) {
$filehash = `$SHA1 $path`;
if ($?) {
print("$pidimage: hash: could not generate SHA1 hash of '$path'\n");
$filehash = "";
$rv = 1;
} else {
if ($filehash =~ /^SHA1.*= ([\da-fA-F]+)$/) {
$filehash = lc($1);
} else {
print("$pidimage: hash: could not parse sha1 hash: '$filehash'\n");
$filehash = "";
}
}
if ($filehash && ($hash ne $filehash)) {
print("$pidimage: hash: DB hash ('$hash') != file hash ('$filehash')\n")
if (!$update || !$quiet);
if ($update) {
print("$pidimage: hash: ")
if (!$quiet);
if ($image->SetHash($1) == 0) {
makehashfile($path, $filehash);
$changed = 1;
print "[FIXED]\n"
if (!$quiet);
} else {
print "[FAILED]\n"
if (!$quiet);
$rv = 1;
}
} else {
$rv = 1;
}
} elsif ($filehash) {
# even if the DB is correct, make sure .sha1 file is correct
if ($update) {
makehashfile($path, $filehash);
}
}
}
}
#
# Check/fix sector range.
#
if ($validate{"all"} || $validate{"range"}) {
my ($lo,$hi,$ssize) = (-1,0,0);
my $isreloc = $relocatable;
my $out = `imageinfo -r $pidimage 2>&1`;
if ($?) {
print("$pidimage: range: could not get sector range:\n$out");
} else {
if ($out =~ /minsect=(\d+).*maxsect=(\d+).*secsize=(\d+)/s) {
$lo = $1;
$hi = $2;
$ssize = $3;
#
# The sector range is actually relative to the slice
# (partition) number that imagezip was told to save.
# Thus a zero offset is actually the start sector of the
# partition and we compensate for that before recording
# the values in the DB.
#
my $off = $image->GetDiskOffset();
if ($off > 0) {
$lo += $off;
$hi += $off;
}
#
# XXX this is unreliable since we also generate a relocation
# for images that do not have a full final sector. Hence, we
# have disabled this.
#
# XXX the relocatable value returned by imageinfo is only a
# heuristic. It says only that relocations exist in the image.
# It is possible for a relocatable image to not actually
# have any imagezip relocations. Hence we only change the
# DB relocatable value from 0 -> 1 if explicitly asked and
# there are relocations in the image file.
#
#if ($setreloc && $relocatable == 0 &&
# $out =~ /relocatable=1/s) {
# $isreloc = 1;
#}
} else {
print("$pidimage: range: could not parse imageinfo output:\n$out");
}
if ($lo >= 0 &&
($lo != $lbalo || $hi != $lbahi || $ssize != $lbasize ||
$isreloc != $relocatable)) {
print("$pidimage: range: DB range ([$lbalo-$lbahi]/$lbasize) != file range ([$lo-$hi]/$ssize)\n")
if (!$update || !$quiet);
if ($update) {
print("$pidimage: range: ")
if (!$quiet);
if ($image->SetRange($lo, $hi, $ssize, $isreloc) == 0) {
$changed = 1;
print "[FIXED]\n"
if (!$quiet);
} else {
print "[FAILED]\n"
if (!$quiet);
$rv = 1;
}
} else {
$rv = 1;
}
}
}
}
#
# Set update time to match mtime of image
#
if ($changed) {
print("$pidimage: mtime: ")
if (!$quiet);
my $uuser = ($nouser ? undef : $user);
# XXX if running as root and no current user, set to image creator
if ($UID == 0 && !defined($image->updater())) {
$uuser = User->LookupByUid($image->creator());
}
if ($image->MarkUpdate($uuser, $mtime) == 0) {
print "[FIXED]\n"
if (!$quiet);
} else {
print "[FAILED]\n"
if (!$quiet);
$rv = 1;
}
}
return $rv;
}