Commit 1f618343 authored by Leigh B Stoller's avatar Leigh B Stoller

Update getimages (used on geniracks) to use image_import instead.

Update image_import to handle image refresh more easily with -r option.
Had this in my devel tree for a long time, time to try it out for real.
parent e54fc488
#!/usr/bin/perl -w
#
# Copyright (c) 2003-2014 University of Utah and the Flux Group.
# Copyright (c) 2003-2015 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -39,12 +39,10 @@ sub usage()
{
print STDERR "Usage: getimages [-d] [-n]\n";
print STDERR " -h This message\n";
print STDERR " -n Impotent mode; just check and report.\n";
exit(-1);
}
my $optlist = "hndt:";
my $optlist = "hdt:";
my $debug = 1;
my $impotent = 0;
my $testfile;
# Protos
......@@ -54,14 +52,23 @@ sub fatal($);
# Configure variables
#
my $TB = "@prefix@";
my $METAURL = "http://www.emulab.net/genirack-imageinfo.xml";
my $FETCH = "/usr/bin/fetch";
my $METAURL = "http://www.emulab.net/genirack-imageinfo-v2.xml";
my $SUDO = "/usr/local/bin/sudo";
my $FETCH = "/usr/bin/fetch";
my $PROTOUSER = "elabman";
my $WAP = "$TB/sbin/withadminprivs";
my $metadata = "/tmp/imageinfo-$$.xml";
my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez";
my $MAXATTEMPTS = 5;
my $IMAGEIMPORT = "$TB/sbin/image_import";
my $FBSD_VERSION = 4;
if (`uname -r` =~ /^(\d+\.\d*)/) {
$FBSD_VERSION = $1;
}
# Extra fetch options on FreeBSD 10.
my $FETCHOPTIONS = "";
if ($FBSD_VERSION >= 10.0) {
$FETCHOPTIONS = "--no-verify-peer --no-verify-hostname";
}
#
# Testbed Support libraries
......@@ -104,9 +111,6 @@ if (! getopts($optlist, \%options)) {
if (defined($options{'h'})) {
usage();
}
if (defined($options{'n'})) {
$impotent = 1;
}
if (defined($options{'d'})) {
$debug = 1;
}
......@@ -117,7 +121,7 @@ usage()
if (@ARGV);
# Only root.
if ($UID && !$impotent) {
if ($UID) {
die("*** $0:\n".
" Must run this as root!\n");
}
......@@ -165,17 +169,13 @@ foreach my $imageid (keys(%{ $xmlparse->{'image'} })) {
}
my $metaurl = $attributes->{'metaurl'};
my $imageurl = $attributes->{'imageurl'};
my $hashurl = $attributes->{'hashurl'};
my $newtime = timegm(strptime($attributes->{'modtime'}));
#
# If we have an entry in the DB, we use the modtime as a serial
# number to determine if we need to go the next step and compare
# hashes.
# number to determine if we need to go the next step.
#
# XXX What if the local site has its own more recent version?
# Need to deal with this at some point.
#
# Lookup will sanity check the imageid string.
#
......@@ -193,47 +193,15 @@ foreach my $imageid (keys(%{ $xmlparse->{'image'} })) {
next;
}
}
print "$imageid timestamp has changed. Checking hash.\n";
print "$imageid timestamp has changed.\n";
}
else {
print "$imageid does not exist.\n";
}
#
# Grab the hash file from the server.
#
my ($fh, $hashfilename) = tempfile(UNLINK => !$debug);
fatal("Could not create temporary file")
if (!defined($fh));
close($fh);
print "Fetching $hashurl\n";
system("$FETCH -o $hashfilename $hashurl") == 0
or fatal("Could not fetch $hashurl");
my $hash = `cat $hashfilename`;
if ($hash =~ /^SHA1.*= (\w*)$/) {
$hash = $1;
}
else {
fatal("Could not parse the sha1 hash: '$hash'")
}
#
# If we have the image defined and the hash matches, then done.
#
if (defined($image)) {
if (defined($image->hash()) && $image->hash() eq $hash) {
print "Image hash has not changed, skipping ...\n"
if ($debug);
# Update the timestamp to avoid repeat.
$image->MarkUpdate($elabman, $newtime);
next;
}
}
#
# Grab the metadata file
# Grab the metadata so we can get the imageurl out of it, we need
# to fix up local images below.
#
my ($fh2, $metafilename) = tempfile(UNLINK => !$debug);
fatal("Could not create temporary file")
......@@ -241,103 +209,44 @@ foreach my $imageid (keys(%{ $xmlparse->{'image'} })) {
close($fh2);
print "Fetching $metaurl\n";
system("$FETCH -o $metafilename $metaurl") == 0
system("$FETCH $FETCHOPTIONS -o $metafilename $metaurl") == 0
or fatal("Could not fetch $metaurl");
chmod(0666, $metafilename);
my $metaparse = eval { XMLin($metafilename,
VarAttr => 'name',
ContentKey => '-content',
SuppressEmpty => undef); };
fatal($@)
if ($@);
my $imageurl = $metaparse->{'attribute'}->{"imagefile_url"}->{'value'};
#
# Load up the descriptor if we do not have it.
# We use image_import to bring in or update the image.
#
if (!defined($image)) {
# Do this as admin cause of admin only options in the descriptor.
system("$SUDO -u $PROTOUSER $WAP $NEWIMAGE_EZ -a $metafilename") == 0
or fatal("Could not create descriptor for $imageid");
$image = Image->Lookup(TBOPSPID(), $imageid);
if (!defined($image)) {
fatal("Could not lookup newly created descriptor for $imageid");
}
$image->MarkReady();
$image->MarkReleased();
# XXX Running this script means you want Utah to manage
# your images, so we ignore locally updated images at the
# moment and force the existing image metadata url to what
# Utah has said it should be.
#
my $cmd = "$SUDO -u $PROTOUSER $WAP $IMAGEIMPORT -d -g -u $PROTOUSER ";
if (defined($image)) {
$cmd .= " -r " . $image->imageid();
$image->Update({"metadata_url" => $metaurl,
"imagefile_url" => $imageurl});
}
else {
$cmd .= " -p " . TBOPSPID() . " '$metaurl'";
}
print "Running '$cmd'\n" if ($debug);
system($cmd);
fatal("Could not import image from $metaurl")
if ($?);
my $imagefilename = "$TB/images/${imageid}.ndz";
my $tmpfilename = "${imagefilename}.new";
unlink($tmpfilename)
if (-e $tmpfilename);
# Try a few times, so we don't fail on temporary errors. Restart
# from the beginning if what we have is hopeless; attempt to continue
# if we have part of a file and aren't convinced it's useless.
my $attempt = 1;
while( 1 ) {
print "Fetching $imageurl (attempt $attempt of $MAXATTEMPTS)...\n";
my $expectedsize = `$FETCH -s $imageurl`;
chomp( $expectedsize );
if( $expectedsize !~ /^[0-9]+$/ ) {
print "Could not retrieve image size, retrying...\n";
next;
}
# Ignore exit code from fetch. Maybe we got lucky and got a
# good transfer before it died; maybe it gave up halfway through
# and we have half a good file and can continue where we left off.
system("$FETCH -a -F -R -r -o $tmpfilename $imageurl");
if( -s $tmpfilename < $expectedsize ) {
print "Local file appears truncated, retrying...\n";
next;
}
#
# Do an integrity check.
#
print "Doing an integrity check ...\n";
my $filehash = `/sbin/sha1 -q $tmpfilename`;
if ($?) {
print "Could not generate sha1 of $tmpfilename\n";
unlink( $tmpfilename );
next;
}
chomp($filehash);
if ($filehash ne $hash) {
print "Integrity check failure. $hash ne $filehash\n";
# Looking bad. But let's start again and give it another try
# just in case.
unlink( $tmpfilename );
next;
}
# We got it!
last;
} continue {
$attempt++;
fatal( "Could not retrieve $imageurl after $MAXATTEMPTS tries." )
if( $attempt > $MAXATTEMPTS );
$image = Image->Lookup(TBOPSPID(), $imageid);
if (!defined($image)) {
fatal("Could not lookup image even though import succeeded");
}
if ($impotent) {
print "Impotent mode is on; not installing the new image.\n";
print "Hash:$hash, time:$newtime\n";
next;
}
#
# So move it into place.
#
print "Moving new image into place\n";
unlink("${imagefilename}.old")
if (-e "${imagefilename}.old");
rename("${imagefilename}", "${imagefilename}.old")
if (-e "${imagefilename}");
rename("$tmpfilename", "${imagefilename}") or
fatal("Could not rename $tmpfilename: $!");
#
# Now update the descriptor to reflect new hash.
#
$image->SetHash($hash) == 0
or fatal("Could not update hash for $image");
#
# Mark this too, so that we do not repeat the first test above.
#
......
......@@ -33,8 +33,9 @@ use URI::Escape;
#
sub usage()
{
print("Usage: import_image [-d] [-v] [-u <user>] [-g] [-p pid] <url>\n");
print(" import_image [-d] [-u <user>] [-g] -i <imageid>\n");
print("Usage: import_image [-d] [-v] [-u <user>] [-g] [-p pid] ".
"[-i name] <url>\n");
print(" import_image [-d] [-u <user>] -r <imageid>\n");
print("Options:\n");
print(" -d - Turn on debugging\n");
print(" -v - Verify XML description only\n");
......@@ -42,10 +43,12 @@ sub usage()
print(" -u uid - Create image as user instead of caller\n");
print(" -p pid - Create image in the specified project.\n".
" Defaults to emulab-ops.\n");
print(" -i id - Update existing imported image.\n");
print(" -i name - Use name for imagename.\n".
" Defaults to name in the desciptor\n");
print(" -r - Refresh (update ndz file) imported image.\n");
exit(-1);
}
my $optlist = "dvu:p:giI";
my $optlist = "dvu:p:gi:Ir";
my $debug = 0;
my $verify = 0;
my $getimage= 0;
......@@ -53,6 +56,7 @@ my $update = 0;
my $force = 0;
my $user;
my $group;
my $imagename;
#
# Configure variables
......@@ -113,7 +117,7 @@ my $url;
# Protos
sub fatal($);
sub FetchMetadata($);
sub CreateImage($$$$);
sub CreateImage($$$$$);
sub DownLoadImage($$$$);
#
......@@ -140,9 +144,12 @@ if (defined($options{"g"})) {
$getimage = 1;
}
if (defined($options{"i"})) {
$update = 1;
$imagename = $options{i};
}
if (defined($options{"r"})) {
$update = 1;
}
if (defined($options{"I"})) {
if (defined($options{"R"})) {
$update = 1;
$force = 1;
}
......@@ -248,7 +255,7 @@ if (!$query_result ||
}
my $image = Image->LookupByURL($url);
if (!defined($image)) {
$image = CreateImage($url, $xmlparse, $user, $group);
$image = CreateImage($url, $xmlparse, $user, $group, $imagename);
}
DBQueryWarn("select RELEASE_LOCK($safe_url)");
exit(0)
......@@ -337,9 +344,9 @@ exit(0);
# Create a new image descriptor. We have to munge the XML file a bit
# though and write it out.
#
sub CreateImage($$$$)
sub CreateImage($$$$$)
{
my ($url, $xmlparse, $user, $group) = @_;
my ($url, $xmlparse, $user, $group, $imagename) = @_;
my $alltypes = "-a";
my $global = 0;
......@@ -386,7 +393,10 @@ sub CreateImage($$$$)
# not going to worry about concurrent attempts to create a descriptor
# with the same name.
#
if (! exists($xmlparse->{'attribute'}->{"imagename"})) {
if (defined($imagename)) {
$xmlparse->{'attribute'}->{"imagename"}->{'value'} = $imagename;
}
elsif (! exists($xmlparse->{'attribute'}->{"imagename"})) {
$xmlparse->{'attribute'}->{"imagename"}->{'value'} =
substr(TBGenSecretKey(), 0, 12);
}
......@@ -403,7 +413,7 @@ sub CreateImage($$$$)
}
$xmlparse->{'attribute'}->{"imagename"}->{'value'} = $imagename;
}
my $imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
$imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
if ($debug) {
print STDERR "Using imagename: $imagename\n";
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment