Commit f9abc442 authored by Gary Wong's avatar Gary Wong

Try to make getimages resilient to interruption or temporary failure.

The basic approach is to tell fetch to be optimistic (preserve partial
output, attempt to resume transfers when local data exist, etc.).  We
ignore all errors fetch gives us, and rely on the SHA1 hashes only for
integrity verification.  If we think our copy is too small, we just
keep trying to resume.  If our copy is big enough but the checksum
doesn't match, then throw it away and restart.
parent 337a0457
...@@ -61,6 +61,7 @@ my $PROTOUSER = "elabman"; ...@@ -61,6 +61,7 @@ my $PROTOUSER = "elabman";
my $WAP = "$TB/sbin/withadminprivs"; my $WAP = "$TB/sbin/withadminprivs";
my $metadata = "/tmp/imageinfo-$$.xml"; my $metadata = "/tmp/imageinfo-$$.xml";
my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez"; my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez";
my $MAXATTEMPTS = 5;
# #
# Testbed Support libraries # Testbed Support libraries
...@@ -260,25 +261,49 @@ foreach my $imageid (keys(%{ $xmlparse->{'image'} })) { ...@@ -260,25 +261,49 @@ foreach my $imageid (keys(%{ $xmlparse->{'image'} })) {
my $tmpfilename = "${imagefilename}.new"; my $tmpfilename = "${imagefilename}.new";
unlink($tmpfilename) unlink($tmpfilename)
if (-e $tmpfilename); if (-e $tmpfilename);
print "Fetching $imageurl\n"; # Try a few times, so we don't fail on temporary errors. Restart
if (system("$FETCH -o $tmpfilename $imageurl")) { # from the beginning if what we have is hopeless; attempt to continue
unlink($tmpfilename); # if we have part of a file and aren't convinced it's useless.
fatal("Could not fetch $imageurl"); my $attempt = 1;
} while( 1 ) {
print "Fetching $imageurl (attempt $attempt of $MAXATTEMPTS)...\n";
# Ignore exit code from fetch. Maybe we got lucky and got a
# good transfer before it died; maybe it gave up halfway through
# and we have half a good file and can continue where we left off.
my $expectedsize = `$FETCH -s $imageurl`;
system("$FETCH -a -F -R -r -o $tmpfilename $imageurl");
if( -s $tmpfilename < $expectedsize ) {
print "Local file appears truncated, retrying...\n";
next;
}
# #
# Do an integrity check. # Do an integrity check.
# #
print "Doing an integrity check ...\n"; print "Doing an integrity check ...\n";
my $filehash = `/sbin/sha1 -q $tmpfilename`; my $filehash = `/sbin/sha1 -q $tmpfilename`;
if ($?) { if ($?) {
fatal("Could not generate sha1 of $tmpfilename"); print "Could not generate sha1 of $tmpfilename\n";
} unlink( $tmpfilename );
chomp($filehash); next;
if ($filehash ne $hash) { }
fatal("Integrity check failure. $hash ne $filehash"); chomp($filehash);
if ($filehash ne $hash) {
print "Integrity check failure. $hash ne $filehash\n";
# Looking bad. But let's start again and give it another try
# just in case.
unlink( $tmpfilename );
next;
}
# We got it!
last;
} continue {
$attempt++;
fatal( "Could not retrieve $imageurl after $MAXATTEMPTS tries." )
if( $attempt > $MAXATTEMPTS );
} }
if ($impotent) { if ($impotent) {
print "Impotent mode is on; not installing the new image.\n"; print "Impotent mode is on; not installing the new image.\n";
print "Hash:$hash, time:$newtime\n"; print "Hash:$hash, time:$newtime\n";
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment