image_import.in 18.2 KB
Newer Older
Leigh B Stoller's avatar
Leigh B Stoller committed
1 2
#!/usr/bin/perl -w
#
3
# Copyright (c) 2010-2015 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh B Stoller's avatar
Leigh B Stoller committed
23 24 25 26 27 28 29 30 31 32 33 34 35
#
use English;
use strict;
use Getopt::Std;
use XML::Simple;
use Data::Dumper;
use URI::Escape;

#
# Import an image from an external source. 
#
sub usage()
{
36
    print("Usage: image_import [-d] [-v] [-u <user>] [-g] [-p pid] ".
37
	  "[-i name] <url>\n");
38
    print("       image_import [-d] [-u <user>] -r <imageid>\n");
Leigh B Stoller's avatar
Leigh B Stoller committed
39 40 41 42 43
    print("Options:\n");
    print(" -d      - Turn on debugging\n");
    print(" -v      - Verify XML description only\n");
    print(" -g      - Download image after creating descriptor\n");
    print(" -u uid  - Create image as user instead of caller\n");
44 45
    print(" -p pid  - Create image in the specified project.\n".
	  "           Defaults to emulab-ops.\n");
46 47 48
    print(" -i name - Use name for imagename.\n".
	  "           Defaults to name in the desciptor\n");
    print(" -r      - Refresh (update ndz file) imported image.\n");
49
    print(" -s      - With -r just update the sig file\n");
Leigh B Stoller's avatar
Leigh B Stoller committed
50 51
    exit(-1);
}
52
my $optlist = "dvu:p:gi:Irs";
Leigh B Stoller's avatar
Leigh B Stoller committed
53 54 55 56
my $debug   = 0;
my $verify  = 0;
my $getimage= 0;
my $update  = 0;
57
my $dosig   = 0;
58
my $force   = 0;
Leigh B Stoller's avatar
Leigh B Stoller committed
59 60
my $user;
my $group;
61
my $imagename;
Leigh B Stoller's avatar
Leigh B Stoller committed
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77

#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
my $TBAUDIT	= "@TBAUDITEMAIL@";
my $TBGROUP_DIR	= "@GROUPSROOT_DIR@";
my $TBPROJ_DIR	= "@PROJROOT_DIR@";
my $TBBASE	= "@TBBASE@";
my $CONTROL     = "@USERNODE@";
my $WGET	= "/usr/local/bin/wget";
my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez";
my $IMAGEDUMP   = "$TB/bin/imagedump";
my $SHA1	= "/sbin/sha1";
my $SAVEUID	= $UID;
78
my $IMAGEVALIDATE = "$TB/sbin/imagevalidate";
79 80
my $DELETEIMAGE   = "$TB/sbin/delete_image";
my $WITHPROVENANCE= @IMAGEPROVENANCE@;
Leigh B Stoller's avatar
Leigh B Stoller committed
81
my $DOIMAGEDIRS   = @IMAGEDIRECTORIES@;
82
my $doprovenance  = 0;
Leigh B Stoller's avatar
Leigh B Stoller committed
83

Leigh B Stoller's avatar
Leigh B Stoller committed
84 85 86 87 88
#
# When fetching the metadata, we now tell the server what client
# version of the software we are so it gives something we can handle.
# Be sure to update this if you change the version in dumpdescriptor.
#
89
my $METADATA_CLIENTVERSION = 2;
Leigh B Stoller's avatar
Leigh B Stoller committed
90

Leigh B Stoller's avatar
Leigh B Stoller committed
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
    die("*** $0:\n".
	"    Must be setuid! Maybe its a development version?\n");
}

#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use libdb;
115
use EmulabConstants;
116
use EmulabFeatures;
Leigh B Stoller's avatar
Leigh B Stoller committed
117 118 119 120 121 122 123 124 125 126 127 128 129
use libtestbed;
use User;
use Project;
use Group;
use Image;
use OSinfo;

# Locals;
my $url;

# Protos
sub fatal($);
sub FetchMetadata($);
130
sub CreateImage($$$$$);
Leigh B Stoller's avatar
Leigh B Stoller committed
131
sub DownLoadImage($$$$);
132
sub FetchImageFile($$);
133
sub FetchSigFile($);
Leigh B Stoller's avatar
Leigh B Stoller committed
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157

#
# There is no reason to run as root unless we need to ssh over
# to ops to fetch the URL.
#
$EUID = $UID;

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"v"})) {
    $verify = 1;
}
if (defined($options{"g"})) {
    $getimage = 1;
}
158 159 160
if (defined($options{"s"})) {
    $dosig = 1;
}
Leigh B Stoller's avatar
Leigh B Stoller committed
161
if (defined($options{"i"})) {
162 163 164 165
    $imagename = $options{i};
}
if (defined($options{"r"})) {
    $update   = 1;
Leigh B Stoller's avatar
Leigh B Stoller committed
166
}
167
if (defined($options{"R"})) {
168 169 170
    $update = 1;
    $force  = 1;
}
Leigh B Stoller's avatar
Leigh B Stoller committed
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
if (defined($options{"u"})) {
    $user = User->Lookup($options{"u"});
    fatal("User does not exist")
	if (!defined($user));
}
if (defined($options{"p"})) {
    $group = Group->Lookup($options{"p"});
    if (!defined($group)) {
	my $project = Project->Lookup($options{"p"});
	fatal("Project/Group does not exist")
	    if (!defined($project));
	$group = $project->GetProjectGroup();
	fatal("Error getting project group for $project")
	    if (!defined($group));
    }
}
187 188 189 190 191
else {
    $group = Group->Lookup(TBOPSPID(), TBOPSPID());
    fatal("Error getting project group for " . TBOPSPID())
	if (!defined($group));
}
Leigh B Stoller's avatar
Leigh B Stoller committed
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221

if (!defined($user)) {
    $user = User->ThisUser();
    if (! defined($user)) {
	fatal("You ($UID) do not exist!");
    }
}
my $user_uid = $user->uid();

if ($update) {
    usage()
	if (!@ARGV);

    my $image = Image->Lookup($ARGV[0]);
    if (!defined($image)) {
	fatal("Image descriptor does not exist");
    }
    if (!defined($image->metadata_url())) {
	fatal("Not an imported image");
    }
    $url = $image->metadata_url();

    # If the user is not an admin, must have perm on the image.
    if (!$user->IsAdmin() &&
	!$image->AccessCheck($user, TB_IMAGEID_CREATE())) {
	fatal("$user does not have permission $image");
    }
}
else {
    usage()
222
	if (! (@ARGV && defined($group)));
Leigh B Stoller's avatar
Leigh B Stoller committed
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250

    $url = $ARGV[0];
    # We know this is valid, but must taint check anyway for shell command.
    if ($url =~/^(.*)$/) {
	$url = $1;
    }

    # If the user is not an admin, must be a member or have perm in
    # the group.
    if (!$user->IsAdmin() &&
	!$group->AccessCheck($user, TB_PROJECT_MAKEIMAGEID())) {
	fatal("$user does not have permission in $group");
    }
}

my $xmlparse = FetchMetadata($url);

#
# Sanity checks; it must have a hash and a url inside. We let
# newimageid do the rest of the checks though.
#
if (! exists($xmlparse->{'attribute'}->{"hash"}) ||
    ! ($xmlparse->{'attribute'}->{"hash"}->{'value'} =~ /^\w{10,}$/)) {
    fatal("Invalid hash in metadata");
}
if (! exists($xmlparse->{'attribute'}->{"imagefile_url"})) {
    fatal("Invalid imagefile url in metadata");
}
251 252 253 254 255
my $newhash = $xmlparse->{'attribute'}->{"hash"}->{'value'};
# Silly taint check.
if ($newhash =~/^(.*)$/) {
    $newhash = $1;
}
Leigh B Stoller's avatar
Leigh B Stoller committed
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272

#
# See if we already have an image in the DB for this URL.
# If not, we have to create it.
#
# Need to watch for two experiments causing this image to
# get created at the same time. It would be pretty silly,
# but users are users ... 
#
my $safe_url = DBQuoteSpecial($url);
my $query_result = DBQueryWarn("select GET_LOCK($safe_url, 120)");
if (!$query_result ||
    !$query_result->numrows) {
    fatal("Could not get the SQL lock for a long time!");
}
my $image = Image->LookupByURL($url);
if (!defined($image)) {
273
    $image = CreateImage($url, $xmlparse, $user, $group, $imagename);
Leigh B Stoller's avatar
Leigh B Stoller committed
274 275
}
DBQueryWarn("select RELEASE_LOCK($safe_url)");
276

Leigh B Stoller's avatar
Leigh B Stoller committed
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
exit(0)
    if ($verify);

#
# If the image has not been downloaded or if the hash has changed,
# get a new copy.
#
if ($getimage) {
    #
    # We need to get the lock since someone else could already
    # be downloading it. Even worse, someone might be taking a local
    # snapshot, although at the moment we do not support that.
    #
    if ($image->Lock()) {
	print "$image is currently locked. Waiting ...\n";

	my $maxwait = 600;
	while ($maxwait > 0 && $image->WaitLock(60)) {
	    print "$image is currently locked. Waiting ...\n";
	    $maxwait -= 60;
	}
	if (!$image->GotLock()) {
	    fatal("Could not get the image lock after a long time");
	}
    }
302 303
    # Run as root to access /proj
    $EUID = $UID = 0;
Leigh B Stoller's avatar
Leigh B Stoller committed
304
    if (! -e $image->FullImageFile() || $newhash ne $image->hash() || $force) {
305
	$EUID = $UID = $SAVEUID;
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
	#
	# When updating an image ... 
	#
	if ($update && $WITHPROVENANCE) {
	    $doprovenance =
		EmulabFeatures->FeatureEnabled("ImageProvenance",
					       undef, $image->GetProject());
	    if ($doprovenance) {
		#
		# This will include unreleased images (in image_versions, but
		# not the one pointed to by the images table). 
		#
		$image = $image->LookupMostRecent();
		if (!defined($image)) {
		    $image->Unlock();
		    fatal("Cannot lookup most recent version of $image");
		}
		# Reuse is not ready/released.
		if ($image->ready() && $image->released()) {
		    my $clone = $image->NewVersion($user, undef, undef);
		    if (!defined($clone)) {
			$image->Unlock();
			fatal("Could not clone image descriptor $image");
		    }
		    $image = $clone;
		}
	    }
Leigh B Stoller's avatar
Leigh B Stoller committed
333 334 335 336 337
	    if ($update) {
		# Update the imageurl in the descriptor.
		$image->Update({"imagefile_url" =>
		     $xmlparse->{'attribute'}->{"imagefile_url"}->{'value'}});
	    }
338
	}
Leigh B Stoller's avatar
Leigh B Stoller committed
339 340 341 342
	if (DownLoadImage($image, $newhash, $user, $group)) {
	    $image->Unlock();
	    exit(1);
	}
343 344
	# Update DB info. 
	my $versname = $image->versname();
345 346
	# Run as root to access /proj
	$EUID = $UID = 0;
347
	if (system("$IMAGEVALIDATE -u $versname")) {
348 349 350
	    # XXX should this be fatal?
	    print STDERR "Could not update DB info for $image\n";
	}
351
	$EUID = $UID = $SAVEUID;
352 353
	$image->MarkReady();
	$image->Release();
354
	# Its more important to know when we brought the new version in.
Leigh B Stoller's avatar
Leigh B Stoller committed
355
	if ($WITHPROVENANCE && $doprovenance && $update) {
356 357
	    $image->MarkUpdate($user);
	}
Leigh B Stoller's avatar
Leigh B Stoller committed
358
    }
359
    $EUID = $UID = $SAVEUID;
Leigh B Stoller's avatar
Leigh B Stoller committed
360 361
    $image->Unlock();
}
362 363 364 365
elsif ($update && $dosig) {
    FetchSigFile($image) == 0
	or exit(1);
}
366 367 368
else {
    print "Not downloading image ... add -g option\n";
}
Leigh B Stoller's avatar
Leigh B Stoller committed
369 370 371 372 373 374
exit(0);

#
# Create a new image descriptor. We have to munge the XML file a bit
# though and write it out.
#
375
sub CreateImage($$$$$)
Leigh B Stoller's avatar
Leigh B Stoller committed
376
{
377
    my ($url, $xmlparse, $user, $group, $imagename) = @_;
Leigh B Stoller's avatar
Leigh B Stoller committed
378
    my $alltypes = "-a";
379
    my $global   = 0;
Leigh B Stoller's avatar
Leigh B Stoller committed
380 381 382 383 384 385 386 387 388 389 390
    
    $xmlparse->{'attribute'}->{"pid"} = {};
    $xmlparse->{'attribute'}->{"gid"} = {};
    $xmlparse->{'attribute'}->{"pid"}->{'value'} = $group->pid();
    $xmlparse->{'attribute'}->{"gid"}->{'value'} = $group->gid();

    #
    # Look for a parent osid; this means we should set the type
    # to pcvm since the image is for a VM. Well, we also use this
    # for subnodes, but I am not going to worry about that case.
    #
391
    if (exists($xmlparse->{'attribute'}->{"def_parentosid"})) {
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
	#
	# If parent does not exist, then ignore with warning.
	# This can be set later via the web interface. 
	#
	my $posid  = $xmlparse->{'attribute'}->{"def_parentosid"}->{'value'};
	my $parent = Image->Lookup($posid);
	if (defined($parent)) {
	    $xmlparse->{'attribute'}->{"mtype_pcvm"} = {};
	    $xmlparse->{'attribute'}->{"mtype_pcvm"}->{'value'} = 1;
	}
	else {
	    delete($xmlparse->{'attribute'}->{"def_parentosid"});
	    print STDERR
		"*** Parent $posid does not exist, skipping parent.\n";
	    print STDERR
		"    You can set the parent later via the web interface.\n";
	}
Leigh B Stoller's avatar
Leigh B Stoller committed
409 410
    }

411 412 413 414 415 416
    # For setting the path below.
    if (exists($xmlparse->{'attribute'}->{"global"}) &&
	$xmlparse->{'attribute'}->{"global"}->{'value'}) {
	$global = 1;
    }

Leigh B Stoller's avatar
Leigh B Stoller committed
417 418 419 420 421 422
    #
    # We check to see if the imagename is already in use. Hopefully
    # not, but if not we have to make something up. Note that I am
    # not going to worry about concurrent attempts to create a descriptor
    # with the same name. 
    #
423 424 425 426
    if (defined($imagename)) {
	$xmlparse->{'attribute'}->{"imagename"}->{'value'} = $imagename;
    }
    elsif (! exists($xmlparse->{'attribute'}->{"imagename"})) {
Leigh B Stoller's avatar
Leigh B Stoller committed
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
	$xmlparse->{'attribute'}->{"imagename"}->{'value'} = 
	    substr(TBGenSecretKey(), 0, 12);
    }
    elsif (Image->Lookup($group->pid(),
			 $xmlparse->{'attribute'}->{"imagename"}->{'value'})) {
	my $index = 1;
	my $imagename;
	do {
	    $imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
	    $imagename .= "_" . $index++;
	} while ($index < 100 && Image->Lookup($group->pid(), $imagename));
	if ($index >= 100) {
	    fatal("Could not generate a unique image name");
	}
	$xmlparse->{'attribute'}->{"imagename"}->{'value'} = $imagename;
    }
443
    $imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
Leigh B Stoller's avatar
Leigh B Stoller committed
444 445 446 447
    if ($debug) {
	print STDERR "Using imagename: $imagename\n";
    }
    # do not trust path coming in.
448
    if ($global && $user->IsAdmin()) {
Leigh B Stoller's avatar
Leigh B Stoller committed
449
	$xmlparse->{'attribute'}->{"path"}->{'value'} = "$TB/images/";
450 451 452
    }
    else {
	$xmlparse->{'attribute'}->{"path"}->{'value'} =
Leigh B Stoller's avatar
Leigh B Stoller committed
453 454 455
	    "$TBPROJ_DIR/" . $group->pid() . "/images/";
    }
    if ($DOIMAGEDIRS) {
Leigh B Stoller's avatar
Leigh B Stoller committed
456
	$xmlparse->{'attribute'}->{"path"}->{'value'} .= "${imagename}/";
Leigh B Stoller's avatar
Leigh B Stoller committed
457 458
    }
    else {
Leigh B Stoller's avatar
Leigh B Stoller committed
459
	$xmlparse->{'attribute'}->{"path"}->{'value'} .= "${imagename}.ndz";
460
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
    
    #
    # Generate a new XML description to feed into newimageid.
    #
    $xmlparse->{'attribute'}->{"imagefile_url"}->{'value'} =
	uri_escape($xmlparse->{'attribute'}->{"imagefile_url"}->{'value'});
    $xmlparse->{'attribute'}->{"metadata_url"}->{'value'} = uri_escape($url);

    my $newxml = "";
    foreach my $key (keys(%{ $xmlparse->{'attribute'} })) {
	my $value = $xmlparse->{'attribute'}->{"$key"}->{'value'};
	$newxml .=
	    "<attribute name=\"$key\"><value>$value</value></attribute>\n";
    }
    $newxml = "<image>$newxml</image>";
    if ($debug) {
	print STDERR "$newxml\n";
    }
    # Verify first, Use skip admin checks option.
    open(NEW, "| $NEWIMAGE_EZ $alltypes -s -v -")
	or fatal("Cannot start $NEWIMAGE_EZ");
    print NEW $newxml;
    if (!close(NEW)) {
	print STDERR "$newxml\n";
	fatal("Image xml did not verify");
    }
    return undef
	if ($verify);
    
    open(NEW, "| $NEWIMAGE_EZ $alltypes -s -")
	or fatal("Cannot start $NEWIMAGE_EZ");
    print NEW $newxml;
    if (!close(NEW)) {
	print STDERR "$newxml\n";
	fatal("Could not create new image from xml");
    }

    my $image = Image->LookupByURL($url);
    if (!defined($image)) {
	fatal("Could not lookup new image for $url");
    }
502 503
    $image->MarkReady();
    $image->MarkReleased();
Leigh B Stoller's avatar
Leigh B Stoller committed
504 505 506 507 508 509 510 511
    return $image;
}

#
# Download the image file. 
#
sub DownLoadImage($$$$)
{
512
    my ($image, $newhash, $user, $group) = @_;
Leigh B Stoller's avatar
Leigh B Stoller committed
513
    my $image_url = uri_unescape($image->imagefile_url());
Leigh B Stoller's avatar
Leigh B Stoller committed
514
    my $localfile = $image->FullImageFile() . ".new";
Leigh B Stoller's avatar
Leigh B Stoller committed
515

516 517
    if (FetchImageFile($image_url, $localfile)) {
	return -1;
Leigh B Stoller's avatar
Leigh B Stoller committed
518 519 520
    }

    #
521
    # Verify the hash which was created by FetchImageFile().
Leigh B Stoller's avatar
Leigh B Stoller committed
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
    #
    my $newhashfile = $localfile . ".sha1";
    print "Verifying the hash ...\n";
    my $filehash = `cat $newhashfile`;
    if ($?) {
	print STDERR "Could not read sha1 hash file $newhashfile\n";
	return -1;
    }
    chomp($filehash);
    if ($filehash =~ /^SHA1.*= (\w*)$/) {
	$filehash = $1;
    }
    else {
	print STDERR "Could not parse the sha1 hash: '$filehash'\n";
	return -1;
    }
    if ($filehash ne $newhash) {
	print STDERR "sha1 hash of new file did not match\n";
	return -1;
    }

    #
    # Use imagedump to verify the ndz file.
    #
    print "Verifying ndz file format ...\n";
    system("$IMAGEDUMP $localfile");
    if ($?) {
	return -1;
    }
    return 0
	if ($verify);

    #
    # Now rename the image files and update the hash file.
    #
Leigh B Stoller's avatar
Leigh B Stoller committed
557 558
    my $hashfile = $image->FullImageSHA1File();
    my $ndzfile  = $image->FullImageFile();
Leigh B Stoller's avatar
Leigh B Stoller committed
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
    unlink($hashfile)
	if (-e $hashfile);
    system("/bin/mv -f $newhashfile $hashfile");
    if ($?) {
	return -1;
    }
    if (-e $ndzfile) {
	system("/bin/mv -f $ndzfile ${ndzfile}.old");
	if ($?) {
	    return -1;
	}
    }
    system("/bin/mv -f $localfile $ndzfile");
    if ($?) {
	return -1;
    }
575 576 577 578 579 580

    #
    # Try to download a sig file. We have to accept that this might
    # fail, which is okay since Mike says we can generate a new one,
    # it just takes a while to do.
    #
581
    FetchSigFile($image);
Leigh B Stoller's avatar
Leigh B Stoller committed
582 583 584
    return 0;
}

585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
#
# Fetch a file.
#
sub FetchImageFile($$)
{
    my ($url, $localfile) = @_;
    my $safe_url  = User::escapeshellarg($url);
    
    #
    # Build up a new command line to do the fetch on ops
    # But no reason to do this if an admin, which is important
    # when the image is going into /usr/testbed/images.
    #
    if (!$user->IsAdmin()) {
	my $cmdargs = "$TB/bin/fetchtar.proxy -h -u $user_uid";
	my $glist = `/usr/bin/id -G $user_uid`;
	if ($glist =~ /^([\d ]*)$/) {
	    $glist = join(",", split(/\s+/, $1));
	}
	else {
	    print STDERR "Unexpected results from 'id -G $user': $glist\n";
	    return -1;
	}
	$cmdargs .= " -g '$glist' \"$safe_url\" $localfile";

	print "Downloading $url ...\n";
	if ($debug) {
	    print "$cmdargs\n";
	}
	$EUID = $UID = 0;
	system("sshtb -host $CONTROL $cmdargs ");
	if ($?) {
	    $EUID = $UID = $SAVEUID;
	    print STDERR "Fetch of image file failed\n";
	    return -1;
	}
	$UID = $SAVEUID;
    }
    else {
	print "Downloading $url ...\n";
	
	if (! open(GET, "| nice -15 $WGET --no-check-certificate ".
		   "--timeout=30 --waitretry=30 --retry-connrefused ".
		   "-q -O $localfile -i -")) {
	    print STDERR "Cannot start $WGET\n";
	    return -1;
	}
	print GET "$url\n";
	return -1
	    if (!close(GET));

	system("$SHA1 $localfile > ${localfile}.sha1");
	if ($?) {
	    print STDERR "Could not generate sha1 hash of $localfile\n";
	    return -1;
	}
    }
642
    return 0;
643 644
}

Leigh B Stoller's avatar
Leigh B Stoller committed
645 646 647 648 649 650
#
# Fetch the metadata from the provided URL. Return the XML parse,
#
sub FetchMetadata($)
{
    my ($url) = @_;
Leigh B Stoller's avatar
Leigh B Stoller committed
651
    $url .= "&clientversion=" . $METADATA_CLIENTVERSION;
Leigh B Stoller's avatar
Leigh B Stoller committed
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
    my $safe_url = User::escapeshellarg($url);
    my $xml  = "";
    my $opts = ($debug ? "" : "-q");
    my $cmd  = "$WGET $opts --no-check-certificate -O - $safe_url ";

    if ($debug) {
	print "$cmd\n";
    }
    open(META, "$cmd |") or
	fatal("Could not fetch metadata from $url");
    while (<META>) {
	$xml .= $_;
    }
    close(META)
	or fatal("Could not read metadata from $url");

    if ($xml eq "") {
	fatal("Failed to get metadata from $url");
    }

    my $xmlparse = eval { XMLin($xml,
				VarAttr => 'name',
				ContentKey => '-content',
				SuppressEmpty => undef); };
    fatal($@)
	if ($@);

    if ($debug) {
	print STDERR Dumper($xmlparse);
    }

    #
    # We are going to let newimageid do the real checking.
    #
    return $xmlparse;
}

689 690 691 692 693 694 695
#
# Fetch sig file.
#
sub FetchSigFile($)
{
    my ($image) = @_;
    my $image_url  = uri_unescape($image->imagefile_url()) . "&sigfile=1";
Leigh B Stoller's avatar
Leigh B Stoller committed
696 697
    my $localfile  = $image->FullImageSigFile() . ".new";
    my $sigfile    = $image->FullImageSigFile();
698 699 700 701 702 703 704 705 706 707
    
    if (! FetchImageFile($image_url, $localfile)) {
	system("/bin/mv -f $localfile $sigfile");
	# Do not need this.
	unlink("${localfile}.sha1")
	    if (-e "${localfile}.sha1");
    }
    return 0;
}

Leigh B Stoller's avatar
Leigh B Stoller committed
708 709 710 711 712 713 714 715 716
sub fatal($)
{
    my ($mesg) = @_;

    print STDERR "*** $0:\n".
	         "    $mesg\n";
    exit(-1);
}