Commit e468f885 authored by Leigh B Stoller's avatar Leigh B Stoller

Add image import utilities.

image_setup is run from tbprerun to verify and create image
descriptors, and then later from tbswap to actually download
and verify the image (ndz) file.

image_import does the actual work for a specific image (url).
parent 93f057a1
......@@ -46,7 +46,7 @@ SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
elabinelab snmpit.proxy panic node_attributes \
nfstrace plabinelab smbpasswd_setup smbpasswd_setup.proxy \
rmproj snmpit.proxynew snmpit.proxyv2 pool_daemon \
checknodes_daemon snmpit.proxyv3
checknodes_daemon snmpit.proxyv3 image_setup
ifeq ($(ISMAINSITE),1)
SBIN_STUFF += repos_daemon
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003-2012 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
use Socket;
#
# Fetch external image definitions and create local descriptors.
#
sub usage()
{
print "Usage: $0 [-d] [-v] [-g] eid\n";
print("Options:\n");
print(" -d - Turn on debugging\n");
print(" -v - Verify XML descriptions only\n");
print(" -g - Download image after creating descriptors\n");
exit(-1);
}
my $optlist = "dvg";
my $debug = 0;
my $verify = 0;
my $getimages= 0;
#
# Functions
#
sub verifyURL($);
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $IMPORTER = "$TB/sbin/image_import";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libtestbed;
use libdb;
use libtblog;
use Experiment;
use Image;
use User;
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"v"})) {
$verify = 1;
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"g"})) {
$getimages = 1;
}
if (@ARGV != 1) {
usage();
}
#
# Verify user and get his DB uid and other info for later.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
tbdie("You ($UID) do not exist!");
}
my $user_uid = $this_user->uid();
#
# First, make sure the experiment exists
#
my $experiment = Experiment->Lookup($ARGV[0]);
if (! $experiment) {
tbdie("There is no such experiment");
}
my $pid = $experiment->pid();
my $eid = $experiment->eid();
#
# User must have at least MODIFY permissions to use this script
#
if (!$experiment->AccessCheck($this_user, TB_EXPT_MODIFY())) {
tbdie("You are not allowed to modify experiment $eid in project $pid");
}
#
# Look for any nodes that specify a url for the osname.
#
my $result = $experiment->TableLookUp("virt_nodes", "vname,osname");
while (my ($vname, $osname) = $result->fetchrow()) {
my $url;
next
if (! ($osname =~ /^(ftp|http|https):/));
# Verify entire URL and taint check.
if ($osname =~ /^((http|https|ftp)\:\/\/[-\w\.\/\@\:\~\?\=\&]*)$/) {
$url = $1;
}
else {
tbdie("Invalid URL $osname\n");
}
my $safe_url = User::escapeshellarg($url);
#
# See if we have already created this descriptor. If so, we
# do not do anything until later when the experiment is
# being swapped in. At this point, we just want to verify
# the information and create the descriptor. Later we will
# fetch the image file, or refetch if it is stale.
#
my $image = Image->LookupByURL($url);
if (!defined($image)) {
my $opts = "";
$opts .= " -d"
if ($debug);
$opts .= " -v"
if ($verify);
system("$IMPORTER $opts -p $pid $safe_url");
exit(-1)
if ($?);
}
next
if ($verify);
$image = Image->LookupByURL($url);
if (!defined($image)) {
tbdie("Could not look up image object for $url\n");
}
next
if (! $getimages);
my $opts = "";
$opts .= " -d"
if ($debug);
system("$IMPORTER $opts -g -p $pid $safe_url");
exit(-1)
if ($?);
}
exit(0);
......@@ -30,7 +30,7 @@ SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
prereserve grantimage getimages localize_mfs \
management_iface sharevlan check-shared-bw \
addspecialdevice addspecialiface imagehash clone_image \
addvpubaddr imageinfo ctrladdr
addvpubaddr imageinfo ctrladdr image_import
WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \
webwanodecheckin webspewimage webdumpdescriptor
......@@ -44,7 +44,7 @@ CTRLSBIN_SCRIPTS= opsdb_control.proxy daemon_wrapper
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS = create_image
SETUID_SBIN_SCRIPTS = grabwebcams checkquota spewconlog opsdb_control suchown \
anonsendmail readblob
anonsendmail readblob image_import
SETUID_SUEXEC_SCRIPTS = xlogin
#
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010-2012 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use strict;
use Getopt::Std;
use XML::Simple;
use Data::Dumper;
use URI::Escape;
#
# Import an image from an external source.
#
sub usage()
{
print("Usage: import_image [-d] [-v] [-u <user>] [-g] -p pid <url>\n");
print(" import_image [-d] [-u <user>] [-g] -i <imageid>\n");
print("Options:\n");
print(" -d - Turn on debugging\n");
print(" -v - Verify XML description only\n");
print(" -g - Download image after creating descriptor\n");
print(" -u uid - Create image as user instead of caller\n");
print(" -p pid - Create image in the specified project\n");
print(" -i id - Update existing imported image.\n");
exit(-1);
}
my $optlist = "dvu:p:gi";
my $debug = 0;
my $verify = 0;
my $getimage= 0;
my $update = 0;
my $user;
my $group;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $TBAUDIT = "@TBAUDITEMAIL@";
my $TBGROUP_DIR = "@GROUPSROOT_DIR@";
my $TBPROJ_DIR = "@PROJROOT_DIR@";
my $TBBASE = "@TBBASE@";
my $CONTROL = "@USERNODE@";
my $WGET = "/usr/local/bin/wget";
my $NEWIMAGE_EZ = "$TB/bin/newimageid_ez";
my $IMAGEDUMP = "$TB/bin/imagedump";
my $SHA1 = "/sbin/sha1";
my $SAVEUID = $UID;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
die("*** $0:\n".
" Must be setuid! Maybe its a development version?\n");
}
#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use User;
use Project;
use Group;
use Image;
use OSinfo;
# Locals;
my $url;
# Protos
sub fatal($);
sub FetchMetadata($);
sub CreateImage($$$$);
sub DownLoadImage($$$$);
#
# There is no reason to run as root unless we need to ssh over
# to ops to fetch the URL.
#
$EUID = $UID;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"v"})) {
$verify = 1;
}
if (defined($options{"g"})) {
$getimage = 1;
}
if (defined($options{"i"})) {
$update = 1;
}
if (defined($options{"u"})) {
$user = User->Lookup($options{"u"});
fatal("User does not exist")
if (!defined($user));
}
if (defined($options{"p"})) {
$group = Group->Lookup($options{"p"});
if (!defined($group)) {
my $project = Project->Lookup($options{"p"});
fatal("Project/Group does not exist")
if (!defined($project));
$group = $project->GetProjectGroup();
fatal("Error getting project group for $project")
if (!defined($group));
}
}
if (!defined($user)) {
$user = User->ThisUser();
if (! defined($user)) {
fatal("You ($UID) do not exist!");
}
}
my $user_uid = $user->uid();
if ($update) {
usage()
if (!@ARGV);
my $image = Image->Lookup($ARGV[0]);
if (!defined($image)) {
fatal("Image descriptor does not exist");
}
if (!defined($image->metadata_url())) {
fatal("Not an imported image");
}
$url = $image->metadata_url();
# If the user is not an admin, must have perm on the image.
if (!$user->IsAdmin() &&
!$image->AccessCheck($user, TB_IMAGEID_CREATE())) {
fatal("$user does not have permission $image");
}
}
else {
usage()
if (! (@ARGV & defined($group)));
$url = $ARGV[0];
# We know this is valid, but must taint check anyway for shell command.
if ($url =~/^(.*)$/) {
$url = $1;
}
# If the user is not an admin, must be a member or have perm in
# the group.
if (!$user->IsAdmin() &&
!$group->AccessCheck($user, TB_PROJECT_MAKEIMAGEID())) {
fatal("$user does not have permission in $group");
}
}
my $xmlparse = FetchMetadata($url);
#
# Sanity checks; it must have a hash and a url inside. We let
# newimageid do the rest of the checks though.
#
if (! exists($xmlparse->{'attribute'}->{"hash"}) ||
! ($xmlparse->{'attribute'}->{"hash"}->{'value'} =~ /^\w{10,}$/)) {
fatal("Invalid hash in metadata");
}
if (! exists($xmlparse->{'attribute'}->{"imagefile_url"})) {
fatal("Invalid imagefile url in metadata");
}
#
# See if we already have an image in the DB for this URL.
# If not, we have to create it.
#
# Need to watch for two experiments causing this image to
# get created at the same time. It would be pretty silly,
# but users are users ...
#
my $safe_url = DBQuoteSpecial($url);
my $query_result = DBQueryWarn("select GET_LOCK($safe_url, 120)");
if (!$query_result ||
!$query_result->numrows) {
fatal("Could not get the SQL lock for a long time!");
}
my $image = Image->LookupByURL($url);
if (!defined($image)) {
$image = CreateImage($url, $xmlparse, $user, $group);
}
DBQueryWarn("select RELEASE_LOCK($safe_url)");
exit(0)
if ($verify);
#
# If the image has not been downloaded or if the hash has changed,
# get a new copy.
#
my $newhash = $xmlparse->{'attribute'}->{"hash"}->{'value'};
if ($getimage) {
#
# We need to get the lock since someone else could already
# be downloading it. Even worse, someone might be taking a local
# snapshot, although at the moment we do not support that.
#
if ($image->Lock()) {
print "$image is currently locked. Waiting ...\n";
my $maxwait = 600;
while ($maxwait > 0 && $image->WaitLock(60)) {
print "$image is currently locked. Waiting ...\n";
$maxwait -= 60;
}
if (!$image->GotLock()) {
fatal("Could not get the image lock after a long time");
}
}
if (! -e $image->path() || $newhash ne $image->hash()) {
if (DownLoadImage($image, $newhash, $user, $group)) {
$image->Unlock();
exit(1);
}
# Update the hash in the DB.
$image->SetHash($newhash);
}
$image->Unlock();
}
exit(0);
#
# Create a new image descriptor. We have to munge the XML file a bit
# though and write it out.
#
sub CreateImage($$$$)
{
my ($url, $xmlparse, $user, $group) = @_;
my $alltypes = "-a";
$xmlparse->{'attribute'}->{"pid"} = {};
$xmlparse->{'attribute'}->{"gid"} = {};
$xmlparse->{'attribute'}->{"pid"}->{'value'} = $group->pid();
$xmlparse->{'attribute'}->{"gid"}->{'value'} = $group->gid();
#
# Look for a parent osid; this means we should set the type
# to pcvm since the image is for a VM. Well, we also use this
# for subnodes, but I am not going to worry about that case.
#
if (exists($xmlparse->{'attribute'}->{"imagename"})) {
$xmlparse->{'attribute'}->{"mtype_pcvm"} = {};
$xmlparse->{'attribute'}->{"mtype_pcvm"}->{'value'} = 1;
$alltypes = "";
}
#
# We check to see if the imagename is already in use. Hopefully
# not, but if not we have to make something up. Note that I am
# not going to worry about concurrent attempts to create a descriptor
# with the same name.
#
if (! exists($xmlparse->{'attribute'}->{"imagename"})) {
$xmlparse->{'attribute'}->{"imagename"}->{'value'} =
substr(TBGenSecretKey(), 0, 12);
}
elsif (Image->Lookup($group->pid(),
$xmlparse->{'attribute'}->{"imagename"}->{'value'})) {
my $index = 1;
my $imagename;
do {
$imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
$imagename .= "_" . $index++;
} while ($index < 100 && Image->Lookup($group->pid(), $imagename));
if ($index >= 100) {
fatal("Could not generate a unique image name");
}
$xmlparse->{'attribute'}->{"imagename"}->{'value'} = $imagename;
}
my $imagename = $xmlparse->{'attribute'}->{"imagename"}->{'value'};
if ($debug) {
print STDERR "Using imagename: $imagename\n";
}
# do not trust path coming in.
$xmlparse->{'attribute'}->{"path"}->{'value'} =
"$TBPROJ_DIR/" . $group->pid() . "/images/${imagename}.ndz";
#
# Generate a new XML description to feed into newimageid.
#
$xmlparse->{'attribute'}->{"imagefile_url"}->{'value'} =
uri_escape($xmlparse->{'attribute'}->{"imagefile_url"}->{'value'});
$xmlparse->{'attribute'}->{"metadata_url"}->{'value'} = uri_escape($url);
my $newxml = "";
foreach my $key (keys(%{ $xmlparse->{'attribute'} })) {
my $value = $xmlparse->{'attribute'}->{"$key"}->{'value'};
$newxml .=
"<attribute name=\"$key\"><value>$value</value></attribute>\n";
}
$newxml = "<image>$newxml</image>";
if ($debug) {
print STDERR "$newxml\n";
}
# Verify first, Use skip admin checks option.
open(NEW, "| $NEWIMAGE_EZ $alltypes -s -v -")
or fatal("Cannot start $NEWIMAGE_EZ");
print NEW $newxml;
if (!close(NEW)) {
print STDERR "$newxml\n";
fatal("Image xml did not verify");
}
return undef
if ($verify);
open(NEW, "| $NEWIMAGE_EZ $alltypes -s -")
or fatal("Cannot start $NEWIMAGE_EZ");
print NEW $newxml;
if (!close(NEW)) {
print STDERR "$newxml\n";
fatal("Could not create new image from xml");
}
my $image = Image->LookupByURL($url);
if (!defined($image)) {
fatal("Could not lookup new image for $url");
}
return $image;
}
#
# Download the image file.
#
sub DownLoadImage($$$$)
{
my ($image, $user, $group) = @_;
my $image_url = uri_unescape($image->imagefile_url());
my $safe_url = User::escapeshellarg($image_url);
my $localfile = $image->path() . ".new";
#
# Build up a new command line to do the fetch on ops
#
my $cmdargs = "$TB/bin/fetchtar.proxy -h -u $user_uid";
my $glist = `/usr/bin/id -G $user_uid`;
if ($glist =~ /^([\d ]*)$/) {
$glist = join(",", split(/\s+/, $1));
}
else {
print STDERR "Unexpected results from 'id -G $user': $glist\n";
return -1;
}
$cmdargs .= " -g '$glist' \"$safe_url\" $localfile";
print "Downloading $image_url ...\n";
if ($debug) {
print "$cmdargs\n";
}
$EUID = $UID = 0;
system("sshtb -host $CONTROL $cmdargs ");
if ($?) {
$EUID = $UID = $SAVEUID;
print STDERR "Fetch of image file failed\n";
return -1;
}
$UID = $SAVEUID;
#
# Verify the hash.
#
my $newhashfile = $localfile . ".sha1";
print "Verifying the hash ...\n";
my $filehash = `cat $newhashfile`;
if ($?) {
print STDERR "Could not read sha1 hash file $newhashfile\n";
return -1;
}
chomp($filehash);
if ($filehash =~ /^SHA1.*= (\w*)$/) {
$filehash = $1;
}
else {
print STDERR "Could not parse the sha1 hash: '$filehash'\n";
return -1;
}
if ($filehash ne $newhash) {
print STDERR "sha1 hash of new file did not match\n";
return -1;
}
#
# Use imagedump to verify the ndz file.
#
print "Verifying ndz file format ...\n";
system("$IMAGEDUMP $localfile");
if ($?) {
return -1;
}
return 0
if ($verify);
#
# Now rename the image files and update the hash file.
#
my $hashfile = $image->path() . ".sha1";
my $ndzfile = $image->path();
unlink($hashfile)
if (-e $hashfile);
system("/bin/mv -f $newhashfile $hashfile");
if ($?) {
return -1;
}
if (-e $ndzfile) {
system("/bin/mv -f $ndzfile ${ndzfile}.old");
if ($?) {
return -1;
}
}
system("/bin/mv -f $localfile $ndzfile");
if ($?) {
return -1;
}
$EUID = $SAVEUID;
return 0;
}
#
# Fetch the metadata from the provided URL. Return the XML parse,
#
sub FetchMetadata($)
{
my ($url) = @_;
my $safe_url = User::escapeshellarg($url);
my $xml = "";
my $opts = ($debug ? "" : "-q");
my $cmd = "$WGET $opts --no-check-certificate -O - $safe_url ";
if ($debug) {