clone_image.in 19.1 KB
Newer Older
1 2
#!/usr/bin/perl -w
#
3
# Copyright (c) 2000-2017, 2019 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
23 24 25 26 27 28 29
#
use English;
use strict;
use Getopt::Std;
use Data::Dumper;
use File::Temp qw(tempfile);
use CGI;
30
use File::Basename;
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47

#
# Clone an image (descriptor) from a node and then snapshot
# that node into the descriptor. Creates the descriptor if
# if it does not exist. The idea is to use all of the info
# from the current image descriptor that is loaded on the node
# to quickly create a new descriptor by inheriting all of the
# attributes of the original.
#
# We also want to support taking a snapshot of a previously
# created clone. To make everything work properly, require
# that the imagename exist in the experiment project, which
# ensures that we are operating on a clone, not an image in
# some other project or a system image.
#
sub usage()
{
48
    print("Usage: clone_image [-dwe] [-n | -s] <imagename> <node_id>\n".
49
	  "Options:\n".
50 51 52 53
	  "       -d      Turn on debug mode\n".
	  "       -e      Create a whole disk image\n".
	  "       -g 0,1  Override base image global setting\n".
	  "       -r 0,1  Override base image shared (within project) setting\n".
54
	  "       -s      Create descriptor but do not take snapshot\n".
55 56 57 58 59
	  "       -n      Impotent mode\n".
	  "       -F      Create a full image even if deltas are on\n".
	  "       -U      Tell prepare to update master password files\n".
	  "       -B uuid Setup a copyback to origin uuid after snapshot\n".
	  "       -w      Wait for image to be created\n");
60 61
    exit(-1);
}
62
my $optlist     = "densg:wFr:b:UB:";
63 64 65 66
my $debug       = 0;
my $wholedisk   = 0;
my $impotent    = 0;
my $nosnapshot  = 0;
67
my $isvirtnode  = 0;
68
my $waitmode    = 0;
69
my $nodelta     = 0; # To pass to create_image.
70 71
my $global      = 0;
my $shared      = 0;
72
my $update_prepare = 0;
73
my $origin_uuid;
74
my $bsname;
75 76
my $base_image;
my $image;
77 78 79 80 81 82

#
# Configure variables
#
my $TB           = "@prefix@";
my $PROJROOT     = "@PROJROOT_DIR@";
83
my $GROUPROOT    = "@GROUPSROOT_DIR@";
84 85
my $CREATEIMAGE  = "$TB/bin/create_image";
my $NEWIMAGEEZ   = "$TB/bin/newimageid_ez";
86
my $DOPROVENANCE = @IMAGEPROVENANCE@;
87
my $DOIMAGEDIRS  = @IMAGEDIRECTORIES@;
88
my $doprovenance = $DOPROVENANCE;
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108

#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use EmulabConstants;
use emutil;
use User;
use Project;
109 110
use OSImage;
use Image;   # For datasets
111
use Node;
112
use EmulabFeatures;
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136

# Protos
sub fatal($);

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"e"})) {
    $wholedisk = 1;
}
if (defined($options{"n"})) {
    $impotent = 1;
}
if (defined($options{"s"})) {
    $nosnapshot = 1;
}
137 138 139
if (defined($options{"w"})) {
    $waitmode = 1;
}
140 141 142
if (defined($options{"F"})) {
    $nodelta = 1;
}
143 144 145
if (defined($options{"U"})) {
    $update_prepare = 1;
}
146 147 148 149 150 151 152 153 154 155
if (defined($options{"B"})) {
    $origin_uuid = $options{"B"};

    if ($origin_uuid =~ /^([-\w]+)$/) {
	$origin_uuid = $1;
    }
    else {
	fatal("Bad data in $origin_uuid");
    }
}
156 157 158 159 160 161 162 163 164 165
if (defined($options{"b"})) {
    $bsname = $options{"b"};

    if ($bsname =~ /^([-\w]+)$/) {
	$bsname = $1;
    }
    else {
	fatal("Bad data in $bsname.");
    }
}
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
usage()
    if (@ARGV != 2);

my $imagename = shift(@ARGV);
my $node_id   = shift(@ARGV);

#
# Map invoking user to object. 
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    fatal("You ($UID) do not exist!");
}

#
# The node must of course be allocated and the user must have
# permission to clone it. 
#
my $node = Node->Lookup($node_id);
if (!defined($node)) {
    fatal("No such node");
}
188 189
$isvirtnode = $node->isvirtnode();

190 191 192
if (!$node->AccessCheck($this_user, TB_NODEACCESS_LOADIMAGE())) {
    fatal("Not enough permission");
}
193 194 195 196
if ($node->IsTainted()) {
    fatal("$node is tainted - image creation denied!");
}

197 198 199 200 201 202
my $experiment = $node->Reservation();
if (!defined($experiment)) {
    fatal("Node is not reserved");
}
my $pid     = $experiment->pid();
my $group   = $experiment->GetGroup();
203
my $gid     = $group->gid();
204 205 206 207
my $project = $experiment->GetProject();
if (! (defined($project) && defined($group))) {
    fatal("Could not get project/group for $experiment");
}
208 209 210 211 212 213 214 215 216 217
if (defined($bsname)) {
    #
    # Datasets are still special, they have no osinfo. 
    #
    $image = Image->Lookup($project->pid(), $imagename);
    if (!defined($image)) {
	fatal("Dataset must already exist before it can be cloned");
    }
    elsif (!$image->isdataset()) {
	fatal("$image is not a dataset for $bsname");
218 219 220 221
    }
    $base_image = $image;
}
else {
222 223 224
    $image = OSImage->Lookup($project->pid(), $imagename);
    $base_image = $node->RunningOsImage();

225
    # No support for cloning MFSs, so there will always be a base image.
226 227
    if (! defined($base_image)) {
	fatal("Could not determine osimage for $node_id");
228
    }
229
    print "$node_id is running $base_image\n"
230
	if ($debug);
231 232
}

233
# See if enabled.
234
if ($DOPROVENANCE) {
235 236 237 238 239
    # But allow feature override.
    if (EmulabFeatures->Lookup("ImageProvenance")) {
	$doprovenance =
	    EmulabFeatures->FeatureEnabled("ImageProvenance", undef, $group);
    }
240 241 242
    # Temporary override for all geni projects until we can export deltas.
    if ($project->IsNonLocal()) {
	$nodelta = 1;
243 244 245
    }
}

246 247 248 249 250
#
# The simple case is that the descriptor already exists. So it is just
# a simple snapshot to the image file. 
#
if (defined($image)) {
251 252
    my $needdelete = 0;
    
253
    #
254
    # Only EZ images or Datasets via this interface.
255
    #
256 257
    if (!($image->ezid() || $image->isdataset())) {
	fatal("Only EZ images or datasets on this path.");
258
    }
259

260 261 262 263 264 265 266 267 268
    #
    # Only project members or an admin can snapshot a noclone image.
    #
    if ($image->noclone() &&
	!$image->AccessCheck($this_user, TB_IMAGEID_CREATE) &&
	!$this_user->IsAdmin()) {
	fatal("You are not allowed to snapshot this image");
    }

269 270 271 272 273 274 275 276 277 278
    #
    # The access check above determines if the caller has permission
    # to overwrite the image file. 
    # Not that this matters, cause create_image is going to make the
    # same checks.
    #
    if ($impotent) {
	print "Not doing anything in impotent mode\n";
	exit(0);
    }
279 280 281 282 283 284 285

    #
    # Before we do anything destructive, we lock the image.
    #
    if ($image->Lock()) {
	fatal("Image is locked, please try again later!\n");
    }
286
    if ($DOPROVENANCE && $doprovenance && !$image->noversioning()) {
287 288 289 290 291 292 293
        #
	# This will include unreleased images (in image_versions, but
	# not the one pointed to by the images table). 
	#
	$image = $image->LookupMostRecent();
	if (!defined($image)) {
	    $image->Unlock();
294
	    fatal("Cannot lookup most recent version for $image");
295 296 297 298 299 300 301 302 303 304
	}
	
	#
	# We create a new version of the image descriptor for the new
	# snapshot. We mark it as not ready so that others know it is
	# in transition. When we later call createimage, it will make
	# sure the ready bit is clear before trying to use it.
	#
	my $needclone = 1;
	
305
	#
306 307 308 309
	# Does the most recent version in the table not have its ready bit set?
	# If so it means something went wrong with a previous image creation.
	# We can reuse it, but reset the provenance just in case the node got
	# reloaded.
310
	#
311 312
	# There is no provenance for datasets; strictly parent/child.
	#
313
	if (!$image->ready()) {
314 315
	    if (!$image->isdataset()) {
		$image->SetProvenance($base_image);
316
	    }
317 318 319 320
	    $needclone = 0;
	    print "Reusing image version " . $image->version() . ", ".
		"since it was never marked ready.\n";
	    
321
	}
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
	#
	# If the new image is going to based on the exact same base,
	# lets not create another new version, but overwrite the current
	# one. Save lots of space this way. We miss saving intermediate
	# versions, but this is a typical approach to getting an image
	# ready for use; change, snapshot, test, change, snapshot ... 
	#
	if ($needclone && !$image->released() && 
	    ($image->parent_imageid() == $base_image->imageid() &&
	     $image->parent_version() == $base_image->version())) {
	    # For create_image to be happy. 
	    $image->ClearReady(0);
	    $needclone = 0;
	    print "Reusing image version " . $image->version() . ", ".
		"since the base is the same and it was not released.\n";
	}

	if ($needclone) {
340 341
	    my $clone_error;
	    my $clone = $image->NewVersion($this_user,
342
					   $base_image, \$clone_error);
343 344 345 346 347 348 349 350 351 352 353 354
	    if (!defined($clone)) {
		$image->Unlock();
		fatal("Could not clone image descriptor" .
		      (defined($clone_error) ? ": $clone_error" : "") . "\n");
	    }
	    $image = $clone;
	    $needdelete = 1;

	    #
	    # Watch for a system image that is saved elsewhere; see equiv code
	    # in create_image. We change the path to point over to the /proj
	    # directory so that we do not burn up space on boss until it is
355
	    # officially "released". We *can* use this version of the image
356 357
	    # by explicitly using its version number, before it is released. 
	    #
Leigh Stoller's avatar
Leigh Stoller committed
358 359
	    if ($image->IsSystemImage()) {
		my $path;
360
		if ($image->IsDirPath()) {
Leigh Stoller's avatar
Leigh Stoller committed
361 362 363 364
		    $path = $image->SaveDir();
		}
		else {
		    $path = $image->SaveDir() . basename($image->path());
365
		}
366
		if ($image->Update({"path" => $path})) {
367
		    $image->PurgeVersion();
368
		    fatal("Could not update path!");
369 370
		}
	    }
371 372
	}
    }
373 374 375 376 377
    else {
	# For create_image to be happy. 
	$image->ClearReady(0);
    }
    
378 379 380 381 382 383 384 385 386 387 388 389 390
    #
    # If a wholedisk image was requested, we need to change the
    # descriptor, since it might not have started out as a whole disk
    # image, but then the user brought in a new partition on the disk
    # and wants it made part of the image. When provenance is on, we
    # will change just the new version, so the old versions of the
    # image will continue to work properly. But without versioning,
    # a failure will leave the image descriptor as a whole disk image,
    # but the image file will not be, and that will break. 
    #
    if ($wholedisk) {
	;
    }
391 392
    $image->Unlock();
    
393
    if ($nosnapshot) {
394 395
	print "Not taking a snapshot, as directed\n"
	    if ($debug);
396 397
	exit(0);
    }
398 399
    my $opts = "-p $pid ";
    $opts   .= "-w " if ($waitmode);
400
    $opts   .= "-F " if ($nodelta);
401
    $opts   .= "-U " if ($update_prepare);
402
    $opts   .= "-B $origin_uuid " if (defined($origin_uuid));
403
    $opts   .= "-b $bsname " if (defined($bsname));
404 405 406 407 408 409 410

    #
    # Mike says do not pass versioned imagenames to create_image when
    # provenance is turned off. 
    #
    $imagename  = $image->imagename();
    $imagename .= ":" . $image->version() if ($DOPROVENANCE && $doprovenance);
411
    
412
    my $output = emutil::ExecQuiet("$CREATEIMAGE $opts $imagename $node_id");
413
    if ($?) {
414
	$image->PurgeVersion()
415
	    if ($needdelete);
416 417 418 419 420 421
	print STDERR $output;
	fatal("Failed to create image");
    }
    print "Image is being created. This can take 15-30 minutes.\n";
    exit(0);
}
422
DoNew:
423 424

#
425
# Only EZ images via this interface.
426
#
427 428 429 430
if (!$base_image->ezid()) {
    fatal("Cannot clone a non-ez image");
}

431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
#
# Figure out the global/shared bits for below.
#
if (defined($options{"g"})) {
    $global = $options{"g"};
}
elsif (defined($options{"r"})) {
    $shared = $options{"r"};
}
elsif ($experiment->pid() eq $base_image->pid()) {
    # Not crossing projects, so inherit from the base image.
    if ($base_image->global()) {
	$global = 1;
    }
    elsif ($base_image->shared()) {
	$shared = 1;
    }
}
elsif ($base_image->IsSystemImage()) {
    # On the Geni Path, everything starts out global so it can be exported.
    # Note that if we import an non-global image into a project, we will
    # inherit that when cloning, via the clause above.
    if ($experiment->geniflags()) {
	$global = 1;
    }
}

458 459 460 461 462 463 464 465 466
#
# Only project members or an admin can clone a noclone image.
#
if ($base_image->noclone() &&
    !$base_image->AccessCheck($this_user, TB_IMAGEID_CREATE) &&
    !$this_user->IsAdmin()) {
    fatal("You are not allowed to clone $base_image");
}

467 468 469 470 471
#
# Not allowed to derive an image from one that has not been released.
# Maybe relax this in the future, but this is a good simplification for
# now.
#
472
if ($DOPROVENANCE && $doprovenance && !$base_image->released()) {
473 474 475 476
    fatal("Not allowed to derive a new image from unreleased ".
	  "base $base_image");
}

477 478 479 480 481 482
#
# To avoid confusion, we do not allow users to shadow system images
# in their own project. 
#
if (Image->LookupByName($imagename) && !$this_user->IsAdmin()) {
    fatal("Not allowed to shadow snapshot a system image");
483 484
}

485 486 487
# Subgroups change the path, but a global image should still
# go into the project image directory. 
my $path = ($experiment->pid() eq $experiment->gid() || $global ?
488 489
	    "$PROJROOT/$pid/images/" :
	    "$GROUPROOT/$pid/$gid/images/");
490
if ($DOIMAGEDIRS) {
Leigh Stoller's avatar
Leigh Stoller committed
491
    $path .= "${imagename}/";
492 493
}
else {
Leigh Stoller's avatar
Leigh Stoller committed
494
    $path .= "${imagename}.ndz";
495
}
496

497 498 499 500 501 502 503 504 505 506 507 508
#
# Create the image descriptor. We use the backend script to do the
# heavy lifting, but we have to cons up an XML file based on the image
# descriptor that is being cloned.
#
# These are the fields we have to come up with, plus a number
# of mtype_* entries.
#
my %xmlfields =
    ("imagename"	=> $imagename,
     "pid"		=> $project->pid(),
     "gid"		=> $experiment->gid(),
509 510 511
     "description"	=> $base_image->description(),
     "OS"		=> $base_image->OS(),
     "version"		=> $base_image->osversion(),
512
     "path"		=> $path,
513
     "op_mode",		=> $base_image->op_mode(),
514
     "wholedisk",	=> $wholedisk,
515
);
516 517 518 519 520
$xmlfields{"reboot_waittime"} = $base_image->reboot_waittime()
    if (defined($base_image->reboot_waittime()));
$xmlfields{"osfeatures"} = $base_image->osfeatures()
    if (defined($base_image->osfeatures()) &&
	$base_image->osfeatures() ne "");
521
if ($global) {
522 523
    $xmlfields{"global"} = 1;
}
524
elsif ($shared) {
525 526
    $xmlfields{"shared"} = 1;
}
Leigh Stoller's avatar
Leigh Stoller committed
527
    
528 529
if (defined($base_image)) {
    $xmlfields{"mbr_version"}     = $base_image->mbr_version();
530
    $xmlfields{"loadpart"}        = $base_image->loadpart();
531
    $xmlfields{"noexport"}        = $base_image->noexport();
532
    $xmlfields{"noclone"}         = $base_image->noclone();
533 534 535
    if ($base_image->format() ne 'ndz') {
	$xmlfields{"format"} = $base_image->format();
    }
536 537 538 539 540

    # Short form uses wholedisk instead. Should fix this. 
    if ($base_image->loadpart() == 0 && $base_image->loadlength() == 4) {
	$xmlfields{"wholedisk"}   = 1;
	$xmlfields{"loadpart"}    = 1;
541 542 543 544 545 546 547 548 549 550 551 552
	#
	# Ick. we have to tell newimageid_ez the correct loadpart, since
	# it uses that as the boot partition. 
	#
	for (my $i = 1; $i <= 4; $i++) {
	    my $func = "part${i}_osid";
	    my $foo  = $base_image->$func();
	    if (defined($foo) && $foo == $base_image->default_osid()) {
		$xmlfields{"loadpart"} = $i;
		last;
	    }
	}
553
    }
554 555 556 557 558 559 560 561 562 563 564
}
elsif ($isvirtnode) {
    $xmlfields{"reboot_waittime"} = 240;
    $xmlfields{"loadpart"}        = 1;
    $xmlfields{"mtype_pcvm"}      = 1;
    $xmlfields{"wholedisk"}       = 1;
}
else {
    fatal("No base image for $node_id");
}
# This needs more thought.
565 566 567 568
if ($base_image->def_parentosid()) {
    my $parentosimage = OSImage->Lookup($base_image->def_parentosid());
    if (!defined($parentosimage)) {
	fatal("Could not lookup object for def parent of $base_image");
569
    }
570
    $xmlfields{"def_parentosid"} =
571
	$parentosimage->pid() . "," . $parentosimage->imagename();
572 573

    # And this is just plain bogus. 
574
    #$xmlfields{"mbr_version"} = 99;
575
}
576

577 578 579 580 581 582 583
#
# Pass both architecture and typelist. newimageid_ez will figure out
# which one to use.
#
if ($base_image->architecture()) {
    $xmlfields{"architecture"} = $base_image->architecture();
}
584 585 586
#
# Grab the existing type list and generate new mtype_* variables.
#
587 588 589 590 591
my @typelist = $base_image->TypeList();
if (! @typelist && defined($base_image->deleted())) {
    my $sysimage = OSImage->LookupByName($base_image->imagename());
    if (defined($sysimage)) {
	@typelist = $sysimage->TypeList();
592
    }
593 594 595 596 597 598
}
if (! @typelist) {
    fatal("$base_image does not run on any types");
}
foreach my $type (@typelist) {
    my $type_id = $type->type();
599
    
600
    $xmlfields{"mtype_${type_id}"} = 1;
601
}
602

603 604 605 606 607 608 609 610 611 612 613
#
# If we are being told that this new image needs to be copied back to
# its original home, then copy the origin urn to new image from the
# old image.
#
if (defined($origin_uuid)) {
    if (! (defined($base_image) && defined($base_image->origin_urn()))) {
	fatal("No origin_urn in $base_image");
    }
    $xmlfields{"origin_urn"} = $base_image->origin_urn();
}
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632

#
# Create the XML file to pass to newimageid_ez.
#
my ($fh, $filename) = tempfile(UNLINK => 1);
fatal("Could not create temporary file")
    if (!defined($fh));

print $fh "<image>\n";
foreach my $key (keys(%xmlfields)) {
    my $value = $xmlfields{$key};

    print $fh "<attribute name=\"$key\">";
    print $fh "<value>" . CGI::escapeHTML($value) . "</value>";
    print $fh "</attribute>\n";
}
print $fh "</image>\n";
close($fh);

633 634 635 636 637
if ($debug) {
    system("/bin/cat $filename");
}

my $output = emutil::ExecQuiet("$NEWIMAGEEZ -s -v $filename");
638 639
if ($?) {
    print STDERR $output;
640 641
    my $foo = `cat $filename`;
    print STDERR $foo;
642 643 644 645 646 647 648
    fatal("Failed to verify image descriptor from $filename");
}
if ($impotent) {
    print "Not doing anything in impotent mode\n";
    system("cat $filename");
    exit(0);
}
649
$output = emutil::ExecQuiet("$NEWIMAGEEZ -s $filename");
650 651 652 653 654 655 656
if ($?) {
    print STDERR $output;
    my $foo = `cat $filename`;
    print STDERR $foo;
    fatal("Failed to create image descriptor");
}

657
$image = OSImage->Lookup($project->pid(), $imagename);
658 659 660
if (!defined($image)) {
    fatal("Cannot lookup newly created image for $imagename");
}
661
if ($DOPROVENANCE && $doprovenance) {
662
    $image->SetProvenance($base_image);
663 664 665 666 667 668 669 670
    #
    # The portal is telling us that this image is really a snapshot of
    # an image at another cluster which is not doing versioning, so we
    # do not do versioning here.
    #
    if ($origin_uuid && $base_image->noversioning()) {
	$image->SetNoVersioning(1);
    }
671
}
672 673 674 675
if ($debug) {
    print "Created $image\n";
}
if ($nosnapshot) {
676 677
    print "Not taking a snapshot, as directed\n"
	if ($debug);
678 679
    exit(0);
}
680 681 682 683 684 685 686

#
# XXX right now, don't create cross-image deltas.
# Makes image import easier.
#
$nodelta = 1;

687 688
my $opts = "-p $pid ";
$opts   .= "-w " if ($waitmode);
689
$opts   .= "-F " if ($nodelta);
690
$opts   .= "-U " if ($update_prepare);
691
$opts   .= "-B $origin_uuid " if (defined($origin_uuid));
692
$opts   .= "-b $bsname " if (defined($bsname));
693
$output  = emutil::ExecQuiet("$CREATEIMAGE $opts $imagename $node_id");
694 695 696 697 698 699 700 701 702 703 704 705 706 707
if ($?) {
    print STDERR $output;
    fatal("Failed to create image");
}
print "Image is being created. This can take 15-30 minutes.\n";
exit(0);

sub fatal($)
{
    my ($mesg) = @_;

    die("*** $0:\n".
	"    $mesg\n");
}