manage_instance.in 93.9 KB
Newer Older
1 2
#!/usr/bin/perl -w
#
3
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use English;
use strict;
use Getopt::Std;
use XML::Simple;
use Data::Dumper;
use CGI;
use POSIX ":sys_wait_h";
31
use POSIX qw(setsid strftime ceil floor);
32
use Date::Parse;
33 34 35 36 37 38

#
# Back-end script to manage APT profiles.
#
sub usage()
{
Leigh B Stoller's avatar
Leigh B Stoller committed
39 40
    print("Usage: manage_instance snapshot instance ".
	  "[-n node_id] [-i imagename] [-u node|all]\n");
41 42 43
    print("Usage: manage_instance consoleurl instance node\n");
    print("Usage: manage_instance terminate instance\n");
    print("Usage: manage_instance refresh instance\n");
Leigh B Stoller's avatar
Leigh B Stoller committed
44 45 46
    print("Usage: manage_instance reboot instance node_id ...\n");
    print("Usage: manage_instance reload instance node_id ...\n");
    print("Usage: manage_instance deletenodes instance node_id ...\n");
47
    print("Usage: manage_instance monitor instance\n");
48
    print("Usage: manage_instance lockdown instance set|clear user|admin\n");
49
    print("Usage: manage_instance panic instance set|clear\n");
50
    print("Usage: manage_instance linktest instance [-k | level]\n");
51
    print("Usage: manage_instance writecreds instance directory\n");
52
    print("Usage: manage_instance updatekeys instance [uid] \n");
53 54
    print("Usage: manage_instance extend instance [-m message] days [filename]\n");
    print("Usage: manage_instance denyextension instance [-m message] [filename]\n");
55
    print("Usage: manage_instance moreinfo instance [-m message] [filename]\n");
56
    print("Usage: manage_instance extendold instance [-f] seconds\n");
57
    print("Usage: manage_instance utilization instance\n");
58
    print("Usage: manage_instance schedterminate instance [-m message] days [filename]\n");
59
    print("Usage: manage_instance idledata instance\n");
60
    print("Usage: manage_instance openstackstats instance\n");
61 62
    exit(-1);
}
63
my $optlist     = "dt:s";
64
my $debug       = 0;
65
my $silent      = 0;
66
my $webtask_id;
67
my $webtask;
68 69
my $this_user;
my $geniuser;
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95

#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
my $QUICKVM     = "$TB/sbin/protogeni/quickvm";

#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use EmulabConstants;
use emdb;
use emutil;
96
use libEmulab;
97
use libtestbed;
98 99 100 101
use User;
use Project;
use APT_Profile;
use APT_Instance;
102
use APT_Geni;
103 104
use GeniXML;
use GeniHRN;
105 106 107
use Genixmlrpc;
use GeniResponse;
use GeniSlice;
108
use GeniImage;
109
use GeniUser;
110
use WebTask;
111
use EmulabFeatures;
112 113 114

# Protos
sub fatal($);
115
sub UserError($);
116
sub DoSnapshot();
117
sub DoConsole();
118
sub DoTerminate();
119
sub DoSchedTerminate();
120
sub DoExtend();
121
sub DoExtendOld();
122
sub DoDenyOrMoreInfo($);
123
sub DoRefresh();
124
sub DoReboot();
125
sub DoReload();
126
sub DoLockdown();
127
sub DoPanic();
128
sub DoManifests();
129
sub DoLinktest();
130
sub DoUpdateKeys();
Leigh B Stoller's avatar
Leigh B Stoller committed
131
sub DoDeleteNodes();
132
sub DoUtilization();
133
sub DoIdleData();
134
sub DoOpenstack();
135
sub WriteCredentials();
136
sub StartMonitor();
Leigh B Stoller's avatar
Leigh B Stoller committed
137
sub StartMonitorInternal(;$@);
138
sub DoImageTrackerStuff($$$$$$);
139 140
sub DenyExtensionInternal($);
sub ExtendInternal($$$$);
141 142 143 144 145 146 147 148 149 150 151 152

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"t"})) {
    $webtask_id = $options{"t"};
}
153 154 155
if (defined($options{"d"})) {
    $debug++;
}
156 157 158
if (defined($options{"s"})) {
    $silent = 1;
}
159
if (@ARGV < 2) {
160 161
    usage();
}
162
my $action   = shift(@ARGV);
163 164
my $uuid     = shift(@ARGV);
my $instance = APT_Instance->Lookup($uuid);
165 166 167
if (!defined($instance)) {
    $instance = APT_Instance->LookupBySlice($uuid);
}
168 169 170
if (!defined($instance)) {
    fatal("No such instance $uuid");
}
171 172 173 174 175 176 177 178 179 180
if (getpwuid($UID) eq "nobody") {
    $this_user = User->ImpliedUser();
}
else  {
    $this_user = User->ThisUser();
}
# If a guest user, we will not have an actual user, which is okay.
if (defined($this_user)) {
    $geniuser = GeniUser->CreateFromLocal($this_user);
}
181

182
if ($action eq "snapshot") {
183 184
    DoSnapshot();
}
185 186 187
if ($action eq "extend") {
    DoExtend();
}
188 189 190 191
if ($action eq "extendold") {
    DoExtendOld();
}
elsif ($action eq "denyextension") {
192 193 194 195
    DoDenyOrMoreInfo("deny")
}
elsif ($action eq "moreinfo") {
    DoDenyOrMoreInfo("info")
196
}
197
elsif ($action eq "consoleurl") {
198 199
    DoConsole()
}
200 201 202
elsif ($action eq "terminate") {
    DoTerminate()
}
203 204 205
elsif ($action eq "schedterminate") {
    DoSchedTerminate()
}
206 207 208
elsif ($action eq "refresh") {
    DoRefresh()
}
209 210 211
elsif ($action eq "reboot") {
    DoReboot()
}
212 213 214
elsif ($action eq "reload") {
    DoReload()
}
215 216 217
elsif ($action eq "monitor") {
    StartMonitor()
}
218 219 220
elsif ($action eq "lockdown") {
    DoLockdown()
}
221 222 223
elsif ($action eq "panic") {
    DoPanic()
}
224 225 226
elsif ($action eq "linktest") {
    DoLinktest()
}
227 228 229
elsif ($action eq "updatekeys") {
    DoUpdateKeys()
}
230 231 232
elsif ($action eq "writecreds") {
    WriteCredentials()
}
233 234 235
elsif ($action eq "getmanifests") {
    DoManifests()
}
Leigh B Stoller's avatar
Leigh B Stoller committed
236 237 238
elsif ($action eq "deletenodes") {
    DoDeleteNodes()
}
239 240 241
elsif ($action eq "utilization") {
    DoUtilization()
}
242 243 244
elsif ($action eq "idledata") {
    DoIdleData()
}
245 246 247
elsif ($action eq "openstackstats") {
    DoOpenstack()
}
248 249 250
else {
    usage();
}
251 252 253 254 255 256 257
exit(0);

#
# Take a snapshot. Implies a single node instance, for now.
#
sub DoSnapshot()
{
258 259
    my $errmsg;
    my $logfile;
260
    my $errcode        = -1;
261 262
    my $needunlock     = 0;
    my $old_status     = $instance->status();
263 264
    my $node_id;
    my $imagename;
Leigh B Stoller's avatar
Leigh B Stoller committed
265
    my $cloneprofile;
266
    my $update_profile;
267 268
    my $copyback_uuid;
    my $copyback_urn;
269
    my $update_prepare = 0;
270 271
    my $doversions = 0;
    my $usetracker = 0;
272

Leigh B Stoller's avatar
Leigh B Stoller committed
273
    my $optlist = "n:i:u:Uc:";
274 275 276 277 278 279 280 281 282 283
    my %options = ();
    if (! getopts($optlist, \%options)) {
	usage();
    }
    if (defined($options{"n"})) {
	$node_id = $options{"n"};
    }
    if (defined($options{"i"})) {
	$imagename = $options{"i"};
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
284 285 286
    if (defined($options{"c"})) {
	$cloneprofile = $options{"c"};
    }
287 288 289 290 291 292
    if (defined($options{"u"})) {
	$update_profile = $options{"u"};
	if ($update_profile !~ /^(node|all)$/) {
	    usage();
	}
    }
293 294 295
    if (defined($options{"U"})) {
	$update_prepare = 1;
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
296 297 298 299 300 301
    if (defined($cloneprofile) && defined($update_profile)) {
	fatal("Not allowed to update profile when cloning a profile");
    }
    if (defined($cloneprofile) && !defined($imagename)) {
	fatal("Must supply image name when cloning a profile");
    }
302 303 304 305 306 307 308
    if ($old_status ne "ready") {
	fatal("Instance must be in the ready state to take a snapshot");
    }
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for quick VM: $uuid");
    }
309
    
310
    #
311
    # Might be a clone (manage_profile).
312
    #
313
    my $sliver_urn;
314 315
    my $aggregate;
    my $node;
Leigh B Stoller's avatar
Leigh B Stoller committed
316 317 318 319 320 321 322 323
    my $profile;

    if (defined($cloneprofile)) {
	$profile = APT_Profile->Lookup($cloneprofile);
    }
    else {
	$profile = APT_Profile->Lookup($instance->profile_id());
    }
324
    if (!defined($profile)) {
Leigh B Stoller's avatar
Leigh B Stoller committed
325 326
	fatal("Could not lookup profile for " .
	      (defined($cloneprofile) ? "cloning" : "snapshot"));
327
    }
328 329
    my $project = Project->Lookup($profile->pid_idx());
    if (!defined($project)) {
Leigh B Stoller's avatar
Leigh B Stoller committed
330
	fatal("Could not lookup project for $profile");
331 332
    }
    
333
    #
334
    # Sanity checks. 
335
    #
336
    my @aggs = $instance->AggregateList();
337 338
    if (! @aggs) {
	fatal("No slivers for instance!");
339
    }
340
    if (!defined($node_id)) {
341
	# We snapshot the one node in the instance.
342 343 344 345 346 347 348 349 350
	if (@aggs != 1) {
	    fatal("Too many aggregates (> 1) to snapshot");
	}
	my ($agg) = @aggs;
	my $manifest = GeniXML::Parse($agg->manifest());
	if (! defined($manifest)) {
	    fatal("Could not parse manifest for $agg");
	}
	my @nodes = GeniXML::FindNodes("n:node", $manifest)->get_nodelist();
351 352 353
	if (@nodes != 1) {
	    fatal("Too many nodes (> 1) to snapshot");
	}
354
	($node)     = @nodes;
355
	$sliver_urn = GeniXML::GetSliverId($node);
356 357
	$node_id    = GeniXML::GetVirtualId($node);
	$aggregate  = $agg;
358 359 360
	# Profile Snapshot, always use the profile name. Clone passes in name.
	if (!defined($imagename)) {
	    $imagename = $profile->name();
361
	}
362
    }
363
    else {
364 365
	my $nodecount = 0;
	
366 367 368 369 370 371 372 373
	# Find the node in its manifest.
	foreach my $agg (@aggs) {
	    my $manifest = GeniXML::Parse($agg->manifest());
	    if (! defined($manifest)) {
		fatal("Could not parse manifest for $agg");
	    }
	    foreach my $ref (GeniXML::FindNodes("n:node",
						$manifest)->get_nodelist()) {
374 375
		$nodecount++;
		
376 377 378
		my $client_id   = GeniXML::GetVirtualId($ref);
		my $manager_urn = GetManagerId($ref);
		my $urn          = GeniXML::GetSliverId($ref);
379 380 381

		# No sliver urn or a different aggregate.
		next
382 383 384
		    if (! (defined($urn) &&
			   defined($manager_urn) &&
			   $manager_urn eq $agg->aggregate_urn()));
385 386 387 388 389 390 391

		if ($node_id eq $client_id) {
		    $node = $ref;
		    $sliver_urn = $urn;
		    $aggregate = $agg;
		    last;
		}
392 393 394 395
	    }
	}
	if (!defined($sliver_urn)) {
	    fatal("Could not find node '$node_id' in manifest");
396
	}
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
	#
	# So, we want Profile snapshot above (of a single node profile) and
	# Node snapshot in a single node profile to behave the same wrt the
	# image name, so look at the nodecount to see if need to append the
	# nodeid to the imagename. 
	#
	if (!defined($imagename)) {
	    $imagename = $profile->name();
	    if ($nodecount > 1) {
		$imagename .= "." . $node_id;
	    }
	}
    }
    #
    # Make sure a valid imagename. This a local test of course, but this
    # only works on IG aggregates anyway.
    #
    if (! TBcheck_dbslot($imagename, "images",
			 "imagename", TBDB_CHECKDBSLOT_ERROR)) {
	$imagename = $profile->profileid();
	$imagename .= "." . $node_id
	    if (defined($node_id));
    }
    
    #
    # Instruct the remote cluster to copy the image back to its origin,
    # but we need to ask the IMS for uuid of the image that is running,
    # so we can tell the cluster, which then tells the origin cluster.
Leigh B Stoller's avatar
Leigh B Stoller committed
425 426
    # We also need to know what the new URN of the image will be, for
    # updating the profile. 
427 428 429 430 431 432
    #
    if (GetSiteVar("protogeni/use_imagetracker") &&	
	EmulabFeatures->FeatureEnabled("APT_UseImageTracker",
					   $this_user, $project)) {
	$usetracker = 1;

Leigh B Stoller's avatar
Leigh B Stoller committed
433 434 435 436
	#
	# When cloning, we use the URN returned by the cluster; it is
	# the origin of the new image.
	#
437 438 439 440 441 442 443 444 445 446 447 448 449
	if (!defined($cloneprofile)) {
	    my $rval = DoImageTrackerStuff($aggregate, $node, $project,
					   \$copyback_uuid, \$copyback_urn,
					   \$errmsg);
	    if ($rval) {
		if ($rval < 0) {
		    fatal($errmsg);
		}
		else {
		    $errcode = 1;
		    goto uerror;
		}
	    }
450 451 452 453
	}
    }
    if (0) {
	fatal("$copyback_uuid, $copyback_urn\n");
454 455
    }

456
    #
457 458 459
    # We are not going to allow this if the instance is on a different
    # cluster then where the image was originally created, since otherwise
    # the image provenancewill look like spaghetti. 
460
    #
461
    if (defined($update_profile)) {
462 463
	my $diskref = GeniXML::GetDiskImage($node);
	if (defined($diskref)) {
464
	    my $authority = $aggregate->GetGeniAuthority();
465
	    my $image_url = GeniXML::GetText("url", $diskref);
466
	    if (defined($image_url) && !$usetracker) {
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
		require URI;

		# Get the hostname for the image URL.
		my $uri = URI->new($image_url);
		if (!defined($uri)) {
		    fatal("Could not parse $image_url");
		}
		my $image_host = $uri->host();

		# Get the hostname for the authority.
		$uri = URI->new($authority->url());
		if (!defined($uri)) {
		    fatal("Could not parse authority URL");
		}
		my $authority_host = $uri->host();

		# Compare domains.
		$image_host =~ s/^([^.]+\.)//;
		$authority_host =~ s/^([^.]+\.)//;
	
		if ($image_host ne $authority_host) {
		    $errmsg  = "Not allowed to take a snapshot on this cluster";
		    $errcode = 1;
490
		    goto uerror;
491 492 493
		}
	    }
	}
494 495 496 497
	# Do this here to avoid output to logfile.
	$doversions =
	    EmulabFeatures->FeatureEnabled("APT_ProfileVersions",
					   $this_user, $project);
498 499
    }
    if ($slice->Lock()) {
500 501 502
	$errmsg  = "Experiment is busy, please try again later.";
	$errcode = 1;
	goto uerror;
503 504
    }
    $needunlock = 1;
505 506 507 508 509 510 511 512 513

    #
    # Create the webtask object, but AFTER locking the slice so we do
    # not destroy one in use.
    #
    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
	# Convenient.
	$webtask->AutoStore(1);
514 515 516 517 518
	# This is convenience for the web server.
	if (defined($webtask)) {
	    $webtask->aggregate_urn($aggregate->aggregate_urn());
	    $webtask->client_id($node_id);
	}
519
    }
520
    $instance->SetStatus("imaging");
521
    $aggregate->SetStatus("imaging");
522

523 524
    #
    # This returns pretty fast, and then the imaging takes place in
525
    # the background at the aggregate. 
526
    #
527
    my $response =
528 529
	$aggregate->CreateImage($sliver_urn, $imagename,
				$update_prepare, $copyback_uuid);
530 531 532
    if (!defined($response)) {
	$errmsg = "Internal error creating image";
	$instance->SetStatus($old_status);
533
	$aggregate->SetStatus($old_status);
534
	goto uerror;
535
    }
536 537
    if ($response->code() != GENIRESPONSE_SUCCESS) {
	$errmsg = "Could not create image: " . $response->output() . "\n";
538 539
	$errcode = 1
	    if ($response->code() == GENIRESPONSE_BUSY ||
540 541
		$response->code() == GENIRESPONSE_SERVER_UNAVAILABLE ||
		$response->code() == GENIRESPONSE_FORBIDDEN);
542
	$instance->SetStatus($old_status);
543
	$aggregate->SetStatus($old_status);
544
	goto uerror;
545 546 547 548 549 550 551 552 553 554
    }
    my ($image_urn, $image_url,
	$version_urn, $version_url) = @{ $response->value() };
    if (!defined($version_urn)) {
	$version_urn = $image_urn;
	$version_url = $image_url
    }
    if (defined($webtask)) {
	$webtask->image_urn($version_urn);
	$webtask->image_url($version_url);
Leigh B Stoller's avatar
Leigh B Stoller committed
555
	my $image_name;
556

Leigh B Stoller's avatar
Leigh B Stoller committed
557 558 559 560 561 562 563 564 565 566 567 568 569 570
	if ($usetracker) {
	    # DoImageTrackerStuff determined that we use whatever the cluster
	    # tells us, cause it is the home of the image.
	    if (!defined($copyback_urn)) {
		$image_name = $version_urn;
	    }
	    else {
		$image_name = $copyback_urn;
	    }
	}
	else {
	    $image_name = $version_url;
	}
	$webtask->image_name($image_name);
571

Leigh B Stoller's avatar
Leigh B Stoller committed
572 573 574 575 576
	# We tell the web interface that the image has to be copied
	# back,
	if (defined($copyback_uuid)) {
	    $webtask->copyback_uuid($copyback_uuid);
	}
577 578
    }
    else {
579
	print "$image_urn,$image_url\n";
580 581 582 583 584 585
    }

    #
    # Exit and leave child to poll.
    #
    if (! $debug) {
586 587 588 589
        $logfile = TBMakeLogname("snapshot");
	
	if (my $childpid = TBBackGround($logfile)) {
	    # Parent exits normally, web interface watches.
590 591 592 593 594
	    exit(0);
	}
	# Let parent exit;
	sleep(2);
    }
595 596 597 598 599 600 601 602
    # Bind the process id. This is important when the caller is
    # manage_profile, doing a clone.
    $webtask->SetProcessID($PID)
	if (defined($webtask));

    #
    # Poll for a reasonable amount of time.
    #
603
    my $seconds  = 1500;
604
    my $interval = 15;
605
    my $ready    = 0;
606
    my $sliver_ready = 0;
607
    my $failed   = 0;
608

609
    while ($seconds > 0) {
610 611
	sleep($interval);
	$seconds -= $interval;
612
    
613
	my $response = $aggregate->SliceStatus();
614
	if ($response->code() != GENIRESPONSE_SUCCESS &&
615
	    $response->code() != GENIRESPONSE_RPCERROR &&
616
	    $response->code() != GENIRESPONSE_SERVER_UNAVAILABLE &&
617 618 619 620 621 622
	    $response->code() != GENIRESPONSE_BUSY) {
	    $errmsg = "Sliverstatus failed: ". $response->output() . "\n";
	    $failed = 1;
	    last;
	}
	next
623
	    if ($response->code() == GENIRESPONSE_BUSY ||
624
		$response->code() == GENIRESPONSE_SERVER_UNAVAILABLE ||
625
		$response->code() == GENIRESPONSE_RPCERROR);
626

627
	my $blob = $response->value();
628
	# This is the per-aggregate status, we always set this for web UI.
629
	$aggregate->UpdateWebStatus($blob->{'details'});
630
	
631 632 633 634
	if ($blob->{'status'} eq "failed") {
	    $failed = 1;
	    last;
	}
635 636 637 638
	elsif ($blob->{'status'} eq "ready") {
	    $sliver_ready = 1;
	}
	
639 640 641
	#
	# We are watching for the image status to report ready or failed.
	#
642
	$response = $aggregate->ImageInfo($image_urn);
643
	if ($response->code() != GENIRESPONSE_SUCCESS &&
644
	    $response->code() != GENIRESPONSE_RPCERROR &&
645
	    $response->code() != GENIRESPONSE_SERVER_UNAVAILABLE &&
646 647 648 649 650 651
	    $response->code() != GENIRESPONSE_BUSY) {
	    $errmsg = "Imageinfo failed: ". $response->output() . "\n";
	    $failed = 1;
	    last;
	}
	next
652
	    if ($response->code() == GENIRESPONSE_BUSY ||
653
		$response->code() == GENIRESPONSE_SERVER_UNAVAILABLE ||
654
		$response->code() == GENIRESPONSE_RPCERROR);
655

656
	my $imageblob = $response->value();
657
	if (defined($webtask)) {
658 659 660 661 662 663 664 665 666
	    my %blobcopy = %{ $imageblob };

	    #
	    # If the image is ready, but needs to be copied back to
	    # its origin, hold of ready till later. We will wait for
	    # the copyback to finish, see below.
	    #
	    if ($imageblob->{'status'} eq "ready" && defined($copyback_uuid)) {
		$blobcopy{'status'} = "copying";
Leigh B Stoller's avatar
Leigh B Stoller committed
667
	    }
668 669
	    # This is also being updated by the event system.
	    $instance->UpdateImageStatus(\%blobcopy);
670
	}
671
	if ($imageblob->{'status'} eq "ready") {
672 673 674
	    $ready = 1;
	    last;
	}
675
	elsif ($imageblob->{'status'} eq "failed") {
676 677 678
	    $failed = 1;
	    last;
	}
679
    }
680 681 682 683
    # Cause of image status events.
    $webtask->Refresh()
	if (defined($webtask));
    
684 685 686 687
    if ($failed) {
	$errmsg = "Imaging failed"
	    if (!defined($errmsg));
	goto bad;
688
    }
689 690
    elsif (!$ready) {
	$errmsg  = "Imaging timed out";
691
	$errcode = -2;
692 693
	goto bad;
    }
694
    elsif (defined($update_profile)) {
695 696 697 698 699 700
	#
	# If successful, we create a new version of the profile and
	# update the rspec to reflect the new image version. Note
	# that we expect the CM is doing image versioning, so do not
	# bother to check if the image version is actually new.
	#
701
	if ($doversions) {
702 703 704 705 706 707 708
	    $profile = $profile->NewVersion($this_user);
	    if (!defined($profile)) {
		print STDERR "Could not create new profile version\n";
		$webtask->Exited(70)
		    if (defined($webtask));
		exit(1);
	    }
709
	}
710
	# DoImageTrackerStuff determined that we use whatever the cluster
Leigh B Stoller's avatar
Leigh B Stoller committed
711
	# tells us, cause it is the home of the image.
712 713
	$copyback_urn = $version_urn
	    if ($usetracker && !defined($copyback_urn));
714

715 716 717
	$profile->UpdateDiskImage($node_id,
				  (defined($copyback_urn) ?
				   $copyback_urn : $version_url),
718
				  ($update_profile eq "all" ? 1 : 0));
719
    }
720
    $instance->SetStatus("ready");
721
    $aggregate->SetStatus("ready");
Leigh B Stoller's avatar
Leigh B Stoller committed
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
    
    #
    # If there is a copyback_uuid, we want to wait for that to finish.
    #
    if (defined($copyback_uuid)) {
	#
	# We know the copyback is done when the IMS has the info.
	#
	my $copied  = 0;
	$seconds  = 1000;

	while ($seconds > 0) {
	    sleep($interval);
	    $seconds -= $interval;

	    #
	    # It would clearly be more more efficient to just look in
	    # the IMS database. 
	    #
	    Genixmlrpc->SetContext(APT_Geni::GeniContext());
	    my $blob = GeniImage::GetImageData($copyback_urn, \$errmsg);
	    Genixmlrpc->SetContext(undef);
	    # We get back undefined if the image is not posted yet.
	    if (defined($blob)) {
		$copied = 1;
		last;
	    }
	    sleep($interval);
	}
	# Tell the web interface. 
	if (!$copied) {
	    $errmsg  = "Failed to copy image back to its origin cluster";
	    $errcode = 1;
	    goto bad;
	}
	elsif (defined($webtask)) {
	    $webtask->image_status("ready");
	}
    }
761 762 763 764 765
    # We garbage collect these later, so anyone waiting has a chance
    # to see the exit status
    $webtask->Exited(0)
	if (defined($webtask));
    $slice->UnLock();
766
    if (defined($logfile) && -s $logfile) {
767 768 769 770 771 772
	SENDMAIL($TBOPS,
		 "Instance Snapshot Complete",
		 "Finished taking snapshot of $instance.\n",
		 $TBOPS, undef, $logfile);
	unlink($logfile);
    }
773 774 775 776 777
    if (!$sliver_ready) {
	#
	# Image is ready, but sliver is not. Start a monitor so that
	# web interface is updated.
	#
Leigh B Stoller's avatar
Leigh B Stoller committed
778
	StartMonitorInternal();
779
    }
780
    exit(0);
781
  bad:
782
    if (!$sliver_ready) {
783
	#
784 785
	# Image is ready, but sliver is not. Start a monitor so that
	# web interface is updated.
786
	#
Leigh B Stoller's avatar
Leigh B Stoller committed
787
	StartMonitorInternal();
788
    }
789
    $instance->SetStatus("ready");
790
    $aggregate->SetStatus("ready");
791
    if (defined($logfile)) {
792 793 794 795 796
	SENDMAIL($TBOPS,
		 "Snapshot failed",
		 "Error taking snapshot of $instance:\n\n".
		 "$errmsg\n",
		 $TBOPS, undef, $logfile);
797 798
	unlink($logfile);
    }
799 800 801 802 803 804 805 806 807
  uerror:
    print STDERR "$errmsg\n";
    if (defined($errmsg) && defined($webtask)) {
	$webtask->Exited($errcode);
	$webtask->output($errmsg);
    }
    $slice->UnLock()
	if ($needunlock);

808
    exit($errcode);
809
}
810

811
sub DoImageTrackerStuff($$$$$$)
812
{
813
    my ($aggregate, $node, $project, $puuid, $purn, $perrmsg) = @_;
814 815 816 817 818 819 820 821 822 823 824 825 826 827
    my $node_id = GeniXML::GetVirtualId($node);
    my $errmsg;

    #
    # If we do not have a diskinfo section, we will use the URN we get back
    # from the cluster (it is a snapshot of the default image).
    #
    my $diskinfo = GeniXML::GetDiskImage($node);
    return 0
	if (!defined($diskinfo));

    #
    # This one needs more thought, it might be a URL.
    #
828 829 830 831 832
    my $image_token = GeniXML::GetText("name", $diskinfo);
    if (!defined($image_token)) {
	$image_token = GeniXML::GetText("url", $diskinfo);
	return 0
	    if (!defined($image_token));
833
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
834 835 836
    if (GeniHRN::IsValid($image_token)) {
	my ($auth,$ospid) = GeniHRN::ParseImage($image_token);
	if (!defined($ospid)) {
837
	    $$perrmsg = "Invalid image urn: $image_token";
Leigh B Stoller's avatar
Leigh B Stoller committed
838 839 840 841
	    return 1;
	}
    }
    
842
    Genixmlrpc->SetContext(APT_Geni::GeniContext());
843
    my $blob = GeniImage::GetImageData($image_token, \$errmsg);
844 845 846
    Genixmlrpc->SetContext(undef);
    
    if (!defined($blob)) {
Leigh B Stoller's avatar
Leigh B Stoller committed
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
	if (GeniHRN::IsValid($image_token)) {
	    #
	    # See if this is for a system image (emulab-ops). If it is,
	    # and the domain is not the MS, then retry with a MS URN.
	    #
	    # This is sorta temporary; at some point there will not be any
	    # profiles using the URNs that are not in the image tracker.
	    # Of course a user is free to set the URN to anything the want,
	    # which is why I expect this code to be here for a while. 
	    #
	    my $urn;
	    my $hrn = GeniHRN->new($image_token);
	    my ($auth,$ospid,$os,$vers) = $hrn->ParseImage();
	    if ($ospid eq TBOPSPID() && $auth ne "emulab.net") {
		$urn = GeniHRN::GenerateImage("emulab.net",
						 TBOPSPID(), $os, $vers);
863 864 865
		Genixmlrpc->SetContext(APT_Geni::GeniContext());
		$blob = GeniImage::GetImageData($urn, \$errmsg);
		Genixmlrpc->SetContext(undef);
Leigh B Stoller's avatar
Leigh B Stoller committed
866 867 868
	    }
	}
	if (!defined($blob)) {
869 870 871
	    $$perrmsg = "Could not get info from the image server for ".
		"$image_token:\n" . $errmsg;
	    return 1;
Leigh B Stoller's avatar
Leigh B Stoller committed
872
	}
873 874 875 876 877 878 879 880
    }
    #
    # System Image? We use the URN we get back from CreateSliver().
    # The cluster will be the origin for the new image.
    #
    return 0
	if ($blob->{'issystem'});

881
    my $image_urn     = $blob->{'urn'};
882 883 884 885 886 887 888 889 890 891
    my $copyback_uuid = $blob->{'version_uuid'};
    my $copyback_urn  = $image_urn;

    my $hrn = GeniHRN->Parse($image_urn);
    my (undef,$ospid,$os,$vers) = $hrn->ParseImage();

    #
    # What happens if the user is doing a snapshot on the cluster where
    # the image lives? The copyback (import) makes no sense in that case,
    # but what if its the same cluster but different projects? In this case
892 893
    # we want a standard image clone, and we use whatever URN the cluster
    # hands back to us.
894 895 896
    #
    # Aside; should we allow snapshots (in the web ui) across projects?
    #
897
    if (lc($hrn->domain()) eq lc($aggregate->domain())) {
898 899
	my $projhrn = GeniHRN->Parse($blob->{'project_urn'});
	if (!defined($projhrn)) {
900
	    $$perrmsg = "Could not parse " . $blob->{'project_urn'} . "\n";
901 902
	    return -1;
	}
903
	if (lc($projhrn->subauth()) eq lc($project->pid())) {
904 905 906
	    # We use the URN we get back from CreateSliver().
	    return 0;
	}
907 908
	# Ditto
	return 0;
909 910 911 912 913
    }

    #
    # If we are going to update the profile, we need to know what to
    # change the image urn to, and that depends on what version the
914 915 916 917 918
    # image is currently at, AT THE ORIGIN CLUSTER. The urn we get back
    # from the snapshotting cluster is not what we care about, we need
    # a urn for the origin cluster. But that depends on what version the
    # origin cluster is at (the highest numbered version). But if we are
    # doing a snapshot of an earlier version, we cannot generate the
Leigh B Stoller's avatar
Leigh B Stoller committed
919
    # version here, we have to ask what it will be. 
920 921
    #
    if ($blob->{'isversioned'}) {
922 923 924
	$copyback_urn = GeniHRN::GenerateImage($hrn->authority(),
					       $ospid, $os,
					       $blob->{'maxversion'} + 1);
925 926 927 928 929 930
    }
    $$puuid = $copyback_uuid;
    $$purn  = $copyback_urn;
    return 0;
}

931 932 933 934 935
#
# Ask the console URL for a node in an instance.
#
sub DoConsole()
{
936
    usage()
937 938
	if (!@ARGV);
    my $node_id = shift(@ARGV);
939 940 941 942 943 944

    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate(undef, $webtask_id);
	if (!defined($webtask)) {
	    fatal("Could not lookup/create webtask for $webtask_id");
	}
945 946
	# Convenient.
	$webtask->AutoStore(1);
947
    }
948 949 950 951 952 953
    
    #
    # Sanity check to make sure the node is really in the rspec, since
    # we need its sliver urn to ask for the console url.
    #
    my $sliver_urn;
954
    my $sliver;
955
    foreach my $obj ($instance->AggregateList()) {
956 957 958 959 960 961
	my $manifest = GeniXML::Parse($obj->manifest());
	if (! defined($manifest)) {
	    fatal("Could not parse manifest for $obj");
	}
	my @nodes = GeniXML::FindNodes("n:node", $manifest)->get_nodelist();
	foreach my $node (@nodes) {
Leigh B Stoller's avatar
Leigh B Stoller committed
962 963 964
	    my $client_id   = GeniXML::GetVirtualId($node);
	    my $urn         = GeniXML::GetSliverId($node);
	    my $manager_urn = GetManagerId($node);
965 966 967

	    # No sliver urn or a different aggregate.
	    next
Leigh B Stoller's avatar
Leigh B Stoller committed
968 969 970
		if (! (defined($urn) &&
		       defined($manager_urn) &&
		       $manager_urn eq $obj->aggregate_urn()));
971 972

	    if ($node_id eq $client_id) {
Leigh B Stoller's avatar
Leigh B Stoller committed
973
		$sliver_urn = $urn;
974 975
		$sliver = $obj;
	    }
976 977 978 979 980
	}
    }
    if (!defined($sliver_urn)) {
	fatal("Could not find node '$node_id' in manifest");
    }
981
    my $response = $sliver->ConsoleInfo($sliver_urn);
982
    if (!defined($response)) {
Leigh B Stoller's avatar
Leigh B Stoller committed
983 984
	fatal("RPC Error calling ConsoleInfo");
    }
985 986 987 988
    if ($response->code() == GENIRESPONSE_UNAVAILABLE) {
	print STDERR "Server says there is no console for $node_id\n";
	if (defined($webtask)) {
	    $webtask->output("Sorry, $node_id does not have a console line");
989 990 991 992 993 994 995 996
	    $webtask->Exited($response->code());
	}
	exit($response->code());
    }
    if ($response->code() == GENIRESPONSE_SEARCHFAILED) {
	print STDERR "Server says $node_id has been deallocated\n";
	if (defined($webtask)) {
	    $webtask->output("Sorry, $node_id has been deallocated");
997 998 999 1000
	    $webtask->Exited($response->code());
	}
	exit($response->code());
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
1001
    if ($response->code() != GENIRESPONSE_SUCCESS) {
1002
	$response = $sliver->ConsoleURL($sliver_urn);
Leigh B Stoller's avatar
Leigh B Stoller committed
1003 1004 1005 1006
	if (!defined($response)) {
	    fatal("RPC Error calling ConsoleURL");
	}
	if ($response->code() != GENIRESPONSE_SUCCESS) {
Leigh B Stoller's avatar
Leigh B Stoller committed
1007 1008 1009
	    if ($response->value()) {
		fatal($response->output());
	    }
1010 1011
	    fatal("Server returned error: " .
		  GENIRESPONSE_STRING($response->code));
Leigh B Stoller's avatar
Leigh B Stoller committed
1012 1013 1014 1015
	}
    }
    my $url;
    my $pswd;
1016
    my $logurl;
Leigh B Stoller's avatar
Leigh B Stoller committed
1017 1018 1019 1020 1021
	
    if (ref($response->value())) {
	$url  = $response->value()->{'url'};
	$pswd = $response->value()->{'password'}
	    if (exists($response->value()->{'password'}));
1022 1023 1024 1025
	$logurl = $response->value()->{'logurl'}
	    if (exists($response->value()->{'logurl'}));

	print Dumper($response->value());
Leigh B Stoller's avatar
Leigh B Stoller committed
1026 1027 1028
    }
    else {
	$url = $response->value();
1029
    }
1030
    if (defined($webtask)) {
Leigh B Stoller's avatar
Leigh B Stoller committed
1031 1032 1033 1034 1035 1036
	if ($response->code()) {
	    $webtask->output($response->output());
	}
	else {
	    $webtask->url($url);
	    $webtask->password($pswd) if (defined($pswd));
1037
	    $webtask->logurl($logurl) if (defined($logurl));
Leigh B Stoller's avatar
Leigh B Stoller committed
1038
	}
1039
	$webtask->Exited($response->code());
Leigh B Stoller's avatar
Leigh B Stoller committed
1040
	exit($response->code());
1041 1042 1043 1044 1045
    }
    # For command line operation too.
    if ($response->code()) {
	fatal($response->output());
    }
Leigh B Stoller's avatar
Leigh B Stoller committed
1046 1047
    print $url . "\n";
    print $pswd . "\n" if (defined($pswd));
1048
    print $logurl . "\n" if (defined($logurl));