manage_instance.in 41.2 KB
Newer Older
1 2
#!/usr/bin/perl -w
#
3
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use English;
use strict;
use Getopt::Std;
use XML::Simple;
use Data::Dumper;
use CGI;
use POSIX ":sys_wait_h";
31
use POSIX qw(setsid close);
32 33 34 35 36 37

#
# Back-end script to manage APT profiles.
#
sub usage()
{
Leigh Stoller's avatar
Leigh Stoller committed
38 39
    print("Usage: manage_instance snapshot instance ".
	  "[-n node_id] [-i imagename] [-u node|all]\n");
40
    print("Usage: manage_instance consoleurl instance node\n");
41
    print("Usage: manage_instance extend instance [-f] seconds\n");
42 43
    print("Usage: manage_instance terminate instance\n");
    print("Usage: manage_instance refresh instance\n");
44
    print("Usage: manage_instance reboot instance node_id [node_id ...]\n");
45
    print("Usage: manage_instance reload instance node_id [node_id ...]\n");
46
    print("Usage: manage_instance monitor instance\n");
47
    print("Usage: manage_instance lockdown instance set|clear user|admin\n");
48
    print("Usage: manage_instance writecreds instance directory\n");
49 50
    exit(-1);
}
51
my $optlist     = "dt:s";
52
my $debug       = 0;
53
my $silent      = 0;
54
my $webtask_id;
55
my $webtask;
56 57 58 59 60 61 62 63

#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
my $QUICKVM     = "$TB/sbin/protogeni/quickvm";

64
# Debugging
Leigh Stoller's avatar
Leigh Stoller committed
65
my $usemydevtree = 0;
66

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use EmulabConstants;
use emdb;
use emutil;
85
use libEmulab;
86
use libtestbed;
87 88 89 90
use User;
use Project;
use APT_Profile;
use APT_Instance;
91
use APT_Geni;
92 93
use GeniXML;
use GeniHRN;
94 95 96
use Genixmlrpc;
use GeniResponse;
use GeniSlice;
97
use GeniImage;
98
use WebTask;
99
use EmulabFeatures;
100 101 102 103

# Protos
sub fatal($);
sub DoSnapshot();
104
sub DoConsole();
105 106 107
sub DoTerminate();
sub DoExtend();
sub DoRefresh();
108
sub DoReboot();
109
sub DoReload();
110
sub DoLockdown();
111
sub DoManifests();
112
sub WriteCredentials();
113
sub StartMonitor();
114
sub DoImageTrackerStuff($$$$$);
115 116 117 118 119 120 121 122 123 124 125 126

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"t"})) {
    $webtask_id = $options{"t"};
}
127 128 129
if (defined($options{"d"})) {
    $debug++;
}
130 131 132
if (defined($options{"s"})) {
    $silent = 1;
}
133
if (@ARGV < 2) {
134 135
    usage();
}
136
my $action   = shift(@ARGV);
137 138
my $uuid     = shift(@ARGV);
my $instance = APT_Instance->Lookup($uuid);
139 140 141
if (!defined($instance)) {
    $instance = APT_Instance->LookupBySlice($uuid);
}
142 143 144 145
if (!defined($instance)) {
    fatal("No such instance $uuid");
}

146
if ($action eq "snapshot") {
147 148
    DoSnapshot();
}
149 150 151 152
if ($action eq "extend") {
    DoExtend();
}
elsif ($action eq "consoleurl") {
153 154
    DoConsole()
}
155 156 157 158 159 160
elsif ($action eq "terminate") {
    DoTerminate()
}
elsif ($action eq "refresh") {
    DoRefresh()
}
161 162 163
elsif ($action eq "reboot") {
    DoReboot()
}
164 165 166
elsif ($action eq "reload") {
    DoReload()
}
167 168 169
elsif ($action eq "monitor") {
    StartMonitor()
}
170 171 172
elsif ($action eq "lockdown") {
    DoLockdown()
}
173 174 175
elsif ($action eq "writecreds") {
    WriteCredentials()
}
176 177 178
elsif ($action eq "getmanifests") {
    DoManifests()
}
179 180 181
else {
    usage();
}
182 183 184 185 186 187 188
exit(0);

#
# Take a snapshot. Implies a single node instance, for now.
#
sub DoSnapshot()
{
189 190 191 192 193
    my $errmsg;
    my $logfile;
    my $errcode        = 1;
    my $needunlock     = 0;
    my $old_status     = $instance->status();
194 195 196
    my $node_id;
    my $imagename;
    my $update_profile;
197 198
    my $copyback_uuid;
    my $copyback_urn;
199
    my $update_prepare = 0;
200 201
    my $doversions = 0;
    my $usetracker = 0;
202

203
    my $optlist = "n:i:u:U";
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
    my %options = ();
    if (! getopts($optlist, \%options)) {
	usage();
    }
    if (defined($options{"n"})) {
	$node_id = $options{"n"};
    }
    if (defined($options{"i"})) {
	$imagename = $options{"i"};
    }
    if (defined($options{"u"})) {
	$update_profile = $options{"u"};
	if ($update_profile !~ /^(node|all)$/) {
	    usage();
	}
    }
220 221 222
    if (defined($options{"U"})) {
	$update_prepare = 1;
    }
223 224 225 226 227 228 229 230
    
    if ($old_status ne "ready") {
	fatal("Instance must be in the ready state to take a snapshot");
    }
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for quick VM: $uuid");
    }
231
    
Leigh Stoller's avatar
Leigh Stoller committed
232 233 234 235 236 237 238 239
    # The web interface (and in the future the xmlrpc interface) sets this.
    my $this_user = User->ImpliedUser();
    if (! defined($this_user)) {
	$this_user = User->ThisUser();
	if (!defined($this_user)) {
	    fatal("You ($UID) do not exist!");
	}
    }
240
    
241
    #
242
    # Might be a clone (manage_profile).
243
    #
244
    my $sliver_urn;
245 246
    my $aggregate;
    my $node;
247
    
248
    my $profile = APT_Profile->Lookup($instance->profile_id());
249 250 251
    if (!defined($profile)) {
	fatal("Could not lookup profile for instance");
    }
252 253 254 255 256 257 258 259
    my $project = Project->Lookup($profile->pid_idx());
    if (!defined($project)) {
	fatal("Could not lookup project for profile");
    }
    
    if (defined($node_id)) {
	if (!defined($imagename)) {
	    $imagename = $profile->name() . "." . $node_id;
260
	}
261 262
    }
    else {
263 264
	if (!defined($imagename)) {
	    $imagename = $profile->name();
265
	}
266
    }
267 268 269 270 271 272 273
    #
    # Make sure a valid imagename. This a local test of course, but this
    # only works on IG aggregates anyway.
    #
    if (! TBcheck_dbslot($imagename, "images",
			 "imagename", TBDB_CHECKDBSLOT_ERROR)) {
	$imagename = $profile->profileid();
274
	$imagename .= "." . $node_id
275 276
	    if (defined($node_id));
    }
277

278
    #
279
    # Sanity checks. 
280
    #
281
    my @aggs = $instance->AggregateList();
282 283
    if (! @aggs) {
	fatal("No slivers for instance!");
284
    }
285
    if (!defined($node_id)) {
286
	# We snapshot the one node in the instance.
287 288 289 290 291 292 293 294 295
	if (@aggs != 1) {
	    fatal("Too many aggregates (> 1) to snapshot");
	}
	my ($agg) = @aggs;
	my $manifest = GeniXML::Parse($agg->manifest());
	if (! defined($manifest)) {
	    fatal("Could not parse manifest for $agg");
	}
	my @nodes = GeniXML::FindNodes("n:node", $manifest)->get_nodelist();
296 297 298
	if (@nodes != 1) {
	    fatal("Too many nodes (> 1) to snapshot");
	}
299
	($node)     = @nodes;
300
	$sliver_urn = GeniXML::GetSliverId($node);
301 302
	$node_id    = GeniXML::GetVirtualId($node);
	$aggregate  = $agg;
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
	#
	# Instruct the remote cluster to copy the image back to its origin,
	# but we need to ask the IMS for uuid of the image that is running,
	# so we can tell the cluster, which then tells the origin cluster.
	#
	if (GetSiteVar("protogeni/use_imagetracker") &&	
	    EmulabFeatures->FeatureEnabled("APT_UseImageTracker",
					   $this_user, $project)) {
	    #
	    # If there is no diskinfo or if it references a system image,
	    # then we have to wait until the snapshot is done to find out
	    # what the image was called, via the return from CreateImage().
	    #
	    $usetracker = 1;

	    if (DoImageTrackerStuff($agg, $node, $project,
				    \$copyback_uuid, \$copyback_urn)) {
		fatal("Image tracking error");
	    }
	}
323
    }
324
    else {
325 326 327 328 329 330 331 332
	# Find the node in its manifest.
	foreach my $agg (@aggs) {
	    my $manifest = GeniXML::Parse($agg->manifest());
	    if (! defined($manifest)) {
		fatal("Could not parse manifest for $agg");
	    }
	    foreach my $ref (GeniXML::FindNodes("n:node",
						$manifest)->get_nodelist()) {
333 334 335
		my $client_id   = GeniXML::GetVirtualId($ref);
		my $manager_urn = GetManagerId($ref);
		my $urn          = GeniXML::GetSliverId($ref);
336 337 338

		# No sliver urn or a different aggregate.
		next
339 340 341
		    if (! (defined($urn) &&
			   defined($manager_urn) &&
			   $manager_urn eq $agg->aggregate_urn()));
342 343 344 345 346 347 348

		if ($node_id eq $client_id) {
		    $node = $ref;
		    $sliver_urn = $urn;
		    $aggregate = $agg;
		    last;
		}
349 350 351 352
	    }
	}
	if (!defined($sliver_urn)) {
	    fatal("Could not find node '$node_id' in manifest");
353 354 355
	}
    }

356
    #
357 358 359
    # We are not going to allow this if the instance is on a different
    # cluster then where the image was originally created, since otherwise
    # the image provenancewill look like spaghetti. 
360
    #
361
    if (defined($update_profile)) {
362 363
	my $diskref = GeniXML::GetDiskImage($node);
	if (defined($diskref)) {
364
	    my $authority = $aggregate->GetGeniAuthority();
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
	    my $image_url = GeniXML::GetText("url", $diskref);
	    if (defined($image_url)) {
		require URI;

		# Get the hostname for the image URL.
		my $uri = URI->new($image_url);
		if (!defined($uri)) {
		    fatal("Could not parse $image_url");
		}
		my $image_host = $uri->host();

		# Get the hostname for the authority.
		$uri = URI->new($authority->url());
		if (!defined($uri)) {
		    fatal("Could not parse authority URL");
		}
		my $authority_host = $uri->host();

		# Compare domains.
		$image_host =~ s/^([^.]+\.)//;
		$authority_host =~ s/^([^.]+\.)//;
	
		if ($image_host ne $authority_host) {
		    $errmsg  = "Not allowed to take a snapshot on this cluster";
		    $errcode = 1;
		    goto bad;
		}
	    }
	}
394 395 396 397
	# Do this here to avoid output to logfile.
	$doversions =
	    EmulabFeatures->FeatureEnabled("APT_ProfileVersions",
					   $this_user, $project);
398 399 400 401 402
    }
    if ($slice->Lock()) {
	fatal("Slice is busy, cannot lock it");
    }
    $needunlock = 1;
403 404 405 406 407 408 409 410 411 412 413

    #
    # Create the webtask object, but AFTER locking the slice so we do
    # not destroy one in use.
    #
    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
	# Convenient.
	$webtask->AutoStore(1);
    }

414 415
    #
    # This returns pretty fast, and then the imaging takes place in
416
    # the background at the aggregate. 
417
    #
418
    my $response =
419 420
	$aggregate->CreateImage($sliver_urn, $imagename,
				$update_prepare, $copyback_uuid);
421 422 423 424
    if (!defined($response)) {
	$errmsg = "Internal error creating image";
	$instance->SetStatus($old_status);
	goto bad;
425
    }
426 427 428 429 430 431 432 433 434 435 436 437 438 439
    if ($response->code() != GENIRESPONSE_SUCCESS) {
	$errmsg = "Could not create image: " . $response->output() . "\n";
	$instance->SetStatus($old_status);
	goto bad;
    }
    my ($image_urn, $image_url,
	$version_urn, $version_url) = @{ $response->value() };
    if (!defined($version_urn)) {
	$version_urn = $image_urn;
	$version_url = $image_url
    }
    if (defined($webtask)) {
	$webtask->image_urn($version_urn);
	$webtask->image_url($version_url);
440 441
    }
    else {
442
	print "$image_urn,$image_url\n";
443 444 445 446 447 448
    }

    #
    # Exit and leave child to poll.
    #
    if (! $debug) {
449 450 451 452
        $logfile = TBMakeLogname("snapshot");
	
	if (my $childpid = TBBackGround($logfile)) {
	    # Parent exits normally, web interface watches.
453 454 455 456 457
	    exit(0);
	}
	# Let parent exit;
	sleep(2);
    }
458 459 460 461 462 463 464 465
    # Bind the process id. This is important when the caller is
    # manage_profile, doing a clone.
    $webtask->SetProcessID($PID)
	if (defined($webtask));

    #
    # Poll for a reasonable amount of time.
    #
466 467
    my $seconds  = 1500;
    my $interval = 10;
468
    my $ready    = 0;
469
    my $sliver_ready = 0;
470
    my $failed   = 0;
471

472
    while ($seconds > 0) {
473 474
	sleep($interval);
	$seconds -= $interval;
475
    
476
	my $response = $aggregate->SliceStatus();
477
	if ($response->code() != GENIRESPONSE_SUCCESS &&
478
	    $response->code() != GENIRESPONSE_RPCERROR &&
479 480 481 482 483 484
	    $response->code() != GENIRESPONSE_BUSY) {
	    $errmsg = "Sliverstatus failed: ". $response->output() . "\n";
	    $failed = 1;
	    last;
	}
	next
485 486
	    if ($response->code() == GENIRESPONSE_BUSY ||
		$response->code() == GENIRESPONSE_RPCERROR);
487

488 489
	my $blob = $response->value();
	if (defined($webtask)) {
490
	    # Special for imaging status display
491 492
	    foreach my $urn (keys(%{$blob->{'details'}})) {
		my $details = $blob->{'details'}->{$urn};
493
		
494 495 496 497
		if ($urn eq $sliver_urn) {
		    $webtask->state($details->{'state'});
		    $webtask->rawstate($details->{'rawstate'});
		}
498 499
	    }
	}
500 501 502 503 504 505 506 507 508
	# This is the per-aggregate status, we always set this for web UI.
	my $statusblob = {};
	foreach my $urn (keys(%{$blob->{'details'}})) {
	    my $details = $blob->{'details'}->{$urn};
	    my $node_id = $details->{'client_id'};
	    $statusblob->{$node_id} = $details;
	}
	$aggregate->webtask()->sliverstatus($statusblob);
	
509 510 511 512
	if ($blob->{'status'} eq "failed") {
	    $failed = 1;
	    last;
	}
513 514 515 516
	elsif ($blob->{'status'} eq "ready") {
	    $sliver_ready = 1;
	}
	
517 518 519
	#
	# We are watching for the image status to report ready or failed.
	#
520
	$response = $aggregate->ImageInfo($image_urn);
521
	if ($response->code() != GENIRESPONSE_SUCCESS &&
522
	    $response->code() != GENIRESPONSE_RPCERROR &&
523 524 525 526 527 528
	    $response->code() != GENIRESPONSE_BUSY) {
	    $errmsg = "Imageinfo failed: ". $response->output() . "\n";
	    $failed = 1;
	    last;
	}
	next
529 530
	    if ($response->code() == GENIRESPONSE_BUSY ||
		$response->code() == GENIRESPONSE_RPCERROR);
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546

	$blob = $response->value();
	if (defined($webtask)) {
	    $webtask->image_size($blob->{'size'}) 	
		if (exists($blob->{'size'}));
	    $webtask->image_status($blob->{'status'})
		if (exists($blob->{'status'}));
	}
	if ($blob->{'status'} eq "ready") {
	    $ready = 1;
	    last;
	}
	elsif ($blob->{'status'} eq "failed") {
	    $failed = 1;
	    last;
	}
547
    }
548 549 550 551 552
    if ($failed) {
	$errmsg = "Imaging failed"
	    if (!defined($errmsg));
	$errcode = 1;
	goto bad;
553
    }
554 555 556 557 558
    elsif (!$ready) {
	$errmsg  = "Imaging timed out";
	$errcode = 60;
	goto bad;
    }
559
    elsif (defined($update_profile)) {
560 561 562 563 564 565
	#
	# If successful, we create a new version of the profile and
	# update the rspec to reflect the new image version. Note
	# that we expect the CM is doing image versioning, so do not
	# bother to check if the image version is actually new.
	#
566
	if ($doversions) {
567 568 569 570 571 572 573
	    $profile = $profile->NewVersion($this_user);
	    if (!defined($profile)) {
		print STDERR "Could not create new profile version\n";
		$webtask->Exited(70)
		    if (defined($webtask));
		exit(1);
	    }
574
	}
575 576 577 578 579 580 581 582
	# DoImageTrackerStuff determined that we use whatever the cluster
	# tells us, it is the home of the image.
	$copyback_urn = $version_urn
	    if ($usetracker && !defined($copyback_urn));
	
	$profile->UpdateDiskImage($node_id,
				  (defined($copyback_urn) ?
				   $copyback_urn : $version_url),
583
				  ($update_profile eq "all" ? 1 : 0));
584
    }
585 586 587 588 589 590
    $instance->SetStatus("ready");
    # We garbage collect these later, so anyone waiting has a chance
    # to see the exit status
    $webtask->Exited(0)
	if (defined($webtask));
    $slice->UnLock();
591
    if (defined($logfile) && -s $logfile) {
592 593 594 595 596 597
	SENDMAIL($TBOPS,
		 "Instance Snapshot Complete",
		 "Finished taking snapshot of $instance.\n",
		 $TBOPS, undef, $logfile);
	unlink($logfile);
    }
598 599 600 601 602 603 604
    if (!$sliver_ready) {
	#
	# Image is ready, but sliver is not. Start a monitor so that
	# web interface is updated.
	#
	StartMonitor();
    }
605
    exit(0);
606
  bad:
607 608 609 610 611 612 613 614 615 616 617
    if ($sliver_ready) {
	#
	# If the sliver comes back ready in spite of the imaging failure,
	# then change the instance back to ready. User will already know
	# that the imaging failed.
	#
	$instance->SetStatus("ready");
    }
    else {
	$instance->SetStatus("imaging-failed");
    }
618 619 620 621 622
    print STDERR "$errmsg\n";
    if (defined($errmsg)) {
	$webtask->Exited($errcode);
	$webtask->output($errmsg);
    }
623 624
    $slice->UnLock()
	if ($needunlock);
625
    if (defined($logfile)) {
626 627 628 629 630
	SENDMAIL($TBOPS,
		 "Snapshot failed",
		 "Error taking snapshot of $instance:\n\n".
		 "$errmsg\n",
		 $TBOPS, undef, $logfile);
631 632 633
	unlink($logfile);
    }
    exit($errcode);
634
}
635

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
sub DoImageTrackerStuff($$$$$)
{
    my ($aggregate, $node, $project, $puuid, $purn) = @_;
    my $node_id = GeniXML::GetVirtualId($node);
    my $errmsg;

    #
    # If we do not have a diskinfo section, we will use the URN we get back
    # from the cluster (it is a snapshot of the default image).
    #
    my $diskinfo = GeniXML::GetDiskImage($node);
    return 0
	if (!defined($diskinfo));

    #
    # This one needs more thought, it might be a URL.
    #
    my $image_urn = GeniXML::GetText("name", $diskinfo);
    return 0
	if (!defined($diskinfo));
    
    if (!GeniHRN::IsValid($image_urn)) {
	print STDERR "Invalid disk image URN for $node_id\n";
	return -1;
    }
			
    Genixmlrpc->SetContext(APT_Geni::GeniContext());
    my $blob = GeniImage::GetImageData($image_urn, \$errmsg);
    Genixmlrpc->SetContext(undef);
    
    if (!defined($blob)) {
	print STDERR "Could not get info from the IMS for ".
	    "$image_urn:\n" . $errmsg . "\n";
	return -1;
    }
    #
    # System Image? We use the URN we get back from CreateSliver().
    # The cluster will be the origin for the new image.
    #
    return 0
	if ($blob->{'issystem'});

    my $copyback_uuid = $blob->{'version_uuid'};
    my $copyback_urn  = $image_urn;

    my $hrn = GeniHRN->Parse($image_urn);
    my (undef,$ospid,$os,$vers) = $hrn->ParseImage();

    #
    # What happens if the user is doing a snapshot on the cluster where
    # the image lives? The copyback (import) makes no sense in that case,
    # but what if its the same cluster but different projects? In this case
    # a copyback is not wrong, but sure is inefficient.
    #
    # Aside; should we allow snapshots (in the web ui) across projects?
    #
    if ($hrn->domain() eq $aggregate->domain()) {
	my $projhrn = GeniHRN->Parse($blob->{'project_urn'});
	if (!defined($projhrn)) {
	    print STDERR "Could not parse " . $blob->{'project_urn'} . "\n";
	    return -1;
	}
	if ($projhrn->subauth() eq $project->pid()) {
	    # We use the URN we get back from CreateSliver().
	    return 0;
	}
    }

    #
    # If we are going to update the profile, we need to know what to
    # change the image urn to, and that depends on what version the
    # image is currently at (if the image is being versioned).
    #
    if ($blob->{'isversioned'}) {
	if (defined($vers)) {
	    $vers++;
	    $copyback_urn = GeniHRN::GenerateImage($hrn->authority(),
						   $ospid, $os, $vers);
	}
    }
    $$puuid = $copyback_uuid;
    $$purn  = $copyback_urn;
    return 0;
}

721 722 723 724 725
#
# Ask the console URL for a node in an instance.
#
sub DoConsole()
{
726
    usage()
727 728
	if (!@ARGV);
    my $node_id = shift(@ARGV);
729 730 731 732 733 734

    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate(undef, $webtask_id);
	if (!defined($webtask)) {
	    fatal("Could not lookup/create webtask for $webtask_id");
	}
735 736
	# Convenient.
	$webtask->AutoStore(1);
737
    }
738 739 740 741 742 743
    
    #
    # Sanity check to make sure the node is really in the rspec, since
    # we need its sliver urn to ask for the console url.
    #
    my $sliver_urn;
744
    my $sliver;
745
    foreach my $obj ($instance->AggregateList()) {
746 747 748 749 750 751
	my $manifest = GeniXML::Parse($obj->manifest());
	if (! defined($manifest)) {
	    fatal("Could not parse manifest for $obj");
	}
	my @nodes = GeniXML::FindNodes("n:node", $manifest)->get_nodelist();
	foreach my $node (@nodes) {
752 753 754
	    my $client_id   = GeniXML::GetVirtualId($node);
	    my $urn         = GeniXML::GetSliverId($node);
	    my $manager_urn = GetManagerId($node);
755 756 757

	    # No sliver urn or a different aggregate.
	    next
758 759 760
		if (! (defined($urn) &&
		       defined($manager_urn) &&
		       $manager_urn eq $obj->aggregate_urn()));
761 762

	    if ($node_id eq $client_id) {
763
		$sliver_urn = $urn;
764 765
		$sliver = $obj;
	    }
766 767 768 769 770
	}
    }
    if (!defined($sliver_urn)) {
	fatal("Could not find node '$node_id' in manifest");
    }
771
    my $response = $sliver->ConsoleInfo($sliver_urn);
772
    if (!defined($response)) {
773 774 775
	fatal("RPC Error calling ConsoleInfo");
    }
    if ($response->code() != GENIRESPONSE_SUCCESS) {
776
	$response = $sliver->ConsoleURL($sliver_urn);
777 778 779 780
	if (!defined($response)) {
	    fatal("RPC Error calling ConsoleURL");
	}
	if ($response->code() != GENIRESPONSE_SUCCESS) {
Leigh Stoller's avatar
Leigh Stoller committed
781 782 783
	    if ($response->value()) {
		fatal($response->output());
	    }
784 785
	    fatal("Server returned error: " .
		  GENIRESPONSE_STRING($response->code));
786 787 788 789
	}
    }
    my $url;
    my $pswd;
790
    my $logurl;
791 792 793 794 795
	
    if (ref($response->value())) {
	$url  = $response->value()->{'url'};
	$pswd = $response->value()->{'password'}
	    if (exists($response->value()->{'password'}));
796 797 798 799
	$logurl = $response->value()->{'logurl'}
	    if (exists($response->value()->{'logurl'}));

	print Dumper($response->value());
800 801 802
    }
    else {
	$url = $response->value();
803
    }
804
    if (defined($webtask)) {
805 806 807 808 809 810
	if ($response->code()) {
	    $webtask->output($response->output());
	}
	else {
	    $webtask->url($url);
	    $webtask->password($pswd) if (defined($pswd));
811
	    $webtask->logurl($logurl) if (defined($logurl));
812
	}
813
	$webtask->Exited($response->code());
814
	exit($response->code());
815 816 817 818 819
    }
    # For command line operation too.
    if ($response->code()) {
	fatal($response->output());
    }
820 821
    print $url . "\n";
    print $pswd . "\n" if (defined($pswd));
822
    print $logurl . "\n" if (defined($logurl));
823 824
    exit(0);
}
825

826 827 828 829 830 831 832
#
# Terminate
#
sub DoTerminate()
{
    my $errmsg;
    my $logfile;
833 834 835 836 837 838 839 840 841 842 843
    my $expired = $RECORDHISTORY_TERMINATED;

    if (@ARGV) {
	my $arg = shift(@ARGV);
	if ($arg eq "-e") {
	    $expired = $RECORDHISTORY_EXPIRED;
	}
	else {
	    usage();
	}
    }
844 845 846
    
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
847 848 849 850
	#
	# No slice (typically) means we never got far enough to the
	# get the sliver created on the backend cluster.
	#
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
	goto killit;
    }
    #
    # Lock the slice in case it is doing something else, like taking
    # a disk image.
    #
    if ($slice->Lock()) {
	fatal("Slice is busy, cannot lock it");
    }
    my $old_status = $instance->status();
    $instance->SetStatus("terminating");

    #
    # Exit and let caller poll for status.
    #
866
    if (!$debug) {
867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
        $logfile = TBMakeLogname("terminate");
	
	if (my $childpid = TBBackGround($logfile)) {
	    my $status = 0;
	    #
	    # Wait a couple of seconds to see if there is going to be an
	    # immediate error. Then return and let it continue to run. This
	    # allows the web server to see quick errors. Later errors will
	    # have to be emailed. 
	    #
	    sleep(3);
	    my $foo = waitpid($childpid, &WNOHANG);
	    if ($foo) {
		$status = $? >> 8;
	    }
	    exit($status);
	}
    }
885 886
    my $coderef = sub {
	my ($sliver) = @_;
887
	my $urn = $sliver->aggregate_urn();
888
	my $errmsg;
889

890 891 892
	return 0
	    if ($sliver->status() eq "terminated");

893 894 895 896 897 898 899 900 901 902 903 904 905 906
	my $response = $sliver->Terminate();
	if (!defined($response)) {
	    $errmsg = "RPC Error calling Terminate";
	    goto bad;
	}

	# SEARCHFAILED is success.
	if ($response->code() != GENIRESPONSE_SUCCESS &&
	    $response->code() != GENIRESPONSE_SEARCHFAILED) {
	    if ($response->code() == GENIRESPONSE_BUSY) {
		$errmsg = "Slice was busy for too long; try again later?";
		goto bad;
	    }
	    $errmsg = "Could not delete slice: ". $response->output();
907 908
	    goto bad;
	}
909
	$instance->SetStatus("terminated");
910 911
	return 0;
      bad:
912
	print STDERR "$urn: $errmsg\n";
913 914
	return -1;
    };
915 916
    print STDERR Dumper($instance);
    
917
    my @return_codes = ();
918
    my @agglist =      $instance->AggregateList();
919 920 921 922 923 924 925 926
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@agglist)},
	    \@return_codes, $coderef, @agglist)) {
	#
	# The parent caught a signal. Leave things intact so that we can
	# kill things cleanly later.
	#
	$errmsg = "Internal error calling Terminate()";
927 928
	goto bad;
    }
929 930 931 932 933 934 935 936 937 938
    #
    # Check the exit codes. 
    #
    foreach my $code (@return_codes) {
	if ($code) {
	    $errmsg = "Some slivers would not terminate";
	    goto bad;
	}
    }
    
939
    $slice->Delete();
940
    $instance->RecordHistory($expired);
941 942 943 944 945
  killit:
    $instance->Delete();
    unlink($logfile)
	if (defined($logfile));
    exit(0);
946
    
947 948 949 950 951
  bad:
    print STDERR $errmsg . "\n";
    $instance->SetStatus($old_status);
    $slice->UnLock();
    if (defined($logfile)) {
952 953
	my $instance_name = $instance->name();
	my $slice_uuid    = $slice->uuid();
954 955
	SENDMAIL($TBOPS,
		 "Unable to terminate instance $uuid",
956 957
		 "Name: $instance_name\n".
		 "Slice: $slice_uuid\n\n".
958
		 "$errmsg\n",
959 960
		 $TBOPS, undef, $logfile)
	    if (!$silent);
961 962 963 964 965 966 967 968 969 970
	unlink($logfile);
    }
    exit(1);
}

#
# Extend.
#
sub DoExtend()
{
971
    my $force = 0;
972
    my $lockdown = 0;
973
    
974 975
    usage()
	if (!@ARGV);
976 977 978 979 980 981 982 983 984 985

    if (@ARGV == 2) {
	my $arg = shift(@ARGV);
	if ($arg eq "-f") {
	    $force = 1;
	}
	else {
	    usage();
	}
    }
986 987 988 989 990
    my $seconds = shift(@ARGV);
    if ($seconds !~ /^\d*$/) {
	usage();
    }

991
    if ($instance->status() eq "failed" && !$force) {
992 993 994 995 996 997
	fatal("Cannot extend failed instance!");
    }
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for instance!");
    }
998 999 1000 1001 1002 1003
    # The web interface (and in the future the xmlrpc interface) sets this.
    my $this_user = User->ImpliedUser();
    if (! defined($this_user)) {
	$this_user = User->ThisUser();
    }

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
    #
    # Lock the slice in case it is doing something else, like taking
    # a disk image.
    #
    if ($slice->Lock()) {
	fatal("Slice is busy, cannot lock it");
    }
    # Save in case of error.
    my $oldexpires = $slice->expires();

1014 1015 1016 1017 1018 1019
    # Lockdown on admin extensions longer then XX days. 
    if (defined($this_user) && $this_user->IsAdmin() &&
	($seconds / (24 * 60 * 60)) > 10) {
	$lockdown = 1
    }
    
1020 1021 1022
    # Need to update slice before creating new credential. 
    $slice->AddToExpiration($seconds);
    my $new_expires = $slice->ExpirationGMT();
1023 1024 1025 1026

    my $coderef = sub {
	my ($sliver) = @_;
	my $webtask  = $sliver->webtask();
Leigh Stoller's avatar
Leigh Stoller committed
1027
	my $domain   = $sliver->domain();
1028 1029 1030 1031
	my $errmsg;

	my $response = $sliver->Extend($new_expires);
	if (!defined($response)) {
Leigh Stoller's avatar
Leigh Stoller committed
1032
	    $errmsg = "Internal error calling Renew at $domain";
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
	    goto bad;
	}
	if ($response->code() != GENIRESPONSE_SUCCESS) {
	    # This is something the user should see.
	    if ($response->code() == GENIRESPONSE_REFUSED ||
		$response->code() == GENIRESPONSE_BUSY) {
		print STDERR $response->output() . "\n";
		# For web interface.
		$webtask->output($response->output());
		$webtask->Exited(1);
		return 1;
	    }
Leigh Stoller's avatar
Leigh Stoller committed
1045 1046
	    $errmsg = "Failed to extend slice at $domain: ".
		$response->output();
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
	    goto bad;
	}
	return 0;
      bad:
	print STDERR "$errmsg\n";
	$webtask->output($errmsg);
	$webtask->Exited(-1);
	return -1;
    };
    my @return_codes = ();
1057
    my @agglist =      $instance->AggregateList();
1058 1059 1060 1061 1062 1063 1064 1065 1066
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@agglist)},
	    \@return_codes, $coderef, @agglist)) {
	#
	# The parent caught a signal. Leave things intact so that we can
	# kill things cleanly later.
	#
	print STDERR "Internal error calling Extend\b";
	goto bad;
1067
    }
1068 1069 1070 1071 1072 1073 1074
    #
    # Check the exit codes. 
    #
    foreach my $code (@return_codes) {
	if ($code) {
	    print STDERR "Some slivers could not be extended\n";
	    goto bad;
1075 1076
	}
    }
1077
    # Lockdown.
1078
    if ($lockdown) {
1079 1080 1081 1082 1083 1084 1085 1086
	if (DoLockdownInternal("set", "admin")) {
	    SENDMAIL($TBOPS,
		     "Failed to lock down APT Instance",
		     "Failed to lock down $instance\n".
		     $instance->webURL() . "\n",
		     $TBOPS);
	}
    }
1087 1088
    $slice->UnLock();
    exit(0);
1089 1090 1091 1092 1093 1094

  bad:
    # Reset back to original expiration, sorry.
    $slice->SetExpiration($oldexpires);
    $slice->UnLock();
    exit(-1);
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
}

#
# Refresh; ask the aggregate for status and set the instance status
# accordingly.
#
sub DoRefresh()
{
    my $errmsg;
    
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	print STDERR "No slice for instance\n";
	goto killit;
    }
1110
    
1111 1112 1113 1114 1115
    #
    # Lock the slice in case it is doing something else, like taking
    # a disk image.
    #
    if ($slice->Lock()) {
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
	$errmsg = "Experiment is busy, cannot lock it. Please try again later";
	goto bad;
    }
    #
    # Create the webtask object, but AFTER locking the slice so we do
    # not destroy one in use.
    #
    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
	# Convenient.
	$webtask->AutoStore(1);
1127 1128
    }

1129 1130 1131 1132
    my $coderef = sub {
	my ($sliver) = @_;
	my $webtask  = $sliver->webtask();
	my $errmsg;
1133

1134 1135 1136
	my $response = $sliver->SliceStatus();
	if (!defined($response)) {
	    $errmsg = "RPC Error calling SliceStatus";
1137
	    goto bad;
1138
	}
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149

	if ($response->code() != GENIRESPONSE_SUCCESS) {
	    if ($response->code() == GENIRESPONSE_SEARCHFAILED) {
		$errmsg = "Slice is gone";
		goto bad;
	    }
	    if ($response->code() == GENIRESPONSE_BUSY) {
		$errmsg = "Slice is busy; try again later";
		goto bad;
	    }
	    $errmsg = "Could not get status: ". $response->output();
1150 1151
	    goto bad;
	}
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	my $blob = $response->value();
	if ($blob->{'status'} eq "ready") {
	    $sliver->SetStatus("ready");
	}
	elsif ($blob->{'status'} eq "failed") {
	    $sliver->SetStatus("failed");
	}
	#
	# Convert to something smaller, with info the web interface
	# cares about. 
	#
	my $statusblob = {};
	foreach my $urn (keys(%{$blob->{'details'}})) {
	    my $details = $blob->{'details'}->{$urn};
	    my $node_id = $details->{'client_id'};
	    $statusblob->{$node_id} = $details;
	}
	$webtask->sliverstatus($statusblob);
	if ($debug) {
	    print STDERR Dumper($statusblob);
	}
	return 0;
      bad:
	print STDERR "$errmsg\n";
	$webtask->output($errmsg);
	$webtask->Exited(1);
	return 1;
    };
    my @return_codes = ();
1181
    my @agglist =      $instance->AggregateList();
1182 1183 1184 1185 1186 1187 1188 1189
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@agglist)},
	    \@return_codes, $coderef, @agglist)) {
	#
	# The parent caught a signal. Leave things intact so that we can
	# kill things cleanly later.
	#
	$errmsg = "Internal error calling Refresh";
1190 1191
	goto bad;
    }
1192
    #
1193
    # Check the exit codes. 
1194
    #
1195 1196 1197
    foreach my $agg (@agglist) {
	my $code = shift(@return_codes);
	
1198 1199
	if ($code) {
	    $errmsg = "Some slivers could not be refreshed";
1200 1201 1202
	    if ($agg->output()) {
		$errmsg .= ": " . $agg->output();
	    }
1203 1204
	    goto bad;
	}
1205 1206 1207 1208
    }
    $slice->UnLock();
    exit(0);
  killit:
1209
    $instance->RecordHistory($RECORDHISTORY_TERMINATED);
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
    $instance->Delete();
    exit(0);
  bad:
    $slice->UnLock();
    print STDERR $errmsg . "\n";
    if (defined($webtask)) {
	$webtask->output($errmsg);
	$webtask->Exited(1);
    }
    exit(1);
}

#
1223
# Reboot or Reload nodes.
1224
#
1225
sub DoRebootOrReload($)
1226
{
1227
    my ($which) = @_;
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
    my $errmsg;
    
    usage()
	if (!@ARGV);

    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	print STDERR "No slice for instance\n";
	goto killit;
    }

1239
    my %sliver_urns = ();
1240
    my %node_ids    = ();
1241
    my @slivers     = ();
1242
    foreach my $obj ($instance->AggregateList()) {
1243 1244 1245 1246 1247 1248 1249 1250
	my $manifest = GeniXML::Parse($obj->manifest());
	if (! defined($manifest)) {
	    fatal("Could not parse manifest");
	}
	my @nodes = GeniXML::FindNodes("n:node", $manifest)->get_nodelist();
	foreach my $node (@nodes) {
	    my $client_id = GeniXML::GetVirtualId($node);
	    if (grep {$_ eq $client_id} @ARGV) {
1251 1252
		my $sliver_urn  = GeniXML::GetSliverId($node);
		my $manager_urn = GetManagerId($node);
1253

1254
		# No sliver urn or a different aggregate.
1255
		next
1256 1257 1258
		    if (! (defined($sliver_urn) &&
			   defined($manager_urn) &&
			   $manager_urn eq $obj->aggregate_urn()));
1259 1260 1261 1262 1263 1264

		if (!exists($sliver_urns{$obj->aggregate_urn()})) {
		    $sliver_urns{$obj->aggregate_urn()} = [];
		    push(@slivers, $obj);
		}
		push(@{ $sliver_urns{$obj->aggregate_urn()} }, $sliver_urn);
1265
		$node_ids{$sliver_urn} = $client_id;
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	    }
	}
    }
    
    #
    # Lock the slice in case it is doing something else, like taking
    # a disk image.
    #
    if ($slice->Lock()) {
	$errmsg = "Experiment is busy, cannot lock it. Please try again later";
	goto bad;
    }

    #
    # Create the webtask object, but AFTER locking the slice so we do
    # not destroy one in use.
    #
    if (defined($webtask_id)) {
	$webtask = WebTask->LookupOrCreate($instance->uuid(), $webtask_id);
	# Convenient.
	$webtask->AutoStore(1);
    }

1289 1290 1291 1292 1293
    my $coderef = sub {
	my ($sliver) = @_;
	my $webtask  = $sliver->webtask();
	my @urns     = @{ $sliver_urns{$sliver->aggregate_urn()} };
	my $errmsg;
1294

1295
	my $response = $sliver->SliverAction(\$errmsg, $which, @urns);
1296 1297
	if (!defined($response)) {
	    $errmsg = "RPC Error calling SliverAction";
1298 1299
	    goto bad;
	}
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
	if ($response->code() != GENIRESPONSE_SUCCESS) {
	    if ($response->code() == GENIRESPONSE_SEARCHFAILED) {
		$errmsg = "Slice is gone";
		goto bad;
	    }
	    if ($response->code() == GENIRESPONSE_BUSY) {
		$errmsg = "Experiment is busy; try again later";
		goto bad;
	    }
	    $errmsg = $response->output();
1310 1311
	    goto bad;
	}
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
	# Tell the web interface something is different. Real status will
	# come later when the monitor starts up. 
	if ($webtask->sliverstatus()) {
	    my $blob = $webtask->sliverstatus();
	    foreach my $urn (@urns) {
		my $node_id = $node_ids{$urn};
		$blob->{$node_id}->{'status'} = "changing";
	    }
	    $webtask->sliverstatus($blob);
	}
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
	return 0;
      bad:
	print STDERR "$errmsg\n";
	$webtask->output($errmsg);
	$webtask->Exited(1);
	return 1;
    };

    my @return_codes = ();
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@slivers)},
	    \@return_codes, $coderef, @slivers)) {
	#
	# The parent caught a signal. Leave things intact so that we can
	# kill things cleanly later.
	#
	$errmsg = "Internal error calling SliverAction";
1339 1340
	goto bad;
    }
1341 1342 1343 1344 1345 1346 1347 1348 1349
    #
    # Check the exit codes. 
    #
    foreach my $code (@return_codes) {
	if ($code) {
	    $errmsg = "Some slivers could not be ${which}'ed";
	    goto bad;
	}
    }
1350
    $slice->UnLock();
1351 1352 1353 1354 1355 1356 1357 1358 1359
    if (defined($webtask)) {
	$webtask->Exited(0);
    }
    #
    # Start the monitor so the web interface will see when the node
    # has actually come back up.
    #
    # XXX This will not return unless a monitor is already running.
    StartMonitor();
1360 1361
    exit(0);
  killit:
1362
    $instance->RecordHistory($RECORDHISTORY_TERMINATED);
1363 1364 1365 1366 1367
    $instance->Delete();
    exit(0);
  bad:
    $slice->UnLock();
    print STDERR $errmsg . "\n";
1368 1369 1370 1371
    if (defined($webtask)) {
	$webtask->output($errmsg);
	$webtask->Exited(1);
    }
1372 1373
    exit(1);
}
1374 1375
sub DoReboot() { return DoRebootOrReload("reboot"); }
sub DoReload() { return DoRebootOrReload("reload"); }
1376

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
#
# 
#
sub DoManifests()
{
    my $errmsg;
    
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	print STDERR "No slice for instance\n";
	goto killit;
    }

    my $coderef = sub {
	my ($sliver) = @_;
	my $webtask  = $sliver->webtask();
	my $errmsg;

	my $response = $sliver->GetManifest();
	if (!defined($response)) {
	    $errmsg = "RPC Error calling GetManifest";
	    goto bad;
	}
	return 0;
      bad:
	return 1;
    };

    my @return_codes = ();
    my @agglist      = $instance->AggregateList();
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@agglist)},
	    \@return_codes, $coderef, @agglist)) {
	#
	# The parent caught a signal. Leave things intact so that we can
	# kill things cleanly later.
	#
	$errmsg = "Internal error calling GetManifest";
	goto bad;
    }
    #
    # Check the exit codes. 
    #
    foreach my $code (@return_codes) {
	if ($code) {
	    $errmsg = "Could not get manifest for some slivers";
	    goto bad;
	}
    }
    exit(0);
  bad:
    print STDERR $errmsg . "\n";
    exit(1);
}

1432 1433 1434 1435 1436 1437
#
# Start up the monitor for an instance. Only one though.
#
sub StartMonitor()
{
    my $logfile;
1438
    my $signaled   = 0;
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
    
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for instance");
    }
    if ($instance->monitor_pid()) {
	my $pid = $instance->monitor_pid();
	if (kill(0, $pid)) {
	    print STDERR "Monitor already running ($pid). ".
		"Kill it before starting a new one.\n";
	    exit(0);
	}
	$instance->Update({"monitor_pid" => 0});
    }
    if (!$debug) {
	$logfile = TBMakeLogname("aptmonitor");
	if (TBBackGround($logfile)) {
Leigh Stoller's avatar
Leigh Stoller committed
1456
	    return 0;
1457 1458 1459
	}
    }
    $instance->Update({"monitor_pid" => '$PID'});
Leigh Stoller's avatar
Leigh Stoller committed
1460 1461 1462 1463 1464 1465 1466 1467

    #
    # We just did the operation, no need to ask so soon, and we
    # avoid locking the slice in case the user wants to reboot
    # another node right away. For reboot/reload, nothing interesting
    # is going to be reported for at least 30 seconds (XEN VM).
    #
    sleep(30);
1468 1469 1470 1471
    
    my $seconds  = 1500;
    my $interval = 15;
    # Shorten default timeout now.
1472
    Genixmlrpc->SetTimeout(30);
1473

1474 1475 1476 1477
    my $coderef = sub {
	my ($sliver) = @_;
	my $webtask  = $sliver->webtask();
	my $errmsg;
1478

1479
	my $response = $sliver->SliceStatus();
1480 1481
	if (!defined($response)) {
	    print STDERR "RPC Error calling SliceStatus\n";
1482
	    return GENIRESPONSE_RPCERROR;
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
	}
	if (($response->code() != GENIRESPONSE_SUCCESS &&
	     $response->code() != GENIRESPONSE_BUSY)) {
	    print STDERR "SliverStatus failed";
	    print STDERR ": " . $response->output() . "\n";
	    if (defined($webtask)) {
		if ($response->output() =~ /read timeout/) {
		    $webtask->output("Lost contact with the aggregate. " .
				     "Possibly a network failure, ".
				     "please try again later.");
		}
		else {
		    $webtask->output($response->output());
		}
	    }
1498 1499 1500 1501 1502
	    return -1;
	}
	if ($response->code() == GENIRESPONSE_BUSY) {
	    # Indicate not done.
	    return GENIRESPONSE_BUSY;
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
	}
    
	my $blob = $response->value();
	#
	# Convert to something smaller, with info the web interface
	# cares about. 
	#
	my $statusblob = {};
	foreach my $urn (keys(%{$blob->{'details'}})) {
	    my $details = $blob->{'details'}->{$urn};
	    my $node_id = $details->{'client_id'};
	    $statusblob->{$node_id} = $details;
	}
	if ($debug) {
	    print STDERR Dumper($statusblob);
	}
1519 1520
	$webtask->sliverstatus($statusblob);

1521
	#
1522
	# We poll until the status goes ready. Might not be a good idea.
1523 1524
	#
	if ($blob->{'status'} eq "ready") {
1525 1526 1527 1528 1529 1530 1531 1532
	    return 0;
	}
	# Not done yet. 
	return 1;
    };

    while ($seconds > 0) {
	$seconds -= $interval;
Leigh Stoller's avatar
Leigh Stoller committed
1533
	
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
	#
	# Lock the slice in case it is doing something else, like taking
	# a disk image. Just skip this turn.
	#
	next
	    if ($slice->Lock());

	my $handler = sub {
	    # This is so we can catch when Parrun gets signaled, but not
	    # exit till it exits.
	    $signaled = 1;
	};
	local $SIG{TERM} = $handler;
	if ($debug) {
	    local $SIG{INT} = $handler;
	}
	my @return_codes = ();
1551
	my @agglist =      $instance->AggregateList();
1552 1553 1554 1555
	if (ParRun({"maxwaittime" => 99999,
		    "maxchildren" => scalar(@agglist)},
		   \@return_codes, $coderef, @agglist)) {
	    print STDERR "Internal error calling Status()\n";
1556 1557 1558
	    $slice->UnLock();
	    last;
	}
1559 1560
	local $SIG{TERM} = 'DEFAULT';
	local $SIG{INT}  = 'DEFAULT';
1561
	$slice->UnLock();
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
	
	#
	# Check the exit codes. 
	#
	my $done = 1;
	foreach my $code (@return_codes) {
	    if ($code) {
		last
		    if ($code < 0);
		$done = 0;
	    }
	}
	last
	    if ($done);
	
	sleep($interval);
1578 1579 1580 1581 1582 1583
    }
    unlink($logfile)
	if (defined($logfile));
    exit(0);
}

1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
#
# Experiment lockdown.
#
sub DoLockdownInternal($$)
{
    my ($setclr,$which) = @_;
    
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for instance");
    }
    if ($which eq "all") {
	if ($instance->SetLockdown("user", ($setclr eq "clear" ? 1 : 0))) {
	    print STDERR "Could not update instance lockdown\n";
	    return -1
	}
	$which = "admin"
    }
    if ($instance->SetLockdown($which, ($setclr eq "clear" ? 1 : 0))) {
	print STDERR "Could not update instance lockdown\n";
	return -1
    }
    my $clear = ($instance->admin_lockdown() ||
		 $instance->user_lockdown() ? 0 : 1);
    
    #
    # Have to set/clear the lockdown on the local slice.
    #
    if ($slice->SetLockdown($clear)) {
	print STDERR "Could not update slice lockdown\n";
	return -1
    }
    #
1617
    # And tell the backend clusters to lockdown the slice.
1618
    #
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
    my $coderef = sub {
	my ($sliver) = @_;

	my $response = $sliver->Lockdown($clear);
	if (!defined($response)) {
	    print STDERR "RPC Error calling Lockdown\n";
	    return -1;
	}
	if ($response->code() != GENIRESPONSE_SUCCESS) {
	    print STDERR "Could not lockdown sliver: ".
		$response->output() . "\n";
	    return -1;
	}
	return 0;
    };
    my @return_codes = ();
1635
    my @agglist =      $instance->AggregateList();
1636 1637 1638 1639
    if (ParRun({"maxwaittime" => 99999,
		"maxchildren" => scalar(@agglist)},
	       \@return_codes, $coderef, @agglist)) {
	print STDERR "Internal error calling Lockdown()\n";
1640 1641
	return -1;
    }
1642 1643 1644 1645 1646 1647 1648 1649
    #
    # Check the exit codes. 
    #
    foreach my $code (@return_codes) {
	if ($code) {
	    print STDERR "Some slivers could not be locked down.\n";
	    return -1;
	}
1650
    }
1651
    return 0;
1652
}
1653

1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
sub DoLockdown()
{
    usage()
	if (@ARGV != 2);
    
    my $setclr = shift(@ARGV);
    my $which  = shift(@ARGV);

    fatal("Must specify either 'admin' or 'user'")
	if ($which !~ /^(admin|user|all)$/);
    fatal("Must specify either 'set' or 'clear'")
	if ($setclr !~ /^(set|clear)$/);

1667 1668 1669 1670 1671 1672 1673
    my $slice = $instance->GetGeniSlice();
    if (!defined($slice)) {
	fatal("No slice for instance");
    }
    if ($slice->Lock()) {
	fatal("Experiment is busy, cannot lock it. Please try again later");
    }
1674
    if (DoLockdownInternal($setclr, $which)) {
1675
	$slice->UnLock();
1676 1677
	fatal("Could not lockdown instance!");
    }
1678
    $slice->UnLock();
1679 1680 1681
    exit(0);
}

1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697
#
# Write instance credentials to files.
#
sub WriteCredentials()
{
    usage()
	if (!@ARGV);
    my $directory = shift(@ARGV);
    fatal("$directory does not exist")
	if (! -e $directory);
    fatal("$directory is not a directory")
	if (! -d $directory);

    return $instance->WriteCredentials($directory);
}

1698 1699 1700 1701
sub fatal($)
{
    my ($mesg) = @_;