Commit 4364805f authored by Leigh Stoller's avatar Leigh Stoller

Checkpoint emergency shutdown; not done.

parent b2a14a73
......@@ -103,6 +103,7 @@ CREATE TABLE `geni_slices` (
`exptidx` int(11) default NULL,
`created` datetime default NULL,
`expires` datetime default NULL,
`shutdown` datetime default NULL,
`locked` datetime default NULL,
`creator_uuid` varchar(40) NOT NULL default '',
`name` tinytext,
......
......@@ -38,6 +38,7 @@ my $TBAPPROVAL = "@TBAPPROVALEMAIL@";
my $TBAUDIT = "@TBAUDITEMAIL@";
my $BOSSNODE = "@BOSSNODE@";
my $OURDOMAIN = "@OURDOMAIN@";
my $SLICESHUTDOWN = "$TB/sbin/protogeni/shutdownslice";
#
# Get a credential to use the ClearingHouse. For the moment, the initial
......@@ -663,6 +664,72 @@ sub Remove($)
return GeniResponse->Create(GENIRESPONSE_UNSUPPORTED);
}
#
# Emergency Shutdown of a slice.
#
sub Shutdown($)
{
my ($argref) = @_;
my $cred = $argref->{'credential'};
my $uuid = $argref->{'uuid'};
my $clear = $argref->{'clear'};
if (! (defined($cred) && defined($uuid))) {
return GeniResponse->MalformedArgsResponse();
}
$clear = (defined($clear) ? $clear : 0);
if (! ($uuid =~ /^[-\w]*$/)) {
return GeniResponse->MalformedArgsResponse();
}
my $credential = GeniCredential->CreateFromSigned($cred);
if (!defined($credential)) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create GeniCredential object");
}
#
# Make sure the credential was issued to the caller.
#
if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"This is not your credential!");
}
#
# And that the target of the credential is this registry.
#
my $authority = GeniAuthority->Lookup($ENV{'MYUUID'});
if (!defined($authority)) {
print STDERR "Could not find local authority object\n";
return GeniResponse->Create(GENIRESPONSE_ERROR);
}
if ($credential->target_uuid() ne $authority->uuid()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN,
undef, "This is not your registry!");
}
my $slice = GeniSlice->Lookup($uuid);
if (!defined($slice)) {
print STDERR "No slice record $uuid for shutdown!\n";
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"No such slice registered here");
}
#
# Pass the whole thing off to a script that will contact the
# CMs.
#
my $arg = ($clear ? "resume": "");
# -c option indicates acting as CH.
system("$SLICESHUTDOWN -c $uuid $arg");
if ($?) {
print STDERR "Could not shutdown $uuid!\n";
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Error shutting down slice");
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
#
# This is just a placeholder; return a list of all components. Eventually
# takes an rspec and we do a resource mapping.
......
......@@ -307,7 +307,7 @@ sub GetTicket($)
}
#
# For now all tickets expire ery quickly (minutes), but once the
# For now all tickets expire very quickly (minutes), but once the
# ticket is redeemed, it will expire according to the rspec request.
#
if (exists($rspec->{'valid_until'})) {
......@@ -346,6 +346,12 @@ sub GetTicket($)
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
# Shutdown slices get nothing.
if ($slice->shutdown()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice has been shutdown");
}
# For now, there can be only a single toplevel aggregate per slice.
#
......@@ -508,11 +514,16 @@ sub GetTicket($)
goto bad;
}
# XXX For tunnels that refer to a node on another site.
next
if (exists($linkref->{'link_type'}) &&
$linkref->{'link_type'} eq "tunnel" &&
!exists($namemap{$node_nickname}));
# XXX No wildcarding for tunnels.
if (exists($linkref->{'link_type'}) &&
$linkref->{'link_type'} eq "tunnel") {
if ($node_uuid eq "*") {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Tunnels cannot be wildcarded");
goto bad;
}
next;
}
#
# First map the node if its wildcarded.
......@@ -630,7 +641,6 @@ sub RedeemTicket($)
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
#
# Do not redeem an expired ticket, kill it now.
#
......@@ -640,6 +650,12 @@ sub RedeemTicket($)
return GeniResponse->Create(GENIRESPONSE_EXPIRED, undef,
"Ticket has expired");
}
# Shutdown slices get nothing.
if ($slice->shutdown()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice has been shutdown");
}
#
# For now, ther can be only a single toplevel aggregate per slice.
......@@ -713,6 +729,12 @@ sub UpdateSliver($)
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
# Shutdown slices get nothing.
if ($slice->shutdown()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice has been shutdown");
}
my $response = ModifySliver($aggregate, $slice,
$credential, $rspec, $impotent, $keys);
......@@ -1040,6 +1062,9 @@ sub ModifySliver($$$$$$)
# interfaces, and then combine those two interfaces into an aggregate,
# and then that aggregate goes into the aggregate for toplevel sliver.
#
goto skiplinks
if (!exists($rspec->{'link'}));
foreach my $linkname (keys(%{$rspec->{'link'}})) {
my @linkslivers = ();
if (! ($linkname =~ /^[-\w]*$/)) {
......@@ -1118,7 +1143,7 @@ sub ModifySliver($$$$$$)
}
}
}
skiplinks:
#
# Create a planetlab slice before provisioning (which creates nodes).
#
......@@ -1315,6 +1340,12 @@ sub StartSliver($)
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
# Shutdown slices get nothing.
if ($slice->shutdown()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice has been shutdown");
}
if (!$impotent && $sliver->Start() != 0) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
......@@ -1680,6 +1711,83 @@ sub BindToSlice($)
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
#
# Shutdown a slice. This is brutal at present; kill it completely.
#
sub Shutdown($)
{
my ($argref) = @_;
my $cred = $argref->{'credential'};
my $clear = $argref->{'clear'};
if (! (defined($cred))) {
return GeniResponse->MalformedArgsResponse();
}
$clear = (defined($clear) ? $clear : 0);
my $credential = GeniCredential->CreateFromSigned($cred);
if (!defined($credential)) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create GeniCredential object");
}
my $slice_uuid = $credential->target_uuid();
my $user_uuid = $credential->owner_uuid();
#
# Make sure the credential was issued to the caller.
#
if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"This is not your credential!");
}
#
# Create the slice record, since we do not want a request to come
# in later.
#
my $slice = GeniSlice->Lookup($slice_uuid);
if (!defined($slice)) {
$slice = CreateSliceFromCertificate($credential);
if (!defined($slice)) {
print STDERR "Could not create $slice_uuid\n";
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create slice");
}
# No point in doing anything else ...
$slice->SetShutdown(1);
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
#
# Do not worry about locking when setting the shutdown time.
# This can lead to race though, if a clear shutdown comes in first.
# Seems unlikely though.
#
if (!$clear) {
# Do not overwrite original shutdown time
$slice->SetShutdown(1)
if (!defined($slice->shutdown()) || $slice->shutdown() eq "");
}
else {
$slice->SetShutdown(0);
}
# Always make sure the slice is shutdown.
if ($slice->shutdown()) {
# The expire daemon is going to look for this, so it will get
# taken care of shortly.
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
if (CleanupDeadSlice($slice, 0) != 0) {
SENDMAIL($TBOPS, "Failed to shutdown slice",
"Failed to shutdown slice $slice\n");
print STDERR "Could not shutdown $slice!\n";
# Lets call this a non-error since the local admin person
# is going to have to deal with it anyway.
}
$slice->UnLock();
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
#
# Utility Routines.
#
......@@ -1794,9 +1902,13 @@ sub CreateAuthorityFromRegistry($)
#
# Cleanup a dead slice, releasing all the stuff associated with it.
#
sub CleanupDeadSlice($)
sub CleanupDeadSlice($;$)
{
my ($slice) = @_;
my ($slice, $purge) = @_;
# Default to full purge.
$purge = 1
if (!defined($purge));
# print STDERR "Cleaning up dead slice $slice\n";
......@@ -1875,6 +1987,9 @@ sub CleanupDeadSlice($)
}
}
return 0
if (!$purge);
my $experiment = $slice->GetExperiment();
if (defined($experiment)) {
$experiment->LockDown(0);
......@@ -1922,6 +2037,7 @@ sub GeniExperiment($)
return undef;
}
$experiment = Experiment->Lookup($uuid);
$experiment->Update({"geniflags" => 1});
}
return $experiment;
}
......
......@@ -50,7 +50,7 @@ my $OPENSSL = "/usr/bin/openssl";
$LOCALSA_FLAG = 1;
$LOCALCM_FLAG = 2;
$LOCALMA_FLAG = 3;
@EXPORT_OK = qw(LOCALSA_FLAG LOCALCM_FLAG LOCALMA_FLAG);
@EXPORT_OK = qw($LOCALSA_FLAG $LOCALCM_FLAG $LOCALMA_FLAG);
# Capability Flags.
......
......@@ -220,6 +220,26 @@ sub RemoveSlice($$)
return $self->Remove("Slice", $uuid);
}
#
# Emergency shutdown
#
sub Shutdown($$$)
{
my ($self, $uuid, $onoff) = @_;
my $response =
Genixmlrpc::CallMethod($self->authority(),
$self->context(),
"Shutdown",
{ "credential" => $self->credential(),
"onoff" => $onoff,
"uuid" => $uuid });
return -1
if (!defined($response) || $response->code() != GENIRESPONSE_SUCCESS);
return 0;
}
#
# List components.
#
......
......@@ -36,6 +36,7 @@ use Data::Dumper;
my $TB = "@prefix@";
my $OURDOMAIN = "@OURDOMAIN@";
my $PGENIDOMAIN = "@PROTOGENI_DOMAIN@";
my $SLICESHUTDOWN = "$TB/sbin/protogeni/shutdownslice";
#
# Get a credential for an object. Ignoring the type for now. If no credential
......@@ -727,5 +728,45 @@ sub BindToSlice($)
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
#
# Emergency shutdown a slice. This cannot be undone via this interface.
# An Emulab admin will have to do that.
#
sub Shutdown($)
{
my ($argref) = @_;
my $cred = $argref->{'credential'};
if (!defined($cred)) {
return GeniResponse->MalformedArgsResponse();
}
my $credential = GeniCredential->CreateFromSigned($cred);
if (!defined($credential)) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create GeniCredential object");
}
#
# Make sure the credential was issued to the caller.
#
if ($credential->owner_uuid() ne $ENV{'GENIUUID'}) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"This is not your credential!");
}
my $slice_uuid = $credential->target_uuid();
my $slice = GeniSlice->Lookup($slice_uuid);
if (!defined($slice)) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Unknown slice for this credential");
}
system("$SLICESHUTDOWN $slice_uuid");
if ($?) {
print STDERR "Could not shutdown $slice!\n";
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Error shutting down slice");
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -125,6 +125,7 @@ sub hrn($) { return field($_[0], "hrn"); }
sub uuid($) { return field($_[0], "uuid"); }
sub creator_uuid($) { return field($_[0], "creator_uuid"); }
sub created($) { return field($_[0], "created"); }
sub shutdown($) { return field($_[0], "shutdown"); }
sub expires($) { return field($_[0], "expires"); }
sub sa_uuid($) { return field($_[0], "sa_uuid"); }
sub exptidx($) { return field($_[0], "exptidx"); }
......@@ -565,8 +566,35 @@ sub SetExpiration($$)
return -1
if (!$query_result);
$self->{'SLICE'}->{'expires'} = $expires;
# XXX Wrong format, but harmless.
$self->{'SLICE'}->{'expires'} = time();
return 0;
}
#
# Set the shutdown field.
#
sub SetShutdown($$)
{
my ($self, $shutdown) = @_;
my $uuid = $self->uuid();
my $when;
if ($shutdown) {
$when = "now()";
}
else {
$when = "NULL";
}
my $query_result =
DBQueryWarn("update geni_slices set shutdown=$when " .
"where uuid='$uuid'");
return -1
if (!$query_result);
# XXX Wrong format, but harmless.
$self->{'SLICE'}->{'shutdown'} = ($shutdown ? time() : undef);
return 0;
}
......
......@@ -14,7 +14,7 @@ include $(OBJDIR)/Makeconf
SBIN_STUFF = cleanupslice
PSBIN_STUFF = register_resources expire_daemon gencrl postcrl \
createcerts initsite addauthority getcacerts \
gencrlbundle
gencrlbundle shutdownslice
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS =
......
......@@ -52,10 +52,6 @@ my $this_user = User->ThisUser();
if (! defined($this_user)) {
fatal("You ($UID) do not exist!");
}
if (!$this_user->IsAdmin()) {
die("*** $0:\n".
" You do not have permission to run this script!\n");
}
#
# Parse command arguments. Once we return from getopts, all that should be
......@@ -82,6 +78,13 @@ my $slice = GeniSlice->Lookup($uuid);
if (!defined($slice)) {
fatal("No such slice for $uuid");
}
my $experiment = $slice->GetExperiment();
if (!$this_user->IsAdmin() &&
!$experiment->AccessCheck($this_user, TB_EXPT_DESTROY)) {
die("*** $0:\n".
" You do not have permission to run this script!\n");
}
GeniUtil::FlipToGeniUser();
if (GeniCM::CleanupDeadSlice($slice) != 0) {
......
......@@ -74,14 +74,13 @@ require GeniCM;
use libtestbed;
use POSIX qw(strftime);
GeniUtil::FlipToGeniUser();
# Go to ground.
if (! $debug) {
if (TBBackGround($LOGFILE)) {
exit(0);
}
}
GeniUtil::FlipToGeniUser();
#
# Look for tickets.
......@@ -128,14 +127,16 @@ sub ExpireTickets()
}
#
# Look for slices that need to be expired.
# Look for slices that need to be expired. Do not expire shutdown slices;
# Handled below, and we want to keep the record around.
#
sub ExpireSlices()
{
my $query_result =
GeniDB::DBQueryWarn("select idx from geni_slices ".
"where UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(expires)");
" UNIX_TIMESTAMP(expires) and ".
" shutdown is null");
while (my ($idx) = $query_result->fetchrow_array()) {
my $slice = GeniSlice->Lookup($idx);
......@@ -196,6 +197,43 @@ sub ExpireSlices()
}
}
#
# Look for slices that need to be shutdown
#
sub ShutdownSlices()
{
my $query_result =
GeniDB::DBQueryWarn("select idx from geni_slices ".
"where shutdown is not null");
while (my ($idx) = $query_result->fetchrow_array()) {
my $slice = GeniSlice->Lookup($idx);
if (!defined($slice)) {
# Slice is gone, lets not worry.
next;
}
if ($slice->Lock() != 0) {
print STDERR "Could not lock slice $slice.\n";
goto skip;
}
#
# This will get called everytime, but CleanupDeadSlice does
# not actually do anything if there are no slivers.
#
if (GeniCM::CleanupDeadSlice($slice, 0) != 0) {
print STDERR "Could not shutdown $slice\n";
SENDMAIL($TBOPS, "Could not shutdown slice",
"Could not shutdown slice $slice");
}
$slice->UnLock();
skip:
$slice->Flush()
if (defined($slice));
}
}
#
# Setup a signal handler for newsyslog.
#
......@@ -212,8 +250,9 @@ while (1) {
ExpireTickets();
ExpireSlices();
ShutdownSlices();
sleep(30);
sleep(60);
}
exit(0);
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2008 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
use English;
use Getopt::Std;
#
# Shutdown a slice. This has two modes of operation; 1) on the SA (Emulab)
# it will mark the slice as shutdown and make sure the Clearinghouse is
# notified. 2) On the clearinghouse, it will mark the slice as shutdown,
# and make sure that every CM is notified. This is somewhat crude.
#
sub usage()
{
print "Usage: shutdownslice [-c] <uuid>\n";
exit(1);
}
my $optlist = "c";
my $asch = 0;
sub fatal($);
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# Check args.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"c"})) {
$asch = 1;
}
usage()
if (@ARGV != 1);
use vars qw($GENI_DBNAME);
if ($asch) {
$GENI_DBNAME = "geni-ch";
}
# Now we can load the libraries after setting the proper DB.
use lib '@prefix@/lib';
require GeniDB;
require Genixmlrpc;
use GeniResponse;
require GeniCertificate;
require GeniCredential;
require GeniAuthority;
require GeniSlice;
my $uuid = $ARGV[0];
#
# Taint check.
#
if ($uuid =~ /^([\w\-]+)$/) {
$uuid = $1;
}
else {
die("*** Bad data in $uuid\n");
}
my $slice = GeniSlice->Lookup($uuid);
if (!defined($slice)) {
fatal("No such slice for $uuid");
}
if ($asch) {
#
# The RPC context for this script is as the CH
#
my $EMULAB_PEMFILE = "@prefix@/etc/genich.pem";
my $certificate = GeniCertificate->LoadFromFile($EMULAB_PEMFILE);
if (!defined($certificate)) {
fatal("Could not load certificate from $EMULAB_PEMFILE");
}
Genixmlrpc->SetContext(Genixmlrpc->Context($certificate));
my $authority = GeniAuthority->Lookup($certificate->uuid());
if (!defined($authority)) {
fatal("Could not find local authority object");
}
#
# The credential we use needs to be a slice credential, since that
# makes it easier for anyone to shutdown a slice at a CM.
#
my $slice_credential = GeniCredential->Create($slice, $authority);
if (!defined($slice_credential)) {
fatal("Could not create new credential for $slice");
}
if ($slice_credential->Sign($certificate) != 0) {
fatal("Could not sign credential for $slice");
}
my $query_result =
GeniDB::DBQueryWarn("select uuid from geni_authorities ".
"where type='cm'");
fatal("Could not lookup CM list")
if (!defined($query_result));
while (my ($manager_uuid) = $query_result->fetchrow_array()) {
my $manager = GeniAuthority->Lookup($manager_uuid);
fatal("Could not lookup up CM $manager_uuid")
if (!defined($manager));
print STDERR "Calling out to $manager to do shutdown\n";
my $response =
Genixmlrpc::CallMethod($manager->url(), undef,
"Shutdown",
{ "clear" => 0,
"credential" =>
$slice_credential->asString() });
if (!defined($response) || $response->code() != GENIRESPONSE_SUCCESS) {
fatal("Could not shutdown $slice on $manager: " .
$response->output());
}
}
}
else {
#
# Mark the slice as shutdown but do not overwrite original time.
#
if (!defined($slice->shutdown()) || $slice->shutdown() eq "") {
$slice->SetShutdown() == 0
or fatal("Could not set the shutdown time for $slice");
}
#
# The RPC context for this script is as the SA
#
my $EMULAB_PEMFILE = "@prefix@/etc/genisa.pem";
my $certificate = GeniCertificate->LoadFromFile($EMULAB_PEMFILE);
if (!defined($certificate)) {
fatal("Could not load certificate from $EMULAB_PEMFILE");
}
Genixmlrpc->SetContext(Genixmlrpc->Context($certificate));
my $clearinghouse = GeniRegistry::ClearingHouse