Commit 00563293 authored by Leigh B Stoller's avatar Leigh B Stoller

Add lockdown support, to prevent accidental termination of important

experiments. Lockdown is implemented at the portal and at the backend
cluster (ignore by cm and sa daemons). At present, the only policy is
that admin extensions longer then 14 days, set the lockdown.
parent 7b6f6150
...@@ -345,7 +345,47 @@ sub SetManifest($$) ...@@ -345,7 +345,47 @@ sub SetManifest($$)
$self->{'INSTANCE'}->{'manifest'} = $manifest; $self->{'INSTANCE'}->{'manifest'} = $manifest;
return 0; return 0;
}
#
# Set/Clear the lockdown bits.
#
sub SetLockdown($$$)
{
my ($self,$which,$clear) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $uuid = $self->uuid();
if ($which eq "admin") {
$which = "admin_lockdown";
}
elsif ($which eq "user") {
$which = "user_lockdown";
}
else {
return -1;
}
my $value = ($clear ? 0 : 1);
DBQueryWarn("update apt_instances set ${which}=${value} ".
"where uuid='$uuid'") or
return -1;
$self->{'INSTANCE'}->{$which} = $value;
return 0;
}
# Return a URL for the status page.
sub webURL($)
{
my ($self) = @_;
my $link = $self->Brand()->wwwBase();
$link = $link . "/status.php?uuid=" . $self->uuid();
return $link;
} }
# #
...@@ -432,8 +472,7 @@ sub WarnExpiring($$) ...@@ -432,8 +472,7 @@ sub WarnExpiring($$)
print STDERR "WarnExpiring: no email address for $geniuser\n"; print STDERR "WarnExpiring: no email address for $geniuser\n";
return 0; return 0;
} }
my $link = $self->Brand()->wwwBase(); my $link = $self->webURL();
$link = $link . "/status.php?uuid=" . $self->uuid();
$self->Brand()->SendEmail($geniuser->email(), $self->Brand()->SendEmail($geniuser->email(),
"Your experiment is expiring soon!", "Your experiment is expiring soon!",
...@@ -776,5 +815,38 @@ sub SliverAction($$@) ...@@ -776,5 +815,38 @@ sub SliverAction($$@)
return Genixmlrpc::CallMethod($cmurl, $context, $method, $args); return Genixmlrpc::CallMethod($cmurl, $context, $method, $args);
} }
#
# Lockdown
#
sub Lockdown($$)
{
my ($self, $clear) = @_;
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $slice = $self->GetGeniSlice();
my $context = APT_Geni::GeniContext();
return undef
if (! (defined($geniuser) && defined($authority) &&
defined($slice) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($slice, $geniuser);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
my $args = {
"slice_urn" => $slice->urn(),
"credentials" => [$slice_credential->asString(),
$speaksfor_credential->asString()],
};
$args->{"clear"} = 1
if ($clear);
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
return Genixmlrpc::CallMethod($cmurl, $context, "Lockdown", $args);
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
...@@ -221,6 +221,7 @@ sub ExpireInstances() ...@@ -221,6 +221,7 @@ sub ExpireInstances()
DBQueryWarn("select a.uuid,s.expires from apt_instances as a ". DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
"left join geni.geni_slices as s on s.uuid=a.slice_uuid ". "left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
"where (a.status='ready') and ". "where (a.status='ready') and ".
" a.admin_lockdown=0 and s.lockdown=0 and ".
" (UNIX_TIMESTAMP(now()) > ". " (UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(s.expires))"); " UNIX_TIMESTAMP(s.expires))");
return return
...@@ -259,11 +260,63 @@ sub ExpireInstances() ...@@ -259,11 +260,63 @@ sub ExpireInstances()
} }
} }
#
# Warn about locked down instances that have expired.
#
sub ReportLockdownExpired()
{
my @instances;
my $query_result =
DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
"left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
"where (a.admin_lockdown=1 or s.lockdown=1) and ".
" (UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(s.expires))");
return
if (!$query_result);
while (my ($uuid,$expires) = $query_result->fetchrow_array()) {
my $instance = APT_Instance->Lookup($uuid);
if (!defined($instance)) {
print STDERR "No such instance $uuid\n";
next;
}
print STDERR "$uuid is locked down but expired at $expires\n";
push(@instances, $instance);
}
return
if (!@instances);
my $text = "";
foreach my $instance (@instances) {
my $profile = $instance->Profile();
my $creator = $instance->creator();
my $wwwlink = $instance->webURL();
$text .= $profile->name() . ",$creator: $wwwlink\n";
}
SENDMAIL($TBOPS,
"APT daemon: Locked down instances",
"The following instances are locked down but expired:\n\n".
$text . "\n",
$TBOPS);
}
my $reportcounter = 0;
# Do this once at startup
if (! NoLogins()) {
ReportLockdownExpired();
}
while (1) { while (1) {
if (NoLogins()) { if (NoLogins()) {
sleep(5); sleep(5);
next; next;
} }
$reportcounter += $SLEEP_INTERVAL;
print "Running at ". print "Running at ".
POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n"; POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n";
...@@ -271,6 +324,12 @@ while (1) { ...@@ -271,6 +324,12 @@ while (1) {
KillFailedInstances(); KillFailedInstances();
ExpireInstances(); ExpireInstances();
# Do this once every 24 hours.
if ($reportcounter >= (24 * 60 * 60)) {
ReportLockdownExpired();
$reportcounter = 0;
}
exit(0) exit(0)
if ($oneshot); if ($oneshot);
......
...@@ -43,6 +43,7 @@ sub usage() ...@@ -43,6 +43,7 @@ sub usage()
print("Usage: manage_instance reboot instance node_id [node_id ...]\n"); print("Usage: manage_instance reboot instance node_id [node_id ...]\n");
print("Usage: manage_instance reload instance node_id [node_id ...]\n"); print("Usage: manage_instance reload instance node_id [node_id ...]\n");
print("Usage: manage_instance monitor instance\n"); print("Usage: manage_instance monitor instance\n");
print("Usage: manage_instance lockdown instance set|clear user|admin\n");
exit(-1); exit(-1);
} }
my $optlist = "dt:"; my $optlist = "dt:";
...@@ -101,6 +102,7 @@ sub DoExtend(); ...@@ -101,6 +102,7 @@ sub DoExtend();
sub DoRefresh(); sub DoRefresh();
sub DoReboot(); sub DoReboot();
sub DoReload(); sub DoReload();
sub DoLockdown();
sub StartMonitor(); sub StartMonitor();
# #
...@@ -154,6 +156,9 @@ elsif ($action eq "reload") { ...@@ -154,6 +156,9 @@ elsif ($action eq "reload") {
elsif ($action eq "monitor") { elsif ($action eq "monitor") {
StartMonitor() StartMonitor()
} }
elsif ($action eq "lockdown") {
DoLockdown()
}
else { else {
usage(); usage();
} }
...@@ -583,7 +588,8 @@ sub DoConsole() ...@@ -583,7 +588,8 @@ sub DoConsole()
if ($response->value()) { if ($response->value()) {
fatal($response->output()); fatal($response->output());
} }
fatal("Server returned error: " . $response->code); fatal("Server returned error: " .
GENIRESPONSE_STRING($response->code));
} }
} }
my $url; my $url;
...@@ -735,11 +741,16 @@ sub DoExtend() ...@@ -735,11 +741,16 @@ sub DoExtend()
if ($instance->status() eq "failed") { if ($instance->status() eq "failed") {
fatal("Cannot extend failed instance!"); fatal("Cannot extend failed instance!");
} }
my $slice = $instance->GetGeniSlice(); my $slice = $instance->GetGeniSlice();
if (!defined($slice)) { if (!defined($slice)) {
fatal("No slice for instance!"); fatal("No slice for instance!");
} }
# The web interface (and in the future the xmlrpc interface) sets this.
my $this_user = User->ImpliedUser();
if (! defined($this_user)) {
$this_user = User->ThisUser();
}
# #
# Lock the slice in case it is doing something else, like taking # Lock the slice in case it is doing something else, like taking
# a disk image. # a disk image.
...@@ -764,13 +775,24 @@ sub DoExtend() ...@@ -764,13 +775,24 @@ sub DoExtend()
$slice->SetExpiration($oldexpires); $slice->SetExpiration($oldexpires);
$slice->UnLock(); $slice->UnLock();
# This is something the user should see. # This is something the user should see.
if ($response->code() == GENIRESPONSE_REFUSED) { if ($response->code() == GENIRESPONSE_REFUSED ||
$response->code() == GENIRESPONSE_BUSY) {
print STDERR $response->output() . "\n"; print STDERR $response->output() . "\n";
# For web interface. # For web interface.
exit(1); exit(1);
} }
fatal("Failed to extend slice: ". $response->output()) fatal("Failed to extend slice: ". $response->output())
} }
if (defined($this_user) && $this_user->IsAdmin() &&
($seconds / (24 * 60 * 60)) > 10) {
if (DoLockdownInternal("set", "admin")) {
SENDMAIL($TBOPS,
"Failed to lock down APT Instance",
"Failed to lock down $instance\n".
$instance->webURL() . "\n",
$TBOPS);
}
}
$slice->UnLock(); $slice->UnLock();
exit(0); exit(0);
} }
...@@ -1087,6 +1109,75 @@ sub StartMonitor() ...@@ -1087,6 +1109,75 @@ sub StartMonitor()
exit(0); exit(0);
} }
#
# Experiment lockdown.
#
sub DoLockdownInternal($$)
{
my ($setclr,$which) = @_;
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance");
}
if ($which eq "all") {
if ($instance->SetLockdown("user", ($setclr eq "clear" ? 1 : 0))) {
print STDERR "Could not update instance lockdown\n";
return -1
}
$which = "admin"
}
if ($instance->SetLockdown($which, ($setclr eq "clear" ? 1 : 0))) {
print STDERR "Could not update instance lockdown\n";
return -1
}
my $clear = ($instance->admin_lockdown() ||
$instance->user_lockdown() ? 0 : 1);
#
# Have to set/clear the lockdown on the local slice.
#
if ($slice->SetLockdown($clear)) {
print STDERR "Could not update slice lockdown\n";
return -1
}
#
# And tell the backend cluster to lockdown the slice.
#
my $response = $instance->Lockdown($clear);
if (!defined($response)) {
print STDERR "RPC Error calling Lockdown\n";
# Clear this so we do not think it is locked down for real.
$slice->SetLockdown(0);
return -1;
}
if ($response->code() != GENIRESPONSE_SUCCESS) {
print STDERR "Could not lockdown sliver: ". $response->output() . "\n";
# Clear this so we do not think it is locked down for real.
$slice->SetLockdown(0);
return -1;
}
return 0;
}
sub DoLockdown()
{
usage()
if (@ARGV != 2);
my $setclr = shift(@ARGV);
my $which = shift(@ARGV);
fatal("Must specify either 'admin' or 'user'")
if ($which !~ /^(admin|user|all)$/);
fatal("Must specify either 'set' or 'clear'")
if ($setclr !~ /^(set|clear)$/);
if (DoLockdownInternal($setclr, $which)) {
fatal("Could not lockdown instance!");
}
exit(0);
}
sub fatal($) sub fatal($)
{ {
my ($mesg) = @_; my ($mesg) = @_;
......
...@@ -58,6 +58,7 @@ use GeniStitch; ...@@ -58,6 +58,7 @@ use GeniStitch;
use GeniStd; use GeniStd;
use emutil; use emutil;
use English; use English;
use libtestbed;
use Data::Dumper; use Data::Dumper;
use XML::Simple; use XML::Simple;
use Date::Parse; use Date::Parse;
...@@ -724,6 +725,10 @@ sub DeleteSlice($) ...@@ -724,6 +725,10 @@ sub DeleteSlice($)
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef, return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN"); "Credential does not match the URN");
} }
if ($slice->lockdown()) {
return GeniResponse->Create(GENIRESPONSE_REFUSED(), undef,
"Slice is locked down");
}
if ($slice->Lock() != 0) { if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse(); return GeniResponse->BusyResponse();
} }
...@@ -3785,8 +3790,6 @@ sub DescribeDataset($) ...@@ -3785,8 +3790,6 @@ sub DescribeDataset($)
!defined($image->creator_urn()) || !defined($image->creator_urn()) ||
$image->creator_urn() ne $user->urn()); $image->creator_urn() ne $user->urn());
print STDERR Dumper($image);
$blob->{'state'} = "valid"; $blob->{'state'} = "valid";
$blob->{'type'} = "imdataset"; $blob->{'type'} = "imdataset";
$blob->{"busy"} = 0; $blob->{"busy"} = 0;
...@@ -3836,5 +3839,57 @@ sub DescribeDataset($) ...@@ -3836,5 +3839,57 @@ sub DescribeDataset($)
return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob); return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
} }
#
# Set/Clear the lockdown.
#
sub Lockdown($)
{
my ($argref) = @_;
my $slice_urn = $argref->{'slice_urn'};
my $clear = $argref->{'clear'};
my $credentials = $argref->{'credentials'};
if (! (defined($credentials) && defined($slice_urn))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
}
if (! GeniHRN::IsValid($slice_urn)) {
return GeniResponse->MalformedArgsResponse("Invalid URN");
}
my ($credential,$speaksfor) = GeniStd::CheckCredentials($credentials);
return $credential
if (GeniResponse::IsResponse($credential));
my $user = GeniCM::CreateUserFromCertificate($credential);
return $user
if (GeniResponse::IsResponse($user));
my $authority = GeniCM::CreateAuthorityFromCertificate($credential);
return $authority
if (GeniResponse::IsResponse($authority));
my ($slice, $aggregate) = Credential2SliceAggregate($credential);
return $slice
if (defined($slice) && GeniResponse::IsResponse($slice));
if (! (defined($slice) && defined($aggregate))) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"Slice does not exist");
}
if ($slice_urn ne $slice->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
if ($slice->creator_urn() ne $user->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Not enough permission to access image");
}
main::AddLogfileMetaDataFromSlice($slice);
if ($slice->SetLockdown(defined($clear) && $clear ? 1 : 0)) {
return GeniResponse->Create(GENIRESPONSE_ERROR);
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2008-2014 University of Utah and the Flux Group. # Copyright (c) 2008-2015 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -308,6 +308,7 @@ sub exptidx($) { return field($_[0], "exptidx"); } ...@@ -308,6 +308,7 @@ sub exptidx($) { return field($_[0], "exptidx"); }
sub needsfirewall($) { return field($_[0], "needsfirewall"); } sub needsfirewall($) { return field($_[0], "needsfirewall"); }
sub name($) { return field($_[0], "name"); } sub name($) { return field($_[0], "name"); }
sub registered($) { return field($_[0], "registered"); } sub registered($) { return field($_[0], "registered"); }
sub lockdown($) { return field($_[0], "lockdown"); }
sub isplaceholder($) { return field($_[0], "isplaceholder"); } sub isplaceholder($) { return field($_[0], "isplaceholder"); }
sub monitor_pid($) { return field($_[0], "monitor_pid"); } sub monitor_pid($) { return field($_[0], "monitor_pid"); }
sub speaksfor_urn($) { return field($_[0], "speaksfor_urn"); } sub speaksfor_urn($) { return field($_[0], "speaksfor_urn"); }
...@@ -994,7 +995,7 @@ sub SetDescription($$) ...@@ -994,7 +995,7 @@ sub SetDescription($$)
my $uuid = $self->uuid(); my $uuid = $self->uuid();
my $safe_description = DBQuoteSpecial($description); my $safe_description = DBQuoteSpecial($description);
return -w return -1
if (!DBQueryWarn("update geni_slices set " . if (!DBQueryWarn("update geni_slices set " .
" description=$safe_description ". " description=$safe_description ".
"where uuid='$uuid'")); "where uuid='$uuid'"));
...@@ -1004,6 +1005,21 @@ sub SetDescription($$) ...@@ -1004,6 +1005,21 @@ sub SetDescription($$)
return 0; return 0;
} }
sub SetLockdown($$)
{
my ($self, $clear) = @_;
my $uuid = $self->uuid();
my $value = ($clear ? 0 : 1);
return -1
if (!DBQueryWarn("update geni_slices set lockdown=$value " .
"where uuid='$uuid'"));
$self->{'SLICE'}->{'lockdown'} = $value;
return 0;
}
sub SetRenewLimit($$) sub SetRenewLimit($$)
{ {
my ($self, $limit) = @_; my ($self, $limit) = @_;
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2008-2010, 2012, 2013 University of Utah and the Flux Group. # Copyright (c) 2008-2015 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -117,6 +117,9 @@ if (!defined($slice)) { ...@@ -117,6 +117,9 @@ if (!defined($slice)) {
if ($slice->Lock() && !$force) { if ($slice->Lock() && !$force) {
fatal("Could not lock slice $slice"); fatal("Could not lock slice $slice");
} }
if ($slice->lockdown()) {
fatal("Slice is locked down, will not delete no matter you say!");
}
GeniUtil::FlipToGeniUser(); GeniUtil::FlipToGeniUser();
sub DoCM() sub DoCM()
......
...@@ -275,7 +275,7 @@ sub WarnSlices() ...@@ -275,7 +275,7 @@ sub WarnSlices()
" UNIX_TIMESTAMP(created) ". " UNIX_TIMESTAMP(created) ".
" from geni_slices ". " from geni_slices ".
"where shutdown is null and ". "where shutdown is null and ".
" hosed=0 and isplaceholder=0"); " hosed=0 and lockdown=0 and isplaceholder=0");
while (my ($idx,$expires,$stamp,$created) = while (my ($idx,$expires,$stamp,$created) =
$query_result->fetchrow_array()) { $query_result->fetchrow_array()) {
...@@ -420,7 +420,7 @@ sub ExpireSlices() ...@@ -420,7 +420,7 @@ sub ExpireSlices()
" (isplaceholder=1 and ". " (isplaceholder=1 and ".
" (UNIX_TIMESTAMP(now()) - ".