Skip to content

GitLab

Commit 00563293 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

Add lockdown support, to prevent accidental termination of important 

experiments. Lockdown is implemented at the portal and at the backend
cluster (ignore by cm and sa daemons). At present, the only policy is
that admin extensions longer then 14 days, set the lockdown.
parent 7b6f6150
Hide whitespace changes
Inline Side-by-side
apt/APT_Instance.pm.in
View file @ 00563293
......@@ -345,7 +345,47 @@ sub SetManifest($$)
$self->{'INSTANCE'}->{'manifest'} = $manifest;
return 0;
}
#
# Set/Clear the lockdown bits.
#
sub SetLockdown($$$)
{
my ($self,$which,$clear) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $uuid = $self->uuid();
if ($which eq "admin") {
$which = "admin_lockdown";
}
elsif ($which eq "user") {
$which = "user_lockdown";
}
else {
return -1;
}
my $value = ($clear ? 0 : 1);
DBQueryWarn("update apt_instances set ${which}=${value} ".
"where uuid='$uuid'") or
return -1;
$self->{'INSTANCE'}->{$which} = $value;
return 0;
}
# Return a URL for the status page.
sub webURL($)
{
my ($self) = @_;
my $link = $self->Brand()->wwwBase();
$link = $link . "/status.php?uuid=" . $self->uuid();
return $link;
}
#
......@@ -432,8 +472,7 @@ sub WarnExpiring($$)
print STDERR "WarnExpiring: no email address for $geniuser\n";
return 0;
}
my $link = $self->Brand()->wwwBase();
$link = $link . "/status.php?uuid=" . $self->uuid();
my $link = $self->webURL();
$self->Brand()->SendEmail($geniuser->email(),
"Your experiment is expiring soon!",
......@@ -776,5 +815,38 @@ sub SliverAction($$@)
return Genixmlrpc::CallMethod($cmurl, $context, $method, $args);
}
#
# Lockdown
#
sub Lockdown($$)
{
my ($self, $clear) = @_;
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->GetGeniUser();
my $slice = $self->GetGeniSlice();
my $context = APT_Geni::GeniContext();
return undef
if (! (defined($geniuser) && defined($authority) &&
defined($slice) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($slice, $geniuser);
return undef
if (! (defined($speaksfor_credential) &&
defined($slice_credential)));
my $args = {
"slice_urn" => $slice->urn(),
"credentials" => [$slice_credential->asString(),
$speaksfor_credential->asString()],
};
$args->{"clear"} = 1
if ($clear);
my $cmurl = $authority->url();
$cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
return Genixmlrpc::CallMethod($cmurl, $context, "Lockdown", $args);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
apt/apt_daemon.in
View file @ 00563293
......@@ -221,6 +221,7 @@ sub ExpireInstances()
DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
"left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
"where (a.status='ready') and ".
" a.admin_lockdown=0 and s.lockdown=0 and ".
" (UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(s.expires))");
return
......@@ -259,11 +260,63 @@ sub ExpireInstances()
}
}
#
# Warn about locked down instances that have expired.
#
sub ReportLockdownExpired()
{
my @instances;
my $query_result =
DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
"left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
"where (a.admin_lockdown=1 or s.lockdown=1) and ".
" (UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(s.expires))");
return
if (!$query_result);
while (my ($uuid,$expires) = $query_result->fetchrow_array()) {
my $instance = APT_Instance->Lookup($uuid);
if (!defined($instance)) {
print STDERR "No such instance $uuid\n";
next;
}
print STDERR "$uuid is locked down but expired at $expires\n";
push(@instances, $instance);
}
return
if (!@instances);
my $text = "";
foreach my $instance (@instances) {
my $profile = $instance->Profile();
my $creator = $instance->creator();
my $wwwlink = $instance->webURL();
$text .= $profile->name() . ",$creator: $wwwlink\n";
}
SENDMAIL($TBOPS,
"APT daemon: Locked down instances",
"The following instances are locked down but expired:\n\n".
$text . "\n",
$TBOPS);
}
my $reportcounter = 0;
# Do this once at startup
if (! NoLogins()) {
ReportLockdownExpired();
}
while (1) {
if (NoLogins()) {
sleep(5);
next;
}
$reportcounter += $SLEEP_INTERVAL;
print "Running at ".
POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n";
......@@ -271,6 +324,12 @@ while (1) {
KillFailedInstances();
ExpireInstances();
# Do this once every 24 hours.
if ($reportcounter >= (24 * 60 * 60)) {
ReportLockdownExpired();
$reportcounter = 0;
}
exit(0)
if ($oneshot);
......
apt/manage_instance.in
View file @ 00563293
......@@ -43,6 +43,7 @@ sub usage()
print("Usage: manage_instance reboot instance node_id [node_id ...]\n");
print("Usage: manage_instance reload instance node_id [node_id ...]\n");
print("Usage: manage_instance monitor instance\n");
print("Usage: manage_instance lockdown instance set|clear user|admin\n");
exit(-1);
}
my $optlist = "dt:";
......@@ -101,6 +102,7 @@ sub DoExtend();
sub DoRefresh();
sub DoReboot();
sub DoReload();
sub DoLockdown();
sub StartMonitor();
#
......@@ -154,6 +156,9 @@ elsif ($action eq "reload") {
elsif ($action eq "monitor") {
StartMonitor()
}
elsif ($action eq "lockdown") {
DoLockdown()
}
else {
usage();
}
......@@ -583,7 +588,8 @@ sub DoConsole()
if ($response->value()) {
fatal($response->output());
}
fatal("Server returned error: " . $response->code);
fatal("Server returned error: " .
GENIRESPONSE_STRING($response->code));
}
}
my $url;
......@@ -735,11 +741,16 @@ sub DoExtend()
if ($instance->status() eq "failed") {
fatal("Cannot extend failed instance!");
}
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance!");
}
# The web interface (and in the future the xmlrpc interface) sets this.
my $this_user = User->ImpliedUser();
if (! defined($this_user)) {
$this_user = User->ThisUser();
}
#
# Lock the slice in case it is doing something else, like taking
# a disk image.
......@@ -764,13 +775,24 @@ sub DoExtend()
$slice->SetExpiration($oldexpires);
$slice->UnLock();
# This is something the user should see.
if ($response->code() == GENIRESPONSE_REFUSED) {
if ($response->code() == GENIRESPONSE_REFUSED ||
$response->code() == GENIRESPONSE_BUSY) {
print STDERR $response->output() . "\n";
# For web interface.
exit(1);
}
fatal("Failed to extend slice: ". $response->output())
}
if (defined($this_user) && $this_user->IsAdmin() &&
($seconds / (24 * 60 * 60)) > 10) {
if (DoLockdownInternal("set", "admin")) {
SENDMAIL($TBOPS,
"Failed to lock down APT Instance",
"Failed to lock down $instance\n".
$instance->webURL() . "\n",
$TBOPS);
}
}
$slice->UnLock();
exit(0);
}
......@@ -1087,6 +1109,75 @@ sub StartMonitor()
exit(0);
}
#
# Experiment lockdown.
#
sub DoLockdownInternal($$)
{
my ($setclr,$which) = @_;
my $slice = $instance->GetGeniSlice();
if (!defined($slice)) {
fatal("No slice for instance");
}
if ($which eq "all") {
if ($instance->SetLockdown("user", ($setclr eq "clear" ? 1 : 0))) {
print STDERR "Could not update instance lockdown\n";
return -1
}
$which = "admin"
}
if ($instance->SetLockdown($which, ($setclr eq "clear" ? 1 : 0))) {
print STDERR "Could not update instance lockdown\n";
return -1
}
my $clear = ($instance->admin_lockdown() ||
$instance->user_lockdown() ? 0 : 1);
#
# Have to set/clear the lockdown on the local slice.
#
if ($slice->SetLockdown($clear)) {
print STDERR "Could not update slice lockdown\n";
return -1
}
#
# And tell the backend cluster to lockdown the slice.
#
my $response = $instance->Lockdown($clear);
if (!defined($response)) {
print STDERR "RPC Error calling Lockdown\n";
# Clear this so we do not think it is locked down for real.
$slice->SetLockdown(0);
return -1;
}
if ($response->code() != GENIRESPONSE_SUCCESS) {
print STDERR "Could not lockdown sliver: ". $response->output() . "\n";
# Clear this so we do not think it is locked down for real.
$slice->SetLockdown(0);
return -1;
}
return 0;
}
sub DoLockdown()
{
usage()
if (@ARGV != 2);
my $setclr = shift(@ARGV);
my $which = shift(@ARGV);
fatal("Must specify either 'admin' or 'user'")
if ($which !~ /^(admin|user|all)$/);
fatal("Must specify either 'set' or 'clear'")
if ($setclr !~ /^(set|clear)$/);
if (DoLockdownInternal($setclr, $which)) {
fatal("Could not lockdown instance!");
}
exit(0);
}
sub fatal($)
{
my ($mesg) = @_;
......
protogeni/lib/GeniCMV2.pm.in
View file @ 00563293
......@@ -58,6 +58,7 @@ use GeniStitch;
use GeniStd;
use emutil;
use English;
use libtestbed;
use Data::Dumper;
use XML::Simple;
use Date::Parse;
......@@ -724,6 +725,10 @@ sub DeleteSlice($)
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
if ($slice->lockdown()) {
return GeniResponse->Create(GENIRESPONSE_REFUSED(), undef,
"Slice is locked down");
}
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
......@@ -3785,8 +3790,6 @@ sub DescribeDataset($)
!defined($image->creator_urn()) ||
$image->creator_urn() ne $user->urn());
print STDERR Dumper($image);
$blob->{'state'} = "valid";
$blob->{'type'} = "imdataset";
$blob->{"busy"} = 0;
......@@ -3836,5 +3839,57 @@ sub DescribeDataset($)
return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
}
#
# Set/Clear the lockdown.
#
sub Lockdown($)
{
my ($argref) = @_;
my $slice_urn = $argref->{'slice_urn'};
my $clear = $argref->{'clear'};
my $credentials = $argref->{'credentials'};
if (! (defined($credentials) && defined($slice_urn))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
}
if (! GeniHRN::IsValid($slice_urn)) {
return GeniResponse->MalformedArgsResponse("Invalid URN");
}
my ($credential,$speaksfor) = GeniStd::CheckCredentials($credentials);
return $credential
if (GeniResponse::IsResponse($credential));
my $user = GeniCM::CreateUserFromCertificate($credential);
return $user
if (GeniResponse::IsResponse($user));
my $authority = GeniCM::CreateAuthorityFromCertificate($credential);
return $authority
if (GeniResponse::IsResponse($authority));
my ($slice, $aggregate) = Credential2SliceAggregate($credential);
return $slice
if (defined($slice) && GeniResponse::IsResponse($slice));
if (! (defined($slice) && defined($aggregate))) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"Slice does not exist");
}
if ($slice_urn ne $slice->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
if ($slice->creator_urn() ne $user->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Not enough permission to access image");
}
main::AddLogfileMetaDataFromSlice($slice);
if ($slice->SetLockdown(defined($clear) && $clear ? 1 : 0)) {
return GeniResponse->Create(GENIRESPONSE_ERROR);
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
protogeni/lib/GeniSlice.pm.in
View file @ 00563293
#!/usr/bin/perl -wT
#
# Copyright (c) 2008-2014 University of Utah and the Flux Group.
# Copyright (c) 2008-2015 University of Utah and the Flux Group.
#
# {{{GENIPUBLIC-LICENSE
#
......@@ -308,6 +308,7 @@ sub exptidx($) { return field($_[0], "exptidx"); }
sub needsfirewall($) { return field($_[0], "needsfirewall"); }
sub name($) { return field($_[0], "name"); }
sub registered($) { return field($_[0], "registered"); }
sub lockdown($) { return field($_[0], "lockdown"); }
sub isplaceholder($) { return field($_[0], "isplaceholder"); }
sub monitor_pid($) { return field($_[0], "monitor_pid"); }
sub speaksfor_urn($) { return field($_[0], "speaksfor_urn"); }
......@@ -994,7 +995,7 @@ sub SetDescription($$)
my $uuid = $self->uuid();
my $safe_description = DBQuoteSpecial($description);
return -w
return -1
if (!DBQueryWarn("update geni_slices set " .
" description=$safe_description ".
"where uuid='$uuid'"));
......@@ -1004,6 +1005,21 @@ sub SetDescription($$)
return 0;
}
sub SetLockdown($$)
{
my ($self, $clear) = @_;
my $uuid = $self->uuid();
my $value = ($clear ? 0 : 1);
return -1
if (!DBQueryWarn("update geni_slices set lockdown=$value " .
"where uuid='$uuid'"));
$self->{'SLICE'}->{'lockdown'} = $value;
return 0;
}
sub SetRenewLimit($$)
{
my ($self, $limit) = @_;
......
protogeni/scripts/cleanupslice.in
View file @ 00563293
#!/usr/bin/perl -w
#
# Copyright (c) 2008-2010, 2012, 2013 University of Utah and the Flux Group.
# Copyright (c) 2008-2015 University of Utah and the Flux Group.
#
# {{{GENIPUBLIC-LICENSE
#
......@@ -117,6 +117,9 @@ if (!defined($slice)) {
if ($slice->Lock() && !$force) {
fatal("Could not lock slice $slice");
}
if ($slice->lockdown()) {
fatal("Slice is locked down, will not delete no matter you say!");
}
GeniUtil::FlipToGeniUser();
sub DoCM()
......
protogeni/scripts/expire_daemon.in
View file @ 00563293
......@@ -275,7 +275,7 @@ sub WarnSlices()
" UNIX_TIMESTAMP(created) ".
" from geni_slices ".
"where shutdown is null and ".
" hosed=0 and isplaceholder=0");
" hosed=0 and lockdown=0 and isplaceholder=0");
while (my ($idx,$expires,$stamp,$created) =
$query_result->fetchrow_array()) {
......@@ -420,7 +420,7 @@ sub ExpireSlices()
" (isplaceholder=1 and ".
" (UNIX_TIMESTAMP(now()) - ".
" UNIX_TIMESTAMP(created)) > 3600)) ".
" and shutdown is null");
" and shutdown is null and lockdown=0");
while (my ($idx) = $query_result->fetchrow_array()) {
my $slice = GeniSlice->Lookup($idx);
......@@ -959,7 +959,7 @@ sub CheckIdle()
GeniDB::DBQueryWarn("select idx,UNIX_TIMESTAMP(created) ".
" from geni_slices ".
"where shutdown is null and ".
" hosed=0 and isplaceholder=0");
" hosed=0 and lockdown=0 and isplaceholder=0");
while (my ($idx,$created) = $query_result->fetchrow_array()) {
my $slice = GeniSlice->Lookup($idx);
......
protogeni/scripts/sa_daemon.in
View file @ 00563293
......@@ -203,7 +203,8 @@ sub WarnSlices()
GeniDB::DBQueryWarn("select idx,expires,UNIX_TIMESTAMP(expires), ".
" UNIX_TIMESTAMP(created) ".
" from geni_slices ".
"where shutdown is null and exptidx is null");
"where shutdown is null and lockdown=0 and ".
" exptidx is null");
while (my ($idx,$expires,$stamp,$created) =
$query_result->fetchrow_array()) {
......@@ -338,7 +339,8 @@ sub ExpireSlices()
GeniDB::DBQueryWarn("select idx from geni_slices ".
"where UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(expires) and ".
" shutdown is null and exptidx is null");
" shutdown is null and lockdown=0 and ".
" exptidx is null");
while (my ($idx) = $query_result->fetchrow_array()) {
my $slice = GeniSlice->Lookup($idx);
......
protogeni/xmlrpc/protogeni-cm.pm.in
View file @ 00563293
......@@ -113,6 +113,7 @@ elsif ($GENI_VERSION eq "2.0") {
"DeleteDataset" => \&GeniCMV2::DeleteDataset,
"DescribeDataset" => \&GeniCMV2::DescribeDataset,
"ModifyDataset" => \&GeniCMV2::ModifyDataset,
"Lockdown" => \&GeniCMV2::Lockdown,
};
}
......
www/aptui/instance_defs.php
View file @ 00563293
......@@ -65,6 +65,9 @@ class Instance
function status() { return $this->field('status'); }
function public_url() { return $this->field('public_url'); }
function manifest() { return $this->field('manifest'); }
function admin_lockdown() { return $this->field('admin_lockdown'); }
function user_lockdown(){ return $this->field('user_lockdown'); }
function extension_reason() { return $this->field('extension_reason'); }
function servername() { return $this->field('servername'); }
function IsAPT() {
return preg_match('/aptlab/', $this->servername());
......@@ -263,5 +266,15 @@ class Instance
}
return $am_array;
}
function SetExtensionReason($reason)
{
$uuid = $this->uuid();
$safe_reason = mysql_escape_string($reason);
DBQueryWarn("update apt_instances set ".
" extension_reason='$safe_reason' ".
"where uuid='$uuid'");
}
}
?>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment