Add lockdown support, to prevent accidental termination of important

experiments. Lockdown is implemented at the portal and at the backend
cluster (ignore by cm and sa daemons). At present, the only policy is
that admin extensions longer then 14 days, set the lockdown.

apt/APT_Instance.pm.in

@@ -345,7 +345,47 @@ sub SetManifest($$)
 
     $self->{'INSTANCE'}->{'manifest'} = $manifest;
     return 0;
-    
+}
+
+#
+# Set/Clear the lockdown bits.
+#
+sub SetLockdown($$$)
+{
+    my ($self,$which,$clear) = @_;
+
+    # Must be a real reference. 
+    return -1
+	if (! ref($self));
+
+    my $uuid = $self->uuid();
+    if ($which eq "admin") {
+	$which = "admin_lockdown";
+    }
+    elsif ($which eq "user") {
+	$which = "user_lockdown";
+    }
+    else {
+	return -1;
+    }
+    my $value = ($clear ? 0 : 1);
+
+    DBQueryWarn("update apt_instances set ${which}=${value} ".
+		"where uuid='$uuid'") or
+	return -1;
+
+    $self->{'INSTANCE'}->{$which} = $value;
+    return 0;
+}
+
+# Return a URL for the status page.
+sub webURL($)
+{
+    my ($self) = @_;
+    my $link = $self->Brand()->wwwBase();
+    $link    = $link . "/status.php?uuid=" . $self->uuid();
+
+    return $link;
 }
 
 #
@@ -432,8 +472,7 @@ sub WarnExpiring($$)
 	print STDERR "WarnExpiring: no email address for $geniuser\n";
 	return 0;
     }
-    my $link = $self->Brand()->wwwBase();
-    $link    = $link . "/status.php?uuid=" . $self->uuid();
+    my $link = $self->webURL();
     
     $self->Brand()->SendEmail($geniuser->email(),
 		     "Your experiment is expiring soon!",
@@ -776,5 +815,38 @@ sub SliverAction($$@)
     return Genixmlrpc::CallMethod($cmurl, $context, $method, $args);
 }
 
+#
+# Lockdown
+#
+sub Lockdown($$)
+{
+    my ($self, $clear) = @_;
+    my $authority = $self->GetGeniAuthority();
+    my $geniuser  = $self->GetGeniUser();
+    my $slice     = $self->GetGeniSlice();
+    my $context   = APT_Geni::GeniContext();
+    return undef
+	if (! (defined($geniuser) && defined($authority) &&
+	       defined($slice) && defined($context)));
+
+    my ($slice_credential, $speaksfor_credential) =
+	APT_Geni::GenCredentials($slice, $geniuser);
+    return undef
+	if (! (defined($speaksfor_credential) &&
+	       defined($slice_credential)));
+
+    my $args = {
+	"slice_urn"   => $slice->urn(),
+	"credentials" => [$slice_credential->asString(),
+			  $speaksfor_credential->asString()],
+    };
+    $args->{"clear"} = 1
+	if ($clear);
+    my $cmurl = $authority->url();
+    $cmurl =~ s/protogeni/protogeni\/stoller/ if ($usemydevtree);
+
+    return Genixmlrpc::CallMethod($cmurl, $context, "Lockdown", $args);
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

apt/apt_daemon.in

@@ -221,6 +221,7 @@ sub ExpireInstances()
 	DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
 		    "left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
 		    "where (a.status='ready') and ".
+		    "      a.admin_lockdown=0 and s.lockdown=0 and ".
 		    "      (UNIX_TIMESTAMP(now()) > ".
 		    "       UNIX_TIMESTAMP(s.expires))");
     return
@@ -259,11 +260,63 @@ sub ExpireInstances()
     }
 }
 
+#
+# Warn about locked down instances that have expired. 
+#
+sub ReportLockdownExpired()
+{
+    my @instances;
+    
+    my $query_result =
+	DBQueryWarn("select a.uuid,s.expires from apt_instances as a ".
+		    "left join geni.geni_slices as s on s.uuid=a.slice_uuid ".
+		    "where (a.admin_lockdown=1 or s.lockdown=1) and ".
+		    "      (UNIX_TIMESTAMP(now()) > ".
+		    "       UNIX_TIMESTAMP(s.expires))");
+    return
+	if (!$query_result);
+
+    while (my ($uuid,$expires) = $query_result->fetchrow_array()) {
+	my $instance = APT_Instance->Lookup($uuid);
+	if (!defined($instance)) {
+	    print STDERR "No such instance $uuid\n";
+	    next;
+	}
+	print STDERR "$uuid is locked down but expired at $expires\n";
+	push(@instances, $instance);
+    }
+    return
+	if (!@instances);
+
+    my $text = "";
+    foreach my $instance (@instances) {
+	my $profile = $instance->Profile();
+	my $creator = $instance->creator();
+	my $wwwlink = $instance->webURL();
+
+	$text .= $profile->name() . ",$creator: $wwwlink\n";
+    }
+    SENDMAIL($TBOPS,
+	     "APT daemon: Locked down instances",
+	     "The following instances are locked down but expired:\n\n".
+	     $text . "\n",
+	     $TBOPS);
+    
+}
+
+my $reportcounter = 0;
+
+# Do this once at startup
+if (! NoLogins()) {
+    ReportLockdownExpired();
+}
+
 while (1) {
     if (NoLogins()) {
 	sleep(5);
 	next;
     }
+    $reportcounter += $SLEEP_INTERVAL;
 
     print "Running at ".
 	POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n";
@@ -271,6 +324,12 @@ while (1) {
     KillFailedInstances();
     ExpireInstances();
 
+    # Do this once every 24 hours.
+    if ($reportcounter >= (24 * 60 * 60)) {
+	ReportLockdownExpired();
+	$reportcounter = 0;
+    }
+    
     exit(0)
 	if ($oneshot);
     

apt/manage_instance.in

@@ -43,6 +43,7 @@ sub usage()
     print("Usage: manage_instance reboot instance node_id [node_id ...]\n");
     print("Usage: manage_instance reload instance node_id [node_id ...]\n");
     print("Usage: manage_instance monitor instance\n");
+    print("Usage: manage_instance lockdown instance set|clear user|admin\n");
     exit(-1);
 }
 my $optlist     = "dt:";
@@ -101,6 +102,7 @@ sub DoExtend();
 sub DoRefresh();
 sub DoReboot();
 sub DoReload();
+sub DoLockdown();
 sub StartMonitor();
 
 #
@@ -154,6 +156,9 @@ elsif ($action eq "reload") {
 elsif ($action eq "monitor") {
     StartMonitor()
 }
+elsif ($action eq "lockdown") {
+    DoLockdown()
+}
 else {
     usage();
 }
@@ -583,7 +588,8 @@ sub DoConsole()
 	    if ($response->value()) {
 		fatal($response->output());
 	    }
-	    fatal("Server returned error: " . $response->code);
+	    fatal("Server returned error: " .
+		  GENIRESPONSE_STRING($response->code));
 	}
     }
     my $url;
@@ -735,11 +741,16 @@ sub DoExtend()
     if ($instance->status() eq "failed") {
 	fatal("Cannot extend failed instance!");
     }
-
     my $slice = $instance->GetGeniSlice();
     if (!defined($slice)) {
 	fatal("No slice for instance!");
     }
+    # The web interface (and in the future the xmlrpc interface) sets this.
+    my $this_user = User->ImpliedUser();
+    if (! defined($this_user)) {
+	$this_user = User->ThisUser();
+    }
+
     #
     # Lock the slice in case it is doing something else, like taking
     # a disk image.
@@ -764,13 +775,24 @@ sub DoExtend()
 	$slice->SetExpiration($oldexpires);
 	$slice->UnLock();
 	# This is something the user should see.
-	if ($response->code() == GENIRESPONSE_REFUSED) {
+	if ($response->code() == GENIRESPONSE_REFUSED ||
+	    $response->code() == GENIRESPONSE_BUSY) {
 	    print STDERR $response->output() . "\n";
 	    # For web interface.
 	    exit(1);
 	}
 	fatal("Failed to extend slice: ". $response->output())
     }
+    if (defined($this_user) && $this_user->IsAdmin() &&
+	($seconds / (24 * 60 * 60)) > 10) {
+	if (DoLockdownInternal("set", "admin")) {
+	    SENDMAIL($TBOPS,
+		     "Failed to lock down APT Instance",
+		     "Failed to lock down $instance\n".
+		     $instance->webURL() . "\n",
+		     $TBOPS);
+	}
+    }
     $slice->UnLock();
     exit(0);
 }
@@ -1087,6 +1109,75 @@ sub StartMonitor()
     exit(0);
 }
 
+#
+# Experiment lockdown.
+#
+sub DoLockdownInternal($$)
+{
+    my ($setclr,$which) = @_;
+    
+    my $slice = $instance->GetGeniSlice();
+    if (!defined($slice)) {
+	fatal("No slice for instance");
+    }
+    if ($which eq "all") {
+	if ($instance->SetLockdown("user", ($setclr eq "clear" ? 1 : 0))) {
+	    print STDERR "Could not update instance lockdown\n";
+	    return -1
+	}
+	$which = "admin"
+    }
+    if ($instance->SetLockdown($which, ($setclr eq "clear" ? 1 : 0))) {
+	print STDERR "Could not update instance lockdown\n";
+	return -1
+    }
+    my $clear = ($instance->admin_lockdown() ||
+		 $instance->user_lockdown() ? 0 : 1);
+    
+    #
+    # Have to set/clear the lockdown on the local slice.
+    #
+    if ($slice->SetLockdown($clear)) {
+	print STDERR "Could not update slice lockdown\n";
+	return -1
+    }
+    #
+    # And tell the backend cluster to lockdown the slice.
+    #
+    my $response = $instance->Lockdown($clear);
+    if (!defined($response)) {
+	print STDERR "RPC Error calling Lockdown\n";
+	# Clear this so we do not think it is locked down for real.
+	$slice->SetLockdown(0);
+	return -1;
+    }
+    if ($response->code() != GENIRESPONSE_SUCCESS) {
+	print STDERR "Could not lockdown sliver: ". $response->output() . "\n";
+	# Clear this so we do not think it is locked down for real.
+	$slice->SetLockdown(0);
+	return -1;
+    }
+    return 0;
+}
+sub DoLockdown()
+{
+    usage()
+	if (@ARGV != 2);
+    
+    my $setclr = shift(@ARGV);
+    my $which  = shift(@ARGV);
+
+    fatal("Must specify either 'admin' or 'user'")
+	if ($which !~ /^(admin|user|all)$/);
+    fatal("Must specify either 'set' or 'clear'")
+	if ($setclr !~ /^(set|clear)$/);
+
+    if (DoLockdownInternal($setclr, $which)) {
+	fatal("Could not lockdown instance!");
+    }
+    exit(0);
+}
+
 sub fatal($)
 {
     my ($mesg) = @_;

protogeni/lib/GeniCMV2.pm.in

@@ -58,6 +58,7 @@ use GeniStitch;
 use GeniStd;
 use emutil;
 use English;
+use libtestbed;
 use Data::Dumper;
 use XML::Simple;
 use Date::Parse;
@@ -724,6 +725,10 @@ sub DeleteSlice($)
 	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
 				    "Credential does not match the URN");
     }
+    if ($slice->lockdown()) {
+	return GeniResponse->Create(GENIRESPONSE_REFUSED(), undef,
+				    "Slice is locked down");
+    }
     if ($slice->Lock() != 0) {
 	return GeniResponse->BusyResponse();
     }
@@ -3785,8 +3790,6 @@ sub DescribeDataset($)
 		!defined($image->creator_urn()) ||
 		$image->creator_urn() ne $user->urn());
 
-	print STDERR Dumper($image);
-	
 	$blob->{'state'}    = "valid";
 	$blob->{'type'}     = "imdataset";
 	$blob->{"busy"}     = 0;
@@ -3836,5 +3839,57 @@ sub DescribeDataset($)
     return GeniResponse->Create(GENIRESPONSE_SUCCESS, $blob);
 }
 
+#
+# Set/Clear the lockdown.
+#
+sub Lockdown($)
+{
+    my ($argref) = @_;
+    my $slice_urn    = $argref->{'slice_urn'};
+    my $clear        = $argref->{'clear'};
+    my $credentials  = $argref->{'credentials'};
+    
+    if (! (defined($credentials) && defined($slice_urn))) {
+	return GeniResponse->MalformedArgsResponse("Missing arguments");
+    }
+    if (! GeniHRN::IsValid($slice_urn)) {
+	return GeniResponse->MalformedArgsResponse("Invalid URN");
+    }
+    my ($credential,$speaksfor) = GeniStd::CheckCredentials($credentials);
+    return $credential
+	if (GeniResponse::IsResponse($credential));
+
+    my $user = GeniCM::CreateUserFromCertificate($credential);
+    return $user
+	if (GeniResponse::IsResponse($user));
+
+    my $authority = GeniCM::CreateAuthorityFromCertificate($credential);
+    return $authority
+	if (GeniResponse::IsResponse($authority));
+
+    my ($slice, $aggregate) = Credential2SliceAggregate($credential);
+    return $slice
+	if (defined($slice) && GeniResponse::IsResponse($slice));
+
+    if (! (defined($slice) && defined($aggregate))) {
+	return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
+				    "Slice does not exist");
+    }
+    if ($slice_urn ne $slice->urn()) {
+	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
+				    "Credential does not match the URN");
+    }
+    if ($slice->creator_urn() ne $user->urn()) {
+	return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
+				    "Not enough permission to access image");
+    }
+    main::AddLogfileMetaDataFromSlice($slice);
+
+    if ($slice->SetLockdown(defined($clear) && $clear ? 1 : 0)) {
+	return GeniResponse->Create(GENIRESPONSE_ERROR);
+    }
+    return GeniResponse->Create(GENIRESPONSE_SUCCESS);
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

protogeni/lib/GeniSlice.pm.in

 #!/usr/bin/perl -wT
 #
-# Copyright (c) 2008-2014 University of Utah and the Flux Group.
+# Copyright (c) 2008-2015 University of Utah and the Flux Group.
 # 
 # {{{GENIPUBLIC-LICENSE
 # 
@@ -308,6 +308,7 @@ sub exptidx($)		{ return field($_[0], "exptidx"); }
 sub needsfirewall($)	{ return field($_[0], "needsfirewall"); }
 sub name($)		{ return field($_[0], "name"); }
 sub registered($)	{ return field($_[0], "registered"); }
+sub lockdown($)		{ return field($_[0], "lockdown"); }
 sub isplaceholder($)	{ return field($_[0], "isplaceholder"); }
 sub monitor_pid($)	{ return field($_[0], "monitor_pid"); }
 sub speaksfor_urn($)	{ return field($_[0], "speaksfor_urn"); }
@@ -994,7 +995,7 @@ sub SetDescription($$)
     my $uuid = $self->uuid();
     my $safe_description = DBQuoteSpecial($description);
 
-    return -w
+    return -1
 	if (!DBQueryWarn("update geni_slices set " .
 			 "  description=$safe_description ".
 			 "where uuid='$uuid'"));
@@ -1004,6 +1005,21 @@ sub SetDescription($$)
     return 0;
 }
 
+sub SetLockdown($$)
+{
+    my ($self, $clear) = @_;
+    my $uuid  = $self->uuid();
+    my $value = ($clear ? 0 : 1);
+
+    return -1
+	if (!DBQueryWarn("update geni_slices set lockdown=$value " .
+			 "where uuid='$uuid'"));
+
+    $self->{'SLICE'}->{'lockdown'} = $value;
+
+    return 0;
+}
+
 sub SetRenewLimit($$)
 {
     my ($self, $limit) = @_;

protogeni/scripts/cleanupslice.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2008-2010, 2012, 2013 University of Utah and the Flux Group.
+# Copyright (c) 2008-2015 University of Utah and the Flux Group.
 # 
 # {{{GENIPUBLIC-LICENSE
 # 
@@ -117,6 +117,9 @@ if (!defined($slice)) {
 if ($slice->Lock() && !$force) {
     fatal("Could not lock slice $slice");
 }
+if ($slice->lockdown()) {
+    fatal("Slice is locked down, will not delete no matter you say!");
+}
 GeniUtil::FlipToGeniUser();
 
 sub DoCM()

protogeni/scripts/expire_daemon.in

@@ -275,7 +275,7 @@ sub WarnSlices()
 			    "   UNIX_TIMESTAMP(created) ".
 			    "  from geni_slices ".
 			    "where shutdown is null and ".
-			    "      hosed=0 and isplaceholder=0");
+			    "      hosed=0 and lockdown=0 and isplaceholder=0");
 
     while (my ($idx,$expires,$stamp,$created) =
 	   $query_result->fetchrow_array()) {
@@ -420,7 +420,7 @@ sub ExpireSlices()
 			    "  (isplaceholder=1 and ".
 			    "     (UNIX_TIMESTAMP(now()) - ".
 			    "      UNIX_TIMESTAMP(created)) > 3600)) ".
-			    " and shutdown is null");
+			    " and shutdown is null and lockdown=0");
 
     while (my ($idx) = $query_result->fetchrow_array()) {
 	my $slice = GeniSlice->Lookup($idx);
@@ -959,7 +959,7 @@ sub CheckIdle()
 	GeniDB::DBQueryWarn("select idx,UNIX_TIMESTAMP(created) ".
 			    "  from geni_slices ".
 			    "where shutdown is null and ".
-			    "      hosed=0 and isplaceholder=0");
+			    "      hosed=0 and lockdown=0 and isplaceholder=0");
 
     while (my ($idx,$created) = $query_result->fetchrow_array()) {
 	my $slice = GeniSlice->Lookup($idx);

protogeni/scripts/sa_daemon.in

@@ -203,7 +203,8 @@ sub WarnSlices()
 	GeniDB::DBQueryWarn("select idx,expires,UNIX_TIMESTAMP(expires), ".
 			    "   UNIX_TIMESTAMP(created) ".
 			    "  from geni_slices ".
-			    "where shutdown is null and exptidx is null");
+			    "where shutdown is null and lockdown=0 and ".
+			    "      exptidx is null");
 
     while (my ($idx,$expires,$stamp,$created) =
 	   $query_result->fetchrow_array()) {
@@ -338,7 +339,8 @@ sub ExpireSlices()
 	GeniDB::DBQueryWarn("select idx from geni_slices ".
 			    "where UNIX_TIMESTAMP(now()) > ".
 			    "      UNIX_TIMESTAMP(expires) and ".
-			    "      shutdown is null and exptidx is null");
+			    "      shutdown is null and lockdown=0 and ".
+			    "      exptidx is null");
 
     while (my ($idx) = $query_result->fetchrow_array()) {
 	my $slice = GeniSlice->Lookup($idx);

protogeni/xmlrpc/protogeni-cm.pm.in

@@ -113,6 +113,7 @@ elsif ($GENI_VERSION eq "2.0") {
 	"DeleteDataset"     => \&GeniCMV2::DeleteDataset,
 	"DescribeDataset"   => \&GeniCMV2::DescribeDataset,
 	"ModifyDataset"     => \&GeniCMV2::ModifyDataset,
+	"Lockdown"          => \&GeniCMV2::Lockdown,
     };
 }
 

www/aptui/instance_defs.php

@@ -65,6 +65,9 @@ class Instance
     function status()	    { return $this->field('status'); }
     function public_url()   { return $this->field('public_url'); }
     function manifest()	    { return $this->field('manifest'); }
+    function admin_lockdown() { return $this->field('admin_lockdown'); }
+    function user_lockdown(){ return $this->field('user_lockdown'); }
+    function extension_reason() { return $this->field('extension_reason'); }
     function servername()   { return $this->field('servername'); }
     function IsAPT() {
 	return preg_match('/aptlab/', $this->servername());
@@ -263,5 +266,15 @@ class Instance
         }
         return $am_array;
     }
+
+    function SetExtensionReason($reason)
+    {
+	$uuid = $this->uuid();
+        $safe_reason = mysql_escape_string($reason);
+
+        DBQueryWarn("update apt_instances set ".
+                    "  extension_reason='$safe_reason' ".
+                    "where uuid='$uuid'");
+    }
 }
 ?>