Commit cdd60e03 authored by Leigh B Stoller's avatar Leigh B Stoller

Change previous revision wrt terminate/freeze to allow quarantine or

terminate instead.
parent 7f170620
...@@ -61,7 +61,6 @@ sub usage() ...@@ -61,7 +61,6 @@ sub usage()
print("Usage: manage_instance idledata instance\n"); print("Usage: manage_instance idledata instance\n");
print("Usage: manage_instance openstackstats instance\n"); print("Usage: manage_instance openstackstats instance\n");
print("Usage: manage_instance getmanifests instance\n"); print("Usage: manage_instance getmanifests instance\n");
print("Usage: manage_instance destroy instance\n");
print("Usage: manage_instance warn instance\n"); print("Usage: manage_instance warn instance\n");
print("Usage: manage_instance applyextensionpolicy instance\n"); print("Usage: manage_instance applyextensionpolicy instance\n");
exit(-1); exit(-1);
...@@ -151,7 +150,6 @@ sub WriteCredentials(); ...@@ -151,7 +150,6 @@ sub WriteCredentials();
sub StartMonitor(); sub StartMonitor();
sub StartMonitorInternal(;$); sub StartMonitorInternal(;$);
sub DoImageTrackerStuff($$$$$$$); sub DoImageTrackerStuff($$$$$$$);
sub DoDestroy();
sub DoWarn(); sub DoWarn();
sub DenyExtensionInternal($); sub DenyExtensionInternal($);
sub ExtendInternal($$$$$); sub ExtendInternal($$$$$);
...@@ -231,9 +229,6 @@ elsif ($action eq "consoleurl") { ...@@ -231,9 +229,6 @@ elsif ($action eq "consoleurl") {
elsif ($action eq "terminate") { elsif ($action eq "terminate") {
DoTerminate() DoTerminate()
} }
elsif ($action eq "destroy") {
DoDestroy()
}
elsif ($action eq "warn") { elsif ($action eq "warn") {
DoWarn() DoWarn()
} }
...@@ -1274,7 +1269,7 @@ sub DoTerminate() ...@@ -1274,7 +1269,7 @@ sub DoTerminate()
# Lock the slice in case it is doing something else, like taking # Lock the slice in case it is doing something else, like taking
# a disk image. # a disk image.
# #
# When told to take the lock, we take it go. # When told to take the lock, we take it and go.
# #
if ($takelock) { if ($takelock) {
$slice->TakeLock(); $slice->TakeLock();
...@@ -3160,10 +3155,19 @@ sub DoLockdown() ...@@ -3160,10 +3155,19 @@ sub DoLockdown()
sub DoPanic() sub DoPanic()
{ {
my $errmsg; my $errmsg;
my $errcode = -1; my $errcode = -1;
my $exitcode = -1; my $exitcode = -1;
my $takelock = 0;
my $response; my $response;
my $optlist = "L";
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"L"})) {
$takelock = 1;
}
usage() usage()
if (@ARGV != 1); if (@ARGV != 1);
...@@ -3175,7 +3179,13 @@ sub DoPanic() ...@@ -3175,7 +3179,13 @@ sub DoPanic()
if (!defined($slice)) { if (!defined($slice)) {
fatal("No slice for instance"); fatal("No slice for instance");
} }
if ($slice->Lock()) { #
# When told to take the lock, we take it and go.
#
if ($takelock) {
$slice->TakeLock();
}
elsif ($slice->Lock()) {
$errmsg = "Experiment is busy, cannot lock it. Please try again later"; $errmsg = "Experiment is busy, cannot lock it. Please try again later";
$errcode = GENIRESPONSE_BUSY; $errcode = GENIRESPONSE_BUSY;
$exitcode = 1; $exitcode = 1;
...@@ -3224,7 +3234,14 @@ sub DoPanic() ...@@ -3224,7 +3234,14 @@ sub DoPanic()
if ($code != GENIRESPONSE_SUCCESS) { if ($code != GENIRESPONSE_SUCCESS) {
$errcode = $code; $errcode = $code;
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
if ($code == GENIRESPONSE_REFUSED) {
$exitcode = 1;
$errmsg = $response->error();
}
else {
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
}
next; next;
} }
} }
...@@ -4327,176 +4344,6 @@ sub DoSchedTerminate() ...@@ -4327,176 +4344,6 @@ sub DoSchedTerminate()
# #
sub DoDestroy() sub DoDestroy()
{ {
my $errcode = 1;
my $exitcode = 1;
my $freeze = 0;
my $errmsg;
my $reason;
my $logfile;
my $brand = $instance->Brand();
my $creator = $instance->GetGeniUser();
my $slice = $instance->GetGeniSlice();
my $name = $instance->name();
my $pid = $instance->pid();
my $project = $instance->GetProject();
my $optlist = "f:F";
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"F"})) {
$freeze = 1;
}
if (defined($options{"f"})) {
my $filename = $options{"f"};
if (! -e $filename) {
fatal("$filename does not exist");
}
open(MSG, $filename) or
fatal("Could not open $filename");
$reason = "";
while (<MSG>) {
$reason .= $_;
}
close(MSG);
}
if (!$this_user->IsAdmin()) {
fatal("Only admins can destroy experiments");
}
#
# Lock the slice in case it is doing something else, like taking
# a disk image.
#
if ($slice->Lock()) {
$errcode = GENIRESPONSE_BUSY;
$errmsg ="Experiment is busy, cannot lock it. Try again later.";
if (defined($webtask)) {
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
print STDERR "$errmsg\n";
exit(1);
}
if (defined($reason) &&
!TBcheck_dbslot($reason, "default", "fulltext",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
$errmsg = "Illegal characters in your reason";
$errcode = 1;
goto bad;
}
if ($instance->admin_lockdown()) {
$errmsg = "Must clear the admin lockdown first.";
$errcode = 1;
goto bad;
}
# No free time.
$instance->Update({"extension_disabled" => 1});
# Expiration is now.
$slice->SetExpiration(time());
# Now we can clear this.
if ($instance->user_lockdown()) {
if (DoLockdownInternal("clear", "user", 0, \$errmsg)) {
$errcode = 1;
goto bad;
}
}
# Hurry up the terminate instead of waiting for the daemon to see it.
if (!$debug) {
$logfile = TBMakeLogname("destroy");
if (my $childpid = TBBackGround($logfile)) {
my $status = 0;
#
# Wait a couple of seconds to see if there is going to be an
# immediate error. Then return and let it continue to run. This
# allows the web server to see quick errors. Later errors will
# have to be emailed.
#
sleep(5);
my $foo = waitpid($childpid, &WNOHANG);
if ($foo) {
$status = $? >> 8;
}
exit($status);
}
}
# We pass the lock through.
system("$MANAGEINSTANCE " . (defined($webtask) ? "-t $webtask_id " : "").
" -d -- terminate $uuid -L");
#
# We wait for this to finish since if its the local cluster, we cannot
# freeze before termination is complete cause of PROTOGENI_LOCALUSER.
# If there is an error terminating (other then busy), we are kinda
# screwed.
#
if ($?) {
if ($debug) {
exit ($? >> 8);
}
if (defined($webtask)) {
$webtask->Refresh();
print STDERR $webtask->output() . "\n";
$exitcode = $webtask->exitcode();
}
else {
$exitcode = $? >> 8;
}
my $slice_uuid = $slice->uuid();
my $weburl = $instance->webURL();
SENDMAIL($TBOPS,
"Unable to terminate instance with cause!",
"Pid: $pid\n".
"Name: $name\n".
"Slice: $slice_uuid\n".
"URL: $weburl\n".
"Reason:\n\n" . (defined($reason) ? $reason : "") . "\n",
$TBOPS, undef, $logfile);
unlink($logfile) if (defined($logfile));
exit($exitcode);
}
unlink($logfile) if (defined($logfile));
my $message = "Your experiment, $pid/$name, has been terminated!\n";
if ($freeze) {
$message .= "Your account has been frozen until this is resolved.\n";
}
if (defined($reason)) {
$message .= "Reason:\n\n" . $reason . "\n";
}
else {
$message .= "You will be contacted shortly with an explaination.\n";
}
$brand->SendEmail($creator->email(),
"Your experiment has been terminated with cause!",
$message,
$brand->OpsEmailAddress(),
"CC: " . $project->GetLeader()->email() . "\n" .
"BCC: " . $brand->OpsEmailAddress());
# This will send email if it fails.
if ($freeze) {
my $creator_uid = $instance->creator();
system("$TBACCT -u freeze $creator_uid");
if ($?) {
exit($? >> 8);
}
}
exit(0);
bad:
print STDERR $errmsg . "\n";
if (defined($webtask)) {
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
done:
exit($exitcode);
} }
# #
...@@ -4508,6 +4355,7 @@ sub DoWarn() ...@@ -4508,6 +4355,7 @@ sub DoWarn()
my $exitcode = 1; my $exitcode = 1;
my $freeze = 0; my $freeze = 0;
my $terminate = 0; my $terminate = 0;
my $panic = 0;
my $errmsg; my $errmsg;
my $reason; my $reason;
my $logfile; my $logfile;
...@@ -4520,7 +4368,7 @@ sub DoWarn() ...@@ -4520,7 +4368,7 @@ sub DoWarn()
my $slice_uuid = $slice->uuid(); my $slice_uuid = $slice->uuid();
my $weburl = $instance->webURL(); my $weburl = $instance->webURL();
my $optlist = "f:FT"; my $optlist = "f:FTQ";
my %options = (); my %options = ();
if (! getopts($optlist, \%options)) { if (! getopts($optlist, \%options)) {
usage(); usage();
...@@ -4531,6 +4379,9 @@ sub DoWarn() ...@@ -4531,6 +4379,9 @@ sub DoWarn()
if (defined($options{"T"})) { if (defined($options{"T"})) {
$terminate = 1; $terminate = 1;
} }
elsif (defined($options{"Q"})) {
$panic = 1;
}
if (defined($options{"f"})) { if (defined($options{"f"})) {
my $filename = $options{"f"}; my $filename = $options{"f"};
if (! -e $filename) { if (! -e $filename) {
...@@ -4545,7 +4396,7 @@ sub DoWarn() ...@@ -4545,7 +4396,7 @@ sub DoWarn()
} }
close(MSG); close(MSG);
} }
if (!$freeze && !$terminate && !$reason) { if (!$freeze && !$terminate && !$panic && !$reason) {
fatal("Must supply a message!"); fatal("Must supply a message!");
} }
if (!$this_user->IsAdmin()) { if (!$this_user->IsAdmin()) {
...@@ -4562,7 +4413,7 @@ sub DoWarn() ...@@ -4562,7 +4413,7 @@ sub DoWarn()
# #
# Just a message. # Just a message.
# #
if (!$terminate) { if (!($terminate || $panic)) {
# This will send email if it fails. # This will send email if it fails.
if ($freeze) { if ($freeze) {
my $creator_uid = $instance->creator(); my $creator_uid = $instance->creator();
...@@ -4646,9 +4497,18 @@ sub DoWarn() ...@@ -4646,9 +4497,18 @@ sub DoWarn()
exit($status); exit($status);
} }
} }
# We pass the lock through. if ($panic) {
system("$MANAGEINSTANCE " . (defined($webtask) ? "-t $webtask_id " : ""). # We pass the lock through.
" -d -- terminate $uuid -L"); system("$MANAGEINSTANCE " .
(defined($webtask) ? "-t $webtask_id " : "").
" -d -- panic $uuid -L set");
}
else {
# We pass the lock through.
system("$MANAGEINSTANCE " .
(defined($webtask) ? "-t $webtask_id " : "").
" -d -- terminate $uuid -L");
}
# #
# We wait for this to finish since if its the local cluster, we cannot # We wait for this to finish since if its the local cluster, we cannot
...@@ -4671,7 +4531,9 @@ sub DoWarn() ...@@ -4671,7 +4531,9 @@ sub DoWarn()
$exitcode = $? >> 8; $exitcode = $? >> 8;
} }
SENDMAIL($TBOPS, SENDMAIL($TBOPS,
"Unable to terminate instance with cause!", ($panic ?
"Unable to quarantine instance with cause!" :
"Unable to terminate instance with cause!"),
"Pid: $pid\n". "Pid: $pid\n".
"Name: $name\n". "Name: $name\n".
"Slice: $slice_uuid\n". "Slice: $slice_uuid\n".
...@@ -4683,7 +4545,17 @@ sub DoWarn() ...@@ -4683,7 +4545,17 @@ sub DoWarn()
} }
unlink($logfile) if (defined($logfile)); unlink($logfile) if (defined($logfile));
my $message = "Your experiment, $pid/$name, has been terminated!\n"; my $message;
my $subject;
if ($panic) {
$message = "Your experiment, $pid/$name, has been quarantined!\n";
$subject = "Your experiment $name has been quarantined!";
}
else {
$message = "Your experiment, $pid/$name, has been terminated!\n";
$subject = "Your experiment $name has been terminated with cause!";
}
if ($freeze) { if ($freeze) {
$message .= "Your account has been frozen until this is resolved.\n"; $message .= "Your account has been frozen until this is resolved.\n";
} }
...@@ -4693,11 +4565,14 @@ sub DoWarn() ...@@ -4693,11 +4565,14 @@ sub DoWarn()
else { else {
$message .= "You will be contacted shortly with an explaination.\n"; $message .= "You will be contacted shortly with an explaination.\n";
} }
if ($panic) {
$message .= "\n" . $weburl . "\n";
}
$brand->SendEmail($creator->email(), $brand->SendEmail($creator->email(),
"Your experiment has been terminated with cause!", $subject,
$message, $message,
$brand->OpsEmailAddress(), $brand->OpsEmailAddress(),
"CC: " . $project->GetLeader()->email(), "CC: " . $project->GetLeader()->email() . "\n" .
"BCC: " . $brand->OpsEmailAddress()); "BCC: " . $brand->OpsEmailAddress());
# This will send email if it fails. # This will send email if it fails.
......
...@@ -264,11 +264,7 @@ $(function () ...@@ -264,11 +264,7 @@ $(function ()
lockdown_override}); lockdown_override});
xmlthing.done(callback); xmlthing.done(callback);
}); });
// Warn/Kill an experiment. SetupWarnKill();
$('#warnkill-experiment-button').click(function (event) {
event.preventDefault();
WarnExperiment();
});
// Handler for select/deselect all rows in the list view. // Handler for select/deselect all rows in the list view.
$('#select-all').change(function () { $('#select-all').change(function () {
...@@ -670,6 +666,12 @@ $(function () ...@@ -670,6 +666,12 @@ $(function ()
if (expinfo.admin_lockdown || !window.APT_OPTIONS.canterminate) { if (expinfo.admin_lockdown || !window.APT_OPTIONS.canterminate) {
terminate = 0; terminate = 0;
} }
// Or if paniced, most buttons disabled.
if (expinfo.paniced) {
terminate = extend = snapshot = destroy = 0;
refresh = reloadtopo = 0;
}
ButtonState('terminate', terminate); ButtonState('terminate', terminate);
ButtonState('refresh', refresh); ButtonState('refresh', refresh);
ButtonState('reloadtopo', reloadtopo); ButtonState('reloadtopo', reloadtopo);
...@@ -3461,6 +3463,28 @@ $(function () ...@@ -3461,6 +3463,28 @@ $(function ()
/* /*
* Terminate with cause and optionally freeze user. * Terminate with cause and optionally freeze user.
*/ */
function SetupWarnKill()
{
$('#warnkill-experiment-button').click(function (event) {
event.preventDefault();
WarnExperiment();
});
// The Terminate/quarantine is a radio that can be unselected.
$('#destroy-quarantine-checkbox').change(function () {
if ($('#destroy-quarantine-checkbox').is(':checked')) {
// Flip the other checkbox off
$('#destroy-terminate-checkbox').prop("checked", false);
}
});
$('#destroy-terminate-checkbox').change(function () {
if ($('#destroy-terminate-checkbox').is(':checked')) {
// Flip the other checkbox off
$('#destroy-quarantine-checkbox').prop("checked", false);
}
});
}
function WarnExperiment() function WarnExperiment()
{ {
// Handler for the Snapshot confirm button. // Handler for the Snapshot confirm button.
...@@ -3469,6 +3493,7 @@ $(function () ...@@ -3469,6 +3493,7 @@ $(function ()
event.preventDefault(); event.preventDefault();
var reason = $('#destroy-experiment-reason').val(); var reason = $('#destroy-experiment-reason').val();
var kill = $('#destroy-terminate-checkbox').is(':checked'); var kill = $('#destroy-terminate-checkbox').is(':checked');
var panic = $('#destroy-quarantine-checkbox').is(':checked');
var freeze = $('#destroy-freeze-checkbox').is(':checked'); var freeze = $('#destroy-freeze-checkbox').is(':checked');
var args = {"uuid" : uuid}; var args = {"uuid" : uuid};
if (reason != "") { if (reason != "") {
...@@ -3480,6 +3505,9 @@ $(function () ...@@ -3480,6 +3505,9 @@ $(function ()
if (kill) { if (kill) {
args["terminate"] = true; args["terminate"] = true;
} }
else if (panic) {
args["quarantine"] = true;
}
sup.HideModal("#destroy-experiment-modal", function () { sup.HideModal("#destroy-experiment-modal", function () {
sup.ShowWaitWait(); sup.ShowWaitWait();
sup.CallServerMethod(null, "status", "Warn", args, sup.CallServerMethod(null, "status", "Warn", args,
......
...@@ -1903,10 +1903,13 @@ function Do_WarnExperiment() ...@@ -1903,10 +1903,13 @@ function Do_WarnExperiment()
chmod($filename, 0666); chmod($filename, 0666);
$options = "-f $filename "; $options = "-f $filename ";
} }
# Terminate experiment # Terminate or Quarantine experiment
if (isset($ajax_args["terminate"]) && $ajax_args["terminate"]) { if (isset($ajax_args["terminate"]) && $ajax_args["terminate"]) {
$options .= "-T "; $options .= "-T ";
} }
elseif (isset($ajax_args["quarantine"]) && $ajax_args["quarantine"]) {
$options .= "-Q ";
}
# Freeze user # Freeze user
if (isset($ajax_args["freeze"]) && $ajax_args["freeze"]) { if (isset($ajax_args["freeze"]) && $ajax_args["freeze"]) {
$options .= "-F "; $options .= "-F ";
......
...@@ -18,11 +18,16 @@ ...@@ -18,11 +18,16 @@
class='form-control' rows=5></textarea> class='form-control' rows=5></textarea>
</div> </div>
<input type=checkbox <input type=checkbox
id='destroy-quarantine-checkbox' value=yes>
Quarantine Experiment or
<input type=checkbox style="margin-left: 5px"
id='destroy-terminate-checkbox' value=yes> id='destroy-terminate-checkbox' value=yes>
Terminate Experiment Terminate Experiment
<input type=checkbox style="margin-left: 20px" <div>
<input type=checkbox
id='destroy-freeze-checkbox' value=yes> id='destroy-freeze-checkbox' value=yes>
Freeze User? Freeze User?
</div>
<div style="margin-top: 10px;"> <div style="margin-top: 10px;">
<button class='btn btn-primary' <button class='btn btn-primary'
style="margin-right: 10px;" style="margin-right: 10px;"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment