Commit db0d2d77 authored by Leigh Stoller's avatar Leigh Stoller

First cut at Powder Stop Button:

The basic operation is that user "powdstop" in the holding project
PowderStop gets to press the stop button. And that's all! This user sees
nothing but the Stop button (and logout). We can give the password to
whoever is supposed to be able to press the button.

Once pressed, we show a list of experiments that are going to be put
into quarantine mode (with power off), and then update the list as the
experiments report back that they are successfully in quarantine mode.
parent 0d852349
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2019 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use English;
use strict;
use Getopt::Std;
use Data::Dumper;
use Date::Parse;
#
# Powder Panic Button
#
sub usage()
{
print STDERR "Usage: powder-panic [-d] [-n]\n";
print STDERR "Options:\n";
print STDERR " -d - Turn on debugging\n";
print STDERR " -n - Dry run mode, just show what would be done\n";
exit(-1);
}
my $optlist = "dnt:";
my $debug = 0;
my $impotent = 0;
my $logfile;
my $webtask_id;
my $webtask;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $OURDOMAIN = "@OURDOMAIN@";
my $MAINSITE = @TBMAINSITE@;
my $MYURN = "urn:publicid:IDN+${OURDOMAIN}+authority+cm";
my $MANAGEINSTANCE = "$TB/bin/manage_instance";
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/usr/bin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# Load the Testbed support stuff.
#
use lib "@prefix@/lib";
use EmulabConstants;
use emdb;
use emutil;
use WebTask;
use Brand;
use libtestbed;
use APT_Geni;
use APT_Aggregate;
use APT_Utility;
use APT_Instance;
# Protos
sub fatal($);
sub notify($);
#
# Only at the Mothership
#
if (!$MAINSITE) {
print "No powder here, too warm.\n";
exit(0);
}
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug++;
}
if (defined($options{"n"})) {
$impotent++;
}
if (defined($options{"t"})) {
$webtask_id = $options{"t"};
$webtask = WebTask->Lookup($webtask_id);
if (!defined($webtask)) {
fatal("Could not lookup/create webtask");
}
$webtask->AutoStore(1);
}
#
# Grab the list of instances to kill off.
#
my @killme = ();
my $query_result =
DBQueryFatal("select uuid from apt_instances");
while (my ($uuid) = $query_result->fetchrow_array()) {
my $instance = APT_Instance->Lookup($uuid);
next
if (!defined($instance));
# This will not be set until the instance is actually paniced.
next
if ($instance->paniced());
#
# Look at the list of aggregates in use, if any of them are flagged
# with panicpoweroff, then do it.
#
my $killit = 0;
foreach my $agg ($instance->AggregateList()) {
my $aptagg = $agg->GetAptAggregate();
next
if ($agg->status() eq "deferred");
#
# Skip anything that is not ready. Need to handle imaging.
#
next
if ($agg->status() ne "ready");
if ($aptagg->panicpoweroff()) {
push(@killme, $instance);
last;
}
#
# Need to look at local experiments too ...
#
if ($aptagg->urn() eq $MYURN) {
#
# Find the local experiment, and look at the list of nodes
# to see if any needs to be powered off during a panic.
#
my $experiment = $instance->LocalExperiment();
if (!defined($experiment)) {
print STDERR
"Could not lookup local Experiment for $instance\n";
next;
}
my @nodelist = $experiment->NodeList(0);
my $bad = 0;
foreach my $node (@nodelist) {
my $nodetype = $node->NodeTypeInfo();
if ($nodetype && $nodetype->GetAttribute("panicpoweroff")) {
$bad = 0;
last;
}
}
if ($bad) {
push(@killme, $instance);
last;
}
}
}
}
#
# Tell the web interface what instances are going to shutdown.
#
if (defined($webtask)) {
my @listing = map {$_->uuid()} @killme;
$webtask->instanceList(\@listing);
}
if (!@killme) {
print "Nothing to panic about, take a chill pill\n";
if (defined($webtask)) {
$webtask->Exited(0);
}
exit(0);
}
#
# Go to background and report later.
#
if (! ($debug || $impotent)) {
$logfile = TBMakeLogname("powder-panic");
if (my $childpid = TBBackGround($logfile)) {
print "Backgounding, see progress in $logfile\n";
if (defined($webtask)) {
$webtask->Exited(0);
}
exit(0);
}
notify("Powder panic button pressed!\n" . "Track progress in $logfile");
}
while (@killme) {
my @todo = @killme;
@killme = ();
foreach my $instance (@todo) {
my $uuid = $instance->uuid();
$instance->Refresh();
next
if ($instance->paniced());
if ($impotent) {
print "Would put $instance into panic mode\n";
next;
}
print "Setting quarantine mode (with power off) on $instance\n";
system("$MANAGEINSTANCE panic $uuid -f -p set");
if ($?) {
print "Will try again later\n";
push(@killme, $instance);
}
}
if (@killme) {
sleep(60);
}
}
notify("All experiments are in panic mode");
if (defined($webtask)) {
$webtask->Exited(0);
}
exit(0);
sub fatal($)
{
my ($mesg) = @_;
if (defined($webtask)) {
$webtask->output($mesg);
$webtask->Exited(-1);
}
notify($mesg);
print STDERR "*** $0:\n".
" $mesg\n";
exit(-1);
}
sub notify($)
{
my ($mesg) = @_;
SENDMAIL($TBOPS,
"Powder Panic Notification",
$mesg,
$TBOPS);
}
$(function ()
{
'use strict';
var templates = APT_OPTIONS.fetchTemplateList(['powder-shutdown',
'waitwait-modal',
'oops-modal']);
var mainString = templates['powder-shutdown'];
var waitwaitString = templates['waitwait-modal'];
var oopsString = templates['oops-modal'];
var timerID = null;
var instances = [];
function initialize()
{
window.APT_OPTIONS.initialize(sup);
$('#page-body').html(mainString);
$('#oops_div').html(oopsString);
$('#waitwait_div').html(waitwaitString);
// Setup the shutdown modal.
$('#powder-shutdown-button').click(function (event) {
event.preventDefault();
DoShutdown();
});
}
// Throw up a confirmation modal, with handler bound to confirm.
function DoShutdown()
{
$('#panic-listing-div').addClass("hidden");
// Handler for hide modal to unbind the click handler.
$('#confirm-shutdown-modal').on('hidden.bs.modal', function (event) {
$(this).unbind(event);
$('#confirm-shutdown-button').unbind("click.shutdown");
});
// Confirm button.
$('#confirm-shutdown-button').bind("click.shutdown", function (event) {
sup.HideModal('#confirm-shutdown-modal');
var callback = function(json) {
sup.HideModal('#waitwait-modal', function () {
if (json.code) {
sup.SpitOops("oops",
"Failed to start powder emergency stop: " +
+ json.value);
}
else {
ShutdownStarted(json.value);
}
});
};
sup.ShowModal('#waitwait-modal');
var xmlthing = sup.CallServerMethod(null, "powder-shutdown",
"Shutdown");
xmlthing.done(callback);
});
sup.ShowModal('#confirm-shutdown-modal');
}
// Shutdown has started, build table, then watch and update.
function ShutdownStarted(info)
{
if (info.length == 0) {
sup.ShowModal("#noexperiments-modal");
return;
}
console.info(info);
/*
* Throw up the list of experiments that are going to be
* shutdown.
*/
var html = "";
_.each(info, function(value, idx) {
var name = value.name;
var status = value.status;
// Remember uuid listing for updates.
instances.push(value.uuid);
if (window.ISADMIN) {
// Show a link for admins.
name = "<a href='status.php?uuid=" + value.uuid + "' " +
"target=_blank>" + name + "</a>";
}
html = html +
"<tr id='" + value.uuid + "'>" +
" <td class='text-nowrap'>" + name + "</td>" +
" <td class='text-nowrap'>" + value.creator + "</td>" +
" <td class='text-nowrap format-date'>" +
value.created + "</td>" +
" <td class='text-nowrap exp-status'>" + status + "</td>" +
" <td class='text-nowrap'>" + value.clusters + "</td>" +
"</tr>";
});
$('#experiments-table tbody').html(html);
// Format dates with moment before display.
$('.format-date').each(function() {
var date = $.trim($(this).html());
if (date != "") {
$(this).html(moment($(this).html()).format("ll"));
}
});
$('#experiments-table').tablesorter({
theme : 'green',
});
$('#panic-listing-div .working').removeClass("hidden");
$('#panic-listing-div .finished').addClass("hidden");
$('#panic-listing-div').removeClass("hidden");
timerID = setInterval(ShutdownWatch, 5000);
}
// Poll for shutdown status
function ShutdownWatch()
{
var count = 0;
var callback = function(json) {
console.info(json);
if (json.code) {
console.info("Failed to get new status: " + json.value);
return;
}
_.each(json.value, function(value, idx) {
var uuid = value.uuid;
if (value.paniced != 0) {
$('#' + uuid + " .exp-status").html(
"<span class=text-danger>quarantined</span>");
count++;
}
});
if (count == instances.length) {
// Done!
$('#panic-listing-div .working').addClass("hidden");
$('#panic-listing-div .finished').removeClass("hidden");
clearInterval(timerID);
timerID = null;
}
};
var xmlthing = sup.CallServerMethod(null, "powder-shutdown", "Status",
{"instances" : instances});
xmlthing.done(callback);
}
$(document).ready(initialize);
});
<?php
#
# Copyright (c) 2000-2019 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
chdir("..");
include_once("webtask.php");
chdir("apt");
include_once("profile_defs.php");
include_once("instance_defs.php");
#
# Need to check the permission, since we allow admins to mess with
# other accounts.
#
function CheckPageArgs()
{
global $this_user;
global $ajax_args;
if (!(ISADMIN())) {
$approved = 0;
$project = Project::Lookup("PowderStop");
if (!$project ||
!$project->IsMember($this_user, $approved) || !$approved) {
SPITAJAX_ERROR(-1, "Not enough permission");
return -1;
}
}
return 0;
}
function Do_StartShutdown()
{
global $this_user, $urn_mapping;
global $ajax_args;
$results = array();
if (CheckPageArgs()) {
return;
}
$webtask = WebTask::CreateAnonymous();
$retval = SUEXEC($this_user->uid(), "nobody",
"webpowder_shutdown -t " . $webtask->task_id() . " -d -n",
SUEXEC_ACTION_CONTINUE);
$webtask->Refresh();
if ($retval != 0) {
SPITAJAX_ERROR(-1, "Internal error starting emergency stop");
$webtask->Delete();
return;
}
$instances = $webtask->TaskValue("instanceList");
if (count($instances)) {
for($i = 0; $i < count($instances); $i++) {
$uuid = $instances[$i];
$instance = Instance::Lookup($uuid);
if (!$instance) {
continue;
}
$blob = array("name" => $instance->name(),
"uuid" => $instance->uuid(),
"creator" => $instance->creator(),
"created" => DateStringGMT($instance->created()),
"status" => $instance->status(),
"paniced" => $instance->paniced() ? 1 : 0);
$clusters = array();
foreach ($instance->slivers() as $sliver) {
$clusters[] = $sliver->aggregate_name();
}
$blob["clusters"] = join(",", $clusters);
$results[] = $blob;
}
}
$webtask->Delete();
SPITAJAX_RESPONSE($results);
}
function Do_ShutdownStatus()
{
global $this_user, $urn_mapping;
global $ajax_args;
$results = array();
if (CheckPageArgs()) {
return;
}
if (!isset($ajax_args["instances"])) {
SPITAJAX_ERROR(1, "Missing instance list");
return;
}
$instances = $ajax_args["instances"];
for($i = 0; $i < count($instances); $i++) {
$uuid = $instances[$i];
$instance = Instance::Lookup($uuid);
if (!$instance) {
continue;
}
$blob = array("uuid" => $instance->uuid(),
"status" => $instance->status(),
"paniced" => $instance->paniced() ? 1 : 1);
$results[] = $blob;
}
SPITAJAX_RESPONSE($results);
}
# Local Variables:
# mode:php
# End:
?>
<?php
#
# Copyright (c) 2000-2017, 2019 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
chdir("..");
include("defs.php3");
include_once("geni_defs.php");
chdir("apt");
include("quickvm_sup.php");
# Do not change this without changing quickvm_sup.php
$page_title = "Powder Shutdown";
#
# Get current user.
#
RedirectSecure();
$this_user = CheckLoginOrRedirect();
$isadmin = (ISADMIN() ? 1 : 0);
if (!$isadmin) {
$approved = 0;
$project = Project::Lookup("PowderStop");
if (!$project ||
!$project->IsMember($this_user, $approved) || !$approved) {
SPITUSERERROR("You do not have permission to access this page");
}
}
SPITHEADER(1);
echo "<link rel='stylesheet'
href='css/tablesorter.css'>\n";
# Place to hang the toplevel template.
echo "<div id='page-body'></div>\n";
echo "<script type='text/javascript'>\n";
echo " window.ISADMIN = $isadmin;\n";
echo "</script>\n";
echo "<script src='js/lib/jquery-2.0.3.min.js'></script>\n";
echo "<script src='js/lib/jquery.tablesorter.min.js'></script>\n";
echo "<script src='js/lib/jquery.tablesorter.widgets.min.js'></script>\n";
REQUIRE_UNDERSCORE();
REQUIRE_SUP();
REQUIRE_MOMENT();
SPITREQUIRE("js/powder-shutdown.js");
AddTemplateList(array("powder-shutdown", "waitwait-modal", "oops-modal"));
SPITFOOTER();
?>
......@@ -99,6 +99,7 @@ $PAGEHEADER_FUNCTION = function($thinheader = 0, $nomenu = false,
}
$height = ($thinheader ? 150 : 250);
$drewheader = 1;
$nonav = 0;
#
# Figure out who is logged in, if anyone.
......@@ -123,6 +124,14 @@ $PAGEHEADER_FUNCTION = function($thinheader = 0, $nomenu = false,
header("Location: licenses.php?referrer=$referrer");
return;
}
if ($login_user && $login_uid == "powdstop") {
$cleanmode = 1;
$nonav = 1;
if ($page_title != "Logout" &&
$page_title != "Powder Shutdown") {
header("Location: powder-shutdown.php");
}
}
header("Expires: Mon, 26 Jul 1997 05:00:00 GMT");
header("Cache-Control: no-cache, must-revalidate");
......@@ -260,7 +269,7 @@ $PAGEHEADER_FUNCTION = function($thinheader = 0, $nomenu = false,
# and turn them on inside the action menu.
$hiddenxs = ($showmenus ? "hidden-xs" : "");
SPITNAV($hiddenxs, $navbar_status, $navbar_right, $login_uid);
SPITNAV($hiddenxs, $nonav, $navbar_status, $navbar_right, $login_uid);
# Put announcements, if any, right below the header.
if (!$cleanmode && $login_user && $login_user->IsActive() &&
......@@ -396,7 +405,7 @@ function SPITHEADER($thinheader = 0,
$PAGEHEADER_FUNCTION($thinheader, $ignore1, $ignore2, $ignore3);
}
function SPITNAV($hiddenxs, $navbar_status, $navbar_right, $login_uid)
function SPITNAV($hiddenxs, $nonav, $navbar_status, $navbar_right, $login_uid)
{
global $PORTAL_MANUAL, $APTLOGO, $login_status, $login_user, $TBMAINSITE;
global $THISHOMEBASE, $ISEMULAB, $ISPNET, $ISPOWDER, $TBBASE;
......@@ -423,7 +432,7 @@ echo " <ul class='nav navbar-nav navbar-left apt-left'>";
echo "<li class='local-name apt-left apt-nav-item'>" . $THISHOMEBASE . "</li>";
}
if ($login_user && !($login_status & CHECKLOGIN_WEBONLY)) {
if ($login_user && !$nonav && !($login_status & CHECKLOGIN_WEBONLY)) {
if ($login_user->IsActive()) {
$then = time() - (90 * 3600 * 24);
......@@ -549,7 +558,7 @@ if (!$login_user->portal()) {
data-toggle='dropdown'>
$login_uid <b class='caret'></b></a>
<ul class='dropdown-menu'>\n";
if (! ($login_status & CHECKLOGIN_WEBONLY)) {
if (!$nonav && !($login_status & CHECKLOGIN_WEBONLY)) {
echo "
<li><a href='myaccount.php'>Manage Account</a></li>
<li><a href='signup.php'>Start/Join Project</a></li>
......
......@@ -465,6 +465,13 @@ $routing = array("geni-login" =>
"Do_Reject",
"Request" =>
"Do_Request")),
"powder-shutdown" =>
array("file" => "powder-shutdown.ajax",
"guest" => false,
"methods" => array("Shutdown" =>
"Do_StartShutdown",
"Status" =>
"Do_ShutdownStatus")),
);
#
......
<div class='row'>
<div class='col-lg-12 col-lg-offset-0
col-md-12 col-md-offset-0
col-sm-12 col-sm-offset-0
col-xs-12 col-xs-offset-0'>
<center>
<h3>Powder Emergency Stop</h3>
</center>
<center style="font-size: 16px;">
Insert explanatory text and warnings here.
</center>
<center id="panic-button-div" style="margin-top: 15px;">
<button class='btn btn-lg btn-danger'
id='powder-shutdown-button' type=button>
Emergency Stop</button>
</center>
<div id="panic-listing-div" class="hidden" style="margin-top: 10px;">
<center>
<div class="working"
style="margin-bottom: 10px; font-size: 16px;">
Working ... patience please.
<div>
<img src='images/spinner.gif' />
</div>
</div>
<div class="finished hidden"
style="margin-bottom: 10px; font-size: 16px;">
Done!
</div>
<div>
<table class='tablesorter' id='experiments-table'>
<thead>
<tr>
<th>Name</th>
<th>Creator</th>
<th>Created</th>