panic.in 6.92 KB
Newer Older
Leigh Stoller's avatar
Leigh Stoller committed
1 2
#!/usr/bin/perl -wT
#
3
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh Stoller's avatar
Leigh Stoller committed
23
#
24
use strict;
Leigh Stoller's avatar
Leigh Stoller committed
25 26 27 28 29 30 31 32 33 34
use English;
use Getopt::Std;
use POSIX qw(isatty setsid);

#
# Press the panic button. Also invoked from web interface.
#
sub usage()
{
    print(STDERR
Leigh Stoller's avatar
Leigh Stoller committed
35 36
	  "Usage: panic [-l level] <pid> <eid>\n".
	  "       panic -r <pid> <eid>\n".
Leigh Stoller's avatar
Leigh Stoller committed
37
	  "switches and arguments:\n".
Leigh Stoller's avatar
Leigh Stoller committed
38 39
	  "-l level - Level 1; reboot nodes into the admin MFS\n".
	  "         - Level 2; disable the control network\n".
Leigh Stoller's avatar
Leigh Stoller committed
40
	  "-r       - Reset panic state (admin people only)\n".
41
	  "-w       - From web interface, create a log file.\n".
Leigh Stoller's avatar
Leigh Stoller committed
42 43 44 45
	  "<pid>    - The project the experiment belongs to\n".
	  "<eid>    - The experiment name (id)\n");
    exit(-1);
}
46
my $optlist = "rl:w";
Leigh Stoller's avatar
Leigh Stoller committed
47
my $reset   = 0;
Leigh Stoller's avatar
Leigh Stoller committed
48
my $level   = 1;
49
my $dolog   = 0;
Leigh Stoller's avatar
Leigh Stoller committed
50

51
sub fatal($);
Leigh Stoller's avatar
Leigh Stoller committed
52
sub DoIt();
Leigh Stoller's avatar
Leigh Stoller committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82

#
# Exit codes are important; they tell the web page what has happened so
# it can say something useful to the user. Fatal errors are mostly done
# with die(), but expected errors use this routine. At some point we will
# use the DB to communicate the actual error.
#
# $status < 0 - Fatal error. Something went wrong we did not expect.
# $status = 0 - Termination is proceeding in the background. Notified later.
# $status > 0 - Expected error. User not allowed for some reason. 
# 
sub ExitWithStatus($$)
{
    my ($status, $message) = @_;
    
    if ($status < 0) {
	die("*** $0:\n".
	    "    $message\n");
    }
    else {
	print STDERR "$message\n";
    }
    exit($status);
}

#
# Configure variables
#
my $TB     = "@prefix@";
my $TBOPS  = "@TBOPSEMAIL@";
83
my $snmpit = "$TB/bin/snmpit";
Leigh Stoller's avatar
Leigh Stoller committed
84 85 86 87 88 89 90

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
Leigh Stoller's avatar
Leigh Stoller committed
91
use Firewall;
92 93
use Experiment;
use User;
94 95
use Logfile;
use libaudit;
Leigh Stoller's avatar
Leigh Stoller committed
96

97 98
# Locals
my $logfile;
Leigh Stoller's avatar
Leigh Stoller committed
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
    
#
# Untaint the path
# 
$ENV{'PATH'} = '/bin:/usr/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
115
my %options = ();
Leigh Stoller's avatar
Leigh Stoller committed
116 117 118 119 120 121 122 123 124
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 2) {
    usage();
}
if (defined($options{"r"})) {
    $reset = 1;
}
125 126 127
if (defined($options{"w"})) {
    $dolog = 1;
}
Leigh Stoller's avatar
Leigh Stoller committed
128 129 130 131 132
if (defined($options{"l"})) {
    $level = 1;
    usage()
	if ($level < 1 || $level > 2);
}
Leigh Stoller's avatar
Leigh Stoller committed
133

134 135 136
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    tbdie("You ($UID) do not exist!");
Leigh Stoller's avatar
Leigh Stoller committed
137
}
138 139 140 141 142 143
my $user_uid = $this_user->uid();

# Slowly convert to using Experiment module.
my $experiment = Experiment->Lookup($ARGV[0], $ARGV[1]);
if (!defined($experiment)) {
    tbdie("Could not lookup experiment object!")
Leigh Stoller's avatar
Leigh Stoller committed
144
}
145 146
my $pid = $experiment->pid();
my $eid = $experiment->eid();
Leigh Stoller's avatar
Leigh Stoller committed
147 148

#
149 150
# Verify that this person is allowed to press the panic button.
# Note that any script down the line has to do an admin check also. 
Leigh Stoller's avatar
Leigh Stoller committed
151
#
152 153
if ($UID && !$this_user->IsAdmin() &&
    !$experiment->AccessCheck($this_user, TB_EXPT_MODIFY)) {
Leigh Stoller's avatar
Leigh Stoller committed
154
    die("*** $0:\n".
Leigh Stoller's avatar
Leigh Stoller committed
155
	"    You do not have permission for this experiment!\n");
Leigh Stoller's avatar
Leigh Stoller committed
156 157 158 159 160
}

#
# Get email info for user.
#
161 162
my $user_name  = $this_user->name();
my $user_email = $this_user->email();
Leigh Stoller's avatar
Leigh Stoller committed
163

164 165 166 167
# Get email for current swapper.
my $swapper        = $experiment->GetSwapper();
my $swapper_name   = $swapper->name();
my $swapper_email  = $swapper->email();
Leigh Stoller's avatar
Leigh Stoller committed
168 169 170 171 172 173 174 175

#
# We have to protect against trying to end an experiment that is currently
# in the process of being terminated. We use a "wrapper" state (actually
# a timestamp so we can say when termination was requested) since
# terminating consists of a couple of different experiment states down inside
# the tb scripts. 
#
176 177
$experiment->LockTables() == 0
    or fatal("Could not lock experiment tables for $pid/$eid!");
Leigh Stoller's avatar
Leigh Stoller committed
178 179 180 181 182 183

#
# Called from user (via web interface).
#
if ($reset) {
    ExitWithStatus(1, "Experiment $pid/$eid is not paniced!\n")
Leigh Stoller's avatar
Leigh Stoller committed
184 185 186 187
	if ($experiment->state() ne EXPTSTATE_PANICED &&
	    $experiment->paniced());

    $level = $experiment->paniced();
Leigh Stoller's avatar
Leigh Stoller committed
188 189 190
}
else {
    ExitWithStatus(1, "Experiment $pid/$eid is not active!\n")
191 192 193
	if (! ($experiment->state() eq EXPTSTATE_ACTIVE ||
	       $experiment->state() eq EXPTSTATE_ACTIVATING ||
	       $experiment->state() eq EXPTSTATE_SWAPPING));
Leigh Stoller's avatar
Leigh Stoller committed
194 195 196
}

#
197
# Change experiment state and lock it. Unlock tables at same time.
198
#
199
$experiment->Lock(($reset ? EXPTSTATE_ACTIVE : EXPTSTATE_PANICED), 1) == 0
200
    or fatal("Could not lock $experiment");
Leigh Stoller's avatar
Leigh Stoller committed
201

Leigh Stoller's avatar
Leigh Stoller committed
202 203 204 205
# Force level 2 for firewalled experiments.
if ($experiment->IsFirewalled()) {
    $level = 2
	if (!$reset);
Leigh Stoller's avatar
Leigh Stoller committed
206 207
}

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
#
# Coming from the web, use a logfile for spewing.
#
if ($dolog) {
    $logfile = $experiment->CreateLogFile("panic");
    if (defined($logfile)) {
	# We want it to spew to the web.
	$experiment->SetLogFile($logfile);
	# Mark it open since we are going to start using it right away.
	$logfile->Open();

        # Use libaudit to handle the logging and demonizing.
	my $childpid = LogStart(1, $logfile->filename(),
				LIBAUDIT_NODELETE()|LIBAUDIT_LOGTBOPS());
	if ($childpid) {
	    exit(0);
	}
    }
}

Leigh Stoller's avatar
Leigh Stoller committed
228
#
Leigh Stoller's avatar
Leigh Stoller committed
229 230
# XXX - At this point a failure is going to leave things in an
# inconsistent state.
Leigh Stoller's avatar
Leigh Stoller committed
231
#
Leigh Stoller's avatar
Leigh Stoller committed
232 233
if (Firewall::Panic($experiment, $level,
		    ($reset ?
Mike Hibler's avatar
Mike Hibler committed
234
		     Firewall::PANIC_CLEAR() : Firewall::PANIC_PANIC()))) {
Leigh Stoller's avatar
Leigh Stoller committed
235
    fatal("Failure in Firewall::Panic()");
Leigh Stoller's avatar
Leigh Stoller committed
236
}
237
$experiment->Unlock();
Leigh Stoller's avatar
Leigh Stoller committed
238
print "Panic Button has been ". ($reset ? "cleared" : "pressed") . "\n";
Leigh Stoller's avatar
Leigh Stoller committed
239 240 241 242 243

#
# Send email notification to user *and* to tbops.
#
SENDMAIL("$user_name <$user_email>",
244 245 246
	 "Quarantine notification for Experiment $pid/$eid",
	 "Experiment $pid/$eid has been ".
	 ($reset ? "released from quarantine" : "quarantined") . "\n",
Leigh Stoller's avatar
Leigh Stoller committed
247
	 "$user_name <$user_email>",
248
	 "Cc:  $swapper_name <$swapper_email>\n".
Leigh Stoller's avatar
Leigh Stoller committed
249
	 "Bcc: $TBOPS");
250 251 252 253 254

if (defined($logfile)) {
    # Close up the log file so the webpage stops.
    $experiment->CloseLogFile();
}
Leigh Stoller's avatar
Leigh Stoller committed
255
exit(0);
Leigh Stoller's avatar
Leigh Stoller committed
256 257 258 259 260 261 262 263 264

sub fatal($)
{
    my($mesg) = $_[0];
    
    #
    # Send a message to the testbed list. Append the logfile.
    #
    SENDMAIL("$user_name <$user_email>",
265 266 267
	     "Quarantine Failure for Experiment $pid/$eid",
	     "$user_uid ". ($reset ? "cleared" : "set") .
	       " Quarantine mode for experiment $pid/$eid,\n".
Leigh Stoller's avatar
Leigh Stoller committed
268 269 270
	     "BUT there was a failure!\n\n".
	     "$mesg\n",
	     "$user_name <$user_email>",
271
	     "Cc:  $swapper_name <$swapper_email>\n".
Leigh Stoller's avatar
Leigh Stoller committed
272 273 274 275 276
	     "Bcc: $TBOPS");

    die("*** $0:\n".
	"    $mesg\n");
}