Commit bc4d1802 authored by Leigh Stoller's avatar Leigh Stoller

Run Panic in the background since it can take a long time, and we do not

want the web interface just sitting there. We wait a short time for the
child to catch quick errors, but rely on email notification if something
goes wrong in the backend.
parent 3e17cecd
......@@ -645,13 +645,15 @@ sub SetPanic($$)
my $uuid = $self->uuid();
my $value = ($clear ? 0 : 1);
my $stamp = ($clear ? "NULL" : "now()");
DBQueryWarn("update apt_instances set paniced=${value} ".
(!$clear ? ",paniced_timestamp=now() " : "") .
DBQueryWarn("update apt_instances set ".
" paniced=${value},paniced_timestamp=${stamp} ".
"where uuid='$uuid'") or
return -1;
$self->{'INSTANCE'}->{"paniced"} = $value;
$self->{'INSTANCE'}->{"paniced_timestamp"} = ($clear ? undef : time());
return 0;
}
......@@ -3170,6 +3172,9 @@ sub Panic($$$)
return CredentialError()
if (! defined($slice_credential));
# Bump default timeout, this can take a while for a response
Genixmlrpc->SetTimeout(900);
my $args = {
"slice_urn" => $slice->urn(),
"credentials" => [$slice_credential->asString()],
......
......@@ -3260,10 +3260,12 @@ sub DoPanic()
my $errcode = -1;
my $exitcode = -1;
my $response;
my $logfile;
my $optlist = "Lp";
my $takelock = 0;
my $poweroff = 0;
my $optlist = "Lpf";
my $takelock = 0;
my $poweroff = 0;
my $foreground = 0;
my %options = ();
if (! getopts($optlist, \%options)) {
......@@ -3275,6 +3277,9 @@ sub DoPanic()
if (defined($options{"p"})) {
$poweroff = 1;
}
if (defined($options{"f"})) {
$foreground = 1;
}
usage()
if (@ARGV != 1);
......@@ -3302,7 +3307,11 @@ sub DoPanic()
$instance->Refresh();
if ($setclr eq "set") {
if ($instance->paniced()) {
print "Already in a panic!\n";
print "Already in a panic!\n";
if (defined($webtask)) {
$webtask->output("Already in a panic");
$webtask->Exited(0);
}
$slice->UnLock();
exit(0);
}
......@@ -3310,6 +3319,10 @@ sub DoPanic()
else {
if (!$instance->paniced()) {
print "No longer in a panic!\n";
if (defined($webtask)) {
$webtask->output("No longer in a panic");
$webtask->Exited(0);
}
$slice->UnLock();
exit(0);
}
......@@ -3331,6 +3344,35 @@ sub DoPanic()
@aggregates = $instance->AggregateList();
}
#
# Exit and let caller poll for status.
#
if (!$foreground) {
$logfile = TBMakeLogname("panic");
if (my $childpid = TBBackGround($logfile)) {
#
# Wait a couple of seconds to see if there is going to be an
# immediate error. Then return and let it continue to run. This
# allows the web server to see quick errors. Later errors will
# have to be emailed.
#
sleep(15);
my $foo = waitpid($childpid, &WNOHANG);
if ($foo) {
my $status = $? >> 8;
if ($status) {
print "There was a problem changing panic mode ...\n";
system("/bin/cat $logfile");
}
exit($status);
}
# Child still running.
print "Proceeding in the background ...\n";
exit(0);
}
}
#
# Helper callback to send the proper arguments.
#
......@@ -3341,7 +3383,7 @@ sub DoPanic()
};
# Invoke on all clusters
$errcode = CallMethodOnAggregates($coderef, 1, \$response, @aggregates);
$errcode = CallMethodOnAggregates($coderef, 0, \$response, @aggregates);
if ($errcode) {
$errmsg = $response;
$exitcode = -1;
......@@ -3358,14 +3400,7 @@ sub DoPanic()
if ($code != GENIRESPONSE_SUCCESS) {
$errcode = $code;
if ($code == GENIRESPONSE_REFUSED) {
$exitcode = 1;
$errmsg = $response->error();
}
else {
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
}
next;
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
}
}
goto bad
......@@ -3381,6 +3416,8 @@ sub DoPanic()
goto bad;
}
$slice->UnLock();
unlink($logfile)
if (defined($logfile) && !$debug);
exit(0);
bad:
......@@ -3390,6 +3427,8 @@ sub DoPanic()
$webtask->output($errmsg);
$webtask->Exited($errcode);
}
unlink($logfile)
if (defined($logfile) && !$debug);
exit($exitcode);
}
......@@ -4630,7 +4669,7 @@ sub DoWarn()
# allows the web server to see quick errors. Later errors will
# have to be emailed.
#
sleep(5);
sleep(10);
my $foo = waitpid($childpid, &WNOHANG);
if ($foo) {
$status = $? >> 8;
......@@ -4642,7 +4681,7 @@ sub DoWarn()
# We pass the lock through.
system("$MANAGEINSTANCE " .
(defined($webtask) ? "-t $webtask_id " : "").
" -d -- panic $uuid -L " . ($poweroff ? "-p " : "") . "set");
" -- panic $uuid -f -L " . ($poweroff ? "-p " : "") . "set");
}
else {
# We pass the lock through.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment