Commit bc4d1802 authored by Leigh B Stoller's avatar Leigh B Stoller

Run Panic in the background since it can take a long time, and we do not

want the web interface just sitting there. We wait a short time for the
child to catch quick errors, but rely on email notification if something
goes wrong in the backend.
parent 3e17cecd
...@@ -645,13 +645,15 @@ sub SetPanic($$) ...@@ -645,13 +645,15 @@ sub SetPanic($$)
my $uuid = $self->uuid(); my $uuid = $self->uuid();
my $value = ($clear ? 0 : 1); my $value = ($clear ? 0 : 1);
my $stamp = ($clear ? "NULL" : "now()");
DBQueryWarn("update apt_instances set paniced=${value} ". DBQueryWarn("update apt_instances set ".
(!$clear ? ",paniced_timestamp=now() " : "") . " paniced=${value},paniced_timestamp=${stamp} ".
"where uuid='$uuid'") or "where uuid='$uuid'") or
return -1; return -1;
$self->{'INSTANCE'}->{"paniced"} = $value; $self->{'INSTANCE'}->{"paniced"} = $value;
$self->{'INSTANCE'}->{"paniced_timestamp"} = ($clear ? undef : time());
return 0; return 0;
} }
...@@ -3170,6 +3172,9 @@ sub Panic($$$) ...@@ -3170,6 +3172,9 @@ sub Panic($$$)
return CredentialError() return CredentialError()
if (! defined($slice_credential)); if (! defined($slice_credential));
# Bump default timeout, this can take a while for a response
Genixmlrpc->SetTimeout(900);
my $args = { my $args = {
"slice_urn" => $slice->urn(), "slice_urn" => $slice->urn(),
"credentials" => [$slice_credential->asString()], "credentials" => [$slice_credential->asString()],
......
...@@ -3260,10 +3260,12 @@ sub DoPanic() ...@@ -3260,10 +3260,12 @@ sub DoPanic()
my $errcode = -1; my $errcode = -1;
my $exitcode = -1; my $exitcode = -1;
my $response; my $response;
my $logfile;
my $optlist = "Lp"; my $optlist = "Lpf";
my $takelock = 0; my $takelock = 0;
my $poweroff = 0; my $poweroff = 0;
my $foreground = 0;
my %options = (); my %options = ();
if (! getopts($optlist, \%options)) { if (! getopts($optlist, \%options)) {
...@@ -3275,6 +3277,9 @@ sub DoPanic() ...@@ -3275,6 +3277,9 @@ sub DoPanic()
if (defined($options{"p"})) { if (defined($options{"p"})) {
$poweroff = 1; $poweroff = 1;
} }
if (defined($options{"f"})) {
$foreground = 1;
}
usage() usage()
if (@ARGV != 1); if (@ARGV != 1);
...@@ -3302,7 +3307,11 @@ sub DoPanic() ...@@ -3302,7 +3307,11 @@ sub DoPanic()
$instance->Refresh(); $instance->Refresh();
if ($setclr eq "set") { if ($setclr eq "set") {
if ($instance->paniced()) { if ($instance->paniced()) {
print "Already in a panic!\n"; print "Already in a panic!\n";
if (defined($webtask)) {
$webtask->output("Already in a panic");
$webtask->Exited(0);
}
$slice->UnLock(); $slice->UnLock();
exit(0); exit(0);
} }
...@@ -3310,6 +3319,10 @@ sub DoPanic() ...@@ -3310,6 +3319,10 @@ sub DoPanic()
else { else {
if (!$instance->paniced()) { if (!$instance->paniced()) {
print "No longer in a panic!\n"; print "No longer in a panic!\n";
if (defined($webtask)) {
$webtask->output("No longer in a panic");
$webtask->Exited(0);
}
$slice->UnLock(); $slice->UnLock();
exit(0); exit(0);
} }
...@@ -3331,6 +3344,35 @@ sub DoPanic() ...@@ -3331,6 +3344,35 @@ sub DoPanic()
@aggregates = $instance->AggregateList(); @aggregates = $instance->AggregateList();
} }
#
# Exit and let caller poll for status.
#
if (!$foreground) {
$logfile = TBMakeLogname("panic");
if (my $childpid = TBBackGround($logfile)) {
#
# Wait a couple of seconds to see if there is going to be an
# immediate error. Then return and let it continue to run. This
# allows the web server to see quick errors. Later errors will
# have to be emailed.
#
sleep(15);
my $foo = waitpid($childpid, &WNOHANG);
if ($foo) {
my $status = $? >> 8;
if ($status) {
print "There was a problem changing panic mode ...\n";
system("/bin/cat $logfile");
}
exit($status);
}
# Child still running.
print "Proceeding in the background ...\n";
exit(0);
}
}
# #
# Helper callback to send the proper arguments. # Helper callback to send the proper arguments.
# #
...@@ -3341,7 +3383,7 @@ sub DoPanic() ...@@ -3341,7 +3383,7 @@ sub DoPanic()
}; };
# Invoke on all clusters # Invoke on all clusters
$errcode = CallMethodOnAggregates($coderef, 1, \$response, @aggregates); $errcode = CallMethodOnAggregates($coderef, 0, \$response, @aggregates);
if ($errcode) { if ($errcode) {
$errmsg = $response; $errmsg = $response;
$exitcode = -1; $exitcode = -1;
...@@ -3358,14 +3400,7 @@ sub DoPanic() ...@@ -3358,14 +3400,7 @@ sub DoPanic()
if ($code != GENIRESPONSE_SUCCESS) { if ($code != GENIRESPONSE_SUCCESS) {
$errcode = $code; $errcode = $code;
if ($code == GENIRESPONSE_REFUSED) { ($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
$exitcode = 1;
$errmsg = $response->error();
}
else {
($exitcode, $errmsg) = ResponseErrorMessage($agg, $response);
}
next;
} }
} }
goto bad goto bad
...@@ -3381,6 +3416,8 @@ sub DoPanic() ...@@ -3381,6 +3416,8 @@ sub DoPanic()
goto bad; goto bad;
} }
$slice->UnLock(); $slice->UnLock();
unlink($logfile)
if (defined($logfile) && !$debug);
exit(0); exit(0);
bad: bad:
...@@ -3390,6 +3427,8 @@ sub DoPanic() ...@@ -3390,6 +3427,8 @@ sub DoPanic()
$webtask->output($errmsg); $webtask->output($errmsg);
$webtask->Exited($errcode); $webtask->Exited($errcode);
} }
unlink($logfile)
if (defined($logfile) && !$debug);
exit($exitcode); exit($exitcode);
} }
...@@ -4630,7 +4669,7 @@ sub DoWarn() ...@@ -4630,7 +4669,7 @@ sub DoWarn()
# allows the web server to see quick errors. Later errors will # allows the web server to see quick errors. Later errors will
# have to be emailed. # have to be emailed.
# #
sleep(5); sleep(10);
my $foo = waitpid($childpid, &WNOHANG); my $foo = waitpid($childpid, &WNOHANG);
if ($foo) { if ($foo) {
$status = $? >> 8; $status = $? >> 8;
...@@ -4642,7 +4681,7 @@ sub DoWarn() ...@@ -4642,7 +4681,7 @@ sub DoWarn()
# We pass the lock through. # We pass the lock through.
system("$MANAGEINSTANCE " . system("$MANAGEINSTANCE " .
(defined($webtask) ? "-t $webtask_id " : ""). (defined($webtask) ? "-t $webtask_id " : "").
" -d -- panic $uuid -L " . ($poweroff ? "-p " : "") . "set"); " -- panic $uuid -f -L " . ($poweroff ? "-p " : "") . "set");
} }
else { else {
# We pass the lock through. # We pass the lock through.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment