Commit bde6c94d authored by Leigh Stoller's avatar Leigh Stoller

Recovery mode:

* Add a new Portal context menu option to nodes, to boot into "recovery"
  mode, which will be a Linux MFS (rather then the FreeBSD MFS, which
  99% of user will not know what to do with).

* Plumb all through to the Geni RPC interface, which invokes node_admin
  with a new option, to use the recovery mfs nodetype attribute.

* recoverymfs_osid is a distinct osid from adminmfs_osid, we use that in
  the CM to add an Emulab name space attribute to the manifest, that
  tells the Portal that a node supports recovery mode (and thus gets a
  context menu option).

* Add an inrecovery flag to the sliver status blob, which the Portal
  uses to determine that a node is currently in recovery mode, so that
  we can indicate that in the topology and list tabs.
parent 3b545f35
...@@ -3632,5 +3632,42 @@ sub MaxExtension($$) ...@@ -3632,5 +3632,42 @@ sub MaxExtension($$)
return $response; return $response;
} }
#
# Turn on/off recovery mode for a sliver.
#
sub Recovery($$$)
{
my ($self, $sliver_urn, $clear) = @_;
my $authority = $self->GetGeniAuthority();
my $geniuser = $self->instance()->GetGeniUser();
my $slice = $self->instance()->GetGeniSlice();
my $context = APT_Geni::GeniContext();
return ContextError()
if (! (defined($geniuser) && defined($authority) &&
defined($slice) && defined($context)));
my ($slice_credential, $speaksfor_credential) =
APT_Geni::GenCredentials($slice, $geniuser, undef, 1);
return CredentialError()
if (!defined($slice_credential));
my $credentials = [$slice_credential->asString()];
if (defined($speaksfor_credential)) {
$credentials = [@$credentials, $speaksfor_credential->asString()];
}
my $args = {
"slice_urn" => $slice->urn(),
"sliver_urn" => $sliver_urn,
"credentials" => $credentials,
};
if ($clear) {
$args->{'clear'} = 1;
}
my $cmurl = $authority->url();
$cmurl = devurl($cmurl) if ($usemydevtree);
return Genixmlrpc::CallMethod($cmurl, $context, "Recovery", $args);
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2018 University of Utah and the Flux Group. # Copyright (c) 2000-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -44,6 +44,7 @@ sub usage() ...@@ -44,6 +44,7 @@ sub usage()
print("Usage: manage_instance refresh instance\n"); print("Usage: manage_instance refresh instance\n");
print("Usage: manage_instance reboot instance node_id ...\n"); print("Usage: manage_instance reboot instance node_id ...\n");
print("Usage: manage_instance reload instance node_id ...\n"); print("Usage: manage_instance reload instance node_id ...\n");
print("Usage: manage_instance recovery instance [-c] node_id\n");
print("Usage: manage_instance deletenodes instance node_id ...\n"); print("Usage: manage_instance deletenodes instance node_id ...\n");
print("Usage: manage_instance monitor instance\n"); print("Usage: manage_instance monitor instance\n");
print("Usage: manage_instance lockdown instance set|clear user|admin\n"); print("Usage: manage_instance lockdown instance set|clear user|admin\n");
...@@ -132,6 +133,7 @@ sub DoDenyOrMoreInfo($); ...@@ -132,6 +133,7 @@ sub DoDenyOrMoreInfo($);
sub DoRefresh(); sub DoRefresh();
sub DoReboot(); sub DoReboot();
sub DoReload(); sub DoReload();
sub DoRecovery();
sub DoLockdown(); sub DoLockdown();
sub DoPanic(); sub DoPanic();
sub DoManifests(); sub DoManifests();
...@@ -245,6 +247,9 @@ elsif ($action eq "reboot") { ...@@ -245,6 +247,9 @@ elsif ($action eq "reboot") {
elsif ($action eq "reload") { elsif ($action eq "reload") {
DoReload() DoReload()
} }
elsif ($action eq "recovery") {
DoRecovery()
}
elsif ($action eq "monitor") { elsif ($action eq "monitor") {
StartMonitor() StartMonitor()
} }
...@@ -2537,6 +2542,90 @@ sub DoRebootOrReload($) ...@@ -2537,6 +2542,90 @@ sub DoRebootOrReload($)
sub DoReboot() { return DoRebootOrReload("reboot"); } sub DoReboot() { return DoRebootOrReload("reboot"); }
sub DoReload() { return DoRebootOrReload("reload"); } sub DoReload() { return DoRebootOrReload("reload"); }
#
# Recovery mode.
#
sub DoRecovery()
{
my ($errmsg, $exitcode, $errcode);
my $clear = 0;
my $optlist = "c";
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"c"})) {
$clear = 1;
}
usage()
if (!@ARGV);
my $node_id = shift(@ARGV);
#
# Sanity check to make sure the node is really in the rspec, since
# we need its sliver urn.
#
my $sliver_urn;
my $sliver;
foreach my $obj ($instance->AggregateList()) {
my $manifest = GeniXML::Parse($obj->manifest());
if (! defined($manifest)) {
fatal("Could not parse manifest for $obj");
}
my @nodes = (GeniXML::FindNodes("n:node", $manifest)->get_nodelist(),
GeniXML::FindNodesNS("n:vhost", $manifest,
$GeniXML::EMULAB_NS)->get_nodelist());
foreach my $node (@nodes) {
my $client_id = GeniXML::GetVirtualId($node);
my $urn = GeniXML::GetSliverId($node);
my $manager_urn = GetManagerId($node);
# No sliver urn or a different aggregate.
next
if (! (defined($urn) &&
defined($manager_urn) &&
$manager_urn eq $obj->aggregate_urn()));
if ($node_id eq $client_id) {
$sliver_urn = $urn;
$sliver = $obj;
}
}
}
if (!defined($sliver_urn)) {
fatal("Could not find node '$node_id' in manifest");
}
if ($sliver->GetAptAggregate()->CheckStatus(\$errmsg)) {
print STDERR "$errmsg\n";
if (defined($webtask)) {
$webtask->output($errmsg);
$webtask->Exited(GENIRESPONSE_SERVER_UNAVAILABLE);
}
exit(1);
}
my $response = $sliver->Recovery($sliver_urn, $clear);
if ($response->code() != GENIRESPONSE_SUCCESS) {
$errcode = $response->code();
($exitcode,$errmsg) = ResponseErrorMessage($sliver, $response);
# Important to tell web user about this
if ($response->code() == GENIRESPONSE_FORBIDDEN) {
$exitcode = 1;
}
goto bad;
}
exit(0);
bad:
print STDERR "$errmsg\n";
if (defined($errmsg) && defined($webtask)) {
$webtask->Exited($errcode);
$webtask->output($errmsg);
}
exit($exitcode);
}
# #
# #
# #
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2005-2018 University of Utah and the Flux Group. # Copyright (c) 2005-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -1768,6 +1768,22 @@ sub adminmfs_osid($;$) { ...@@ -1768,6 +1768,22 @@ sub adminmfs_osid($;$) {
return OSImage->Lookup(TBOPSPID(), TB_OSID_FREEBSD_MFS())->osid(); return OSImage->Lookup(TBOPSPID(), TB_OSID_FREEBSD_MFS())->osid();
} }
sub recoverymfs_osid($;$) {
my ($self,$stuff) = @_;
my $val = undef;
require OSImage;
if (NodeAttribute($self, "recoverymfs_osid", \$val) == 0 &&
defined($val)) {
return $val;
}
if (NodeTypeAttribute($self, "recoverymfs_osid", \$val) == 0 &&
defined($val)) {
return $val;
}
return undef;
}
sub diskloadmfs_osid($;$) { sub diskloadmfs_osid($;$) {
my ($self,$stuff) = @_; my ($self,$stuff) = @_;
my $val = undef; my $val = undef;
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2005-2018 University of Utah and the Flux Group. # Copyright (c) 2005-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -417,6 +417,7 @@ sub frequency($;$) {return GetAttribute($_[0], "frequency", $_[1]); } ...@@ -417,6 +417,7 @@ sub frequency($;$) {return GetAttribute($_[0], "frequency", $_[1]); }
sub bios_waittime($;$) {return GetAttribute($_[0], "bios_waittime", $_[1]); } sub bios_waittime($;$) {return GetAttribute($_[0], "bios_waittime", $_[1]); }
sub control_iface($;$) {return GetAttribute($_[0], "control_interface",$_[1]);} sub control_iface($;$) {return GetAttribute($_[0], "control_interface",$_[1]);}
sub adminmfs_osid($;$) {return GetAttribute($_[0], "adminmfs_osid",$_[1]);} sub adminmfs_osid($;$) {return GetAttribute($_[0], "adminmfs_osid",$_[1]);}
sub reoverymfs_osid($;$) {return GetAttribute($_[0], "recoverymfs_osid",$_[1]);}
sub rebootable($;$) {return GetAttribute($_[0], "rebootable",$_[1]);} sub rebootable($;$) {return GetAttribute($_[0], "rebootable",$_[1]);}
sub power_delay($;$) {return GetAttribute($_[0], "power_delay",$_[1]);} sub power_delay($;$) {return GetAttribute($_[0], "power_delay",$_[1]);}
sub shared($;$) {return GetAttribute($_[0], "shared",$_[1]);} sub shared($;$) {return GetAttribute($_[0], "shared",$_[1]);}
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2000-2018 University of Utah and the Flux Group. # Copyright (c) 2000-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -69,7 +69,7 @@ use vars qw(@ISA @EXPORT); ...@@ -69,7 +69,7 @@ use vars qw(@ISA @EXPORT);
TBSetExptFirewallVlan TBClearExptFirewallVlan TBSetExptFirewallVlan TBClearExptFirewallVlan
TBNodeConsoleTail TBExptGetSwapoutAction TBExptGetSwapState TBNodeConsoleTail TBExptGetSwapoutAction TBExptGetSwapState
TBNodeSubNodes TBNodeSubNodes
TBNodeAdminOSID TBNodeNFSAdmin TBNodeDiskloadOSID TBNodeAdminOSID TBNodeRecoveryOSID TBNodeNFSAdmin TBNodeDiskloadOSID
TBNodeType TBNodeTypeProcInfo TBNodeTypeBiosWaittime TBNodeType TBNodeTypeProcInfo TBNodeTypeBiosWaittime
TBExptPortRange TBExptPortRange
TBWideareaNodeID TBTipServers TBWideareaNodeID TBTipServers
...@@ -1527,6 +1527,17 @@ sub TBNodeAdminOSID($) ...@@ -1527,6 +1527,17 @@ sub TBNodeAdminOSID($)
return 0; return 0;
} }
sub TBNodeRecoveryOSID($)
{
my ($nodeid) = @_;
my $node = LocalNodeLookup($nodeid);
if ($node) {
return $node->recoverymfs_osid();
}
return 0;
}
# #
# Returns 1 if node uses NFS-based admin MFS, 0 ow. # Returns 1 if node uses NFS-based admin MFS, 0 ow.
# #
......
...@@ -115,6 +115,7 @@ my $WAP = "$TB/sbin/withadminprivs"; ...@@ -115,6 +115,7 @@ my $WAP = "$TB/sbin/withadminprivs";
my $SHAREVLAN = "$TB/sbin/sharevlan"; my $SHAREVLAN = "$TB/sbin/sharevlan";
my $PANIC = "$TB/sbin/panic"; my $PANIC = "$TB/sbin/panic";
my $LINKTEST = "$TB/sbin/linktest_control"; my $LINKTEST = "$TB/sbin/linktest_control";
my $NODEADMIN = "$TB/bin/node_admin";
my $XMLLINT = "/usr/local/bin/xmllint"; my $XMLLINT = "/usr/local/bin/xmllint";
my $IMAGEINFO = "$TB/sbin/imageinfo"; my $IMAGEINFO = "$TB/sbin/imageinfo";
my $PRERENDER = "$TB/libexec/vis/prerender"; my $PRERENDER = "$TB/libexec/vis/prerender";
...@@ -5964,5 +5965,87 @@ sub RunLinktest($) ...@@ -5964,5 +5965,87 @@ sub RunLinktest($)
"results" => $output}, $output); "results" => $output}, $output);
} }
#
# Another Emulab specific function to put a node into "recovery" mode.
# In other words, a linux MFS so the user can fix what they broke.
#
sub Recovery($)
{
my ($argref) = @_;
my $slice_urn = $argref->{'slice_urn'};
my $credentials = $argref->{'credentials'};
my $sliver_urn = $argref->{'sliver_urn'};
my $clear = $argref->{'clear'};
if (! (defined($credentials) &&
defined($slice_urn) && defined($sliver_urn))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
}
my ($credential,$speaksfor) = GeniStd::CheckCredentials($credentials);
return $credential
if (GeniResponse::IsResponse($credential));
my ($slice, $aggregate) = Credential2SliceAggregate($credential);
return $slice
if (defined($slice) && GeniResponse::IsResponse($slice));
if (! (defined($slice) && defined($aggregate))) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"Sliver does not exist");
}
my $user = GeniCM::CreateUserFromCertificate($credential);
return $user
if (GeniResponse::IsResponse($user));
main::AddLogfileMetaDataFromSlice($slice);
if ($slice_urn ne $slice->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
my $sliver = GeniSliver->Lookup($sliver_urn);
if (!defined($sliver)) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"Sliver does not exist");
}
if ($sliver->slice_uuid() ne $slice->uuid()) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"Sliver is not in slice");
}
my $node_id = $sliver->resource_id();
my $node = Node->Lookup($node_id);
if (!defined($node)) {
return GeniResponse->Create(GENIRESPONSE_SEARCHFAILED, undef,
"No node for sliver urn");
}
if ($node->IsTainted()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"node is tainted - recovery mode denied");
}
if (!defined(GeniCM::FlipToUser($slice, $user))) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"FlipToUser failed");
}
my $experiment = $slice->GetExperiment();
if (!defined($experiment)) {
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"No local experiment for slice");
}
my $output = "";
if ($clear) {
$output = GeniUtil::ExecQuiet("$NODEADMIN -R off $node_id");
}
else {
$output = GeniUtil::ExecQuiet("$NODEADMIN -R on $node_id");
}
if ($?) {
print STDERR $output . "\n";
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Failed to change recovery mode");
}
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2008-2017 University of Utah and the Flux Group. # Copyright (c) 2008-2019 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -944,20 +944,41 @@ sub AnnotateManifest($) ...@@ -944,20 +944,41 @@ sub AnnotateManifest($)
# one, expecting it to be available by the time the user might # one, expecting it to be available by the time the user might
# want to use it. # want to use it.
# #
if (!$node->IsTainted() && if (!$node->IsTainted()) {
(($node->TipServer(\$tipserver) == 0 && defined($tipserver)) || if ($node->TipServer(\$tipserver) == 0 && defined($tipserver) ||
$node->isvirtnode())) { $node->isvirtnode()) {
if (! defined($services)) { if (! defined($services)) {
$services = GeniXML::AddElement("services", $rspec); $services = GeniXML::AddElement("services", $rspec);
}
my $console = GeniXML::FindNodesNS("n:console", $services,
$GeniXML::EMULAB_NS)->pop();
if (defined($console)) {
$services->removeChild($console);
}
$console = GeniXML::AddElement("console",
$services, $GeniXML::EMULAB_NS);
GeniXML::SetText("server", $console,
(defined($tipserver) ? $tipserver : $sshdhost));
}
if ($node->recoverymfs_osid() && !$node->isvirtnode()) {
if (! defined($services)) {
$services = GeniXML::AddElement("services", $rspec);
}
my $recovery = GeniXML::FindNodesNS("n:recovery", $services,
$GeniXML::EMULAB_NS)->pop();
if (!defined($recovery)) {
$recovery = GeniXML::AddElement("recovery",
$services, $GeniXML::EMULAB_NS);
}
GeniXML::SetText("available", $recovery, "true");
} }
my $console = GeniXML::FindNodesNS("n:console", $services, elsif (defined($services)) {
$GeniXML::EMULAB_NS)->pop(); my $recovery = GeniXML::FindNodesNS("n:recovery", $services,
if (defined($console)) { $GeniXML::EMULAB_NS)->pop();
$services->removeChild($console); if (defined($recovery)) {
$services->removeChild($recovery);
}
} }
$console = GeniXML::AddElement("console",$services,$GeniXML::EMULAB_NS);
GeniXML::SetText("server", $console,
(defined($tipserver) ? $tipserver : $sshdhost));
} }
my $adb_port; my $adb_port;
...@@ -1694,6 +1715,12 @@ sub GenerateStatusBlob($) ...@@ -1694,6 +1715,12 @@ sub GenerateStatusBlob($)
$state = "stopped" $state = "stopped"
if ($state eq "new"); if ($state eq "new");
# See if booted into the admin or recovery MFS (not frisbee).
my $recovery = 0;
if ($node->temp_boot_osid()) {
$recovery = 1;
}
# #
# So we have a bunch of timestamps for various bits of state. # So we have a bunch of timestamps for various bits of state.
# Not much atomicity, but I think we are okay. Use the most # Not much atomicity, but I think we are okay. Use the most
...@@ -1714,6 +1741,7 @@ sub GenerateStatusBlob($) ...@@ -1714,6 +1741,7 @@ sub GenerateStatusBlob($)
"rawstate" => $rawstate, "rawstate" => $rawstate,
"nodestatus" => $nodestatus, "nodestatus" => $nodestatus,
"utc" => $utc, "utc" => $utc,
"recovery" => $recovery,
}; };
# #
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2008-2018 University of Utah and the Flux Group. # Copyright (c) 2008-2019 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -197,6 +197,7 @@ $V2_METHODS = { ...@@ -197,6 +197,7 @@ $V2_METHODS = {
"DeleteNodes" => \&GeniCMV2::DeleteNodes, "DeleteNodes" => \&GeniCMV2::DeleteNodes,
"Panic" => \&GeniCMV2::Panic, "Panic" => \&GeniCMV2::Panic,
"RunLinktest" => \&GeniCMV2::RunLinktest, "RunLinktest" => \&GeniCMV2::RunLinktest,
"Recovery" => \&GeniCMV2::Recovery,
}; };
ProtoGeniDefs::AddModule("cm", ProtoGeniDefs::AddModule("cm",
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2008-2017 University of Utah and the Flux Group. # Copyright (c) 2008-2019 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -123,6 +123,7 @@ elsif ($GENI_VERSION eq "2.0") { ...@@ -123,6 +123,7 @@ elsif ($GENI_VERSION eq "2.0") {
"DeleteNodes" => \&GeniCMV2::DeleteNodes, "DeleteNodes" => \&GeniCMV2::DeleteNodes,
"Panic" => \&GeniCMV2::Panic, "Panic" => \&GeniCMV2::Panic,
"RunLinktest" => \&GeniCMV2::RunLinktest, "RunLinktest" => \&GeniCMV2::RunLinktest,
"Recovery" => \&GeniCMV2::Recovery,
}; };
} }
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2005-2017 University of Utah and the Flux Group. # Copyright (c) 2005-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -288,6 +288,8 @@ sub TBAdminMfsBoot($$@) ...@@ -288,6 +288,8 @@ sub TBAdminMfsBoot($$@)
# 'on' 1 to set temp OSID to MFS, 0 to clear # 'on' 1 to set temp OSID to MFS, 0 to clear
# 'clearall' 1 to clear one-shot and partition boot OSIDs # 'clearall' 1 to clear one-shot and partition boot OSIDs
# (if 'on' is set), 0 leaves them alone # (if 'on' is set), 0 leaves them alone
# 'recovery' use the recovery MFS instead of Admin (falling back
# to admin if not defined for the node).
# #
# Returns zero if we successfully set the state for all nodes, # Returns zero if we successfully set the state for all nodes,
# and non-zero otherwise. If the $failed ref is defined, it is an # and non-zero otherwise. If the $failed ref is defined, it is an
...@@ -309,6 +311,7 @@ sub TBAdminMfsSelect($$@) ...@@ -309,6 +311,7 @@ sub TBAdminMfsSelect($$@)
my $me = $args->{'name'}; my $me = $args->{'name'};
my $on = $args->{'on'}; my $on = $args->{'on'};
my $only = $args->{'clearall'}; my $only = $args->{'clearall'};
my $recovery = $args->{'recovery'};
my @good = (); my @good = ();
my @bad = (); my @bad = ();
...@@ -328,7 +331,16 @@ sub TBAdminMfsSelect($$@) ...@@ -328,7 +331,16 @@ sub TBAdminMfsSelect($$@)
# determine the correct admin OSID for all nodes # determine the correct admin OSID for all nodes
my %adminosid = (); my %adminosid = ();
for my $node (@nodes) { for my $node (@nodes) {
my $osid = TBNodeAdminOSID($node); my $osid;
if ($recovery) {
$osid = TBNodeRecoveryOSID($node);
if (!$osid) {
$osid = TBNodeAdminOSID($node);
}
}
else {
$osid = TBNodeAdminOSID($node);
}
push @{$adminosid{$osid}}, $node; push @{$adminosid{$osid}}, $node;
} }
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2018 University of Utah and the Flux Group. # Copyright (c) 2000-2019 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -35,13 +35,15 @@ sub usage() ...@@ -35,13 +35,15 @@ sub usage()
print STDOUT "-h This message\n"; print STDOUT "-h This message\n";
print STDOUT "-n Do not reboot node\n"; print STDOUT "-n Do not reboot node\n";
print STDOUT "-w Wait for node to come back up if rebooted\n"; print STDOUT "-w Wait for node to come back up if rebooted\n";
print STDOUT "-R Boot node into 'recovery' MFS instead\n";
print STDOUT "-e Operate on all nodes in an experiment\n"; print STDOUT "-e Operate on all nodes in an experiment\n";
print STDOUT "-c cmd Run command in MFS and wait for completion\n". print STDOUT "-c cmd Run command in MFS and wait for completion\n".
" (-n and -w apply after the command is run).\n"; " (-n and -w apply after the command is run).\n";
exit(-1); exit(-1);
} }
my $optlist = "hnwe:c:"; my $optlist = "hnwe:c:R";
my $waitmode = 0; my $waitmode = 0;
my $recovery = 0;
my $reboot = 1; my $reboot = 1;
my $runcmd = ""; my $runcmd = "";
my $onoff; my $onoff;
...@@ -97,6 +99,9 @@ if (defined($options{"n"})) { ...@@ -97,6 +99,9 @@ if (defined($options{"n"})) {
if (defined($options{"w"})) { if (defined($options{"w"})) {
$waitmode = 1; $waitmode = 1;
} }
if (defined($options{"R"})) {
$recovery = 1;
}
if (defined($options{"c"})) { if (defined($options{"c"})) {
$runcmd = $options{"c"}; $runcmd = $options{"c"};
$onoff = "on"; $onoff = "on";
...@@ -237,6 +242,7 @@ if ($runcmd ne "") { ...@@ -237,6 +242,7 @@ if ($runcmd ne "") {
$args{'name'} = $0; $args{'name'} = $0;
$args{'on'} = ($onoff eq "on"); $args{'on'} = ($onoff eq "on");
$args{'clearall'} = 0; $args{'clearall'} = 0;
$args{'recovery'} = 1 if ($recovery && $onoff eq "on");
if (TBAdminMfsSelect(\%args, \@bad, @nodeids)) { if (TBAdminMfsSelect(\%args, \@bad, @nodeids)) {
die("*** $0:\n". die("*** $0:\n".
" Could not turn admin mode $onoff for @bad!\n"); " Could not turn admin mode $onoff for @bad!\n");
......
...@@ -29,6 +29,7 @@ $(function () ...@@ -29,6 +29,7 @@ $(function ()
var jacksIDs = {}; var jacksIDs = {};
var jacksSites = {}; var jacksSites = {};
var publicURLs = null; var publicURLs = null;
var inrecovery = {};
var extension_blob = null; var extension_blob = null;
var manifests = {}; var manifests = {};
var status_collapsed = false; var status_collapsed = false;
...@@ -946,18 +947,33 @@ $(function () ...@@ -946,18 +947,33 @@ $(function ()
return; return;
} }
$.each(iblob.details, function(node_id, details) { $.each(iblob.details, function(node_id, details) {
var jacksID = jacksIDs[node_id]; var jacksID = jacksIDs[node_id];
// No manifest yet for this node. // No manifest yet for this node.
if (jacksID === undefined) { if (jacksID === undefined) {
return; return;
} }
$('#listview-row-' + node_id + ' td[name="status"]') // Is the node in recovery.
.html(details.status); var recovery = false;
if (_.has(details, "recovery") && details.recovery != 0) {
recovery = true;
inrecovery[node_id] = true;
}
else {
inrecovery[node_id] = false;
}
$('#listview-row-' + node_id + ' td[name="status"]')
.html(recovery ? "<b>recovery</b>" : details.status);
if (details.status == "ready") { if (details.status == "ready") {
// Greenish. // Greenish.
var color = "#91E388";
if (recovery) {
// warning
color = "#fcf8e3";
}
$('#' + jacksID + ' .node .nodebox') $('#' + jacksID + ' .node .nodebox')
.css("fill", "#91E388"); .css("fill", color);
$('#listview-row-' + node_id + ' td[name="node_id"], ' + $('#listview-row-' + node_id + ' td[name="node_id"], ' +
'#listview-row-' + node_id + ' td[name="client_id"]')