Skip to content

GitLab

Commit 30f7f8b0 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

Redo shutdown to match what it says in the paper; Contact the CMs in 

parallel uding ParRun() so that its a lot faster. Also deal with
transient errors and retry. Note though, that I do not deal with
timeout at the RPC level cause I do not know how to get that error
back yet. Also cleanup credential usage, although not active yet until
the new CM makes it out; we must use a clearinghouse self signed
credential not the bogus slice credential that the clearinghouse was using.
parent d6689392
Hide whitespace changes
Inline Side-by-side
protogeni/scripts/shutdownslice.in
View file @ 30f7f8b0
#!/usr/bin/perl -w
#
# GENIPUBLIC-COPYRIGHT
# Copyright (c) 2008-2009 University of Utah and the Flux Group.
# Copyright (c) 2008-2010 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
......@@ -9,19 +9,23 @@ use English;
use Getopt::Std;
#
# Shutdown a slice. This has two modes of operation; 1) on the SA (Emulab)
# it will mark the slice as shutdown and make sure the Clearinghouse is
# notified. 2) On the clearinghouse, it will mark the slice as shutdown,
# and make sure that every CM is notified. This is somewhat crude.
#
# Shutdown a slice. With the -c option, does what the Clearinghouse
# needs to do; tell all CMs in the federation. Without the -c option,
# operate as the SA, telling the CH to shutdown the slice.
#
sub usage()
{
print "Usage: shutdownslice [-c] <uuid>\n";
print "Usage: shutdownslice [-d] [-c] [-u] <uuid | urn>\n";
exit(1);
}
my $optlist = "c";
my $optlist = "cdun";
my $debug = 0;
my $asch = 0;
my $clear = 0;
my $impotent = 0;
sub fatal($);
sub ShutDownAsCH();
sub ShutDownAsSA();
#
# Configure variables
......@@ -49,8 +53,18 @@ if (! getopts($optlist, \%options)) {
if (defined($options{"c"})) {
$asch = 1;
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"u"})) {
$clear = 1;
}
if (defined($options{"n"})) {
$impotent = 1;
}
usage()
if (@ARGV != 1);
my $slice_urn = $ARGV[0];
use vars qw($GENI_DBNAME);
if ($asch) {
......@@ -60,6 +74,7 @@ if ($asch) {
# Now we can load the libraries after setting the proper DB.
use lib '@prefix@/lib';
use libtestbed;
use emutil;
require GeniDB;
require Genixmlrpc;
use GeniResponse;
......@@ -68,141 +83,239 @@ require GeniCredential;
require GeniAuthority;
require GeniSlice;
my $uuid = $ARGV[0];
exit(($asch ? ShutDownAsCH() : ShutDownAsSA()));
#
# Taint check.
# Shut down a local SA slice.
#
if ($uuid =~ /^([\w\-]+)$/) {
$uuid = $1;
}
else {
die("*** Bad data in $uuid\n");
}
my $slice = GeniSlice->Lookup($uuid);
if (!defined($slice)) {
fatal("No such slice for $uuid");
}
sub ShutDownAsSA()
{
my $slice = GeniSlice->Lookup($slice_urn);
if (!defined($slice)) {
fatal("No such slice found");
}
#
#
#
if ($asch) {
#
# The RPC context for this script is as the CH
# Mark the slice as shutdown but do not overwrite original time.
#
my $EMULAB_PEMFILE = "@prefix@/etc/genich.pem";
if (!defined($slice->shutdown()) || $slice->shutdown() eq "") {
$slice->SetShutdown() == 0
or fatal("Could not set the shutdown time for $slice");
}
#
# The RPC context for this script is as the SA
#
my $EMULAB_PEMFILE = "@prefix@/etc/genisa.pem";
my $certificate = GeniCertificate->LoadFromFile($EMULAB_PEMFILE);
if (!defined($certificate)) {
fatal("Could not load certificate from $EMULAB_PEMFILE");
}
Genixmlrpc->SetContext(Genixmlrpc->Context($certificate));
#
# Generate a slice credential. Used to be a Clearinghouse credential,
# but that is wrong.
#
my $authority = GeniAuthority->Lookup($certificate->uuid());
if (!defined($authority)) {
fatal("Could not find local authority object");
}
my $credential = GeniCredential->Create($slice, $authority);
if (!defined($credential)) {
fatal("Could not create new credential for $slice");
}
if ($credential->Sign($certificate) != 0) {
fatal("Could not sign credential for $slice");
}
#
# Need the CH certificate to make the RPC.
#
my $clearinghouse = GeniRegistry::ClearingHouse->GetCertificate();
if (!defined($clearinghouse)) {
fatal("Could not load the clearinghouse certificate");
}
my $response =
Genixmlrpc::CallMethod($clearinghouse->URL(), undef,
"Shutdown",
{ "clear" => $clear,
"urn" => $slice->urn(),
"credential" => $credential->asString() });
if (!defined($response) || $response->code() != GENIRESPONSE_SUCCESS) {
fatal("Could not shutdown $slice at clearinghouse: " .
(defined($response) ? $response->output() : ""));
}
return 0;
}
sub ShutDownAsCH()
{
my $slice = GeniSlice->Lookup($slice_urn);
my $credential;
my @managers = ();
#
# The credential we use needs to be a slice credential, since that
# makes it easier for anyone to shutdown a slice at a CM.
# The RPC context for this script is as the CH
#
my $certificate = GeniRegistry::ClearingHouse->GetCertificate();
if (!defined($certificate)) {
fatal("Could not load CH certificate");
}
Genixmlrpc->SetContext(Genixmlrpc->Context($certificate));
my $authority = GeniAuthority->Lookup($certificate->uuid());
if (!defined($authority)) {
fatal("Could not find local authority object");
}
#
# It is technically wrong to generate a slice credential here, since
# the target of the credential (slice) will be in a different domain
# then the CH. But we do not actually check that yet. I have changed the
# CMV2 to accept a self credential, but until everyone is running this
# verision, we have to continue use a locally generated slice credential.
#
# Complication: The new SA might not have registered the slice yet,
# so we do not have a slice object in the DB. In this case, go ahead
# and generate the self credential.
#
my $slice_credential = GeniCredential->Create($slice, $authority);
if (!defined($slice_credential)) {
if (defined($slice)) {
$credential = GeniCredential->Create($slice, $authority);
}
else {
$credential = GeniCredential->Create($authority, $authority);
}
if (!defined($credential)) {
fatal("Could not create new credential for $slice");
}
if ($slice_credential->Sign($certificate) != 0) {
if ($credential->Sign($certificate) != 0) {
fatal("Could not sign credential for $slice");
}
my $query_result =
GeniDB::DBQueryWarn("select uuid from geni_authorities ".
"where type='cm'");
"where type='cm' and disabled=0");
fatal("Could not lookup CM list")
if (!defined($query_result));
while (my ($manager_uuid) = $query_result->fetchrow_array()) {
my $manager = GeniAuthority->Lookup($manager_uuid);
print STDERR "Could not lookup up CM $manager_uuid\n"
if (!defined($manager));
push(@managers, $manager);
}
#
# Go into the background now, so as not to block the caller for
# a long time. Mail will be sent if there is an error.
# XXX Cannot use libaudit cause there is already an audit running
# via the xmlrpc invocation.
#
my $logname = TBMakeLogname("shutdownslice");
if (TBBackGround($logname)) {
#
# Parent exits normally.
#
exit(0);
my $logname;
if (!$debug) {
$logname = TBMakeLogname("shutdownslice");
if (TBBackGround($logname)) {
#
# Parent exits normally.
#
exit(0);
}
}
my $errors = 0;
while (my ($manager_uuid) = $query_result->fetchrow_array()) {
my $manager = GeniAuthority->Lookup($manager_uuid);
print STDERR "Could not lookup up CM $manager_uuid\n"
if (!defined($manager));
#
# We want to do this operation in parallel to make it as fast
# as possible and because the set of CMs can be large.
#
my @results = ();
my $coderef = sub {
my ($manager) = @_;
while (1) {
print STDERR "Shutting down slice on $manager\n";
if ($impotent) {
sleep(5);
return 0;
}
my $response =
Genixmlrpc::CallMethod($manager->url(), undef,
"Shutdown",
{ "clear" => 0,
"slice_urn" => $slice_urn,
"credential" => $credential->asString(),
"credentials" => [$credential->asString()],
});
if (!defined($response) ||
($response->code() != GENIRESPONSE_SUCCESS &&
$response->code() != GENIRESPONSE_SEARCHFAILED &&
$response->code() != GENIRESPONSE_BUSY)) {
print STDERR
"Could not shutdown slice on $manager: " .
(defined($response) ? $response->output() : "");
return -1;
}
# Search failed is okay.
# Keep trying on busy.
last
if ($response->code() == GENIRESPONSE_SUCCESS ||
$response->code() == GENIRESPONSE_SEARCHFAILED);
sleep(5);
}
return 0;
};
if (ParRun(undef, \@results, $coderef, @managers)) {
fatal("Internal error shutting down $slice");
}
#
# Check the exit codes. Eventually return specific error info.
#
my $errors = 0;
my $count = 0;
foreach my $result (@results) {
my $manager = $managers[$count];
print STDERR "Calling out to $manager at " .
TBTimeStamp() . " to do shutdown\n";
my $response =
Genixmlrpc::CallMethod($manager->url(), undef,
"Shutdown",
{ "clear" => 0,
"credential" =>
$slice_credential->asString() });
if (!defined($response) || $response->code() != GENIRESPONSE_SUCCESS) {
print STDERR
"Could not shutdown $slice on $manager: " .
$response->output();
if ($result != 0) {
print STDERR "*** Error shutting down slice on $manager\n";
$errors++;
}
$count++;
}
if ($errors) {
SENDMAIL($TBOPS,
"Error doing emergency shutdown",
"Error doing emergency shutdown of slice $uuid",
"Error doing emergency shutdown of $slice_urn",
undef, undef, $logname);
}
else {
SENDMAIL($TBOPS,
"ClearingHouse did an emergency shutdown",
"ClearingHouse did an emergency shutdown of slice $uuid",
"ClearingHouse did an emergency shutdown of $slice_urn",
undef, undef, $logname);
}
exit($errors);
}
else {
#
# Mark the slice as shutdown but do not overwrite original time.
#
if (!defined($slice->shutdown()) || $slice->shutdown() eq "") {
$slice->SetShutdown() == 0
or fatal("Could not set the shutdown time for $slice");
}
#
# The RPC context for this script is as the SA
#
my $EMULAB_PEMFILE = "@prefix@/etc/genisa.pem";
my $certificate = GeniCertificate->LoadFromFile($EMULAB_PEMFILE);
if (!defined($certificate)) {
fatal("Could not load certificate from $EMULAB_PEMFILE");
}
Genixmlrpc->SetContext(Genixmlrpc->Context($certificate));
my $clearinghouse = GeniRegistry::ClearingHouse->Create();
if (!defined($clearinghouse)) {
fatal("Could not create client for clearinghouse");
}
if ($clearinghouse->Shutdown($uuid, 1) != 0) {
fatal("Could not tell the clearinghouse to shutdown $slice");
}
}
exit(0);
sub fatal($) {
my ($msg) = @_;
print STDERR "$msg\n";
SENDMAIL($TBOPS,
"Error shutting down slice",
"Could not shutdown $slice_urn:\n\n" . "$msg\n",
$TBOPS);
exit(1);
}
tmcd/freebsd/rc.bridges 0 → 100755
View file @ 30f7f8b0
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
use strict;
use English;
use Getopt::Std;
#
# Set up the bridges.
#
sub usage()
{
print "Usage: " .
scriptname() . " [-j vnodeid] boot|shutdown|reconfig|reset\n";
exit(1);
}
my $optlist = "j:";
my $action = "boot";
my $vnodeid;
# Turn off line buffering on output
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
# Only root.
if ($EUID != 0) {
die("*** $0:\n".
" Must be root to run this script!\n");
}
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
use libtmcc;
use librc;
#
# Not all clients support this.
#
exit(0)
if (MFS() || REMOTE() || JAILED() || INXENVM() || PLAB());
# Protos.
sub doboot();
sub doshutdown();
sub doreconfig();
sub docleanup();
# Parse command line.
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{'j'})) {
$vnodeid = $options{'j'};
libsetup_setvnodeid($vnodeid);
}
# Allow default above.
if (@ARGV) {
$action = $ARGV[0];
}
my $FBSD_VERSION = `uname -v`;
if ($FBSD_VERSION =~ /FreeBSD ([0-9]).*/) {
$FBSD_VERSION = int($1);
}
else {
fatal(" Cannot determine OS version\n");
}
# Execute the action.
SWITCH: for ($action) {
/^boot$/i && do {
doboot();
last SWITCH;
};
/^shutdown$/i && do {
doshutdown();
last SWITCH;
};
/^reconfig$/i && do {
doreconfig();
last SWITCH;
};
/^reset$/i && do {
docleanup();
last SWITCH;
};
fatal("Invalid action: $action\n");
}
exit(0);
#
# Boot Action.
#
sub doboot()
{
my $upcmds = "";
my $downcmds = "";
my %bridges = ();
my @tmccresults;
print STDOUT "Checking Testbed bridge configuration ... \n";
if (tmcc(TMCCCMD_BRIDGES, undef, \@tmccresults) < 0) {
fatal("Could not get bridge info from server!");
}
return 0
if (! @tmccresults);
foreach my $str (@tmccresults) {
if ($str =~
/^BRIDGE IDX=(\d+) IFACE=(\w+) VNODE=([-\w]+) LINKNAME=([-\w]+)/){
my $idx = $1;
my $iface = $2;
my $vnode = $3;
my $vname = $4;
if (! defined($bridges{"$idx"})) {
$bridges{"$idx"} = {};
}
$bridges->{"$idx"}->{$iface} = [$iface, $vnode, $vname];
}
else {
warning("Bad bridge line: $str");
}
}
open(BR, ">" . TMBRIDGES)
or fatal("Could not open " . TMBRIDGES . ": $!");
print BR "#!/bin/sh\n";
print BR "# auto-generated by delaysetup.pm, DO NOT EDIT\n";
print BR "if [ x\$1 = x ]; then action=enable; else action=\$1; fi\n";
#
# Shutdown: undo only the stuff that might get redone during
# a reconfig.
#
print BR "if [ \"\$action\" = \"disable\" ]; then\n";
# bridging
if ($FBSD_VERSION <= 4) {
print BR " sysctl -w net.link.ether.bridge=0\n";
print BR " sysctl -w net.link.ether.bridge_ipfw=0\n";
}
elsif ($FBSD_VERSION <= 6) {
print BR " sysctl -w net.link.ether.bridge.enable=0\n";
print BR " sysctl -w net.link.ether.bridge.ipfw=0\n";
}
else {
foreach my $bix (keys(%bridges)) {
my ($bix) = @$bridge;
print BR " ifconfig bridge$bix down destroy\n";
}
print BR " sysctl -w net.link.bridge.ipfw=0\n";
print BR " sysctl -w net.link.bridge.ipfw_arp=0\n";
}
print BR " exit 0\n";
print BR "fi\n";
#
# For FreeBSD 6 and beyond we must configure the number of kernel
# mbuf clusters via sysctl, there is no param in the kernel anymore.
#
# We also load the bridge module.
#
if ($FBSD_VERSION >= 7) {
my $MPATH = "/boot/kernel";
print BR "if ! kldstat -q -m if_bridge >/dev/null 2>&1; then\n";
print BR " echo \"Loading bridge module...\"\n";
print BR " kldload $MPATH/if_bridge.ko || {\n";
print BR " echo ' *** Could not load bridge code'\n";
print BR " exit 1\n";
print BR " }\n";
print BR "fi\n";
}
elsif ($FBSD_VERSION >= 6) {
my $MPATH = "/boot/kernel";
print BR "if ! sysctl net.link.ether.bridge >/dev/null 2>&1; then\n";
print BR " kldload $MPATH/bridge.ko || {\n";
print BR " echo \"Could not load bridge code\"\n";
print BR " exit 1\n";
print BR " }\n";
print BR "fi\n";
}
#
# Starting with FreeBSD 7, we use the new if_bridge module
#
if ($FBSD_VERSION >= 7) {
# Turn these off before setting bridge_cfg.
print BR "sysctl -w net.link.bridge.ipfw=0\n";
print BR "sysctl -w net.link.bridge.ipfw_arp=0\n";
foreach my $bix (keys(%bridges)) {
print BR "ifconfig bridge$bix create || {\n";
print BR " echo \"Could not create bridge$bix\"\n";
print BR " exit 1\n";
print BR "}\n";
print BR "ifconfig bridge$bix ";
foreach my $iface (keys($bridges->{$bix})) {
print BR "addm $iface ";
}
print BR "up || {\n";
print BR " echo \"Could not add $if1/$if2 to bridge$bix\"\n";
print BR " ifconfig bridge$bix down destroy\n";
print BR " exit 1\n";
print BR "}\n";
}
# Now turn them back on.
print BR "sysctl -w net.link.bridge.ipfw=1\n";
print BR "sysctl -w net.link.bridge.ipfw_arp=1\n";
} else {
# Turn these off before setting bridge_cfg.
if ($FBSD_VERSION <= 4) {
print BR "sysctl -w net.link.ether.bridge=0\n";
print BR "sysctl -w net.link.ether.bridge_ipfw=0\n";
print BR "sysctl -w net.link.ether.bridge_cfg=";
}
else {
print BR "sysctl -w net.link.ether.bridge.enable=0\n";
print BR "sysctl -w net.link.ether.bridge.ipfw=0\n";
print BR "sysctl -w net.link.ether.bridge.config=";