Commit 081daac5 authored by Leigh B Stoller's avatar Leigh B Stoller

Commit my two helper scripts for keeping geniracks in sync.

parent a3ae01a9
#!/usr/bin/perl
use strict;
use Getopt::Std;
use POSIX qw(strftime);
use lib "/usr/testbed/lib";
use emutil;
# wap /usr/testbed/sbin/update_sitevars
# wap /usr/testbed/bin/editnodetype ../emulab-devel/install/ctrltype.xml
# wap /usr/testbed/sbin/addservers
# wap setsitevar general/arplockdown staticonly
sub usage {
print "Usage: $0 [options] [rack]\n";
print "Options:\n";
print "-r - Rsync software to rack.\n";
print "-b - Build the software (with reconfig).\n";
print "-i - Install on each rack.\n";
print "-p arg - Update shared pool on each rack.\n";
print " arg is type,func where type=xen|openvz\n";
print "-u - Do Utah rack.\n";
print "-d - Do DDC rack.\n";
print "-U - Skip Utah rack.\n";
print "-D - Skip DDC rack.\n";
print "-7 - Just G7 racks.\n";
print "-8 - Just G8 racks.\n";
print "-f - Run function instead. Add -F to shutdown testbed\n";
print "-s - No parallelization in -r, -f, or -b.\n";
print "-t - Tag source with instageni-YYYYMMDD\n";
print "rack - Specific rack, or all racks\n";
exit(1);
}
my $optlist = "binuUdDhfFrlc78tsp:";
my $rebuild = 0;
my $install = 0;
my $rsync = 0;
my $dofunc = 0;
my $dotag = 0;
my $nopar = 0;
my $dopool;
my $rack;
my $TB = "/usr/testbed";
my $UTAHRACK = "boss.utah.geniracks.net";
my $DDCRACK = "boss.utahddc.geniracks.net",
my %G7RACKS = ("bbn" => "boss.instageni.gpolab.bbn.com",
"nwu" => "boss.instageni.northwestern.edu",
"uky" => "boss.lan.sdn.uky.edu",
"kettering" => "boss.geni.kettering.edu",
"gatech" => "boss.instageni.rnoc.gatech.edu",
"princeton" => "boss.instageni.cs.princeton.edu",
"clemson" => "boss.instageni.clemson.edu",
"kansas" => "boss.instageni.ku.gpeni.net",
"nyu" => "boss.genirack.nyu.edu",
"idaho" => "boss.instageni.uidaho.edu",
);
my @G7RACKS = values(%G7RACKS);
my %G8RACKS = ("max" => "boss.instageni.maxgigapop.net",
"nysernet" => "boss.instageni.nysernet.org",
"sox" => "boss.instageni.sox.net",
"urbana" => "boss.instageni.illinois.edu",
"missouri" => "boss.instageni.rnet.missouri.edu",
"wisc" => "boss.instageni.wisc.edu",
"rutgers" => "boss.instageni.rutgers.edu",
"stanford" => "boss.instageni.stanford.edu",
"cornell" => "boss.geni.it.cornell.edu",
"lsu" => "boss.instageni.lsu.edu",
"case" => "boss.geni.case.edu",
"moxi" => "boss.instageni.iu.edu",
"chicago" => "boss.geni.uchicago.edu",
"metro" => "boss.instageni.metrodatacenter.com",
"nps" => "boss.instageni.nps.edu",
"ohio" => "boss.instageni.osu.edu",
"umkc" => "boss.instageni.umkc.edu",
"ucla" => "boss.instageni.idre.ucla.edu",
);
my @G8RACKS = values(%G8RACKS);
my @ALLRACKS = (@G7RACKS, @G8RACKS);
my @ALLCONTROL = (
"utahddc.control.geniracks.net",
"gpolab.control-nodes.geniracks.net",
"nu.control-nodes.geniracks.net",
"uky.control-nodes.geniracks.net",
"kettering.control-nodes.geniracks.net",
"gatech.control-nodes.geniracks.net",
"princeton.control-nodes.geniracks.net",
"clemson.control-nodes.geniracks.net",
"kansas.control-nodes.geniracks.net",
"nyu.control-nodes.geniracks.net",
"max.control-nodes.geniracks.net",
"nysernet.control-nodes.geniracks.net",
"sox.control-nodes.geniracks.net",
"missouri.control-nodes.geniracks.net",
# Illinois
"urbana.control-nodes.geniracks.net",
"rutgers.control-nodes.geniracks.net",
"stanford.control-nodes.geniracks.net",
"cornell.control-nodes.geniracks.net",
"lsu.control-nodes.geniracks.net",
"wisconsin.control-nodes.geniracks.net",
"casewestern.control-nodes.geniracks.net",
"chicago.control-nodes.geniracks.net",
"moxi.control-nodes.geniracks.net",
"dublin.control-nodes.geniracks.net",
"nps.control-nodes.geniracks.net",
"idaho.control-nodes.geniracks.net",
);
my @TODO = ($UTAHRACK, $DDCRACK, @ALLRACKS);
my %SKIP = ();
my $HOME = "/home/stoller";
sub fatal($)
{
my ($msg) = @_;
die("*** $0:\n".
" $msg\n");
}
sub SSH($$);
sub TagSchema();
#
# Turn off line buffering on output
#
$| = 1;
#
# Check args.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"h"})) {
usage();
}
if (defined($options{"l"})) {
foreach my $name (defined($options{"c"}) ? @ALLCONTROL : @ALLRACKS) {
print "$name\n";
}
exit(0);
}
if (defined($options{"i"})) {
$install = 1;
}
if (defined($options{"s"})) {
$nopar = 1;
}
if (defined($options{"f"})) {
$dofunc = 1;
if (defined($options{"F"})) {
$dofunc++;
}
}
if (defined($options{"b"})) {
$rebuild = 1;
}
if (defined($options{"r"})) {
$rsync = 1;
}
if (defined($options{"p"})) {
$dopool = $options{"p"};
$dofunc = 1;
}
if (defined($options{"u"}) || defined($options{"d"})) {
@TODO = ();
push(@TODO, $UTAHRACK)
if (defined($options{"u"}));
push(@TODO, $DDCRACK)
if (defined($options{"d"}));
}
elsif (defined($options{"7"})) {
@TODO = @G7RACKS;
if (! defined($options{"U"})) {
@TODO = ($UTAHRACK, @TODO);
}
}
elsif (defined($options{"8"})) {
@TODO = @G8RACKS;
if (! defined($options{"D"})) {
@TODO = ($DDCRACK, @TODO);
}
}
elsif (@ARGV) {
@TODO = ();
foreach my $arg (@ARGV) {
if ($arg =~ /\./) {
push(@TODO, $arg);
}
elsif (exists($G7RACKS{$arg})) {
push(@TODO, $G7RACKS{$arg});
}
elsif (exists($G8RACKS{$arg})) {
push(@TODO, $G8RACKS{$arg});
}
else {
fatal("No such rack: $arg");
}
}
}
else {
@TODO = @ALLRACKS;
if (! defined($options{"D"})) {
@TODO = ($DDCRACK, @TODO);
}
if (! defined($options{"U"})) {
@TODO = ($UTAHRACK, @TODO);
}
}
if (defined($options{"t"})) {
TagSchema();
}
#
# Just a way to run some little bits of code on the target.
#
if ($dofunc && !$install) {
my $coderef = sub {
my ($rack) = @_;
print "Running function on $rack ...\n";
if ($dofunc > 1) {
print "-> Shutting down testbed ...\n";
if (SSH($rack,
"(sudo $TB/sbin/testbed-control shutdown >& shutdown.log)")) {
print STDERR "** could not shutdown!\n";
return 1;
}
}
my $devel = "emulab-devel/emulab-devel";
print "-> Running function ...\n";
my $command = "";
if (defined($dopool)) {
my ($type,$func,$limit) = split(',', $dopool);
usage()
if (!(defined($type) && defined($func)));
$command = "$devel/update-shared.pl -t $type -f $func ".
(defined($limit) ? "-l $limit" : "");
}
elsif (1) {
$command = "/usr/testbed/sbin/wap ".
"/usr/testbed/sbin/delete_image -p ".
"ch-geni-net,OVSxenOpenFlowTutorial";
}
elsif (0) {
$command = "sudo -u geniuser /usr/testbed/sbin/wap ".
"/usr/testbed/sbin/image_import -g -p ch-geni-net ".
"https://www.utahddc.geniracks.net/image_metadata.php\\\?uuid=da660c93-2134-11e3-85ef-000000000000";
}
elsif (0) {
$command = "sudo scp ".
"$devel/clientside/tmcc/common/mkvnode.pl ".
"$devel/clientside/tmcc/common/libsetup.pm ".
"$devel/clientside/tmcc/linux/xen/libvnode_xen.pm ".
" vhost3.shared-nodes.emulab-ops:/usr/local/etc/emulab";
}
elsif (0) {
$command = "sudo scp $devel/xendomains ".
" vhost3.shared-nodes.emulab-ops:/usr/local/etc/emulab";
}
elsif (1) {
$command =
"cd emulab-devel/obj/rc.d; sudo gmake install";
}
elsif (0) {
$command = "sudo ssh vhost1.shared-nodes.emulab-ops ".
"sysctl -w net.netfilter.nf_conntrack_generic_timeout=120 ".
" net.netfilter.nf_conntrack_tcp_timeout_established=54000 ".
" net.netfilter.nf_conntrack_max=131071";
}
elsif (0) {
$command = "sudo ssh vhost3.shared-nodes.emulab-ops ".
"/etc/rc3.d/S23ntp restart";
}
elsif (0) {
# Utah Rack is vhost2. Sheesh.
$command =
"sudo ssh vhost1.shared-nodes.emulab-ops ".
" systemctl restart ntpd.service; " .
"sudo ssh vhost2.shared-nodes.emulab-ops ".
" systemctl restart ntpd.service; ";
}
elsif (0) {
$command =
"cd emulab-devel/obj/db; sudo gmake /usr/testbed/lib/Node.pm";
}
elsif (0) {
$command = "cd emulab-devel/obj/ntpd; ".
"sudo gmake install; ".
"sudo /etc/rc.d/ntpd restart";
}
elsif (0) {
$command =
"ssh ops \"(cd emulab-devel/obj/ntpd; ".
"sudo gmake control-install; ".
"sudo /etc/rc.d/ntpd restart)\"";
}
elsif (0) {
$command =
"chmod 664 emulab-devel/defs-genirack; ".
" echo 'BROWSER_CONSOLE_ENABLE=1' >> emulab-devel/defs-genirack; ".
" sudo scp $devel/capture-nossl ".
" vhost3.shared-nodes.emulab-ops:/usr/local/etc/emulab/capture";
}
elsif (0) {
#$command = "sudo /usr/testbed/sbin/getimages";
#$command = "/usr/testbed/sbin/wap /usr/testbed/sbin/grantimage -a -x emulab-ops,Ubuntu12-64-OVS";
$command = "emulab-devel/emulab-devel/runsonxen.pl -r ".
"emulab-ops,FEDORA15-STD";
}
else {
$command = "cat emulab-devel/emulab-devel/foo.sql | mysql tbdb";
}
#$command = "($command >& /tmp/function.log)";
if (SSH($rack, $command)) {
print STDERR "Error running '$command' on $rack\n";
return 2;
}
if ($dofunc > 1) {
print "-> Booting the testbed ...\n";
if (SSH($rack,
"(sudo $TB/sbin/testbed-control boot >& boot.log)")) {
print STDERR "** could not boot!\n";
return 3;
next;
}
}
};
# List of racks that we can proceed with.
my @doracks = @TODO;
# Return codes for each rack.
my @results = ();
if (ParRun({"maxwaittime" => 99999, "maxchildren" => ($nopar ? 1 : 6)},
\@results, $coderef, @doracks)) {
fatal("ParRun failed!");
}
#
# Check the exit codes.
#
my $count = 0;
foreach my $result (@results) {
my ($rack) = $doracks[$count];
if ($result) {
$result = $result >> 8;
if ($result == 1) {
$SKIP{$rack} = "could not shutdown";
}
elsif ($result == 2) {
$SKIP{$rack} = "could not run function";
}
elsif ($result == 3) {
$SKIP{$rack} = "could not retstart";
}
}
$count++;
}
if (keys(%SKIP)) {
print "The following racks failed!\n";
foreach my $rack (keys(%SKIP)) {
my $reason = $SKIP{$rack};
print "$rack: $reason\n";
}
}
exit(scalar(keys(%SKIP)));
}
#
# First do all of the rsyncs.
#
if ($rsync) {
my $coderef = sub {
my ($rack) = @_;
print "rsyncing $rack ...\n";
print "-> rsyncing emulab-devel\n";
system("rsync -a --timeout=30 --delete ".
"--exclude-from .rsyncignore ".
" $HOME/testbed-noelvin/emulab-devel ".
" $HOME/testbed-noelvin/reconfig.rack ".
" elabman\@${rack}:emulab-devel");
if ($?) {
print STDERR "** $rack: error rsyncing emulab-devel\n";
return 1;
}
print "-> rsyncing pubsub\n";
system("rsync -a --timeout=30 --delete ".
"--exclude-from $HOME/.rsyncignore ".
" $HOME/testbed-noelvin/pubsub elabman\@${rack}:");
if ($?) {
print STDERR "** $rack: error rsyncing pubsub\n";
return 1;
}
print "-> rsyncing shellinabox\n";
system("rsync -a --timeout=30 --delete ".
"--exclude-from $HOME/.rsyncignore ".
" $HOME/testbed-noelvin/shellinabox elabman\@${rack}:");
if ($?) {
print STDERR "** $rack: error rsyncing shellinabox\n";
return 1;
}
};
# List of racks that we can proceed with.
my @doracks = @TODO;
# Return codes for each rack.
my @results = ();
if (ParRun({"maxwaittime" => 99999, "maxchildren" => ($nopar ? 1 : 6)},
\@results, $coderef, @doracks)) {
fatal("ParRun failed!");
}
#
# Check the exit codes.
#
my $count = 0;
foreach my $result (@results) {
my ($rack) = $doracks[$count];
if ($result) {
$SKIP{$rack} = "could not rsync";
}
$count++;
}
if (keys(%SKIP)) {
print "The following racks failed!\n";
foreach my $rack (keys(%SKIP)) {
my $reason = $SKIP{$rack};
print "$rack: $reason\n";
}
}
}
if ($rebuild) {
my $coderef = sub {
my ($rack) = @_;
print "rebuilding on $rack ...\n";
print "-> $rack: Starting reconfig ...\n";
if (SSH($rack,
"(cd emulab-devel; ./reconfig.rack >& reconfig.log)")) {
print STDERR "** $rack: could not reconfig!\n";
return 1;
}
print "-> $rack: Starting clean ...\n";
if (SSH($rack,
"(cd emulab-devel/obj; sudo gmake clean >& clean.log)")) {
print STDERR "** $rack: could not clean!\n";
return 2;
}
print "-> $rack: Starting rebuild ...\n";
if (SSH($rack,
"(cd emulab-devel/obj; gmake >& rebuild.log)")) {
print STDERR "** $rack: could not rebuild!\n";
return 3;
}
return 0;
};
# List of racks that we can proceed with.
my @doracks = ();
foreach my $rack (@TODO) {
if (exists($SKIP{$rack})) {
print "skipping rebuild on $rack\n";
next;
}
push(@doracks, $rack);
}
# Return codes for each rack.
my @results = ();
if (ParRun({"maxwaittime" => 99999, "maxchildren" => ($nopar ? 1 : 8)},
\@results, $coderef, @doracks)) {
fatal("ParRun failed!");
}
#
# Check the exit codes.
#
my $count = 0;
foreach my $result (@results) {
my ($rack) = $doracks[$count];
if ($result) {
$result = $result >> 8;
if ($result == 1) {
$SKIP{$rack} = "could not reconfig";
}
elsif ($result == 2) {
$SKIP{$rack} = "could not clean";
}
elsif ($result == 3) {
$SKIP{$rack} = "could not rebuild";
}
}
$count++;
}
}
if ($install) {
foreach my $rack (@TODO) {
if (exists($SKIP{$rack})) {
print "skipping install on $rack\n";
next;
}
print "installing on $rack ...\n";
print "-> Shutting down testbed ...\n";
if (SSH($rack,
"(sudo $TB/sbin/testbed-control shutdown >& /tmp/shutdown.log)")) {
print STDERR "** could not shutdown!\n";
$SKIP{$rack} = "could not shutdown";
next;
}
print "-> Starting install on boss ...\n";
if (SSH($rack, "(cd emulab-devel/obj; ".
" sudo gmake update-testbed-nostop >& /tmp/install.log)")) {
print STDERR "** could not install on boss!\n";
$SKIP{$rack} = "could not install on boss";
next;
}
print "-> Starting install on ops ...\n";
my $rackops = $rack;
$rackops =~ s/^boss/ops/;
if (SSH($rackops, "(cd emulab-devel/obj/clientside; ".
" sudo gmake control-install >>& /tmp/install.log)")) {
print STDERR "** could not install on ops!\n";
$SKIP{$rack} = "could not install ops";
next;
}
if ($dofunc) {
print "-> Running function ...\n";
my $command = "/bin/ls /";
if (1) {
$command = "cd emulab-devel/obj/install; ".
" sudo perl emulab-install -b -u -i boss/mfs boss ";
}
if (defined($command) &&
SSH($rack, "($command >& /tmp/function.log)")) {
print STDERR "Error running '$command' on $rack\n";
$SKIP{$rack} = "could not run function";
next;
}
}
print "-> Booting the testbed ...\n";
if (SSH($rack,
"(sudo $TB/sbin/testbed-control boot >& /tmp/boot.log)")) {
print STDERR "** could not boot!\n";
$SKIP{$rack} = "could not boot";
next;
}
}
}
if (keys(%SKIP)) {
print "The following racks failed!\n";
foreach my $rack (keys(%SKIP)) {
my $reason = $SKIP{$rack};
print "$rack: $reason\n";
}
}
sub SSH($$)
{
my ($host, $cmd, $timeout) = @_;
$timeout = 2500 if (!defined($timeout));
my $childpid = fork();
if ($childpid) {
local $SIG{ALRM} = sub { kill("TERM", $childpid); };
alarm $timeout;
waitpid($childpid, 0);
alarm 0;
my $stat = $?;
#
# Any failure, revert to plain reboot below.
#
if ($?) {
return -1;
}
return 0;
}
else {
exec("ssh elabman\@${host} '$cmd'");
exit(1);
}
}
sub TagSchema()
{
my $tag = POSIX::strftime("instageni-20%y%m%d", localtime(time()));
print "Tagging with $tag\n";
system("git tag -f -m 'Push to InstaGeni Racks' $tag");
fatal("Could not tag repo!")
if ($?);
system("git push --tags");
fatal("Could not push tag up!")
if ($?);
return 0;
}