backup.in

#!/usr/bin/perl -w
#
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use strict;
use English;
use Getopt::Std;

#
# Back up the DB, rolling the base and update logs. The approach is to
# open up the index file and see what the name of the current update file
# is. It has a numeric extension. Rename the base log to base.XXX,
# and then snapshot the DB into backup.XXX So, the combination of
# update.XXX and base.XXX is the DB history since the last time the script
# was run. The file backup.XXX corresponds to the DB at this point in time.
#
# To restore (or track the changes of) a DB, simply take backup.XXX and 
# apply the changes that are stored in update.XXX+1 (which are the changes
# made since backup.XXX was made). This should give you a DB that is the
# same as backup.XXX+1. You can go back further, and just apply all the
# subsequent update.XXX files. 
#
sub usage()
{
    print("Usage: backup [-o] [-c -y]\n");
    exit(-1);
}
my $optlist = "docy";
my $debug   = 0;
my $opsmode = 0;
my $clean   = 0;

#
# Configure variables
#
my $TB		= "@prefix@";
my $DBNAME	= "@TBDBNAME@";
my $TBOPS       = "@TBOPSEMAIL@";
my $MAINSITE    = @TBMAINSITE@;
my $PGENISUPPORT= @PROTOGENI_SUPPORT@;
my $BACKUPDIR	= "$TB/backup";
my $LOGDIR	= "$TB/log/mysql";
my $HOTCOPY     = "/usr/local/bin/mysqlhotcopy";
my $MYSQLDUMP   = "/usr/local/bin/mysqldump";
my $SETSITEVAR  = "$TB/sbin/setsitevar";
my $DBCONF      = "$TB/etc/mysqld.pwd";
my $TAR         = "/usr/bin/tar";
my $BASE	= "base";
my $UPD		= "update";
my $SLOW	= "slowqueries";
my $BACK	= "tbdb";
my $BACKUPDAYS  = "30";
my $extension;
my @updatefiles = ();
my $dohotcopy   = 0;
my $dbname      = "mysql";
my $dbuser      = "root";
my $dbpass;

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1; 

# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libtestbed;

#
# Function prototypes
#
sub fatal($);

#
# Only real root can call this.
# 
if ($UID != 0) {
    print STDERR "You must be root to run this script!\n";
    exit(-1);
}

#
# Parse command arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"c"})) {
    $clean++;
}
if (defined($options{"y"})) {
    $clean++;
}
if (defined($options{"o"})) {
    $opsmode = 1;
}
if ($opsmode && ($dohotcopy || $clean)) {
    fatal("Cannot do hotcopy or clean mode on ops");
}

if ($clean) {
    if (! chdir($BACKUPDIR)) {
	fatal("Could not chdir to $BACKUPDIR: $!");
    }
    my $doit = ($clean > 1 ? "-delete" : "");

    print "These files will be removed in $BACKUPDIR\n";
    if ($clean == 1) {
	print "Add the -y option to actually remove.\n";
    }
    system("find . \\\( -name 'tbdb.*.gz' \\\) -mtime +${BACKUPDAYS} -print $doit");

    if (! chdir($LOGDIR)) {
	fatal("Could not chdir to $LOGDIR: $!");
    }
    print "These files will be removed in $LOGDIR\n";
    if ($clean == 1) {
	print "Add the -y option to actually remove.\n";
    }
    system("find . \\\( -name 'base.*' -o -name 'update.*' ".
	   "         -o -name 'slowqueries.*' \\\) ".
	   "-mtime +${BACKUPDAYS} -print $doit");
    exit(0);
}

#
# Create a temporary name for a log file and untaint it.
#
#
# Form a temp name.
#
my $logname = `mktemp /tmp/dbbackup.XXXXXX`;

if ($logname =~ /^([-\@\w.\/]+)$/) {
    $logname = $1;
} else {
    die "Bad data in $logname";
}

#
# Reopen both stdout and stderr so that we can record all the output for
# later mailing.
# 
open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");

if ($opsmode) {
    require libtbdb;
    import libtbdb;

    if (`cat $DBCONF` =~ /^([\w]*)$/) {
	$dbpass = $1;
    }
    else {
	fatal("Could not get mysql password from $DBCONF!");
    }
    if (TBDBConnect($dbname, $dbuser, $dbpass) < 0) {
	fatal("Could not connect to ops database!");
    }
}
else {
    if (! chdir($BACKUPDIR)) {
	fatal("Could not chdir to $BACKUPDIR: $!");
    }
    #
    # Let people know the system will be sluggish.
    #
    system("$SETSITEVAR web/message '<font color=red>".
	   "Nightly backup in progress; system might be sluggish</font>'");
}

#
# Open up the index file to see what the current update file extension is.
# The base/backup files correspond to this most recent update file, and
# so should be named with that extension.
#
# Read through to the last line and get its extension. This becomes
# the extension for the other files. No locking is needed.
#
open(IDX, "< $LOGDIR/update.index") or
    fatal("Could not open $LOGDIR/update.index: $!");

while (<IDX>) {
    my $file = $_;
    chomp($file);
    
    if ($file =~ /^.*\.([0-9]*)$/) {
	$extension = $1;
    }
    #
    # Watch for uncompressed update files, we now get more then one
    # per day on super busy clusters. 
    #
    if (-e $file) {
	push(@updatefiles, $file);
    }
}
close(IDX);
print "Extension:        $extension\n";

#
# Use mysqlhotcopy if installed.
#
if ($dohotcopy) {
    #
    # We use hotcopy, so make sure the target directory we are going to use
    # does not already exist.
    #
    if (-e "tbdb") {
	system("/bin/rm -r tbdb") == 0
	    or fatal("Could not remove old backup directory!");
    }
    $BACK .= ".tgz";
}

my $backname  = "$BACK.$extension";
my $basename  = "$BASE.$extension";
my $slowname  = "$SLOW.$extension";

print "Backup file name: $backname\n"
    if (!$opsmode);
print "Base file name:   $basename\n";
print "Slow file name:   $slowname\n";

#
# Move base log out of the way since flush-logs will reset it too.
#
if (-e "$LOGDIR/$BASE") {
    if (system("/bin/mv $LOGDIR/$BASE $LOGDIR/$basename")) {
	print STDERR "Could not move $LOGDIR/$BASE to $LOGDIR/$basename!";
    }
}
if (-e "$LOGDIR/$SLOW") {
    if (system("/bin/mv $LOGDIR/$SLOW $LOGDIR/$slowname")) {
	print STDERR "Could not move $LOGDIR/$SLOW to $LOGDIR/$slowname!";
    }
}

if ($opsmode) {
    #
    # This will reset the log files.
    #
    if (! DBQueryWarn("flush logs")) {
	fatal("mysqladmin failed!");
    }
}
elsif ($dohotcopy) {
    #
    # Do a hotcopy. This will reset the log files.
    #
    if (system("$HOTCOPY --noindices --flushlog $DBNAME $BACKUPDIR")) {
	fatal("$HOTCOPY failed!");
    }

    #
    # Tar and compress the directory.
    # 
    if (system("$TAR -zcf $backname $DBNAME")) {
	fatal("$TAR -zcf $backname failed!");
    }
}
else {
    #
    # Do a mysqldump. This will reset the log files.
    #
    my @DBNAMES = ($DBNAME);
    push(@DBNAMES, "ims")
	if ($MAINSITE);
    push(@DBNAMES, ("geni", "geni-cm", "geni-ch"))
	if ($PGENISUPPORT);
    
    if (system("$MYSQLDUMP --create-options ".
	       "--flush-logs --lock-tables --databases @DBNAMES > $backname")) {
	fatal("mysqldump failed!");
    }

    #
    # Compress the files.
    # 
    if (system("nice gzip $backname")) {
	fatal("gzip $backname failed!");
    }
}

if (! chdir($LOGDIR)) {
    fatal("Could not chdir to $LOGDIR: $!");
}

if (-e "$basename" && system("nice gzip $basename")) {
    fatal("gzip $basename failed!");
}

if (-e "$slowname" && system("nice gzip $slowname")) {
    fatal("gzip $slowname failed!");
}

foreach my $updname (@updatefiles) {
    if (-e "$updname" && system("nice gzip $updname")) {
	fatal("gzip $updname failed!");
    }
}

# Do this now that the backup is complete.
if ($dohotcopy && -e "$BACKUPDIR/tbdb") {
    system("/bin/rm -r $BACKUPDIR/tbdb") == 0
	or fatal("Could not remove old backup directory!");
}

#SENDMAIL("stoller@flux.utah.edu",
#	 "DB Backup Finished", "", undef, undef, ($logname));

system("$SETSITEVAR web/message -")
    if (!$opsmode);
unlink("$logname");
exit 0;

sub fatal($) {
    my ($msg) = @_;

    print STDERR "$msg\n";

    SENDMAIL($TBOPS, "DB Backup Failed", $msg, undef, undef, ($logname));
    system("$SETSITEVAR web/message -");
    unlink("$logname");
    exit(1);
}