backup.in 8.54 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh Stoller's avatar
Leigh Stoller committed
2
#
3
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh Stoller's avatar
Leigh Stoller committed
23
#
24
use strict;
25
use English;
26
use Getopt::Std;
27

28 29 30 31 32 33 34 35 36 37 38 39 40
#
# Back up the DB, rolling the base and update logs. The approach is to
# open up the index file and see what the name of the current update file
# is. It has a numeric extension. Rename the base log to base.XXX,
# and then snapshot the DB into backup.XXX So, the combination of
# update.XXX and base.XXX is the DB history since the last time the script
# was run. The file backup.XXX corresponds to the DB at this point in time.
#
# To restore (or track the changes of) a DB, simply take backup.XXX and 
# apply the changes that are stored in update.XXX+1 (which are the changes
# made since backup.XXX was made). This should give you a DB that is the
# same as backup.XXX+1. You can go back further, and just apply all the
# subsequent update.XXX files. 
41 42 43
#
sub usage()
{
44
    print("Usage: backup [-o] [-c -y]\n");
45 46
    exit(-1);
}
47
my $optlist = "docy";
48 49
my $debug   = 0;
my $opsmode = 0;
50
my $clean   = 0;
51

52 53 54 55 56
#
# Configure variables
#
my $TB		= "@prefix@";
my $DBNAME	= "@TBDBNAME@";
57
my $TBOPS       = "@TBOPSEMAIL@";
58 59
my $MAINSITE    = @TBMAINSITE@;
my $PGENISUPPORT= @PROTOGENI_SUPPORT@;
60
my $BACKUPDIR	= "$TB/backup";
61
my $LOGDIR	= "$TB/log/mysql";
62 63
my $HOTCOPY     = "/usr/local/bin/mysqlhotcopy";
my $MYSQLDUMP   = "/usr/local/bin/mysqldump";
64
my $SETSITEVAR  = "$TB/sbin/setsitevar";
65
my $DBCONF      = "$TB/etc/mysqld.pwd";
66
my $TAR         = "/usr/bin/tar";
67
my $BASE	= "base";
Mac Newbold's avatar
Mac Newbold committed
68
my $UPD		= "update";
69
my $SLOW	= "slowqueries";
70
my $BACK	= "tbdb";
71
my $BACKUPDAYS  = "30";
72
my $extension;
73
my @updatefiles = ();
74
my $dohotcopy   = 0;
75 76 77
my $dbname      = "mysql";
my $dbuser      = "root";
my $dbpass;
78 79

# un-taint path
80
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
81 82
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

83 84 85 86 87 88 89 90 91
#
# Turn off line buffering on output
#
$| = 1; 

# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libtestbed;

92 93 94 95 96
#
# Function prototypes
#
sub fatal($);

97 98 99 100 101 102 103 104
#
# Only real root can call this.
# 
if ($UID != 0) {
    print STDERR "You must be root to run this script!\n";
    exit(-1);
}

105 106 107 108 109 110 111 112 113 114
#
# Parse command arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
115 116 117 118 119 120
if (defined($options{"c"})) {
    $clean++;
}
if (defined($options{"y"})) {
    $clean++;
}
121 122 123
if (defined($options{"o"})) {
    $opsmode = 1;
}
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
if ($opsmode && ($dohotcopy || $clean)) {
    fatal("Cannot do hotcopy or clean mode on ops");
}

if ($clean) {
    if (! chdir($BACKUPDIR)) {
	fatal("Could not chdir to $BACKUPDIR: $!");
    }
    my $doit = ($clean > 1 ? "-delete" : "");

    print "These files will be removed in $BACKUPDIR\n";
    if ($clean == 1) {
	print "Add the -y option to actually remove.\n";
    }
    system("find . \\\( -name 'tbdb.*.gz' \\\) -mtime +${BACKUPDAYS} -print $doit");

    if (! chdir($LOGDIR)) {
	fatal("Could not chdir to $LOGDIR: $!");
    }
    print "These files will be removed in $LOGDIR\n";
    if ($clean == 1) {
	print "Add the -y option to actually remove.\n";
    }
147 148
    system("find . \\\( -name 'base.*' -o -name 'update.*' ".
	   "         -o -name 'slowqueries.*' \\\) ".
149 150
	   "-mtime +${BACKUPDAYS} -print $doit");
    exit(0);
151 152
}

153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
#
# Create a temporary name for a log file and untaint it.
#
#
# Form a temp name.
#
my $logname = `mktemp /tmp/dbbackup.XXXXXX`;

if ($logname =~ /^([-\@\w.\/]+)$/) {
    $logname = $1;
} else {
    die "Bad data in $logname";
}

#
# Reopen both stdout and stderr so that we can record all the output for
# later mailing.
# 
open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");

174 175 176
if ($opsmode) {
    require libtbdb;
    import libtbdb;
177

178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
    if (`cat $DBCONF` =~ /^([\w]*)$/) {
	$dbpass = $1;
    }
    else {
	fatal("Could not get mysql password from $DBCONF!");
    }
    if (TBDBConnect($dbname, $dbuser, $dbpass) < 0) {
	fatal("Could not connect to ops database!");
    }
}
else {
    if (! chdir($BACKUPDIR)) {
	fatal("Could not chdir to $BACKUPDIR: $!");
    }
    #
    # Let people know the system will be sluggish.
    #
    system("$SETSITEVAR web/message '<font color=red>".
	   "Nightly backup in progress; system might be sluggish</font>'");
}
198

199 200 201 202 203 204 205 206 207 208 209 210
#
# Open up the index file to see what the current update file extension is.
# The base/backup files correspond to this most recent update file, and
# so should be named with that extension.
#
# Read through to the last line and get its extension. This becomes
# the extension for the other files. No locking is needed.
#
open(IDX, "< $LOGDIR/update.index") or
    fatal("Could not open $LOGDIR/update.index: $!");

while (<IDX>) {
211 212 213 214
    my $file = $_;
    chomp($file);
    
    if ($file =~ /^.*\.([0-9]*)$/) {
215 216
	$extension = $1;
    }
217 218 219 220 221 222 223
    #
    # Watch for uncompressed update files, we now get more then one
    # per day on super busy clusters. 
    #
    if (-e $file) {
	push(@updatefiles, $file);
    }
224 225 226 227
}
close(IDX);
print "Extension:        $extension\n";

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
#
# Use mysqlhotcopy if installed.
#
if ($dohotcopy) {
    #
    # We use hotcopy, so make sure the target directory we are going to use
    # does not already exist.
    #
    if (-e "tbdb") {
	system("/bin/rm -r tbdb") == 0
	    or fatal("Could not remove old backup directory!");
    }
    $BACK .= ".tgz";
}

243 244
my $backname  = "$BACK.$extension";
my $basename  = "$BASE.$extension";
245
my $slowname  = "$SLOW.$extension";
246

247 248
print "Backup file name: $backname\n"
    if (!$opsmode);
249
print "Base file name:   $basename\n";
250
print "Slow file name:   $slowname\n";
251

252 253 254
#
# Move base log out of the way since flush-logs will reset it too.
#
255 256 257
if (-e "$LOGDIR/$BASE") {
    if (system("/bin/mv $LOGDIR/$BASE $LOGDIR/$basename")) {
	print STDERR "Could not move $LOGDIR/$BASE to $LOGDIR/$basename!";
258 259
    }
}
260 261 262 263 264
if (-e "$LOGDIR/$SLOW") {
    if (system("/bin/mv $LOGDIR/$SLOW $LOGDIR/$slowname")) {
	print STDERR "Could not move $LOGDIR/$SLOW to $LOGDIR/$slowname!";
    }
}
265

266 267 268 269 270 271 272 273 274
if ($opsmode) {
    #
    # This will reset the log files.
    #
    if (! DBQueryWarn("flush logs")) {
	fatal("mysqladmin failed!");
    }
}
elsif ($dohotcopy) {
275 276 277 278 279 280 281 282 283 284 285 286 287
    #
    # Do a hotcopy. This will reset the log files.
    #
    if (system("$HOTCOPY --noindices --flushlog $DBNAME $BACKUPDIR")) {
	fatal("$HOTCOPY failed!");
    }

    #
    # Tar and compress the directory.
    # 
    if (system("$TAR -zcf $backname $DBNAME")) {
	fatal("$TAR -zcf $backname failed!");
    }
288
}
289 290 291 292
else {
    #
    # Do a mysqldump. This will reset the log files.
    #
293 294 295 296 297 298
    my @DBNAMES = ($DBNAME);
    push(@DBNAMES, "ims")
	if ($MAINSITE);
    push(@DBNAMES, ("geni", "geni-cm", "geni-ch"))
	if ($PGENISUPPORT);
    
299
    if (system("$MYSQLDUMP --create-options ".
300
	       "--flush-logs --lock-tables --databases @DBNAMES > $backname")) {
301 302
	fatal("mysqldump failed!");
    }
303

304 305 306 307 308 309
    #
    # Compress the files.
    # 
    if (system("nice gzip $backname")) {
	fatal("gzip $backname failed!");
    }
310 311 312
}

if (! chdir($LOGDIR)) {
313
    fatal("Could not chdir to $LOGDIR: $!");
314 315
}

316
if (-e "$basename" && system("nice gzip $basename")) {
317
    fatal("gzip $basename failed!");
318 319
}

320
if (-e "$slowname" && system("nice gzip $slowname")) {
321 322 323
    fatal("gzip $slowname failed!");
}

324 325 326 327 328 329
foreach my $updname (@updatefiles) {
    if (-e "$updname" && system("nice gzip $updname")) {
	fatal("gzip $updname failed!");
    }
}

330 331 332 333 334 335
# Do this now that the backup is complete.
if ($dohotcopy && -e "$BACKUPDIR/tbdb") {
    system("/bin/rm -r $BACKUPDIR/tbdb") == 0
	or fatal("Could not remove old backup directory!");
}

336 337
#SENDMAIL("stoller@flux.utah.edu",
#	 "DB Backup Finished", "", undef, undef, ($logname));
338

339 340
system("$SETSITEVAR web/message -")
    if (!$opsmode);
341
unlink("$logname");
342
exit 0;
343

344 345 346 347
sub fatal($) {
    my ($msg) = @_;

    print STDERR "$msg\n";
348 349

    SENDMAIL($TBOPS, "DB Backup Failed", $msg, undef, undef, ($logname));
350
    system("$SETSITEVAR web/message -");
351 352 353
    unlink("$logname");
    exit(1);
}