install-tarfile 15.8 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2010 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
5 6
# All rights reserved.
#
7 8
use English;
use Getopt::Std;
9
use POSIX qw(mktime);
10

11 12 13
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

14 15
#
# Install a tarfile. This script is run from the setup code on client nodes.
16 17 18
# By default the tarfile is accessed directly via NFS, if '-c' is specified
# the tar file is copied over first either via NFS (the default) or tmcc
# (-t option).
19 20 21 22 23 24 25
#
# Exit Value Matters!: 0 if installed okay
#                      1 if already installed
#                     -1 if something goes wrong.
#
sub usage()
{
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
    print STDOUT 
	"Usage: install-tarfile [-hVdfvct] [-n nodeid] [-u user] <installdir> ".
	    "<filename>\n".
	"       install-tarfile [-l]\n".
	"Options:\n".
	"  -h          Display this message\n".
	"  -V          Print version information and exit\n".
        "  -d          Output debugging messages\n".
	"  -f          Force the installation\n".
	"  -v          Verify an installation only, do not attempt an install\n".
	"  -c          Copy the tar file to the local disk (recommended)\n".
	"  -t          Download the tarfile from Emulab\n".
	"  -n nodeid   Override the default node ID when downloading from Emulab\n".
	"  -u user     User that should own files with unknown uid/gid\n".
	"  -l          List the currently installed tar files and exit\n".
	"\n".
	"Required Arguments:".
	"  installdir  The absolute path of the install directory.\n".
	"  filename    The absolute path of the tar file to install.\n";

46 47
    exit(-1);
}
48
my $optlist  = "hVlvcdftn:u:";
49 50 51 52 53 54 55

#
# Turn off line buffering on output
#
$| = 1;

#
56
# Untaint env.
57 58 59 60 61 62 63 64 65 66
# 
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# No configure vars.
#
my $IDENTFILE      = "/var/db/testbed.tarfiles";
my $tarfile        = "";
my $decompressflag = "";
my $installdir     = "/";
67 68
my $unknownuser	   = "";
my $force	   = 0;
69
my $usewget	   = 0;
70
my $listmode	   = 0;
71
my $copymode	   = 0;
72
my $verifymode	   = 0;
73 74 75 76
my $debug	   = 0;
my $copyfile;
my $nodeid;
my $keyhash;
77 78 79 80 81
my $filemd5;
my $filestamp;
my $oldmd5;
my $oldstamp;
my @identlines     = ();
82 83 84 85 86 87

#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself. 
#
use libsetup;
88
use libtmcc;
89

90
# Protos
91 92 93
sub GetTarFile($$$$$$$);
sub GetMD5($);
sub WriteIdentFile();
94

95 96 97 98
#
# Must be running as root to work. 
#
if ($EUID != 0) {
99
    die("Must be run as root! Try using sudo or su1!\n");
100 101 102 103 104 105 106
}

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
107
if (! getopts($optlist, \%options) || defined($options{"h"})) {
108 109
    usage();
}
110 111 112 113 114 115 116
if (defined($options{"V"})) {
    print STDOUT "0.5";
    exit(-1);
}
if (defined($options{"l"})) {
    $listmode = 1;
}
117 118 119
if (defined($options{"c"})) {
    $copymode = 1;
}
120 121 122
if (defined($options{"d"})) {
    $debug = 1;
}
123 124 125
if (defined($options{"f"})) {
    $force = 1;
}
126 127
if (defined($options{"t"})) {
    $copymode = 1;
128
    $usewget  = 1;
129
}
130 131 132
if (defined($options{"v"})) {
    $verifymode = 1;
}
133
if (defined($options{"n"})) {
134 135 136 137 138 139
    $nodeid = $options{"n"};
    if ($nodeid =~ /^([-\w]+)$/) {
	$nodeid = $1;
    }
    else {
	fatal("Tainted nodeid: $nodeid");
140 141
    }
}
142 143 144 145 146 147 148 149 150
if (defined($options{"u"})) {
    $unknownuser = $options{"u"};
    if ($unknownuser =~ /^([-\w]+)$/) {
	$unknownuser = $1;
    }
    else {
	fatal("Tainted user: $unknownuser");
    }
}
151
# XXX compat
152
if (defined($options{"j"})) {
153
    $copymode = 1;
154
    $usewget  = 1;
155
}
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
if (!$listmode) {
    if (@ARGV != 2) {
	usage();
    }
    $installdir = $ARGV[0];
    $tarfile    = $ARGV[1];
    
    #
    # Untaint the arguments.
    #
    # Note different taint check (allow /).
    if ($tarfile =~ /^([-\w.\/\+]+)$/) {
	$tarfile = $1;
    }
    else {
	fatal("Tainted filename: $tarfile");
    }
    if ($installdir =~ /^([-\w.\/]+)$/) {
	$installdir = $1;
    }
    else {
	fatal("Tainted directory name: $installdir");
    }
179

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
    if (! $tarfile =~ /^\//) {
	fatal("Tar file must be an absolute path.");
    }
    
    if (! $installdir =~ /^\//) {
	fatal("Install directory must be an absolute path.");
    }
    
    #
    # Make sure the installdir exists!
    #
    if (! -d $installdir &&
	! mkdir($installdir, 0775)) {
	fatal("Could not make directory $installdir: $!");
    }
195
}
196
   
197
#
198 199 200 201 202 203 204 205
# Check to make sure this tarfile has not already been installed.
# If so, we get the old timestamp and md5 so we can compare against
# current ones.
# We need to update the stamp/md5 in place in case it has changed, so
# copy out all the identlines so we can write them back later. We do not
# copyout the current one of course; we make up a new line at the end
# of this script based on the new info.
# 
206
if (-e $IDENTFILE) {
207 208
    if (!open(IDENT, $IDENTFILE)) {
	fatal("Could not open $IDENTFILE: $!");
209
    }
210
    while (<IDENT>) {
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
	if ($_ =~ /^([-\w\.\/\+]*) ([\d]*) ([\w]*) ([-\w\.\/\+]*)$/) {
	    my $file = $1;
	    my $stamp= $2;
	    my $md5  = $3;
	    my $idir = $4;

	    if ($listmode) {
		# XXX Perl ignorance here...  I can do ("\t" * 3) in python
		# to get a string with three tabs, how do you do that in perl?
		print STDOUT "$idir" .
		    (length($idir) >= 24 ? "  " :
		     length($idir) >= 16 ? "\t" :
		     length($idir) >=  8 ? "\t\t" :
		                           "\t\t\t") 
			. "$file\n";
		# TODO: Tell them which ones are up-to-date?
		next;
	    }
	    elsif (($file eq $tarfile) && ($idir eq $installdir)) {
		#
		# Save the info and continue;
		#
		$oldstamp = $stamp;
		$oldmd5   = $md5;
		next;
	    }
	    push(@identlines, "$file $stamp $md5 $idir");
	}
	elsif ($_ =~ /^([-\w\.\/\+]*) ([\d]*) ([\w]*)$/) {
	    # Continue to accept the old school format...
241 242 243 244
	    my $file = $1;
	    my $stamp= $2;
	    my $md5  = $3;

245 246 247 248 249 250
	    if ($listmode) {
		print STDOUT "(unknown)\t\t$file\n";
		# TODO: Tell them which ones are up-to-date?
		next;
	    }
	    elsif ($file eq $tarfile) {
251 252 253 254 255 256 257 258 259 260 261 262 263 264
		#
		# Save the info and continue;
		#
		$oldstamp = $stamp;
		$oldmd5   = $md5;
		next;
	    }
	    push(@identlines, "$file $stamp $md5");
	}
	else {
	    warn("*** WARNING: Bad line in $IDENTFILE: $_\n");
	}
    }
    close(IDENT);
265
}
266

267 268 269 270
if ($listmode) {
    exit(0);
}

271
#
272
# Must be able to see the tarfile if not copying. The front end
273
# ensures that its in a reasonable place, but have to make sure here.
274
#
275
if (! $copymode) {
276 277 278 279 280 281
    #
    # Make sure its really there.
    #
    if (! -r $tarfile) {
	fatal("$tarfile does not exist or is not accessible!");
    }
282 283 284 285 286 287 288

    #
    # Compare timestamp. If no change, we are done. 
    #
    (undef,undef,undef,undef,undef,undef,
     undef,undef,undef,$filestamp) = stat($tarfile);

289
    if (defined($oldstamp) && $oldstamp >= $filestamp && !$force) {
290 291 292
	print STDOUT "Tarfile $tarfile has already been installed!\n";
	exit(1);
    }
293

294 295 296 297
    #
    # Otherwise compare MD5.
    #
    $filemd5 = GetMD5($tarfile);
298
    if (defined($oldmd5) && $filemd5 eq $oldmd5 && !$force) {
299 300
	print STDOUT "Tarfile $tarfile has already been installed!\n";
	# Must write a new ident file to avoid repeated checks.
301
	push(@identlines, "$tarfile $filestamp $filemd5 $installdir");
302 303 304
	WriteIdentFile();
	exit(1);
    }
305 306
}
else {
307
    $copyfile = `mktemp /var/tmp/tarball.XXXXXX`;
308

309 310
    if ($copyfile =~ /^([-\@\w\.\/]+)$/) {
	$copyfile = $1;
311 312
    }
    else {
313
	die("Bad data in copyfile name: $copyfile");
314 315
    }
    #
316 317 318
    # Dies on any failure.
    # Returns >0 if server copy has not been modifed.
    # Returns =0 if okay to install, and gives us new stamp/md5.
319
    #
320 321 322 323 324
    if (GetTarFile($tarfile, $copyfile, $usewget,
		   $oldstamp, $oldmd5, \$filestamp, \$filemd5)) {
	print STDOUT "Tarfile $tarfile has already been installed!\n";
	if (defined($filestamp) && $filestamp != $oldstamp) {
	    # Must write a new ident file to avoid repeated checks.
325
	    push(@identlines, "$tarfile $filestamp $oldmd5 $installdir");
326 327 328 329 330 331
	    WriteIdentFile();
	}
	unlink($copyfile)
	    if (-e $copyfile);
	exit(1);
    }
332 333
}

334 335 336 337
if ($verifymode) {
    exit(0);
}

338
#
339 340
# Okay, add new info to the list for update.
#
341
push(@identlines, "$tarfile $filestamp $filemd5 $installdir");
342 343 344 345 346

#
# Figure what decompression flag is required, based on file extension.
#
SWITCH: for ($tarfile) {
347
    /^.*\.tar\.Z$/   && do {$decompressflag = "-z"; last SWITCH; } ;
348
    /^.*\.tar\.gz$/  && do {$decompressflag = "-z"; last SWITCH; } ;
349
    /^.*\.tgz$/      && do {$decompressflag = "-z"; last SWITCH; } ;
350
    /^.*\.tar\.bz2$/ && do {$decompressflag = "-j"; last SWITCH; } ;
351 352 353 354 355 356 357 358 359 360
    /^.*\.tar$/      && do {last SWITCH; } ;
}

#
# Install tar file from root?
# 
if (! chdir($installdir)) {
    fatal("Could not chdir to $installdir: $!\n");
}

361 362 363 364 365 366 367
my ($uuname,$uupasswd,$uuuid,$uugid) = getpwnam($unknownuser)
    if ($unknownuser);
    
if ($unknownuser && ! $uuname) {
    fatal("No such user: $unknownuser");
}

368 369 370
#
# Run the tarfile. 
#
371 372
if ($copymode) {
    $tarfile = $copyfile;
373
}
374 375 376 377 378 379 380
$tarlist = `mktemp /var/tmp/tarlist.XXXXXX`;
if ($tarlist =~ /^([-\@\w\.\/]+)$/) {
    $tarlist = $1;
}
else {
    die("Bad data in tarlist name: $tarlist");
}
381
my $oumask = umask(0);
382 383 384
my $TAR = `which gtar`;
chomp $TAR;
$TAR = "tar" if $TAR eq '';
385 386 387 388 389 390 391 392
open VER, "$TAR --version 2>&1 |";
my $GNU_TAR = 0;
while (<VER>) {
    $GNU_TAR = 1 if /GNU tar/;
}
my $TAR_FLAGS = $GNU_TAR ? "-xvf" : "-xf";

system("tar $decompressflag $TAR_FLAGS $tarfile >$tarlist 2>&1");
393
$exit_status = $? >> 8;
394
umask($oumask);
395 396 397
if ($copymode) {
    unlink($copyfile);
}
398

399
if ($unknownuser && $GNU_TAR) {
400 401 402 403
    open(TARIN, "< $tarlist");
    while (my $line = <TARIN>) {
	chop $line;
	if ($line =~ /^(.*)$/) {
404 405 406 407 408 409 410
	    my $file = $1;

	    # XXX hack for bsdtar
	    if (! -e "$installdir/$file" && $file =~ /^x (.*)$/) {
		$file = $1;
	    }
	    $line = "$installdir/$file";
411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
	    if (-e $line &&
		(my (undef,undef,undef,undef,$fuid,$fgid) = stat $line)) {
		if ($fuid < 100) {
		    if ($debug) {
			print STDERR "exempt file $line $fuid $fgid\n";
		    }
		}
		elsif (! getpwuid $fuid) {
		    print STDERR "install-tarfile: changing ownership of $line to $uuuid/$uugid";
		    chown($uuuid, $uugid, $line);
		}
		elsif (! getgrgid $fgid) {
		    print STDERR "install-tarfile: changing ownership of $line to $fuid/$uugid";
		    chown($fuid, $uugid, $line);
		}
		elsif ($debug) {
		    print STDERR "not changing $line $fuid $fgid\n";
		}
	    }
	    else {
		print STDERR "ok: $line\n";
	    }
	}
    }
    close(TARIN);
436 437
} elsif ($unknownuser) {
    warn("*** WARNING: -u option requires GNU tar, ignoring.\n");
438 439 440
}
unlink($tarlist);

441 442 443 444 445 446
#
# Recreate the index file if the install was okay.
#
if (!$exit_status) {
    WriteIdentFile();
}
447 448 449 450 451
exit($exit_status);

sub fatal {
    local($msg) = $_[0];

452
    if ($copymode && defined($copyfile) && -e $copyfile) {
453 454 455 456
	unlink($copyfile);
    }
    die("*** $0:\n".
	"    $msg\n");
457 458
}

459 460 461
#
# Get a tarfile from the server via tmcc and stash.
#
462
sub GetTarFile($$$$$$$)
463
{
464 465
    my ($tarfile, $copyfile, $usewget,
	$oldstamp, $oldmd5, $filestamp, $filemd5) = @_;
466 467 468
    my $buf;

    #
469
    # If copying via NFS, must watch for read errors and retry.
470
    #
471
    if (! $usewget) {
472 473 474 475 476 477
	#
	# Compare timestamp. If no change, we are done. 
	#
	my (undef,undef,undef,undef,undef,undef,undef,$bytelen,
	    undef,$stamp) = stat($tarfile);

478
	if (defined($oldstamp) && $oldstamp >= $stamp && !$force) {
479 480 481 482
	    print STDOUT "Timestamp ($stamp) for $tarfile unchanged!\n"
		if ($debug);
	    return 1;
	}
483
	
484 485 486 487
	# Must do this for caller so that if the MD5 has not changed,
	# the caller can update the timestamp in the ident file.
	$$filestamp = $stamp;
	
488 489
	open(TMCC, "< $tarfile")
	    or fatal("Could not open tarfile on server!");
490

491 492
	binmode TMCC;

493
	#
494
	# Open the target file and start dumping the data in.
495
	#
496 497
	open(JFILE, "> $copyfile")
	    or fatal("Could not open local file $copyfile: $!");
498

499 500
	binmode JFILE;

501 502 503 504 505
	#
	# Deal with NFS read failures
	#
	my $foffset = 0;
	my $retries = 5;
506

507 508
	while ($bytelen) {
	    my $rlen = sysread(TMCC, $buf, 8192);
509

510 511 512 513
	    if (! defined($rlen)) {
		#
		# If we are copying the file via NFS, retry a few times
		# on error to avoid the changing-exports-file server problem.
514 515 516 517 518 519 520
		if ($retries > 0 && sysseek(TMCC, $foffset, 0)) {
		    warn("*** WARNING retrying read of $tarfile ".
			 "at offset $foffset\n");
		    $retries--;
		    sleep(2);
		    next;
		}
521 522 523 524 525 526 527 528 529 530 531 532 533 534
		fatal("Error reading tarball $tarfile: $!");
	    }
	    if ($rlen == 0) {
		last;
	    }
	    if (! syswrite(JFILE, $buf)) {
		fatal("Error writing tarfile $copyfile: $!");
	    }
	    $foffset += $rlen;
	    $bytelen -= $rlen;
	    $retries = 5;
	}
	close(JFILE);
	close(TMCC);
535 536 537 538 539

	#
	# Compare md5.
	#
	my $md5 = GetMD5($copyfile);
540
	if (defined($oldmd5) && $oldmd5 eq $md5 && !$force) {
541 542 543 544 545
	    print STDOUT "MD5 ($md5) for $tarfile unchanged!\n"
		if ($debug);
	    return 2;
	}
	$$filemd5   = $md5;
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
    }
    else {
	#
	# Need the nodeid and the keyhash. We allow the nodeid to be
	# overridden on the command line, but thats just a debugging
	# feature.
	#
	if (!defined($nodeid)) {
	    #
	    # Eventually, use tmcc which will cache the result. 
	    # 
	    open(FD, "< " . TMNODEID()) or
		fatal("Could not open ". TMNODEID() . ": $!");
	    $nodeid = <FD>;
	    close(FD);
	    fatal("Could not get our nodeid!")
		if (!defined($nodeid));

	    if ($nodeid =~ /^([-\w]+)$/) {
		$nodeid = $1;
566
	    }
567
	}
568 569 570 571 572 573 574 575 576 577 578
	#
	# Eventually, use tmcc which will cache the result. 
	# 
	open(FD, "< " . TMKEYHASH()) or
	    fatal("Could not open ". TMKEYHASH() . ": $!");
	$keyhash = <FD>;
	close(FD);
	fatal("Could not get our keyhash!")
		if (!defined($keyhash));
	if ($keyhash =~ /^([\w]+)$/) {
	    $keyhash = $1;
579
	}
580 581 582

	#
	# Lastly, need our boss node.
583 584
	#
	my ($www) = tmccbossname();
585 586 587

	if ($www =~ /^[-\w]+\.(.*)$/) {
	    $www = "www.${1}";
588
	}
589 590 591
	else {
	    fatal("Tainted bossinfo $www!");
	}
592 593 594
	$www  = "https://${www}";
	#$www = "https://${www}/dev/stoller";
	#$www = "http://golden-gw.ballmoss.com:9876/~stoller/testbed";
595 596 597 598

	#
	# Okay, run wget with the proper arguments. 
	#
599
	my $cmd = "wget -nv -O $copyfile ".
600
	          ($debug ? "--server-response " : "") .
601 602 603 604 605 606
	          "'${www}/spewrpmtar.php3".
	          "?nodeid=${nodeid}&file=${tarfile}&key=${keyhash}" .
		  (defined($oldstamp) ? "&stamp=$oldstamp" : "") .
		  (defined($oldmd5)   ? "&md5=$oldmd5" : "") .
		  "'";

607 608 609 610 611
    
	if ($debug) {
	    print STDERR "$cmd\n";
	}

612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
	#
	# We need to read back the response to see if the file was
	# unchanged. This is dumb; why doesn't wget exit with reasonable
	# error codes?
	#
	my $nochange = 0;
	if (!open(WGET, "$cmd 2>&1 |")) {
	    fatal("Cannot start wget: $!\n");
	}
	while (<WGET>) {
	    print $_
		if ($debug);
	    
	    # Ick!
	    if ($_ =~ /^.* ERROR 304.*$/i) {
		$nochange = 1;
	    }
	}
	if (! close(WGET)) {
	    if ($?) {
		fatal("Could not retrieve $tarfile from $www")
		    if (!$nochange);
		# Otherwise, not modifed. 
		print STDOUT "Timestamp for $tarfile unchanged!\n"
		    if ($debug);
		return 1;
	    }
	    else {
		fatal("Error closing wget pipe: $!\n");
	    }
	}
	# Must do this for caller so that if the MD5 has not changed,
	# the caller can update the timestamp in the ident file.
	#
	# Always use GM time for this. The server expects it.
	$$filestamp = mktime(gmtime(time()));
	
	#
	# We got a file. Compare the MD5 now. 
	#
	my $md5 = GetMD5($copyfile);
653
	if (defined($oldmd5) && $oldmd5 eq $md5 && !$force) {
654 655 656 657 658
	    print STDOUT "MD5 ($md5) for $tarfile unchanged!\n"
		if ($debug);
	    return 2;
	}
	$$filemd5 = $md5;
659 660 661
    }
    return 0;
}
662 663 664 665 666 667 668 669 670

#
# Get MD5 of file.
#
sub GetMD5($)
{
    my ($file) = @_;
    my $md5;

671
    if ($OSNAME eq "linux" || $OSNAME eq "cygwin") {
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
	$md5 = `md5sum $file`;
    
	if ($md5 =~ /^([\w]*)\s*.*$/) {
	    $md5 = $1;
	}
	else {
	    fatal("Bad MD5 for $file: $md5.");
	}
    }
    elsif ($OSNAME eq "freebsd") {
	$md5 = `md5 -q $file`;
    
	if ($md5 =~ /^([\w]*)$/) {
	    $md5 = $1;
	}
	else {
	    fatal("Bad MD5 for $file: $md5.");
	}
    }
    else {
	fatal("Do not know how to compute MD5s!");
    }
    return $md5;
}

#
# Recreate the ident file.
#
sub WriteIdentFile()
{
    if (!open(IDENT, "> $IDENTFILE")) {
	fatal("Could not open $IDENTFILE for writing: $!");
    }
    foreach my $id (@identlines) {
	print IDENT "$id\n";
    }
    close(IDENT);
}