Commit 763c6aca authored by Mike Hibler's avatar Mike Hibler

Client half of the fetch-tarballs-via-the-web change.

For every tarball and rpm, tmcd will now pass a SERVER= string as well
telling the client where the file should be downloaded from (if using
the web rather than NFS). Right now this value is the same for all
tarballs and rpms, and is hardwired in tmcd as either "www" (if
SPEWFROMOPS=0) or "users" (if 1). Note: BUMPED THE TMCC VERSION NUMBER
for this.

Made a common routine for doing an error-check-and-retry copy of a file
across "racy" NFS. This is used by install-{tarfile,rpm} and rc.topomap.
parent b4c12259
......@@ -46,4 +46,4 @@
* it in clientside/tmcc/common/libsetup.pm!
*/
#define DEFAULT_VERSION 2
#define CURRENT_VERSION 35
#define CURRENT_VERSION 36
......@@ -43,10 +43,10 @@ BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
sub usage()
{
print STDOUT "Usage: install-rpm [-d] [-ct] [-n nodeid] <filename>\n";
print STDOUT "Usage: install-rpm [-d] [-ct] [-n nodeid] [-S server] <filename>\n";
exit(-1);
}
my $optlist = "dctn:";
my $optlist = "dctn:S:";
#
# Turn off line buffering on output
......@@ -64,6 +64,7 @@ delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
my $IDENTFILE = "$DBDIR/testbed.rpms";
my $rpm = "";
my $usewget = 0;
my $wgetserver = "";
my $copymode = 0;
my $debug = 0;
my $copyfile;
......@@ -81,7 +82,7 @@ use libsetup;
use libtmcc;
# Protos
sub GetRPMFile($$$$$$$);
sub GetRPMFile($$$$$$$$);
sub GetMD5($);
sub WriteIdentFile();
......@@ -110,6 +111,12 @@ if (defined($options{"t"})) {
$usewget = 1;
$copymode = 1;
}
if (defined($options{"S"})) {
$wgetserver = $options{"S"};
if ($wgetserver =~ /^([-\w\.]+)$/) {
$wgetserver = $1;
}
}
if (defined($options{"n"})) {
$nodeid = $options{"n"};
if ($nodeid =~ /^([-\w]+)$/) {
......@@ -190,6 +197,7 @@ if (! $copymode) {
print STDOUT "RPM $rpm has already been installed!\n";
exit(1);
}
#
# Otherwise compare MD5.
#
......@@ -216,7 +224,7 @@ else {
# Returns >0 if server copy has not been modifed.
# Returns =0 if okay to install, and gives us new stamp/md5.
#
if (GetRPMFile($rpm, $copyfile, $usewget,
if (GetRPMFile($rpm, $copyfile, $usewget, $wgetserver,
$oldstamp, $oldmd5, \$filestamp, \$filemd5)) {
print STDOUT "RPM $rpm has already been installed!\n";
if (defined($filestamp) && $filestamp != $oldstamp) {
......@@ -270,78 +278,41 @@ sub fatal {
#
# Get an RPM from the server via tmcc and stash.
#
sub GetRPMFile($$$$$$$)
sub GetRPMFile($$$$$$$$)
{
my ($rpm, $copyfile, $usewget,
my ($rpm, $copyfile, $usewget, $wgetserver,
$oldstamp, $oldmd5, $filestamp, $filemd5) = @_;
my $buf;
#
# If copying via NFS, must watch for read errors and retry.
#
if (! $usewget) {
print STDOUT "Copying RPM $rpm across NFS\n"
if ($debug);
#
# Compare timestamp. If no change, we are done.
#
my (undef,undef,undef,undef,undef,undef,undef,$bytelen,
undef,$stamp) = stat($rpm);
my (undef,undef,undef,undef,undef,undef,undef,undef,undef,$stamp) =
stat($rpm);
if (defined($oldstamp) && $oldstamp >= $stamp) {
print STDOUT "Timestamp ($stamp) for $rpm unchanged!\n"
if ($debug);
return 1;
}
#
# Must do this for caller so that if the MD5 has not changed,
# the caller can update the timestamp in the ident file.
#
$$filestamp = $stamp;
open(TMCC, "< $rpm")
or fatal("Could not open rpm on server!");
binmode TMCC;
#
# Open the target file and start dumping the data in.
#
open(JFILE, "> $copyfile")
or fatal("Could not open local file $copyfile: $!");
binmode JFILE;
#
# Deal with NFS read failures
# If copying via NFS, use special copy routine which retries on error.
#
my $foffset = 0;
my $retries = 5;
while ($bytelen) {
my $rlen = sysread(TMCC, $buf, 8192);
if (! defined($rlen)) {
#
# If we are copying the file via NFS, retry a few times
# on error to avoid the changing-exports-file server problem.
if ($retries > 0 && sysseek(TMCC, $foffset, 0)) {
warn("*** WARNING retrying read of $rpm ".
"at offset $foffset\n");
$retries--;
sleep(2);
next;
}
fatal("Error reading tarball $rpm: $!");
}
if ($rlen == 0) {
last;
}
if (! syswrite(JFILE, $buf)) {
fatal("Error writing rpm $copyfile: $!");
}
$foffset += $rlen;
$bytelen -= $rlen;
$retries = 5;
if (!copyfilefromnfs($rpm, $copyfile, 1)) {
unlink($copyfile);
fatal("Could not copy RPM $rpm from server!");
}
close(JFILE);
close(TMCC);
#
# Compare md5.
......@@ -355,6 +326,9 @@ sub GetRPMFile($$$$$$$)
$$filemd5 = $md5;
}
else {
print STDOUT "Fetching RPM $rpm from $wgetserver via HTTP\n"
if ($debug);
#
# Need the nodeid and the keyhash. We allow the nodeid to be
# overridden on the command line, but thats just a debugging
......@@ -389,30 +363,30 @@ sub GetRPMFile($$$$$$$)
}
#
# Lastly, need our boss node.
# Lastly, need the server.
# For compat, use boss (aka, www) if not specified.
#
my ($www) = tmccbossname();
if ($www =~ /^[-\w]+\.(.*)$/) {
$www = "www.${1}";
}
else {
fatal("Tainted bossinfo $www!");
if ($wgetserver eq "") {
($wgetserver) = tmccbossname();
if ($wgetserver =~ /^[-\w]+\.(.*)$/) {
$wgetserver = "www.${1}";
}
else {
fatal("Tainted bossinfo $wgetserver!");
}
}
$www = "https://${www}";
#$www = "https://${www}/dev/stoller";
#$www = "http://golden-gw.ballmoss.com:9876/~stoller/testbed";
my $www = "https://$wgetserver";
#
# Okay, run wget with the proper arguments.
#
my $cmd = "wget -nv -O $copyfile ".
my $cmd = "wget -nv -O --no-check-certificate $copyfile ".
($debug ? "--server-response " : "") .
"'${www}/spewrpmtar.php3".
"?nodeid=${nodeid}&file=${rpm}&key=${keyhash}".
(defined($oldstamp) ? "&stamp=$oldstamp" : "") .
(defined($oldmd5) ? "&md5=$oldmd5" : "") .
"'";
"'";
if ($debug) {
print STDERR "$cmd\n";
......
......@@ -41,7 +41,7 @@ BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
sub usage()
{
print STDOUT
"Usage: install-tarfile [-hVdfvct] [-n nodeid] [-u user] <installdir> ".
"Usage: install-tarfile [-hVdfvct] [-n nodeid] [-u user] [-S server] <installdir> ".
"<filename>\n".
" install-tarfile [-l]\n".
"Options:\n".
......@@ -55,6 +55,7 @@ sub usage()
" -n nodeid Override the default node ID when downloading from Emulab\n".
" -u user User that should own files with unknown uid/gid\n".
" -l List the currently installed tar files and exit\n".
" -S server Specifies the Emulab server (for -t)\n".
"\n".
"Required Arguments:".
" installdir The absolute path of the install directory.\n".
......@@ -62,7 +63,7 @@ sub usage()
exit(-1);
}
my $optlist = "hVlvcdftn:u:";
my $optlist = "hVlvcdftn:u:S:";
#
# Turn off line buffering on output
......@@ -84,6 +85,7 @@ my $installdir = "/";
my $unknownuser = "";
my $force = 0;
my $usewget = 0;
my $wgetserver = "";
my $listmode = 0;
my $copymode = 0;
my $verifymode = 0;
......@@ -105,7 +107,7 @@ use libsetup;
use libtmcc;
# Protos
sub GetTarFile($$$$$$$);
sub GetTarFile($$$$$$$$);
sub GetMD5($);
sub WriteIdentFile();
......@@ -144,6 +146,15 @@ if (defined($options{"t"})) {
$copymode = 1;
$usewget = 1;
}
if (defined($options{"S"})) {
$wgetserver = $options{"S"};
if ($wgetserver =~ /^([-\w\.]+)$/) {
$wgetserver = $1;
}
else {
fatal("Tainted server: $wgetserver");
}
}
if (defined($options{"v"})) {
$verifymode = 1;
}
......@@ -340,7 +351,7 @@ else {
# Returns >0 if server copy has not been modifed.
# Returns =0 if okay to install, and gives us new stamp/md5.
#
if (GetTarFile($tarfile, $copyfile, $usewget,
if (GetTarFile($tarfile, $copyfile, $usewget, $wgetserver,
$oldstamp, $oldmd5, \$filestamp, \$filemd5)) {
print STDOUT "Tarfile $tarfile has already been installed!\n";
if (defined($filestamp) && $filestamp != $oldstamp) {
......@@ -483,79 +494,41 @@ sub fatal {
#
# Get a tarfile from the server via tmcc and stash.
#
sub GetTarFile($$$$$$$)
sub GetTarFile($$$$$$$$)
{
my ($tarfile, $copyfile, $usewget,
my ($tarfile, $copyfile, $usewget, $wgetserver,
$oldstamp, $oldmd5, $filestamp, $filemd5) = @_;
my $buf;
#
# If copying via NFS, must watch for read errors and retry.
#
if (! $usewget) {
print STDOUT "Copying tarball $tarfile across NFS\n"
if ($debug);
#
# Compare timestamp. If no change, we are done.
#
my (undef,undef,undef,undef,undef,undef,undef,$bytelen,
undef,$stamp) = stat($tarfile);
my (undef,undef,undef,undef,undef,undef,undef,undef,undef,$stamp) =
stat($tarfile);
if (defined($oldstamp) && $oldstamp >= $stamp && !$force) {
print STDOUT "Timestamp ($stamp) for $tarfile unchanged!\n"
if ($debug);
return 1;
}
#
# Must do this for caller so that if the MD5 has not changed,
# the caller can update the timestamp in the ident file.
#
$$filestamp = $stamp;
open(TMCC, "< $tarfile")
or fatal("Could not open tarfile on server!");
binmode TMCC;
#
# Open the target file and start dumping the data in.
#
open(JFILE, "> $copyfile")
or fatal("Could not open local file $copyfile: $!");
binmode JFILE;
#
# Deal with NFS read failures
# If copying via NFS, use special copy routine which retries on error.
#
my $foffset = 0;
my $retries = 5;
while ($bytelen) {
my $rlen = sysread(TMCC, $buf, 8192);
if (! defined($rlen)) {
#
# If we are copying the file via NFS, retry a few times
# on error to avoid the changing-exports-file server problem.
if ($retries > 0 && sysseek(TMCC, $foffset, 0)) {
warn("*** WARNING retrying read of $tarfile ".
"at offset $foffset\n");
$retries--;
sleep(2);
next;
}
fatal("Error reading tarball $tarfile: $!");
}
if ($rlen == 0) {
last;
}
if (! syswrite(JFILE, $buf)) {
fatal("Error writing tarfile $copyfile: $!");
}
$foffset += $rlen;
$bytelen -= $rlen;
$retries = 5;
if (!copyfilefromnfs($tarfile, $copyfile, 1)) {
unlink($copyfile);
fatal("Could not copy tarfile $tarfile from server!");
}
close(JFILE);
close(TMCC);
#
# Compare md5.
......@@ -569,6 +542,9 @@ sub GetTarFile($$$$$$$)
$$filemd5 = $md5;
}
else {
print STDOUT "Fetching tarball $tarfile from $wgetserver via HTTP\n"
if ($debug);
#
# Need the nodeid and the keyhash. We allow the nodeid to be
# overridden on the command line, but thats just a debugging
......@@ -603,31 +579,31 @@ sub GetTarFile($$$$$$$)
}
#
# Lastly, need our boss node.
# Lastly, need the server.
# For compat, use boss (aka, www) if not specified.
#
my ($www) = tmccbossname();
if ($www =~ /^[-\w]+\.(.*)$/) {
$www = "www.${1}";
}
else {
fatal("Tainted bossinfo $www!");
if ($wgetserver eq "") {
($wgetserver) = tmccbossname();
if ($wgetserver =~ /^[-\w]+\.(.*)$/) {
$wgetserver = "www.${1}";
}
else {
fatal("Tainted bossinfo $wgetserver!");
}
}
$www = "https://${www}";
#$www = "https://${www}/dev/stoller";
#$www = "http://golden-gw.ballmoss.com:9876/~stoller/testbed";
my $www = "https://$wgetserver";
#
# Okay, run wget with the proper arguments.
#
my $cmd = "wget -nv -O $copyfile ".
"--no-check-certificate ".
($debug ? "--server-response " : "") .
"'${www}/spewrpmtar.php3".
"?nodeid=${nodeid}&file=${tarfile}&key=${keyhash}" .
(defined($oldstamp) ? "&stamp=$oldstamp" : "") .
(defined($oldmd5) ? "&md5=$oldmd5" : "") .
"'";
if ($debug) {
print STDERR "$cmd\n";
......@@ -645,7 +621,7 @@ sub GetTarFile($$$$$$$)
while (<WGET>) {
print $_
if ($debug);
# Ick!
if ($_ =~ /^.* ERROR 304.*$/i) {
$nochange = 1;
......
......@@ -111,7 +111,7 @@ sub doboot()
{
my @rpms = ();
my $errors = 0;
print STDOUT "Checking Testbed RPM configuration ... \n";
if (tmcc(TMCCCMD_RPM, undef, \@rpms) < 0) {
......@@ -128,19 +128,50 @@ sub doboot()
#
# Note: Windows also uses the -t option since root does not have access
# to NFS mounted FSes.
#
my $installoption = ((REMOTE() || SHADOW() || WINDOWS()) ? "-t" : "-c");
#
my $MAXCPSIZE = (256 * 1024 * 1024);
my $useweb = (REMOTE() || SHADOW() || WINDOWS()) ? 1 : 0;
foreach my $rpm (@rpms) {
if ($rpm =~ /RPM=(.+)/) {
my $rpmline = sprintf($RPMINSTALL, $installoption, $1);
if ($rpm =~ /(?:SERVER=(\S+)\s+)?RPM=(.+)/) {
my $server = $1;
my $rpm = $2;
my $installoption = $useweb ? "-t" : "-c";
#
# We also use -t if the file is large. This way we avoid
# thrashing the NFS server's link for an extended period
# (in particular, if we are using UDP-based NFS) and also
# avoids the stupid server race mentioned above. But we only
# do this if SERVER=user, since otherwise boss will have to
# get the file across NFS from the server before sending it
# to us via the web, which doesn't save any wear and tear
# on the NFS server link.
#
# Why "user" instead of "fs"? Because in a configuration
# where the fs node is seperate from the user node, we don't
# currently install a web server on fs.
#
if (!$useweb && $server && $server =~ /^users\./ &&
-s "$rpm" > $MAXCPSIZE) {
print STDOUT "NOTICE: downloading large RPM via https\n";
$installoption = "-t";
}
if ($server && $installoption eq "-t") {
$installoption .= " -S $server";
}
my $rpmline = sprintf($RPMINSTALL, $installoption, $rpm);
print STDOUT "Installing RPM $1\n";
$server = "www" if (!$server);
print STDOUT "$server:$rpm: Installing RPM\n";
my $stamp = time();
if (system($rpmline)) {
if ($? >> 8 == 255) {
$errors++;
}
}
print STDOUT "$server:$rpm: finished in ", time() - $stamp, " seconds\n";
}
else {
warning("Bad RPM line: $rpm\n");
......
......@@ -128,19 +128,51 @@ sub doboot()
#
# Note: Windows also uses the -t option since root does not have access
# to NFS mounted FSes.
#
my $installoption = ((REMOTE() || SHADOW() || WINDOWS()) ? "-t" : "-c");
#
my $MAXCPSIZE = (256 * 1024 * 1024);
my $useweb = (REMOTE() || SHADOW() || WINDOWS()) ? 1 : 0;
foreach my $tarball (@tarballs) {
if ($tarball =~ /DIR=(.+)\s+TARBALL=(.+)/) {
my $tbline = sprintf($TARINSTALL, $installoption, $1, $2);
if ($tarball =~ /(?:SERVER=(\S+)\s+)?DIR=(.+)\s+TARBALL=(.+)/) {
my $server = $1;
my $idir = $2;
my $tball = $3;
my $installoption = $useweb ? "-t" : "-c";
#
# We also use -t if the file is large. This way we avoid
# thrashing the NFS server's link for an extended period
# (in particular, if we are using UDP-based NFS) and also
# avoids the stupid server race mentioned above. But we only
# do this if SERVER=user, since otherwise boss will have to
# get the file across NFS from the server before sending it
# to us via the web, which doesn't save any wear and tear
# on the NFS server link.
#
# Why "user" instead of "fs"? Because in a configuration
# where the fs node is seperate from the user node, we don't
# currently install a web server on fs.
#
if (!$useweb && $server && $server =~ /^users\./ &&
-s "$tball" > $MAXCPSIZE) {
print STDOUT "NOTICE: downloading large tarball via https\n";
$installoption = "-t";
}
if ($server && $installoption eq "-t") {
$installoption .= " -S $server";
}
my $tbline = sprintf($TARINSTALL, $installoption, $idir, $tball);
print STDOUT "Installing Tarball $2 in dir $1\n";
$server = "www" if (!$server);
print STDOUT "$server:$tball: Installing tarball in $idir\n";
my $stamp = time();
if (system($tbline)) {
if ($? >> 8 == 255) {
$errors++;
}
}
print STDOUT "$server:$tball: finished in ", time() - $stamp, " seconds\n";
}
else {
warning("Bad Tarballs line: $tarball\n");
......
#!/usr/bin/perl -w
#
# Copyright (c) 2004-2010 University of Utah and the Flux Group.
# Copyright (c) 2004-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -170,15 +170,15 @@ sub dotopomap()
unlink("$templtp");
if (! (USESFS() || WINDOWS() || REMOTEDED() || (FSRVTYPE() eq "LOCAL"))) {
if (!forcecopy($topomap, $tempmap)) {
if (!copyfilefromnfs($topomap, $tempmap, 0)) {
warning("NFS cp of $topomap failed, will try tmcc ...");
unlink("$tempmap");
}
if (!forcecopy($ltmap, $templt)) {
if (!copyfilefromnfs($ltmap, $templt, 0)) {
warning("NFS cp of $ltmap failed, will try tmcc ...");
unlink("$templt");
}
if (!forcecopy($ltpmap, $templtp)) {
if (!copyfilefromnfs($ltpmap, $templtp, 0)) {
warning("NFS cp of $ltpmap failed, will try tmcc ...");
unlink("$templtp");
}
......
......@@ -40,7 +40,7 @@ use Exporter;
gettraceconfig genhostsfile getmotelogconfig calcroutes fakejailsetup
getlocalevserver genvnodesetup getgenvnodeconfig stashgenvnodeconfig
getlinkdelayconfig getloadinfo getbootwhat getnodeattributes
forcecopy getnodeuuid getarpinfo
copyfilefromnfs getnodeuuid getarpinfo
getmanifest fetchmanifestblobs runbootscript runhooks
build_fake_macs
......@@ -81,7 +81,7 @@ use librc;
# IMPORTANT NOTE: if you change the version here, you must also change it
# in clientside/lib/tmcd/tmcd.h!
#
sub TMCD_VERSION() { 35; };
sub TMCD_VERSION() { 36; };
libtmcc::configtmcc("version", TMCD_VERSION());
# Control tmcc timeout.
......@@ -2207,31 +2207,95 @@ sub getbootwhat($)
}
#
# Do everything in our power to copy a file.
# The main "specialness" about this function is that it tries to work
# around the old FreeBSD NFS server race with changing the exports list--
# we retry the copy several times before failing.
# Returns one on success, zero on failure.
# Copy a file from an NFS filesystem.
# Supports retry on errors when the NFS filesystem is known to be "racy."
# On error, it is up to the caller to remove the target.
# Returns 1 on success, 0 otherwise.
#
sub forcecopy($$)
sub copyfilefromnfs($$$)
{
my ($ffile, $tfile) = @_;
my ($ffile, $tfile, $showerrs) = @_;
my $tries = 1;
#
# If the file server has NFS races, we try operations multiple
# times in case we hit the EPERM window.
# If the file server doesn't have the BSD mountd NFS export race
# we just use the system cp command.
#
if (FSRVTYPE() eq "NFS-RACY") {
$tries = 5;
if (FSRVTYPE() ne "NFS-RACY") {
my $redir = ">/dev/null 2>&1";
if ($showerrs) {
$redir = "";
}
if (system("cp -fp $ffile $tfile $redir") == 0) {
return 1;
}
return 0;
}
for (my $i = 0; $i < $tries; $i++) {
if (system("cp -fp $ffile $tfile >/dev/null 2>&1") == 0) {
return 1;
if (!open(IN, "< $ffile")) {
if ($showerrs) {
print STDERR "$ffile: could not open for read: $!\n";
}
return 0;
}
return 0;
binmode IN;
if (!open(OUT, "> $tfile")) {
if ($showerrs) {
print STDERR "$tfile: could not open for write: $!\n";
}
return 0;
}
binmode OUT;
#
# Deal with NFS read failures
#
my $foffset = 0;
my $retries = 5;
my $rval = 1;
while (1) {
my $buf;
my $rlen = sysread(IN, $buf, 8192);
if (!defined($rlen)) {
#
# If we are copying the file via NFS, retry a few times
# on error to avoid the changing-exports-file server problem.
#
if ($retries > 0 && sysseek(IN, $foffset, 0)) {
if ($showerrs) {
print STDERR "*** WARNING retrying read of $ffile ".
"at offset $foffset\n";
}
$retries--;
sleep(1);
next;
}
if ($showerrs) {
print STDERR "$ffile: error reading file: $!\n";
}
$rval = 0;
last;
}
if ($rlen == 0) {
last;
}
if (!syswrite(OUT, $buf)) {
if ($showerrs) {
print STDERR "$tfile: error writing file: $!\n";
}
$rval = 0;
last;
}
$foffset += $rlen;
$retries = 5;
}
close(OUT);
close(IN);
return $rval;
}
my %fwvars = ();
......
......@@ -56,3 +56,5 @@
#undef HAVE_MEZZANINE
#undef OPSVM_ENABLE
#undef OPSVM_MOUNTPOINT
#undef SPEWFROMOPS
......@@ -5078,6 +5078,11 @@ done
......@@ -5494,6 +5499,12 @@ _ACEOF
#define OPSVM_MOUNTPOINT $OPSVM_MOUNTPOINT
_ACEOF
fi
if test $SPEWFROMOPS -eq 1; then
cat >>confdefs.h <<_ACEOF
#define SPEWFROMOPS 1
_ACEOF
fi
LOG_TESTBED=`echo "LOG_$TBLOGFACIL" | tr a-z A-Z`
......