Merge branch 'master' into arm64

clientside/os/create-versioned-image

@@ -355,6 +355,10 @@ foreach my $iid (sort keys %iinfo) {
 	print STDERR "Must specify imagename\n";
 	exit(0);
     }
+    if (!defined($iinfo{$iid}{'method'})) {
+	print STDERR "Must specify method\n";
+	exit(0);
+    }
     if ($iinfo{$iid}{'method'} eq "frisbee") {
 	$dofrisbee++;
 	if (!defined($iinfo{$iid}{'server'})) {

clientside/tmcc/linux/libvnode.pm

@@ -31,6 +31,7 @@ use Exporter;
 	      existsBridge findBridge findBridgeIfaces
               downloadImage getKernelVersion createExtraFS
               forwardPort removePortForward lvSize DoIPtables restartDHCP
+              computeStripeSize
             );
 
 use Data::Dumper;
@@ -318,6 +319,30 @@ isdisk:
     return %retval;
 }
 
+#
+# Attempt to compute a stripe size, based on physical number of
+# devices in the volume group.
+#
+sub computeStripeSize($)
+{
+    my ($vgname) = @_;
+    my $command  = "vgdisplay -v $vgname 2>/dev/null | ".
+	"awk '/PV Name/ {print \$3}'";
+    my %devices = ();
+    my $count   = 1;
+    
+    if (open(PFD, "$command |")) {
+	while (my $line = <PFD>) {
+	    if ($line =~ /\/dev\/(\w{2,3})/) {
+		$devices{$1} = $1;
+	    }
+	}
+	close(PFD);
+	$count = scalar(keys(%devices));
+    }
+    return $count;
+}
+
 my %if2mac = ();
 my %mac2if = ();
 my %ip2if = ();

clientside/tmcc/linux/prepare

@@ -43,6 +43,7 @@ my $NTPDRIFT1   = "/etc/ntp/drift";
 my $NTPDRIFT2   = "/var/lib/ntp/drift";
 my $SFSUSERS    = "/etc/sfs/sfs_users";
 my $SFSHOSTKEY  = "/etc/sfs/sfs_host_key";
+my $SSHDCONFIG  = "/etc/ssh/sshd_config";
 my $VARACCTDIR  = "/var/account";
 my $VARACCTSDIR = "/var/log/sa";
 my $IFTAB       = "/etc/iftab";
@@ -416,6 +417,14 @@ foreach my $file (@SOCKETS) {
     }
 }
 
+print "Removing extra Ports from $SSHDCONFIG\n";
+if (-e $SSHDCONFIG) {
+    # Remove high numbered ports.
+    system("sed -i.bak -e '/^Port [0-9]\{3,\}/d' $SSHDCONFIG");
+    warn("*** Could not remove Port lines\n")
+	if ($?);
+}
+
 # Leave this print statement here; create_image depends on it.
 print "prepare ran successfully!\n";
 exit 0;

clientside/tmcc/linux/xen/create-xen-image

@@ -30,7 +30,8 @@ BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
 
 my $VIMAGE	= "/usr/local/bin/create-versioned-image";
 my $zipper      = "/usr/local/bin/imagezip";
-my $uploader    = "/usr/local/bin/frisupload";
+my $uploader    = "$BINDIR/frisupload";
+my $uploader_alt= "/usr/local/bin/frisupload";
 my $VNODESETUP	= "$BINDIR/vnodesetup";
 my $CAPTURE	= "$BINDIR/capturevm.pl";
 my $EXTRAFS	= "/capture";
@@ -66,6 +67,10 @@ use libsetup;
 use libvnode_xen;
 use libutil;
 
+# Location confusion.
+$uploader = $uploader_alt
+    if (! -x $uploader);
+
 #
 # No configure vars.
 #

clientside/tmcc/linux/xen/libvnode_xen.pm

@@ -222,6 +222,9 @@ my $RTTABLES       = "/etc/iproute2/rt_tables";
 # Temporary; later kernel version increases this.
 my $MAXROUTETTABLE = 255;
 
+# Striping
+my $STRIPE_COUNT   = 1;
+
 # Whether or not to use only unpartitioned (unused) disks to form the Xen VG.
 my $LVM_FULLDISKONLY = 0;
 
@@ -310,16 +313,10 @@ sub init($)
     }
     getXenInfo();
 
-    #
-    # The 4.4 image has to use the openvswitch package instead of building
-    # from source since the current release does not support the kernel.
-    # And the package is installed in a different place.
-    #
-    if ($xeninfo{xen_minor} >= 4) {
-	$OVSCTL   = "/usr/bin/ovs-vsctl";
-	$OVSSTART = "/usr/share/openvswitch/scripts/ovs-ctl";
+    # Compute the strip size for new lvms.
+    if (-e "/var/run/xen.ready") {
+	$STRIPE_COUNT = computeStripeSize($VGNAME);
     }
-
     return 0;
 }
 
@@ -470,9 +467,9 @@ sub rootPreConfig($)
     }
 
     #
-    # Make sure pieces are at least a GiB.
+    # Make sure pieces are at least a 5GiB.
     #
-    my %devs = libvnode::findSpareDisks(1 * 1024);
+    my %devs = libvnode::findSpareDisks(5 * 1024);
 
     #
     # Turn on write caching. Hacky. 
@@ -580,7 +577,8 @@ sub rootPreConfig($)
 		" $MAX_VNODES VMs ($size < $XEN_MIN_VGSIZE)\n";
 	}
     }
-
+    $STRIPE_COUNT = computeStripeSize($VGNAME);
+    
     #
     # Make sure our volumes are active -- they seem to become inactive
     # across reboots
@@ -1484,8 +1482,10 @@ sub vnodePreConfig($$$$$){
 	# Testing a theory; remove all this iscsi stuff to see if that
 	# is causing problems with the control network interface going
 	# offline after boot.
-	mysystem2("/bin/rm -vf $vnoderoot/etc/init/*iscsi*");
-	mysystem2("/bin/rm -vf $vnoderoot/etc/init.d/*iscsi*");
+	if ($xeninfo{xen_minor} < 4) {
+	    mysystem2("/bin/rm -vf $vnoderoot/etc/init/*iscsi*");
+	    mysystem2("/bin/rm -vf $vnoderoot/etc/init.d/*iscsi*");
+	}
     }
     else {
 	#
@@ -2467,7 +2467,8 @@ sub CreatePrimaryDisk($$$$)
 	    $lv_size += $extrafs * (1024 * 1024);
 	}
     }
-    if (mysystem2("lvcreate -L ${lv_size}k -n $target $VGNAME")) {
+    if (mysystem2("lvcreate -i${STRIPE_COUNT} ".
+		  "   -L ${lv_size}k -n $target $VGNAME")) {
 	print STDERR "libvnode_xen: could not create disk for $target\n";
 	return -1;
     }

db/Image.pm.in

@@ -313,6 +313,13 @@ sub LookupByURL($$)
     }
     return undef;
 }
+# Image is local if the metadata_url is null.
+sub IsLocal($)
+{
+    my ($self) = @_;
+
+    return (defined($self->metadata_url()) ? 0 : 1);
+}
 
 #
 # Lookup a "system" osid, meaning one in the emulab-ops project.

protogeni/lib/GeniCredential.pm.in

@@ -1029,7 +1029,7 @@ sub HasActualPrivilege($$)
     my ($self, $p) = @_;
 
     return 0
-	if (!exists($self->{'capabilities'}));
+	if (!defined($self->{'capabilities'}));
 
     return exists($self->{'capabilities'}->{$p}) ? 1 : 0;
 }

protogeni/lib/GeniXML.pm.in

@@ -720,7 +720,14 @@ sub SetDiskImage($$)
 	    $diskref = AddElement("disk_image", $type);
 	}
     }
-    GeniXML::SetText("name", $diskref, $urn);
+    if (GeniHRN::IsValid($urn)) {
+	$diskref->removeAttribute("url");
+	GeniXML::SetText("name", $diskref, $urn);
+    }
+    else {
+	$diskref->removeAttribute("name");
+	GeniXML::SetText("url", $diskref, $urn);
+    }
     return $diskref;
 }
 

tbsetup/image_setup.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2003-2012 University of Utah and the Flux Group.
+# Copyright (c) 2003-2014 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -149,7 +149,15 @@ while (my ($vname, $osname) = $result->fetchrow()) {
     # fetch the image file, or refetch if it is stale.
     #
     my $image = Image->LookupByURL($url);
-    if (!defined($image)) {
+    if (defined($image)) {
+	#
+	# LookupByURL might return a local image, which of course
+	# we want to skip; later phases know its a local image.
+	#
+	next
+	    if ($image->IsLocal());
+    }
+    else {
 	my $opts = "";
 	$opts .= " -d"
 	    if ($debug);
@@ -162,9 +170,11 @@ while (my ($vname, $osname) = $result->fetchrow()) {
     next
 	if ($verify);
 
-    $image = Image->LookupByURL($url);
     if (!defined($image)) {
-	tbdie("Could not look up image object for $url\n");
+	$image = Image->LookupByURL($url);
+	if (!defined($image)) {
+	    tbdie("Could not look up image object for $url\n");
+	}
     }
     next
 	if (! $getimages);

utils/create_image.in

@@ -720,10 +720,12 @@ if (defined($webtask)) {
 }
 
 #
-# Be sure to kill off running frisbee. If a node is trying to load that
-# image, well tough.
+# Slight problem here; killing off the running frisbeed will cause
+# any experiment trying to load that image, to blow up. So we
+# do not want to do this for system images, but for project images
+# this is generally okay to do. 
 #
-if (0) {
+if ($pid ne TBOPSPID()) {
     system("$friskiller -k $imageid");
     if ($?) {
 	fatal("Could not kill running frisbee for $imageid!");