More ZFS_NOEXPORT improvements. Okay I mean kludges.

account/tbacct.in

@@ -160,6 +160,7 @@ use libdb;
 use libtestbed;
 use User;
 use Project;
+use emutil;
 
 #
 # Function prototypes
@@ -474,6 +475,9 @@ sub AddUser()
 	    system("$SSH -host $CONTROL $ADDHOOK $user");
 	}
     }
+    $UID  = $SAVEUID;
+    $EUID = $UID;
+
     if ($WITHZFS) {
 	if ($ZFS_NOEXPORT) {
 	    #
@@ -486,23 +490,16 @@ sub AddUser()
 	}
 	
 	#
-	# There is some lag before the automounter can mount the new
-	# volume. Lets delay until we can access the directory.
+	# There is some lag before the automounter can mount the new volume.
 	#
-	for (my $i = 0; $i < 10; $i++) {
-	    sleep(1)
-		if (! -r "$HOMEDIR/$user/.cshrc");
-	}
-	if (! -r "$HOMEDIR/$user/.cshrc") {
+	if (emutil::waitForMount("$HOMEDIR/$user") < 0) {
 	    fatal("Could not access new user directory");
 	}
     }
-    $UID = $SAVEUID;
 
     #
-    # Do the ssh thing. Invoke as real user for auditing.
+    # Do the ssh thing.
     #
-    $EUID = $UID;
     if ($user ne $PROTOUSER && system("$ADDKEY -i $user")) {
 	fatal("Could not generate initial ssh key for $user");
     }

db/emutil.pm.in

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2000-2014 University of Utah and the Flux Group.
+# Copyright (c) 2000-2015 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -770,5 +770,47 @@ sub TBDateStringLocal($)
     return POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime($date));
 }
 
+sub isMounted($)
+{
+    my ($dir) = @_;
+    my $rval  = 0;
+    my $MOUNT = "/sbin/mount";
+
+    #
+    # Grab the output of the mount command and parse.
+    #
+    if (! open(MOUNT, "$MOUNT|")) {
+	print "Cannot run mount command\n";
+	return 0;
+    }
+    while (<MOUNT>) {
+	if ($_ =~ /^([-\w\.\/:\(\)]+) on ([-\w\.\/]+) \((.*)\)$/) {
+	    # Search for nfs string in the option list.
+	    foreach my $opt (split(',', $3)) {
+		if ($opt eq "nfs" && $2 eq $dir) {
+		    $rval = 1;
+		}
+	    }
+	}
+    }
+    close(MOUNT);
+    return $rval;
+}
+
+sub waitForMount($;$)
+{
+    my ($dir, $delay) = @_;
+    $delay = 10 if (!defined($delay));
+
+    for (my $i = 0; $i < $delay; $i++) {
+	if (isMounted($dir)) {
+	    return 0;
+	}
+	sleep(1);
+	system("/bin/ls $dir >/dev/null 2>&1");
+    }
+    return -1;
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

tbsetup/exports_setup.in

@@ -269,8 +269,7 @@ if ($WITHZFS && $ZFS_NOEXPORT) {
     $active_result =
 	DBQueryFatal("select pid from project_stats ".
 		     "where ((UNIX_TIMESTAMP(now()) - ".
-		     "       UNIX_TIMESTAMP(last_activity)) <= $limit) and ".
-		     "      approved=1");
+		     "       UNIX_TIMESTAMP(last_activity)) <= $limit)");
     while (my ($pid) = $active_result->fetchrow_array()) {
 	$bossexports{"$projdir/$pid"}  = "$projdir/$pid";
 	$bossexports{"$groupdir/$pid"} = "$groupdir/$pid";

tbsetup/mkgroup.in

@@ -96,6 +96,7 @@ use libdb;
 use libtestbed;
 use User;
 use Group;
+use emutil;
 
 # Defined in libtestbed.
 my $PROJROOT    = PROJROOT();
@@ -227,11 +228,7 @@ if ($WITHZFS) {
     # There is some lag before the automounter can mount the new
     # volume. Lets delay until we can access the project directory.
     #
-    for (my $i = 0; $i < 10; $i++) {
-	sleep(1)
-	    if (! -r $projdir);
-    }
-    if (! -r $projdir) {
+    if (emutil::waitForMount($projdir) < 0) {
 	fatal("Could not access new project directory: $projdir");
     }
 }

tbsetup/mkproj.in

@@ -74,6 +74,8 @@ my $BUGDBSUPPORT  = @BUGDBSUPPORT@;
 my $OPSDBSUPPORT  = @OPSDBSUPPORT@;
 my $CVSSUPPORT    = @CVSSUPPORT@;
 my $MAILMANSUPPORT= @MAILMANSUPPORT@;
+my $WITHZFS       = @WITHZFS@;
+my $ZFS_NOEXPORT  = @ZFS_NOEXPORT@;
 my $ADDWIKIPROJ   = "$TB/sbin/addwikiproj";
 my $ADDBUGDBPROJ  = "$TB/sbin/addbugdbproj";
 my $ADDMMLIST     = "$TB/sbin/addmmlist";
@@ -103,6 +105,7 @@ use libdb;
 use libtestbed;
 use User;
 use Project;
+use emutil;
 
 my $PROJROOT     = PROJROOT();
 my $GRPROOT      = GROUPROOT();
@@ -335,7 +338,13 @@ my (undef,undef,$gid) = getgrnam($unix_name)
 #
 # Okay, do it.
 #
-if (! -e "$PROJROOT/$pid") {
+if ($WITHZFS) {
+    # Automounter lag after exports_setup
+    if (emutil::waitForMount("$PROJROOT/$pid") < 0) {
+	fatal("Could not access project directory");
+    }
+}
+elsif (! -e "$PROJROOT/$pid") {
     if (! mkdir("$PROJROOT/$pid", 0770)) {
 	fatal("Could not make directory $PROJROOT/$pid: $!");
     }
@@ -431,7 +440,13 @@ if ($CVSSUPPORT && !$isnonlocal) {
 #
 # Create groups directory.
 #
-if (! -e "$GRPROOT/$pid") {
+if ($WITHZFS) {
+    # Automounter lag after exports_setup
+    if (emutil::waitForMount("$GRPROOT/$pid") < 0) {
+	fatal("Could not access group directory");
+    }
+}
+elsif (! -e "$GRPROOT/$pid") {
     if (! mkdir("$GRPROOT/$pid", 0770)) {
 	fatal("Could not make directory $GRPROOT/$pid: $!");
     }

tbsetup/setgroups.in

 #!/usr/bin/perl -wT
 
 #
-# Copyright (c) 2000-2013 University of Utah and the Flux Group.
+# Copyright (c) 2000-2013, 2015 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -65,9 +65,12 @@ my $ELABINELAB    = @ELABINELAB@;
 my $WIKISUPPORT   = @WIKISUPPORT@;
 my $BUGDBSUPPORT  = @BUGDBSUPPORT@;
 my $OPSDBSUPPORT  = @OPSDBSUPPORT@;
+my $WITHZFS       = @WITHZFS@;
+my $ZFS_NOEXPORT  = @ZFS_NOEXPORT@;
 my $SETWIKIGROUPS = "$TB/sbin/setwikigroups";
 my $SETBUGDBGROUPS= "$TB/sbin/setbugdbgroups";
 my $OPSDBCONTROL  = "$TB/sbin/opsdb_control";
+my $EXPORTSSETUP  = "$TB/sbin/exports_setup";
 
 my $SSH     = "$TB/bin/sshtb";
 my $USERMOD = "/usr/sbin/pw usermod";
@@ -106,6 +109,7 @@ use libaudit;
 use libdb;
 use libtestbed;
 use User;
+use emutil;
 
 #
 # We do not want to run this script unless its the real version.
@@ -194,11 +198,6 @@ if (AuditStart(0)) {
     exit(0);
 }
 
-#
-# All this stuff must be done as root (ssh).
-#
-$UID = $EUID;
-
 #
 # If no user list provided, we have to do this for the entire project
 # member list since we have no idea who got changed.
@@ -216,6 +215,35 @@ if (! scalar(@userlist)) {
     }
 }
 
+if ($WITHZFS && $ZFS_NOEXPORT) {
+    #
+    # Have to force the directories to be exported.
+    # See ZFS code in exports_setup
+    #
+    foreach my $token (@userlist) {
+	my $user = User->Lookup($token);
+	$user->BumpActivity()
+	    if (defined($user));
+    }
+    $EUID = $UID;
+    system($EXPORTSSETUP) == 0 or
+	fatal("$EXPORTSSETUP failed");
+    $EUID = 0;
+
+    # There is some lag before the automounter can mount the directories
+    foreach my $uid (@userlist) {
+	my $homedir = USERROOT() . "/$uid";
+	if (emutil::waitForMount($homedir) < 0) {
+	    fatal("Could not access user directory: $homedir");
+	}
+    }
+}
+
+#
+# All this stuff must be done as root (ssh).
+#
+$UID = $EUID;
+
 #
 # Loop through user set, building up the group set and issuing commands.
 #