Commit 8404af03 authored by Leigh Stoller's avatar Leigh Stoller

Another set of group changes. As discussed in email and meetings,

group directories are now created in a different tree than the
the project directory so that they can be exported independently of
the project tree to the nodes in a group experiment. The tree is
routed at /groups on boss/users and on nodes.

1. mkgroup,rmgroup,mkproj - Minor changes to reflect new group
   directory location (/groups). We leave a symlink in the old spot to
   maintain compatability, and to reduce the number of different
   directories that a person needs to worry about. So, when a group is
   made, you get a real directory /groups/pid/gid, and a symlink
   /proj/pid/groups/gid that points to the former.

2. tmcd/tmcd.c - Minor change to add the additional group directory mount
   in the mounts command. Only done when pid!=gid for the experiment.

3. tmcd/libsetup.pm and friends - Minor changes to fix the fact that
   mkdir does not create subdirs along the way unless the -p option is
   specified. Needed to create the local directory for the mounts
   returned by tmcd for group dirs. Pushed them out to the sup trees,
   although 6.2 images older than the most recent one are not going to
   work right. No one is using those images though, and we should just
   flush the sup trees.

4. exports_setup.in - Ah, the crux of the issue. I really dislike NFS
   at this point. The original idea was to export a third set of
   directories to nodes that were part of a group experiment. Those
   nodes would get /groups/pid/gid exported, and /proj/pid read-only.
   Well, no such luck. On users, /groups and /proj are both really on
   /q, and the old restriction of mountd not allowing an IP to
   specified more than once on the right hand side for any FS, reared
   its ugly head again. As far as mountd is concerned, /q/groups and
   /q/proj are the same thing, and so it bombed when I tried to export
   them on different lines, since that meant an IP was repeated twice.
   So, I reworked exports_setup, and now for any node that is part of
   a group experiment, it gets this:

	/q/proj/pid /q/groups/pid/gid -maproot=root 155.101.132.26

   which at least allows the individual group dirs to be protected
   from each other, but does not allow /proj/pid to be exported read
   only. Sigh.
parent 0cd744a4
......@@ -26,6 +26,7 @@ my $exportstail = "/var/tmp/exports.tail";
my $lockfile = "/var/tmp/testbed_exports_lockfile";
my $projdir = "/q/proj";
my $usersdir = "/users";
my $groupdir = "/q/groups";
my $dbg = 1;
my @row;
......@@ -99,16 +100,24 @@ print MAP "\n";
# the control network card number, so we can find the IP address for that
# card.
#
# XXX - Because of mountd sillyness, group experiments cannot be exported
# here since the groups directory is in the same FS as the proj.
# This bothers mountd, since the same IP would be listed more than once
# on the right hand side. So, this first join eliminates nodes that
# part of group experiments (pid!=gid).
#
$db_result =
DBQueryFatal("select reserved.node_id,reserved.pid,interfaces.IP,a.pid ".
"from reserved ".
"left join exppid_access as a ".
" on a.exp_pid=reserved.pid and a.exp_eid=reserved.eid ".
"left join experiments as e on ".
" reserved.pid=e.pid and reserved.eid=e.eid ".
"left join nodes on reserved.node_id=nodes.node_id ".
"left join node_types on node_types.type=nodes.type ".
"left join interfaces on reserved.node_id=interfaces.node_id".
" and interfaces.card=node_types.control_net ".
"where interfaces.IP!='NULL' ".
"where interfaces.IP!='NULL' and e.gid=e.pid ".
"order by reserved.pid,interfaces.IP");
#
......@@ -138,56 +147,72 @@ while (@row = $db_result->fetchrow_array) {
}
}
print MAP "#\n";
print MAP "# Export Project directories\n";
print MAP "#\n";
#
# Sort each of the lists of projects (per IP).
# This takes the iphash array and spits out the mount lists using the
# function provided.
#
foreach my $ip ( @iplist ) {
my @pids = @{ $iphash{$ip} };
$iphash{$ip} = [ sort( @pids ) ];
}
spitmounts(sub { my $pid = shift; print MAP "$projdir/$pid " });
#
# Next go through and combine sequential IPs that have exactly the same set
# of exports. Not perfect, but should result in a big enough reduction for
# now. At some point we should have our own exportfs program do this stuff
# instead of going through mountd.
# Next determine what group directories need to be exported. This is done
# only for experiments in which pid!=gid.
#
# Shift off the first IP to initialize the sequence.
# XXX - See XXX above. The printer function below has to include the proj
# directory.
#
my @ipseq = ( shift(@iplist) );
my @lastpids = @{ $iphash{$ipseq[0]} };
$db_result =
DBQueryFatal("select reserved.node_id,reserved.pid,interfaces.IP,e.gid ".
"from reserved ".
"left join experiments as e on ".
" reserved.pid=e.pid and reserved.eid=e.eid ".
"left join nodes on reserved.node_id=nodes.node_id ".
"left join node_types on node_types.type=nodes.type ".
"left join interfaces on reserved.node_id=interfaces.node_id".
" and interfaces.card=node_types.control_net ".
"where interfaces.IP!='NULL' and e.pid!=e.gid ".
"order by reserved.pid,e.gid,interfaces.IP");
print MAP "#\n";
print MAP "# Export Project directories\n";
print MAP "#\n";
#
# Go through and build up a list of exported mounts for each IP.
#
%iphash = ();
@iplist = ();
foreach my $ip ( @iplist ) {
my @pids = @{ $iphash{$ip} };
while (@row = $db_result->fetchrow_array) {
my $node_id = $row[0];
my $pid = $row[1];
my $ip = $row[2];
my $gid = $row[3];
#
# Whenever the set changes, spit out what we have saved up, and then
# start over.
#
if (! listequal(\@pids, \@lastpids)) {
foreach my $pid ( @lastpids ) {
print MAP "$projdir/$pid $projdir/$pid/images ";
}
print MAP "-maproot=root @ipseq\n";
@ipseq = ();
if (! defined($iphash{$ip})) {
$iphash{$ip} = [ "$pid/$gid" ];
push(@iplist, $ip);
}
else {
push(@{ $iphash{$ip} }, "$pid/$gid");
}
push(@ipseq, $ip);
@lastpids = @pids;
}
print MAP "#\n";
print MAP "# Export Group directories\n";
print MAP "#\n";
#
# Last set of IPs.
# This takes the iphash array and spits out the mount lists using function
# provided. Note that the function also prints include the proj dir. See
# the XXX comments above.
#
if (@ipseq) {
foreach my $pid ( @lastpids ) {
print MAP "$projdir/$pid $projdir/$pid/images ";
}
print MAP "-maproot=root @ipseq\n";
if (@iplist) {
my $printer = sub {
my($pid, $gid) = split('/', $_[0]);
print MAP "$projdir/$pid $groupdir/$pid/$gid ";
};
spitmounts($printer);
}
#
......@@ -196,7 +221,7 @@ if (@ipseq) {
# experiments. Also, since we again want the IPs, we need all of that crazy
# join again. Even worse, we cannot list an IP address more than once, so
# we end up with mulitple exports for any particular user (cause they can be
# in multiple experiments withing a project), but with non overlapping
# in multiple experiments within a project), but with non overlapping
# sets of IP addresses. For example:
#
# /users/stoller /users/mike 101.101.101.100 101.101.101.101
......@@ -235,57 +260,15 @@ while (@row = $db_result->fetchrow_array) {
}
}
#
# Sort each of the lists of users (per IP).
#
foreach my $ip ( @iplist ) {
my @uids = @{ $iphash{$ip} };
$iphash{$ip} = [ sort( @uids ) ];
}
#
# Next go through and combine sequential IPs that have exactly the same set
# of exports. Not perfect, but should result in a big enough reduction for
# now. At some point we should have our own exportfs program do this stuff
# instead of going through mountd.
#
# Shift off the first IP to initialize the sequence.
#
@ipseq = ( shift(@iplist) );
my @lastuids = @{ $iphash{$ipseq[0]} };
print MAP "#\n";
print MAP "# Export User directories\n";
print MAP "#\n";
foreach my $ip ( @iplist ) {
my @uids = @{ $iphash{$ip} };
#
# Whenever the set changes, spit out what we have saved up, and then
# start over.
#
if (! listequal(\@uids, \@lastuids)) {
foreach my $uid ( @lastuids ) {
print MAP "$usersdir/$uid ";
}
print MAP "-maproot=root @ipseq\n";
@ipseq = ();
}
push(@ipseq, $ip);
@lastuids = @uids;
}
#
# Last set of IPs.
# This takes the iphash array and spits out the mount lists using the
# function provided.
#
if (@ipseq) {
foreach my $uid ( @lastuids ) {
print MAP "$usersdir/$uid ";
}
print MAP "-maproot=root @ipseq\n";
}
spitmounts(sub { my $uid = shift; print MAP "$usersdir/$uid " });
print MAP "\n";
close(MAP);
......@@ -321,6 +304,63 @@ sub fatal {
die($msg);
}
#
# Spit out a set of mounts. The argument is the printer function to
# format the mount line.
#
sub spitmounts($)
{
my $printer = shift;
#
# Sort each of the lists (per IP).
#
foreach my $ip ( @iplist ) {
my @lis = @{ $iphash{$ip} };
$iphash{$ip} = [ sort( @lis ) ];
}
#
# Next go through and combine sequential IPs that have exactly the
# same set of exports. Not perfect, but should result in a big
# enough reduction for now. At some point we should have our own
# exportfs program do this stuff instead of going through mountd.
#
# Shift off the first IP to initialize the sequence.
#
my @ipseq = ( shift(@iplist) );
my @lastset = @{ $iphash{$ipseq[0]} };
foreach my $ip ( @iplist ) {
my @set = @{ $iphash{$ip} };
#
# Whenever the set changes, spit out what we have saved up, and then
# start over.
#
if (! listequal(\@set, \@lastset)) {
foreach my $el ( @lastset ) {
&$printer($el);
}
print MAP "-maproot=root @ipseq\n";
@ipseq = ();
}
push(@ipseq, $ip);
@lastset = @set;
}
#
# Last set of IPs.
#
if (@ipseq) {
foreach my $el ( @lastset ) {
&$printer($el);
}
print MAP "-maproot=root @ipseq\n";
}
}
#
# Compare two lists of tokens.
#
......@@ -339,3 +379,5 @@ sub listequal ($$)
return 1;
}
......@@ -25,6 +25,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
my $CONTROL = "@USERNODE@";
my $PROJROOT = "/proj";
my $GRPROOT = "/groups";
my $SSH = "$TB/bin/sshtb";
my $GROUPADD = "/usr/sbin/pw groupadd";
......@@ -34,7 +35,6 @@ my $dbuid;
my @db_row;
my $query_result;
my $leader;
my $groupdir;
my $logname;
my $user_name;
my $user_email;
......@@ -169,7 +169,8 @@ if (! ($leader = GroupLeader($pid, $gid))) {
#
# The group directory lives here.
#
$groupdir = "$PROJROOT/$pid/groups/$gid";
my $groupdir = "$GRPROOT/$pid/$gid";
my $grouplink = "$PROJROOT/$pid/groups/$gid";
#
# Unix info for the group
......@@ -274,6 +275,11 @@ if (! -e $groupdir && $pid ne $gid) {
if (! chown($unix_uid, $unix_gid, "$groupdir")) {
fatal("*** Could not chown $groupdir to $leader/$gid: $!");
}
if (! -e $grouplink) {
symlink($groupdir, $grouplink) or
fatal("*** Could not symlink($groupdir, $grouplink): $!");
}
}
print "Group Creation Completed!\n";
......
......@@ -19,6 +19,7 @@ my $MKGROUP = "$TB/sbin/mkgroup";
my $MKACCT = "$TB/sbin/mkacct";
my $PROJROOT = "/proj";
my $GRPROOT = "/groups";
my $TFTPROOT = "/tftpboot";
my @DIRLIST = ("exp", "images", "logs", "deltas", "tarfiles", "rpms",
"groups", "tiplogs");
......@@ -176,6 +177,19 @@ if (! chown($uid, $gid, "$TFTPROOT/proj/$pid")) {
fatal("Could not chown $TFTPROOT/proj/$pid to $uid/$gid: $!");
}
#
# Create groups directory.
#
if (! mkdir("$GRPROOT/$pid", 0770)) {
fatal("Could not make directory $GRPROOT/$pid: $!");
}
if (! chmod(0777, ""$GRPROOT/$pid")) {
fatal("Could not chmod directory $GRPROOT/$pid: $!");
}
if (! chown($uid, $gid, "$GRPROOT/$pid")) {
fatal("Could not chown $GRPROOT/$pid to $uid/$gid: $!");
}
exit(0);
sub fatal {
......
......@@ -17,6 +17,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $CONTROL = "@USERNODE@";
my $PROJROOT = "/proj";
my $GRPROOT = "/groups";
my $SSH = "$TB/bin/sshtb";
my $GROUPDEL = "/usr/sbin/pw groupdel";
my $errors = 0;
......@@ -94,19 +95,21 @@ if (! TBGroupUnixInfo($pid, $gid, \$unix_gid, \$unix_name)) {
#
# Rename the group directory.
#
my $newname = "$gid-" . TBDateTimeFSSafe();
my $olddir = "$PROJROOT/$pid/groups";
if (! chdir($olddir)) {
die("*** Could not chdir to $olddir: $!\n");
my $groupdir = "$GRPROOT/$pid";
my $oldname = "$groupdir/$gid";
my $newname = "$oldname-" . TBDateTimeFSSafe();
my $grouplink = "$PROJROOT/$pid/groups/$gid";
if (-e $grouplink) {
unlink($grouplink) or
die("*** Could not unlink $grouplink: $!\n");
}
if (-e $gid) {
print "Renaming $gid to $newname in $olddir.\n";
if (-e $oldname) {
print "Renaming $oldname to $newname.\n";
if (! rename($gid, $newname)) {
die("*** Could not rename group directory $olddir/$gid to ".
"$newname: $!\n");
if (! rename($oldname, $newname)) {
die("*** Could not rename $oldname to $newname: $!\n");
}
#
......
......@@ -244,7 +244,7 @@ sub domounts()
}
if (! -e $local) {
if (! mkdir($local, 0770)) {
if (! os_mkdir($local, 0770)) {
warn "*** WARNING: Could not make directory $local: $!\n";
next;
}
......
......@@ -9,7 +9,7 @@ use Exporter;
@EXPORT =
qw ( $CP $EGREP $MOUNT $UMOUNT $TMPASSWD
os_cleanup_node os_ifconfig_line os_etchosts_line
os_setup os_groupadd os_useradd os_userdel os_usermod
os_setup os_groupadd os_useradd os_userdel os_usermod os_mkdir
os_rpminstall_line update_delays
);
......@@ -46,6 +46,7 @@ my $IFC_100MBS = "media 100baseTX";
my $IFC_10MBS = "media 10baseT/UTP";
my $IFC_FDUPLEX = "mediaopt full-duplex";
my $RPMINSTALL = "/usr/local/bin/rpm -i %s";
my $MKDIR = "/bin/mkdir";
#
# Delay node configuration goop.
......@@ -215,6 +216,19 @@ sub os_rpminstall_line($)
return sprintf($RPMINSTALL, $rpm);
}
#
# Create a directory including all intermediate directories.
#
sub os_mkdir($$)
{
my ($dir, $mode) = @_;
if (system("$MKDIR -p -m $mode $dir")) {
return 0;
}
return 1;
}
#
# OS Dependent configuration.
#
......
......@@ -244,7 +244,7 @@ sub domounts()
}
if (! -e $local) {
if (! mkdir($local, 0770)) {
if (! os_mkdir($local, 0770)) {
warn "*** WARNING: Could not make directory $local: $!\n";
next;
}
......
......@@ -9,7 +9,7 @@ use Exporter;
@EXPORT =
qw ( $CP $EGREP $MOUNT $UMOUNT $TMPASSWD
os_cleanup_node os_ifconfig_line os_etchosts_line
os_setup os_groupadd os_useradd os_userdel os_usermod
os_setup os_groupadd os_useradd os_userdel os_usermod os_mkdir
os_rpminstall_line
);
......@@ -48,6 +48,7 @@ my $IFC_FDUPLEX = "FD";
my $IFC_HDUPLEX = "HD";
my $RPMINSTALL = "/bin/rpm -i %s";
my @LOCKFILES = ("/etc/group.lock", "/etc/gshadow.lock");
my $MKDIR = "/bin/mkdir";
#
# OS dependent part of cleanup node state.
......@@ -201,6 +202,19 @@ sub os_rpminstall_line($)
return sprintf($RPMINSTALL, $rpm);
}
#
# Create a directory including all intermediate directories.
#
sub os_mkdir($$)
{
my ($dir, $mode) = @_;
if (system("$MKDIR -p -m $mode $dir")) {
return 0;
}
return 1;
}
#
# OS Dependent configuration.
#
......
......@@ -18,8 +18,10 @@
* XXX This needs to be localized!
*/
#define FSPROJDIR "fs.emulab.net:/q/proj"
#define FSGROUPDIR "fs.emulab.net:/q/groups"
#define FSUSERDIR "fs.emulab.net:/users"
#define PROJDIR "/proj"
#define GROUPDIR "/groups"
#define USERDIR "/users"
#define RELOADPID "emulab-ops"
#define RELOADEID "reloading"
......@@ -1846,6 +1848,16 @@ domounts(int sock, struct in_addr ipaddr, char *rdata, int tcp)
FSPROJDIR, pid, PROJDIR, pid);
client_writeback(sock, buf, strlen(buf), tcp);
/*
* If pid!=gid, then this is group experiment, and we return
* a mount for the group directory too.
*/
if (strcmp(pid, gid)) {
sprintf(buf, "REMOTE=%s/%s/%s LOCAL=%s/%s/%s\n",
FSGROUPDIR, pid, gid, GROUPDIR, pid, gid);
client_writeback(sock, buf, strlen(buf), tcp);
}
/*
* Now check for aux project access. Return a list of mounts for
* those projects.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment