Commit 609c547e authored by Mike Hibler's avatar Mike Hibler

Yet another attempt to Get It Right based on our latest half-assed

understanding of how mountd operates.

Things that should be fixed:

1. It iterates over every node calculating what directories are exported,
   what FSes they are on, etc.  Most of that work only needs to happen
   per experiment.

2. The algorithm to determine what FS a directory is on is a hack.  I just
   take the first component of the path provided.  As long as we mount all
   our FSes at the top level and configure with canonical pathes, this is
   ok.  Other solutions require calling out to the ops node to get actual
   mount info.

3. Once shared experiments are revived, the code to determine exported
   directories will need to change.  The algorithm for computing the
   exports lines should still be correct.
parent 4f02800a
......@@ -95,184 +95,105 @@ print MAP "#\n";
print MAP "\n";
#
# We export to particular nodes, based on the experiment that is allocated
# to the node. Since we want the exports file to be based on IP numbers,
# we need this crazy join to find out the type of the node, so we can find
# the control network card number, so we can find the IP address for that
# card.
# First gather up all the nodes that are reserved and the required info.
#
# XXX - Because of mountd sillyness, group experiments cannot be exported
# here since the groups directory is in the same FS as the proj.
# This bothers mountd, since the same IP would be listed more than once
# on the right hand side. So, this first join eliminates nodes that
# part of group experiments (pid!=gid).
#
$db_result =
DBQueryFatal("select reserved.node_id,reserved.pid,interfaces.IP,a.pid ".
"from reserved ".
"left join exppid_access as a ".
" on a.exp_pid=reserved.pid and a.exp_eid=reserved.eid ".
"left join experiments as e on ".
" reserved.pid=e.pid and reserved.eid=e.eid ".
"left join nodes on reserved.node_id=nodes.node_id ".
$nodes_result =
DBQueryFatal("select r.node_id,r.pid,r.eid,e.gid,i.IP from reserved as r ".
"left join experiments as e on r.pid=e.pid and r.eid=e.eid ".
"left join nodes on r.node_id=nodes.node_id ".
"left join node_types on node_types.type=nodes.type ".
"left join interfaces on reserved.node_id=interfaces.node_id".
" and interfaces.card=node_types.control_net ".
"where interfaces.IP!='NULL' and e.gid=e.pid ".
"order by reserved.pid,interfaces.IP");
#
# Go through and build up a list of exported mounts for each IP.
#
my %iphash = ();
my @iplist = ();
while (@row = $db_result->fetchrow_array) {
my $node_id = $row[0];
my $pid = $row[1];
my $ip = $row[2];
my $subpid = $row[3];
"left join interfaces as i on r.node_id=i.node_id ".
" and i.card=node_types.control_net ".
" where i.IP!='NULL' and r.node_id not like 'sh%' ".
"order by r.eid,e.gid,r.pid,nodes.priority");
if (! defined($iphash{$ip})) {
$iphash{$ip} = [ $pid ];
push(@iplist, $ip);
if (defined($subpid)) {
push(@{ $iphash{$ip} }, $subpid);
}
}
else {
if (defined($subpid)) {
push(@{ $iphash{$ip} }, $subpid);
}
}
}
print MAP "#\n";
print MAP "# Export Project directories\n";
print MAP "#\n";
my %ipgroups = ();
#
# This takes the iphash array and spits out the mount lists using the
# function provided.
# For each node:
# determine the list of directories accessible
# split the list into sublists based on filesystems
# (i.e., all directories in the same FS are in the same sublist)
# add the node to each sublist
#
if (@iplist) {
spitmounts(sub { my $pid = shift; print MAP "$projdir/$pid " });
}
#
# Next determine what group directories need to be exported. This is done
# only for experiments in which pid!=gid.
#
# XXX - See XXX above. The printer function below has to include the proj
# directory.
# Note that we could do this per experiment rather than per node,
# adding all nodes from an experiment to the sublists created.
#
$db_result =
DBQueryFatal("select reserved.node_id,reserved.pid,interfaces.IP,e.gid ".
"from reserved ".
"left join experiments as e on ".
" reserved.pid=e.pid and reserved.eid=e.eid ".
"left join nodes on reserved.node_id=nodes.node_id ".
"left join node_types on node_types.type=nodes.type ".
"left join interfaces on reserved.node_id=interfaces.node_id".
" and interfaces.card=node_types.control_net ".
"where interfaces.IP!='NULL' and e.pid!=e.gid ".
"order by reserved.pid,e.gid,interfaces.IP");
#
# Go through and build up a list of exported mounts for each IP.
#
%iphash = ();
@iplist = ();
while (@row = $db_result->fetchrow_array) {
while (@row = $nodes_result->fetchrow_array) {
my $node_id = $row[0];
my $pid = $row[1];
my $ip = $row[2];
my $eid = $row[2];
my $gid = $row[3];
my $ip = $row[4];
my %fslist = ();
my @dirlist = ();
if (! defined($iphash{$ip})) {
$iphash{$ip} = [ "$pid/$gid" ];
push(@iplist, $ip);
}
else {
push(@{ $iphash{$ip} }, "$pid/$gid");
#
# Construct a list of directories accessible from this node.
# First the project and group directories.
# XXX needs to be fixed for shared experiments?
#
push(@dirlist, "$projdir/$pid");
if ($gid ne $pid) {
push(@dirlist, "$groupdir/$pid/$gid");
}
}
print MAP "#\n";
print MAP "# Export Group directories\n";
print MAP "#\n";
#
# Determine the users that can access this node, and add those
# users' directories to the list.
# XXX needs to be fixed for shared experiments?
#
$users_result =
DBQueryFatal("select distinct uid from group_membership ".
"where pid='$pid' and gid='$gid' and trust!='none'");
#
# This takes the iphash array and spits out the mount lists using function
# provided. Note that the function also prints include the proj dir. See
# the XXX comments above.
#
if (@iplist) {
my $printer = sub {
my($pid, $gid) = split('/', $_[0]);
while (@usersrow = $users_result->fetchrow_array) {
my $uid = $usersrow[0];
print MAP "$projdir/$pid $groupdir/$pid/$gid ";
};
spitmounts($printer);
}
push(@dirlist, "$usersdir/$uid");
}
#
# Now we need the users list for exporting from /users. We need to export
# all of the project members to all of the machines in that projects
# experiments. Also, since we again want the IPs, we need all of that crazy
# join again. Even worse, we cannot list an IP address more than once, so
# we end up with mulitple exports for any particular user (cause they can be
# in multiple experiments within a project), but with non overlapping
# sets of IP addresses. For example:
#
# /users/stoller /users/mike 101.101.101.100 101.101.101.101
# /users/stoller 101.101.101.102
# /users/mike 101.101.101.103
#
$db_result =
DBQueryFatal("select distinct reserved.node_id,reserved.pid,".
"g.uid,interfaces.IP from reserved ".
"left join exppid_access as a ".
" on a.exp_pid=reserved.pid and a.exp_eid=reserved.eid ".
"left join group_membership as g on g.pid=a.pid or ".
" (g.pid=reserved.pid and g.gid=g.pid) ".
"left join nodes on reserved.node_id=nodes.node_id ".
"left join node_types on node_types.type=nodes.type ".
"left join interfaces on reserved.node_id=interfaces.node_id ".
"and interfaces.card=node_types.control_net ".
"where interfaces.IP!='NULL' and g.trust!='none' ".
"order by reserved.pid,reserved.node_id");
#
# Build up filesystem sub-lists.
# Iterate through directory list dividing it according to filesystem.
#
foreach my $dir ( @dirlist ) {
my $fs = fsof($dir);
%iphash = ();
@iplist = ();
if (! defined($fslist{$fs})) {
$fslist{$fs} = [ $dir ];
}
else {
push(@{ $fslist{$fs} }, $dir);
}
}
while (@row = $db_result->fetchrow_array) {
my $node_id = $row[0];
my $pid = $row[1];
my $uid = $row[2];
my $ip = $row[3];
#
# For each FS directory list, create a hash key out of its directory list.
#
foreach my $fs ( keys(%fslist) ) {
#
# Convert dirlist to a string and use that as a hash index to group
# IPs together with the same set of mounts.
#
my $str = join(" ", sort(@{ $fslist{$fs} }));
if (! defined($iphash{$ip})) {
$iphash{$ip} = [ $uid ];
push(@iplist, $ip);
if (! defined($ipgroups{$str})) {
$ipgroups{$str} = [ $ip ];
}
else {
push(@{ $iphash{$ip} }, $uid);
push(@{ $ipgroups{$str} }, $ip);
}
}
}
print MAP "#\n";
print MAP "# Export User directories\n";
print MAP "#\n";
#
# This takes the iphash array and spits out the mount lists using the
# function provided.
# Now spit out each group!
#
if (@iplist) {
spitmounts(sub { my $uid = shift; print MAP "$usersdir/$uid " });
foreach my $str ( keys(%ipgroups) ) {
my @iplist = @{ $ipgroups{$str} };
print MAP "$str -maproot=root @iplist\n";
}
print MAP "\n";
......@@ -310,79 +231,21 @@ sub fatal {
}
#
# Spit out a set of mounts. The argument is the printer function to
# format the mount line.
# Return a unique (per-FS) string identifying the filesystem of
# the given path.
#
sub spitmounts($)
{
my $printer = shift;
#
# Sort each of the lists (per IP).
#
foreach my $ip ( @iplist ) {
my @lis = @{ $iphash{$ip} };
$iphash{$ip} = [ sort( @lis ) ];
}
#
# Next go through and combine sequential IPs that have exactly the
# same set of exports. Not perfect, but should result in a big
# enough reduction for now. At some point we should have our own
# exportfs program do this stuff instead of going through mountd.
#
# Shift off the first IP to initialize the sequence.
#
my @ipseq = ( shift(@iplist) );
my @lastset = @{ $iphash{$ipseq[0]} };
foreach my $ip ( @iplist ) {
my @set = @{ $iphash{$ip} };
#
# Whenever the set changes, spit out what we have saved up, and then
# start over.
#
if (! listequal(\@set, \@lastset)) {
foreach my $el ( @lastset ) {
&$printer($el);
}
print MAP "-maproot=root @ipseq\n";
@ipseq = ();
}
push(@ipseq, $ip);
@lastset = @set;
}
#
# Last set of IPs.
#
if (@ipseq) {
foreach my $el ( @lastset ) {
&$printer($el);
}
print MAP "-maproot=root @ipseq\n";
}
}
# XXX for the moment, we just return the first component of the path
# assuming everything is mounted at the top level. Clearly this needs
# to be fixed.
#
# Compare two lists of tokens.
# Fixing probably involves either parsing the output of mount or df to
# compare against or using $dev as returned by the stat call. Both of
# these options require ssh callouts to the ops node where the actual
# filesystems are.
#
sub listequal ($$)
{
my($lista, $listb) = @_;
sub fsof($) {
my($path) = @_;
if (@$lista != @$listb) {
return 0;
}
for ($i = 0; $i < @$lista; $i++) {
if ($$lista[$i] ne $$listb[$i]) {
return 0;
}
}
return 1;
$path =~ s#^(/[^/]*).*#$1#;
return $path;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment