Commit fd0fd225 authored by Leigh Stoller's avatar Leigh Stoller

Changes for dealing with group/password file locking errors:

* Move user mod (gecos,password) into the accountsetup proxy instead of
  ssh chpass. Wrap all usermod/chpass system calls in a loop that looks
  for the busy file error, back off and try again for a while.

* Add same wrapping to local (boss) calls of usermod/chpass. I put that
  function into emutil.

* Rename old modgroups in the proxy to setgroups, since that it is what
  it was actually doing.
parent bc6355f5
...@@ -42,6 +42,7 @@ sub usage() ...@@ -42,6 +42,7 @@ sub usage()
print "Usage: accountsetup adduser ...\n"; print "Usage: accountsetup adduser ...\n";
print " accountsetup deluser ...\n"; print " accountsetup deluser ...\n";
print " accountsetup moduser ...\n"; print " accountsetup moduser ...\n";
print " accountsetup setgroups ...\n";
print " accountsetup chpass ...\n"; print " accountsetup chpass ...\n";
print " accountsetup addproject ...\n"; print " accountsetup addproject ...\n";
print " accountsetup addgroup ...\n"; print " accountsetup addgroup ...\n";
...@@ -74,6 +75,7 @@ my $ZFS_ROOT = "@ZFS_ROOT@"; ...@@ -74,6 +75,7 @@ my $ZFS_ROOT = "@ZFS_ROOT@";
my $ZFS_QUOTA_USER = "@ZFS_QUOTA_USER@"; my $ZFS_QUOTA_USER = "@ZFS_QUOTA_USER@";
my $ZFS_QUOTA_PROJECT = "@ZFS_QUOTA_PROJECT@"; my $ZFS_QUOTA_PROJECT = "@ZFS_QUOTA_PROJECT@";
my $ZFS_QUOTA_GROUP = "@ZFS_QUOTA_GROUP@"; my $ZFS_QUOTA_GROUP = "@ZFS_QUOTA_GROUP@";
my $PW = "/usr/sbin/pw";
my $USERADD = "/usr/sbin/pw useradd"; my $USERADD = "/usr/sbin/pw useradd";
my $USERDEL = "/usr/sbin/pw userdel"; my $USERDEL = "/usr/sbin/pw userdel";
my $USERMOD = "/usr/sbin/pw usermod"; my $USERMOD = "/usr/sbin/pw usermod";
...@@ -124,6 +126,7 @@ my @GDIRLIST = ("exp", "images", "logs", "tarfiles", "rpms", "tiplogs"); ...@@ -124,6 +126,7 @@ my @GDIRLIST = ("exp", "images", "logs", "tarfiles", "rpms", "tiplogs");
# #
sub AddUser(); sub AddUser();
sub DeleteUser(); sub DeleteUser();
sub SetGroups();
sub ModifyUser(); sub ModifyUser();
sub ChangePassword(); sub ChangePassword();
sub AddProject(); sub AddProject();
...@@ -141,6 +144,7 @@ sub ZFSexists($); ...@@ -141,6 +144,7 @@ sub ZFSexists($);
sub MakeDir($$); sub MakeDir($$);
sub WhackDir($$); sub WhackDir($$);
sub mysystem($); sub mysystem($);
sub runBusyLoop($);
# #
# Check args. # Check args.
...@@ -179,6 +183,10 @@ SWITCH: for ($cmd) { ...@@ -179,6 +183,10 @@ SWITCH: for ($cmd) {
ModifyUser(); ModifyUser();
last SWITCH; last SWITCH;
}; };
/^setgroups$/ && do {
SetGroups();
last SWITCH;
};
/^chpass$/ && do { /^chpass$/ && do {
ChangePassword(); ChangePassword();
last SWITCH; last SWITCH;
...@@ -248,8 +256,8 @@ sub AddUser() ...@@ -248,8 +256,8 @@ sub AddUser()
} }
if (mysystem("egrep -q -s '^${user}:' /etc/passwd") && if (mysystem("egrep -q -s '^${user}:' /etc/passwd") &&
mysystem("$USERADD $user -u $uid -c \"$name\" ". runBusyLoop("$USERADD $user -u $uid -c \"$name\" ".
"-k $SKEL -h - -d $hdir -g $gid -s $shell")) { "-k $SKEL -h - -d $hdir -g $gid -s $shell")) {
if (($? >> 8) != $USEREXISTS) { if (($? >> 8) != $USEREXISTS) {
fatal("$USERADD: could not add account"); fatal("$USERADD: could not add account");
} }
...@@ -307,7 +315,7 @@ sub DeleteUser() ...@@ -307,7 +315,7 @@ sub DeleteUser()
# Note that this does NOT remove the user's homedir. # Note that this does NOT remove the user's homedir.
# We remove/rename it below... # We remove/rename it below...
# #
if (mysystem("$USERDEL $user")) { if (runBusyLoop("$USERDEL $user")) {
if (($? >> 8) != $NOSUCHUSER) { if (($? >> 8) != $NOSUCHUSER) {
fatal("Could not remove user $user"); fatal("Could not remove user $user");
} }
...@@ -322,14 +330,33 @@ sub DeleteUser() ...@@ -322,14 +330,33 @@ sub DeleteUser()
return 0; return 0;
} }
#
# Usage: moduser username shell full_name
#
sub ModifyUser()
{
if (@ARGV != 3) {
fatal("moduser: Wrong number of arguments");
}
my $user = shift(@ARGV);
my $shell = shift(@ARGV);
my $name = shift(@ARGV);
if (runBusyLoop("$USERMOD $user -c \"$name\" -s $shell")) {
fatal("$USERMOD: could not modify account");
}
return 0;
}
# #
# Usage: username group1 [ group2 ... groupN ] # Usage: username group1 [ group2 ... groupN ]
# XXX this is specific to what is required by setgroups. # XXX this is specific to what is required by setgroups.
# #
sub ModifyUser() sub SetGroups()
{ {
if (@ARGV < 2) { if (@ARGV < 2) {
fatal("moduser: Wrong number of arguments"); fatal("setgroups: Wrong number of arguments");
} }
my $user = shift(@ARGV); my $user = shift(@ARGV);
my $pgroup = shift(@ARGV); my $pgroup = shift(@ARGV);
...@@ -337,7 +364,7 @@ sub ModifyUser() ...@@ -337,7 +364,7 @@ sub ModifyUser()
if (@ARGV > 0) { if (@ARGV > 0) {
$grouplist = "-G '" . join(',', @ARGV) . "'"; $grouplist = "-G '" . join(',', @ARGV) . "'";
} }
if (mysystem("$USERMOD $user -g $pgroup $grouplist")) { if (runBusyLoop("$USERMOD $user -g $pgroup $grouplist")) {
fatal("Could not modify user $user to add groups!\n"); fatal("Could not modify user $user to add groups!\n");
} }
SetDotGroup($user, $pgroup); SetDotGroup($user, $pgroup);
...@@ -352,7 +379,7 @@ sub ChangePassword() ...@@ -352,7 +379,7 @@ sub ChangePassword()
my $user = shift(@ARGV); my $user = shift(@ARGV);
my $hash = shift(@ARGV); my $hash = shift(@ARGV);
if (mysystem("$CHPASS -p '$hash' $user")) { if (runBusyLoop("$CHPASS -p '$hash' $user")) {
fatal("Could not change password"); fatal("Could not change password");
} }
return 0; return 0;
...@@ -375,7 +402,7 @@ sub DeactivateUser() ...@@ -375,7 +402,7 @@ sub DeactivateUser()
fatal("Could not set ZFS dir $zfsdir to mountpoint=none"); fatal("Could not set ZFS dir $zfsdir to mountpoint=none");
} }
} }
if (mysystem("$USERMOD $user -d /var/empty -s $NOLOGIN")) { if (runBusyLoop("$USERMOD $user -d /var/empty -s $NOLOGIN")) {
fatal("Could not set shell to $NOLOGIN"); fatal("Could not set shell to $NOLOGIN");
} }
return 0; return 0;
...@@ -402,7 +429,7 @@ sub ReactivateUser() ...@@ -402,7 +429,7 @@ sub ReactivateUser()
fatal("Could not set ZFS dir $zfsdir to mountpoint=$hdir"); fatal("Could not set ZFS dir $zfsdir to mountpoint=$hdir");
} }
} }
if (mysystem("$USERMOD $user -d $hdir")) { if (runBusyLoop("$USERMOD $user -d $hdir")) {
fatal("Could not set home directory back to $hdir"); fatal("Could not set home directory back to $hdir");
} }
# #
...@@ -435,7 +462,7 @@ sub AddProject() ...@@ -435,7 +462,7 @@ sub AddProject()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) { if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name ...\n"; print "Adding group $unix_name ...\n";
if (mysystem("$GROUPADD $unix_name -g $unix_gid")) { if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add group $unix_name ($unix_gid)!\n"); fatal("Could not add group $unix_name ($unix_gid)!\n");
} }
} }
...@@ -521,7 +548,7 @@ sub AddGroup() ...@@ -521,7 +548,7 @@ sub AddGroup()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) { if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name ...\n"; print "Adding group $unix_name ...\n";
if (mysystem("$GROUPADD $unix_name -g $unix_gid")) { if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add group $unix_name ($unix_gid)!\n"); fatal("Could not add group $unix_name ($unix_gid)!\n");
} }
} }
...@@ -577,7 +604,7 @@ sub DelProject() ...@@ -577,7 +604,7 @@ sub DelProject()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) { if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) {
print "Deleting project $unix_name ...\n"; print "Deleting project $unix_name ...\n";
if (mysystem("$GROUPDEL $unix_name")) { if (runBusyLoop("$GROUPDEL $unix_name")) {
fatal("Could not delete group $unix_name!\n"); fatal("Could not delete group $unix_name!\n");
} }
} }
...@@ -608,7 +635,7 @@ sub DelGroup() ...@@ -608,7 +635,7 @@ sub DelGroup()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) { if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) {
print "Deleting group $unix_name ...\n"; print "Deleting group $unix_name ...\n";
if (mysystem("$GROUPDEL $unix_name")) { if (runBusyLoop("$GROUPDEL $unix_name")) {
fatal("Could not delete group $unix_name!\n"); fatal("Could not delete group $unix_name!\n");
} }
} }
...@@ -986,6 +1013,60 @@ sub mysystem($) ...@@ -986,6 +1013,60 @@ sub mysystem($)
return system($cmd); return system($cmd);
} }
#
# Run pw/chpass, checking for a locked passwd/group file. The pw routines
# exit with non specific error code 1 for everything, so there is no way
# to tell that its a busy file except by looking at the error message. Then
# wait for a bit and try again. Silly.
#
sub runBusyLoop($)
{
my $command = shift;
my $maxtries = 10;
print STDERR "accountsetup: '$command'\n";
if (open(FD, ">>/usr/testbed/log/accountsetup.log")) {
my $tstamp = POSIX::strftime("%b %e %H:%M:%S", localtime());
print FD "$tstamp: $command\n";
close(FD);
}
while ($maxtries--) {
my $output = "";
#
# This open implicitly forks a child, which goes on to execute the
# command. The parent is going to sit in this loop and capture the
# output of the child. We do this so that we have better control
# over the descriptors.
#
my $pid = open(PIPE, "-|");
if (!defined($pid)) {
print STDERR "runBusyLoop; popen failed!\n";
return -1;
}
if ($pid) {
while (<PIPE>) {
$output .= $_;
}
close(PIPE);
print $output;
return 0
if (!$?);
if ($output =~ /(group|db) file is busy/m) {
print "runBusyLoop; waiting a few seconds before trying again\n";
sleep(3);
}
}
else {
open(STDERR, ">&STDOUT");
exec($command);
}
}
return -1;
}
sub fatal($) { sub fatal($) {
my ($msg) = @_; my ($msg) = @_;
......
...@@ -383,6 +383,10 @@ SWITCH: for ($cmd) { ...@@ -383,6 +383,10 @@ SWITCH: for ($cmd) {
# an account on. # an account on.
# #
TBNodeUpdateAccountsByUID($user); TBNodeUpdateAccountsByUID($user);
if ($PGENISUPPORT) {
require APT_Utility;
APT_Utility::UpdateInstancesByUser($target_user);
}
exit(0); exit(0);
...@@ -433,8 +437,8 @@ sub AddUser() ...@@ -433,8 +437,8 @@ sub AddUser()
if (system("egrep -q -s '^${user}:' /etc/passwd")) { if (system("egrep -q -s '^${user}:' /etc/passwd")) {
print "Adding user $user ($user_number) to local node.\n"; print "Adding user $user ($user_number) to local node.\n";
if (system("$USERADD $user -u $user_number -c \"$fullname\" ". if (runBusyLoop("$USERADD $user -u $user_number -c \"$fullname\" ".
"-h - -d $HOMEDIR/$user ". "-h - -d $HOMEDIR/$user ".
"-g $default_groupname -s $PBAG")) { "-g $default_groupname -s $PBAG")) {
fatal("Could not add user $user to local node."); fatal("Could not add user $user to local node.");
} }
...@@ -610,7 +614,7 @@ sub DelUser() ...@@ -610,7 +614,7 @@ sub DelUser()
$UID = 0; $UID = 0;
if (system("$USERDEL $user")) { if (runBusyLoop("$USERDEL $user")) {
if (($? >> 8) != $NOSUCHUSER) { if (($? >> 8) != $NOSUCHUSER) {
fatal("Could not remove user $user from local node."); fatal("Could not remove user $user from local node.");
} }
...@@ -888,7 +892,7 @@ sub UpdateUser(;$) ...@@ -888,7 +892,7 @@ sub UpdateUser(;$)
# Shell is different on local vs control node. # Shell is different on local vs control node.
if ($freezeopt) { if ($freezeopt) {
$locshellarg = "-s $NOLOGIN"; $locshellarg = "-s $NOLOGIN";
$remshellarg = "-s $NOLOGIN"; $remshellarg = "$NOLOGIN";
} }
else { else {
# Admin users get a local shell. # Admin users get a local shell.
...@@ -900,16 +904,16 @@ sub UpdateUser(;$) ...@@ -900,16 +904,16 @@ sub UpdateUser(;$)
} }
if (!defined($usr_shell) || if (!defined($usr_shell) ||
!exists($shellpaths{$usr_shell})) { !exists($shellpaths{$usr_shell})) {
$remshellarg = "-s " . $shellpaths{"tcsh"}; $remshellarg = $shellpaths{"tcsh"};
} }
else { else {
$remshellarg = "-s " . $shellpaths{$usr_shell}; $remshellarg = $shellpaths{$usr_shell};
} }
} }
print "Updating user $user ($user_number) on local node.\n"; print "Updating user $user ($user_number) on local node.\n";
$UID = 0; $UID = 0;
if (system("$USERMOD $user $locshellarg -c \"$fullname\" ")) { if (runBusyLoop("$USERMOD $user $locshellarg -c \"$fullname\" ")) {
fatal("Could not modify user $user on local node."); fatal("Could not modify user $user on local node.");
} }
...@@ -923,7 +927,7 @@ sub UpdateUser(;$) ...@@ -923,7 +927,7 @@ sub UpdateUser(;$)
print "Updating user $user ($user_number) on $CONTROL\n"; print "Updating user $user ($user_number) on $CONTROL\n";
if (system("$SSH -host $CONTROL ". if (system("$SSH -host $CONTROL ".
"'$USERMOD $user $remshellarg -c \"$fullname\"'")) { "'$ACCOUNTPROXY moduser $user $remshellarg \"$fullname\"'")){
fatal("Could not modify user $user record on $CONTROL."); fatal("Could not modify user $user record on $CONTROL.");
} }
} }
...@@ -1077,7 +1081,7 @@ sub ThawUser() ...@@ -1077,7 +1081,7 @@ sub ThawUser()
AddUser() == 0 AddUser() == 0
or fatal("Cannot thaw $user"); or fatal("Cannot thaw $user");
system("$USERMOD -n $user -s /bin/tcsh"); runBusyLoop("$USERMOD -n $user -s /bin/tcsh");
} }
else { else {
UpdateUser(0) == 0 UpdateUser(0) == 0
...@@ -1119,7 +1123,7 @@ sub DeactivateUser() ...@@ -1119,7 +1123,7 @@ sub DeactivateUser()
# Shell goes to nologin on the CONTROL node. # Shell goes to nologin on the CONTROL node.
# #
$UID = 0; $UID = 0;
if (system("$USERMOD $user -s $NOLOGIN")) { if (runBusyLoop("$USERMOD $user -s $NOLOGIN")) {
fatal("Could not set shell to $NOLOGIN for $user on local node."); fatal("Could not set shell to $NOLOGIN for $user on local node.");
} }
if ($CONTROL ne $BOSSNODE) { if ($CONTROL ne $BOSSNODE) {
......
...@@ -33,7 +33,7 @@ use vars qw(@ISA @EXPORT); ...@@ -33,7 +33,7 @@ use vars qw(@ISA @EXPORT);
TBDB_CHECKDBSLOT_ERROR TBcheck_dbslot TBFieldErrorString TBDB_CHECKDBSLOT_ERROR TBcheck_dbslot TBFieldErrorString
TBGetUniqueIndex ParRun VersionInfo UpdateVersionInfo TBGetUniqueIndex ParRun VersionInfo UpdateVersionInfo
SpanningTree GenFakeMac BackTraceOnWarning PassWordHash SpanningTree GenFakeMac BackTraceOnWarning PassWordHash
SSHwithTimeout TBDateStringGMT TBDateStringLocal SSHwithTimeout TBDateStringGMT TBDateStringLocal runBusyLoop
); );
use emdb; use emdb;
...@@ -901,6 +901,51 @@ sub waitForMount($;$) ...@@ -901,6 +901,51 @@ sub waitForMount($;$)
return -1; return -1;
} }
#
# Run pw/chpass, checking for a locked passwd/group file. The pw routines
# exit with non specific error code 1 for everything, so there is no way
# to tell that its a busy file except by looking at the error message. Then
# wait for a bit and try again. Silly.
#
sub runBusyLoop($)
{
my $command = shift;
my $maxtries = 10;
while ($maxtries--) {
my $output = "";
#
# This open implicitly forks a child, which goes on to execute the
# command. The parent is going to sit in this loop and capture the
# output of the child. We do this so that we have better control
# over the descriptors.
#
my $pid = open(PIPE, "-|");
if (!defined($pid)) {
print STDERR "runBusyLoop; popen failed!\n";
return -1;
}
if ($pid) {
while (<PIPE>) {
$output .= $_;
}
close(PIPE);
print $output;
return 0
if (!$?);
if ($output =~ /(group|db) file is busy/m) {
print "runBusyLoop; waiting a few seconds before trying again\n";
sleep(3);
}
}
else {
open(STDERR, ">&STDOUT");
exec($command);
}
}
return -1;
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2015 University of Utah and the Flux Group. # Copyright (c) 2000-2016 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -194,7 +194,7 @@ my $groupdir = "$GRPROOT/$pid/$gid"; ...@@ -194,7 +194,7 @@ my $groupdir = "$GRPROOT/$pid/$gid";
if (system("egrep -q -s '^${unix_name}:' /etc/group")) { if (system("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name to local node ...\n"; print "Adding group $unix_name to local node ...\n";
if (system("$GROUPADD $unix_name -g $unix_gid")) { if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add $unix_gid ($unix_gid) to local node!\n"); fatal("Could not add $unix_gid ($unix_gid) to local node!\n");
} }
} }
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2015 University of Utah and the Flux Group. # Copyright (c) 2000-2016 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -92,6 +92,7 @@ $| = 1; ...@@ -92,6 +92,7 @@ $| = 1;
use lib "@prefix@/lib"; use lib "@prefix@/lib";
use libaudit; use libaudit;
use libdb; use libdb;
use emutil;
use libtestbed; use libtestbed;
use User; use User;
use Group; use Group;
...@@ -190,7 +191,7 @@ if (system("grep -q '^${unix_gid}:' /etc/group")) { ...@@ -190,7 +191,7 @@ if (system("grep -q '^${unix_gid}:' /etc/group")) {
# #
print "Removing group $unix_name ($unix_gid) on local node.\n"; print "Removing group $unix_name ($unix_gid) on local node.\n";
if (system("$GROUPDEL $unix_name")) { if (runBusyLoop("$GROUPDEL $unix_name")) {
if (($? >> 8) != 65) { if (($? >> 8) != 65) {
fatal("Could not remove group $unix_name from local node!"); fatal("Could not remove group $unix_name from local node!");
} }
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2015 University of Utah and the Flux Group. # Copyright (c) 2000-2016 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -389,7 +389,7 @@ foreach my $token (@userlist) { ...@@ -389,7 +389,7 @@ foreach my $token (@userlist) {
# #
# MAKE SURE not to update anything else! # MAKE SURE not to update anything else!
# #
if (system("$USERMOD $uid -g $project $groupargument")) { if (runBusyLoop("$USERMOD $uid -g $project $groupargument")) {
fatal("Could not modify user $uid on local node."); fatal("Could not modify user $uid on local node.");
} }
...@@ -398,7 +398,7 @@ foreach my $token (@userlist) { ...@@ -398,7 +398,7 @@ foreach my $token (@userlist) {
if ($control_node ne $BOSSNODE) { if ($control_node ne $BOSSNODE) {
$groupargument = join(' ', @groupnames); $groupargument = join(' ', @groupnames);
if (system("$SSH -host $control_node ". if (system("$SSH -host $control_node ".
"$ACCOUNTPROXY moduser $uid $project $groupargument")) { "$ACCOUNTPROXY setgroups $uid $project $groupargument")) {
fatal("Could not modify user $uid record on $control_node."); fatal("Could not modify user $uid record on $control_node.");
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment