Commit fd0fd225 authored by Leigh Stoller's avatar Leigh Stoller

Changes for dealing with group/password file locking errors:

* Move user mod (gecos,password) into the accountsetup proxy instead of
  ssh chpass. Wrap all usermod/chpass system calls in a loop that looks
  for the busy file error, back off and try again for a while.

* Add same wrapping to local (boss) calls of usermod/chpass. I put that
  function into emutil.

* Rename old modgroups in the proxy to setgroups, since that it is what
  it was actually doing.
parent bc6355f5
......@@ -42,6 +42,7 @@ sub usage()
print "Usage: accountsetup adduser ...\n";
print " accountsetup deluser ...\n";
print " accountsetup moduser ...\n";
print " accountsetup setgroups ...\n";
print " accountsetup chpass ...\n";
print " accountsetup addproject ...\n";
print " accountsetup addgroup ...\n";
......@@ -74,6 +75,7 @@ my $ZFS_ROOT = "@ZFS_ROOT@";
my $ZFS_QUOTA_USER = "@ZFS_QUOTA_USER@";
my $ZFS_QUOTA_PROJECT = "@ZFS_QUOTA_PROJECT@";
my $ZFS_QUOTA_GROUP = "@ZFS_QUOTA_GROUP@";
my $PW = "/usr/sbin/pw";
my $USERADD = "/usr/sbin/pw useradd";
my $USERDEL = "/usr/sbin/pw userdel";
my $USERMOD = "/usr/sbin/pw usermod";
......@@ -124,6 +126,7 @@ my @GDIRLIST = ("exp", "images", "logs", "tarfiles", "rpms", "tiplogs");
#
sub AddUser();
sub DeleteUser();
sub SetGroups();
sub ModifyUser();
sub ChangePassword();
sub AddProject();
......@@ -141,6 +144,7 @@ sub ZFSexists($);
sub MakeDir($$);
sub WhackDir($$);
sub mysystem($);
sub runBusyLoop($);
#
# Check args.
......@@ -179,6 +183,10 @@ SWITCH: for ($cmd) {
ModifyUser();
last SWITCH;
};
/^setgroups$/ && do {
SetGroups();
last SWITCH;
};
/^chpass$/ && do {
ChangePassword();
last SWITCH;
......@@ -248,7 +256,7 @@ sub AddUser()
}
if (mysystem("egrep -q -s '^${user}:' /etc/passwd") &&
mysystem("$USERADD $user -u $uid -c \"$name\" ".
runBusyLoop("$USERADD $user -u $uid -c \"$name\" ".
"-k $SKEL -h - -d $hdir -g $gid -s $shell")) {
if (($? >> 8) != $USEREXISTS) {
fatal("$USERADD: could not add account");
......@@ -307,7 +315,7 @@ sub DeleteUser()
# Note that this does NOT remove the user's homedir.
# We remove/rename it below...
#
if (mysystem("$USERDEL $user")) {
if (runBusyLoop("$USERDEL $user")) {
if (($? >> 8) != $NOSUCHUSER) {
fatal("Could not remove user $user");
}
......@@ -322,14 +330,33 @@ sub DeleteUser()
return 0;
}
#
# Usage: moduser username shell full_name
#
sub ModifyUser()
{
if (@ARGV != 3) {
fatal("moduser: Wrong number of arguments");
}
my $user = shift(@ARGV);
my $shell = shift(@ARGV);
my $name = shift(@ARGV);
if (runBusyLoop("$USERMOD $user -c \"$name\" -s $shell")) {
fatal("$USERMOD: could not modify account");
}
return 0;
}
#
# Usage: username group1 [ group2 ... groupN ]
# XXX this is specific to what is required by setgroups.
#
sub ModifyUser()
sub SetGroups()
{
if (@ARGV < 2) {
fatal("moduser: Wrong number of arguments");
fatal("setgroups: Wrong number of arguments");
}
my $user = shift(@ARGV);
my $pgroup = shift(@ARGV);
......@@ -337,7 +364,7 @@ sub ModifyUser()
if (@ARGV > 0) {
$grouplist = "-G '" . join(',', @ARGV) . "'";
}
if (mysystem("$USERMOD $user -g $pgroup $grouplist")) {
if (runBusyLoop("$USERMOD $user -g $pgroup $grouplist")) {
fatal("Could not modify user $user to add groups!\n");
}
SetDotGroup($user, $pgroup);
......@@ -352,7 +379,7 @@ sub ChangePassword()
my $user = shift(@ARGV);
my $hash = shift(@ARGV);
if (mysystem("$CHPASS -p '$hash' $user")) {
if (runBusyLoop("$CHPASS -p '$hash' $user")) {
fatal("Could not change password");
}
return 0;
......@@ -375,7 +402,7 @@ sub DeactivateUser()
fatal("Could not set ZFS dir $zfsdir to mountpoint=none");
}
}
if (mysystem("$USERMOD $user -d /var/empty -s $NOLOGIN")) {
if (runBusyLoop("$USERMOD $user -d /var/empty -s $NOLOGIN")) {
fatal("Could not set shell to $NOLOGIN");
}
return 0;
......@@ -402,7 +429,7 @@ sub ReactivateUser()
fatal("Could not set ZFS dir $zfsdir to mountpoint=$hdir");
}
}
if (mysystem("$USERMOD $user -d $hdir")) {
if (runBusyLoop("$USERMOD $user -d $hdir")) {
fatal("Could not set home directory back to $hdir");
}
#
......@@ -435,7 +462,7 @@ sub AddProject()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name ...\n";
if (mysystem("$GROUPADD $unix_name -g $unix_gid")) {
if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add group $unix_name ($unix_gid)!\n");
}
}
......@@ -521,7 +548,7 @@ sub AddGroup()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name ...\n";
if (mysystem("$GROUPADD $unix_name -g $unix_gid")) {
if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add group $unix_name ($unix_gid)!\n");
}
}
......@@ -577,7 +604,7 @@ sub DelProject()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) {
print "Deleting project $unix_name ...\n";
if (mysystem("$GROUPDEL $unix_name")) {
if (runBusyLoop("$GROUPDEL $unix_name")) {
fatal("Could not delete group $unix_name!\n");
}
}
......@@ -608,7 +635,7 @@ sub DelGroup()
if (mysystem("egrep -q -s '^${unix_name}:' /etc/group") == 0) {
print "Deleting group $unix_name ...\n";
if (mysystem("$GROUPDEL $unix_name")) {
if (runBusyLoop("$GROUPDEL $unix_name")) {
fatal("Could not delete group $unix_name!\n");
}
}
......@@ -986,6 +1013,60 @@ sub mysystem($)
return system($cmd);
}
#
# Run pw/chpass, checking for a locked passwd/group file. The pw routines
# exit with non specific error code 1 for everything, so there is no way
# to tell that its a busy file except by looking at the error message. Then
# wait for a bit and try again. Silly.
#
sub runBusyLoop($)
{
my $command = shift;
my $maxtries = 10;
print STDERR "accountsetup: '$command'\n";
if (open(FD, ">>/usr/testbed/log/accountsetup.log")) {
my $tstamp = POSIX::strftime("%b %e %H:%M:%S", localtime());
print FD "$tstamp: $command\n";
close(FD);
}
while ($maxtries--) {
my $output = "";
#
# This open implicitly forks a child, which goes on to execute the
# command. The parent is going to sit in this loop and capture the
# output of the child. We do this so that we have better control
# over the descriptors.
#
my $pid = open(PIPE, "-|");
if (!defined($pid)) {
print STDERR "runBusyLoop; popen failed!\n";
return -1;
}
if ($pid) {
while (<PIPE>) {
$output .= $_;
}
close(PIPE);
print $output;
return 0
if (!$?);
if ($output =~ /(group|db) file is busy/m) {
print "runBusyLoop; waiting a few seconds before trying again\n";
sleep(3);
}
}
else {
open(STDERR, ">&STDOUT");
exec($command);
}
}
return -1;
}
sub fatal($) {
my ($msg) = @_;
......
......@@ -383,6 +383,10 @@ SWITCH: for ($cmd) {
# an account on.
#
TBNodeUpdateAccountsByUID($user);
if ($PGENISUPPORT) {
require APT_Utility;
APT_Utility::UpdateInstancesByUser($target_user);
}
exit(0);
......@@ -433,7 +437,7 @@ sub AddUser()
if (system("egrep -q -s '^${user}:' /etc/passwd")) {
print "Adding user $user ($user_number) to local node.\n";
if (system("$USERADD $user -u $user_number -c \"$fullname\" ".
if (runBusyLoop("$USERADD $user -u $user_number -c \"$fullname\" ".
"-h - -d $HOMEDIR/$user ".
"-g $default_groupname -s $PBAG")) {
fatal("Could not add user $user to local node.");
......@@ -610,7 +614,7 @@ sub DelUser()
$UID = 0;
if (system("$USERDEL $user")) {
if (runBusyLoop("$USERDEL $user")) {
if (($? >> 8) != $NOSUCHUSER) {
fatal("Could not remove user $user from local node.");
}
......@@ -888,7 +892,7 @@ sub UpdateUser(;$)
# Shell is different on local vs control node.
if ($freezeopt) {
$locshellarg = "-s $NOLOGIN";
$remshellarg = "-s $NOLOGIN";
$remshellarg = "$NOLOGIN";
}
else {
# Admin users get a local shell.
......@@ -900,16 +904,16 @@ sub UpdateUser(;$)
}
if (!defined($usr_shell) ||
!exists($shellpaths{$usr_shell})) {
$remshellarg = "-s " . $shellpaths{"tcsh"};
$remshellarg = $shellpaths{"tcsh"};
}
else {
$remshellarg = "-s " . $shellpaths{$usr_shell};
$remshellarg = $shellpaths{$usr_shell};
}
}
print "Updating user $user ($user_number) on local node.\n";
$UID = 0;
if (system("$USERMOD $user $locshellarg -c \"$fullname\" ")) {
if (runBusyLoop("$USERMOD $user $locshellarg -c \"$fullname\" ")) {
fatal("Could not modify user $user on local node.");
}
......@@ -923,7 +927,7 @@ sub UpdateUser(;$)
print "Updating user $user ($user_number) on $CONTROL\n";
if (system("$SSH -host $CONTROL ".
"'$USERMOD $user $remshellarg -c \"$fullname\"'")) {
"'$ACCOUNTPROXY moduser $user $remshellarg \"$fullname\"'")){
fatal("Could not modify user $user record on $CONTROL.");
}
}
......@@ -1077,7 +1081,7 @@ sub ThawUser()
AddUser() == 0
or fatal("Cannot thaw $user");
system("$USERMOD -n $user -s /bin/tcsh");
runBusyLoop("$USERMOD -n $user -s /bin/tcsh");
}
else {
UpdateUser(0) == 0
......@@ -1119,7 +1123,7 @@ sub DeactivateUser()
# Shell goes to nologin on the CONTROL node.
#
$UID = 0;
if (system("$USERMOD $user -s $NOLOGIN")) {
if (runBusyLoop("$USERMOD $user -s $NOLOGIN")) {
fatal("Could not set shell to $NOLOGIN for $user on local node.");
}
if ($CONTROL ne $BOSSNODE) {
......
......@@ -33,7 +33,7 @@ use vars qw(@ISA @EXPORT);
TBDB_CHECKDBSLOT_ERROR TBcheck_dbslot TBFieldErrorString
TBGetUniqueIndex ParRun VersionInfo UpdateVersionInfo
SpanningTree GenFakeMac BackTraceOnWarning PassWordHash
SSHwithTimeout TBDateStringGMT TBDateStringLocal
SSHwithTimeout TBDateStringGMT TBDateStringLocal runBusyLoop
);
use emdb;
......@@ -901,6 +901,51 @@ sub waitForMount($;$)
return -1;
}
#
# Run pw/chpass, checking for a locked passwd/group file. The pw routines
# exit with non specific error code 1 for everything, so there is no way
# to tell that its a busy file except by looking at the error message. Then
# wait for a bit and try again. Silly.
#
sub runBusyLoop($)
{
my $command = shift;
my $maxtries = 10;
while ($maxtries--) {
my $output = "";
#
# This open implicitly forks a child, which goes on to execute the
# command. The parent is going to sit in this loop and capture the
# output of the child. We do this so that we have better control
# over the descriptors.
#
my $pid = open(PIPE, "-|");
if (!defined($pid)) {
print STDERR "runBusyLoop; popen failed!\n";
return -1;
}
if ($pid) {
while (<PIPE>) {
$output .= $_;
}
close(PIPE);
print $output;
return 0
if (!$?);
if ($output =~ /(group|db) file is busy/m) {
print "runBusyLoop; waiting a few seconds before trying again\n";
sleep(3);
}
}
else {
open(STDERR, ">&STDOUT");
exec($command);
}
}
return -1;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -194,7 +194,7 @@ my $groupdir = "$GRPROOT/$pid/$gid";
if (system("egrep -q -s '^${unix_name}:' /etc/group")) {
print "Adding group $unix_name to local node ...\n";
if (system("$GROUPADD $unix_name -g $unix_gid")) {
if (runBusyLoop("$GROUPADD $unix_name -g $unix_gid")) {
fatal("Could not add $unix_gid ($unix_gid) to local node!\n");
}
}
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -92,6 +92,7 @@ $| = 1;
use lib "@prefix@/lib";
use libaudit;
use libdb;
use emutil;
use libtestbed;
use User;
use Group;
......@@ -190,7 +191,7 @@ if (system("grep -q '^${unix_gid}:' /etc/group")) {
#
print "Removing group $unix_name ($unix_gid) on local node.\n";
if (system("$GROUPDEL $unix_name")) {
if (runBusyLoop("$GROUPDEL $unix_name")) {
if (($? >> 8) != 65) {
fatal("Could not remove group $unix_name from local node!");
}
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -389,7 +389,7 @@ foreach my $token (@userlist) {
#
# MAKE SURE not to update anything else!
#
if (system("$USERMOD $uid -g $project $groupargument")) {
if (runBusyLoop("$USERMOD $uid -g $project $groupargument")) {
fatal("Could not modify user $uid on local node.");
}
......@@ -398,7 +398,7 @@ foreach my $token (@userlist) {
if ($control_node ne $BOSSNODE) {
$groupargument = join(' ', @groupnames);
if (system("$SSH -host $control_node ".
"$ACCOUNTPROXY moduser $uid $project $groupargument")) {
"$ACCOUNTPROXY setgroups $uid $project $groupargument")) {
fatal("Could not modify user $uid record on $control_node.");
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment