genelists.in 15.5 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh Stoller's avatar
Leigh Stoller committed
2 3

#
4
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh Stoller's avatar
Leigh Stoller committed
24
#
25
use Fcntl ':flock';
26
use English;
27
use Getopt::Std;
28 29

sub usage() {
30 31 32
    print("Usage: genelists [-d] [-n] -a\n".
	  "Usage: genelists [-d] [-n] [-m] -u user\n".
	  "Usage: genelists [-d] [-n] -p project\n".
33
	  "Usage: genelists [-d] [-n] [-P | -t]\n".
34 35 36 37 38
	  "where:\n".
	  "  -d    - Turn on debugging\n".
	  "  -n    - Impotent mode\n".
	  "  -u    - Generate lists for a user; add -m for new email address\n".
	  "  -p    - Generate lists for a project (includes subgroups)\n".
39
	  "  -P    - Generate lists for all projects (includes subgroups)\n".
40
	  "  -t    - Generate activity lists\n".
41
	  "  -c    - Generate just the current users list\n".
42
	  "  -a    - Generate all email lists; careful ...\n");
43 44
    exit(-1);
}
45 46 47 48

sub ActiveUsers();
sub RecentUsers();
sub RecentProjects();
49
sub RecentProjectLeaders();
50 51 52 53
sub Users();
sub WideAreaPeople();
sub ProjectLeaders();
sub ProjectLists($$);
54
sub genelist($$$$);
55

56
my $optlist = "anu:p:tdmfcP";
57 58 59 60
my $debug   = 0;
my $all     = 0;
my $update  = 0;
my $activity= 0;
61
my $projects= 0;
62
my $current = 0;
63
my $impotent= 0;
64
my $force   = 0;
65 66
my $pid;
my $user;
67 68 69 70

# Configure variables
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
71
my $USERS       = "@USERNODE@";
72
my $OURDOMAIN   = "@OURDOMAIN@";
73 74
my $TBACTIVE    = "@TBACTIVEARCHIVE@";
my $TBALL       = "@TBUSERSARCHIVE@";
75
my $ELISTS      = "$TB/lists";
76
my $ELABINELAB  = @ELABINELAB@;
77 78
my $MAILMANSUPPORT= @MAILMANSUPPORT@;
my $MMPROG	= "$TB/sbin/setmmlistmembers";
79
my $PGENISUPPORT= @PROTOGENI_SUPPORT@;
80 81

# Note no -n option. We redirect stdin from the new exports file below.
82
my $SSH		= "$TB/bin/sshtb -l root -host $USERS";
83
my $PROG	= "/usr/testbed/sbin/genelists.proxy";
84
my $lockfile    = "/var/tmp/testbed_genelists_lockfile";
85
my $tempfile    = "/var/tmp/testbed_genelists_tempfile";
86
my $SAVEUID	= $UID;
87

88 89 90
#
# Turn off line buffering on output
#
91
$| = 1;
92 93

# Load the Testbed support stuff.
94
use lib "@prefix@/lib";
95
use emdbi;
96 97
use libdb;
use libtestbed;
98
use libtblog;
99
use User;
100

101 102 103 104
#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
Leigh Stoller's avatar
Leigh Stoller committed
105 106
    die("*** $0:\n".
	"    Must be root! Maybe its a development version?\n");
107 108
}
# XXX Hacky!
109
if (0 && $TB ne "/usr/testbed") {
Leigh Stoller's avatar
Leigh Stoller committed
110 111
    die("*** $0:\n".
	"    Wrong version. Maybe its a development version?\n");
112
}
113 114 115 116 117 118 119

#
# un-taint path
#
$ENV{'PATH'} = '/bin:/usr/bin:/usr/sbin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

120 121 122 123 124 125 126 127 128 129 130
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV) {
    usage();
}
131
if (defined($options{"d"})) {
132
    $debug++;
133
}
134 135 136
if (defined($options{"f"})) {
    $force = 1;
}
137 138 139
if (defined($options{"c"})) {
    $current = 1;
}
140
if (defined($options{"a"})) {
141 142 143 144 145
    $all = 1;
}
if (defined($options{"m"})) {
    $update = 1;
}
146 147 148
if (defined($options{"P"})) {
    $projects = 1;
}
149 150
if (defined($options{"t"})) {
    $activity = 1;
151 152
}
if (defined($options{"n"})) {
153 154 155 156 157
    $impotent = 1;
}
if (defined($options{"u"})) {
    $user = $options{"u"};
    
158 159 160
    #
    # Untaint.
    #
161 162
    if ($user =~ /^([-\w]+)$/) {
	$user = $1;
163 164
    }
    else {
165
	die("Tainted argument $user!\n");
166 167
    }
}
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186

if (defined($options{"p"})) {
    $pid = $options{"p"};
    
    #
    # Untaint.
    #
    if ($pid =~ /^([-\w]+)$/) {
	$pid = $1;
    }
    else {
	die("Tainted argument $pid!\n");
    }
}

if (defined($user) && defined($pid)) {
    usage();
}
if ($update && !defined($user)) {
187 188 189
    usage();
}

190 191
#
# We need to serialize this script to avoid a trashed map file. Use
192
# a dummy file in /var/tmp, opened for writing and flock'ed.
193
#
194 195
open(LOCK, ">>$lockfile") || fatal("Couldn't open $lockfile\n");
$count = 0;
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
if (flock(LOCK, LOCK_EX|LOCK_NB) == 0) {
    #
    # If we don't get it the first time, we wait for:
    # 1) The lock to become free, in which case we do our thing
    # 2) The time on the lock to change, in which case we wait for that process
    #    to finish
    #
    my $oldlocktime = (stat(LOCK))[9];
    my $gotlock = 0;
    while (1) {
	print "Another genelists in progress, waiting for it to finish\n";
	if (flock(LOCK, LOCK_EX|LOCK_NB) != 0) {
	    # OK, got the lock, we can do what we're supposed to
	    $gotlock = 1;
	    last;
	}
	$locktime = (stat(LOCK))[9];
	if ($locktime != $oldlocktime) {
	    $oldlocktime = $locktime;
	    last;
	}
	if ($count++ > 20)  {
	    fatal("Could not get the lock after a long time!\n");
	}
	sleep(1);
    }

    $count = 0;
    #
    # If we didn't get the lock, wait for the processes that did to finish
    #
    if (!$gotlock) {
228 229 230 231 232 233 234 235
	while (1) {
	    if ((stat(LOCK))[9] != $oldlocktime) {
		exit(0);
	    }
	    if (flock(LOCK, LOCK_EX|LOCK_NB) != 0) {
		close(LOCK);
		exit(0);
	    }
236
	    if ($count++ > 30)  {
237
		fatal("Process with the lock didn't finish after a long time!\n");
238
	    }
239
	    sleep(1);
240
	}
241 242 243
    }
}

244 245 246 247 248 249
#
# Perl-style touch(1)
#
my $now = time;
utime $now, $now, $lockfile;

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
#
# Grab the mailman admin password for TBOPS.
#
my $admin_address;

if ($MAILMANSUPPORT) {
    my $mailman_password;
    
    if (! TBGetSiteVar("general/mailman/password", \$mailman_password)) {
	fatal("Could not mailman admin password from sitevars!");
    }
    $admin_address = "$TBOPS $mailman_password 'Emulab Operations'";
}
else {
    $admin_address = $TBOPS;
}

267
ActiveUsers()
268
    if ($all || $activity || $update || $current);
269 270 271 272 273 274 275

RecentUsers()
    if ($all || $activity || $update);

RecentProjects()
    if ($all || $activity || $update);

276 277 278
RecentProjectLeaders()
    if ($all || $activity || $update);

279 280 281 282 283 284 285 286 287
Users()
    if ($all || defined($user));

WideAreaPeople()
    if ($all || defined($user));

ProjectLeaders()
    if ($all || defined($user) || defined($pid));

288
if ($all || $projects || defined($user) || defined($pid)) {
289 290 291 292
    my $query;
    my $phash = {};
    my $query_result;
    
293
    if ($all || $projects) {
294 295
	$query = "select g.pid,g.gid from groups as g ".
		 "left join projects as p on p.pid=g.pid ".
296 297
		 "where p.approved=1 and p.nonlocal_id is null ".
		 "order by g.pid,g.gid";
298 299
    }
    elsif ($user) {
300 301 302 303 304
	$query  = "select g.pid,g.gid from group_membership as g ".
	          "left join projects as p on p.pid=g.pid ".
		  "where g.uid='$user' and p.approved=1 and ".
		  "    p.nonlocal_id is null " .
		  "order by g.pid,g.gid";
305 306
    }
    else {
307 308
	$query  = "select g.pid,g.gid from groups as g ".
	          "left join projects as p on p.pid=g.pid ".
309
		  "where g.pid='$pid' and p.nonlocal_id is null " .
310
		  "order by g.pid,g.gid";
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
    }

    if (! ($query_result = DBQuery($query))) {
	DBFatal("Getting Project List!");
    }
    while (my ($pid,$gid) = $query_result->fetchrow_array()) {
	ProjectLists($pid, $gid);
    }
}

#
# Close the lock file. Exiting releases it, but might as well.
#
close(LOCK);
exit 0;

#
# All active users on the testbed
#
sub ActiveUsers()
{
332
    my $userlist;
333 334 335 336 337 338 339
    my $query_result;

    print "Getting Active Users\n" if $debug;
    
    if (! ($query_result =
	   DBQuery("SELECT DISTINCT u.usr_email from experiments as e ".
		   "left join group_membership as p ".
340
		   "     on e.pid_idx=p.pid_idx and p.pid_idx=p.gid_idx ".
341
		   "left join users as u on u.uid_idx=p.uid_idx ".
342 343 344 345 346 347 348
		   "where u.status='active' and ".
		   "      e.state='active' ".
		   "order by u.usr_email"))) {
	DBFatal("Getting Active Users!");
    }
    $userlist = "$TBOPS\n".
	        "$TBACTIVE";
349

350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if ($PGENISUPPORT) {
	require GeniDB;
	require GeniUser;
	require GeniHRN;

	if (emdbi::DBExists(GeniDB::GENICM_DBNAME())) {
	    # Connect to the proper DB.
	    GeniDB::DBConnect(GeniDB::GENICM_DBNAME());

	    my $geni_users =
		GeniDB::DBQueryFatal("select distinct u.email ".
				     "  from geni_aggregates as a ".
				     "left join geni_users as u on ".
				     "     a.creator_uuid=u.uuid ".
				     "where u.email is not null");
	    
	    while (my ($email) = $geni_users->fetchrow_array()) {
		$userlist = "$email\n" . $userlist;
	    }
	}
    }
371
    genelist($query_result, $userlist, "emulab-active-users", 0);
372 373 374 375 376 377 378 379 380 381 382 383 384 385
}

#
# Recently active users.
# 
sub RecentUsers()
{
    my $userlist;
    my $query_result;

    my $limit = (60 * 60 * 24) * TBGetSiteVar("general/recently_active");
    print "Getting Recently Active Users\n" if $debug;

    if (! ($query_result =
386
	   DBQuery("select distinct u.usr_email from user_stats as s ".
387
		   "left join users as u on u.uid_idx=s.uid_idx ".
388 389
		   "where ((UNIX_TIMESTAMP(now()) - ".
		   "       UNIX_TIMESTAMP(s.last_activity)) <= $limit) ".
390 391
		   "order by u.usr_email"))) {
	DBFatal("Getting Recently Active Users!");
392
    }
393 394
    $userlist  = "$TBOPS\n";
    $userlist .= "$TBACTIVE";
395

396
    genelist($query_result, $userlist, "emulab-recently-active-users", 0);
397 398 399 400 401 402 403 404 405 406 407 408 409 410
}

#
# Recently active projects (members).
#
sub RecentProjects()
{
    my $userlist;
    my $query_result;

    my $limit = (60 * 60 * 24) * TBGetSiteVar("general/recently_active");
    print "Getting Recently Active Projects (members)\n" if $debug;

    if (! ($query_result =
411 412
	   DBQuery("select distinct u.usr_email from project_stats as s ".
		   "left join group_membership as g on ".
413
		   "  g.pid_idx=s.pid_idx and g.gid_idx=g.pid_idx ".
414
		   "left join users as u on u.uid_idx=g.uid_idx ".
415 416 417
		   "where u.status='active' and ".
		   "      ((UNIX_TIMESTAMP(now()) - ".
		   "       UNIX_TIMESTAMP(s.last_activity)) <= $limit) ".
418 419
		   "order by u.usr_email"))) {
	DBFatal("Getting Recently Active Projects!");
420
    }
421 422
    $userlist  = "$TBOPS\n";
    $userlist .= "$TBACTIVE";
423

424
    genelist($query_result, $userlist, "emulab-recently-active-projects", 0);
425 426
}

427 428 429 430 431 432 433 434 435 436 437 438 439
#
# Recently active projects (leaders).
#
sub RecentProjectLeaders()
{
    my $userlist;
    my $query_result;

    my $limit = (60 * 60 * 24) * TBGetSiteVar("general/recently_active");

    if (! ($query_result =
	   DBQuery("select distinct u.usr_email from project_stats as s ".
		   "left join group_membership as g on ".
440
		   "  g.pid_idx=s.pid_idx and g.gid_idx=g.pid_idx ".
441
		   "left join users as u on u.uid_idx=g.uid_idx ".
442
                   "left join projects as p on u.uid_idx=p.head_idx ".
443 444 445 446 447 448 449 450 451 452 453 454 455 456
		   "where u.status='active' and ".
		   "      ((UNIX_TIMESTAMP(now()) - ".
		   "       UNIX_TIMESTAMP(s.last_activity)) <= $limit) ".
                   " and p.pid is not null " .
		   "order by u.usr_email"))) {
	DBFatal("Getting Recently Active Project Leaders!");
    }
    $userlist  = "$TBOPS\n";
    $userlist .= "$TBACTIVE";

    genelist($query_result, $userlist,
             "emulab-recently-active-project-leaders", 0);
}

457 458 459 460 461 462 463 464 465 466 467 468 469 470
#
# All active users of the testbed.
# 
sub Users()
{
    my $userlist;
    my $query_result;

    print "Getting All Users\n" if $debug;

    if (! ($query_result =
	   DBQuery("SELECT DISTINCT usr_email FROM users ".
		   "where status='active' order by usr_email"))) {
	DBFatal("Getting Users!");
471
    }
472 473
    $userlist  = "$TBOPS\n";
    $userlist .= "$TBALL";
474
    
475
    genelist($query_result, $userlist, "emulab-allusers", 0);
476 477
}

478 479 480
#
# Special list for people approved to use the widearea-nodes.
#
481 482 483 484
sub WideAreaPeople()
{
    my $query_result =
	DBQueryFatal("SELECT DISTINCT u.usr_email from projects as p ".
485
		     "left join group_membership as m on m.pid_idx=p.pid_idx ".
486
		     "left join users as u on u.uid_idx=m.uid_idx ".
487 488 489 490
		     "where p.approved!=0 and p.pcremote_ok is not null ".
		     "      and m.trust!='none' and u.status='active' ".
		     "order by usr_email");

491
    genelist($query_result, "$TBOPS", "emulab-widearea-users", 0);
492
}
493

494
#
495
# Another list of project leaders.
496
#
497 498 499
sub ProjectLeaders()
{
    my $query_result =
500 501 502 503
	DBQueryFatal("SELECT DISTINCT u.usr_email ".
		     ($MAILMANSUPPORT ?
		      ", u.uid ,u.usr_name, u.mailman_password " : "") .
		     "  from projects as p ".
504
		     "left join users as u on u.uid_idx=p.head_idx ".
505 506
		     "where p.approved!=0 ".
		     "order by usr_email");
507

508
    genelist($query_result, "$TBOPS", "emulab-project-leaders", 0);
509
}
510

511
#
512
# Regen project lists. 
513
#
514 515 516
sub ProjectLists($$)
{
    my ($pid, $gid) = @_;
517 518
    my $proj_result;

519
    print "Getting project members for $pid/$gid\n" if $debug;
520

521
    my $query_result =
522 523 524 525
	DBQueryFatal("SELECT distinct u.usr_email ".
		     ($MAILMANSUPPORT ?
		      ", u.uid ,u.usr_name, u.mailman_password " : "") .
		     " from group_membership as p ".
526
		     "left join users as u on u.uid_idx=p.uid_idx ".
527
		     "where p.pid='$pid' and p.gid='$gid' and ".
528
		     " p.trust!='none' and u.status='active' ".
529
		     "order by u.usr_email");
530

531 532
    if ($query_result->numrows) {
	if ($pid eq $gid) {
533 534
	    genelist($query_result, undef, "$pid-users",
		     ($pid eq "CloudLab" ? 0 : 1));
535 536
	}
	else {
537 538
	    genelist($query_result, undef, "$pid-$gid-users",
		     ($ELABINELAB ? 0 : 1));
539 540 541 542 543 544 545
	}
    }
}

#
# Generate and fire over a list.
#
546
sub genelist($$$$)
547
{
548
    my($query_result, $inituserlist, $listname, $usemailman) = @_;
549 550

    print "Processing $listname at: \t".time()." \t(".
551
      $query_result->numrows()." entries)\n" if $debug>1;
552

553 554
    open(LIST,"> $tempfile") ||
	fatal("Couldn't open $tempfile: $!\n");
555

556 557 558
    print LIST "#\n";
    print LIST "# WARNING! THIS FILE IS AUTOGENERATED. DO NOT EDIT!\n";
    print LIST "#\n";
559 560 561
    if (defined($inituserlist)) {
	print LIST "$inituserlist\n";
    }
562 563

    for ($i = 0; $i < $query_result->numrows; $i++) {
564 565 566
	my ($user_email, $uid, $user_name, $mailman_password) =
	    $query_result->fetchrow_array();
	
567 568 569
	if (! defined($user_email)) {
	    next;
	}
570 571
	# HACK! These special accounts should be flagged in the DB
	next 
572 573 574
	    if ($usemailman && $MAILMANSUPPORT &&
		($uid eq "elabman" || $uid eq "elabckup" ||
		 $uid eq "operator"));
575
	
576
	if ($usemailman && $MAILMANSUPPORT) {
577
	    print LIST "$uid $user_email $mailman_password '$user_name'\n";
578 579 580 581 582
	}
	else {
	    print LIST "$user_email\n";
	    print "$user_email\n" if $debug>1;
	}
583 584
    }
    close(LIST);
585 586 587 588 589 590
    chmod(0664, $tempfile);

    if (! -d $ELISTS) {
	if (! mkdir($ELISTS, 0770)) {
	    fatal("Could not make directory $ELISTS: $!");
	}
591

592 593 594 595 596 597 598
	if (! chmod(0775, $ELISTS)) {
	    fatal("Could not chmod directory $ELISTS: $!");
	}
    }

    if (-e "$ELISTS/$listname" &&
	system("cmp -s $tempfile $ELISTS/$listname") == 0) {
599
	print "$listname has not changed. Skipping.\n"
600
	    if ($debug && !$force);
601 602 603 604
	if (!$force) {
	    unlink("$tempfile");
	    return;
	}
605 606
    }

Leigh Stoller's avatar
Leigh Stoller committed
607
    system("/bin/cp -pf $tempfile $ELISTS/$listname") == 0 ||
608 609
	fatal("Could not move $tempfile to $ELISTS/$listname: $!");
    
610 611 612
    #
    # Fire the new file over to the fileserver to finish up.
    #
613
    if (!$impotent) {
614 615 616 617 618 619 620 621 622 623 624 625 626 627
	if ($usemailman && $MAILMANSUPPORT) {
	    my $optarg = ($debug ? "-d" : "");

	    $EUID = $UID;
	    system("$MMPROG $optarg $listname $tempfile") == 0 or
		fatal("Failed: $MMPROG $listname $tempfile: $?");
	    $EUID = 0;
	}
	else {
	    $UID = 0;
	    system("$SSH $PROG $listname < $tempfile") == 0 or
		fatal("Failed: $SSH $PROG $listname < $tempfile: $?");
	    $UID = $SAVEUID;
	}
628
    }
629
    unlink("$tempfile");
630 631
}

632 633
sub fatal {
  local($msg) = $_[0];
634
  SENDMAIL($TBOPS, "Failure Generating Email Lists", $msg);
635 636
  die($msg);
}