Commit ff9061d4 authored by Leigh Stoller's avatar Leigh Stoller

Big set of changes intended to solve a couple of problems with long

term archiving of firstclass objects like users, projects, and of
course templates.

* Projects, Users, and Groups are now uniquely identified inside the
  DB by a index value that will not be reused. If necessary, this
  could easily be a globally unique identifier, but without federation
  there is no reason to do that yet.

* Currently, pid, gid, and uid still need to be locally unique until
  all of the changes are in place (which is going to take a fairly
  long time since the entire system operates in terms of those, except
  for the few places that I had to change to get the ball rolling).

* We currently archive deleted users to the deleted_users table (their
  user_stats are kept forever since they are indexed by the new index
  column). Eventually do the same with projects (not sure about
  groups) but since we rarely if ever delete a project, there is no
  rush on this one.

* At the same time, I have started a large reorg of the code, to move
  all of the user, group, project code into modules, both in php and
  perl, turning them into first class "objects" (as far as that goes
  in php and perl). Eventually, the number of query statements
  scattered around the code will be manageable, or so I hope.

* Another related part of this reorg is to make it easier to move the
  new user/project/group code in the perl backend so that it can be
  made available via the xmlrpc interface (without duplication of the
  code).
parent 52411ea0
......@@ -20,7 +20,7 @@ SBIN_SCRIPTS = avail inuse showgraph if2port backup webcontrol node_status \
dumperrorlog
LIBEXEC_SCRIPTS = webnodelog webnfree webnewwanode webidlemail xmlconvert
LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \
NodeType.pm Interface.pm
NodeType.pm Interface.pm User.pm Group.pm Project.pm
# Stuff installed on plastic.
USERSBINS = genelists.proxy dumperrorlog.proxy
......
This diff is collapsed.
......@@ -17,7 +17,7 @@ use vars qw(@ISA @EXPORT);
use lib '@prefix@/lib';
use libdb;
use libtestbed;
use libtblog;
use Group;
use English;
use Data::Dumper;
use File::Basename;
......@@ -27,3 +27,197 @@ use overload ('""' => 'Stringify');
my $TB = "@prefix@";
my $BOSSNODE = "@BOSSNODE@";
my $CONTROL = "@USERNODE@";
# Cache of instances to avoid regenerating them.
my %projects = ();
my $debug = 0;
# Little helper and debug function.
sub mysystem($)
{
my ($command) = @_;
print STDERR "Running '$command'\n"
if ($debug);
return system($command);
}
#
# Lookup by idx.
#
sub Lookup($$)
{
my ($class, $pid_idx) = @_;
# Look in cache first
return $projects{"$pid_idx"}
if (exists($projects{"$pid_idx"}));
my $query_result =
DBQueryWarn("select * from projects where pid_idx=$pid_idx");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'PROJECT'} = $query_result->fetchrow_hashref();
bless($self, $class);
# Add to cache.
$projects{"$pid_idx"} = $self;
return $self;
}
# accessors
sub field($$) { return ((! ref($_[0])) ? -1 : $_[0]->{'PROJECT'}->{$_[1]}); }
sub pid($) { return field($_[0], "pid"); }
sub gid($) { return field($_[0], "gid"); }
sub pid_idx($) { return field($_[0], "pid_idx"); }
sub gid_idx($) { return field($_[0], "gid_idx"); }
sub leader($) { return field($_[0], "leader"); }
sub created($) { return field($_[0], "created"); }
sub description($) { return field($_[0], "description"); }
sub unix_gid($) { return field($_[0], "unix_gid"); }
sub unix_name($) { return field($_[0], "unix_name"); }
sub expt_count($) { return field($_[0], "expt_count"); }
sub expt_last($) { return field($_[0], "expt_last"); }
sub wikiname($) { return field($_[0], "wikiname"); }
sub mailman_password($) { return field($_[0], "mailman_password"); }
#
# Lookup given pid For backwards compat.
#
sub LookupByPid($$)
{
my ($class, $pid) = @_;
my $query_result =
DBQueryWarn("select pid_idx from projects where pid='$pid'");
return undef
if (! $query_result || !$query_result->numrows);
my ($pid_idx) = $query_result->fetchrow_array();
return Project->Lookup($pid_idx);
}
#
# Refresh a class instance by reloading from the DB.
#
sub Refresh($)
{
my ($self) = @_;
return -1
if (! ref($self));
my $pid_idx = $self->pid_idx();
my $query_result =
DBQueryWarn("select * from projects where pid_idx=$pid_idx");
return -1
if (!$query_result || !$query_result->numrows);
$self->{'PROJECT'} = $query_result->fetchrow_hashref();
return 0;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $pid = $self->pid();
my $pid_idx = $self->pid_idx();
return "[Project: $pid, IDX: $pid_idx]";
}
#
# Perform some updates ...
#
sub Update($$)
{
my ($self, $argref) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $pid_idx = $self->pid_idx();
my $query = "update projects set ".
join(",", map("$_='" . $argref->{$_} . "'", keys(%{$argref})));
$query .= " where pid_idx='$pid_idx'";
return -1
if (! DBQueryWarn($query));
return Refresh($self);
}
#
# Delete a user from the project (and all subgroups of course). No checks
# are made; that should be done higher up. Optionally return list of groups
# from which the user was deleted.
#
sub DeleteUser($$;$)
{
my ($self, $user, $plist) = @_;
my @grouplist = ();
# Must be a real reference.
return -1
if (! (ref($self) && ref($user)));
my $uid_idx = $user->uid_idx();
my $pid_idx = $self->pid_idx();
my $gid_idx = $pid_idx;
# Need a list of all groups for the user in this project.
# Should probably use a "class" method in the Groups module.
my $query_result =
DBQueryWarn("select gid_idx from group_membership ".
"where pid_idx=$pid_idx and uid_idx=$uid_idx");
return -1
if (! $query_result);
return 0
if (! $query_result->numrows);
while (my ($idx) = $query_result->fetchrow_array()) {
# Do main group last.
next
if ($idx == $gid_idx);
my $group = Group->Lookup($idx);
return -1
if (!defined($group));
return -1
if ($group->DeleteMemberShip($user) < 0);
push(@grouplist, $group);
}
my $group = Group->Lookup($gid_idx);
return -1
if (!defined($group));
return -1
if ($group->DeleteMemberShip($user) < 0);
if (defined($plist)) {
@$plist = (@grouplist, $group);
}
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -17,7 +17,6 @@ use vars qw(@ISA @EXPORT);
use lib '@prefix@/lib';
use libdb;
use libtestbed;
use libtblog;
use English;
use Data::Dumper;
use File::Basename;
......@@ -27,3 +26,202 @@ use overload ('""' => 'Stringify');
my $TB = "@prefix@";
my $BOSSNODE = "@BOSSNODE@";
my $CONTROL = "@USERNODE@";
# Cache of instances to avoid regenerating them.
my %users = ();
my $debug = 0;
# Little helper and debug function.
sub mysystem($)
{
my ($command) = @_;
print STDERR "Running '$command'\n"
if ($debug);
return system($command);
}
#
# Lookup by idx.
#
sub Lookup($$)
{
my ($class, $uid_idx) = @_;
# Look in cache first
return $users{"$uid_idx"}
if (exists($users{"$uid_idx"}));
my $query_result =
DBQueryWarn("select * from users where uid_idx=$uid_idx");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'USER'} = $query_result->fetchrow_hashref();
bless($self, $class);
# Add to cache.
$users{"$uid_idx"} = $self;
return $self;
}
# accessors
sub field($$) { return ((! ref($_[0])) ? -1 : $_[0]->{'USER'}->{$_[1]}); }
sub uid_idx($) { return field($_[0], "uid_idx"); }
sub uid($) { return field($_[0], "uid"); }
sub created($) { return field($_[0], "usr_created"); }
sub expires($) { return field($_[0], "usr_expires"); }
sub modified($) { return field($_[0], "usr_modified"); }
sub name($) { return field($_[0], "usr_name"); }
sub title($) { return field($_[0], "usr_title"); }
sub affil($) { return field($_[0], "usr_affil"); }
sub email($) { return field($_[0], "usr_email"); }
sub URL($) { return field($_[0], "usr_URL"); }
sub addr($) { return field($_[0], "usr_addr"); }
sub addr2($) { return field($_[0], "usr_addr2"); }
sub city($) { return field($_[0], "usr_city"); }
sub state($) { return field($_[0], "usr_state"); }
sub zip($) { return field($_[0], "usr_zip"); }
sub country($) { return field($_[0], "usr_country"); }
sub phone($) { return field($_[0], "usr_phone"); }
sub shell($) { return field($_[0], "usr_shell"); }
sub pswd($) { return field($_[0], "usr_pswd"); }
sub w_pswd($) { return field($_[0], "usr_w_pswd"); }
sub unix_uid($) { return field($_[0], "unix_uid"); }
sub status($) { return field($_[0], "status"); }
sub admin($) { return field($_[0], "admin"); }
sub foreign_admin($) { return field($_[0], "foreign_admin"); }
sub dbedit($) { return field($_[0], "dbedit"); }
sub stud($) { return field($_[0], "stud"); }
sub webonly($) { return field($_[0], "webonly"); }
sub pswd_expires($) { return field($_[0], "pswd_expires"); }
sub cvsweb($) { return field($_[0], "cvsweb"); }
sub emulab_pubkey($) { return field($_[0], "emulab_pubkey"); }
sub home_pubkey($) { return field($_[0], "home_pubkey"); }
sub adminoff($) { return field($_[0], "adminoff"); }
sub verify_key($) { return field($_[0], "verify_key"); }
sub widearearoot($) { return field($_[0], "widearearoot"); }
sub wideareajailroot($) { return field($_[0], "wideareajailroot"); }
sub notes($) { return field($_[0], "notes"); }
sub weblogin_frozen($) { return field($_[0], "weblogin_frozen"); }
sub weblogin_failcount($){return field($_[0], "weblogin_failcount");}
sub weblogin_failstamp($){return field($_[0], "weblogin_failstamp");}
sub plab_user($) { return field($_[0], "plab_user"); }
sub user_interface($) { return field($_[0], "user_interface"); }
sub chpasswd_key($) { return field($_[0], "chpasswd_key"); }
sub chpasswd_expires($) { return field($_[0], "chpasswd_expires"); }
sub wikiname($) { return field($_[0], "wikiname"); }
sub wikionly($) { return field($_[0], "wikionly"); }
sub mailman_password($) { return field($_[0], "mailman_password"); }
#
# Lookup user given a plain uid. For backwards compat.
#
sub LookupByUid($$)
{
my ($class, $uid) = @_;
my $query_result =
DBQueryWarn("select uid_idx from users where uid='$uid'");
return undef
if (! $query_result || !$query_result->numrows);
my ($uid_idx) = $query_result->fetchrow_array();
return User->Lookup($uid_idx);
}
#
# Lookup user given the unix uid ($UID typically).
#
sub LookupByUnixId($$)
{
my ($class, $unix_uid) = @_;
my $query_result =
DBQueryFatal("select uid,uid_idx from users ".
"where unix_uid='$unix_uid'");
return undef
if (! $query_result || !$query_result->numrows);
my ($uid, $uid_idx) = $query_result->fetchrow_array();
# Sanity check against password file before returning.
my ($pwname) = getpwuid($unix_uid) or
die("*** $unix_uid is not in the password file!");
if ($uid ne $pwname) {
warn("*** WARNING: LookupByUnixId: $pwname does not match $uid\n");
return undef
}
return User->Lookup($uid_idx);
}
#
# Refresh a class instance by reloading from the DB.
#
sub Refresh($)
{
my ($self) = @_;
return -1
if (! ref($self));
my $uid_idx = $self->uid_idx();
my $query_result =
DBQueryWarn("select * from users where uid_idx=$uid_idx");
return -1
if (!$query_result || !$query_result->numrows);
$self->{'USER'} = $query_result->fetchrow_hashref();
return 0;
}
#
# Stringify for output.
#
sub Stringify($)
{
my ($self) = @_;
my $uid = $self->uid();
my $uid_idx = $self->uid_idx();
return "[User: $uid, IDX: $uid_idx]";
}
#
# Perform some updates ...
#
sub Update($$)
{
my ($self, $argref) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $uid_idx = $self->uid_idx();
my $query = "update users set ".
join(",", map("$_='" . $argref->{$_} . "'", keys(%{$argref})));
$query .= " where uid_idx='$uid_idx'";
return -1
if (! DBQueryWarn($query));
return Refresh($self);
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -239,6 +239,9 @@ use libtblog_simple;
use English;
use File::Basename;
use POSIX qw(strftime);
require User;
require Project;
require Group;
require Node;
require NodeType;
require Mysql;
......@@ -5001,6 +5004,12 @@ sub GatherSwapStats($$$$$;$)
$lastswapuid = $uid
if (!defined($lastswapuid) || $lastswapuid eq "");
# Need these for user_stats below. Eventually do major rewrite!
my $thisuser = User->LookupByUid($uid);
my $lastuser = User->LookupByUid($lastswapuid);
my $this_uid_idx = $thisuser->uid_idx();
my $last_uid_idx = $thisuser->uid_idx();
#
# A non-zero ecode indicates error. If op is a preload/swapin/start/modify
# then we do not want to gather anymore stats beyond the error code
......@@ -5198,10 +5207,10 @@ sub GatherSwapStats($$$$$;$)
" allexpt_vnode_duration+($vnodes * ${duration}), ".
" allexpt_pnode_duration=".
" allexpt_pnode_duration+($pnodes * ${duration}) ".
"where uid=".
"where uid_idx=".
((($mode eq TBDB_STATS_SWAPOUT) ||
($mode eq TBDB_STATS_SWAPMODIFY)) ?
"'$lastswapuid'" : "'$uid'"));
"'$last_uid_idx'" : "'$this_uid_idx'"));
#
# Project/group aggregate is a little more convenient to work with
......@@ -5297,7 +5306,7 @@ sub GatherSwapStats($$$$$;$)
# We do these here since even failed operations implies activity
DBQueryWarn("update user_stats set last_activity=now() ".
"where uid='$uid'");
"where uid_idx='$this_uid_idx'");
DBQueryWarn("update project_stats set last_activity=now() ".
"where pid='$pid'");
DBQueryWarn("update group_stats set last_activity=now() ".
......
......@@ -284,7 +284,7 @@ CREATE TABLE delays (
CREATE TABLE deleted_users (
uid varchar(8) NOT NULL default '',
uid_idx smallint(5) unsigned NOT NULL default '0',
uid_idx mediumint(8) unsigned NOT NULL default '0',
usr_created datetime default NULL,
usr_deleted datetime default NULL,
usr_name tinytext,
......@@ -996,12 +996,16 @@ CREATE TABLE group_membership (
uid varchar(8) NOT NULL default '',
gid varchar(16) NOT NULL default '',
pid varchar(12) NOT NULL default '',
uid_idx mediumint(8) unsigned NOT NULL default '0',
pid_idx mediumint(8) unsigned NOT NULL default '0',
gid_idx mediumint(8) unsigned NOT NULL default '0',
trust enum('none','user','local_root','group_root','project_root') default NULL,
date_applied date default NULL,
date_approved datetime default NULL,
PRIMARY KEY (uid,gid,pid),
PRIMARY KEY (uid_idx,gid_idx),
KEY pid (pid),
KEY gid (gid)
KEY gid (gid),
UNIQUE KEY uid (uid,pid,gid)
) TYPE=MyISAM;
--
......@@ -1024,6 +1028,7 @@ CREATE TABLE group_policies (
CREATE TABLE group_stats (
pid varchar(12) NOT NULL default '',
gid varchar(12) NOT NULL default '',
gid_idx mediumint(8) unsigned NOT NULL default '0',
exptstart_count int(11) unsigned default '0',
exptstart_last datetime default NULL,
exptpreload_count int(11) unsigned default '0',
......@@ -1040,7 +1045,8 @@ CREATE TABLE group_stats (
allexpt_vnode_duration int(11) unsigned default '0',
allexpt_pnodes int(11) unsigned default '0',
allexpt_pnode_duration int(11) unsigned default '0',
PRIMARY KEY (pid,gid)
PRIMARY KEY (gid_idx),
UNIQUE KEY pidgid (pid,gid)
) TYPE=MyISAM;
--
......@@ -1050,6 +1056,8 @@ CREATE TABLE group_stats (
CREATE TABLE groups (
pid varchar(12) NOT NULL default '',
gid varchar(12) NOT NULL default '',
pid_idx mediumint(8) unsigned NOT NULL default '0',
gid_idx mediumint(8) unsigned NOT NULL default '0',
leader varchar(8) NOT NULL default '',
created datetime default NULL,
description tinytext,
......@@ -1059,10 +1067,12 @@ CREATE TABLE groups (
expt_last date default NULL,
wikiname tinytext,
mailman_password tinytext,
PRIMARY KEY (pid,gid),
PRIMARY KEY (gid_idx),
KEY unix_gid (unix_gid),
KEY gid (gid),
KEY pid (pid)
KEY pid (pid),
KEY pididx (pid_idx,gid_idx),
UNIQUE KEY pidgid (pid,gid)
) TYPE=MyISAM;
--
......@@ -2079,6 +2089,7 @@ CREATE TABLE proj_memb (
CREATE TABLE project_stats (
pid varchar(12) NOT NULL default '',
pid_idx mediumint(8) unsigned NOT NULL default '0',
exptstart_count int(11) unsigned default '0',
exptstart_last datetime default NULL,
exptpreload_count int(11) unsigned default '0',
......@@ -2095,7 +2106,8 @@ CREATE TABLE project_stats (
allexpt_vnode_duration int(11) unsigned default '0',
allexpt_pnodes int(11) unsigned default '0',
allexpt_pnode_duration int(11) unsigned default '0',
PRIMARY KEY (pid)
PRIMARY KEY (pid_idx),
UNIQUE KEY pid (pid)
) TYPE=MyISAM;
--
......@@ -2104,6 +2116,7 @@ CREATE TABLE project_stats (
CREATE TABLE projects (
pid varchar(12) NOT NULL default '',
pid_idx mediumint(8) unsigned NOT NULL default '0',
created datetime default NULL,
expires date default NULL,
name tinytext,
......@@ -2130,11 +2143,12 @@ CREATE TABLE projects (
default_user_interface enum('emulab','plab') NOT NULL default 'emulab',
linked_to_us tinyint(4) NOT NULL default '0',
cvsrepo_public tinyint(1) NOT NULL default '0',
PRIMARY KEY (pid),
PRIMARY KEY (pid_idx),
KEY unix_gid (unix_gid),
KEY approved (approved),
KEY approved_2 (approved),
KEY pcremote_ok (pcremote_ok)
KEY pcremote_ok (pcremote_ok),
UNIQUE KEY pid (pid)
) TYPE=MyISAM;
--
......@@ -2547,7 +2561,7 @@ CREATE TABLE user_sslcerts (
CREATE TABLE user_stats (
uid varchar(8) NOT NULL default '',
uid_idx smallint(5) unsigned NOT NULL default '0',
uid_idx mediumint(8) unsigned NOT NULL default '0',
weblogin_count int(11) unsigned default '0',
weblogin_last datetime default NULL,
exptstart_count int(11) unsigned default '0',
......@@ -2575,6 +2589,7 @@ CREATE TABLE user_stats (
CREATE TABLE users (
uid varchar(8) NOT NULL default '',
uid_idx mediumint(8) unsigned NOT NULL default '0',
usr_created datetime default NULL,
usr_expires datetime default NULL,
usr_modified datetime default NULL,
......@@ -2619,9 +2634,10 @@ CREATE TABLE users (
wikiname tinytext,
wikionly tinyint(1) default '0',
mailman_password tinytext,
PRIMARY KEY (uid),
PRIMARY KEY (uid_idx),
KEY unix_uid (unix_uid),
KEY status (status)
KEY status (status),
UNIQUE KEY uid (uid)
) TYPE=MyISAM;
--
......
......@@ -3771,3 +3771,60 @@ last_net_act,last_cpu_act,last_ext_act);
add column need_more_info tinyint(1) null after inferred,
add column tblog_revision varchar(8) not null after mesg;
4.100: Add uid_idx column to users table, to become the new unique id
for all users.
Add pid_idx column to projects table, to become the new unique id
for all projects.
Add gid_idx column to groups table, to become the new unique id
for all groups.
Order matters ...
alter table users add
uid_idx mediumint(8) unsigned NOT NULL default '0' after uid;
alter table users drop primary key;
alter table users add UNIQUE KEY uid (uid);
alter table user_stats change uid_idx
uid_idx mediumint(8) unsigned NOT NULL default '0';
alter table deleted_users change uid_idx
uid_idx mediumint(8) unsigned NOT NULL default '0';
alter table projects add
pid_idx mediumint(8) unsigned NOT NULL default '0' after pid;
alter table projects drop primary key;
alter table projects add UNIQUE KEY pid (pid);
alter table project_stats add
pid_idx mediumint(8) unsigned NOT NULL default '0' after pid;
alter table project_stats drop primary key;
alter table project_stats add UNIQUE KEY pid (pid);
alter table groups add
pid_idx mediumint(8) unsigned NOT NULL default '0' after gid;
alter table groups add
gid_idx mediumint(8) unsigned NOT NULL default '0' after pid_idx;
alter table groups drop primary key;
alter table groups add KEY pididx (pid_idx,gid_idx);
alter table groups add UNIQUE KEY pidgid (pid,gid);
alter table group_stats add
gid_idx mediumint(8) unsigned NOT NULL default '0' after gid;
alter table group_stats drop primary key;
alter table group_stats add UNIQUE KEY pidgid (pid,gid);
alter table group_membership add
uid_idx mediumint(8) unsigned NOT NULL default '0' after pid;
alter table group_membership add
pid_idx mediumint(8) unsigned NOT NULL default '0' after uid_idx;
alter table group_membership add
gid_idx mediumint(8) unsigned NOT NULL default '0' after pid_idx;
alter table group_membership drop primary key;
alter table group_membership add UNIQUE KEY uid (uid,pid,gid);
# Now run this script to populate the indexes and create new KEYS.
./init_newids.pl
4.101:
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use lib "/usr/testbed/lib";
use libdb;
use libtestbed;
#
# Untaint the path
#
$ENV{'PATH'} = '/bin:/usr/bin:/usr/sbin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
# We want to change the group_membership table later, so save the ids.
my %uids = ();
my %gids = ();
# Nice place to start.
my $next_gid = 10000;
# Locals
my $TBOPSPID = TBOPSPID();
#
# Generate new unique indexes for all existing users.
#
my $query_result =
DBQueryFatal("select uid,unix_uid from users ".
"order by usr_created");
while (my ($uid,$unix_uid) = $query_result->fetchrow_array()) {
#print "$uid -> $unix_uid\n";
DBQueryFatal("update users set uid_idx=$unix_uid ".
"where uid='$uid'");
# These were left out in initial testbed setup.
if ($uid eq "elabckup" || $uid eq "elabman" || $uid eq "operator") {
DBQueryFatal("replace into user_stats set ".
" uid='$uid',uid_idx=$unix_uid");
}
$uids{$uid} = $unix_uid;
}
DBQuery("alter table users drop primary key");
DBQueryFatal("alter table users add PRIMARY KEY (uid_idx)");
DBQuery("alter table projects drop PRIMARY KEY");
DBQuery("alter table project_stats drop PRIMARY KEY");
DBQuery("alter table groups drop PRIMARY KEY");
DBQuery("alter table group_stats drop PRIMARY KEY");
DBQuery("alter table group_membership drop PRIMARY KEY");
# These were left out in initial testbed setup.
DBQueryFatal("replace into group_stats set ".
" pid='$TBOPSPID',gid='$TBOPSPID',gid_idx=0");
DBQueryFatal("replace into project_stats set ".
" pid='$TBOPSPID',pid_idx=0");