New stats tables. There is a new script in this directory called

initstats.pl to fill in the tables as best as possible. Mostly a
hack job since we need Mac's scripts to go over the logs and really
fill in the info.

sql/database-create.sql

@@ -169,6 +169,42 @@ CREATE TABLE eventlist (
 -- Table structure for table 'experiments'
 --
 
+CREATE TABLE experiment_stats (
+  pid varchar(12) NOT NULL default '',
+  eid varchar(32) NOT NULL default '',
+  creator varchar(8) NOT NULL default '',
+  idx int(10) unsigned NOT NULL default '0',
+  gid varchar(16) NOT NULL default '',
+  created datetime default NULL,
+  destroyed datetime default NULL,
+  swapin_count smallint(5) unsigned default '0',
+  swapin_last datetime default NULL,
+  swapout_count smallint(5) unsigned default '0',
+  swapout_last datetime default NULL,
+  swapmodify_count smallint(5) unsigned default '0',
+  swapmodify_last datetime default NULL,
+  swapin_duration int(10) unsigned default '0',
+  vnodes smallint(5) unsigned default '0',
+  pnodes smallint(5) unsigned default '0',
+  wanodes smallint(5) unsigned default '0',
+  simnodes smallint(5) unsigned default '0',
+  jailnodes smallint(5) unsigned default '0',
+  delaynodes smallint(5) unsigned default '0',
+  linkdelays smallint(5) unsigned default '0',
+  walinks smallint(5) unsigned default '0',
+  links smallint(5) unsigned default '0',
+  lans smallint(5) unsigned default '0',
+  shapedlinks smallint(5) unsigned default '0',
+  shapedlans smallint(5) unsigned default '0',
+  minlinks tinyint(3) unsigned default '0',
+  maxlinks tinyint(3) unsigned default '0',
+  PRIMARY KEY  (eid,pid,idx)
+) TYPE=MyISAM;
+
+--
+-- Table structure for table 'experiments'
+--
+
 CREATE TABLE experiments (
   eid varchar(32) NOT NULL default '',
   pid varchar(12) NOT NULL default '',
@@ -261,6 +297,31 @@ CREATE TABLE group_membership (
   KEY gid (gid)
 ) TYPE=MyISAM;
 
+--
+-- Table structure for table 'group_stats'
+--
+
+CREATE TABLE group_stats (
+  pid varchar(12) NOT NULL default '',
+  gid varchar(12) NOT NULL default '',
+  exptstart_count int(11) unsigned default '0',
+  exptstart_last datetime default NULL,
+  exptpreload_count int(11) unsigned default '0',
+  exptpreload_last datetime default NULL,
+  exptswapin_count int(11) unsigned default '0',
+  exptswapin_last datetime default NULL,
+  exptswapout_count int(11) unsigned default '0',
+  exptswapout_last datetime default NULL,
+  exptswapmodify_count int(11) unsigned default '0',
+  exptswapmodify_last datetime default NULL,
+  allexpt_duration int(11) unsigned default '0',
+  allexpt_vnodes int(11) unsigned default '0',
+  allexpt_vnode_duration int(11) unsigned default '0',
+  allexpt_pnodes int(11) unsigned default '0',
+  allexpt_pnode_duration int(11) unsigned default '0',
+  PRIMARY KEY  (pid,gid)
+) TYPE=MyISAM;
+
 --
 -- Table structure for table 'groups'
 --
@@ -871,6 +932,30 @@ CREATE TABLE proj_memb (
   PRIMARY KEY  (uid,pid)
 ) TYPE=MyISAM;
 
+--
+-- Table structure for table 'project_stats'
+--
+
+CREATE TABLE project_stats (
+  pid varchar(12) NOT NULL default '',
+  exptstart_count int(11) unsigned default '0',
+  exptstart_last datetime default NULL,
+  exptpreload_count int(11) unsigned default '0',
+  exptpreload_last datetime default NULL,
+  exptswapin_count int(11) unsigned default '0',
+  exptswapin_last datetime default NULL,
+  exptswapout_count int(11) unsigned default '0',
+  exptswapout_last datetime default NULL,
+  exptswapmodify_count int(11) unsigned default '0',
+  exptswapmodify_last datetime default NULL,
+  allexpt_duration int(11) unsigned default '0',
+  allexpt_vnodes int(11) unsigned default '0',
+  allexpt_vnode_duration int(11) unsigned default '0',
+  allexpt_pnodes int(11) unsigned default '0',
+  allexpt_pnode_duration int(11) unsigned default '0',
+  PRIMARY KEY  (pid)
+) TYPE=MyISAM;
+
 --
 -- Table structure for table 'projects'
 --
@@ -1124,6 +1209,33 @@ CREATE TABLE user_sfskeys (
   PRIMARY KEY  (uid,comment)
 ) TYPE=MyISAM;
 
+
+--
+-- Table structure for table 'user_stats'
+--
+
+CREATE TABLE user_stats (
+  uid varchar(8) NOT NULL default '',
+  weblogin_count int(11) unsigned default '0',
+  weblogin_last datetime default NULL,
+  exptstart_count int(11) unsigned default '0',
+  exptstart_last datetime default NULL,
+  exptpreload_count int(11) unsigned default '0',
+  exptpreload_last datetime default NULL,
+  exptswapin_count int(11) unsigned default '0',
+  exptswapin_last datetime default NULL,
+  exptswapout_count int(11) unsigned default '0',
+  exptswapout_last datetime default NULL,
+  exptswapmodify_count int(11) unsigned default '0',
+  exptswapmodify_last datetime default NULL,
+  allexpt_duration int(11) unsigned default '0',
+  allexpt_vnodes int(11) unsigned default '0',
+  allexpt_vnode_duration int(11) unsigned default '0',
+  allexpt_pnodes int(11) unsigned default '0',
+  allexpt_pnode_duration int(11) unsigned default '0',
+  PRIMARY KEY  (uid)
+) TYPE=MyISAM;
+
 --
 -- Table structure for table 'users'
 --

sql/database-migrate.txt

@@ -238,3 +238,110 @@ last_net_act,last_cpu_act,last_ext_act);
        alter table new_interfaces drop column node_id;
        alter table new_interfaces add column new_node_id int not null first,
               add primary key(new_node_id,iface);
+
+1.133: Rob?
+
+1.134: New stats tables. I've checked in a script in this directory to
+       fill in the tables as best as possible. See ./initstats.pl. Just
+       run it and pipe the output into "mysql tbdb". Its a hack job,
+       since there is not enough info in the system to do a good job.
+       Need Mac's scripts to make it work properly.
+
+	CREATE TABLE experiment_stats (
+	  pid varchar(12) NOT NULL default '',
+	  eid varchar(32) NOT NULL default '',
+	  creator varchar(8) NOT NULL default '',
+	  idx int(10) unsigned NOT NULL default '0',
+	  gid varchar(16) NOT NULL default '',
+	  created datetime default NULL,
+	  destroyed datetime default NULL,
+	  swapin_count smallint(5) unsigned default '0',
+	  swapin_last datetime default NULL,
+	  swapout_count smallint(5) unsigned default '0',
+	  swapout_last datetime default NULL,
+	  swapmodify_count smallint(5) unsigned default '0',
+	  swapmodify_last datetime default NULL,
+	  swapin_duration int(10) unsigned default '0',
+	  vnodes smallint(5) unsigned default '0',
+	  pnodes smallint(5) unsigned default '0',
+	  wanodes smallint(5) unsigned default '0',
+	  simnodes smallint(5) unsigned default '0',
+	  jailnodes smallint(5) unsigned default '0',
+	  delaynodes smallint(5) unsigned default '0',
+	  linkdelays smallint(5) unsigned default '0',
+	  walinks smallint(5) unsigned default '0',
+	  links smallint(5) unsigned default '0',
+	  lans smallint(5) unsigned default '0',
+	  shapedlinks smallint(5) unsigned default '0',
+	  shapedlans smallint(5) unsigned default '0',
+	  minlinks tinyint(3) unsigned default '0',
+	  maxlinks tinyint(3) unsigned default '0',
+	  PRIMARY KEY  (eid,pid,idx)
+	) TYPE=MyISAM;
+
+
+	CREATE TABLE group_stats (
+	  pid varchar(12) NOT NULL default '',
+	  gid varchar(12) NOT NULL default '',
+	  exptstart_count int(11) unsigned default '0',
+	  exptstart_last datetime default NULL,
+	  exptpreload_count int(11) unsigned default '0',
+	  exptpreload_last datetime default NULL,
+	  exptswapin_count int(11) unsigned default '0',
+	  exptswapin_last datetime default NULL,
+	  exptswapout_count int(11) unsigned default '0',
+	  exptswapout_last datetime default NULL,
+	  exptswapmodify_count int(11) unsigned default '0',
+	  exptswapmodify_last datetime default NULL,
+	  allexpt_duration int(11) unsigned default '0',
+	  allexpt_vnodes int(11) unsigned default '0',
+	  allexpt_vnode_duration int(11) unsigned default '0',
+	  allexpt_pnodes int(11) unsigned default '0',
+	  allexpt_pnode_duration int(11) unsigned default '0',
+	  PRIMARY KEY  (pid,gid)
+	) TYPE=MyISAM;
+
+
+	CREATE TABLE project_stats (
+	  pid varchar(12) NOT NULL default '',
+	  exptstart_count int(11) unsigned default '0',
+	  exptstart_last datetime default NULL,
+	  exptpreload_count int(11) unsigned default '0',
+	  exptpreload_last datetime default NULL,
+	  exptswapin_count int(11) unsigned default '0',
+	  exptswapin_last datetime default NULL,
+	  exptswapout_count int(11) unsigned default '0',
+	  exptswapout_last datetime default NULL,
+	  exptswapmodify_count int(11) unsigned default '0',
+	  exptswapmodify_last datetime default NULL,
+	  allexpt_duration int(11) unsigned default '0',
+	  allexpt_vnodes int(11) unsigned default '0',
+	  allexpt_vnode_duration int(11) unsigned default '0',
+	  allexpt_pnodes int(11) unsigned default '0',
+	  allexpt_pnode_duration int(11) unsigned default '0',
+	  PRIMARY KEY  (pid)
+	) TYPE=MyISAM;
+
+
+	CREATE TABLE user_stats (
+	  uid varchar(8) NOT NULL default '',
+	  weblogin_count int(11) unsigned default '0',
+	  weblogin_last datetime default NULL,
+	  exptstart_count int(11) unsigned default '0',
+	  exptstart_last datetime default NULL,
+	  exptpreload_count int(11) unsigned default '0',
+	  exptpreload_last datetime default NULL,
+	  exptswapin_count int(11) unsigned default '0',
+	  exptswapin_last datetime default NULL,
+	  exptswapout_count int(11) unsigned default '0',
+	  exptswapout_last datetime default NULL,
+	  exptswapmodify_count int(11) unsigned default '0',
+	  exptswapmodify_last datetime default NULL,
+	  allexpt_duration int(11) unsigned default '0',
+	  allexpt_vnodes int(11) unsigned default '0',
+	  allexpt_vnode_duration int(11) unsigned default '0',
+	  allexpt_pnodes int(11) unsigned default '0',
+	  allexpt_pnode_duration int(11) unsigned default '0',
+	  PRIMARY KEY  (uid)
+	) TYPE=MyISAM;
+

sql/initstats.pl

+#!/usr/bin/perl -wT
+use English;
+
+use lib "/usr/testbed/lib";
+use libdb;
+use libtestbed;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/sbin';
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+$query_result =
+    DBQueryFatal("select pid,gid from groups");
+
+while (($pid,$gid) = $query_result->fetchrow_array()) {
+    if ($pid eq $gid) {
+	print "insert into project_stats (pid) values ('$pid');\n";
+    }
+    print "insert into group_stats (pid,gid) values ('$pid', '$gid');\n";
+}
+
+$query_result =
+    DBQueryFatal("select u.uid,time from users as u ".
+		 "left join lastlogin as l on u.uid=l.uid");
+
+while (($uid,$stamp) = $query_result->fetchrow_array()) {
+    my $count;
+    if (defined($stamp)) {
+	$stamp = "'$stamp'";
+	$count = 1;
+    }
+    else {
+	$stamp = "NULL";
+	$count = 0;
+    }
+
+    print "insert into user_stats (uid, weblogin_last, weblogin_count) ".
+	"values ('$uid', $stamp, $count);\n";
+}
+
+$query_result =
+    DBQueryFatal("select state,eid,pid,expt_head_uid,idx,gid,expt_created, ".
+		 "       expt_swapped ".
+		 "from experiments order by expt_swapped");
+
+while (($state,$eid,$pid,$creator,$idx,$gid,$created,$swapped) =
+       $query_result->fetchrow_array()) {
+    print "insert into experiment_stats ".
+	"(eid, pid, creator, idx, gid, created) ".
+        "VALUES ('$eid', '$pid', '$creator', $idx, '$gid', '$created');\n";
+
+    if ($state eq EXPTSTATE_NEW ||
+	($state eq EXPTSTATE_SWAPPED && !defined($swapped))) {
+	print("update group_stats set ".
+	      "  exptpreload_count=exptpreload_count+1, ".
+	      "  exptpreload_last='$created' ".
+	      "where pid='$pid' and gid='$gid';\n");
+	print("update project_stats set ".
+	      "  exptpreload_count=exptpreload_count+1, ".
+	      "  exptpreload_last='$created' ".
+	      "where pid='$pid';\n");
+	print("update user_stats set ".
+	      "  exptpreload_count=exptpreload_count+1, ".
+	      "  exptpreload_last='$created' ".
+	      "where uid='$creator';\n");
+    }
+    if ($state eq EXPTSTATE_ACTIVE && defined($swapped)) {
+	my $pnodes = 0;
+	my $pnodes_result =
+	    DBQueryWarn("select r.node_id from reserved as r ".
+			"left join nodes as n on r.node_id=n.node_id ".
+			"where r.pid='$pid' and r.eid='$eid' and ".
+			"      n.role='testnode'");
+	$pnodes = $pnodes_result->numrows
+	    if ($pnodes_result);
+	
+	print("update experiment_stats set ".
+	      "  swapin_count=swapin_count+1, ".
+	      "  swapin_last='$swapped', ".
+	      "  pnodes=$pnodes ".
+	      "where pid='$pid' and eid='$eid' and idx=$idx;\n");
+	print("update group_stats set ".
+	      "  exptswapin_count=exptswapin_count+1, ".
+	      "  exptswapin_last='$swapped' ".
+	      "where pid='$pid' and gid='$gid';\n");
+	print("update project_stats set ".
+	      "  exptswapin_count=exptswapin_count+1, ".
+	      "  exptswapin_last='$swapped' ".
+	      "where pid='$pid';\n");
+	print("update user_stats set ".
+	      "  exptswapin_count=exptswapin_count+1, ".
+	      "  exptswapin_last='$swapped' ".
+	      "where uid='$creator';\n");
+    }
+    if ($state eq EXPTSTATE_SWAPPED && defined($swapped)) {
+	print("update experiment_stats set ".
+	      "  swapout_count=swapout_count+1, ".
+	      "  swapout_last='$swapped' ".
+	      "where pid='$pid' and eid='$eid' and idx=$idx;\n");
+	print("update group_stats set ".
+	      "  exptswapout_count=exptswapout_count+1, ".
+	      "  exptswapout_last='$swapped' ".
+	      "where pid='$pid' and gid='$gid';\n");
+	print("update project_stats set ".
+	      "  exptswapout_count=exptswapout_count+1, ".
+	      "  exptswapout_last='$swapped' ".
+	      "where pid='$pid';\n");
+	print("update user_stats set ".
+	      "  exptswapout_count=exptswapout_count+1, ".
+	      "  exptswapout_last='$swapped' ".
+	      "where uid='$creator';\n");
+    }
+}
+
+