Commit 5e5508bf authored by Leigh Stoller's avatar Leigh Stoller

Add support for new {user,group,project,experiment}_stats tables.

The first three are aggregate tables, while the experiment stats table
gets a record for each new experiment, and is updated when an
experiment is swapped in/out/modify or terminated. Look at the table
to see what is tracked. Once the experiment_stats record is updated,
the aggregate tables are updated as necessary. There are a bunch of
ugly changes to assign_wrapper to get the stats. Note that pnodes is
not incremented until an experiment sucessfully swaps in. This is in
leu of getting status codes; I'm not tracking failed operations yet,
nor creating the log file that Jay wants. I'll do that in the next
round of changes when we see how useful these numbers are.

Most of the changes are to create/delete table entries where
appropriate, and to display the records. Display is only under admin
mode, and the display is raw; just a dump of the assoc tables in php.
The last 100 experiment stats records are available via the Experiment
List page, using the "Stats" show option at the top. Bad place, but
will do for now.
parent cfdcc39b
......@@ -220,6 +220,31 @@ my $forcelinkdelays = 0;
my %uselinkdelay = ();
my %nobwshaping = ();
#
# This is for stats gathering. It might duplicate other stuff, but
# thats okay.
#
my %expt_stats = (# pnodes include jailnodes and delaynodes.
# We let the wrapper determine pnodes once the
# experiment is fully swapped in so that the record
# is not "committed" until successful swapin.
jailnodes => 0,
vnodes => 0,
# vnodes include wanodes.
wanodes => 0,
simnodes => 0,
delaynodes => 0,
linkdelays => 0,
links => 0,
walinks => 0,
lans => 0,
shapedlinks => 0,
shapedlans => 0,
minlinks => 100000,
# includes emulated links. Maybe thats wrong.
maxlinks => 0,
);
my $query_result =
DBQueryFatal("SELECT uselinkdelays,forcelinkdelays from experiments ".
"where pid='$pid' and eid='$eid'");
......@@ -330,7 +355,17 @@ while (($vname,$ips,$type,$fixed,$isremote,$isvirt) =
$isremotenode{$vname} = $isremote;
$remotecount++
if ($isremote);
foreach $ipinfo (split(" ",$ips)) {
# stats
my @iplist = split(" ", $ips);
my $ipcount = scalar(@iplist);
$expt_stats{"maxlinks"} = $ipcount
if ($ipcount > $expt_stats{"maxlinks"});
$expt_stats{"minlinks"} = $ipcount
if ($ipcount < $expt_stats{"minlinks"});
foreach $ipinfo (@iplist) {
($port,$ip) = split(":",$ipinfo);
$ips{"$vname:$port"} = $ip;
if( $type eq "sim" ) {
......@@ -348,6 +383,11 @@ while (($vname,$ips,$type,$fixed,$isremote,$isvirt) =
}
$result->finish;
# Stats
$expt_stats{"vnodes"} = $virtcount;
$expt_stats{"wanodes"} = $remotecount;
$expt_stats{"simnodes"} = scalar(@simnodelist);
printdb "Loading virt_lans.\n";
$result =
DBQueryFatal("select vname,member,delay,bandwidth,lossrate," .
......@@ -387,6 +427,7 @@ while (my ($vname,$member,$delay,$bandwidth,$lossrate,
}
push(@{$tunnels{$vname}},$member);
$rnodelans{$vname} = 1;
$expt_stats{"walinks"} += 1;
printdb " Added $member to tunnels of $vname\n";
next;
}
......@@ -546,6 +587,7 @@ foreach $lan (keys(%lans)) {
my $emulated = $emulated{$lan};
my $uselinkdelay = $uselinkdelay{$lan};
my $mustdelay = $mustdelay{$lan};
my $isdelayed = 0;
my $nobwshaping = $nobwshaping{$lan};
$trivial_ok = 0;
foreach $member (@members) {
......@@ -576,6 +618,7 @@ foreach $lan (keys(%lans)) {
$trivial_ok = 1;
}
if ($#members == 1) {
$expt_stats{"links"} += 1;
($nodeport0,$nodeport1) = @members;
$node0 = (split(":",$nodeport0))[0];
$node1 = (split(":",$nodeport1))[0];
......@@ -615,6 +658,7 @@ foreach $lan (keys(%lans)) {
# simulated and a real node, we might need to put in delay
# nodes
($realnodes != 0)) {
$isdelayed = 1;
#
# We use a linkdelay if the link is emulated, globally forced,
# globally preferred if the link is shaped, or if the per-link
......@@ -685,8 +729,10 @@ foreach $lan (keys(%lans)) {
}
print TOPFILE "\n";
}
$expt_stats{"shapedlinks"} += $isdelayed;
} elsif ($#members != 0) {
print TOPFILE "node lan/$lan lan\n";
$expt_stats{"lans"} += 1;
$lannodes{"lan/$lan"} = 1;
foreach $member (@members) {
($delay,$bw,$loss,
......@@ -715,6 +761,7 @@ foreach $lan (keys(%lans)) {
# if we have 1 real node in the LAN, we may need to create
# a lan
($realnodes != 0)) {
$isdelayed = 1;
#
# We use a linkdelay if the link is emulated, globally forced,
# globally preferred if the link is shaped, or if the per-link
......@@ -776,6 +823,7 @@ foreach $lan (keys(%lans)) {
print TOPFILE "\n";
}
}
$expt_stats{"shapedlans"} += $isdelayed;
}
# If a LAN has only one member we don't do anything.
}
......@@ -1687,6 +1735,8 @@ foreach $delayid (keys(%linkdelays)) {
" idx from event_objecttypes where ".
" event_objecttypes.type='LINK'");
$expt_stats{"linkdelays"} += 1;
#
# We used to post-pass the eventlist to set the vnode where the agent
# would be running, but this is no longer needed since the virt_agents
......@@ -2088,7 +2138,7 @@ foreach my $vnode (keys(%v2pmap)) {
DBQueryFatal("insert into v2pmap (pid,eid,vname,node_id) values " .
"('$pid','$eid','$vnode','$pnode')");
}
UploadStats();
TBDebugTimeStamp("uploading finished");
TBDebugTimeStamp("assign_wrapper finished");
......@@ -2218,6 +2268,8 @@ sub InitPnode($pnode, $vnode)
DBQueryFatal("update reserved set vname='$vname' " .
"where node_id='$pnode'");
$expt_stats{"jailnodes"} += 1;
}
else {
#
......@@ -2265,6 +2317,8 @@ sub InitPnode($pnode, $vnode)
DBQueryFatal("update reserved set vname='$vname' " .
"where node_id='$pnode'");
$expt_stats{"delaynodes"} += 1;
}
}
printdb("reserved vname: $pnode $vname\n");
......@@ -2498,3 +2552,18 @@ sub UploadVlans()
}
}
#
# Write the stats record to the DB.
#
sub UploadStats()
{
my @updates = ();
foreach my $key (keys(%expt_stats)) {
$val = $expt_stats{$key};
push (@updates, "$key=$val");
}
DBQueryFatal("update experiment_stats set " . join(",", @updates) . " ".
"where pid='$pid' and eid='$eid'");
}
......@@ -116,7 +116,7 @@ if (! TBProjAccessCheck($dbuid, $pid, $gid, TB_PROJECT_CREATEEXPT)) {
# Create an experiment record. The pid/eid has to be unique, so lock the
# table for the check/insert.
#
DBQueryFatal("lock tables experiments write");
DBQueryFatal("lock tables experiments write, experiment_stats write");
$query_result =
DBQueryFatal("SELECT pid,eid FROM experiments ".
......@@ -142,6 +142,19 @@ if (! DBQueryWarn("INSERT INTO experiments ".
die("*** $0:\n".
" Database error inserting record for $pid/$eid!\n");
}
# Need the idx.
if (! ($query_result =
DBQueryWarn("select idx from experiments ".
"where pid='$pid' and eid='$eid'"))) {
DBQueryWarn("unlock tables");
fatal("Database error inserting experiment stats record for $pid/$eid!\n");
}
my ($idx) = $query_result->fetchrow_array;
# Insert stats record.
DBQueryWarn("INSERT INTO experiment_stats ".
"(eid, pid, creator, idx, gid, created) ".
"VALUES ('$eid', '$pid', '$dbuid', $idx, '$gid', now())");
if (! DBQueryWarn("unlock tables")) {
fatal("Unexpected DB Error!");
......@@ -229,7 +242,15 @@ sub fatal($)
#
# Clear the record and cleanup.
#
TBExptDestroy($pid, $eid);
TBExptDestroy($pid, $eid);
#
# Generally, we do not delete the stats record, but if we failed
# at this point, no point in keeping the record. Just a waste of
# space.
#
DBQueryWarn("DELETE from experiment_stats ".
"WHERE eid='$eid' and pid='$pid'");
exit(-1);
}
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -279,6 +279,11 @@ if ($estate eq EXPTSTATE_ACTIVE) {
if ($estate ne EXPTSTATE_SWAPPED) {
fatal("Experiment is in the wrong state: $estate\n");
}
#
# Gather statistics for the swapout.
#
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT);
}
if ($estate eq EXPTSTATE_SWAPPED) {
......@@ -290,6 +295,11 @@ if ($estate eq EXPTSTATE_SWAPPED) {
if ($estate ne EXPTSTATE_TERMINATED) {
fatal("Experiment is in the wrong state: $estate\n");
}
#
# Gather statistics for the swapout.
#
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_TERMINATE);
}
if ($estate ne EXPTSTATE_TERMINATED && $estate ne EXPTSTATE_NEW) {
......
......@@ -46,7 +46,7 @@ if ($EUID != 0) {
" Must be root! Maybe its a development version?\n");
}
# XXX Hacky!
if (1 && $TB ne "/usr/testbed") {
if (0 && $TB ne "/usr/testbed") {
print STDERR "*** $0:\n".
" Wrong version. Maybe its a development version?\n";
#
......
......@@ -224,6 +224,9 @@ foreach my $tipserver ( TBTipServers() ) {
DBQueryFatal("delete from group_membership ".
"where pid='$pid' and gid='$gid'");
DBQueryFatal("delete from group_stats ".
"where pid='$pid' and gid='$gid'");
DBQueryFatal("delete from groups ".
"where pid='$pid' and gid='$gid'");
......
......@@ -202,10 +202,11 @@ $EUID = 0;
#
# Then the project table itself, plus a few other bits and pieces
#
DBQueryFatal("delete FROM projects where pid='$pid'");
DBQueryFatal("delete FROM images where pid='$pid'");
DBQueryFatal("delete FROM os_info where pid='$pid'");
DBQueryFatal("delete FROM nodetypeXpid_permissions where pid='$pid'");
DBQueryFatal("delete FROM project_stats where pid='$pid'");
DBQueryFatal("delete FROM projects where pid='$pid'");
print "Project $pid has been removed!\n";
exit(0);
......
......@@ -234,6 +234,7 @@ DBQueryFatal("delete from nodeuidlastlogin where uid='$user'");
DBQueryFatal("delete from uidnodelastlogin where uid='$user'");
DBQueryFatal("delete from unixgroup_membership where uid='$user'");
DBQueryFatal("delete from userslastlogin where uid='$user'");
DBQueryFatal("delete from user_stats where uid='$user'");
#
# Remove user from both local and control node.
......
......@@ -182,8 +182,6 @@ my $estate = $row{'state'};
my $expt_path = $row{'path'};
my $expt_name = $row{'expt_name'};
my $expt_created= $row{'expt_created'};
my $expt_expires= $row{'expt_expires'};
my $expt_started;
if ($estate ne EXPTSTATE_NEW) {
die("*** $0:\n".
......@@ -269,22 +267,19 @@ if (system("$tbdir/tbreport -b $pid $eid 2>&1 > $repfile") != 0) {
}
#
# Increment the project experiment count/lastdate. This is informational.
# Must unlock before exit.
#
if (! $frontend) {
DBQueryWarn("update projects ".
"set expt_count=expt_count+1, expt_last=now() ".
"where pid='$pid'");
}
TBUnLockExp($pid, $eid);
#
# Gen up a date for the started field of the record, and insert it.
# Unlock the experiment at the same time.
#
$expt_started = DBDateTime();
DBQueryWarn("update experiments set ".
"expt_start='$expt_started', expt_locked=NULL ".
"WHERE eid='$eid' and pid='$pid'");
# Gather statistics.
#
if ($frontend) {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_PRELOAD);
}
else {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_START);
}
#
# In batchmode, send the report to stdout for the batch daemon.
......@@ -349,8 +344,6 @@ $message .=
"GID: $gid\n" .
"Name: $expt_name\n" .
"Created: $expt_created\n" .
"Expires: $expt_expires\n" .
"Started: $expt_started\n" .
"Directory: $expt_path\n".
"\n".
"Appended at the end is the output of the experiment setup. If you\n" .
......
......@@ -47,6 +47,7 @@ my $logname;
my $dbuid;
my $user_name;
my $user_email;
my @allnodes;
my @row;
my $action;
......@@ -387,6 +388,19 @@ system("cp -Rfp $workdir/ $userdir/tbdata/");
#
TBUnLockExp($pid, $eid);
#
# Gather stats.
#
if ($inout eq "in") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPIN);
}
elsif ($inout eq "out") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT);
}
elsif ($inout eq "modify") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPMODIFY);
}
print "Done!\n";
#
......
......@@ -671,6 +671,8 @@ if (! $returning) {
"'$encoding', NULL, 'newuser', ".
"date_add(now(), interval 1 year), now())");
DBQueryFatal("INSERT INTO user_stats (uid) VALUES ('$joining_uid')");
$key = TBGenVerificationKey($joining_uid);
TBMAIL("$usr_name '$joining_uid' <$usr_email>",
......
......@@ -119,6 +119,10 @@ DBQueryFatal("INSERT INTO groups ".
"VALUES ('$group_pid', '$group_id', '$group_leader', now(), ".
" '$group_description', NULL, '$unix_gname')");
DBQueryFatal("INSERT INTO group_stats ".
"(pid, gid) ".
"VALUES ('$group_pid', '$group_id')");
DBQueryFatal("insert into group_membership ".
"(uid, pid, gid, trust, date_applied, date_approved) ".
"values ('$group_leader','$group_pid','$group_id', ".
......
......@@ -922,6 +922,8 @@ if (! $returning) {
"'$usr_phone', '$encoding', NULL, 'newuser', ".
"date_add(now(), interval 1 year), now())");
DBQueryFatal("INSERT INTO user_stats (uid) VALUES ('$proj_head_uid')");
$key = TBGenVerificationKey($proj_head_uid);
TBMAIL("$usr_name '$proj_head_uid' <$usr_email>",
......@@ -961,11 +963,15 @@ DBQueryFatal("INSERT INTO projects ".
" '$proj_funders', NULL, $proj_plabpcs, $proj_ronpcs, ".
" $public, '$proj_whynotpublic')");
DBQueryFatal("INSERT INTO project_stats (pid) VALUES ('$pid')");
DBQueryFatal("INSERT INTO groups ".
"(pid, gid, leader, created, description, unix_gid, unix_name) ".
"VALUES ('$pid', '$pid', '$proj_head_uid', now(), ".
" 'Default Group', NULL, '$pid')");
DBQueryFatal("INSERT INTO group_stats (pid, gid) VALUES ('$pid', '$pid')");
DBQueryFatal("insert into group_membership ".
"(uid, gid, pid, trust, date_applied) ".
"values ('$proj_head_uid','$pid','$pid','none', now())");
......
......@@ -157,6 +157,14 @@ SUBPAGEEND();
#
SHOWNODES($exp_pid, $exp_eid);
if (ISADMIN($uid)) {
echo "<center>
<h3>Experiment Stats</h3>
</center>\n";
SHOWEXPTSTATS($exp_pid, $exp_eid);
}
#
# Standard Testbed Footer
#
......
......@@ -52,6 +52,7 @@ if ($isadmin) {
} else {
echo "Idle, ";
}
echo "<a class='static' href='showexpstats.php3'>Stats</a>, ";
}
if ($showtype != "all") {
......
<?php
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
include("defs.php3");
include("showstuff.php3");
#
# Standard Testbed Header
#
PAGEHEADER("Show Experiment Information");
#
# Only known and logged in users can end experiments.
#
$uid = GETLOGIN();
LOGGEDINORDIE($uid);
$isadmin = ISADMIN($uid);
#
# Only admins
#
if (! $isadmin) {
USERERROR("You do not have permission to view this page!", 1);
}
#
# Right now we show just the last 50 records entered.
#
$query_result =
DBQueryFatal("select * from experiment_stats order by idx desc limit 100");
if (mysql_num_rows($query_result) == 0) {
USERERROR("No experiment records in the system!", 1);
}
#
# Use first row to get the column headers (no pretty printing yet).
#
$row = mysql_fetch_assoc($query_result);
echo "<table align=center border=1>\n";
echo "<tr>\n";
foreach($row as $key => $value) {
$key = str_replace("_", " ", $key);
echo "<th><font size=-1>$key</font></th>\n";
}
echo "</tr>\n";
mysql_data_seek($query_result, 0);
while ($row = mysql_fetch_assoc($query_result)) {
echo "<tr>\n";
foreach($row as $key => $value) {
echo "<td nowrap>$value</td>\n";
}
echo "</tr>\n";
}
echo "</table>\n";
#
# Standard Testbed Footer
#
PAGEFOOTER();
?>
......@@ -149,6 +149,14 @@ if (mysql_num_rows($query_result)) {
echo "</table>\n";
}
if ($isadmin) {
echo "<center>
<h3>Group Stats</h3>
</center>\n";
SHOWGROUPSTATS($pid, $gid);
}
#
# Standard Testbed Footer
#
......
<?php
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
include("defs.php3");
......@@ -148,6 +148,14 @@ echo "<p><center>
SHOWPROJECT($pid, $uid);
if ($isadmin) {
echo "<center>
<h3>Project Stats</h3>
</center>\n";
SHOWPROJSTATS($pid);
}
if ($isadmin) {
echo "<p>
<A href='deleteproject.php3?pid=$pid'>
......
......@@ -1737,6 +1737,107 @@ function SHOWWIDEAREANODE($node_id, $embedded = 0) {
}
}
#
# Stats
#
function SHOWUSERSTATS($uid) {
$query_result =
DBQueryFatal("SELECT * from user_stats where uid='$uid'");
if (! mysql_num_rows($query_result)) {
return;
}
$row = mysql_fetch_assoc($query_result);
#
# Not pretty printed yet.
#
echo "<table align=center border=1>\n";
foreach($row as $key => $value) {
echo "<tr>
<td>$key:</td>
<td>$value</td>
</tr>\n";
}
echo "</table>\n";
}
function SHOWPROJSTATS($pid) {
$query_result =
DBQueryFatal("SELECT * from project_stats where pid='$pid'");
if (! mysql_num_rows($query_result)) {
return;
}
$row = mysql_fetch_assoc($query_result);
#
# Not pretty printed yet.
#
echo "<table align=center border=1>\n";
foreach($row as $key => $value) {
echo "<tr>
<td>$key:</td>
<td>$value</td>
</tr>\n";
}
echo "</table>\n";
}
function SHOWGROUPSTATS($pid, $gid) {
$query_result =
DBQueryFatal("SELECT * from group_stats ".
"where pid='$pid' and gid='$gid'");
if (! mysql_num_rows($query_result)) {
return;
}
$row = mysql_fetch_assoc($query_result);
#
# Not pretty printed yet.
#
echo "<table align=center border=1>\n";
foreach($row as $key => $value) {
echo "<tr>
<td>$key:</td>
<td>$value</td>
</tr>\n";
}
echo "</table>\n";
}
function SHOWEXPTSTATS($pid, $eid) {
$query_result =
DBQueryFatal("SELECT * from experiment_stats ".
"where pid='$pid' and eid='$eid'");
if (! mysql_num_rows($query_result)) {
return;
}
$row = mysql_fetch_assoc($query_result);
#
# Not pretty printed yet.
#
echo "<table align=center border=1>\n";
foreach($row as $key => $value) {
echo "<tr>
<td>$key:</td>
<td>$value</td>
</tr>\n";
}
echo "</table>\n";
}
#
# This is an included file.
#
......
......@@ -221,6 +221,14 @@ SUBMENUEND();
SHOWUSER($target_uid);
SUBPAGEEND();
if ($isadmin) {
echo "<center>
<h3>User Stats</h3>
</center>\n";
SHOWUSERSTATS($target_uid);
}
#
# Standard Testbed Footer
#
......
......@@ -383,10 +383,10 @@ function DOLOGIN($uid, $password, $adminmode) {
#
# Usage stats.
#
# DBQueryFatal("update user_stats set ".
# " weblogin_count=weblogin_count+1, ".
# " weblogin_last=now() ".
# "where uid='$uid'");
DBQueryFatal("update user_stats set ".
" weblogin_count=weblogin_count+1, ".
" weblogin_last=now() ".
"where uid='$uid'");
#
# Issue the cookie requests so that subsequent pages come back
......@@ -509,11 +509,11 @@ function LASTWEBLOGIN($uid) {
global $TBDBNAME;
$query_result =
DBQueryFatal("SELECT time from lastlogin where uid='$uid'");
DBQueryFatal("SELECT weblogin_last from user_stats where uid='$uid'");
if (mysql_num_rows($query_result)) {
$lastrow = mysql_fetch_array($query_result);
return $lastrow[time];
return $lastrow[weblogin_last];
}
return 0;
}
......
......@@ -871,6 +871,8 @@ if (! $returning) {
"'$usr_URL', '$usr_title', '$usr_affil', ".
"'$usr_phone', '$encoding', NULL, 'newuser', ".
"date_add(now(), interval 1 year), now(), 1)");
DBQueryFatal("INSERT INTO user_stats (uid) VALUES ('$usr_uid')");
$key = TBGenVerificationKey($usr_uid);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment