Commit 52396569 authored by Mac Newbold's avatar Mac Newbold

First big commit/checkpoint of web changes for new slothd.

 - Added a bunch of functions to get time of last activity, hours idle,
   and staleness for nodes and experiments.
 - Fixed the showexp_list page to use the new idle measures, including
   idle view. Idle view now runs in about 1.3 seconds instead of 13.
 - Changed all the "show" functions for nodes and expts to show idleness.

More changes will be coming, but I wanted to commit this before I install
it for real.
parent 483c14b2
......@@ -1293,7 +1293,8 @@ function TBEventCount($pid, $eid)
function TBExpUidLastLogins($pid, $eid)
{
$query_result =
DBQueryFatal("select n.*, to_days(now())-to_days(n.date) as daysidle ".
DBQueryFatal("select n.*, date_format(n.date,'%c/%e') as shortdate, ".
"to_days(now())-to_days(n.date) as daysidle ".
"from nodeuidlastlogin as n ".
"left join reserved as r on n.node_id=r.node_id ".
"left join experiments as e ".
......@@ -1420,6 +1421,153 @@ function TBIPtoNodeID($ip) {
return $row[node_id];
}
#
# Functions related to idle times
#
$lastact_query = "greatest(last_tty_act,last_net_act,last_cpu_act,".
"last_ext_act)";
#
# Gets the time of last activity for a node
#
function TBGetNodeLastAct($pc) {
global $lastact_query;
$query_result =
DBQueryWarn("select $lastact_query as last_actfrom node_activity ".
"where node_id='$pc'");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
return $row[last_act];
}
#
# Gets the time of idleness for a node, in hours
#
function TBGetNodeIdleTime($pc) {
global $lastact_query;
$query_result =
DBQueryWarn("select (unix_timestamp(now()) - unix_timestamp( ".
"$lastact_query))/3600 as idle_time from node_activity ".
"where node_id='$pc'");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
$t = $row[idle_time];
# if it is less than 5 minutes, it is not idle at all...
return ($t < 0.0834 ? 0 : $t);
}
#
# Find out if a node's idle reports are stale
#
function TBGetNodeIdleStale($pc) {
global $lastact_query;
# We currently have a 5 minute interval for slothd between reports
# So give some slack in case a node reboots without reporting for a while
$staletime = 10; # in minutes
$stalesec = 60 * $staletime;
$query_result =
DBQueryWarn("select (unix_timestamp(now()) - ".
"unix_timestamp(last_report )) as t ".
"from node_activity where node_id='$pc'");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
return ($row[t]>$stalesec);
}
#
# Gets the time of last activity for an expt
#
function TBGetExptLastAct($pid,$eid) {
global $lastact_query;
$query_result =
DBQueryWarn("select max($lastact_query) as last_act ".
"from node_activity as na ".
"left join reserved as r on na.node_id=r.node_id ".
"where pid='$pid' and eid='$eid' ".
"group by pid,eid");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
return $row[last_act];
}
#
# Gets the time of idleness for an expt, in hours
#
function TBGetExptIdleTime($pid,$eid) {
global $lastact_query;
$query_result =
DBQueryWarn("select (unix_timestamp(now()) - unix_timestamp( ".
"max($lastact_query) ))/3600 as idle_time ".
"from node_activity as na ".
"left join reserved as r on na.node_id=r.node_id ".
"where pid='$pid' and eid='$eid' ".
"group by pid,eid");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
$t = $row[idle_time];
# if it is less than 5 minutes, it is not idle at all...
return ($t < 0.0834 ? 0 : $t);
}
#
# Find out if an expt's idle report is stale
#
function TBGetExptIdleStale($pid,$eid) {
global $lastact_query;
# We currently have a 5 minute interval for slothd between reports
# So give some slack in case a node reboots without reporting for a while
$staletime = 10; # in minutes
$stalesec = 60 * $staletime;
$query_result =
DBQueryWarn("select (unix_timestamp(now()) - ".
"unix_timestamp(min(last_report) )) as t ".
"from node_activity as na ".
"left join reserved as r on na.node_id=r.node_id ".
"where pid='$pid' and eid='$eid' ".
"group by pid,eid");
if (mysql_num_rows($query_result) == 0) {
return -1;
}
$row = mysql_fetch_array($query_result);
return ($row[t]>$stalesec);
}
#
# Find out if an expt's idleness should be ignored
#
function TBGetExptIdleIgnore($pid,$eid) {
global $lastact_query;
$query_result =
DBQueryWarn("select idle_ignore from experiments ".
"where pid='$pid' and eid='$eid'");
if (mysql_num_rows($query_result) == 0) {
return 0;
}
$row = mysql_fetch_array($query_result);
return $row[idle_ignore];
}
#
# DB Interface.
#
......
......@@ -70,6 +70,8 @@ echo "</b><br />\n";
#echo "\n <a class='static' href='showexp_list.php3?showtype=all&sortby=$sortby&thumb=$thumb'>all</a>.
# </b><br />\n";
# Default value for showlastlogin
$showlastlogin = 1;
#
# Handle showtype
......@@ -80,6 +82,9 @@ if (! strcmp($showtype, "all")) {
elseif (! strcmp($showtype, "active")) {
# Active is now defined as "having nodes reserved" - we just depend on
# the fact that we skip expts with no nodes further down...
# But for speed, explicitly exclude expts that say they are "swapped"
# (leave in activating, swapping, etc.)
$clause = "e.state !='$TB_EXPTSTATE_SWAPPED'";
$title = "Active";
$having = "having (ncount>0)";
$active = 1;
......@@ -90,11 +95,14 @@ elseif (! strcmp($showtype, "batch")) {
}
elseif ((!strcmp($showtype, "idle")) && $isadmin ) {
# Do not put active in the clause for same reason as active
# However, it takes a long time with so many swapped expts
$clause = "e.state !='$TB_EXPTSTATE_SWAPPED'";
$title = "Idle";
#$having = "having (lastswap>=1)"; # At least one day since swapin
$having = "having (lastswap>=0)";
$idle = 1;
$idlehours = 2;
$showlastlogin = 0; # don't show a lastlogin column
}
else {
# See active above
......@@ -210,15 +218,17 @@ else
# Show a menu of all experiments for all projects that this uid
# is a member of. Or, if an admin type person, show them all!
#
if ($clause) {
$clause = "and ($clause)";
} else {
$clause = "";
}
if ($isadmin) {
if ($clause)
$clause = "and ($clause)";
else
$clause = "";
$experiments_result =
DBQueryFatal("select e.*,".
"date_format(expt_swapped,\"%Y-%m-%d\") as d, ".
"date_format(expt_swapped,'%c/%e') as dshort, ".
"(to_days(now())-to_days(expt_swapped)) as lastswap, ".
"count(r.node_id) as ncount, swap_requests, ".
"round((unix_timestamp(now()) - ".
......@@ -233,43 +243,12 @@ if ($isadmin) {
"group by e.pid,e.eid ".
"$having ".
"order by $order");
if ($idle) {
# Run idlecheck and get the info
#print "<pre>Running idlecheck\n";
$x=exec("$TBSUEXEC_PATH $uid $TBADMINGROUP ".
"webidlecheck -s -u -t $idlehours",
$l, $rv);
reset($l);
while(list($index,$i) = each ($l)) {
list($ipid,$ieid,$word1, $word2, $word3) = split("[ \t/]+",$i);
#print "$ipid + $ieid + $word1 + $word2 + $word3\n";
$expt = "$ipid/$ieid";
$set = array($word1,$word2,$word3);
while(list($index,$tag) = each($set)) {
if ($tag == "") { break; }
if ($tag == "inactive") { $inactive[$expt]=1; }
elseif ($tag == "stale") { $stale[$expt]=1; }
elseif ($tag == "unswappable") { $unswap[$expt]=1; }
else {
if (defined($other[$expt])) { $other[$expt].=$tag; }
else {$other[$expt]=$tag; }
}
}
}
#print "\nDone idlechecking...\n</pre>";
}
}
else {
if ($clause)
$clause = "and ($clause)";
else
$clause = "";
$experiments_result =
DBQueryFatal("select distinct e.*, ".
"date_format(expt_swapped,'%Y-%m-%d') as d, ".
"date_format(expt_swapped,'%c/%e') as dshort, ".
"count(r.node_id) as ncount, ".
"ve.thumb_hash as thumb_hash ".
"from group_membership as g ".
......@@ -284,7 +263,7 @@ else {
"and e.eid is not null $clause ".
"group by e.pid,e.eid ".
"$having ".
"order by $order");
"order by $order");
}
if (! mysql_num_rows($experiments_result)) {
USERERROR("There are no experiments running in any of the projects ".
......@@ -347,7 +326,7 @@ if ($thumb && !$idle) {
$eid = $row["eid"];
$huid = $row["expt_head_uid"];
$name = stripslashes($row["expt_name"]);
$date = $row["d"];
$date = $row["dshort"];
$thumb_hash = $row["thumb_hash"];
if ($idle && ($str=="&nbsp;" || !$pcs)) { continue; }
......@@ -418,11 +397,11 @@ if ($thumb && !$idle) {
$daysidle=$lastexpnodelogins["daysidle"];
#if ($idle && $daysidle < 1)
# continue;
$lastlogin .= $lastexpnodelogins["date"] . " " .
$lastlogin .= $lastexpnodelogins["shortdate"] . " " .
"(" . $lastexpnodelogins["uid"] . ")";
} elseif ($state=="active") {
$daysidle=$row["lastswap"];
$lastlogin .= "$date Swapped In";
$lastlogin .= "$date swapin";
}
# if ($lastlogin=="") { $lastlogin="<td>&nbsp;</td>\n"; }
if ($lastlogin != "") {
......@@ -477,14 +456,16 @@ if ($thumb && !$idle) {
EID</a></th>
<th align=center width=3%>
<a class='static' href='showexp_list.php3?showtype=$showtype&sortby=pcs'>
PCs</a><br>[<b>1</b>]</th>\n";
PCs</a><br>[<b>1</b>]</th>
<th align=center width=3%>
Hours Idle</th>\n";
if ($isadmin && !$idle)
if ($showlastlogin)
echo "<th width=17% align=center>Last Login</th>\n";
if ($idle)
if ($idle) {
# "<th width=4% align=center>Days Idle</th>\n";
echo "<th width=4% align=center>Slothd Info</th>
<th width=4% align=center colspan=2>Swap Request</th>\n";
echo "<th width=4% align=center colspan=2>Swap Request</th>\n";
}
echo " <th width=60%>
<a class='static' href='showexp_list.php3?showtype=$showtype&sortby=name'>
......@@ -499,11 +480,14 @@ if ($thumb && !$idle) {
$eid = $row["eid"];
$huid = $row["expt_head_uid"];
$name = stripslashes($row["expt_name"]);
$date = $row["d"];
$date = $row["dshort"];
$state= $row["state"];
$isidle = $row["swap_requests"];
$daysidle=0;
$idletime = round(TBGetExptIdleTime($pid,$eid),1);
$stale = TBGetExptIdleStale($pid,$eid);
$ignore = TBGetExptIdleIgnore($pid,$eid);
if ($isadmin) {
$swappable= $row["swappable"];
$swapreq=$row["swap_requests"];
......@@ -513,66 +497,47 @@ if ($thumb && !$idle) {
$daysidle=$lastexpnodelogins["daysidle"];
#if ($idle && $daysidle < 1)
# continue;
$lastlogin .= $lastexpnodelogins["date"] . " " .
$lastlogin .= $lastexpnodelogins["shortdate"] . " " .
"(" . $lastexpnodelogins["uid"] . ")";
} elseif ($state=="active") {
$daysidle=$row["lastswap"];
$lastlogin .= "$date Swapped In";
$lastlogin .= "$date swapin";
}
$lastlogin.="</td>\n";
if ($lastlogin=="<td></td>\n") { $lastlogin="<td>&nbsp;</td>\n"; }
}
$inactive = $idletime >= $idlehours;
if ($idle) {
# If it is ignored, skip it now.
if ($ignore) { continue; }
#$lastlogin .= "<td align=center>$daysidle</td>\n";
$foo = "<td align=left>";
$expt = "$pid/$eid";
$str="";
if ($inactive[$expt]==1) {
if ($stale[$expt]==1) {
$str .= "possibly&nbsp;inactive, ";
} elseif ($unswap[$expt]==1) {
$str .= "<b>probably&nbsp;inactive, unswappable</b>";
} else {
$str .= "<b>inactive</b>";
}
if ($other[$expt]) { $str .= " ($other[$expt]) "; }
}
if ($stale[$expt]==1) { $str .= "<b>no&nbsp;report</b> "; }
# For now, don't show this tag, it's redundant
#if ($unswap[$expt]==1) { $str .= "unswappable"; }
if ($str=="") { $str="&nbsp;"; }
# sanity check
$slothderr=0;
# This check won't work anymore now that a login today
# doesn't mean it is active.
#if ($daysidle==0 && $inactive[$expt]==1 && $stale[$expt]==0) {
# $str .= " (recently logged into)\n";
# $slothderr=1;
#}
$foo .= "$str</td>\n";
if (isset($perexp_usage["$pid:$eid"]) &&
isset($perexp_usage["$pid:$eid"]["pc"])) {
$pcs = $perexp_usage["$pid:$eid"]["pc"];
} else { $pcs=0; }
$foo .= "<td align=center valign=center>\n";
if ($inactive[$expt]==1 && $stale[$expt]!=1 &&
!$slothderr && $pcs) {
$fooswap = "<td><a href=\"request_swapexp.php3?pid=$pid&eid=$eid\">".
"<img border=0 src=\"redball.gif\"></a></td>\n" ;
$foo = "<td align=center valign=center>\n";
if ($inactive && !$stale && !$ignore && $pcs) {
$fooswap = "<td><a ".
"href=\"request_swapexp.php3?pid=$pid&eid=$eid\">".
"<img border=0 src=\"redball.gif\"></a></td>\n" ;
} else {
$fooswap = "<td></td>";
if (!$pcs) { $foo .= "(no PCs)\n"; }
else { $foo .="&nbsp;"; }
$label = "";
if ($stale) { $label .= "stale "; }
if ($ignore) { $label .= "ignore "; }
if ($label == "") { $label = "&nbsp;"; }
$fooswap = "<td>$label</td>";
if (!$pcs) { $foo .= "(no PCs)\n"; }
else { $foo .="&nbsp;"; }
}
if ($swapreq > 0) {
$foo .= "&nbsp;$swapreq&nbsp;sent<br />".
"<font size=-2>(${lastswapreq}&nbsp;hours&nbsp;ago)</font>\n";
"<font size=-2>(${lastswapreq}&nbsp;hrs&nbsp;ago)</font>\n";
}
$foo .= "</td>" . $fooswap . "\n";
}
if ($idle && ($str=="&nbsp;" || !$pcs)) { continue; }
if ($idle && ($str=="&nbsp;" || !$pcs || !$inactive)) { continue; }
$nodes = 0;
$special = 0;
......@@ -612,7 +577,9 @@ if ($thumb && !$idle) {
else
echo "<td>$nodes</td>\n";
if ($isadmin && !$idle) echo "$lastlogin\n";
echo "<td>$idletime</td>\n";
if ($showlastlogin) echo "$lastlogin\n";
if ($idle) echo "$foo\n";
echo "<td>$name</td>
......
......@@ -600,6 +600,20 @@ function SHOWEXP($pid, $eid) {
<td class=\"left\">$nodes</td>
</tr>\n";
$lastact = TBGetExptLastAct($pid,$eid);
$idletime = TBGetExptIdleTime($pid,$eid);
echo "<tr>
<td>Last Activity: </td>
<td class=\"left\">$lastact</td>
</tr>\n";
echo "<tr>
<td>Idle Time: </td>
<td class=\"left\">$idletime hours</td>
</tr>\n";
if ($batchmode) {
echo "<tr>
<td>Batch Mode: </td>
......@@ -651,20 +665,22 @@ function SHOWNODES($pid, $eid) {
<th>Node ID</th>
<th>$nodename</th>
<th>Type</th>
<th>Default<br>OSID</th>
<th>Default<br>Path</th>
<th>Default<br>Cmdline</th>
<th>Boot<br>Status[<b>1</b>]</th>
<th>Startup<br>Command</th>
<th>Startup<br>Status[<b>2</b>]</th>
<th>Ready<br>Status[<b>3</b>]</th>
<th>Default OSID</th>
<th>Node<br>Status</th>
<th>Last Active</th>
<th>Startup<br>Status[<b>1</b>]</th>
<th>Ready<br>Status[<b>2</b>]</th>
</tr>\n";
$query_result =
DBQueryFatal("SELECT nodes.*,reserved.vname, ".
"lower(date_format(greatest(last_tty_act,last_net_act,".
"last_cpu_act,last_ext_act),\"%c/%d&nbsp;%l:%i:%s&nbsp;%p\"))".
"as acttime, ".
"date_format(rsrv_time,\"%Y-%m-%d&nbsp;%T\") as rsrvtime ".
"FROM nodes LEFT JOIN reserved ".
"ON nodes.node_id=reserved.node_id ".
"FROM nodes LEFT JOIN node_activity ".
"on nodes.node_id=node_activity.node_id ".
"LEFT JOIN reserved ON nodes.node_id=reserved.node_id ".
"WHERE reserved.eid=\"$eid\" and reserved.pid=\"$pid\" ".
"ORDER BY type,priority");
......@@ -672,26 +688,13 @@ function SHOWNODES($pid, $eid) {
$node_id = $row[node_id];
$vname = $row[$vnamefield];
$type = $row[type];
$def_boot_osid = $row[def_boot_osid];
$def_boot_path = $row[def_boot_path];
$def_boot_cmd_line = $row[def_boot_cmd_line];
$next_boot_path = $row[next_boot_path];
$next_boot_cmd_line = $row[next_boot_cmd_line];
$startupcmd = $row[startupcmd];
$startstatus = $row[startstatus];
$readystatus = $row[ready];
$bootstatus = $row[bootstatus];
if (!$def_boot_cmd_line)
$def_boot_cmd_line = "NULL";
if (!$def_boot_path)
$def_boot_path = "NULL";
if (!$next_boot_path)
$next_boot_path = "NULL";
if (!$next_boot_cmd_line)
$next_boot_cmd_line = "NULL";
if (!$startupcmd)
$startupcmd = "NULL";
$def_boot_osid = $row[def_boot_osid];
$startstatus = $row[startstatus];
$readystatus = $row[ready];
$status = $row[status];
$bootstate = $row[eventstate];
$acttime = $row[acttime];
if (!$vname)
$vname = "--";
if ($readystatus)
......@@ -711,11 +714,14 @@ function SHOWNODES($pid, $eid) {
}
else
echo "<td>&nbsp</td>\n";
if ($bootstate != "ISUP") {
echo " <td>$status ($bootstate)</td>\n";
} else {
echo " <td>$status</td>\n";
}
echo " <td>$def_boot_path</td>
<td>$def_boot_cmd_line</td>
<td align=center>$bootstatus</td>
<td>$startupcmd</td>
echo " <td>$acttime</td>
<td align=center>$startstatus</td>
<td align=center>$readylabel</td>
</tr>\n";
......@@ -723,7 +729,6 @@ function SHOWNODES($pid, $eid) {
echo "</table>\n";
echo "<h4><blockquote><blockquote><blockquote>
<ol>
<li>Node has rebooted successfully after experiment creation.
<li>Exit value of the node startup command. A value of
666 indicates a testbed internal error.
<li>User application ready status, reported via TMCC.
......@@ -1107,10 +1112,13 @@ function SHOWIMAGEID($imageid, $edit, $isadmin = 0) {
#
function SHOWNODE($node_id) {
$query_result =
DBQueryFatal("select n.*,r.vname,r.pid,r.eid,i.IP, ".
DBQueryFatal("select n.*,na.*,r.vname,r.pid,r.eid,i.IP, ".
"greatest(last_tty_act,last_net_act,last_cpu_act,".
"last_ext_act) as last_act, ".
" t.isvirtnode,t.isremotenode ".
" from nodes as n ".
"left join reserved as r on n.node_id=r.node_id ".
"left join node_activity as na on n.node_id=na.node_id ".
"left join node_types as t on t.type=n.type ".
"left join interfaces as i on i.card=t.control_net ".
" and i.node_id=n.node_id ".
......@@ -1141,6 +1149,8 @@ function SHOWNODE($node_id) {
$routertype = $row[routertype];
$eventstate = $row[eventstate];
$state_timestamp = $row[state_timestamp];
$allocstate = $row[allocstate];
$allocstate_timestamp= $row[allocstate_timestamp];
$op_mode = $row[op_mode];
$op_mode_timestamp = $row[op_mode_timestamp];
$IP = $row[IP];
......@@ -1149,7 +1159,13 @@ function SHOWNODE($node_id) {
$ipport_low = $row[ipport_low];
$ipport_next = $row[ipport_next];
$ipport_high = $row[ipport_high];
$last_act = $row[last_act];
$last_tty_act = $row[last_tty_act];
$last_net_act = $row[last_net_act];
$last_cpu_act = $row[last_cpu_act];
$last_ext_act = $row[last_ext_act];
$last_report = $row[last_report];
if (!$def_boot_cmd_line)
$def_boot_cmd_line = "&nbsp";
if (!$def_boot_path)
......@@ -1211,16 +1227,6 @@ function SHOWNODE($node_id) {
<td class=left>$type</td>
</tr>\n";
echo "<tr>
<td>Control Net IP:</td>
<td class=left>$IP</td>
</tr>\n";
echo "<tr>
<td>Bios Version:</td>
<td class=left>$bios</td>
</tr>\n";
echo "<tr>
<td>Def Boot OS:</td>
<td class=left>";
......@@ -1228,6 +1234,85 @@ function SHOWNODE($node_id) {
echo " </td>
</tr>\n";
if ($eventstate) {
$when = strftime("20%y-%m-%d %H:%M:%S", $state_timestamp);
echo "<tr>
<td>EventState:</td>
<td class=left>$eventstate ($when)</td>
</tr>\n";
}
if ($op_mode) {
$when = strftime("20%y-%m-%d %H:%M:%S", $op_mode_timestamp);
echo "<tr>
<td>Operating Mode:</td>
<td class=left>$op_mode ($when)</td>
</tr>\n";
}
if ($allocstate) {
$when = strftime("20%y-%m-%d %H:%M:%S", $allocstate_timestamp);
echo "<tr>
<td>AllocState:</td>
<td class=left>$allocstate ($when)</td>
</tr>\n";
}
#
# We want the last login for this node, but only if its *after* the
# experiment was created (or swapped in).
#