Commit 283cf638 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Change testbed_stats to record the start time, in addition to the

final time, so that we can see how long things take. As per Jay's
request.
parent 67981a64
[This file is not kept entirely up to date.] [This file is not kept entirely up to date.]
* Fix tmcd/fork problem (events out of order).
* Supply form args to beginexp page, and allow inline NS files.
* From: Jay Lepreau <lepreau@flux.utah.edu>
Subject: expt phase transition durations
Date: Fri, 18 Jul 2003 14:03:48 MDT
Some sub-times would be really useful too,
like how long assign took to run.
* Feed back jail changes to FreeBSD.org.
* I think these swap/modify errors should include the ns file.
Well, certainly if it's a parsing error.
* From: Jay Lepreau <lepreau@cs.utah.edu> * From: Jay Lepreau <lepreau@cs.utah.edu>
Date: Thu, 10 Jul 2003 10:56:37 MDT Date: Thu, 10 Jul 2003 10:56:37 MDT
...@@ -94,7 +110,11 @@ ...@@ -94,7 +110,11 @@
modify. We should also save the XML representation to avoid having to modify. We should also save the XML representation to avoid having to
reparse old NS files, although there is some versioning issues with this. reparse old NS files, although there is some versioning issues with this.
* Auto discovery of new nodes. * Auto discovery of new nodes.
* Rob is working on this.
* Viz can't handle multiple links between nodes.
* Change netbuild to speak XML (in both directions). * Change netbuild to speak XML (in both directions).
......
...@@ -108,6 +108,7 @@ use Exporter; ...@@ -108,6 +108,7 @@ use Exporter;
TBDB_STATS_PRELOAD TBDB_STATS_START TBDB_STATS_TERMINATE TBDB_STATS_PRELOAD TBDB_STATS_START TBDB_STATS_TERMINATE
TBDB_STATS_SWAPIN TBDB_STATS_SWAPOUT TBDB_STATS_SWAPMODIFY TBDB_STATS_SWAPIN TBDB_STATS_SWAPOUT TBDB_STATS_SWAPMODIFY
TBDB_STATS_FLAGS_IDLESWAP TBDB_STATS_FLAGS_PREMODIFY TBDB_STATS_FLAGS_IDLESWAP TBDB_STATS_FLAGS_PREMODIFY
TBDB_STATS_FLAGS_START
TBDB_JAILIPBASE TBDB_JAILIPMASK TBDB_JAILIPBASE TBDB_JAILIPMASK
...@@ -149,6 +150,8 @@ use Exporter; ...@@ -149,6 +150,8 @@ use Exporter;
TBExptRemovePhysicalState TBExptBackupPhysicalState TBExptRemovePhysicalState TBExptBackupPhysicalState
TBExptRestorePhysicalState TBExptRestorePhysicalState
TBExptPortRange
TBDB_WIDEAREA_LOCALNODE TBDB_WIDEAREA_LOCALNODE
TBWideareaNodeID TBTipServers TBWideareaNodeID TBTipServers
...@@ -486,6 +489,9 @@ sub TBDB_STATS_SWAPOUT() { "swapout"; } ...@@ -486,6 +489,9 @@ sub TBDB_STATS_SWAPOUT() { "swapout"; }
sub TBDB_STATS_SWAPMODIFY() { "swapmod"; } sub TBDB_STATS_SWAPMODIFY() { "swapmod"; }
sub TBDB_STATS_FLAGS_IDLESWAP() { 0x01; } sub TBDB_STATS_FLAGS_IDLESWAP() { 0x01; }
sub TBDB_STATS_FLAGS_PREMODIFY(){ 0x02; } sub TBDB_STATS_FLAGS_PREMODIFY(){ 0x02; }
sub TBDB_STATS_FLAGS_START() { 0x04; }
# Do not export this variable!
my $TBDB_STATS_STARTCLOCK;
# Jail. # Jail.
sub TBDB_JAILIPBASE() { "@JAILIPBASE@"; } sub TBDB_JAILIPBASE() { "@JAILIPBASE@"; }
...@@ -2499,14 +2505,13 @@ sub TBGetSiteVar($) ...@@ -2499,14 +2505,13 @@ sub TBGetSiteVar($)
"virt_vtypes", "virt_vtypes",
"nseconfigs", "nseconfigs",
"eventlist", "eventlist",
#"ipport_ranges",
"ipsubnets"); "ipsubnets");
@physicalTables = ("delays", @physicalTables = ("delays",
"vlans", "vlans",
"tunnels", "tunnels",
"v2pmap",
"ipport_ranges", "ipport_ranges",
"v2pmap",
"linkdelays", "linkdelays",
"portmap"); "portmap");
...@@ -2678,6 +2683,30 @@ sub TBExptSetThumbNail($$$) ...@@ -2678,6 +2683,30 @@ sub TBExptSetThumbNail($$$)
"where idx=$rsrcidx"); "where idx=$rsrcidx");
} }
#
# Get the port range for an experiment.
#
# usage TBControlNetIP(char *pid, char *eid, int \*low, int \*high)
# Return 1 if success.
# Return 0 if error.
#
sub TBExptPortRange($$$$)
{
my ($pid, $eid, $high, $low) = @_;
my $query_result =
DBQueryFatal("select low,high from ipport_ranges ".
"where pid='$pid' and eid='$eid'");
if ($query_result->numrows == 0) {
return 0;
}
my @row = $query_result->fetchrow_array();
$$low = $row[0];
$$high = $row[1];
return 1;
}
# #
# Get the control network IP for a node (underlying physical node!). # Get the control network IP for a node (underlying physical node!).
# #
...@@ -3271,6 +3300,16 @@ sub GatherSwapStats($$$$$;$) ...@@ -3271,6 +3300,16 @@ sub GatherSwapStats($$$$$;$)
# Optional argument to modify the stats gathering. # Optional argument to modify the stats gathering.
$flags = 0 $flags = 0
if (!defined($flags)); if (!defined($flags));
#
# If this is a start time marker, then just record the time in a global
# variable and return. This is cheezy, but the interface I'm providing
# allows for fancier stuff later if desired.
#
if ($flags & TBDB_STATS_FLAGS_START) {
$TBDB_STATS_STARTCLOCK = time();
return;
}
local $DBQUERY_MAXTRIES = 5; local $DBQUERY_MAXTRIES = 5;
...@@ -3521,11 +3560,22 @@ sub GatherSwapStats($$$$$;$) ...@@ -3521,11 +3560,22 @@ sub GatherSwapStats($$$$$;$)
# worth worrying about. # worth worrying about.
# #
logit: logit:
DBQueryWarn("insert into testbed_stats ". my $starttime = (!defined($TBDB_STATS_STARTCLOCK) ? "NULL" :
"FROM_UNIXTIME($TBDB_STATS_STARTCLOCK)");
if ("@OURDOMAIN@" eq "ballmoss.com") {
DBQueryWarn("insert into testbed_stats ".
"(idx, uid, start_time, end_time, exptidx, rsrcidx, ".
" action, exitcode) ".
"values (0, '$uid', $starttime, now(), $exptidx, $rsrcidx, ".
" '$mode', $ecode)");
}
else {
DBQueryWarn("insert into testbed_stats ".
"(idx, uid, tstamp, exptidx, rsrcidx, ". "(idx, uid, tstamp, exptidx, rsrcidx, ".
" action, exitcode) ". " action, exitcode) ".
"values (0, '$uid', now(), $exptidx, $rsrcidx, ". "values (0, '$uid', now(), $exptidx, $rsrcidx, ".
" '$mode', $ecode)"); " '$mode', $ecode)");
}
} }
# #
......
...@@ -1140,7 +1140,8 @@ CREATE TABLE switch_stacks ( ...@@ -1140,7 +1140,8 @@ CREATE TABLE switch_stacks (
CREATE TABLE testbed_stats ( CREATE TABLE testbed_stats (
idx int(10) unsigned NOT NULL auto_increment, idx int(10) unsigned NOT NULL auto_increment,
tstamp datetime default NULL, start_time datetime default NULL,
end_time datetime default NULL,
exptidx int(10) unsigned NOT NULL default '0', exptidx int(10) unsigned NOT NULL default '0',
rsrcidx int(10) unsigned NOT NULL default '0', rsrcidx int(10) unsigned NOT NULL default '0',
action varchar(16) NOT NULL default '', action varchar(16) NOT NULL default '',
......
...@@ -757,21 +757,21 @@ last_net_act,last_cpu_act,last_ext_act); ...@@ -757,21 +757,21 @@ last_net_act,last_cpu_act,last_ext_act);
drop table vis_experiments; drop table vis_experiments;
1.161: Add a node_type_features table that lists the assign 'features' for each 1.162: Add a node_type_features table that lists the assign 'features' for each
node type: node type:
create table node_type_features (type varchar(30) not null, create table node_type_features (type varchar(30) not null,
feature varchar(30) not null, weight float not null default 0.0, feature varchar(30) not null, weight float not null default 0.0,
primary key (type,feature)); primary key (type,feature));
1.162: Add columns to the new_interfaces table indicating where the interface 1.163: Add columns to the new_interfaces table indicating where the interface
is plugged into a switch: is plugged into a switch:
alter table new_interfaces add column switch_id char(10); alter table new_interfaces add column switch_id char(10);
alter table new_interfaces add column switch_card tinyint(3); alter table new_interfaces add column switch_card tinyint(3);
alter table new_interfaces add column switch_port tinyint(3); alter table new_interfaces add column switch_port tinyint(3);
1.163: Chage the new_interfaces table to contain card numbers, rather than 1.164: Chage the new_interfaces table to contain card numbers, rather than
iface names: iface names:
alter table new_interfaces drop primary key; alter table new_interfaces drop primary key;
...@@ -780,12 +780,12 @@ last_net_act,last_cpu_act,last_ext_act); ...@@ -780,12 +780,12 @@ last_net_act,last_cpu_act,last_ext_act);
new_node_id; new_node_id;
alter table new_interfaces add primary key (new_node_id,card); alter table new_interfaces add primary key (new_node_id,card);
1.164: Added a new column to new_nodes, so we can tell the IP address that 1.165: Added a new column to new_nodes, so we can tell the IP address that
the node was temporarily given while it's configuring: the node was temporarily given while it's configuring:
alter table new_nodes add column temporary_IP varchar(15) after IP; alter table new_nodes add column temporary_IP varchar(15) after IP;
1.165: Kill max_ports and max_cards, replace with max_interfaces. 1.166: Kill max_ports and max_cards, replace with max_interfaces.
alter table node_types drop max_ports; alter table node_types drop max_ports;
alter table node_types drop max_cards; alter table node_types drop max_cards;
...@@ -795,4 +795,15 @@ last_net_act,last_cpu_act,last_ext_act); ...@@ -795,4 +795,15 @@ last_net_act,last_cpu_act,last_ext_act);
update node_types set max_interfaces=5 where type='pc600'; update node_types set max_interfaces=5 where type='pc600';
update node_types set max_interfaces=5 where type='pc850'; update node_types set max_interfaces=5 where type='pc850';
update node_types set max_interfaces=3 where type='pcL440GX'; update node_types set max_interfaces=3 where type='pcL440GX';
\ No newline at end of file
1.168: As per Jay's request, start recording the start time as well as
the end time, in the testbed_stats table. So we know how long
things like swapin take, etc. Below, we convert the current
timestamp into the "end" time, and add a new "start" time,
which is null for all existing records.
alter table testbed_stats change tstamp \
end_time datetime default NULL;
alter table testbed_stats add start_time datetime \
default NULL after idx;
...@@ -284,6 +284,9 @@ if (! $batch) { ...@@ -284,6 +284,9 @@ if (! $batch) {
# Sanity check states in case someone changes something. # Sanity check states in case someone changes something.
# #
if ($estate eq EXPTSTATE_ACTIVE) { if ($estate eq EXPTSTATE_ACTIVE) {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT, 0,
TBDB_STATS_FLAGS_START);
print STDOUT "Running 'tbswap out' with arguments: $pid $eid\n"; print STDOUT "Running 'tbswap out' with arguments: $pid $eid\n";
if (system("$tbdir/tbswap out $pid $eid") != 0) { if (system("$tbdir/tbswap out $pid $eid") != 0) {
fatal("tbswap out failed!\n"); fatal("tbswap out failed!\n");
...@@ -301,6 +304,9 @@ if ($estate eq EXPTSTATE_ACTIVE) { ...@@ -301,6 +304,9 @@ if ($estate eq EXPTSTATE_ACTIVE) {
} }
if ($estate eq EXPTSTATE_SWAPPED) { if ($estate eq EXPTSTATE_SWAPPED) {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_TERMINATE, 0,
TBDB_STATS_FLAGS_START);
print STDOUT "Running tbend with arguments: $pid $eid\n"; print STDOUT "Running tbend with arguments: $pid $eid\n";
if (system("$tbdir/tbend $pid $eid") != 0) { if (system("$tbdir/tbend $pid $eid") != 0) {
fatal("tbend failed!\n"); fatal("tbend failed!\n");
......
...@@ -238,6 +238,17 @@ if (! $batch) { ...@@ -238,6 +238,17 @@ if (! $batch) {
exit(0); exit(0);
} }
} }
#
# Gather statistics; start the clock ticking.
#
if ($frontend) {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_PRELOAD, 0,
TBDB_STATS_FLAGS_START);
}
else {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_START, 0,
TBDB_STATS_FLAGS_START);
}
# #
# Run the various scripts. We want to propogate the error from tbprerun # Run the various scripts. We want to propogate the error from tbprerun
......
...@@ -401,6 +401,22 @@ if (! $batch) { ...@@ -401,6 +401,22 @@ if (! $batch) {
} }
} }
#
# Gather stats; start clock ticking
#
if ($inout eq "in") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPIN, 0,
TBDB_STATS_FLAGS_START);
}
elsif ($inout eq "out") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPOUT, 0,
TBDB_STATS_FLAGS_START);
}
elsif ($inout eq "modify") {
GatherSwapStats($pid, $eid, $dbuid, TBDB_STATS_SWAPMODIFY, 0,
TBDB_STATS_FLAGS_START);
}
# #
# Remove old report file since its contents are going to be invalid. # Remove old report file since its contents are going to be invalid.
# #
......
...@@ -150,12 +150,12 @@ else { ...@@ -150,12 +150,12 @@ else {
$query_result = $query_result =
DBQueryFatal("select t.exptidx,s.pid,s.eid,t.action,t.exitcode,t.uid, ". DBQueryFatal("select t.exptidx,s.pid,s.eid,t.action,t.exitcode,t.uid, ".
" r.pnodes,t.idx as statno,t.tstamp as ttstamp ". " r.pnodes,t.idx as statno,t.start_time,t.end_time ".
" from testbed_stats as t ". " from testbed_stats as t ".
"left join experiment_stats as s on s.exptidx=t.exptidx ". "left join experiment_stats as s on s.exptidx=t.exptidx ".
"left join experiment_resources as r on r.idx=t.rsrcidx ". "left join experiment_resources as r on r.idx=t.rsrcidx ".
"$wclause ". "$wclause ".
"order by t.tstamp desc,t.idx desc limit $records"); "order by t.end_time desc,t.idx desc limit $records");
if (mysql_num_rows($query_result) == 0) { if (mysql_num_rows($query_result) == 0) {
USERERROR("No testbed stats records in the system!", 1); USERERROR("No testbed stats records in the system!", 1);
...@@ -168,7 +168,8 @@ echo "<table align=center border=1> ...@@ -168,7 +168,8 @@ echo "<table align=center border=1>
<th>Pid</th> <th>Pid</th>
<th>Eid</th> <th>Eid</th>
<th>ExptIdx</th> <th>ExptIdx</th>
<th>Time</th> <th>Start</th>
<th>End</th>
<th>Action (Nodes)</th> <th>Action (Nodes)</th>
<th>ECode</th> <th>ECode</th>
</tr>\n"; </tr>\n";
...@@ -179,10 +180,14 @@ while ($row = mysql_fetch_assoc($query_result)) { ...@@ -179,10 +180,14 @@ while ($row = mysql_fetch_assoc($query_result)) {
$pid = $row[pid]; $pid = $row[pid];
$eid = $row[eid]; $eid = $row[eid];
$uid = $row[uid]; $uid = $row[uid];
$when = $row[ttstamp]; $start = $row[start_time];
$end = $row[end_time];
$action = $row[action]; $action = $row[action];
$ecode = $row[exitcode]; $ecode = $row[exitcode];
$pnodes = $row[pnodes]; $pnodes = $row[pnodes];
if (!isset($end))
$end = "&nbsp";
echo "<tr> echo "<tr>
<td>$idx</td> <td>$idx</td>
...@@ -190,7 +195,8 @@ while ($row = mysql_fetch_assoc($query_result)) { ...@@ -190,7 +195,8 @@ while ($row = mysql_fetch_assoc($query_result)) {
<td>$pid</td> <td>$pid</td>
<td>$eid</td> <td>$eid</td>
<td><a href=showexpstats.php3?record=$exptidx>$exptidx</a></td> <td><a href=showexpstats.php3?record=$exptidx>$exptidx</a></td>
<td>$when</td>\n"; <td>$start</td>
<td>$end</td>\n";
if (!$ecode && (strcmp($action, "preload") && if (!$ecode && (strcmp($action, "preload") &&
strcmp($action, "destroy"))) { strcmp($action, "destroy"))) {
echo "<td>$action ($pnodes)</td>\n"; echo "<td>$action ($pnodes)</td>\n";
......
...@@ -321,7 +321,7 @@ function showrange ($showby, $sortby, $range) { ...@@ -321,7 +321,7 @@ function showrange ($showby, $sortby, $range) {
$query_result = $query_result =
DBQueryFatal("select s.pid,s.eid,t.uid,t.action,t.exptidx, ". DBQueryFatal("select s.pid,s.eid,t.uid,t.action,t.exptidx, ".
" r1.pnodes as pnodes1,r2.pnodes as pnodes2, ". " r1.pnodes as pnodes1,r2.pnodes as pnodes2, ".
" UNIX_TIMESTAMP(t.tstamp) as ttstamp ". " UNIX_TIMESTAMP(t.end_time) as ttstamp ".
" from testbed_stats as t ". " from testbed_stats as t ".
"left join experiment_stats as s on ". "left join experiment_stats as s on ".
" s.exptidx=t.exptidx ". " s.exptidx=t.exptidx ".
...@@ -330,9 +330,9 @@ function showrange ($showby, $sortby, $range) { ...@@ -330,9 +330,9 @@ function showrange ($showby, $sortby, $range) {
"left join experiment_resources as r2 on ". "left join experiment_resources as r2 on ".
" r2.idx=r1.lastidx and r1.lastidx is not null ". " r2.idx=r1.lastidx and r1.lastidx is not null ".
"where t.exitcode = 0 && ". "where t.exitcode = 0 && ".
" ((UNIX_TIMESTAMP(now())-UNIX_TIMESTAMP(t.tstamp)) ". " ((UNIX_TIMESTAMP(now())-UNIX_TIMESTAMP(t.end_time))".
" < $wclause) ". " < $wclause) ".
"order by t.tstamp"); "order by t.end_time");
# Experiment start time, indexed by pid:eid. # Experiment start time, indexed by pid:eid.
$expt_start = array(); $expt_start = array();
...@@ -362,7 +362,7 @@ function showrange ($showby, $sortby, $range) { ...@@ -362,7 +362,7 @@ function showrange ($showby, $sortby, $range) {
$swapper_result = $swapper_result =
DBQueryFatal("select action from testbed_stats ". DBQueryFatal("select action from testbed_stats ".
"where exptidx=$idx and ". "where exptidx=$idx and ".
" UNIX_TIMESTAMP(tstamp)<$tstamp ". " UNIX_TIMESTAMP(end_time)<$tstamp ".
"order by tstamp desc"); "order by tstamp desc");
while ($srow = mysql_fetch_assoc($swapper_result)) { while ($srow = mysql_fetch_assoc($swapper_result)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment