From 761639208e66e30acc157517b964af3bce1a8bc0 Mon Sep 17 00:00:00 2001 From: Leigh B Stoller Date: Thu, 24 May 2012 17:45:33 -0600 Subject: [PATCH] More improvements. Thanks to Gary for figuring out a great query; see comments in the code. --- tbsetup/node_history.in | 253 +++++++++++++++++++++++---------------- www/node_defs.php | 54 ++++++--- www/shownodehistory.php3 | 96 ++++++++------- 3 files changed, 244 insertions(+), 159 deletions(-) diff --git a/tbsetup/node_history.in b/tbsetup/node_history.in index 5be889b5a..03d59d7d6 100644 --- a/tbsetup/node_history.in +++ b/tbsetup/node_history.in @@ -8,7 +8,6 @@ use English; use Getopt::Std; use Date::Parse; use Time::Local; -use Math::BigInt; # # Drill down the node history data in the DB @@ -46,15 +45,18 @@ sub usage { " use negative value for last num records\n". " -r reverse order (most recent first)\n". " -s show a summary of node's past usage\n". + " -d date Start at YYYY-MM-DD HH:MM:SS\n". + " -c used with -d, show current status of all nodes\n". " -v verbose output\n". " -w print warnings about anomolous records\n"); exit(1); } -my $optlist = "ARS:aln:rswvd:i:"; +my $optlist = "ARS:aln:rswvd:i:c"; my $warnme = 0; my $verbose = 0; my $showall = 0; +my $showcurrent = 0; my $showalloconly = 0; my $list = 0; my $summary = 0; @@ -66,18 +68,12 @@ my $ip; # # Sort stuff. sortby value should correspond to record field format: # -# sortby 0: node -# sortby 1: pideid -# sortby 2: uid -# sortby 3: date -# sortby 4: elapsed -# my %sortmap = ( - "node" => 0, - "pideid" => 1, - "uid" => 2, - "date" => 3, - "elapsed" => 4 + "node" => 0, + "pideid" => 6, + "uid" => 3, + "date" => 1, + "elapsed" => 2 ); my $sortbydate = $sortmap{date}; my $sortby = $sortbydate; @@ -115,7 +111,7 @@ if (defined($options{"R"})) { $raw = 1; } if (defined($options{"S"})) { - if (!defined($sortmap{$options{"S"}})) { + if (!exists($sortmap{$options{"S"}})) { print STDERR "invalid sort field '$options{S}'\n"; usage(); } @@ -140,6 +136,11 @@ if (defined($options{"d"})) { $datetime = timelocal(strptime($options{"d"})); $summary = 0; } +if (defined($options{"c"})) { + $showcurrent = 1; + usage() + if (!defined($datetime)); +} if (defined($options{"i"})) { $ip = $options{"i"}; $summary = 0; @@ -150,7 +151,7 @@ if (defined($options{"w"})) { if (defined($options{"v"})) { $verbose = 1; } -if (!$showall && @ARGV == 0) { +if (!$showall && @ARGV == 0 && !defined($ip)) { usage(); } my @nodes = @ARGV; @@ -159,13 +160,90 @@ if (!$list && !$summary) { $list = 1; } +my %nodeinfo; # [ expt, starttime, uid ] +my @rawrecords; +my %startinfo; + # # Common case: a single node. # Just fetch data for it, otherwise fetch data for all. # +# +# If we have a datetime, then we want to start with records greater +# then the stamp, but we will not know the state at that point, since +# it will be the most recent record *before* the requested stamp, that +# says what the node is doing at the time. So, this query does some +# amazing magic to find those records without an explosion of terms +# that takes forever. Kudos to Gary for figuring out this query. +# +if ($datetime) { + my $querymod = ""; + if (@nodes) { + $querymod = " AND (" . + join(" or ", map {"h1.node_id='$_'"} @nodes) . ")"; + } + elsif (defined($ip)) { + $querymod = " AND h1.cnet_ip='$ip'"; + } + + my $query_result = + DBQueryFatal("SELECT h1.node_id,h1.op,h1.stamp,h1.uid,h1.exptidx,". + " s.pid,s.eid ". + " FROM node_history as h1 ". + "JOIN (SELECT h3.node_id,MAX(h3.stamp) as stamp ". + " FROM node_history as h3 ". + " WHERE stamp < $datetime GROUP BY h3.node_id ". + " ) AS h2 ON h1.node_id=h2.node_id AND ". + " h1.stamp = h2.stamp ". + "left join nodes as n on n.node_id=h1.node_id ". + "left join node_types as t on t.type=n.type ". + "left join experiment_stats as s on s.exptidx=h1.exptidx ". + "where t.isplabdslice=0 and t.isplabphysnode=0 and ". + " (n.role='testnode' or n.role='virtnode') ". + "$querymod order by h1.node_id"); + + while (my %row = $query_result->fetchhash()) { + my $pid = $row{pid}; + my $eid = $row{eid}; + my $pideid = "$pid/$eid"; + my $exptidx = $row{"exptidx"}; + my $node = $row{node_id}; + my $stamp = $row{stamp}; + my $uid = $row{uid}; + my $op = $row{op}; + my $cstamp = ctime($stamp); + my $diff = $datetime - $stamp; + chomp($cstamp); + + if ($op eq "alloc" || $op eq "move" || $op eq "create") { + if ($showcurrent) { + next + if ($pid eq TBOPSPID()); + + if ($verbose) { + print "$node '$stamp' $diff $uid $pid $eid\n"; + } + else { + print "$node REC $stamp $diff $uid $pid $eid\n"; + } + } + else { + $nodeinfo{$node} = [ $pideid, $exptidx, $stamp, $uid ]; + } + } + elsif ($op eq "free" || $op eq "destroy") { + if (!$showcurrent) { + $nodeinfo{$node} = [ "", undef, $stamp, $uid ]; + } + } + } +} +exit(0) + if ($showcurrent); + my $querymod = ""; -if (@nodes == 1) { - $querymod = " AND node_id='$nodes[0]'"; +if (@nodes) { + $querymod = " AND (" . join(" or ", map {"node_id='$_'"} @nodes) . ")"; } elsif (defined($ip)) { $querymod = " AND cnet_ip='$ip'"; @@ -181,10 +259,9 @@ my $query_result = " experiment_stats.exptidx,cnet_ip,phys_nodeid ". "FROM node_history,experiment_stats ". "WHERE node_history.exptidx=experiment_stats.exptidx ". - "$querymod $orderby $limitby"); - -my %nodeinfo; # [ expt, starttime, uid ] -my @records; + $querymod . + ($datetime ? " and stamp>='$datetime' " : ""). + "$orderby $limitby"); while (my %row = $query_result->fetchhash()) { my $pideid = "$row{pid}/$row{eid}"; @@ -254,45 +331,33 @@ while (my %row = $query_result->fetchhash()) { } $nodeinfo{$node} = [ "", undef, $stamp, $uid ]; } - + #print "R: $node, $opideid, $oidx, $ouid, $ostamp, $elapsed\n"; + # save off the record - push(@records, [ $node, $opideid, $oidx, $ouid, $ostamp, $elapsed ]); -} - -# Include the current state of nodes in a final record -my $stamp = time(); -for $node (keys(%nodeinfo)) { - my ($opideid, $oidx, $ostamp, $ouid) = @{$nodeinfo{$node}}; - my $elapsed = $stamp - $ostamp; - push(@records, [ $node, $opideid, $oidx, $ouid, $ostamp, $elapsed ]); + push(@rawrecords, [ $node, $opideid, $oidx, $ouid, $ostamp, $elapsed ]); } -# Prune the list based on date range (someday) - -# Sort the list as desired -if ($sortby ne "date") { - @records = sort byfield @records; -} -if ($revorder) { - @records = reverse(@records); -} - -# Prune to the proper number of entries (first/last $numrecs entries) -if ($numrecs && $numrecs < $#records) { - if ($numrecs > 0) { - @records = @records[0 .. $numrecs-1]; - } else { - @records = @records[$numrecs .. -1 ]; +if (!$datetime) { + # Include the current state of nodes in a final record + my $stamp = time(); + for $node (keys(%nodeinfo)) { + my ($opideid, $oidx, $ostamp, $ouid) = @{$nodeinfo{$node}}; + my $elapsed = $stamp - $ostamp; + push(@rawrecords, [ $node, $opideid, $oidx, $ouid, $ostamp, $elapsed ]); } } # -# Loop over the remaining records, computing summary stats -# and printing (if desired). +# Loop over the raw records, computing summary stats and creating +# a another set of records to print (if desired). # -for my $rec (@records) { +my @records = (); + +for my $rec (@rawrecords) { my ($node, $pideid, $exptidx, $uid, $stamp, $elapsed) = @{$rec}; + #print "RR: $node, $elapsed\n"; + if (!defined($nodestats{$node})) { $nodestats{$node} = [ 0, 0, 0, 0 ]; } @@ -312,65 +377,51 @@ for my $rec (@records) { $nodestats{$node} = [ $ftime, $atime, $rtime, $dtime ]; if ($list) { - if ($verbose) { - my $start = ctime($stamp); - chomp($start); - my $end = ctime($stamp + $elapsed); - chomp($end); - - if ($datetime) { - my $d = Math::BigInt->new($datetime); - my $s = Math::BigInt->new($stamp); - my $e = Math::BigInt->new($elapsed); - $e->badd($s); - my $c1 = $s->bcmp($d); - my $c2 = $e->bcmp($d); + my ($pid, $eid); + if ($pideid eq "") { + $pid = $eid = ""; + $exptidx = 0; + } else { + ($pid, $eid) = split("/", $pideid); + } + push(@records, + [$node, $stamp, $elapsed, $uid, $pid, $eid, $exptidx, "$pid/$eid"]) + if (!$showalloconly || $isalloced); + } +} - next - if ($c1 < 0 && $c2 <= 0); +# Sort the list as desired +if ($sortby ne $sortbydate) { + @records = sort byfield @records; +} +if ($revorder) { + @records = reverse(@records); +} - if ($c1 <= 0 && $c2 > 0) { - print "$node: $pideid($exptidx) from $start til $end ". - "($elapsed sec)\n" - if ($isalloced); - last; - } - } - print "$node: $pideid($exptidx) from $start til $end ($elapsed sec)\n" - if (!$showalloconly || $isalloced); - } else { - my ($pid, $eid); - if ($pideid eq "") { - $pid = $eid = ""; - $exptidx = 0; - } else { - ($pid, $eid) = split("/", $pideid); - } - if ($datetime) { - my $d = Math::BigInt->new($datetime); - my $s = Math::BigInt->new($stamp); - my $e = Math::BigInt->new($elapsed); - $e->badd($s); - my $c1 = $s->bcmp($d); - my $c2 = $e->bcmp($d); +# Prune to the proper number of entries (first/last $numrecs entries) +if ($numrecs && $numrecs < $#records) { + if ($numrecs > 0) { + @records = @records[0 .. $numrecs-1]; + } else { + @records = @records[$numrecs .. -1 ]; + } +} - next - if ($c1 < 0 && $c2 <= 0); +if (@records) { + foreach my $record (@records) { + my ($node, $stamp, $elapsed, $uid, $pid, $eid) = @$record; - if ($c1 <= 0 && $c2 > 0) { - print "$node REC $stamp $elapsed $uid $pid $eid $exptidx\n"; - last; - } - } - print "$node REC $stamp $elapsed $uid $pid $eid $exptidx\n" - if (!$showalloconly || $isalloced); + if ($verbose) { + $stamp = ctime($stamp); + chomp($stamp); + print "$node '$stamp' $elapsed $uid $pid $eid\n"; + } + else { + print "$node REC $stamp $elapsed $uid $pid $eid\n"; } } } -print scalar(@records) . " records\n" - if (0 && $list && $verbose); - # # Print out summary information # diff --git a/www/node_defs.php b/www/node_defs.php index 9c16effb6..629feb4cc 100644 --- a/www/node_defs.php +++ b/www/node_defs.php @@ -252,6 +252,20 @@ class Node return $row["isremotenode"]; } + function IsVirtNode() { + $type = $this->type(); + + $query_result = + DBQueryFatal("select isvirtnode from node_types ". + "where type='$type'"); + + if (mysql_num_rows($query_result) == 0) { + return 0; + } + $row = mysql_fetch_array($query_result); + return $row["isremotenode"]; + } + function NodeStatus() { $node_id = $this->node_id(); @@ -1331,7 +1345,7 @@ class Node # # Show history. # -function ShowNodeHistory($node = null, +function ShowNodeHistory($node_id = null, $showall = 0, $count = 20, $reverse = 1, $date = null, $IP = null) { global $TBSUEXEC_PATH; @@ -1341,33 +1355,43 @@ function ShowNodeHistory($node = null, $rtime = 0; $dtime = 0; $nodestr = ""; - + $arg = ""; $opt = "-ls"; if (!$showall) { $opt .= "a"; } - if ($node) { - $node_id = $node->node_id(); + if ($date) { + $opt .= " -d " . escapeshellarg($date); + } + if ($node_id || $IP) { + if ($IP) { + $opt .= " -i " . escapeshellarg($IP); + $nodestr = "Node"; + } + else { + $arg = escapeshellarg($node_id); + } } else { $node_id = ""; - $opt .= "A"; + $opt .= " -A"; $nodestr = "Node"; + # + # When supplying a date, we want a summary of all nodes at that + # point in time, not a listing. + # + if ($date) { + $opt .= " -c"; + } } if ($reverse) { - $opt .= "r"; + $opt .= " -r"; } if ($count) { $opt .= " -n $count"; } - if ($date) { - $opt .= " -d " . escapeshellarg($date); - } - if ($IP) { - $opt .= " -i " . escapeshellarg($IP); - } if ($fp = popen("$TBSUEXEC_PATH nobody nobody ". - " webnode_history $opt $node_id", "r")) { + " webnode_history $opt $arg", "r")) { if (!$showall) { $str = "Allocation"; } else { @@ -1457,8 +1481,10 @@ function ShowNodeHistory($node = null, } if ($node_id == "") { + $nodeurl = CreateURL("shownodehistory", + URLARG_NODEID, $nodeid); echo " - $nodeid + $nodeid $pid $eid"; if ($PROTOGENI) { diff --git a/www/shownodehistory.php3 b/www/shownodehistory.php3 index 9b3d9308c..e05c4f663 100644 --- a/www/shownodehistory.php3 +++ b/www/shownodehistory.php3 @@ -26,7 +26,9 @@ $optargs = OptionalPageArguments("showall", PAGEARG_BOOLEAN, "count", PAGEARG_INTEGER, "datetime", PAGEARG_STRING, "IP", PAGEARG_STRING, - "node", PAGEARG_NODE); + # To allow for pcvm search, since they are + # transient and will not map to a node. + "node_id", PAGEARG_STRING); # # Standard Testbed Header @@ -37,35 +39,52 @@ if (!isset($showall)) { $showall = 0; } if (!isset($count)) { - $count = 20; + $count = 200; } if (!isset($reverse)) { $reverse = 1; } -if (!isset($datetime)) { - $datetime = ""; +if (isset($datetime) && $datetime != "") { + if (! strtotime($datetime)) { + USERERROR("Invalid date specified", 1); + } + $dateopt = "&datetime=" . urlencode($datetime); +} +else { + $dateopt = ""; } if (isset($IP)) { if (! preg_match('/^[0-9\.]+$/', $IP)) { USERERROR("Does not look like a valid IP address.", 1); } $node = Node::LookupByIP($IP); + # - # No record might mean the node does not exist, or that it - # is a virtual node. We are going to pass IP through to the - # backend in either case. + # Switch to a node_id if its a physical node. Otherwise, + # continue with the IP. # - if ($node && $node->IsRemote()) { - unset($node); + if ($node && !$node->IsVirtNode()) { + $node_id = $node->node_id(); + $IP = null; } } else { $IP = null; } -$node_id = (isset($node) ? $node->node_id() : ""); -$node_opt = (isset($node) ? "&node_id=$node_id" : ""); +if (isset($node_id)) { + $node_opt = "&node_id=$node_id"; + $form_opt = ""; +} +else if (isset($IP)) { + $node_opt = "&IP=$IP"; + $form_opt = ""; +} +else { + $node_opt = ""; + $form_opt = ""; +} -$opts="count=$count&reverse=$reverse$node_opt"; +$opts="count=$count&reverse=$reverse$node_opt$dateopt"; echo "Show records: "; if ($showall) { echo "allocated only, @@ -75,7 +94,7 @@ if ($showall) { all"; } -$opts="count=$count&showall=$showall$node_opt"; +$opts="count=$count&showall=$showall$node_opt$dateopt"; echo "
Order by: "; if ($reverse == 0) { echo "lastest first, @@ -85,17 +104,17 @@ if ($reverse == 0) { earliest first"; } -$opts="showall=$showall&reverse=$reverse$node_opt"; +$opts="showall=$showall&reverse=$reverse$node_opt$dateopt"; echo "
Show number: "; -if ($count != 20) { - echo "first 20, "; +if ($count != 200) { + echo "first 200, "; } else { - echo "first 20, "; + echo "first 200, "; } -if ($count != -20) { - echo "last 20, "; +if ($count != -200) { + echo "last 200, "; } else { - echo "last 20, "; + echo "last 200, "; } if ($count != 0) { echo "all"; @@ -104,58 +123,47 @@ if ($count != 0) { } # -# Spit out a date form. +# Spit out the various search forms. # -if ($datetime == "") { - $datetime = "mm/dd/yy HH:MM"; -} echo "
"; echo "\n"; -# Only display search form for a specific node. -if ($node_id != "") { - echo " +echo " + size=20 + value=\"" . ($datetime ? $datetime : "mm/dd/yy HH:MM") . "\"> + + + $form_opt \n"; - echo "\n"; -} +echo "\n"; echo " + + \n"; echo "\n"; - echo " + + \n"; echo "\n"; echo "
Show Datetime:
Search for Node:
Search for IP:

\n"; -if ($node_id != "" && $datetime != "" && $datetime != "mm/dd/yy HH:MM") { - if (strtotime($datetime)) { - ShowNodeHistory($node, 1, 1, 0, $datetime, $IP); - } - else { - USERERROR("Invalid date specified", 1); - } -} -else { - ShowNodeHistory((isset($node) ? $node : null), - $showall, $count, $reverse, null, $IP); -} +ShowNodeHistory($node_id, $showall, $count, $reverse, $datetime, $IP); # # Standard Testbed Footer -- GitLab