From 4982b9cdd96a4a6564d2288133985856e9938dfc Mon Sep 17 00:00:00 2001
From: David Johnson <johnsond@flux.utah.edu>
Date: Thu, 16 Feb 2006 04:42:03 +0000
Subject: [PATCH] * Makeconf.in, configure, configure.in, defs-default,
defs-johnsond-emulab: - added a new defs var, TBROBOCOPSEMAIL
* tbsetup/power_mail.pm.in:
- add some new info to robot powerup mails
* db/libdb.pm.in:
- add a new function to determine if an experiment contains nodes of a
given class/type
* tbsetup/swapexp.in:
- check if exp is a robot exp; that is, if it has robots or motes; if
so, cc error msgs to TBROBOCOPSEMAIL in addition to TBOPS
---
Makeconf.in | 1 +
configure | 24 +++++++++-----
configure.in | 4 +++
db/libdb.pm.in | 29 ++++++++++++++++
defs-default | 1 +
defs-johnsond-emulab | 1 +
tbsetup/power_mail.pm.in | 71 +++++++++++++++++++++++++++++++++++-----
tbsetup/swapexp.in | 23 ++++++++++++-
8 files changed, 135 insertions(+), 19 deletions(-)
diff --git a/Makeconf.in b/Makeconf.in
index 190d7b016c..a816f5ccb2 100644
--- a/Makeconf.in
+++ b/Makeconf.in
@@ -27,6 +27,7 @@ TBROOT = @prefix@
TBDBNAME = @TBDBNAME@
TBADMINGROUP = @TBADMINGROUP@
TBOPSEMAIL = @TBOPSEMAIL@
+TBROBOCOPSEMAIL = @TBROBOCOPSEMAIL@
TBLOGSEMAIL = @TBLOGSEMAIL@
TBAUDITEMAIL = @TBAUDITEMAIL@
TBACTIVEARCHIVE = @TBACTIVEARCHIVE@
diff --git a/configure b/configure
index c445bbc4be..2a8080cdd9 100755
--- a/configure
+++ b/configure
@@ -1397,6 +1397,8 @@ done
+
+
@@ -1610,6 +1612,8 @@ fi
#
TBOPSEMAIL_NOSLASH="$TBOPSEMAIL"
TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`"
+TBROBOCOPSEMAIL_NOSLASH="$TBROBOCOPSEMAIL"
+TBROBOCOPSEMAIL="`echo $TBROBOCOPSEMAIL | sed -e 's/@/\\\@/'`"
TBLOGSEMAIL_NOSLASH="$TBLOGSEMAIL"
TBLOGSEMAIL="`echo $TBLOGSEMAIL | sed -e 's/@/\\\@/'`"
TBAUDITEMAIL_NOSLASH="$TBAUDITEMAIL"
@@ -1947,17 +1951,17 @@ for ac_hdr in ulxmlrpcpp/ulxr_config.h
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
-echo "configure:1951: checking for $ac_hdr" >&5
+echo "configure:1955: checking for $ac_hdr" >&5
if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 1956 "configure"
+#line 1960 "configure"
#include "confdefs.h"
#include <$ac_hdr>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:1961: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:1965: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
@@ -1996,17 +2000,17 @@ for ac_hdr in linux/videodev.h
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
-echo "configure:2000: checking for $ac_hdr" >&5
+echo "configure:2004: checking for $ac_hdr" >&5
if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2005 "configure"
+#line 2009 "configure"
#include "confdefs.h"
#include <$ac_hdr>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:2010: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:2014: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
@@ -2039,7 +2043,7 @@ done
# Extract the first word of "gtk-config", so it can be a program name with args.
set dummy gtk-config; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2043: checking for $ac_word" >&5
+echo "configure:2047: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_GTK_CONFIG'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
@@ -2118,7 +2122,7 @@ fi
# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
# ./install, which can be erroneously created by make from ./install.sh.
echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6
-echo "configure:2122: checking for a BSD compatible install" >&5
+echo "configure:2126: checking for a BSD compatible install" >&5
if test -z "$INSTALL"; then
if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
@@ -2179,7 +2183,7 @@ esac
# Extract the first word of "rsync", so it can be a program name with args.
set dummy rsync; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2183: checking for $ac_word" >&5
+echo "configure:2187: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_RSYNC'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
@@ -2651,6 +2655,8 @@ s%@DELAYTHRESH@%$DELAYTHRESH%g
s%@PELABSUPPORT@%$PELABSUPPORT%g
s%@TBOPSEMAIL@%$TBOPSEMAIL%g
s%@TBOPSEMAIL_NOSLASH@%$TBOPSEMAIL_NOSLASH%g
+s%@TBROBOCOPSEMAIL@%$TBROBOCOPSEMAIL%g
+s%@TBROBOCOPSEMAIL_NOSLASH@%$TBROBOCOPSEMAIL_NOSLASH%g
s%@TBLOGSEMAIL@%$TBLOGSEMAIL%g
s%@TBLOGSEMAIL_NOSLASH@%$TBLOGSEMAIL_NOSLASH%g
s%@TBWWWEMAIL@%$TBWWWEMAIL%g
diff --git a/configure.in b/configure.in
index bb4f343733..c1d53dfdf4 100755
--- a/configure.in
+++ b/configure.in
@@ -162,6 +162,8 @@ AC_SUBST(PELABSUPPORT)
#
AC_SUBST(TBOPSEMAIL)
AC_SUBST(TBOPSEMAIL_NOSLASH)
+AC_SUBST(TBROBOCOPSEMAIL)
+AC_SUBST(TBROBOCOPSEMAIL_NOSLASH)
AC_SUBST(TBLOGSEMAIL)
AC_SUBST(TBLOGSEMAIL_NOSLASH)
AC_SUBST(TBWWWEMAIL)
@@ -314,6 +316,8 @@ fi
#
TBOPSEMAIL_NOSLASH="$TBOPSEMAIL"
TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`"
+TBROBOCOPSEMAIL_NOSLASH="$TBROBOCOPSEMAIL"
+TBROBOCOPSEMAIL="`echo $TBROBOCOPSEMAIL | sed -e 's/@/\\\@/'`"
TBLOGSEMAIL_NOSLASH="$TBLOGSEMAIL"
TBLOGSEMAIL="`echo $TBLOGSEMAIL | sed -e 's/@/\\\@/'`"
TBAUDITEMAIL_NOSLASH="$TBAUDITEMAIL"
diff --git a/db/libdb.pm.in b/db/libdb.pm.in
index c85eaf3109..b5a1f4e897 100644
--- a/db/libdb.pm.in
+++ b/db/libdb.pm.in
@@ -223,6 +223,8 @@ use vars qw(@ISA @EXPORT);
TBRobotLabExpt
+ TBExptContainsNodeCT
+
);
# Must come after package declaration!
@@ -3301,6 +3303,33 @@ sub TBRobotLabExpt($$)
return 1;
}
+#
+# is a certain type/class node present?
+# args: pid, eid, valid type/class
+#
+sub TBExptContainsNodeCT($$$)
+{
+ my ($pid,$eid,$ntc) = @_;
+
+ # find out if this is a valid class or type...
+ my $dbq = DBQueryWarn("select v.pid,v.eid,v.type from virt_nodes as v " .
+ "left join node_types as nt on v.type=nt.type " .
+ "where v.pid='$pid' and v.eid='$eid' and " .
+ "(nt.class='$ntc' or nt.type='$ntc')");
+
+
+#"select r.pid,r.eid,r.node_id from reserved as r " .
+#"left join nodes as n on n.node_id=r.node_id " .
+#"left join node_types as nt on nt.type=n.type " .
+#"where r.pid='$pid' and r.eid='$eid' and " .
+#"(nt.class='$ntc' or nt.type='$ntc')");
+
+ return 0
+ if (!$dbq || !$dbq->numrows());
+
+ return 1;
+}
+
#
# List of tables used for experiment removal/backup/restore.
#
diff --git a/defs-default b/defs-default
index 152b62b071..a1d4eb3015 100644
--- a/defs-default
+++ b/defs-default
@@ -9,6 +9,7 @@ TBDBNAME=tbdb
TBADMINGROUP=flux
TBOPSEMAIL=testbed-ops@flux.utah.edu
TBLOGSEMAIL=testbed-logs@flux.utah.edu
+TBROBOCOPSEMAIL=testbed-robocops@flux.utah.edu
TBWWWEMAIL=testbed-www@flux.utah.edu
TBAPPROVALEMAIL=testbed-approval@flux.utah.edu
TBAUDITEMAIL=testbed-audit@flux.utah.edu
diff --git a/defs-johnsond-emulab b/defs-johnsond-emulab
index fc01be3dfa..cfa8b4f870 100644
--- a/defs-johnsond-emulab
+++ b/defs-johnsond-emulab
@@ -4,6 +4,7 @@
. defs-default
TBOPSEMAIL=johnsond@flux.utah.edu
+TBROBOCOPSEMAIL=johnsond@cs.utah.edu
TBLOGSEMAIL=johnsond@flux.utah.edu
TBWWWEMAIL=johnsond@flux.utah.edu
TBAPPROVALEMAIL=johnsond@flux.utah.edu
diff --git a/tbsetup/power_mail.pm.in b/tbsetup/power_mail.pm.in
index 20acbe3665..2b1b5fd81b 100644
--- a/tbsetup/power_mail.pm.in
+++ b/tbsetup/power_mail.pm.in
@@ -108,9 +108,11 @@ sub mailctrl($@) {
}
$cond_str .= " node_id='$node'";
}
- $cond_str .= ") group by email";
+ $cond_str .= ")";
- my $dbres = DBQueryFatal("select email from location_info ".$cond_str);
+ my $dbres = DBQueryFatal("select email from location_info " .
+ $cond_str . " group by email");
+
if ($dbres->num_rows() != 0) {
my $row;
while (($row = $dbres->fetchrow_hashref())) {
@@ -124,19 +126,70 @@ sub mailctrl($@) {
push @emails, $TBOPS;
}
+ my $email_body =
+ "Someone needs to power $cmd the following nodes:\n" .
+ "\t\n" . join(" ",@nodes) . "\n\nfor $pid/$eid, " .
+ "swapped in by $swapper_uid.\n" .
+ "\nAnd update power time through this web page:\n" .
+ "\n https://$WWW/powertime.php3?node_id=" . join(",",@nodes) .
+ "\n";
+
+ $dbres = DBQueryFatal("select node_id,battery_voltage as v, " .
+ "battery_percentage as p, " .
+ "(UNIX_TIMESTAMP(NOW()) - battery_timestamp)".
+ " as tdelta from nodes " .
+ "where battery_voltage is not NULL");
+ my $row;
+ my %powinfo = ();
+ while (($row = $dbres->fetchrow_hashref())) {
+ $powinfo{$row->{'node_id'}} = { 'v' => $row->{'v'},
+ 'p' => $row->{'p'},
+ 'tdelta' => $row->{'tdelta'}
+ };
+ }
+
+ $email_body .=
+ "\nHere's the last known battery info for these robots. If \n" .
+ "it's been more than 3 days since last power update, or if \n" .
+ "the remaining percent is below 50 or the voltage is below 7.5,\n".
+ "you should probably replace the battery.\n\n";
+
+ foreach $bot (@nodes) {
+ if (defined($powinfo{$bot})) {
+ my $ts = $powinfo{$bot}{'tdelta'};
+ my $time_str;
+ if ($ts > (3600*24)) {
+ my $tts = sprintf("%.2f",($ts / (3600*24)));
+ $time_str = $tts . " days since last update.";
+ }
+ else {
+ my $tts = sprintf("%.2f",($ts / 3600));
+ $time_str = $tts . " hours since last update.";
+ }
+
+ $email_body .=
+ $bot . ": " . sprintf("%.2f",$powinfo{$bot}{'p'}) .
+ "%, " . sprintf("%.2f",$powinfo{$bot}{'v'}) .
+ "V, " . $time_str . "\n";
+ }
+ else {
+ $email_body .= "$bot: no info!!!\n";
+ }
+ }
+
+ $email_body .= "\nThe Power Control Dude.\n";
+
foreach $email (@emails) {
+ #print "Sending to $email\n\n";
if ($email ne "") {
SENDMAIL($email,
- "Power $cmd nodes for $pid/$eid\n" .
- "Someone needs to power $cmd the following nodes:\n" .
- "\t\n" . join(" ",@nodes) . "\n\nfor $pid/$eid, " .
- "swapped in by $swapper_uid.\n" .
- "\nAnd update power time through this web page:\n" .
- "\n https://$WWW/powertime.php3?node_id=" . join(",",@nodes),
- $TBOPS);
+ "Power $cmd nodes for $pid/$eid\n",
+ $email_body);
}
}
+
+
foreach my $node (keys %actual) {
my $tries = $default_tries;
my $ok = 0;
diff --git a/tbsetup/swapexp.in b/tbsetup/swapexp.in
index f0217ba948..70da8f241c 100644
--- a/tbsetup/swapexp.in
+++ b/tbsetup/swapexp.in
@@ -65,6 +65,7 @@ sub ExitWithStatus($$)
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
+my $TBROBOCOPS = "@TBROBOCOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
my $TBINFO = "$TB/expinfo";
my $TBDOCBASE = "@TBDOCBASE@";
@@ -102,6 +103,7 @@ my $eventsys_restart = 0;
my $errorstat= -1;
my $modifyHosed = 0;
my $modifySwapped = 0;
+my $robotexp = 0;
my $inout;
my $logname;
@@ -321,6 +323,16 @@ if ($overquota) {
#
my $firewalled = TBExptFirewall($pid, $eid);
+#
+# see if we've got a robot exp (this isn't the only check; if this is a
+# swapmod, we've got to check tbprerun as well...
+#
+$robotexp =
+ TBExptContainsNodeCT($pid,$eid,'robot') ||
+ TBExptContainsNodeCT($pid,$eid,'mote') ||
+ TBExptContainsNodeCT($pid,$eid,'motehost') ||
+ TBExptContainsNodeCT($pid,$eid,'powermon');
+
#
# We have to protect against trying to end an experiment that is currently
# in the process of being terminated. We use a "wrapper" state (actually
@@ -1291,6 +1303,15 @@ sub cleanup()
#
# Send a message to the testbed list. Append the logfile.
#
+
+ #
+ # also try to send mail to robocops if it was a robot/mote exp
+ #
+ my $rcops = '';
+ if ($robotexp) {
+ $rcops = "\nCc: $TBROBOCOPS";
+ }
+
SENDMAIL("$user_name <$user_email>",
"Swap ${inout} Failure: $pid/$eid",
"Please look at the log below to see what happened. If the error\n".
@@ -1305,7 +1326,7 @@ sub cleanup()
"#BatchMode\n",
($idleswap ? $TBOPS : "$user_name <$user_email>"),
"Cc: $expt_head_name <$expt_head_email>\n".
- "Cc: $TBOPS",
+ "Cc: $TBOPS $rcops",
(($logname), (defined($modnsfile) ? ($modnsfile) : ())));
if ($modifyHosed) {
--
GitLab