Commit 4982b9cd authored by David Johnson's avatar David Johnson

* Makeconf.in, configure, configure.in, defs-default, defs-johnsond-emulab:

    - added a new defs var, TBROBOCOPSEMAIL

  * tbsetup/power_mail.pm.in:
    - add some new info to robot powerup mails

  * db/libdb.pm.in:
    - add a new function to determine if an experiment contains nodes of a
      given class/type

  * tbsetup/swapexp.in:
    - check if exp is a robot exp; that is, if it has robots or motes; if
      so, cc error msgs to TBROBOCOPSEMAIL in addition to TBOPS
parent 753dc231
......@@ -27,6 +27,7 @@ TBROOT = @prefix@
TBDBNAME = @TBDBNAME@
TBADMINGROUP = @TBADMINGROUP@
TBOPSEMAIL = @TBOPSEMAIL@
TBROBOCOPSEMAIL = @TBROBOCOPSEMAIL@
TBLOGSEMAIL = @TBLOGSEMAIL@
TBAUDITEMAIL = @TBAUDITEMAIL@
TBACTIVEARCHIVE = @TBACTIVEARCHIVE@
......
......@@ -1397,6 +1397,8 @@ done
......@@ -1610,6 +1612,8 @@ fi
#
TBOPSEMAIL_NOSLASH="$TBOPSEMAIL"
TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`"
TBROBOCOPSEMAIL_NOSLASH="$TBROBOCOPSEMAIL"
TBROBOCOPSEMAIL="`echo $TBROBOCOPSEMAIL | sed -e 's/@/\\\@/'`"
TBLOGSEMAIL_NOSLASH="$TBLOGSEMAIL"
TBLOGSEMAIL="`echo $TBLOGSEMAIL | sed -e 's/@/\\\@/'`"
TBAUDITEMAIL_NOSLASH="$TBAUDITEMAIL"
......@@ -1947,17 +1951,17 @@ for ac_hdr in ulxmlrpcpp/ulxr_config.h
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
echo "configure:1951: checking for $ac_hdr" >&5
echo "configure:1955: checking for $ac_hdr" >&5
if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
#line 1956 "configure"
#line 1960 "configure"
#include "confdefs.h"
#include <$ac_hdr>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
{ (eval echo configure:1961: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
{ (eval echo configure:1965: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
......@@ -1996,17 +2000,17 @@ for ac_hdr in linux/videodev.h
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
echo "configure:2000: checking for $ac_hdr" >&5
echo "configure:2004: checking for $ac_hdr" >&5
if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
#line 2005 "configure"
#line 2009 "configure"
#include "confdefs.h"
#include <$ac_hdr>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
{ (eval echo configure:2010: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
{ (eval echo configure:2014: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
......@@ -2039,7 +2043,7 @@ done
# Extract the first word of "gtk-config", so it can be a program name with args.
set dummy gtk-config; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
echo "configure:2043: checking for $ac_word" >&5
echo "configure:2047: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_GTK_CONFIG'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
......@@ -2118,7 +2122,7 @@ fi
# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
# ./install, which can be erroneously created by make from ./install.sh.
echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6
echo "configure:2122: checking for a BSD compatible install" >&5
echo "configure:2126: checking for a BSD compatible install" >&5
if test -z "$INSTALL"; then
if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
......@@ -2179,7 +2183,7 @@ esac
# Extract the first word of "rsync", so it can be a program name with args.
set dummy rsync; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
echo "configure:2183: checking for $ac_word" >&5
echo "configure:2187: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_RSYNC'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
......@@ -2651,6 +2655,8 @@ s%@DELAYTHRESH@%$DELAYTHRESH%g
s%@PELABSUPPORT@%$PELABSUPPORT%g
s%@TBOPSEMAIL@%$TBOPSEMAIL%g
s%@TBOPSEMAIL_NOSLASH@%$TBOPSEMAIL_NOSLASH%g
s%@TBROBOCOPSEMAIL@%$TBROBOCOPSEMAIL%g
s%@TBROBOCOPSEMAIL_NOSLASH@%$TBROBOCOPSEMAIL_NOSLASH%g
s%@TBLOGSEMAIL@%$TBLOGSEMAIL%g
s%@TBLOGSEMAIL_NOSLASH@%$TBLOGSEMAIL_NOSLASH%g
s%@TBWWWEMAIL@%$TBWWWEMAIL%g
......
......@@ -162,6 +162,8 @@ AC_SUBST(PELABSUPPORT)
#
AC_SUBST(TBOPSEMAIL)
AC_SUBST(TBOPSEMAIL_NOSLASH)
AC_SUBST(TBROBOCOPSEMAIL)
AC_SUBST(TBROBOCOPSEMAIL_NOSLASH)
AC_SUBST(TBLOGSEMAIL)
AC_SUBST(TBLOGSEMAIL_NOSLASH)
AC_SUBST(TBWWWEMAIL)
......@@ -314,6 +316,8 @@ fi
#
TBOPSEMAIL_NOSLASH="$TBOPSEMAIL"
TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`"
TBROBOCOPSEMAIL_NOSLASH="$TBROBOCOPSEMAIL"
TBROBOCOPSEMAIL="`echo $TBROBOCOPSEMAIL | sed -e 's/@/\\\@/'`"
TBLOGSEMAIL_NOSLASH="$TBLOGSEMAIL"
TBLOGSEMAIL="`echo $TBLOGSEMAIL | sed -e 's/@/\\\@/'`"
TBAUDITEMAIL_NOSLASH="$TBAUDITEMAIL"
......
......@@ -223,6 +223,8 @@ use vars qw(@ISA @EXPORT);
TBRobotLabExpt
TBExptContainsNodeCT
);
# Must come after package declaration!
......@@ -3301,6 +3303,33 @@ sub TBRobotLabExpt($$)
return 1;
}
#
# is a certain type/class node present?
# args: pid, eid, valid type/class
#
sub TBExptContainsNodeCT($$$)
{
my ($pid,$eid,$ntc) = @_;
# find out if this is a valid class or type...
my $dbq = DBQueryWarn("select v.pid,v.eid,v.type from virt_nodes as v " .
"left join node_types as nt on v.type=nt.type " .
"where v.pid='$pid' and v.eid='$eid' and " .
"(nt.class='$ntc' or nt.type='$ntc')");
#"select r.pid,r.eid,r.node_id from reserved as r " .
#"left join nodes as n on n.node_id=r.node_id " .
#"left join node_types as nt on nt.type=n.type " .
#"where r.pid='$pid' and r.eid='$eid' and " .
#"(nt.class='$ntc' or nt.type='$ntc')");
return 0
if (!$dbq || !$dbq->numrows());
return 1;
}
#
# List of tables used for experiment removal/backup/restore.
#
......
......@@ -9,6 +9,7 @@ TBDBNAME=tbdb
TBADMINGROUP=flux
TBOPSEMAIL=testbed-ops@flux.utah.edu
TBLOGSEMAIL=testbed-logs@flux.utah.edu
TBROBOCOPSEMAIL=testbed-robocops@flux.utah.edu
TBWWWEMAIL=testbed-www@flux.utah.edu
TBAPPROVALEMAIL=testbed-approval@flux.utah.edu
TBAUDITEMAIL=testbed-audit@flux.utah.edu
......
......@@ -4,6 +4,7 @@
. defs-default
TBOPSEMAIL=johnsond@flux.utah.edu
TBROBOCOPSEMAIL=johnsond@cs.utah.edu
TBLOGSEMAIL=johnsond@flux.utah.edu
TBWWWEMAIL=johnsond@flux.utah.edu
TBAPPROVALEMAIL=johnsond@flux.utah.edu
......
......@@ -108,9 +108,11 @@ sub mailctrl($@) {
}
$cond_str .= " node_id='$node'";
}
$cond_str .= ") group by email";
$cond_str .= ")";
my $dbres = DBQueryFatal("select email from location_info ".$cond_str);
my $dbres = DBQueryFatal("select email from location_info " .
$cond_str . " group by email");
if ($dbres->num_rows() != 0) {
my $row;
while (($row = $dbres->fetchrow_hashref())) {
......@@ -124,19 +126,70 @@ sub mailctrl($@) {
push @emails, $TBOPS;
}
my $email_body =
"Someone needs to power $cmd the following nodes:\n" .
"\t\n" . join(" ",@nodes) . "\n\nfor $pid/$eid, " .
"swapped in by $swapper_uid.\n" .
"\nAnd update power time through this web page:\n" .
"\n https://$WWW/powertime.php3?node_id=" . join(",",@nodes) .
"\n";
$dbres = DBQueryFatal("select node_id,battery_voltage as v, " .
"battery_percentage as p, " .
"(UNIX_TIMESTAMP(NOW()) - battery_timestamp)".
" as tdelta from nodes " .
"where battery_voltage is not NULL");
my $row;
my %powinfo = ();
while (($row = $dbres->fetchrow_hashref())) {
$powinfo{$row->{'node_id'}} = { 'v' => $row->{'v'},
'p' => $row->{'p'},
'tdelta' => $row->{'tdelta'}
};
}
$email_body .=
"\nHere's the last known battery info for these robots. If \n" .
"it's been more than 3 days since last power update, or if \n" .
"the remaining percent is below 50 or the voltage is below 7.5,\n".
"you should probably replace the battery.\n\n";
foreach $bot (@nodes) {
if (defined($powinfo{$bot})) {
my $ts = $powinfo{$bot}{'tdelta'};
my $time_str;
if ($ts > (3600*24)) {
my $tts = sprintf("%.2f",($ts / (3600*24)));
$time_str = $tts . " days since last update.";
}
else {
my $tts = sprintf("%.2f",($ts / 3600));
$time_str = $tts . " hours since last update.";
}
$email_body .=
$bot . ": " . sprintf("%.2f",$powinfo{$bot}{'p'}) .
"%, " . sprintf("%.2f",$powinfo{$bot}{'v'}) .
"V, " . $time_str . "\n";
}
else {
$email_body .= "$bot: no info!!!\n";
}
}
$email_body .= "\nThe Power Control Dude.\n";
foreach $email (@emails) {
#print "Sending to $email\n\n";
if ($email ne "") {
SENDMAIL($email,
"Power $cmd nodes for $pid/$eid\n" .
"Someone needs to power $cmd the following nodes:\n" .
"\t\n" . join(" ",@nodes) . "\n\nfor $pid/$eid, " .
"swapped in by $swapper_uid.\n" .
"\nAnd update power time through this web page:\n" .
"\n https://$WWW/powertime.php3?node_id=" . join(",",@nodes),
$TBOPS);
"Power $cmd nodes for $pid/$eid\n",
$email_body);
}
}
foreach my $node (keys %actual) {
my $tries = $default_tries;
my $ok = 0;
......
......@@ -65,6 +65,7 @@ sub ExitWithStatus($$)
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $TBROBOCOPS = "@TBROBOCOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
my $TBINFO = "$TB/expinfo";
my $TBDOCBASE = "@TBDOCBASE@";
......@@ -102,6 +103,7 @@ my $eventsys_restart = 0;
my $errorstat= -1;
my $modifyHosed = 0;
my $modifySwapped = 0;
my $robotexp = 0;
my $inout;
my $logname;
......@@ -321,6 +323,16 @@ if ($overquota) {
#
my $firewalled = TBExptFirewall($pid, $eid);
#
# see if we've got a robot exp (this isn't the only check; if this is a
# swapmod, we've got to check tbprerun as well...
#
$robotexp =
TBExptContainsNodeCT($pid,$eid,'robot') ||
TBExptContainsNodeCT($pid,$eid,'mote') ||
TBExptContainsNodeCT($pid,$eid,'motehost') ||
TBExptContainsNodeCT($pid,$eid,'powermon');
#
# We have to protect against trying to end an experiment that is currently
# in the process of being terminated. We use a "wrapper" state (actually
......@@ -1291,6 +1303,15 @@ sub cleanup()
#
# Send a message to the testbed list. Append the logfile.
#
#
# also try to send mail to robocops if it was a robot/mote exp
#
my $rcops = '';
if ($robotexp) {
$rcops = "\nCc: $TBROBOCOPS";
}
SENDMAIL("$user_name <$user_email>",
"Swap ${inout} Failure: $pid/$eid",
"Please look at the log below to see what happened. If the error\n".
......@@ -1305,7 +1326,7 @@ sub cleanup()
"#BatchMode\n",
($idleswap ? $TBOPS : "$user_name <$user_email>"),
"Cc: $expt_head_name <$expt_head_email>\n".
"Cc: $TBOPS",
"Cc: $TBOPS $rcops",
(($logname), (defined($modnsfile) ? ($modnsfile) : ())));
if ($modifyHosed) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment