Commit a46a911b authored by David Johnson's avatar David Johnson

Process WFA logs to detect success/failure; maybe send email.

We look for Traceback strings as error indicators; and specific
strings to indicate success.  Then if specific email-related env
vars are set, we'll send emails on failure and maybe on success,
with key bits of logfiles attached.  The env vars you would set are

TESTEMAIL=you@mail.wherever
TESTMAILFAILURE=1
TESTMAILSUCCESS=1

You must set TESTEMAIL to get any mail.  If you set TESTMAILFAILURE
to 1, you'll get failure notifications.  If you set TESTMAILSUCCESS
to 1, you'll also get successful test notifications.

Now I don't have to stay up all night babysitting these test runs.
parent 265047ce
Pipeline #1896 passed with stage
in 2 seconds
......@@ -195,15 +195,29 @@ done
#
# Wait for everything to finish, then sync back logfiles.
#
CONTROLLERLOGFILE="/var/tmp/${BRIDGE}-controller.log"
UWFALOGFILE="/var/tmp/wfagent.${UTENANTWFANAME}.${UTENANTWFAID}.log"
SWFALOGFILE="/var/tmp/wfagent.${STENANTWFANAME}.${STENANTWFAID}.log"
CONTROLLERLOGFILE_SHORT="${BRIDGE}-controller.log"
CONTROLLERLOGFILE="/var/tmp/${CONTROLLERLOGFILE_SHORT}"
UWFALOGFILE_SHORT="wfagent.${UTENANTWFANAME}.${UTENANTWFAID}.log"
UWFALOGFILE="/var/tmp/${UWFALOGFILE_SHORT}"
SWFALOGFILE_SHORT="wfagent.${STENANTWFANAME}.${STENANTWFAID}.log"
SWFALOGFILE="/var/tmp/${SWFALOGFILE_SHORT}"
STATUS_UWFA=1
STATUS_SWFA=2
while [ 1 -eq 1 ]; do
$SSH $NETWORKMANAGER \
grep "'^DONE$'" $UWFALOGFILE
if [ $? -eq 0 ]; then
if [ ! $STATUS_UWFA -eq 0 ]; then
$SSH $NETWORKMANAGER \
grep -E '\(^DONE\$\)\|\(^Traceback\)' $UWFALOGFILE
STATUS_UWFA=$?
fi
if [ ! $STATUS_SWFA -eq 0 ]; then
$SSH $NETWORKMANAGER \
grep -E "'(^Finished setting up Hadoop)|(^Traceback)'" $SWFALOGFILE
STATUS_SWFA=$?
fi
if [ $STATUS_UWFA -eq 0 -a $STATUS_SWFA -eq 0 ]; then
break
fi
sleep 10
done
echo "*** Test has completed, fetching logs ..."
......@@ -221,6 +235,56 @@ $PSSH $PNHOSTS \
-e $TESTDIR/err.flowtables-post-hadoop \
ovs-ofctl --protocol=OpenFlow13 dump-flows $BRIDGE
echo "*** Done ; results in $TESTDIR"
#
# Process the results a bit.
#
UERR=0
UDONE=0
SERR=0
grep -E '^Traceback' $TESTDIR/${UWFALOGFILE_SHORT}
UERR=$?
if [ $UERR -eq 0 ]; then
UMSG="uWfaErr=Exception!"
else
UMSG=""
fi
grep -E '^DONE$' $TESTDIR/${UWFALOGFILE_SHORT}
UDONE=$?
if [ $UDONE -eq 0 ]; then
DONEMSG="uWfaDone=Nope!"
else
DONEMSG=""
fi
grep -E '^Traceback' $TESTDIR/${SWFALOGFILE_SHORT}
SERR=$?
if [ $SERR -eq 0 ]; then
SMSG="sWfaErr=Exception!"
else
SMSG=""
fi
if [ $UERR -ne 0 -a $UDONE -eq 0 -a $SERR -ne 0 ]; then
touch $TESTDIR/SUCCESS
if [ "x$TESTEMAIL" != "x" -a "x$TESTMAILSUCCESS" != "x" -a $TESTMAILSUCCESS -eq 1 ]; then
echo "Success ${TESTNAME}" \
| mailx -s "Success ${TESTNAME}" \
-a $TESTDIR/${UWFALOGFILE_SHORT} -a $TESTDIR/${SWFALOGFILE_SHORT} \
"$TESTEMAIL"
fi
else
touch $TESTDIR/FAILURE
if [ -n "$TESTMAILFAILURE" -a $TESTMAILFAILURE -eq 1 ]; then
# Controller logfiles get large; just send last 500 lines in attachment
TMPF=`mktemp`
tail -500 $TESTDIR/${CONTROLLERLOGFILE_SHORT} > $TMPF
echo "ERROR ${TESTNAME} $DONEMSG $UMSG $SMSG (UERR=$UERR UDONE=$UDONE SERR=$SERR)" \
| mailx -s "ERROR ${TESTNAME} $DONEMSG $UMSG $SMSG" \
-a $TESTDIR/${UWFALOGFILE_SHORT} -a $TESTDIR/${SWFALOGFILE_SHORT} \
-a $TMPF "$TESTEMAIL"
rm -f $TMPF
fi
fi
echo "*** Done ($TESTNAME); results in $TESTDIR"
exit 0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment