audit.in 5.12 KB
Newer Older
1 2 3
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
5 6 7
# All rights reserved.
#
use English;
8
use Getopt::Std;
9
use POSIX qw(strftime);
10 11 12

#
# Audit the DB, looking for things we want to catch. Run once a day from cron.
13 14 15 16 17 18
#
sub usage() {
    print STDOUT "Usage: audit [-d]\n".
	"Use the -d option to see debugging output instead of emailing it.\n";
    exit(-1);
}
19
sub fatal($);
20 21
my $optlist = "d";
my $debug   = 0;
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50

#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1; 

# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libdb;
use libtestbed;

#
# Only real root can call this.
# 
if ($UID != 0) {
    print STDERR "You must be root to run this script!\n";
    exit(-1);
}

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV) {
    usage();
}
if (defined($options{"d"})) {
    $debug++;
}

66 67 68 69 70 71 72 73 74
#
# Form a temp name.
#
my $logname = TBMakeLogname("audit");
my $query_result;

#
# Reopen both stdout and stderr so that we can record all the output for
# later mailing.
75 76 77 78 79
#
if (! $debug) {
    open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
    open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");
}
80 81 82 83 84 85

#
# Look for experiments running longer than 1 day, using real nodes.
#
if (! ($query_result = 
       DBQueryWarn("select e.pid,e.eid,e.expt_head_uid,expt_swapped, ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
86 87 88 89
		   "       count(r.node_id) as ncount, ".
		   "       UNIX_TIMESTAMP(now()) - UNIX_TIMESTAMP(expt_swapped)".
		   "         as swapsec ".
		   "  from experiments as e ".
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
		   "left join reserved as r on e.pid=r.pid and e.eid=r.eid ".
		   "left join nodes as n on n.node_id=r.node_id ".
		   "left join node_types as nt on nt.type=n.type ".
		   "where e.state='active' and ".
		   "      e.expt_swapped < date_sub(now(), interval 1 day) and ".
		   "      nt.isvirtnode=0 and nt.isremotenode=0 ".
		   "group by e.pid,e.eid ".
		   "having ncount>0 ".
		   "order by expt_swapped desc"))) {
    fatal("Error accessing the database.");
}

if ($query_result->numrows) {
    print "\n";
    print "Experiments swapped in longer than 1 day.\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
105 106 107 108
    printf("%-12s %-8s %-12s %-12s %-22s %s\n",
           "PID", "Hours", "EID", "Creator", "Swapped", "PCs");
    print "------------ -------- ------------ ------------ ".
	  "---------------------- ----\n";
109
    
Leigh B. Stoller's avatar
Leigh B. Stoller committed
110 111
    while (my ($pid,$eid,$creator,$swapped,$count,$seconds) =
	   $query_result->fetchrow()) {
112 113
	next
	    if ($count == 0);
Leigh B. Stoller's avatar
Leigh B. Stoller committed
114
	my $hours = int($seconds / (60 * 60));
115

Leigh B. Stoller's avatar
Leigh B. Stoller committed
116 117
	printf("%-12s %-8s %-12s %-12s %-22s %s\n",
	       $pid, $hours, $eid, $creator, $swapped, $count);
118 119 120
    }
}

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
#
# Once a week (Sunday) look for anyone that has used the wireless nodes.
#
my $today = POSIX::strftime("%u", localtime());
if ($today == 7) {
    my $query_result =
	DBQueryWarn("select e.exptidx,pid,eid,swapin_last,wirelesslans ".
		    "   from experiment_resources as r ".
		    "left join experiment_stats as e on e.exptidx=r.exptidx ".
		    "where wirelesslans!=0 and swapin_last is not null and ".
		    "UNIX_TIMESTAMP(swapin_last) > ".
		    "  UNIX_TIMESTAMP(now()) - (7 * 60 * 60 * 24) ".
		    "order by e.exptidx");
    fatal("Could not get wireless experiment stats")
	if (!$query_result);

    if ($query_result->numrows) {
	print "\n";
	print "Experiments that have used wireless nodes in the last week\n";
	printf("%-10s %-12s %-14s %-22s %-5s\n",
	       "ExptIDX", "PID", "EID", "Swapin Last", "Wlans");
	print "---------- ------------ -------------- ".
	    "---------------------- -----\n";
    
	while (my ($exptidx,$pid,$eid,$swapin_last,$wlans) =
	       $query_result->fetchrow()) {

	    printf("%-10s %-12s %-14s %-22s %-5s\n",
		   $exptidx, $pid, $eid, $swapin_last, $wlans);
	}
    }
}

154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
#
# Look for real nodes in hwdown.
#
$DPID = NODEDEAD_PID();
$DEID = NODEDEAD_EID();

if (! ($query_result = 
       DBQueryWarn("select r.node_id,rsrv_time from reserved as r ".
		   "left join nodes as n on n.node_id=r.node_id ".
		   "left join node_types as nt on nt.type=n.type ".
		   "where r.pid='$DPID' and r.eid='$DEID' and ".
		   "      nt.isremotenode=0 ".
		   "order by rsrv_time desc"))) {
    fatal("Error accessing the database.");
}

if ($query_result->numrows) {
    print "\n";
    print "----------------------------------------------------------------\n";
    print "\n";
174
    print "Local nodes stuck in $DPID/$DEID:\n";
175 176 177 178 179 180 181
    
    while (my ($node_id) = $query_result->fetchrow()) {
	print "$node_id ";
    }
    print "\n";
}

182 183 184
#
# Send email if anything was reported.
#
185
if (!$debug && -s $logname) {
186 187 188 189
    SENDMAIL($TBOPS, "Testbed Audit Results", "Testbed Audit Results",
	     $TBOPS, undef, ($logname));
}

190 191
unlink("$logname")
    if (-e $logname);
192 193 194 195 196 197 198
exit 0;

sub fatal($) {
    my ($msg) = @_;

    print STDERR "$msg\n";
    SENDMAIL($TBOPS, "Testbed Audit Failed", $msg, undef, undef, ($logname));
199 200
    unlink("$logname")
	if (-e $logname);
201 202
    exit(1);
}