Commit 94dfc872 authored by Timothy Stack's avatar Timothy Stack

Next phase of nfstrace support, transfer the set of files accessed by

the experiment from ops' db to boss'.  File names are stored in the
accessed_files table and are associated with the experiment via the
fs_resources table.
parent d2c02774
......@@ -33,6 +33,10 @@ if os.path.exists(MAINT_PATH):
VERBOSITY = 1
STYLE_MACHINE = 0
STYLE_HUMAN = 1
OUTPUT_STYLE = STYLE_HUMAN
# Tables that should be cleaned when a node transitions from one experiment
# to another.
GC_TABLES = [
......@@ -62,17 +66,22 @@ GC_TABLES = [
def human_readable(n, suffix='B', places=2):
'''Return a human friendly approximation of n, using SI prefixes'''
prefixes = [' ','K','M','G','T']
base, step, limit = 10, 3, 100
if n == 0:
magnitude = 0 #cannot take log(0)
if OUTPUT_STYLE == STYLE_HUMAN:
prefixes = [' ','K','M','G','T']
base, step, limit = 10, 3, 100
if n == 0:
magnitude = 0 #cannot take log(0)
pass
else:
magnitude = math.log(n, base)
order = int(round(magnitude)) // step
return '%.1f %s%s' % (float(n)/base**(order*step), \
prefixes[order], suffix)
pass
else:
magnitude = math.log(n, base)
order = int(round(magnitude)) // step
return '%.1f %s%s' % (float(n)/base**(order*step), \
prefixes[order], suffix)
return str(n)
def usage():
print "Usage: nfstrace.proxy [-h] <action> [...]"
......@@ -366,7 +375,7 @@ def resolve_fh(fh, depth=0):
return retval
def print_files(type, seen, links, used_links):
def process_files(type, seen, links, used_links):
global cur
retval = 0
......@@ -424,7 +433,7 @@ def do_get(args):
mount_map = []
try:
opts, args = getopt.getopt(args, "m:", [ "map=" ])
opts, args = getopt.getopt(args, "m:", [ "map=", ])
for opt, val in opts:
if opt in ("-m", "--map="):
l = val.split(':')
......@@ -494,7 +503,7 @@ def do_get(args):
"LEFT JOIN file_checkpoint as fc on (fa.fh=fc.fh) "
"GROUP BY fa.fh", (pid, eid))
total_size += print_files("r", seen, links, used_links)
total_size += process_files("r", seen, links, used_links)
cur.execute("SELECT fw.fh,fc.size,"
" IFNULL(MAX(fw.last_access)>MAX(fd.last_remove),1) "
......@@ -506,12 +515,15 @@ def do_get(args):
"LEFT JOIN file_checkpoint as fc on (fw.fh=fc.fh) "
"GROUP BY fw.fh", (pid, eid))
total_size += print_files("w", seen, links, used_links)
total_size += process_files("w", seen, links, used_links)
keys = seen.keys()
keys.sort()
for key in keys:
value = seen[key]
if value[0] == "r":
value[0] = "r "
pass
print "%2s %10s %s" % (value[0], value[1], value[2])
pass
......@@ -523,8 +535,10 @@ def do_get(args):
pass
pass
print ("-" * 79)
print "%10s Total" % (human_readable(total_size),)
if OUTPUT_STYLE == STYLE_HUMAN:
print ("-" * 79)
print "%10s Total" % (human_readable(total_size),)
pass
if missing > 0:
sys.stderr.write(`missing` + " unknown file(s) accessed.\n")
......@@ -653,8 +667,8 @@ ACTIONS = {
try:
opts, req_args = getopt.getopt(sys.argv[1:],
"h",
[ "help", ])
"hm",
[ "help", "machine" ])
if len(req_args) > 0:
action = req_args[0].lower()
......@@ -670,6 +684,9 @@ try:
pass
sys.exit()
pass
elif opt in ("-m", "--machine"):
OUTPUT_STYLE = STYLE_MACHINE
pass
pass
if len(req_args) < 1:
......
......@@ -35,7 +35,7 @@ my @row;
# Handle command-line arguments
#
sub usage() {
warn "Usage: $0 <gc|get|stats> pid eid\n";
warn "Usage: $0 <gc|get|transfer|stats> pid eid\n";
return 1;
}
......@@ -45,7 +45,7 @@ if (!$cmd) {
}
# Check the operation type
if ($cmd !~ /^(gc|get|stats)$/) {
if ($cmd !~ /^(gc|get|transfer|stats)$/) {
exit usage();
} else {
# Untaint operation
......@@ -114,9 +114,43 @@ use libtestbed;
$UID = 0;
if ($cmd ne "stats") {
my $expstate;
if (! ($expstate = ExpState($pid, $eid))) {
die("*** $0:\n".
" No such experiment $pid/$eid!\n");
}
if ($expstate ne EXPTSTATE_ACTIVE &&
$expstate ne EXPTSTATE_ACTIVATING &&
$expstate ne EXPTSTATE_MODIFY_RESWAP &&
$expstate ne EXPTSTATE_SWAPPING) {
die("*** $0:\n".
" Experiment $pid/$eid must be active (or swapping)! $expstate\n");
}
}
my $exptidx;
my $rsrcidx;
my $query_result;
my $cmdline = "$SSH $PROG $cmd";
if ($cmd eq "get") {
if ($cmd eq "get" || $cmd eq "transfer") {
if ($cmd eq "transfer") {
$cmdline = "$SSH $PROG -m get";
$query_result =
DBQueryFatal("SELECT e.idx,s.rsrcidx FROM experiments as e ".
"left join experiment_stats as s on e.idx=s.exptidx ".
"where e.pid='$pid' and e.eid='$eid'");
if ($query_result->num_rows() != 1) {
die("*** $0:\n".
" Experiment $pid/$eid has no stats!\n");
}
($exptidx, $rsrcidx) = $query_result->fetchrow_array();
}
$cmdline .= " -m $FSDIR_GROUPS:/groups";
$cmdline .= " -m $FSDIR_PROJ:/proj";
$cmdline .= " -m $FSDIR_USERS:/users";
......@@ -157,8 +191,59 @@ if ($cmd eq "gc" && defined($pid)) {
close(NT_WRITER);
sub InsertFile($)
{
my ($fn) = @_;
my $retval = -1;
$fn = DBQuoteSpecial($fn);
$query = "INSERT IGNORE INTO accessed_files (fn) VALUES ($fn)";
$query_result = DBQueryWarn($query);
if (!$query_result) {
print STDERR
"*** WARNING $0:\n".
" Failed to insert a new file access record for ".
"$pid/$eid\n";
}
$query_result =
DBQueryFatal("SELECT idx FROM accessed_files WHERE fn=$fn");
if ($query_result->num_rows() != 1) {
die("*** $0:\n".
" Missing file $fn!\n");
}
($retval) = $query_result->fetchrow_array();
return $retval;
}
while (<NT_READER>) {
print "$_";
if ($cmd eq "transfer") {
if ($_ =~ /^\s?(r|w|rw)\s+(\d+)\s+(.+)$/) {
my $fileidx = InsertFile($3);
$query = "REPLACE INTO fs_resources ".
"(rsrcidx, fileidx, exptidx, type, size) VALUES (".
"$rsrcidx, $fileidx, $exptidx, '$1', $2)";
DBQueryFatal($query);
}
elsif ($_ =~ /^\s+link\s+(.+)$/) {
my $fileidx = InsertFile($1);
$query = "REPLACE INTO fs_resources ".
"(rsrcidx, fileidx, exptidx, type, size) VALUES (".
"$rsrcidx, $fileidx, $exptidx, 'l', 0)";
DBQueryFatal($query);
}
else {
print "No match: $_";
}
}
else {
print "$_";
}
}
close(NT_READER);
......
......@@ -406,17 +406,7 @@ sub doSwapout($) {
if ($NFSTRACESUPPORT) {
print "Getting files accessed via NFS.\n";
TBDebugTimeStamp("nfstrace started");
if (lc $THISHOMEBASE eq "emulab.net") {
if (defined($umail)) {
SENDMAIL("$TBLOGS",
"Files accessed by $pid/$eid via NFS",
`nfstrace get $pid $eid`,
"$uname <$umail>");
}
}
else {
system("nfstrace get $pid $eid");
}
system("nfstrace transfer $pid $eid");
TBDebugTimeStamp("nfstrace finished");
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment