Commit b0cfd986 authored by Leigh Stoller's avatar Leigh Stoller

Add long term storage of log files in /usr/testbed/logs/logfiles.

New Store() method will gzip and move the file from current location
to long term storage, and update the DB row to reflect the change.
Change spewlogfile to handle these compressed files. Also added a
"public" flag that says a log file can be requested by an anonymous
user that knows the logid (md5 of bits). Change spewlogfile to allow
anonymous requests.

Add logfile_metadata to hold key,value pairs associated with a
logfile, eventually for search but initially to include when returning
the contents of a logfile via spewlogfile.
parent 5921c861
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2010 University of Utah and the Flux Group.
# Copyright (c) 2007-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -31,14 +31,16 @@ use vars qw(@ISA @EXPORT);
@EXPORT = qw ( );
# Must come after package declaration!
use libdb;
use emdb;
use emutil;
use libtestbed;
use English;
use Data::Dumper;
# Configure variables
my $TB = "@prefix@";
my $TBAUDIT = "@TBAUDITEMAIL@";
my $TBOPS = "@TBOPSEMAIL@";
my $GZIP = "/usr/bin/gzip";
#
# Lookup by uuid. For now, just knowing the uuid says you can read the file.
......@@ -64,20 +66,53 @@ sub Lookup($$)
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'LOGFILE'} = $query_result->fetchrow_hashref();
bless($self, $class);
my $self = (ref($class) ? $class : {});
$self->{'LOGFILE'} = $query_result->fetchrow_hashref();
$self->{'METADATA'} = undef;
# Ordered list.
$self->{'METADATALIST'} = undef;
my $logidx = $self->{'LOGFILE'}->{'logidx'};
my $metadata_result =
DBQueryWarn("select metakey,metaval from logfile_metadata ".
"where logidx='$logidx' order by idx");
if ($metadata_result && $metadata_result->numrows) {
$self->{'METADATA'} = {};
$self->{'METADATALIST'} = [];
my $idx = 0;
while (my ($key,$val) = $metadata_result->fetchrow_array()) {
$self->{'METADATA'}->{$key} = $val;
$self->{"METADATALIST"}->[$idx] = [$key, $val];
$idx++;
}
}
bless($self, $class) if (!ref($class));
return $self;
}
# accessors
sub field($$) { return ((! ref($_[0])) ? -1 : $_[0]->{'LOGFILE'}->{$_[1]}); }
sub logid($) { return field($_[0], "logid"); }
sub logidx($) { return field($_[0], "logidx"); }
sub filename($) { return field($_[0], "filename"); }
sub isopen($) { return field($_[0], "isopen"); }
sub gid_idx($) { return field($_[0], "gid_idx"); }
sub date_created($) { return field($_[0], "date_created"); }
sub public($) { return field($_[0], "public"); }
sub compressed($) { return field($_[0], "compressed"); }
sub stored($) { return field($_[0], "stored"); }
sub Metadata($) { return $_[0]->{'METADATA'}; }
sub MetadataList($) { return $_[0]->{'METADATALIST'}; }
# Break circular reference someplace to avoid exit errors.
sub DESTROY {
my $self = shift;
$self->{"LOGFILE"} = undef;
$self->{"METADATA"} = undef;
$self->{"METADATALIST"} = undef;
}
#
# Refresh a class instance by reloading from the DB.
......@@ -86,20 +121,7 @@ sub Refresh($)
{
my ($self) = @_;
return -1
if (! ref($self));
my $logid = $self->logid();
my $query_result =
DBQueryWarn("select * from logfiles where logid='$logid'");
return -1
if (!$query_result || !$query_result->numrows);
$self->{'LOGFILE'} = $query_result->fetchrow_hashref();
return 0;
return $self->Lookup($self->logid());
}
#
......@@ -108,19 +130,30 @@ sub Refresh($)
#
sub Create($$;$)
{
my ($class, $gid_idx, $filename) = @_;
my ($class, $group, $filename) = @_;
my $gid_idx = (ref($group) ? $group->gid_idx() : $group);
return undef
if (ref($class));
$filename = TBMakeLogname("logfile")
if (!defined($filename));
# Plain secret key, which is used to reference the file.
my $logid = TBGenSecretKey();
my $logid = TBGenSecretKey();
my $logidx = TBGetUniqueIndex('next_logfile', 1);
# This creates the file, so the owner/group are current user/group.
if (!defined($filename)) {
$filename = TBMakeLogname("logfile");
# So tbops people can read the files ...
if (!chmod(0664, $filename)) {
print STDERR "Could not chmod $filename to 0644: $!\n";
unlink($filename);
return undef;
}
}
if (!DBQueryWarn("insert into logfiles set ".
" logid='$logid', ".
" logid='$logid', logidx='$logidx', ".
" isopen=0, ".
" filename='$filename', ".
" gid_idx='$gid_idx', ".
......@@ -144,10 +177,14 @@ sub Delete($;$)
if (!defined($delete));
my $logid = $self->logid();
my $logidx = $self->logidx();
my $filename = $self->filename();
if ($delete) {
unlink($filename);
}
return -1
if (!DBQueryWarn("delete from logfile_metadata ".
"where logidx='$logidx'"));
return -1
if (!DBQueryWarn("delete from logfiles where logid='$logid'"));
......@@ -167,6 +204,10 @@ sub AccessCheck($$)
return 0
if (! ref($self));
# Public? Anyone can look at it.
return 1
if ($self->public());
# Admins do whatever they want.
return 1
if ($user->IsAdmin());
......@@ -216,5 +257,111 @@ sub Close($)
return $self->Refresh();
}
#
# Is file empty?
#
sub Empty($)
{
my ($self) = @_;
my $filename = $self->filename();
return 1
if (! -e $filename || ! -s $filename);
return 0;
}
#
# Mark public or private.
#
sub SetPublic($$)
{
my ($self, $public) = @_;
$public = ($public ? 1 : 0);
return -1
if (!ref($self));
my $logid = $self->logid();
DBQueryWarn("update logfiles set public='$public' where logid='$logid'")
or return -1;
return $self->Refresh();
}
#
# Move a log file from its current location to long term storage,
# compressing on the way. Then reset its location in the DB and
# mark it as compressed.
#
sub Store($)
{
my ($self) = @_;
return -1
if (!ref($self));
my $logid = $self->logid();
my $filename = $self->filename();
my $newname = "/usr/testbed/log/logfiles";
# Might as well.
$self->Close();
#
# Use first two letters of the id for the subdir.
#
my $dir = lc(substr($logid, 0, 2));
$newname = "$newname/$dir/${logid}.gz";
if (-e $newname) {
print STDERR "Logfile::Store ($logid) - $newname exists\n";
return -1;
}
my $output = emutil::ExecQuiet("$GZIP -c $filename > $newname");
if ($?) {
SENDMAIL($TBOPS, "Logfile Store Error",
"Failed to store $filename to $newname\n\n".
"-----------------\n".
"$output\n", $TBOPS);
return -1
}
unlink($filename);
DBQueryWarn("update logfiles set ".
" stored=1,compressed=1,filename='$newname' ".
"where logid='$logid'")
or return -1;
return 0;
}
#
# Set the metadata (stuff to search on) for a log. Argument is an
# array of header,value pairs
#
sub SetMetadata($$$)
{
my ($self, $argref, $purge) = @_;
return -1
if (!ref($self));
my $logidx = $self->logidx();
# Purge old data
DBQueryWarn("delete from logfile_metadata where logidx='$logidx'")
if ($purge);
foreach my $ref (@{$argref}) {
my ($key,$val) = @{$ref};
$key = DBQuoteSpecial($key);
$val = DBQuoteSpecial($val);
return -1
if (! DBQueryWarn("replace into logfile_metadata set ".
" logidx='$logidx',metakey=$key,metaval=$val"));
}
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -2193,11 +2193,31 @@ CREATE TABLE `log` (
DROP TABLE IF EXISTS `logfiles`;
CREATE TABLE `logfiles` (
`logid` varchar(40) NOT NULL default '',
`logidx` int(10) unsigned NOT NULL default '0',
`filename` tinytext,
`isopen` tinyint(4) NOT NULL default '0',
`gid_idx` mediumint(8) unsigned NOT NULL default '0',
`date_created` datetime default NULL,
PRIMARY KEY (`logid`)
`public` tinyint(1) NOT NULL default '0',
`compressed` tinyint(1) NOT NULL default '0',
`stored` tinyint(1) NOT NULL default '0',
PRIMARY KEY (`logid`),
KEY `logidx` (`logidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `logfile_metadata`
--
DROP TABLE IF EXISTS `logfile_metadata`;
CREATE TABLE `logfile_metadata` (
`logidx` int(10) unsigned NOT NULL default '0',
`idx` int(10) unsigned NOT NULL auto_increment,
`metakey` tinytext,
`metaval` tinytext,
PRIMARY KEY (`logidx`,`idx`),
UNIQUE KEY `logidxkey` (`logidx`,`metakey`(128)),
KEY `headervalue` (`metakey`(64),`metaval`(128))
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......
#
# Add a public and compressed flag to logfiles.
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if (!DBSlotExists("logfiles", "public")) {
DBQueryFatal("alter table logfiles add ".
" `public` tinyint(1) NOT NULL default '0'");
}
if (!DBSlotExists("logfiles", "compressed")) {
DBQueryFatal("alter table logfiles add ".
" `compressed` tinyint(1) NOT NULL default '0'");
}
if (!DBSlotExists("logfiles", "stored")) {
DBQueryFatal("alter table logfiles add ".
" `stored` tinyint(1) NOT NULL default '0'");
}
if (!DBSlotExists("logfiles", "idx")) {
DBQueryFatal("alter table logfiles add ".
" `logidx` int(10) unsigned NOT NULL default '0' ".
" after logid");
}
if (!DBKeyExists("logfiles", "logidx")) {
DBQueryFatal("alter table logfiles add ".
" KEY `logidx` (`logidx`)");
}
if (!DBTableExists("logfile_metadata")) {
DBQueryFatal("CREATE TABLE `logfile_metadata` ( ".
" `logidx` int(10) unsigned NOT NULL default '0', ".
" `idx` int(10) unsigned NOT NULL auto_increment, ".
" `metakey` tinytext, ".
" `metaval` tinytext, ".
" PRIMARY KEY (`logidx`,`idx`), ".
" UNIQUE KEY `logidxkey` (`logidx`,`metakey`(128)), ".
" KEY `headervalue` (`metakey`(64),`metaval`(128)) ".
") ENGINE=MyISAM DEFAULT CHARSET=latin1");
}
#
# Need to assign indicies to all of the existing records.
#
my $query_result =
DBQueryFatal("select logid from logfiles ".
"where logidx=0 order by date_created asc");
while (my ($logid) = $query_result->fetchrow_array()) {
my $idx = TBGetUniqueIndex('next_logfile', 1);
DBQueryFatal("update logfiles set logidx='$idx' ".
"where logid='$logid'");
}
return 0;
}
1;
# Local Variables:
# mode:perl
# End:
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2010 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -40,8 +40,9 @@ sub usage()
"Spew a logfile to stdout, as for the web interface\n");
exit(-1);
}
my $optlist = "we:t:i:";
my $optlist = "wi:a";
my $fromweb = 0;
my $anon = 0;
#
# Configure variables
......@@ -49,6 +50,7 @@ my $fromweb = 0;
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
my $GUNZIP = "/usr/bin/gunzip";
my $logname;
my $isopen;
......@@ -81,6 +83,9 @@ if (! getopts($optlist, \%options)) {
if (defined($options{"w"})) {
$fromweb = 1;
}
if (defined($options{"a"})) {
$anon = 1;
}
if (defined($options{"i"})) {
$logfile = Logfile->Lookup($options{"i"});
if (! $logfile) {
......@@ -100,19 +105,26 @@ if ($UID == 0) {
" Please do not run this as root! Its already setuid!\n");
}
#
# Allow for anonymous users to request a public file.
# Verify user and get his DB uid and other info for later.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
die("*** $0:\n".
" You ($UID) do not exist!");
if (!$anon) {
my $this_user = User->ThisUser();
if (! defined($this_user)) {
die("*** $0:\n".
" You ($UID) do not exist!");
}
#
# Verify that this person is allowed to do this.
#
if (!$logfile->AccessCheck($this_user)) {
die("*** $0:\n".
" You do not have permission to view logfile!\n");
}
}
#
# Verify that this person is allowed to do this.
#
if (!$logfile->AccessCheck($this_user)) {
elsif (!$logfile->public()) {
die("*** $0:\n".
" You do not have permission to view logfile!\n");
}
......@@ -123,6 +135,28 @@ use Fcntl;
use IO::Handle;
STDOUT->autoflush(1);
#
# if the file is closed do it the quick way. No locking, so this could
# mess up sometimes.
#
if (!$isopen) {
if ($logfile->MetadataList()) {
foreach my $ref (@{ $logfile->MetadataList() }) {
my ($key,$val) = @{$ref};
printf("%-15s %s\n", "$key:", $val);
}
print "\n";
print "---------------------\n";
}
if ($logfile->compressed()) {
system("$GUNZIP -c $logname");
}
else {
system("/bin/cat $logname");
}
exit(0);
}
#
# Open the file up while still root. We verified permission above, and the
# added check using the filesystems permissions if more of a pain then it
......@@ -150,6 +184,15 @@ if ($fromweb && $isopen && $size < 1024) {
print "\n";
}
if ($logfile->MetadataList()) {
foreach my $ref (@{ $logfile->MetadataList() }) {
my ($key,$val) = @{$ref};
printf("%-15s %s\n", "$key:", $val);
}
print "\n";
print "---------------------\n";
}
#
# Loop reading the file in nonblocking mode. Sleep between loops, and
# check for a change in status.
......
<?php
#
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -24,10 +24,11 @@
include("defs.php3");
#
# Only known and logged in users.
# Get current user, but allow for anon access.
#
$this_user = CheckLoginOrDie();
$uid = $this_user->uid();
$this_user = CheckLogin($check_status);
$uid = ($this_user ? $this_user->uid() : "nobody");
$anonopt = ($this_user ? "" : "-a");
#
# Verify page arguments.
......@@ -61,8 +62,9 @@ function SPEWCLEANUP()
ignore_user_abort(1);
register_shutdown_function("SPEWCLEANUP");
if ($fp = popen("$TBSUEXEC_PATH $uid nobody ".
"spewlogfile -w -i " . escapeshellarg($logfileid), "r")) {
if ($fp =
popen("$TBSUEXEC_PATH $uid nobody ".
"spewlogfile $anonopt -w -i " . escapeshellarg($logfileid), "r")) {
header("Content-Type: text/plain");
header("Expires: Mon, 26 Jul 1997 05:00:00 GMT");
header("Cache-Control: no-cache, must-revalidate");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment