Commit c1cff09b authored by Leigh Stoller's avatar Leigh Stoller

This started out as a simple change to turn the datastore into a CVS

sandbox, and that I did. It falls back to the older archive when
the template is older then CVS repos.

But along the way I got annoyed with the fact that template instantiation
does not provide a logfile to the web interface. The reason is that
the current logfile stuff is very experiment centric; there has to be an
experiment and an attached logfile. An instance does not have an experiment
until really late in the game so the code was just not bothering.

Anyway, I've started to generalize the logfile stuff with a new table
and the approach that a logfile is named by a random key, and if you
know the key you can look at the logfile in the web (since without an
experiment it is hard to do permission checks unless we make logfiles
uid/gid owned, and I did not want to do that.
parent 760e3a13
...@@ -25,7 +25,7 @@ WEB_BIN_SCRIPTS = webnfree ...@@ -25,7 +25,7 @@ WEB_BIN_SCRIPTS = webnfree
LIBEXEC_SCRIPTS = $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS) xmlconvert LIBEXEC_SCRIPTS = $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS) xmlconvert
LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \ LIB_SCRIPTS = libdb.pm Node.pm libdb.py libadminctrl.pm Experiment.pm \
NodeType.pm Interface.pm User.pm Group.pm Project.pm \ NodeType.pm Interface.pm User.pm Group.pm Project.pm \
Image.pm OSinfo.pm Archive.pm Image.pm OSinfo.pm Archive.pm Logfile.pm
# Stuff installed on plastic. # Stuff installed on plastic.
USERSBINS = genelists.proxy dumperrorlog.proxy USERSBINS = genelists.proxy dumperrorlog.proxy
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2007 University of Utah and the Flux Group.
# All rights reserved.
#
package Logfile;
use strict;
use Exporter;
use vars qw(@ISA @EXPORT);
@ISA = "Exporter";
@EXPORT = qw ( );
# Must come after package declaration!
use lib '@prefix@/lib';
use libdb;
use libtestbed;
use English;
use Data::Dumper;
# Configure variables
my $TB = "@prefix@";
my $TBAUDIT = "@TBAUDITEMAIL@";
#
# Lookup by uuid. For now, just knowing the uuid says you can read the file.
#
sub Lookup($$)
{
my ($class, $logid) = @_;
my $logfile;
#
# Argument must be alphanumeric.
#
if ($logid =~ /^([\w]*)$/) {
$logid = $1;
}
else {
return undef;
}
my $query_result =
DBQueryWarn("select * from logfiles where logid='$logid'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'LOGFILE'} = $query_result->fetchrow_hashref();
bless($self, $class);
return $self;
}
# accessors
sub field($$) { return ((! ref($_[0])) ? -1 : $_[0]->{'LOGFILE'}->{$_[1]}); }
sub logid($) { return field($_[0], "logid"); }
sub filename($) { return field($_[0], "filename"); }
sub isopen($) { return field($_[0], "isopen"); }
sub date_created($) { return field($_[0], "date_created"); }
#
# Refresh a class instance by reloading from the DB.
#
sub Refresh($)
{
my ($self) = @_;
return -1
if (! ref($self));
my $logid = $self->logid();
my $query_result =
DBQueryWarn("select * from logfiles where logid='$logid'");
return -1
if (!$query_result || !$query_result->numrows);
$self->{'LOGFILE'} = $query_result->fetchrow_hashref();
return 0;
}
#
# Create a new logfile. We are given the optional filename, otherwise
# generate one.
#
sub Create($;$)
{
my ($class, $filename) = @_;
return undef
if (ref($class));
$filename = TBMakeLogname("logfile")
if (!defined($filename));
# Plain secret key, which is used to reference the file.
my $logid = TBGenSecretKey();
if (!DBQueryWarn("insert into logfiles set ".
" logid='$logid', ".
" isopen=0, ".
" filename='$filename', ".
" date_created=now()")) {
return undef;
}
return Logfile->Lookup($logid);
}
#
# Delete a logfile record. Optionally delete the logfile too.
#
sub Delete($;$)
{
my ($self, $delete) = @_;
return -1
if (!ref($self));
$delete = 0
if (!defined($delete));
my $logid = $self->logid();
my $filename = $self->filename();
if ($delete) {
unlink($filename);
}
return -1
if (!DBQueryWarn("delete from logfiles where logid='$logid'"));
return 0;
}
#
# Mark a file open so that the web interface knows to watch it.
#
sub Open($)
{
my ($self) = @_;
return -1
if (!ref($self));
my $logid = $self->logid();
DBQueryWarn("update logfiles set isopen=1 where logid='$logid'")
or return -1;
return $self->Refresh();
}
#
# Mark file closed, which is used to stop the web interface from spewing.
#
sub Close($)
{
my ($self) = @_;
return -1
if (!ref($self));
my $logid = $self->logid();
DBQueryWarn("update logfiles set isopen=0 where logid='$logid'")
or return -1;
return $self->Refresh();
}
# _Always_ make sure that this 1 is at the end of the file...
1;
...@@ -772,6 +772,7 @@ CREATE TABLE `experiment_template_instances` ( ...@@ -772,6 +772,7 @@ CREATE TABLE `experiment_template_instances` (
`eid` varchar(32) NOT NULL default '', `eid` varchar(32) NOT NULL default '',
`uid` varchar(8) NOT NULL default '', `uid` varchar(8) NOT NULL default '',
`uid_idx` mediumint(8) unsigned NOT NULL default '0', `uid_idx` mediumint(8) unsigned NOT NULL default '0',
`logfileid` varchar(40) default NULL,
`description` tinytext, `description` tinytext,
`start_time` datetime default NULL, `start_time` datetime default NULL,
`stop_time` datetime default NULL, `stop_time` datetime default NULL,
...@@ -1560,6 +1561,19 @@ CREATE TABLE `log` ( ...@@ -1560,6 +1561,19 @@ CREATE TABLE `log` (
KEY `stamp` (`stamp`) KEY `stamp` (`stamp`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1; ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `logfiles`
--
DROP TABLE IF EXISTS `logfiles`;
CREATE TABLE `logfiles` (
`logid` varchar(40) NOT NULL default '',
`filename` tinytext,
`isopen` tinyint(4) NOT NULL default '0',
`date_created` datetime default NULL,
PRIMARY KEY (`logid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-- --
-- Table structure for table `login` -- Table structure for table `login`
-- --
......
...@@ -4181,3 +4181,16 @@ last_net_act,last_cpu_act,last_ext_act); ...@@ -4181,3 +4181,16 @@ last_net_act,last_cpu_act,last_ext_act);
`created` datetime default NULL, `created` datetime default NULL,
PRIMARY KEY (`parent_guid`,`parent_vers`,`uid_idx`,`name`) PRIMARY KEY (`parent_guid`,`parent_vers`,`uid_idx`,`name`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1; ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
4.134: Start cleaning up logfiles.
CREATE TABLE `logfiles` (
`logid` varchar(40) NOT NULL default '',
`filename` tinytext,
`isopen` tinyint(4) NOT NULL default '0',
`date_created` datetime default NULL,
PRIMARY KEY (`logid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
alter table experiment_template_instances add \
`logfileid` varchar(40) default NULL after uid_idx;
...@@ -26,6 +26,7 @@ use Project; ...@@ -26,6 +26,7 @@ use Project;
use User; use User;
use Experiment; use Experiment;
use Group; use Group;
use Logfile;
use English; use English;
use HTML::Entities; use HTML::Entities;
use overload ('""' => 'Stringify'); use overload ('""' => 'Stringify');
...@@ -2135,6 +2136,7 @@ sub runtime($) { return field($_[0], 'runtime'); } ...@@ -2135,6 +2136,7 @@ sub runtime($) { return field($_[0], 'runtime'); }
sub locked($) { return field($_[0], 'locked'); } sub locked($) { return field($_[0], 'locked'); }
sub locker_pid($) { return field($_[0], 'locker_pid'); } sub locker_pid($) { return field($_[0], 'locker_pid'); }
sub description($) { return field($_[0], 'description'); } sub description($) { return field($_[0], 'description'); }
sub logfileid($) { return field($_[0], 'logfileid'); }
sub template($) { return ((!ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}); } sub template($) { return ((!ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}); }
# The path is the path of the experiment. # The path is the path of the experiment.
...@@ -2445,6 +2447,12 @@ sub Delete($) ...@@ -2445,6 +2447,12 @@ sub Delete($)
"where exptidx='$exptidx'") "where exptidx='$exptidx'")
or return -1; or return -1;
# No reason to keep the log entry around.
if ($self->logfileid()) {
my $logfile = Logfile->Lookup($self->logfileid());
$logfile->Delete();
}
# #
# Also delete the binding records for the instance. # Also delete the binding records for the instance.
# #
...@@ -3526,12 +3534,12 @@ sub CopyTemplateEvents($) ...@@ -3526,12 +3534,12 @@ sub CopyTemplateEvents($)
# #
# Create a log file for an instance, in the template directory. # Create a log file for an instance, in the template directory.
# #
sub CreateLogFile($$$) sub CreateLogFile($$)
{ {
my($self, $token, $ppath) = @_; my ($self, $token) = @_;
# Must be a real reference. # Must be a real reference.
return -1 return undef
if (! ref($self)); if (! ref($self));
my $idx = $self->idx(); my $idx = $self->idx();
...@@ -3539,17 +3547,28 @@ sub CreateLogFile($$$) ...@@ -3539,17 +3547,28 @@ sub CreateLogFile($$$)
my $logdir = "$path/logs"; my $logdir = "$path/logs";
my $logname = "$logdir/instance${idx}.${token}"; my $logname = "$logdir/instance${idx}.${token}";
return -1 return undef
if (-e $logname); if (-e $logname);
return -1 return undef
if (! -d $logdir && !mkdir($logdir, 0775)); if (! -d $logdir && !mkdir($logdir, 0775));
Template::mysystem("touch $logname") == 0 Template::mysystem("touch $logname") == 0
or return -1; or return undef;
$$ppath = $logname; my $logfile = Logfile->Create($logname);
return 0; if (!defined($logfile)) {
unlink($logname);
return undef;
}
if ($self->Update(0, {'logfileid' => $logfile->logid()})) {
$logfile->Delete();
unlink($logname)
if (!defined($logfile));
return undef;
}
return $logfile;
} }
sub WriteEnvVariables($) sub WriteEnvVariables($)
......
...@@ -21,10 +21,11 @@ sub usage() ...@@ -21,10 +21,11 @@ sub usage()
{ {
print("Usage: spewlogfile -e pid,eid\n". print("Usage: spewlogfile -e pid,eid\n".
" spewlogfile -t guid,vers\n". " spewlogfile -t guid,vers\n".
" spewlogfile -i logid\n".
"Spew the logfile for an experiment or template.\n"); "Spew the logfile for an experiment or template.\n");
exit(-1); exit(-1);
} }
my $optlist = "we:t:"; my $optlist = "we:t:i:";
my $fromweb = 0; my $fromweb = 0;
# #
...@@ -38,6 +39,7 @@ my $logname; ...@@ -38,6 +39,7 @@ my $logname;
my $isopen; my $isopen;
my $experiment; my $experiment;
my $template; my $template;
my $logfile;
# #
# Load the Testbed support stuff. # Load the Testbed support stuff.
...@@ -48,6 +50,7 @@ use libtestbed; ...@@ -48,6 +50,7 @@ use libtestbed;
use Experiment; use Experiment;
use Template; use Template;
use User; use User;
use Logfile;
# un-taint path # un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin'; $ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
...@@ -81,8 +84,15 @@ elsif (defined($options{"t"})) { ...@@ -81,8 +84,15 @@ elsif (defined($options{"t"})) {
" No such template in the Emulab Database.\n"); " No such template in the Emulab Database.\n");
} }
} }
elsif (defined($options{"i"})) {
$logfile = Logfile->Lookup($options{"i"});
if (! $logfile) {
die("*** $0:\n".
" No such logfile in the Emulab Database.\n");
}
}
usage() usage()
if (@ARGV || !($experiment || $template)); if (@ARGV || !($experiment || $template || $logfile));
# #
# This script is setuid, so please do not run it as root. Hard to track # This script is setuid, so please do not run it as root. Hard to track
...@@ -118,7 +128,7 @@ if ($experiment) { ...@@ -118,7 +128,7 @@ if ($experiment) {
" There is no logfile to view for $experiment!\n"); " There is no logfile to view for $experiment!\n");
} }
} }
else { elsif ($template) {
if (!$template->AccessCheck($this_user, TB_EXPT_READINFO)) { if (!$template->AccessCheck($this_user, TB_EXPT_READINFO)) {
die("*** $0:\n". die("*** $0:\n".
" You do not have permission to view log for $template!\n"); " You do not have permission to view log for $template!\n");
...@@ -131,6 +141,14 @@ else { ...@@ -131,6 +141,14 @@ else {
" There is no logfile to view for $template!\n"); " There is no logfile to view for $template!\n");
} }
} }
else {
#
# Presently, just knowing the ID grants you access.
#
$logname = $logfile->filename();
$isopen = $logfile->isopen();
}
use Fcntl; use Fcntl;
use IO::Handle; use IO::Handle;
...@@ -138,9 +156,11 @@ STDOUT->autoflush(1); ...@@ -138,9 +156,11 @@ STDOUT->autoflush(1);
# #
# If not an admin type, flip back to the UID now to enforce normal # If not an admin type, flip back to the UID now to enforce normal
# permissions. # permissions.
# #
if (!$this_user->IsAdmin()) { # XXX For a logfile ID, do not flip back either.
#
if (!$this_user->IsAdmin() || !$logfile) {
$EUID = $UID; $EUID = $UID;
} }
...@@ -150,8 +170,9 @@ sysopen(LOG, $logname, O_RDONLY | O_NONBLOCK) or ...@@ -150,8 +170,9 @@ sysopen(LOG, $logname, O_RDONLY | O_NONBLOCK) or
# #
# If an admin type, flip back to the UID now that the file is open. # If an admin type, flip back to the UID now that the file is open.
# XXX Ditto for logfile IDs.
# #
if ($this_user->IsAdmin()) { if ($this_user->IsAdmin() || $logfile) {
$EUID = $UID; $EUID = $UID;
} }
...@@ -186,11 +207,16 @@ while (1) { ...@@ -186,11 +207,16 @@ while (1) {
if ($experiment->GetLogFile(\$tmp, \$isopen) || if ($experiment->GetLogFile(\$tmp, \$isopen) ||
!$isopen || $tmp ne $logname); !$isopen || $tmp ne $logname);
} }
else { elsif ($template) {
last last
if ($template->GetLogFile(\$tmp, \$isopen) || if ($template->GetLogFile(\$tmp, \$isopen) ||
!$isopen || $tmp ne $logname); !$isopen || $tmp ne $logname);
} }
else {
last
if ($logfile->Refresh() != 0 || !$logfile->isopen());
}
sleep(2); sleep(2);
} }
close(LOG); close(LOG);
......
...@@ -81,7 +81,7 @@ my $EVhandle; ...@@ -81,7 +81,7 @@ my $EVhandle;
my $exptidx; my $exptidx;
my $template; my $template;
my $instance; my $instance;
my $logname; my $logfile;
my $template_tag; my $template_tag;
my @ExptStates = (); my @ExptStates = ();
# For the END block below. # For the END block below.
...@@ -101,6 +101,8 @@ my $swapin = "$TB/bin/template_swapin"; ...@@ -101,6 +101,8 @@ my $swapin = "$TB/bin/template_swapin";
my $endexp = "$TB/bin/endexp"; my $endexp = "$TB/bin/endexp";
my $dbcontrol = "$TB/sbin/opsdb_control"; my $dbcontrol = "$TB/sbin/opsdb_control";
my $archcontrol = "$TB/bin/archive_control"; my $archcontrol = "$TB/bin/archive_control";
my $CVSBIN = "/usr/bin/cvs";
my $RLOG = "/usr/bin/rlog";
# Protos # Protos
sub ParseArgs(); sub ParseArgs();
...@@ -121,8 +123,8 @@ use User; ...@@ -121,8 +123,8 @@ use User;
use event; use event;
use libaudit; use libaudit;
# Be careful not to exit on transient error # In libdb
$libdb::DBQUERY_MAXTRIES = 0; my $projroot = PROJROOT();
# #
# Turn off line buffering on output # Turn off line buffering on output
...@@ -354,11 +356,14 @@ $SIG{TERM} = \&sighandler; ...@@ -354,11 +356,14 @@ $SIG{TERM} = \&sighandler;
# Use the logonly option to audit so that we get a record mailed. # Use the logonly option to audit so that we get a record mailed.
# #
if (! ($foreground || $batchmode)) { if (! ($foreground || $batchmode)) {
if ($instance->CreateLogFile("swapin", \$logname) < 0) { $logfile = $instance->CreateLogFile("swapin");
if (!defined($logfile)) {
fatal(-1, "Could not create logfile!"); fatal(-1, "Could not create logfile!");
} }
# Mark it open, since it exists.
$logfile->Open();
if (my $childpid = AuditStart(LIBAUDIT_DAEMON, $logname, if (my $childpid = AuditStart(LIBAUDIT_DAEMON, $logfile->filename(),
LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) { LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) {
# #
# Parent exits normally, unless in waitmode. We have to set # Parent exits normally, unless in waitmode. We have to set
...@@ -367,31 +372,14 @@ if (! ($foreground || $batchmode)) { ...@@ -367,31 +372,14 @@ if (! ($foreground || $batchmode)) {
$justexit = 1; $justexit = 1;
if (!$waitmode) { if (!$waitmode) {
# my $idx = $instance->idx();
# Before we can actually exit, we need to wait. Totally ick;
# the logfile stuff needs work. # XXX The web interface depends on this line.
# print("Instance $pid/$eid ($idx) is now being instantiated.\n")
while (1) { if (! $quiet);
my ($tmp1, $tmp2);
last
if (TBExptGetLogFile($pid, $eid, \$tmp1, \$tmp2));
sleep(2);
}
if ($batchmode) {
print("Experiment $pid/$eid has entered the batch system.\n".
"You will be notified when it is fully instantiated.\n")
if (! $quiet);
}
else {
print("Experiment $pid/$eid is now being instantiated.\n".
"You will be notified via email when this is done.\n")
if (! $quiet);
}
exit(0); exit(0);
} }
print("Waiting for experiment $eid to fully instantiate.\n") print("Waiting for instance $pid/$eid to fully instantiate.\n")
if (! $quiet); if (! $quiet);
if (-t STDIN && !$quiet) { if (-t STDIN && !$quiet) {
...@@ -479,14 +467,6 @@ if ($STAMPS) { ...@@ -479,14 +467,6 @@ if ($STAMPS) {
$instance->Stamp("template_instantiate", "batchexp done"); $instance->Stamp("template_instantiate", "batchexp done");
} }
#
# Now we can do this ...
#
if (defined($logname) && ! ($foreground || $batchmode)) {
TBExptSetLogFile($pid, $eid, $logname);
TBExptOpenLogFile($pid, $eid);
}
# Grab the experiment record for below. # Grab the experiment record for below.
my $experiment = Experiment->Lookup($pid, $eid); my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) { if (!defined($experiment)) {
...@@ -546,21 +526,46 @@ if ($paramfile) { ...@@ -546,21 +526,46 @@ if ($paramfile) {
# This is essentially a copy. # This is essentially a copy.
# #
my $instance_path = $userdir; my $instance_path = $userdir;
my $datastore_tag = $template_tag;
# #
# But if this is a replay, then use the tag corresponding to the run being # Using the repository now ... the archive will eventually go away.
# replayed, so that we get the datastore that was in place when the run was, #
# well, originally run. my $cvstag = "T${guid}-${version}";
my $cvsdir = "$projroot/$pid/templates/$guid/cvsrepo";
#
# But if this is a replay, then use the tag corresponding to the
# run being replayed, so that we get the datastore that was in
# place when the run was, well, originally run.
# #
if (defined($replay_instance)) { if (defined($replay_instance)) {
$datastore_tag = $replay_run->start_tag(); $cvstag = "T${guid}-" . $replay_instance->vers();
}
my $revision = `$RLOG -h $cvsdir/setup/.template,v | grep '${cvstag}:'`;
if (! $?) {
print "Checking out a copy of the template datastore ($cvstag)\n";
System("cd $instance_path; $CVSBIN -d $cvsdir ".
" checkout -r '$cvstag' -d datastore setup/datastore")
== 0 or fatal(-1, "Could not checkout from $cvsdir");
} }
else {
my $datastore_tag = $template_tag;
print "Checking out a copy of the template datastore ($datastore_tag)\n"; #
$instance->CopyDataStore($datastore_tag, # But if this is a replay, then use the tag corresponding to the
"$instance_path", $replay_instance) == 0 # run being replayed, so that we get the datastore that was in
or fatal(-1, "Could not copy datastore to instance"); # place when the run was, well, originally run.
#
if (defined($replay_instance)) {
$datastore_tag = $replay_run->start_tag();
}
print "Checking out a copy of the template datastore ($datastore_tag)\n";
$instance->CopyDataStore($datastore_tag,
"$instance_path", $replay_instance) == 0
or fatal(-1, "Could not copy datastore to instance");
}
# Ditto for dynamic events. # Ditto for dynamic events.
$instance->CopyTemplateEvents() == 0 $instance->CopyTemplateEvents() == 0
...@@ -599,8 +604,8 @@ if ($STAMPS) { ...@@ -599,8 +604,8 @@ if ($STAMPS) {
} }
# Stop the web interface from spewing. # Stop the web interface from spewing.
TBExptCloseLogFile($pid, $eid) $logfile->Close()
if (defined($logname) && !$batchmode); if (defined($logfile));
# Email is sent from libaudit at exit ... # Email is sent from libaudit at exit ...
exit(0); exit(0);
...@@ -771,12 +776,16 @@ sub cleanup() ...@@ -771,12 +776,16 @@ sub cleanup()
$experiment->End("-f") == 0 $experiment->End("-f") == 0
or exit(-1); or exit(-1);
} }
# The web interface will stop spewing when the instance is deleted
# cause instance Delete removes the logfile entry too.
$instance->Delete() $instance->Delete()
if (defined($instance)); if (defined($instance));
# Stop the web interface from spewing. #
TBExptCloseLogFile($pid, $eid) # Cleanup DB state for the experiment now that instance is gone.
if (defined($logname) && !$batchmode); #
$experiment->Delete()
if (defined($experiment));
} }
sub fatal($$) sub fatal($$)
......
...@@ -632,6 +632,54 @@ function TBGetUniqueIndex($name) ...@@ -632,6 +632,54 @@ function TBGetUniqueIndex($name)
return $curidx; return $curidx;
} }
#
# Trivial wrapup of Logile table so we can use it in url_defs.
#
class Logfile