Commit fe9aa6a4 authored by Leigh Stoller's avatar Leigh Stoller

The template "datastore" ...

Each template has a datastore, which is really just a subdirectory that can
be populated with files, and committed to the subversion archive.  Note,
the datastore os specific to the template itself. The Template Archive link
on the Show Template page takes you to the subdirectory, which by
convention I am calling "datastore".

The directory actually lives in /proj/pid/exp/eid/TGUID-VERS ... but that
path is printed out for you on the archive page.

Anyway, put stuff in the datastore directory, and then commit the template
archive so there is a tag associated with it.

When an instance is created, a checkout of the datastore is placed in the
experiment directory (/proj/pid/eid/exp/template_datastore). The current
tag (from above) is stored with the instance so that we can later recreate
the enviroment for the instance, say for rerun.

Tarfiles and rpms in the datastore can be referenced as xxx://foo.rpm (in
your NS file).  tarfiles_setup transforms those when the instance is
swapped in, sorta like it does other URLs, only it does not actually fetch
them, just need to rewrite the paths so they reference datastore.

The program agent gets another environment variable so you can refer to the
datastore without hardwiring paths ($DATASTORE). Eventually I want to move
the checkout someplace else, but it was easy to drop it into the experiment
directory for now.
parent 720c0986
......@@ -587,6 +587,7 @@ CREATE TABLE experiment_template_instances (
start_time datetime default NULL,
stop_time datetime default NULL,
runidx int(10) unsigned default NULL,
template_tag varchar(64) default NULL,
PRIMARY KEY (idx),
KEY exptidx (exptidx),
KEY parent_guid (parent_guid,parent_vers),
......
......@@ -3423,3 +3423,10 @@ last_net_act,last_cpu_act,last_ext_act);
This immediately logs everyone out from the web interface. but
thats not really a big deal.
4.69: Add template_tag to experiment_template_instances table.
**** Skip this if you just did 4.41 above.
alter table experiment_template_instances add \
template_tag varchar(64) default NULL;
......@@ -29,6 +29,7 @@ my $MD5 = "/sbin/md5";
my $makegraph = "$TB/bin/template_graph";
my $TEVC = "$TB/bin/tevc";
my $DBCONTROL = "$TB/sbin/opsdb_control";
my $RSYNC = "/usr/local/bin/rsync";
# Cache of template instances to avoid regenerating them.
my %templates = ();
......@@ -103,6 +104,7 @@ sub pid($) { return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'pid'}); }
sub gid($) { return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'gid'}); }
sub eid($) { return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'eid'}); }
sub tid($) { return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'tid'}); }
sub path($) { return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'path'}); }
sub archive_idx($) {
return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'archive_idx'}); }
......@@ -1218,6 +1220,26 @@ sub InstanceList($$$)
return 0;
}
#
# Copy the datastore from parent template to child template.
#
sub CopyDataStore($$$)
{
my ($self, $parent) = @_;
# Must be a real reference.
return -1
if (! (ref($self) && ref($parent)));
my $from_path = $parent->path();
my $to_path = $self->path();
system("cd $from_path; $RSYNC -rtgoDlz datastore $to_path") == 0
or return -1;
return 0;
}
############################################################################
package Template::Instance;
......@@ -1253,7 +1275,7 @@ sub LookupByID($$)
return undef
if (!defined($template));
$self->{'TEMPLATE'} = $template;
bless($self, $class);
return $self;
}
......@@ -1893,7 +1915,36 @@ sub ArchiveTag($$)
return 0;
}
#
# Copy the datastore from the template down to the instance. Temporary ...
#
sub CopyDataStore($$$)
{
my ($self, $archive_tag, $to_path) = @_;
# Must be a real reference.
return -1
if (! ref($self));
# From the template itself.
my $exptidx;
TBExptIDX($self->template()->pid(), $self->template()->eid(), \$exptidx)
or return -1;
if (! -e $to_path) {
system("mkdir -p $to_path") == 0
or return -1;
}
libArchive::TBCheckoutExperimentArchivebyExptIDX($exptidx,
$to_path,
$archive_tag,
"datastore")
== 0 or return -1;
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
......
......@@ -346,7 +346,12 @@ sub ArchiveAdd($$;$$$)
if ($special) {
#
# Last parth of path must be a directory.
# What does this do?
# Basically, we copy the last part (directory) to / of the checkin.
# eg: cp /proj/pid/exp/eid/archive/... /archive of the checkins.
# This avoids pid/eid tokens in the archive.
#
# Last part of path must be a directory.
#
if (! -d $pathname) {
print STDERR "ArchiveAdd: Must be a direcotory: $pathname\n";
......@@ -354,10 +359,6 @@ sub ArchiveAdd($$;$$$)
}
my ($filename,$directory,undef) = fileparse($pathname);
#
# Basically, copy the last part (directory) to / of the checkin.
# eg: cp /proj/pid/exp/eid/archive/... /archive of the checkins.
#
$rootdir = $filename;
$sourcedir = $directory;
$sourcefile = $filename . "/";
......@@ -1308,6 +1309,32 @@ sub TBExperimentArchiveInfo($$$$)
return 0;
}
#
# Grab the current tag for an experiment.
#
sub TBExperimentArchiveTag($$$)
{
my ($pid, $eid, $tagp) = @_;
my $query_result =
DBQueryWarn("select r.archive_tag from experiments as e ".
"left join experiment_stats as s on s.exptidx=e.idx ".
"left join experiment_resources as r on r.idx=s.rsrcidx ".
"where e.pid='$pid' and e.eid='$eid'");
return -1
if (!$query_result || $query_result->numrows == 0);
my ($archive_tag) = $query_result->fetchrow_array();
# Need to deal with no archive yet!
return 1
if (!defined($archive_tag));
$$tagp = $archive_tag;
return 0;
}
#
# Create a new archive for an experiment. This has to update the
# experiment_stats table with the newly created archive index.
......@@ -1479,6 +1506,17 @@ sub TBExperimentArchiveAddUserFiles($$)
return $rval
if ($rval != 0);
}
# Another special directory ... specifically for templates.
my $userstore = "$userdir/datastore";
if (-e $userstore) {
$rval = ArchiveAdd($archive_idx, $userstore, $view, 1, 1);
return $rval
if ($rval != 0);
}
return 0;
}
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003, 2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2003, 2005, 2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -40,7 +40,7 @@ my $projroot = PROJROOT();
my $grouproot= GROUPROOT();
my $tbdata = "tbdata";
my @dirlist = ($tbdata, "bin", "tmp", "logs", "archive",
my @dirlist = ($tbdata, "bin", "tmp", "logs", "archive", "datastore",
"tftpboot", "swapinfo");
my $exitval;
......
......@@ -254,10 +254,13 @@ namespace eval TBCOMPAT {
# @param url The URL to check.
# @return True if "url" looks like a URL.
#
# What is xxx:// you might ask? Its part of experimental template code.
#
proc verify-url {url} {
if {[string match "http://*" $url] ||
[string match "https://*" $url] ||
[string match "ftp://*" $url]} {
[string match "ftp://*" $url] ||
[string match "xxx://*" $url]} {
set retval 1
} else {
set retval 0
......
......@@ -50,6 +50,7 @@ use libosload;
use libtestbed;
use libtblog;
use libArchive;
use Template;
TBDebugTimeStampsOn();
......@@ -319,8 +320,6 @@ while (my %row = $db_result->fetchhash()) {
if (! -f $rpm) {
die_noretry("RPM $rpm for node $node does not exist!");
}
libArchive::TBExperimentArchiveAddFile($pid, $eid, $rpm) == 0 or
die_noretry("*** Failed to add RPM $rpm to the archive!");
}
#
......@@ -332,8 +331,6 @@ while (my %row = $db_result->fetchhash()) {
if (! -f $tar) {
die_noretry("Tarfile $tar for node $node does not exist!");
}
libArchive::TBExperimentArchiveAddFile($pid, $eid, $tar) == 0 or
die_noretry("*** Failed to add Tarfile $tar to the archive!");
}
#
......
......@@ -1097,10 +1097,8 @@ if (! -e "$archivedir/nsdata") {
or fatal("Failed to mkdir $archivedir/nsdata");
}
if ($inout eq "modify") {
system("cp -fp $workdir/$eid.ns $archivedir/nsdata/nsfile.ns") == 0
or fatal("Failed to copy nsfile to $archivedir/tbdata");
}
system("cp -fp $workdir/$eid.ns $archivedir/nsdata/nsfile.ns") == 0
or fatal("Failed to copy nsfile to $archivedir/tbdata");
if (defined($logname)) {
system("cp -fp $logname $archivedir/tbdata/log.$tag") == 0
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2003, 2004 University of Utah and the Flux Group.
# Copyright (c) 2003, 2004, 2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -79,6 +79,8 @@ use lib "@prefix@/lib";
use libtestbed;
use libdb;
use libtblog;
use Template;
use libArchive;
if (@ARGV != 2) {
usage();
......@@ -86,6 +88,8 @@ if (@ARGV != 2) {
my ($pid, $eid) = @ARGV;
my $dbuid;
my %tofetch = ();
my %toarchive = ();
#
# Verify user and get his DB uid.
......@@ -116,6 +120,11 @@ if (!$expdir) {
tbdie("Unable to get experiment directory");
}
# Need this below for template specific code.
my $exptidx;
TBExptIDX($pid, $eid, \$exptidx) or
tbdie("unable to get experiment index!");
#
# Get a list of all RPMs and tarballs to fetch
#
......@@ -171,8 +180,35 @@ while (my ($vname, $rpms, $tarfiles) = $result->fetchrow()) {
# will be uploaded to the nodes table
#
$tofetch{$URL} = $localfile;
$toarchive{$localfile} = $localfile;
$rpms =~ s/$URL/$localfile/g;
}
elsif ($rpm =~ /^xxx:\/\/(.*)$/) {
#
# XXX (well, what else did you expect!). This is a template
# "url" that refers to a file in the template datastore tree.
#
my $instance = Template::Instance->LookupByExptidx($exptidx);
if (!defined($instance)) {
tbdie("Invalid RPM URL for non-template experiment: $rpm");
}
my $localfile = $instance->template()->path() . "/datastore/" . $1;
tbdie("$rpm cannot be found; $localfile does not exist")
if (! -e $localfile);
# no need to archive these since they are saved in the template
$rpms =~ s/$rpm/$localfile/g;
}
else {
#
# Should be a regular path.
#
tbdie("$rpm cannot be found; local file does not exist")
if (! -e $rpm);
$toarchive{$rpm} = $rpm;
}
}
#
......@@ -195,9 +231,37 @@ while (my ($vname, $rpms, $tarfiles) = $result->fetchrow()) {
$ext =~ /(\.tar|\.tar\.Z|\.tar\.gz|\.tgz|\.tar\.bz2)$/;
$ext = $1;
my $localfile = $expdir . "/" . $md5 . $ext;
$toarchive{$localfile} = $localfile;
$tofetch{$URL} = $localfile;
$tarfiles =~ s/$URL/$localfile/g;
}
elsif ($tarfile =~ /^xxx:\/\/(.*)$/) {
#
# XXX (well, what else did you expect!). This is a template
# "url" that refers to a file in the template datastore tree.
#
my $instance = Template::Instance->LookupByExptidx($exptidx);
if (!defined($instance)) {
tbdie("Invalid tarball URL for non-template experiment: ".
"$tarfile");
}
my $localfile = $instance->template()->path() . "/datastore/" . $1;
tbdie("$tarfile cannot be found; $localfile does not exist")
if (! -e $localfile);
# no need to archive these since they are saved in the template
$tarfiles =~ s/$tarfile/$localfile/g;
}
else {
#
# Should be a regular path.
#
tbdie("$tarfile cannot be found; local file does not exist")
if (! -e $tarfile);
$toarchive{$tarfile} = $tarfile;
}
}
#
......@@ -247,6 +311,14 @@ while (my ($URL, $localfile) = each %tofetch) {
}
}
#
# Now add to the archive.
#
while (my ($localfile, $ignored) = each %toarchive) {
libArchive::TBExperimentArchiveAddFile($pid, $eid, $localfile) == 0 or
tbdie("Failed to add $localfile to the archive!");
}
#
# Check to make sure a URL for a tarball or RPM is valid, and return an
# untained version of it. Returns undefined if the URL is not valid.
......
......@@ -258,6 +258,8 @@ else {
# And update the record.
%args = ();
$args{'archive_idx'} = $archive_idx;
# The archive code below requires this be set!
$args{'path'} = TBExptUserDir($pid, $eid);
$template->Update(\%args) == 0
or fatal(-1, "Could not update template record!");
......@@ -267,7 +269,28 @@ $template->Update(\%args) == 0
# but it is not committed. We need to do that now cause this experiment
# is never actually swapped in. Instead each instance is a new fork, and if
# the archive were not committed, it would not look correct.
# Before we do that though, we want to copy the datastore directory to the
# child since we *do* want that stuff shared.
#
if ($modify) {
print "Committing archive before copying data store\n";
libArchive::TBCommitExperimentArchive($pid, $eid, "template_modify")
>= 0 or fatal(-1, "Failed to commit experiment archive!");
$template->CopyDataStore($parent_template, $dbuid) == 0
or fatal(-1, "Failed to copy data store");
# and tell the archive library about the above files.
print "Add new files\n";
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal(-1, "Failed to add datastore files to the archive!");
# and then do a savepoint prior to the commit below.
print "Doing another savepoint\n";
libArchive::TBExperimentArchiveSavePoint($pid, $eid, "CopyDataStore") >= 0
or fatal(-1, "Failed to do a savepoint on the experiment archive!");
}
print "Doing final commit\n";
libArchive::TBCommitExperimentArchive($pid, $eid, "TemplateCreate")
>= 0 or fatal(-1, "Failed to commit experiment archive!");
......
......@@ -82,6 +82,7 @@ my $exptidx;
my $template;
my $instance;
my $logname;
my $template_tag;
my @ExptStates = ();
# For the END block below.
my $cleaning = 0;
......@@ -305,11 +306,20 @@ if (! TBExptIDX($pid, $eid, \$exptidx)) {
fatal(-1, "Could not get experiment index for $pid,$eid!");
}
#
# Need the current archive tag so we know how to find the datastore that
# corresponds to this instance, at some later date.
#
libArchive::TBExperimentArchiveTag($template->pid(),
$template->eid(), \$template_tag) == 0
or fatal(-1, "Could not get current tag for $template");
#
# Update the Instance
#
my %args = ();
$args{'exptidx'} = $exptidx;
$args{'exptidx'} = $exptidx;
$args{'template_tag'} = $template_tag;
$instance->Update(0, \%args) == 0
or fatal(-1, "Could not update experiment instance record!");
......@@ -425,6 +435,21 @@ foreach my $name (keys(%parameters)) {
" pid='$pid', eid='$eid'");
}
#
# Copy the datastore into the experiment directory. This really needs
# to be a checkout, and it really needs to go someplace other then the
# exp directory. Anyway, once that is done, add an environment
# variable the user can reference for finding files in the datastore.
#
my $instance_path = TBExptUserDir($pid, $eid);
$instance->CopyDataStore($template_tag,
"$instance_path/template_datastore") == 0
or fatal(-1, "Could not copy datastore ($template_tag) to instance");
DBQueryFatal("replace into virt_user_environment set ".
" name='DATASTORE', value='$instance_path/template_datastore',".
" pid='$pid', eid='$eid'");
#
# Now do the swapin (or it gets queued if a batch experiment).
#
......
......@@ -82,6 +82,7 @@ my $exptidx;
my $template;
my $instance;
my $logname;
my $template_tag;
my @ExptStates = ();
# For the END block below.
my $cleaning = 0;
......@@ -305,11 +306,20 @@ if (! TBExptIDX($pid, $eid, \$exptidx)) {
fatal(-1, "Could not get experiment index for $pid,$eid!");
}
#
# Need the current archive tag so we know how to find the datastore that
# corresponds to this instance, at some later date.
#
libArchive::TBExperimentArchiveTag($template->pid(),
$template->eid(), \$template_tag) == 0
or fatal(-1, "Could not get current tag for $template");
#
# Update the Instance
#
my %args = ();
$args{'exptidx'} = $exptidx;
$args{'exptidx'} = $exptidx;
$args{'template_tag'} = $template_tag;
$instance->Update(0, \%args) == 0
or fatal(-1, "Could not update experiment instance record!");
......@@ -425,6 +435,21 @@ foreach my $name (keys(%parameters)) {
" pid='$pid', eid='$eid'");
}
#
# Copy the datastore into the experiment directory. This really needs
# to be a checkout, and it really needs to go someplace other then the
# exp directory. Anyway, once that is done, add an environment
# variable the user can reference for finding files in the datastore.
#
my $instance_path = TBExptUserDir($pid, $eid);
$instance->CopyDataStore($template_tag,
"$instance_path/template_datastore") == 0
or fatal(-1, "Could not copy datastore ($template_tag) to instance");
DBQueryFatal("replace into virt_user_environment set ".
" name='DATASTORE', value='$instance_path/template_datastore',".
" pid='$pid', eid='$eid'");
#
# Now do the swapin (or it gets queued if a batch experiment).
#
......
......@@ -7,6 +7,11 @@
include("defs.php3");
include("showstuff.php3");
#
# Standard Testbed Header
#
PAGEHEADER("Commit and Tag");
#
# Only known and logged in users can look at experiments.
#
......@@ -46,12 +51,7 @@ if (!TBCurrentExperiment($exptidx)) {
function SPITFORM($formfields, $errors)
{
global $isadmin, $exptidx, $TBDB_ARCHIVE_TAGLEN;
#
# Standard Testbed Header
#
PAGEHEADER("Commit/Tag archive for experiment $exptidx");
global $isadmin, $exptidx, $TBDB_ARCHIVE_TAGLEN, $referrer;
echo "<center>
Commit/Tag Archive
......@@ -82,6 +82,10 @@ function SPITFORM($formfields, $errors)
echo "<table align=center border=1>
<form action=archive_tag.php3?exptidx=$exptidx method=post>\n";
if (isset($referrer)) {
echo "<input type=hidden name=referrer value=$referrer>\n";
}
echo "<tr>
<td align=center>
<b>Please enter a tag[<b>1</b>]</b>
......@@ -138,6 +142,9 @@ function SPITFORM($formfields, $errors)
#
if (! $submit) {
$defaults = array();
if (!isset($referrer))
$referrer = $_SERVER['HTTP_REFERER'];
SPITFORM($defaults, 0);
PAGEFOOTER();
return;
......@@ -208,12 +215,19 @@ if (count($errors)) {
return;
}
STARTBUSY("Committing and Tagging!");
#
# First lets make sure the tag is unique.
#
$retval = SUEXEC($uid, "$pid,$gid",
"webarchive_control checktag $pid $eid $tag",
SUEXEC_ACTION_IGNORE);
/* Clear the various 'loading' indicators. */
if ($retval)
STOPBUSY();
#
# Fatal Error.
#
......@@ -234,5 +248,15 @@ SUEXEC($uid, "$pid,$gid",
"webarchive_control $tagarg $message -u commit $pid $eid",
SUEXEC_ACTION_DIE);
header("Location: archive_view.php3/$exptidx/?exptidx=$exptidx");
STOPBUSY();
if (!isset($referrer))
$referrer = "archive_view.php3/$exptidx/?exptidx=$exptidx";
PAGEREPLACE($referrer);
#
# Standard Testbed Footer
#
PAGEFOOTER();
?>
......@@ -6,6 +6,7 @@
#
include("defs.php3");
include("showstuff.php3");
include_once("template_defs.php");
#
# Only known and logged in users can look at experiments.
......@@ -45,13 +46,17 @@ if (!$isadmin &&
USERERROR("You do not have permission to view tags for ".
"archive in $pid/$gid ($exptidx)!", 1);
}
$current = TBCurrentExperiment($exptidx);
$current = TBCurrentExperiment($exptidx);
$template = Template::LookupbyEid($pid, $eid);
$url = preg_replace("/archive_view/", "cvsweb/cvsweb",
$_SERVER['REQUEST_URI']);
# This is how you get forms to align side by side across the page.
if ($current) {
if ($template) {
$style = 'style="float:left; width:50%;"';
}
elseif ($current) {
$style = 'style="float:left; width:33%;"';
}
else {
......@@ -59,10 +64,23 @@ else {
}
echo "<center>\n";
echo "<font size=+1>
This is the Subversion archive for experiment $exptidx.<br></font>";
echo "<font size=+1>";
if ($template) {
$guid = $template->guid();
$vers = $template->vers();
$path = $template->path();
echo "This is the (Subversion) datastore for Template $guid/$vers<br>";
echo "(located at $USERNODE:$path)\n";
}
else {
echo "This is the (Subversion) archive for experiment $exptidx.";
}
echo "<br><br></font>";
if ($current) {
if ($current || $template) {
echo "<form action='${TBBASE}/archive_tag.php3' $style method=get>\n";
echo "<b><input type=submit name=tag value='Tag Archive'></b>";
echo "<input type=hidden name=exptidx value='$exptidx'>";
......@@ -74,7 +92,7 @@ echo "<b><input type=submit name=tag value='Show Tags'></b>";
echo "<input type=hidden name=exptidx value='$exptidx'>";
echo "</form>";
if ($current) {
if ($current && !$template) {
echo "<form action='${TBBASE}/archive_missing.php3' $style method=get>";
echo "<b><input type=submit name=missing value='Show Missing Files'></b>";
echo "<input type=hidden name=exptidx value='$exptidx'>";
......
......@@ -11,6 +11,7 @@
class Template
{
var $template;
var $experiment;
function Template($guid, $vers) {
$guid = addslashes($guid);
......@@ -25,6 +26,22 @@ class Template
return;
}
$this->template = mysql_fetch_array($query_result);
#
# Underlying experiment for easy access. Experiment should be its own class!
#
$pid = $this->pid();
$eid = $this->eid();
$query_result =
DBQueryWarn("select * from experiments ".
"where pid='$pid' and eid='$eid'");
if (!$query_result || !mysql_num_rows($query_result)) {
$this->template = NULL;
return;
}
$this->experiment = mysql_fetch_array($query_result);
}
# Hmm, how does one cause an error in a php constructor?
......@@ -48,6 +65,25 @@ class Template
return $foo;
return null;
}
# Look up by pid,eid is which also unique across templates.
function LookupbyEid($pid, $eid) {
$query_result =
DBQueryWarn("select guid,vers from experiment_templates ".
"where pid='$pid' and eid='$eid'");
if (!$query_result || !mysql_num_rows($query_result))
return null;
$row = mysql_fetch_array($query_result);
$guid = $row['guid'];
$vers = $row['vers'];
$foo = new Template($guid, $vers);
if ($foo->IsValid())
return $foo;
return null;
}
#
# Refresh a template instance by reloading from the DB.
......@@ -93,6 +129,9 @@ class Template
function uid() {
return (is_null($this->template) ? -1 : $this->template['uid']);
}
function path() {
return (is_null($this->template) ? -1 : $this->experiment['path']);
}
function IsHidden() {
return (is_null($this->template) ? -1 : $this->template['hidden']);
}
......
......@@ -140,8 +140,9 @@ WRITESUBMENUBUTTON("Add Metadata",
"template_metadata.php?action=add&".
"guid=$guid&version=$version");
# We show the user the datastore for the template; the rest of it is not important.
WRITESUBMENUBUTTON("Template Archive",
"archive_view.php3?exptidx=$exptidx");
"archive_view.php3/$exptidx/trunk?exptidx=$exptidx");
WRITESUBMENUBUTTON("Template Record",
"template_history.php?guid=$guid&version=$version");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment