Commit 32682011 authored by Leigh Stoller's avatar Leigh Stoller

Lets checkpoint a couple of weeks of work on templates.

parent 6cfd387a
......@@ -173,7 +173,9 @@ use vars qw(@ISA @EXPORT);
TBSaveExpLogFiles TBExptWorkDir TBExptUserDir TBExptLogDir
TBExptDestroy TBIPtoNodeID TBNodeBootReset TBNodeStateWait
TBLeaderMailList ExpGroup TBExptSetSwapUID TBExptSetThumbNail
TBNodeAllocCheck TBPlabNodeUsername MarkPhysNodeDown TBExptIsElabInElab
TBNodeAllocCheck TBPlabNodeUsername MarkPhysNodeDown
TBExptIsElabInElab TBBatchUnLockExp TBExptIsBatchExp
TBExptFirewall TBNodeFirewall TBExptFirewallAndPort
TBSetExptFirewallVlan TBClearExptFirewallVlan
TBNodeConsoleTail TBExptGetSwapoutAction TBExptGetSwapState
......@@ -1613,6 +1615,35 @@ sub TBUnLockExp($$;$)
return 1;
}
#
# Helper function for batch system.
#
sub TBBatchUnLockExp($$;$)
{
my($pid, $eid, $newstate) = @_;
my $BSTATE_UNLOCKED = BATCHSTATE_UNLOCKED;
my $query_result =
DBQueryWarn("update experiments set expt_locked=NULL, ".
" batchstate='$BSTATE_UNLOCKED' ".
(defined($newstate) ? ",state='$newstate' " : "") .
"where eid='$eid' and pid='$pid'");
if (! $query_result ||
$query_result->numrows == 0) {
return 0;
}
if ($EVENTSYS && defined($newstate)) {
EventSendWarn(objtype => TBDB_TBEVENT_EXPTSTATE,
objname => "$pid/$eid",
eventtype => $newstate,
expt => "$pid/$eid",
host => $BOSSNODE);
}
return 1;
}
#
# Set cancel flag,
#
......@@ -3752,6 +3783,25 @@ sub TBExptIsElabInElab($$$;$)
return 1;
}
#
# Similar function for batchmode.
#
sub TBExptIsBatchExp($$$)
{
my ($pid, $eid, $batchmode) = @_;
my $query_result =
DBQueryFatal("select batchmode from experiments ".
"where pid='$pid' and eid='$eid'");
if ($query_result->numrows == 0) {
return 0;
}
my @row = $query_result->fetchrow_array();
$$batchmode = $row[0];
return 1;
}
#
# Get the control network IP for a node (underlying physical node!).
#
......
......@@ -595,18 +595,30 @@ CREATE TABLE experiment_template_instances (
--
CREATE TABLE experiment_template_metadata (
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
metadata_guid varchar(16) NOT NULL default '',
metadata_vers smallint(5) unsigned NOT NULL default '0',
internal tinyint(1) NOT NULL default '0',
PRIMARY KEY (parent_guid, parent_vers, metadata_guid, metadata_vers)
) TYPE=MyISAM;
--
-- Table structure for table `experiment_template_metadata_items`
--
CREATE TABLE experiment_template_metadata_items (
guid varchar(16) NOT NULL default '',
vers smallint(5) unsigned NOT NULL default '0',
parent_guid varchar(16) default NULL,
parent_vers smallint(5) unsigned NOT NULL default '0',
template_guid varchar(16) NOT NULL default '',
template_vers smallint(5) unsigned NOT NULL default '0',
name varchar(64) NOT NULL default '',
value tinytext,
created datetime default NULL,
PRIMARY KEY (guid,vers,name),
KEY parent_guid (parent_guid,parent_vers),
KEY template_guid (template_guid,template_vers)
PRIMARY KEY (guid, vers),
KEY parent (parent_guid,parent_vers),
KEY template (template_guid)
) TYPE=MyISAM;
--
......
......@@ -3309,5 +3309,5 @@ last_net_act,last_cpu_act,last_ext_act);
4.54: Minor changes to schema; add some quotes around field names that
are now reserved words mysql 5.X. Skip to the next entry ...
4.55: Changes to templates.sql. Skip this revision for now.
......@@ -156,26 +156,42 @@ CREATE TABLE experiment_template_settings (
) TYPE=MyISAM;
#
# This is versioned metadata that goes with each template. Not sure what
# goes into this table yet ...
# This is versioned metadata that goes with each template. We store the
# guid and version of the corresponding record in the table below.
#
CREATE TABLE experiment_template_metadata (
-- Globally Unique ID. Okay, how global is global?
-- Globally Unique ID of the ExperimentTemplate this record belongs to.
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
-- GUID of the metadata item.
metadata_guid varchar(16) NOT NULL default '',
metadata_vers smallint(5) unsigned NOT NULL default '0',
-- Internal metadata items, handled specially.
internal tinyint(1) NOT NULL default '0',
PRIMARY KEY (parent_guid, parent_vers, metadata_guid, metadata_vers)
) TYPE=MyISAM;
#
# The actual versioned metadata.
#
CREATE TABLE experiment_template_metadata_items (
-- Globally Unique ID.
guid varchar(16) NOT NULL default '',
-- Version number for tracking modifications
vers smallint(5) unsigned NOT NULL default '0',
-- Backlink to the previous version of this metadata item.
parent_guid varchar(16) default NULL,
parent_vers smallint(5) unsigned NOT NULL default '0',
-- Backlink to the template this metadata item belongs to.
-- Record the template GUID this metadata associated with. This is for
-- permission checks and for deletion.
template_guid varchar(16) NOT NULL default '',
template_vers smallint(5) unsigned NOT NULL default '0',
-- Key/Value pairs.
name varchar(64) NOT NULL default '',
value tinytext,
created datetime default NULL,
PRIMARY KEY (guid, vers, name),
KEY (parent_guid,parent_vers),
KEY (template_guid,template_vers)
PRIMARY KEY (guid, vers),
KEY parent (parent_guid,parent_vers),
KEY template (template_guid)
) TYPE=MyISAM;
#
......
......@@ -23,7 +23,7 @@ BIN_STUFF = power snmpit tbend tbprerun tbreport \
tbswap nseswap tarfiles_setup node_history tbrsync \
node_attributes archive_control template_create \
template_swapin template_swapout template_graph \
template_exprun
template_exprun template_delete
SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
batch_daemon exports_setup reload_daemon sched_reserve \
......@@ -52,7 +52,8 @@ LIBEXEC_STUFF = rmproj wanlinksolve wanlinkinfo \
webnodereboot webrmuser webidleswap switchmac \
spewrpmtar webtarfiles_setup webfrisbeekiller gentopofile \
webnodeattributes webarchive_control webtemplate_create \
webtemplate_swapin webtemplate_swapout webtemplate_exprun
webtemplate_swapin webtemplate_swapout webtemplate_exprun \
webtemplate_graph
LIB_STUFF = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm power_rpc27.pm \
......
......@@ -175,13 +175,22 @@ sub DeleteTemplateRecord($$)
{
my ($guid, $vers) = @_;
DeleteTemplateMetadata($guid, $vers) == 0
or return -1;
# Make sure the experiment_templates table is always last, in case
# something goes wrong.
my @tables = ("experiment_templates");
my @tables = ("experiment_templates", "experiment_template_parameters");
foreach my $table (@tables) {
DBQueryFatal("delete from $table ".
"where guid='$guid' and vers='$vers'");
if ($table eq "experiment_templates") {
DBQueryFatal("delete from $table ".
"where guid='$guid' and vers='$vers'");
}
else {
DBQueryFatal("delete from $table ".
"where parent_guid='$guid' and parent_vers='$vers'");
}
}
}
......@@ -213,9 +222,39 @@ sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'");
#
# Delete the run records first since they will not mean much after
# this record is gone.
#
my $query_result =
DBQueryWarn("select exptidx from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'");
return -1
if (!$query_result);
while (my ($exptidx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_run_bindings ".
"where exptidx='$exptidx'")
or return -1;
DBQueryWarn("delete from experiment_runs ".
"where exptidx='$exptidx'")
or return -1;
}
#
# Also delete the binding records for the instance.
#
DeleteTemplateInstanceBindingRecords($guid, $vers, $idx) == 0
or return -1;
# And finally the instance record.
DBQueryWarn("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'")
or return -1;
return 0;
}
......@@ -351,9 +390,10 @@ sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" instance_idx='$idx'");
DBQueryWarn("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" instance_idx='$idx'")
or return -1;
return 0;
}
......@@ -450,22 +490,26 @@ sub DeleteExperimentRunBindingRecords($$)
#
# Utility routine to get the pid,tid,gid of a template.
#
sub TemplateInfo($$$$;$)
sub TemplateInfo($$;$$$$)
{
my ($guid, $version, $ppid, $ptid, $pgid) = @_;
my ($guid, $version, $ppid, $ptid, $pgid, $peid) = @_;
my $query_result =
DBQueryWarn("select pid,tid,gid from experiment_templates ".
DBQueryWarn("select pid,tid,gid,eid from experiment_templates ".
"where guid='$guid' and vers='$version'");
return -1
if (!$query_result || !$query_result->numrows);
my ($pid, $tid, $gid) = $query_result->fetchrow_array();
$$ppid = $pid;
$$ptid = $tid;
my ($pid, $tid, $gid, $eid) = $query_result->fetchrow_array();
$$ppid = $pid
if (defined($ppid));
$$ptid = $tid
if (defined($ptid));
$$pgid = $gid
if (defined($pgid));
$$peid = $eid
if (defined($peid));
return 0;
}
......@@ -616,6 +660,140 @@ sub DeleteTemplateInputFiles($$)
return 0;
}
#
# Add a metadata record. This is used for new templates.
#
sub NewTemplateMetadata($$$$)
{
my ($template_guid, $template_version, $name, $value) = @_;
my $guid;
my $version = 1;
return -1
if (libTemplates::NewGUID(\$guid) < 0);
$name = DBQuoteSpecial($name);
$value = DBQuoteSpecial($value);
my $query_result =
DBQueryWarn("insert into experiment_template_metadata_items set ".
" guid='$guid', vers='$version', ".
" template_guid='$template_guid', ".
" name=$name, value=$value, created=now()");
return -1
if (!$query_result);
DBQueryWarn("insert into experiment_template_metadata set ".
" parent_guid='$template_guid', ".
" parent_vers='$template_version', ".
" metadata_guid='$guid', ".
" metadata_vers='$version', ".
" internal=1")
or return -1;
return 0;
}
#
# Delete all template metadata; thats all we need here at the moment
#
sub DeleteTemplateMetadata($$)
{
my ($template_guid, $template_version) = @_;
my $query_result =
DBQueryWarn("select metadata_guid ".
" from experiment_template_metadata ".
"where parent_guid='$template_guid' and ".
" parent_vers='$template_version'");
return -1
if (!$query_result);
while (my ($metadata_guid) = $query_result->fetchrow_array()) {
my @versions = ();
#
# Delete all versions for each record. This is wrong if we ever
# want to share entries between templates.
#
my $metadata_result =
DBQueryWarn("select vers from experiment_template_metadata_items ".
"where guid='$metadata_guid'");
return -1
if (!$metadata_result);
next
if (!$metadata_result->numrows);
while (my ($metadata_vers) = $metadata_result->fetchrow_array()) {
push(@versions, $metadata_vers);
}
my $clause = join(" or ", map("vers='$_'", @versions));
DBQueryWarn("delete from experiment_template_metadata_items ".
"where guid='$metadata_guid' and ($clause)")
or return -1;
}
DBQueryWarn("delete from experiment_template_metadata ".
"where parent_guid='$template_guid' and ".
" parent_vers='$template_version'")
or return -1;
return 0;
}
#
# Copy exiting template metadata to a child. This is likely to change
# since we probably want to share at some point.
#
sub CopyTemplateMetadata($$$)
{
my ($from_guid, $from_version, $to_version) = @_;
#
# Copy the toplevel items.
#
my $query_result =
DBQueryWarn("select name,value ".
" from experiment_template_metadata as m ".
"left join experiment_template_metadata_items as i on ".
" i.guid=m.metadata_guid and ".
" i.vers=m.metadata_vers ".
"where m.parent_guid='$from_guid' and ".
" m.parent_vers='$from_version' and m.internal=0")
or return -1;
while (my ($name, $value) = $query_result->fetchrow_array()) {
my $guid;
my $version = 1;
$name = DBQuoteSpecial($name);
$value = DBQuoteSpecial($value);
return -1
if (libTemplates::NewGUID(\$guid) < 0);
DBQueryWarn("insert into experiment_template_metadata set ".
" parent_guid='$from_guid', ".
" parent_vers='$to_version', ".
" metadata_guid='$guid', ".
" metadata_vers='$version', ".
" internal=0")
or return -1;
DBQueryWarn("insert into experiment_template_metadata_items set ".
" guid='$guid', vers='$version', ".
" template_guid='$from_guid', ".
" name=$name, value=$value, created=now()")
or return -1;
}
return 0;
}
#
# Find out if an experiment is a template instantiation; used by existing
# pages to alter what they do.
......
......@@ -53,6 +53,7 @@ if ($UID) {
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use libTemplates;
# Be careful not to exit on transient error; 0 means infinite retry.
$libdb::DBQUERY_MAXTRIES = 0;
......@@ -66,6 +67,9 @@ my $batchlog = "$TB/log/batchlog";
my $projroot = "/proj";
my $debug = 0;
# New template stuff.
my $template_swapout = "$TB/bin/template_swapout";
my $BSTATE_POSTED = EXPTSTATE_QUEUED;
my $BSTATE_ACTIVATING = EXPTSTATE_ACTIVATING;
my $BSTATE_RUNNING = EXPTSTATE_ACTIVE;
......@@ -88,6 +92,9 @@ my $userdir;
my $workdir;
my $user_name = "Testbed Operations";
my $user_email = "$TBOPS";
my $istemplate;
my $guid;
my $version;
#
# Turn off line buffering on output
......@@ -348,6 +355,18 @@ sub dosomething($$)
#
$logname = TBExptCreateLogFile($pid, $eid, "${dowhat}-batch");
#
# If this is a template instantiation, then do not set the logfile.
# We still generate the log file, but it is not available via the
# web interface. Needs more thought.
#
my $exptidx = $exphash->{'idx'};
$istemplate = libTemplates::IsTemplateInstanceExperiment($exptidx);
if ($istemplate) {
libTemplates::MapExptidxtoTemplate($exptidx, \$guid, \$version) == 0
or fatal("Error template info the experiment $pid/$eid");
}
#
# Start up a child to run the guts. The parent waits. If the
# experiment configures okay, the parent can return to try something
......@@ -574,7 +593,12 @@ sub swapexp($;$)
my $running = ($exphash->{'state'} eq EXPTSTATE_ACTIVE);
if ($running) {
system("$swapexp -b -s out $pid $eid");
if ($istemplate) {
system("$template_swapout -b -e $eid $guid/$version");
}
else {
system("$swapexp -b -s out $pid $eid");
}
if ($?) {
#
# TB admin is going to have to clean up.
......@@ -613,7 +637,12 @@ sub cancelexp($)
# It does not matter if the experiment is running; endexp does the
# right thing.
#
system("$endexp -b $pid $eid");
if ($istemplate) {
system("$template_swapout -b -e $eid $guid/$version");
}
else {
system("$endexp -b $pid $eid");
}
if ($?) {
#
# TB admin is going to have to clean up.
......@@ -756,13 +785,3 @@ sub donotify($$$)
($logname, "assign.log", $nsfile));
}
sub TBBatchUnLockExp($$;$)
{
my($pid, $eid, $newstate) = @_;
DBQueryWarn("update experiments set expt_locked=NULL, ".
" batchstate='$BSTATE_UNLOCKED' ".
(defined($newstate) ? ",state='$newstate' " : "") .
"where eid='$eid' and pid='$pid'");
return 1;
}
......@@ -175,13 +175,22 @@ sub DeleteTemplateRecord($$)
{
my ($guid, $vers) = @_;
DeleteTemplateMetadata($guid, $vers) == 0
or return -1;
# Make sure the experiment_templates table is always last, in case
# something goes wrong.
my @tables = ("experiment_templates");
my @tables = ("experiment_templates", "experiment_template_parameters");
foreach my $table (@tables) {
DBQueryFatal("delete from $table ".
"where guid='$guid' and vers='$vers'");
if ($table eq "experiment_templates") {
DBQueryFatal("delete from $table ".
"where guid='$guid' and vers='$vers'");
}
else {
DBQueryFatal("delete from $table ".
"where parent_guid='$guid' and parent_vers='$vers'");
}
}
}
......@@ -213,9 +222,39 @@ sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'");
#
# Delete the run records first since they will not mean much after
# this record is gone.
#
my $query_result =
DBQueryWarn("select exptidx from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'");
return -1
if (!$query_result);
while (my ($exptidx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_run_bindings ".
"where exptidx='$exptidx'")
or return -1;
DBQueryWarn("delete from experiment_runs ".
"where exptidx='$exptidx'")
or return -1;
}
#
# Also delete the binding records for the instance.
#
DeleteTemplateInstanceBindingRecords($guid, $vers, $idx) == 0
or return -1;
# And finally the instance record.
DBQueryWarn("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" idx='$idx'")
or return -1;
return 0;
}
......@@ -351,9 +390,10 @@ sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" instance_idx='$idx'");
DBQueryWarn("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" instance_idx='$idx'")
or return -1;
return 0;
}
......@@ -450,22 +490,26 @@ sub DeleteExperimentRunBindingRecords($$)
#
# Utility routine to get the pid,tid,gid of a template.
#
sub TemplateInfo($$$$;$)
sub TemplateInfo($$;$$$$)
{
my ($guid, $version, $ppid, $ptid, $pgid) = @_;
my ($guid, $version, $ppid, $ptid, $pgid, $peid) = @_;
my $query_result =
DBQueryWarn("select pid,tid,gid from experiment_templates ".
DBQueryWarn("select pid,tid,gid,eid from experiment_templates ".
"where guid='$guid' and vers='$version'");
return -1
if (!$query_result || !$query_result->numrows);
my ($pid, $tid, $gid) = $query_result->fetchrow_array();
$$ppid = $pid;
$$ptid = $tid;
my ($pid, $tid, $gid, $eid) = $query_result->fetchrow_array();
$$ppid = $pid
if (defined($ppid));
$$ptid = $tid
if (defined($ptid));
$$pgid = $gid
if (defined($pgid));
$$peid = $eid
if (defined($peid));
return 0;
}
......@@ -616,6 +660,140 @@ sub DeleteTemplateInputFiles($$)
return 0;
}
#
# Add a metadata record. This is used for new templates.
#
sub NewTemplateMetadata($$$$)
{
my ($template_guid, $template_version, $name, $value) = @_;
my $guid;
my $version = 1;
return -1
if (libTemplates::NewGUID(\$guid) < 0);
$name = DBQuoteSpecial($name);
$value = DBQuoteSpecial($value);
my $query_result =
DBQueryWarn("insert into experiment_template_metadata_items set ".
" guid='$guid', vers='$version', ".
" template_guid='$template_guid', ".
" name=$name, value=$value, created=now()");
return -1
if (!$query_result);
DBQueryWarn("insert into experiment_template_metadata set ".
" parent_guid='$template_guid', ".
" parent_vers='$template_version', ".
" metadata_guid='$guid', ".
" metadata_vers='$version', ".
" internal=1")
or return -1;
return 0;
}
#
# Delete all template metadata; thats all we need here at the moment
#
sub DeleteTemplateMetadata($$)
{
my ($template_guid, $template_version) = @_;
my $query_result =
DBQueryWarn("select metadata_guid ".
" from experiment_template_metadata ".
"where parent_guid='$template_guid' and ".
" parent_vers='$template_version'");
return -1
if (!$query_result);
while (my ($metadata_guid) = $query_result->fetchrow_array()) {
my @versions = ();
#
# Delete all versions for each record. This is wrong if we ever
# want to share entries between templates.
#
my $metadata_result =
DBQueryWarn("select vers from experiment_template_metadata_items ".
"where guid='$metadata_guid'");
return -1
if (!$metadata_result);
next
if (!$metadata_result->numrows);
while (my ($metadata_vers) = $metadata_result->fetchrow_array()) {
push(@versions, $metadata_vers);
}
my $clause = join(" or ", map("vers='$_'", @versions));
DBQueryWarn("delete from experiment_template_metadata_items ".
"where guid='$metadata_guid' and ($clause)")
or return -1;
}
DBQueryWarn("delete from experiment_template_metadata ".
"where parent_guid='$template_guid' and ".
" parent_vers='$template_version'")
or return -1;
return 0;
}
#
# Copy exiting template metadata to a child. This is likely to change
# since we probably want to share at some point.
#
sub CopyTemplateMetadata($$$)
{
my ($from_guid, $from_version, $to_version) = @_;
#
# Copy the toplevel items.
#
my $query_result =
DBQueryWarn("select name,value ".
" from experiment_template_metadata as m ".
"left join experiment_template_metadata_items as i on ".
" i.guid=m.metadata_guid and ".
" i.vers=m.metadata_vers ".
"where m.parent_guid='$from_guid' and ".
" m.parent_vers='$from_version' and m.internal=0")
or return -1;