Commit 74bf8d1d authored by Leigh Stoller's avatar Leigh Stoller

More experiment run stuff. Also the NS file parsing support that was

needed (reparsing of NS file with current bindings).

Mostly I decided that I made a huge mistake in using the exptidx as the
main cross index. I started to change that, which took a bunch of time.
Still more tables to fix up, but I think all of this code is gonna need
a major reorg once I am no longer under the gun to get it done.
parent 122ab861
......@@ -556,6 +556,7 @@ CREATE TABLE experiment_template_inputs (
--
CREATE TABLE experiment_template_instance_bindings (
instance_idx int(10) unsigned NOT NULL default '0',
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
exptidx int(10) unsigned NOT NULL default '0',
......@@ -563,7 +564,7 @@ CREATE TABLE experiment_template_instance_bindings (
eid varchar(32) NOT NULL default '',
name varchar(64) NOT NULL default '',
value tinytext NOT NULL,
PRIMARY KEY (exptidx,name),
PRIMARY KEY (instance_idx, name),
KEY parent_guid (parent_guid,parent_vers),
KEY pidtid (pid,eid)
) TYPE=MyISAM;
......@@ -573,6 +574,7 @@ CREATE TABLE experiment_template_instance_bindings (
--
CREATE TABLE experiment_template_instances (
idx int(10) unsigned NOT NULL auto_increment,
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
exptidx int(10) unsigned NOT NULL default '0',
......@@ -582,10 +584,10 @@ CREATE TABLE experiment_template_instances (
start_time datetime default NULL,
stop_time datetime default NULL,
runidx int(10) unsigned default NULL,
PRIMARY KEY (exptidx),
PRIMARY KEY (idx),
KEY exptidx (exptidx),
KEY parent_guid (parent_guid,parent_vers),
KEY pid (pid,eid),
KEY exptidx (exptidx, runidx)
KEY pid (pid,eid)
) TYPE=MyISAM;
--
......@@ -776,6 +778,7 @@ CREATE TABLE experiments (
savedisk tinyint(1) NOT NULL default '0',
locpiper_pid int(11) default '0',
locpiper_port int(11) default '0',
instance_idx int(10) unsigned NOT NULL default '0',
PRIMARY KEY (eid,pid),
KEY idx (idx),
KEY batchmode (batchmode)
......
......@@ -3268,3 +3268,8 @@ last_net_act,last_cpu_act,last_ext_act);
4.44: Changes to templates.sql. Skip this revision for now.
4.45: More changes to templates.sql. Also:
alter table experiments add column instance_idx int(10) \
unsigned NOT NULL default '0' AFTER locpiper_port;
......@@ -238,6 +238,8 @@ CREATE TABLE virt_parameters (
# to "ExperimentRecord."
#
CREATE TABLE experiment_template_instances (
-- Auto generated unique index.
idx int(10) unsigned NOT NULL auto_increment,
-- Backlink to the template.
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
......@@ -254,10 +256,10 @@ CREATE TABLE experiment_template_instances (
stop_time datetime default NULL,
-- The current experiment that is running (see below). One at a time!
runidx int(10) unsigned default NULL,
PRIMARY KEY (exptidx),
PRIMARY KEY (idx),
KEY (exptidx),
KEY (parent_guid,parent_vers),
KEY (pid,eid),
KEY (exptidx, runidx)
) TYPE=MyISAM;
#
......@@ -270,6 +272,8 @@ CREATE TABLE experiment_template_instances (
# are permanent.
#
CREATE TABLE experiment_template_instance_bindings (
-- Backlink to the instance above.
instance_idx int(10) unsigned NOT NULL default '0',
-- Backlink to the template.
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
......@@ -281,8 +285,8 @@ CREATE TABLE experiment_template_instance_bindings (
eid varchar(32) NOT NULL default '',
name varchar(64) NOT NULL default '',
value tinytext NOT NULL,
PRIMARY KEY (exptidx, name),
KEY (parent_guid,parent_vers),
PRIMARY KEY (instance_idx, name),
KEY parent_guid (parent_guid,parent_vers),
KEY pidtid (pid,eid)
) TYPE=MyISAM;
......
......@@ -171,20 +171,21 @@ sub DeleteTemplateRecord($$)
#
# Create a new template instance record from the args provided.
#
# Usage: int NewTemplateInstanceRecord(\%args)
# Usage: int NewTemplateInstanceRecord(\%args, \$idx)
#
sub NewTemplateInstanceRecord($)
sub NewTemplateInstanceRecord($$)
{
my ($args) = @_;
my ($args, $pidx) = @_;
my $query = "insert into experiment_template_instances set ".
join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
# Append some default value stuff.
$query .= ",start_time=now()";
my $query_result = DBQueryWarn($query);
return -1
if (! DBQueryWarn($query));
if (! $query_result);
# Grab the insert record.
$$pidx = $query_result->insertid;
return 0;
}
......@@ -193,11 +194,11 @@ sub NewTemplateInstanceRecord($)
#
sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" idx='$idx'");
return 0;
}
......@@ -209,14 +210,15 @@ sub DeleteTemplateInstanceRecord($$$)
#
sub FinalizeTemplateInstanceRecord($$$;$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $args) = @_;
#
# Get the runidx and do it first.
#
my $runidx;
my $exptidx;
return -1
if (TemplateInstanceInfo($exptidx, \$runidx) < 0);
if (TemplateInstanceInfo($idx, \$runidx, undef, \$exptidx) < 0);
if (defined($runidx)) {
return -1
......@@ -230,7 +232,7 @@ sub FinalizeTemplateInstanceRecord($$$;$)
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -240,18 +242,23 @@ sub FinalizeTemplateInstanceRecord($$$;$)
#
# Update an instance record.
#
sub UpdateTemplateInstanceRecord($$$$)
sub UpdateTemplateInstanceRecord($$$$$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $start_time, $args) = @_;
my $query = "update experiment_template_instances set ";
my $query = "update experiment_template_instances ";
$query .= "start_time=now() "
if ($start_time);
if (defined($args) && scalar(keys%{$args})) {
$query .= ",";
$query .= ","
if ($start_time);
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -261,21 +268,45 @@ sub UpdateTemplateInstanceRecord($$$$)
#
# Get info about a template run.
#
sub TemplateInstanceInfo($$;$)
sub TemplateInstanceInfo($$;$$)
{
my ($idx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx,exptidx ".
" from experiment_template_instances ".
"where idx='$idx'");
return -1
if (! $query_result);
my ($eid,$runidx,$exptidx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $exptidx
if (defined($pidx));
return 0;
}
sub TemplateInstanceInfoByExptidx($$;$$)
{
my ($exptidx, $prunidx, $peid) = @_;
my ($exptidx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx from experiment_template_instances ".
DBQueryWarn("select eid,runidx,idx ".
" from experiment_template_instances ".
"where exptidx='$exptidx'");
return -1
if (! $query_result);
my ($eid,$runidx) = $query_result->fetchrow_array();
my ($eid,$runidx,$idx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $idx
if (defined($pidx));
return 0;
}
......@@ -301,11 +332,11 @@ sub NewTemplateInstanceBindingRecord($)
#
sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" instance_idx='$idx'");
return 0;
}
......@@ -452,7 +483,7 @@ sub TemplateInfo($$$$;$)
sub AddTemplateInputFile($$$)
{
my ($guid, $version, $inputfile) = @_;
my $input_idx;
my $input_data_idx;
my $data_string = `cat $inputfile`;
return -1
if ($?);
......@@ -479,15 +510,21 @@ sub AddTemplateInputFile($$$)
my $md5 = `$MD5 -q $inputfile`;
chomp($md5);
DBQueryWarn("lock tables experiment_template_inputs write, ".
" experiment_template_input_data write")
or return -1;
my $query_result =
DBQueryWarn("select idx from experiment_template_input_data ".
"where md5='$md5'");
return -1
if (!$query_result);
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
if ($query_result->numrows) {
($input_idx) = $query_result->fetchrow_array();
($input_data_idx) = $query_result->fetchrow_array();
}
else {
$query_result =
......@@ -495,22 +532,72 @@ sub AddTemplateInputFile($$$)
"(idx, md5, input) ".
"values (NULL, '$md5', $data_string)");
return -1
if (!$query_result);
$input_idx = $query_result->insertid;
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
$input_data_idx = $query_result->insertid;
}
$query_result =
DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_data_idx')");
DBQueryWarn("unlock tables");
return -1
if (!DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_idx')"));
if (!$query_result);
}
return 0;
}
#
# Delete an input file, say for a template create/modify that fails.
#
sub DeleteTemplateInputFiles($$)
{
my ($guid, $version) = @_;
DBQueryWarn("lock tables experiment_template_inputs as i write, ".
" experiment_template_inputs as j write, ".
" experiment_template_input_data write")
or return -1;
#
# The point of this query is to see if any of the input files in this
# template are shared with some other template, and thus should not
# be deleted from the input_data table.
#
my $query_result =
DBQueryWarn("select i.idx,i.input_idx,count(j.input_idx) as count ".
" from experiment_template_inputs as i ".
"left join experiment_template_inputs as j on ".
" j.input_idx=i.input_idx ".
"where i.parent_guid='$guid' and ".
" i.parent_vers='$version' ".
"group by j.input_idx having count > 1");
if (! $query_result) {
DBQueryWarn("unlock tables");
return -1;
}
while (my ($input_idx, $data_idx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_template_input_data ".
"where idx='$data_idx'");
}
$query_result =
DBQueryWarn("delete from experiment_template_inputs ".
"where parent_guid='$guid' and parent_vers='$version'");
DBQueryWarn("unlock tables");
return -1
if (! $query_result);
return 0;
}
#
# Find out if an experiment is a template instantiation; used by existing
# pages to alter what they do.
......
......@@ -171,20 +171,21 @@ sub DeleteTemplateRecord($$)
#
# Create a new template instance record from the args provided.
#
# Usage: int NewTemplateInstanceRecord(\%args)
# Usage: int NewTemplateInstanceRecord(\%args, \$idx)
#
sub NewTemplateInstanceRecord($)
sub NewTemplateInstanceRecord($$)
{
my ($args) = @_;
my ($args, $pidx) = @_;
my $query = "insert into experiment_template_instances set ".
join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
# Append some default value stuff.
$query .= ",start_time=now()";
my $query_result = DBQueryWarn($query);
return -1
if (! DBQueryWarn($query));
if (! $query_result);
# Grab the insert record.
$$pidx = $query_result->insertid;
return 0;
}
......@@ -193,11 +194,11 @@ sub NewTemplateInstanceRecord($)
#
sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" idx='$idx'");
return 0;
}
......@@ -209,14 +210,15 @@ sub DeleteTemplateInstanceRecord($$$)
#
sub FinalizeTemplateInstanceRecord($$$;$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $args) = @_;
#
# Get the runidx and do it first.
#
my $runidx;
my $exptidx;
return -1
if (TemplateInstanceInfo($exptidx, \$runidx) < 0);
if (TemplateInstanceInfo($idx, \$runidx, undef, \$exptidx) < 0);
if (defined($runidx)) {
return -1
......@@ -230,7 +232,7 @@ sub FinalizeTemplateInstanceRecord($$$;$)
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -240,18 +242,23 @@ sub FinalizeTemplateInstanceRecord($$$;$)
#
# Update an instance record.
#
sub UpdateTemplateInstanceRecord($$$$)
sub UpdateTemplateInstanceRecord($$$$$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $start_time, $args) = @_;
my $query = "update experiment_template_instances set ";
my $query = "update experiment_template_instances ";
$query .= "start_time=now() "
if ($start_time);
if (defined($args) && scalar(keys%{$args})) {
$query .= ",";
$query .= ","
if ($start_time);
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -261,21 +268,45 @@ sub UpdateTemplateInstanceRecord($$$$)
#
# Get info about a template run.
#
sub TemplateInstanceInfo($$;$)
sub TemplateInstanceInfo($$;$$)
{
my ($idx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx,exptidx ".
" from experiment_template_instances ".
"where idx='$idx'");
return -1
if (! $query_result);
my ($eid,$runidx,$exptidx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $exptidx
if (defined($pidx));
return 0;
}
sub TemplateInstanceInfoByExptidx($$;$$)
{
my ($exptidx, $prunidx, $peid) = @_;
my ($exptidx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx from experiment_template_instances ".
DBQueryWarn("select eid,runidx,idx ".
" from experiment_template_instances ".
"where exptidx='$exptidx'");
return -1
if (! $query_result);
my ($eid,$runidx) = $query_result->fetchrow_array();
my ($eid,$runidx,$idx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $idx
if (defined($pidx));
return 0;
}
......@@ -301,11 +332,11 @@ sub NewTemplateInstanceBindingRecord($)
#
sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" instance_idx='$idx'");
return 0;
}
......@@ -452,7 +483,7 @@ sub TemplateInfo($$$$;$)
sub AddTemplateInputFile($$$)
{
my ($guid, $version, $inputfile) = @_;
my $input_idx;
my $input_data_idx;
my $data_string = `cat $inputfile`;
return -1
if ($?);
......@@ -479,15 +510,21 @@ sub AddTemplateInputFile($$$)
my $md5 = `$MD5 -q $inputfile`;
chomp($md5);
DBQueryWarn("lock tables experiment_template_inputs write, ".
" experiment_template_input_data write")
or return -1;
my $query_result =
DBQueryWarn("select idx from experiment_template_input_data ".
"where md5='$md5'");
return -1
if (!$query_result);
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
if ($query_result->numrows) {
($input_idx) = $query_result->fetchrow_array();
($input_data_idx) = $query_result->fetchrow_array();
}
else {
$query_result =
......@@ -495,22 +532,72 @@ sub AddTemplateInputFile($$$)
"(idx, md5, input) ".
"values (NULL, '$md5', $data_string)");
return -1
if (!$query_result);
$input_idx = $query_result->insertid;
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
$input_data_idx = $query_result->insertid;
}
$query_result =
DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_data_idx')");
DBQueryWarn("unlock tables");
return -1
if (!DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_idx')"));
if (!$query_result);
}
return 0;
}
#
# Delete an input file, say for a template create/modify that fails.
#
sub DeleteTemplateInputFiles($$)
{
my ($guid, $version) = @_;
DBQueryWarn("lock tables experiment_template_inputs as i write, ".
" experiment_template_inputs as j write, ".
" experiment_template_input_data write")
or return -1;
#
# The point of this query is to see if any of the input files in this
# template are shared with some other template, and thus should not
# be deleted from the input_data table.
#
my $query_result =
DBQueryWarn("select i.idx,i.input_idx,count(j.input_idx) as count ".
" from experiment_template_inputs as i ".
"left join experiment_template_inputs as j on ".
" j.input_idx=i.input_idx ".
"where i.parent_guid='$guid' and ".
" i.parent_vers='$version' ".
"group by j.input_idx having count > 1");
if (! $query_result) {
DBQueryWarn("unlock tables");
return -1;
}
while (my ($input_idx, $data_idx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_template_input_data ".
"where idx='$data_idx'");
}
$query_result =
DBQueryWarn("delete from experiment_template_inputs ".
"where parent_guid='$guid' and parent_vers='$version'");
DBQueryWarn("unlock tables");
return -1
if (! $query_result);
return 0;
}
#
# Find out if an experiment is a template instantiation; used by existing
# pages to alter what they do.
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -589,7 +589,29 @@ sub GenDefsFile($)
}
}
# Be sure to initialize this to something ...
print TCL "set elabinelab_maxpcs $maxnodes\n";
print TCL "set elabinelab_maxpcs $maxnodes\n\n";
#
# For Templates.
#
$query_result =
DBQueryFatal("select instance_idx from experiments ".
"where pid='$pid' and eid='$eid'");
my ($instance_idx) = $query_result->fetchrow_array();
if ($instance_idx) {
print TCL "# Template goo\n";
$query_result =
DBQueryFatal("select * from experiment_template_instance_bindings ".
"where instance_idx='$instance_idx'");
while (my %row = $query_result->fetchhash()) {
my $name = $row{'name'};
my $value = $row{'value'};
print TCL "set parameter_list_defaults($name) \"$value\"\n";
}
}
print TCL "}\n";
close(TCL);
......
......@@ -1088,15 +1088,18 @@ Simulator instproc add_topography {tg} {
return 0
}
Simulator instproc define_template_parameter {name args} {
Simulator instproc define-template-parameter {name value} {
$self instvar parameter_list
var_import ::TBCOMPAT::parameter_list_defaults
if {[llength $args] == 0} {
set parameter_list($name) {}
} else {
set parameter_list($name) [lindex $args 0]
if {[info exists parameter_list_defaults($name)]} {
set value $parameter_list_defaults($name)
}
set parameter_list($name) $value
# And install the name/value in the outer environment.
uplevel 1 real_set \{$name\} \{$value\}
return 0
}
......
# -*- tcl -*-
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -102,6 +102,10 @@ namespace eval TBCOMPAT {
variable reserved_node
set reserved_list {}
# Input parameters for Templates
variable parameter_list_defaults
array set parameter_list_defaults {}
# Physical node names
variable physnodes
......
......@@ -255,10 +255,6 @@ if (libTemplates::NewTemplateRecord(\%args) < 0) {
# Now safe to unlock; the tid/eid is reserved;
DBQueryFatal("unlock tables");
# Input files are kept in the DB, with the template.
exit(-1)
if (libTemplates::AddTemplateInputFile($guid, $version, $inputfile) < 0);
# Now invoke batchexp to preload the experiment. Note special -x option.
system("$batchexp -x -q -i -f ".
"-E 'Experiment Template Preload $guid/$version' ".
......@@ -269,6 +265,9 @@ fatal($? >> 8, "Oops")
# Need to kill the experiment if we fail after this point.
$exptcreated = 1;
# Input files are kept in the DB, with the template.
exit(-1)
if (libTemplates::AddTemplateInputFile($guid, $version, $inputfile) < 0);
#
# Generate the graph for the template.
#
......@@ -430,6 +429,9 @@ sub cleanup()
# And delete all the other stuff?
}
libTemplates::DeleteTemplateInputFiles($guid, $version)
if (defined($guid));
libTemplates::DeleteTemplateRecord($guid, $version)
if (defined($guid));
}
......
......@@ -57,6 +57,7 @@ my $guid;
my $version;
my $inputfile;