Commit 74bf8d1d authored by Leigh B. Stoller's avatar Leigh B. Stoller

More experiment run stuff. Also the NS file parsing support that was

needed (reparsing of NS file with current bindings).

Mostly I decided that I made a huge mistake in using the exptidx as the
main cross index. I started to change that, which took a bunch of time.
Still more tables to fix up, but I think all of this code is gonna need
a major reorg once I am no longer under the gun to get it done.
parent 122ab861
......@@ -556,6 +556,7 @@ CREATE TABLE experiment_template_inputs (
--
CREATE TABLE experiment_template_instance_bindings (
instance_idx int(10) unsigned NOT NULL default '0',
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
exptidx int(10) unsigned NOT NULL default '0',
......@@ -563,7 +564,7 @@ CREATE TABLE experiment_template_instance_bindings (
eid varchar(32) NOT NULL default '',
name varchar(64) NOT NULL default '',
value tinytext NOT NULL,
PRIMARY KEY (exptidx,name),
PRIMARY KEY (instance_idx, name),
KEY parent_guid (parent_guid,parent_vers),
KEY pidtid (pid,eid)
) TYPE=MyISAM;
......@@ -573,6 +574,7 @@ CREATE TABLE experiment_template_instance_bindings (
--
CREATE TABLE experiment_template_instances (
idx int(10) unsigned NOT NULL auto_increment,
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
exptidx int(10) unsigned NOT NULL default '0',
......@@ -582,10 +584,10 @@ CREATE TABLE experiment_template_instances (
start_time datetime default NULL,
stop_time datetime default NULL,
runidx int(10) unsigned default NULL,
PRIMARY KEY (exptidx),
PRIMARY KEY (idx),
KEY exptidx (exptidx),
KEY parent_guid (parent_guid,parent_vers),
KEY pid (pid,eid),
KEY exptidx (exptidx, runidx)
KEY pid (pid,eid)
) TYPE=MyISAM;
--
......@@ -776,6 +778,7 @@ CREATE TABLE experiments (
savedisk tinyint(1) NOT NULL default '0',
locpiper_pid int(11) default '0',
locpiper_port int(11) default '0',
instance_idx int(10) unsigned NOT NULL default '0',
PRIMARY KEY (eid,pid),
KEY idx (idx),
KEY batchmode (batchmode)
......
......@@ -3268,3 +3268,8 @@ last_net_act,last_cpu_act,last_ext_act);
4.44: Changes to templates.sql. Skip this revision for now.
4.45: More changes to templates.sql. Also:
alter table experiments add column instance_idx int(10) \
unsigned NOT NULL default '0' AFTER locpiper_port;
......@@ -238,6 +238,8 @@ CREATE TABLE virt_parameters (
# to "ExperimentRecord."
#
CREATE TABLE experiment_template_instances (
-- Auto generated unique index.
idx int(10) unsigned NOT NULL auto_increment,
-- Backlink to the template.
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
......@@ -254,10 +256,10 @@ CREATE TABLE experiment_template_instances (
stop_time datetime default NULL,
-- The current experiment that is running (see below). One at a time!
runidx int(10) unsigned default NULL,
PRIMARY KEY (exptidx),
PRIMARY KEY (idx),
KEY (exptidx),
KEY (parent_guid,parent_vers),
KEY (pid,eid),
KEY (exptidx, runidx)
) TYPE=MyISAM;
#
......@@ -270,6 +272,8 @@ CREATE TABLE experiment_template_instances (
# are permanent.
#
CREATE TABLE experiment_template_instance_bindings (
-- Backlink to the instance above.
instance_idx int(10) unsigned NOT NULL default '0',
-- Backlink to the template.
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
......@@ -281,8 +285,8 @@ CREATE TABLE experiment_template_instance_bindings (
eid varchar(32) NOT NULL default '',
name varchar(64) NOT NULL default '',
value tinytext NOT NULL,
PRIMARY KEY (exptidx, name),
KEY (parent_guid,parent_vers),
PRIMARY KEY (instance_idx, name),
KEY parent_guid (parent_guid,parent_vers),
KEY pidtid (pid,eid)
) TYPE=MyISAM;
......
......@@ -171,20 +171,21 @@ sub DeleteTemplateRecord($$)
#
# Create a new template instance record from the args provided.
#
# Usage: int NewTemplateInstanceRecord(\%args)
# Usage: int NewTemplateInstanceRecord(\%args, \$idx)
#
sub NewTemplateInstanceRecord($)
sub NewTemplateInstanceRecord($$)
{
my ($args) = @_;
my ($args, $pidx) = @_;
my $query = "insert into experiment_template_instances set ".
join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
# Append some default value stuff.
$query .= ",start_time=now()";
my $query_result = DBQueryWarn($query);
return -1
if (! DBQueryWarn($query));
if (! $query_result);
# Grab the insert record.
$$pidx = $query_result->insertid;
return 0;
}
......@@ -193,11 +194,11 @@ sub NewTemplateInstanceRecord($)
#
sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" idx='$idx'");
return 0;
}
......@@ -209,14 +210,15 @@ sub DeleteTemplateInstanceRecord($$$)
#
sub FinalizeTemplateInstanceRecord($$$;$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $args) = @_;
#
# Get the runidx and do it first.
#
my $runidx;
my $exptidx;
return -1
if (TemplateInstanceInfo($exptidx, \$runidx) < 0);
if (TemplateInstanceInfo($idx, \$runidx, undef, \$exptidx) < 0);
if (defined($runidx)) {
return -1
......@@ -230,7 +232,7 @@ sub FinalizeTemplateInstanceRecord($$$;$)
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -240,18 +242,23 @@ sub FinalizeTemplateInstanceRecord($$$;$)
#
# Update an instance record.
#
sub UpdateTemplateInstanceRecord($$$$)
sub UpdateTemplateInstanceRecord($$$$$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $start_time, $args) = @_;
my $query = "update experiment_template_instances set ";
my $query = "update experiment_template_instances ";
$query .= "start_time=now() "
if ($start_time);
if (defined($args) && scalar(keys%{$args})) {
$query .= ",";
$query .= ","
if ($start_time);
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -261,21 +268,45 @@ sub UpdateTemplateInstanceRecord($$$$)
#
# Get info about a template run.
#
sub TemplateInstanceInfo($$;$)
sub TemplateInstanceInfo($$;$$)
{
my ($idx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx,exptidx ".
" from experiment_template_instances ".
"where idx='$idx'");
return -1
if (! $query_result);
my ($eid,$runidx,$exptidx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $exptidx
if (defined($pidx));
return 0;
}
sub TemplateInstanceInfoByExptidx($$;$$)
{
my ($exptidx, $prunidx, $peid) = @_;
my ($exptidx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx from experiment_template_instances ".
DBQueryWarn("select eid,runidx,idx ".
" from experiment_template_instances ".
"where exptidx='$exptidx'");
return -1
if (! $query_result);
my ($eid,$runidx) = $query_result->fetchrow_array();
my ($eid,$runidx,$idx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $idx
if (defined($pidx));
return 0;
}
......@@ -301,11 +332,11 @@ sub NewTemplateInstanceBindingRecord($)
#
sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" instance_idx='$idx'");
return 0;
}
......@@ -452,7 +483,7 @@ sub TemplateInfo($$$$;$)
sub AddTemplateInputFile($$$)
{
my ($guid, $version, $inputfile) = @_;
my $input_idx;
my $input_data_idx;
my $data_string = `cat $inputfile`;
return -1
if ($?);
......@@ -479,15 +510,21 @@ sub AddTemplateInputFile($$$)
my $md5 = `$MD5 -q $inputfile`;
chomp($md5);
DBQueryWarn("lock tables experiment_template_inputs write, ".
" experiment_template_input_data write")
or return -1;
my $query_result =
DBQueryWarn("select idx from experiment_template_input_data ".
"where md5='$md5'");
return -1
if (!$query_result);
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
if ($query_result->numrows) {
($input_idx) = $query_result->fetchrow_array();
($input_data_idx) = $query_result->fetchrow_array();
}
else {
$query_result =
......@@ -495,22 +532,72 @@ sub AddTemplateInputFile($$$)
"(idx, md5, input) ".
"values (NULL, '$md5', $data_string)");
return -1
if (!$query_result);
$input_idx = $query_result->insertid;
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
$input_data_idx = $query_result->insertid;
}
$query_result =
DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_data_idx')");
DBQueryWarn("unlock tables");
return -1
if (!DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_idx')"));
if (!$query_result);
}
return 0;
}
#
# Delete an input file, say for a template create/modify that fails.
#
sub DeleteTemplateInputFiles($$)
{
my ($guid, $version) = @_;
DBQueryWarn("lock tables experiment_template_inputs as i write, ".
" experiment_template_inputs as j write, ".
" experiment_template_input_data write")
or return -1;
#
# The point of this query is to see if any of the input files in this
# template are shared with some other template, and thus should not
# be deleted from the input_data table.
#
my $query_result =
DBQueryWarn("select i.idx,i.input_idx,count(j.input_idx) as count ".
" from experiment_template_inputs as i ".
"left join experiment_template_inputs as j on ".
" j.input_idx=i.input_idx ".
"where i.parent_guid='$guid' and ".
" i.parent_vers='$version' ".
"group by j.input_idx having count > 1");
if (! $query_result) {
DBQueryWarn("unlock tables");
return -1;
}
while (my ($input_idx, $data_idx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_template_input_data ".
"where idx='$data_idx'");
}
$query_result =
DBQueryWarn("delete from experiment_template_inputs ".
"where parent_guid='$guid' and parent_vers='$version'");
DBQueryWarn("unlock tables");
return -1
if (! $query_result);
return 0;
}
#
# Find out if an experiment is a template instantiation; used by existing
# pages to alter what they do.
......
......@@ -171,20 +171,21 @@ sub DeleteTemplateRecord($$)
#
# Create a new template instance record from the args provided.
#
# Usage: int NewTemplateInstanceRecord(\%args)
# Usage: int NewTemplateInstanceRecord(\%args, \$idx)
#
sub NewTemplateInstanceRecord($)
sub NewTemplateInstanceRecord($$)
{
my ($args) = @_;
my ($args, $pidx) = @_;
my $query = "insert into experiment_template_instances set ".
join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
# Append some default value stuff.
$query .= ",start_time=now()";
my $query_result = DBQueryWarn($query);
return -1
if (! DBQueryWarn($query));
if (! $query_result);
# Grab the insert record.
$$pidx = $query_result->insertid;
return 0;
}
......@@ -193,11 +194,11 @@ sub NewTemplateInstanceRecord($)
#
sub DeleteTemplateInstanceRecord($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instances ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" idx='$idx'");
return 0;
}
......@@ -209,14 +210,15 @@ sub DeleteTemplateInstanceRecord($$$)
#
sub FinalizeTemplateInstanceRecord($$$;$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $args) = @_;
#
# Get the runidx and do it first.
#
my $runidx;
my $exptidx;
return -1
if (TemplateInstanceInfo($exptidx, \$runidx) < 0);
if (TemplateInstanceInfo($idx, \$runidx, undef, \$exptidx) < 0);
if (defined($runidx)) {
return -1
......@@ -230,7 +232,7 @@ sub FinalizeTemplateInstanceRecord($$$;$)
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -240,18 +242,23 @@ sub FinalizeTemplateInstanceRecord($$$;$)
#
# Update an instance record.
#
sub UpdateTemplateInstanceRecord($$$$)
sub UpdateTemplateInstanceRecord($$$$$)
{
my ($guid, $vers, $exptidx, $args) = @_;
my ($guid, $vers, $idx, $start_time, $args) = @_;
my $query = "update experiment_template_instances set ";
my $query = "update experiment_template_instances ";
$query .= "start_time=now() "
if ($start_time);
if (defined($args) && scalar(keys%{$args})) {
$query .= ",";
$query .= ","
if ($start_time);
$query .= join(",", map("$_='" . $args->{$_} . "'", keys(%{$args})));
}
$query .= " where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'";
" idx='$idx'";
return -1
if (! DBQueryWarn($query));
......@@ -261,21 +268,45 @@ sub UpdateTemplateInstanceRecord($$$$)
#
# Get info about a template run.
#
sub TemplateInstanceInfo($$;$)
sub TemplateInstanceInfo($$;$$)
{
my ($idx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx,exptidx ".
" from experiment_template_instances ".
"where idx='$idx'");
return -1
if (! $query_result);
my ($eid,$runidx,$exptidx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $exptidx
if (defined($pidx));
return 0;
}
sub TemplateInstanceInfoByExptidx($$;$$)
{
my ($exptidx, $prunidx, $peid) = @_;
my ($exptidx, $prunidx, $peid, $pidx) = @_;
my $query_result =
DBQueryWarn("select eid,runidx from experiment_template_instances ".
DBQueryWarn("select eid,runidx,idx ".
" from experiment_template_instances ".
"where exptidx='$exptidx'");
return -1
if (! $query_result);
my ($eid,$runidx) = $query_result->fetchrow_array();
my ($eid,$runidx,$idx) = $query_result->fetchrow_array();
$$prunidx = $runidx;
$$peid = $eid
if (defined($peid));
$$pidx = $idx
if (defined($pidx));
return 0;
}
......@@ -301,11 +332,11 @@ sub NewTemplateInstanceBindingRecord($)
#
sub DeleteTemplateInstanceBindingRecords($$$)
{
my ($guid, $vers, $exptidx) = @_;
my ($guid, $vers, $idx) = @_;
DBQueryFatal("delete from experiment_template_instance_bindings ".
"where parent_guid='$guid' and parent_vers='$vers' and ".
" exptidx='$exptidx'");
" instance_idx='$idx'");
return 0;
}
......@@ -452,7 +483,7 @@ sub TemplateInfo($$$$;$)
sub AddTemplateInputFile($$$)
{
my ($guid, $version, $inputfile) = @_;
my $input_idx;
my $input_data_idx;
my $data_string = `cat $inputfile`;
return -1
if ($?);
......@@ -479,15 +510,21 @@ sub AddTemplateInputFile($$$)
my $md5 = `$MD5 -q $inputfile`;
chomp($md5);
DBQueryWarn("lock tables experiment_template_inputs write, ".
" experiment_template_input_data write")
or return -1;
my $query_result =
DBQueryWarn("select idx from experiment_template_input_data ".
"where md5='$md5'");
return -1
if (!$query_result);
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
if ($query_result->numrows) {
($input_idx) = $query_result->fetchrow_array();
($input_data_idx) = $query_result->fetchrow_array();
}
else {
$query_result =
......@@ -495,22 +532,72 @@ sub AddTemplateInputFile($$$)
"(idx, md5, input) ".
"values (NULL, '$md5', $data_string)");
return -1
if (!$query_result);
$input_idx = $query_result->insertid;
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
$input_data_idx = $query_result->insertid;
}
$query_result =
DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_data_idx')");
DBQueryWarn("unlock tables");
return -1
if (!DBQueryWarn("insert into experiment_template_inputs ".
" (idx, parent_guid, parent_vers, ".
" pid, tid, input_idx) values ".
" (NULL, '$guid', '$version', '$pid', '$tid', ".
" '$input_idx')"));
if (!$query_result);
}
return 0;
}
#
# Delete an input file, say for a template create/modify that fails.
#
sub DeleteTemplateInputFiles($$)
{
my ($guid, $version) = @_;
DBQueryWarn("lock tables experiment_template_inputs as i write, ".
" experiment_template_inputs as j write, ".
" experiment_template_input_data write")
or return -1;
#
# The point of this query is to see if any of the input files in this
# template are shared with some other template, and thus should not
# be deleted from the input_data table.
#
my $query_result =
DBQueryWarn("select i.idx,i.input_idx,count(j.input_idx) as count ".
" from experiment_template_inputs as i ".
"left join experiment_template_inputs as j on ".
" j.input_idx=i.input_idx ".
"where i.parent_guid='$guid' and ".
" i.parent_vers='$version' ".
"group by j.input_idx having count > 1");
if (! $query_result) {
DBQueryWarn("unlock tables");
return -1;
}
while (my ($input_idx, $data_idx) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from experiment_template_input_data ".
"where idx='$data_idx'");
}
$query_result =
DBQueryWarn("delete from experiment_template_inputs ".
"where parent_guid='$guid' and parent_vers='$version'");
DBQueryWarn("unlock tables");
return -1
if (! $query_result);
return 0;
}
#
# Find out if an experiment is a template instantiation; used by existing
# pages to alter what they do.
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -589,7 +589,29 @@ sub GenDefsFile($)
}
}
# Be sure to initialize this to something ...
print TCL "set elabinelab_maxpcs $maxnodes\n";
print TCL "set elabinelab_maxpcs $maxnodes\n\n";
#
# For Templates.
#
$query_result =
DBQueryFatal("select instance_idx from experiments ".
"where pid='$pid' and eid='$eid'");
my ($instance_idx) = $query_result->fetchrow_array();
if ($instance_idx) {
print TCL "# Template goo\n";
$query_result =
DBQueryFatal("select * from experiment_template_instance_bindings ".
"where instance_idx='$instance_idx'");
while (my %row = $query_result->fetchhash()) {
my $name = $row{'name'};
my $value = $row{'value'};
print TCL "set parameter_list_defaults($name) \"$value\"\n";
}
}
print TCL "}\n";
close(TCL);
......
......@@ -1088,15 +1088,18 @@ Simulator instproc add_topography {tg} {
return 0
}
Simulator instproc define_template_parameter {name args} {
Simulator instproc define-template-parameter {name value} {
$self instvar parameter_list
var_import ::TBCOMPAT::parameter_list_defaults
if {[llength $args] == 0} {
set parameter_list($name) {}
} else {
set parameter_list($name) [lindex $args 0]
if {[info exists parameter_list_defaults($name)]} {
set value $parameter_list_defaults($name)
}
set parameter_list($name) $value
# And install the name/value in the outer environment.
uplevel 1 real_set \{$name\} \{$value\}
return 0
}
......
# -*- tcl -*-
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2005 University of Utah and the Flux Group.
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -102,6 +102,10 @@ namespace eval TBCOMPAT {
variable reserved_node
set reserved_list {}
# Input parameters for Templates
variable parameter_list_defaults
array set parameter_list_defaults {}
# Physical node names
variable physnodes
......
......@@ -255,10 +255,6 @@ if (libTemplates::NewTemplateRecord(\%args) < 0) {
# Now safe to unlock; the tid/eid is reserved;
DBQueryFatal("unlock tables");
# Input files are kept in the DB, with the template.
exit(-1)
if (libTemplates::AddTemplateInputFile($guid, $version, $inputfile) < 0);
# Now invoke batchexp to preload the experiment. Note special -x option.
system("$batchexp -x -q -i -f ".
"-E 'Experiment Template Preload $guid/$version' ".
......@@ -269,6 +265,9 @@ fatal($? >> 8, "Oops")
# Need to kill the experiment if we fail after this point.
$exptcreated = 1;
# Input files are kept in the DB, with the template.
exit(-1)
if (libTemplates::AddTemplateInputFile($guid, $version, $inputfile) < 0);
#
# Generate the graph for the template.
#
......@@ -430,6 +429,9 @@ sub cleanup()
# And delete all the other stuff?
}
libTemplates::DeleteTemplateInputFiles($guid, $version)
if (defined($guid));
libTemplates::DeleteTemplateRecord($guid, $version)
if (defined($guid));
}
......
......@@ -57,6 +57,7 @@ my $guid;
my $version;