Lets checkpoint a couple of weeks of work on templates.

db/libdb.pm.in

@@ -173,7 +173,9 @@ use vars qw(@ISA @EXPORT);
 	 TBSaveExpLogFiles TBExptWorkDir TBExptUserDir TBExptLogDir
 	 TBExptDestroy TBIPtoNodeID TBNodeBootReset TBNodeStateWait
 	 TBLeaderMailList ExpGroup TBExptSetSwapUID TBExptSetThumbNail
-	 TBNodeAllocCheck TBPlabNodeUsername MarkPhysNodeDown TBExptIsElabInElab
+	 TBNodeAllocCheck TBPlabNodeUsername MarkPhysNodeDown
+	 TBExptIsElabInElab TBBatchUnLockExp TBExptIsBatchExp
+	 
 	 TBExptFirewall TBNodeFirewall TBExptFirewallAndPort
 	 TBSetExptFirewallVlan TBClearExptFirewallVlan
 	 TBNodeConsoleTail TBExptGetSwapoutAction TBExptGetSwapState
@@ -1613,6 +1615,35 @@ sub TBUnLockExp($$;$)
     return 1;
 }
 
+#
+# Helper function for batch system.
+#
+sub TBBatchUnLockExp($$;$)
+{
+    my($pid, $eid, $newstate) = @_;
+    my $BSTATE_UNLOCKED       = BATCHSTATE_UNLOCKED;
+    
+    my $query_result =
+	DBQueryWarn("update experiments set expt_locked=NULL, ".
+		    "       batchstate='$BSTATE_UNLOCKED' ".
+		    (defined($newstate) ? ",state='$newstate' " : "") .
+		    "where eid='$eid' and pid='$pid'");
+    
+    if (! $query_result ||
+	$query_result->numrows == 0) {
+	return 0;
+    }
+    
+    if ($EVENTSYS && defined($newstate)) {
+	EventSendWarn(objtype   => TBDB_TBEVENT_EXPTSTATE,
+		      objname   => "$pid/$eid",
+		      eventtype => $newstate,
+		      expt      => "$pid/$eid",
+		      host      => $BOSSNODE);
+    }
+    return 1;
+}
+
 #
 # Set cancel flag,
 #
@@ -3752,6 +3783,25 @@ sub TBExptIsElabInElab($$$;$)
     return 1;
 }
 
+#
+# Similar function for batchmode.
+#
+sub TBExptIsBatchExp($$$)
+{
+    my ($pid, $eid, $batchmode) = @_;
+
+    my $query_result =
+	DBQueryFatal("select batchmode from experiments ".
+		     "where pid='$pid' and eid='$eid'");
+
+    if ($query_result->numrows == 0) {
+	return 0;
+    }
+    my @row = $query_result->fetchrow_array();
+    $$batchmode     = $row[0];
+    return 1;
+}
+
 #
 # Get the control network IP for a node (underlying physical node!).
 #

sql/database-create.sql

@@ -595,18 +595,30 @@ CREATE TABLE experiment_template_instances (
 --
 
 CREATE TABLE experiment_template_metadata (
+  parent_guid varchar(16) NOT NULL default '',
+  parent_vers smallint(5) unsigned NOT NULL default '0',
+  metadata_guid varchar(16) NOT NULL default '',
+  metadata_vers smallint(5) unsigned NOT NULL default '0',
+  internal tinyint(1) NOT NULL default '0',
+  PRIMARY KEY  (parent_guid, parent_vers, metadata_guid, metadata_vers)
+) TYPE=MyISAM;
+
+--
+-- Table structure for table `experiment_template_metadata_items`
+--
+
+CREATE TABLE experiment_template_metadata_items (
   guid varchar(16) NOT NULL default '',
   vers smallint(5) unsigned NOT NULL default '0',
   parent_guid varchar(16) default NULL,
   parent_vers smallint(5) unsigned NOT NULL default '0',
   template_guid varchar(16) NOT NULL default '',
-  template_vers smallint(5) unsigned NOT NULL default '0',
   name varchar(64) NOT NULL default '',
   value tinytext,
   created datetime default NULL,
-  PRIMARY KEY  (guid,vers,name),
-  KEY parent_guid (parent_guid,parent_vers),
-  KEY template_guid (template_guid,template_vers)
+  PRIMARY KEY (guid, vers),
+  KEY parent (parent_guid,parent_vers),
+  KEY template (template_guid)
 ) TYPE=MyISAM;
 
 --

sql/database-migrate.txt

@@ -3309,5 +3309,5 @@ last_net_act,last_cpu_act,last_ext_act);
 4.54: Minor changes to schema; add some quotes around field names that
       are now reserved words mysql 5.X. Skip to the next entry ...
 
-
+4.55: Changes to templates.sql. Skip this revision for now.
 

sql/templates.sql

@@ -156,26 +156,42 @@ CREATE TABLE experiment_template_settings (
 ) TYPE=MyISAM;
 
 #
-# This is versioned metadata that goes with each template. Not sure what
-# goes into this table yet ...
+# This is versioned metadata that goes with each template. We store the
+# guid and version of the corresponding record in the table below.
 #
 CREATE TABLE experiment_template_metadata (
-  -- Globally Unique ID. Okay, how global is global? 
+  -- Globally Unique ID of the ExperimentTemplate this record belongs to.
+  parent_guid varchar(16) NOT NULL default '',
+  parent_vers smallint(5) unsigned NOT NULL default '0',
+  -- GUID of the metadata item.
+  metadata_guid varchar(16) NOT NULL default '',
+  metadata_vers smallint(5) unsigned NOT NULL default '0',
+  -- Internal metadata items, handled specially.
+  internal tinyint(1) NOT NULL default '0',
+  PRIMARY KEY  (parent_guid, parent_vers, metadata_guid, metadata_vers)
+) TYPE=MyISAM;
+
+#
+# The actual versioned metadata.
+#
+CREATE TABLE experiment_template_metadata_items (
+  -- Globally Unique ID. 
   guid varchar(16) NOT NULL default '',
   -- Version number for tracking modifications
   vers smallint(5) unsigned NOT NULL default '0',
   -- Backlink to the previous version of this metadata item.
   parent_guid varchar(16) default NULL,
   parent_vers smallint(5) unsigned NOT NULL default '0',
-  -- Backlink to the template this metadata item belongs to.
+  -- Record the template GUID this metadata associated with. This is for
+  -- permission checks and for deletion.
   template_guid varchar(16) NOT NULL default '',
-  template_vers smallint(5) unsigned NOT NULL default '0',
+  -- Key/Value pairs.
   name varchar(64) NOT NULL default '',
   value tinytext,
   created datetime default NULL,
-  PRIMARY KEY  (guid, vers, name),
-  KEY  (parent_guid,parent_vers),
-  KEY  (template_guid,template_vers)
+  PRIMARY KEY (guid, vers),
+  KEY parent (parent_guid,parent_vers),
+  KEY template (template_guid)
 ) TYPE=MyISAM;
 
 #

tbsetup/GNUmakefile.in

@@ -23,7 +23,7 @@ BIN_STUFF	= power snmpit tbend tbprerun tbreport \
 		  tbswap nseswap tarfiles_setup node_history tbrsync \
 		  node_attributes archive_control template_create \
 		  template_swapin template_swapout template_graph \
-		  template_exprun
+		  template_exprun template_delete
 
 SBIN_STUFF	= resetvlans console_setup.proxy sched_reload named_setup \
 		  batch_daemon exports_setup reload_daemon sched_reserve \
@@ -52,7 +52,8 @@ LIBEXEC_STUFF	= rmproj wanlinksolve wanlinkinfo \
 		  webnodereboot webrmuser webidleswap switchmac \
 		  spewrpmtar webtarfiles_setup webfrisbeekiller gentopofile \
 		  webnodeattributes webarchive_control webtemplate_create \
-		  webtemplate_swapin webtemplate_swapout webtemplate_exprun
+		  webtemplate_swapin webtemplate_swapout webtemplate_exprun \
+		  webtemplate_graph
 
 LIB_STUFF       = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
                   snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm power_rpc27.pm \

tbsetup/Template.pm.in

@@ -175,13 +175,22 @@ sub DeleteTemplateRecord($$)
 {
     my ($guid, $vers) = @_;
 
+    DeleteTemplateMetadata($guid, $vers) == 0
+	or return -1;
+
     # Make sure the experiment_templates table is always last, in case
     # something goes wrong. 
-    my @tables = ("experiment_templates");
+    my @tables = ("experiment_templates", "experiment_template_parameters");
 
     foreach my $table (@tables) {
-	DBQueryFatal("delete from $table ".
-		     "where guid='$guid' and vers='$vers'");
+        if ($table eq "experiment_templates") {
+            DBQueryFatal("delete from $table ".
+                         "where guid='$guid' and vers='$vers'");
+        }
+        else {
+            DBQueryFatal("delete from $table ".
+                         "where parent_guid='$guid' and parent_vers='$vers'");
+        }
     }
 }
 
@@ -213,9 +222,39 @@ sub DeleteTemplateInstanceRecord($$$)
 {
     my ($guid, $vers, $idx) = @_;
 
-    DBQueryFatal("delete from experiment_template_instances ".
-		 "where parent_guid='$guid' and parent_vers='$vers' and ".
-		 "      idx='$idx'");
+    #
+    # Delete the run records first since they will not mean much after
+    # this record is gone.
+    #
+    my $query_result =
+	DBQueryWarn("select exptidx from experiment_template_instances ".
+		    "where parent_guid='$guid' and parent_vers='$vers' and ".
+		    "      idx='$idx'");
+
+    return -1
+	if (!$query_result);
+
+    while (my ($exptidx) = $query_result->fetchrow_array()) {
+	DBQueryWarn("delete from experiment_run_bindings ".
+		    "where exptidx='$exptidx'")
+	    or return -1;
+
+	DBQueryWarn("delete from experiment_runs ".
+		    "where exptidx='$exptidx'")
+	    or return -1;
+    }
+
+    #
+    # Also delete the binding records for the instance.
+    #
+    DeleteTemplateInstanceBindingRecords($guid, $vers, $idx) == 0
+	or return -1;
+
+    # And finally the instance record.
+    DBQueryWarn("delete from experiment_template_instances ".
+		"where parent_guid='$guid' and parent_vers='$vers' and ".
+		"      idx='$idx'")
+	or return -1;
 
     return 0;
 }
@@ -351,9 +390,10 @@ sub DeleteTemplateInstanceBindingRecords($$$)
 {
     my ($guid, $vers, $idx) = @_;
 
-    DBQueryFatal("delete from experiment_template_instance_bindings ".
-		 "where parent_guid='$guid' and parent_vers='$vers' and ".
-		 "      instance_idx='$idx'");
+    DBQueryWarn("delete from experiment_template_instance_bindings ".
+		"where parent_guid='$guid' and parent_vers='$vers' and ".
+		"      instance_idx='$idx'")
+	or return -1;
 
     return 0;
 }
@@ -450,22 +490,26 @@ sub DeleteExperimentRunBindingRecords($$)
 #
 # Utility routine to get the pid,tid,gid of a template.
 #
-sub TemplateInfo($$$$;$)
+sub TemplateInfo($$;$$$$)
 {
-    my ($guid, $version, $ppid, $ptid, $pgid) = @_;
+    my ($guid, $version, $ppid, $ptid, $pgid, $peid) = @_;
 
     my $query_result =
-	DBQueryWarn("select pid,tid,gid from experiment_templates ".
+	DBQueryWarn("select pid,tid,gid,eid from experiment_templates ".
 		    "where guid='$guid' and vers='$version'");
 
     return -1
 	if (!$query_result || !$query_result->numrows);
 
-    my ($pid, $tid, $gid) = $query_result->fetchrow_array();
-    $$ppid = $pid;
-    $$ptid = $tid;
+    my ($pid, $tid, $gid, $eid) = $query_result->fetchrow_array();
+    $$ppid = $pid
+	if (defined($ppid));
+    $$ptid = $tid
+	if (defined($ptid));
     $$pgid = $gid
 	if (defined($pgid));
+    $$peid = $eid
+	if (defined($peid));
     return 0;
 }
 
@@ -616,6 +660,140 @@ sub DeleteTemplateInputFiles($$)
     return 0;
 }
 
+#
+# Add a metadata record. This is used for new templates.
+#
+sub NewTemplateMetadata($$$$)
+{
+    my ($template_guid, $template_version, $name, $value) = @_;
+    my $guid;
+    my $version = 1;
+
+    return -1
+	if (libTemplates::NewGUID(\$guid) < 0);
+
+    $name  = DBQuoteSpecial($name);
+    $value = DBQuoteSpecial($value);
+
+    my $query_result =
+	DBQueryWarn("insert into experiment_template_metadata_items set ".
+		    "   guid='$guid', vers='$version', ".
+		    "   template_guid='$template_guid', ".
+		    "   name=$name, value=$value, created=now()");
+    return -1
+	if (!$query_result);
+
+    DBQueryWarn("insert into experiment_template_metadata set ".
+		"   parent_guid='$template_guid', ".
+		"   parent_vers='$template_version', ".
+		"   metadata_guid='$guid', ".
+		"   metadata_vers='$version', ".
+		"   internal=1")
+	or return -1;
+    
+    return 0;
+}
+
+#
+# Delete all template metadata; thats all we need here at the moment
+#
+sub DeleteTemplateMetadata($$)
+{
+    my ($template_guid, $template_version) = @_;
+
+    my $query_result =
+	DBQueryWarn("select metadata_guid ".
+		    "    from experiment_template_metadata ".
+		    "where parent_guid='$template_guid' and ".
+		    "      parent_vers='$template_version'");
+
+    return -1
+	if (!$query_result);
+
+    while (my ($metadata_guid) = $query_result->fetchrow_array()) {
+	my @versions = ();
+
+	#
+	# Delete all versions for each record. This is wrong if we ever
+	# want to share entries between templates.
+	#
+	my $metadata_result =
+	    DBQueryWarn("select vers from experiment_template_metadata_items ".
+			"where guid='$metadata_guid'");
+
+	return -1
+	    if (!$metadata_result);
+
+	next
+	    if (!$metadata_result->numrows);
+
+	while (my ($metadata_vers) = $metadata_result->fetchrow_array()) {
+	    push(@versions, $metadata_vers);
+	}
+
+	my $clause = join(" or ", map("vers='$_'", @versions));
+
+	DBQueryWarn("delete from experiment_template_metadata_items ".
+		    "where guid='$metadata_guid' and ($clause)")
+	    or return -1;
+    }
+    DBQueryWarn("delete from experiment_template_metadata ".
+		"where parent_guid='$template_guid' and ".
+		"      parent_vers='$template_version'")
+	or return -1;
+
+    return 0;
+}
+
+#
+# Copy exiting template metadata to a child. This is likely to change
+# since we probably want to share at some point.
+#
+sub CopyTemplateMetadata($$$)
+{
+    my ($from_guid, $from_version, $to_version) = @_;
+
+    #
+    # Copy the toplevel items.
+    #
+    my $query_result =
+	DBQueryWarn("select name,value ".
+		    "  from experiment_template_metadata as m ".
+		    "left join experiment_template_metadata_items as i on ".
+		    "     i.guid=m.metadata_guid and ".
+		    "     i.vers=m.metadata_vers ".
+		    "where m.parent_guid='$from_guid' and ".
+		    "      m.parent_vers='$from_version' and m.internal=0")
+	or return -1;
+
+    while (my ($name, $value) = $query_result->fetchrow_array()) {
+	my $guid;
+	my $version = 1;
+	
+	$name   = DBQuoteSpecial($name);
+	$value  = DBQuoteSpecial($value);
+
+	return -1
+	    if (libTemplates::NewGUID(\$guid) < 0);
+
+	DBQueryWarn("insert into experiment_template_metadata set ".
+		    "   parent_guid='$from_guid', ".
+		    "   parent_vers='$to_version', ".
+		    "   metadata_guid='$guid', ".
+		    "   metadata_vers='$version', ".
+		    "   internal=0")
+	    or return -1;
+
+	DBQueryWarn("insert into experiment_template_metadata_items set ".
+		    "   guid='$guid', vers='$version', ".
+		    "   template_guid='$from_guid', ".
+		    "   name=$name, value=$value, created=now()")
+	    or return -1;
+    }
+
+    return 0;
+}
+
 #
 # Find out if an experiment is a template instantiation; used by existing
 # pages to alter what they do.

tbsetup/batch_daemon.in

@@ -53,6 +53,7 @@ if ($UID) {
 use lib "@prefix@/lib";
 use libdb;
 use libtestbed;
+use libTemplates;
 
 # Be careful not to exit on transient error; 0 means infinite retry.
 $libdb::DBQUERY_MAXTRIES = 0;
@@ -66,6 +67,9 @@ my $batchlog = "$TB/log/batchlog";
 my $projroot = "/proj";
 my $debug    = 0;
 
+# New template stuff.
+my $template_swapout  = "$TB/bin/template_swapout";
+
 my $BSTATE_POSTED	= EXPTSTATE_QUEUED;
 my $BSTATE_ACTIVATING	= EXPTSTATE_ACTIVATING;
 my $BSTATE_RUNNING	= EXPTSTATE_ACTIVE;
@@ -88,6 +92,9 @@ my $userdir;
 my $workdir;
 my $user_name  = "Testbed Operations";
 my $user_email = "$TBOPS";
+my $istemplate;
+my $guid;
+my $version;
 
 #
 # Turn off line buffering on output
@@ -348,6 +355,18 @@ sub dosomething($$)
     #
     $logname = TBExptCreateLogFile($pid, $eid, "${dowhat}-batch");
 
+    #
+    # If this is a template instantiation, then do not set the logfile.
+    # We still generate the log file, but it is not available via the
+    # web interface. Needs more thought.
+    #
+    my $exptidx = $exphash->{'idx'};
+    $istemplate = libTemplates::IsTemplateInstanceExperiment($exptidx);
+    if ($istemplate) {
+	libTemplates::MapExptidxtoTemplate($exptidx, \$guid, \$version) == 0
+	    or fatal("Error template info the experiment $pid/$eid");
+    }
+
     #
     # Start up a child to run the guts. The parent waits. If the
     # experiment configures okay, the parent can return to try something
@@ -574,7 +593,12 @@ sub swapexp($;$)
     my $running   = ($exphash->{'state'} eq EXPTSTATE_ACTIVE);
 
     if ($running) {
-	system("$swapexp -b -s out $pid $eid");
+	if ($istemplate) {
+	    system("$template_swapout -b -e $eid $guid/$version");
+	}
+	else {
+	    system("$swapexp -b -s out $pid $eid");
+	}
 	if ($?) {
 	    #
 	    # TB admin is going to have to clean up. 
@@ -613,7 +637,12 @@ sub cancelexp($)
     # It does not matter if the experiment is running; endexp does the
     # right thing.
     # 
-    system("$endexp -b $pid $eid");
+    if ($istemplate) {
+	system("$template_swapout -b -e $eid $guid/$version");
+    }
+    else {
+	system("$endexp -b $pid $eid");
+    }
     if ($?) {
 	#
 	# TB admin is going to have to clean up. 
@@ -756,13 +785,3 @@ sub donotify($$$)
 	     ($logname, "assign.log", $nsfile));
 }
 
-sub TBBatchUnLockExp($$;$)
-{
-    my($pid, $eid, $newstate) = @_;
-
-    DBQueryWarn("update experiments set expt_locked=NULL, ".
-		"       batchstate='$BSTATE_UNLOCKED' ".
-		(defined($newstate) ? ",state='$newstate' " : "") .
-		"where eid='$eid' and pid='$pid'");
-    return 1;
-}

tbsetup/libTemplates.pm.in

@@ -175,13 +175,22 @@ sub DeleteTemplateRecord($$)
 {
     my ($guid, $vers) = @_;
 
+    DeleteTemplateMetadata($guid, $vers) == 0
+	or return -1;
+
     # Make sure the experiment_templates table is always last, in case
     # something goes wrong. 
-    my @tables = ("experiment_templates");
+    my @tables = ("experiment_templates", "experiment_template_parameters");
 
     foreach my $table (@tables) {
-	DBQueryFatal("delete from $table ".
-		     "where guid='$guid' and vers='$vers'");
+        if ($table eq "experiment_templates") {
+            DBQueryFatal("delete from $table ".
+                         "where guid='$guid' and vers='$vers'");
+        }
+        else {
+            DBQueryFatal("delete from $table ".
+                         "where parent_guid='$guid' and parent_vers='$vers'");
+        }
     }
 }
 
@@ -213,9 +222,39 @@ sub DeleteTemplateInstanceRecord($$$)
 {
     my ($guid, $vers, $idx) = @_;
 
-    DBQueryFatal("delete from experiment_template_instances ".
-		 "where parent_guid='$guid' and parent_vers='$vers' and ".
-		 "      idx='$idx'");
+    #
+    # Delete the run records first since they will not mean much after
+    # this record is gone.
+    #
+    my $query_result =
+	DBQueryWarn("select exptidx from experiment_template_instances ".
+		    "where parent_guid='$guid' and parent_vers='$vers' and ".
+		    "      idx='$idx'");
+
+    return -1
+	if (!$query_result);
+
+    while (my ($exptidx) = $query_result->fetchrow_array()) {
+	DBQueryWarn("delete from experiment_run_bindings ".
+		    "where exptidx='$exptidx'")
+	    or return -1;
+
+	DBQueryWarn("delete from experiment_runs ".
+		    "where exptidx='$exptidx'")
+	    or return -1;
+    }
+
+    #
+    # Also delete the binding records for the instance.
+    #
+    DeleteTemplateInstanceBindingRecords($guid, $vers, $idx) == 0
+	or return -1;
+
+    # And finally the instance record.
+    DBQueryWarn("delete from experiment_template_instances ".
+		"where parent_guid='$guid' and parent_vers='$vers' and ".
+		"      idx='$idx'")
+	or return -1;
 
     return 0;
 }
@@ -351,9 +390,10 @@ sub DeleteTemplateInstanceBindingRecords($$$)
 {
     my ($guid, $vers, $idx) = @_;
 
-    DBQueryFatal("delete from experiment_template_instance_bindings ".
-		 "where parent_guid='$guid' and parent_vers='$vers' and ".
-		 "      instance_idx='$idx'");
+    DBQueryWarn("delete from experiment_template_instance_bindings ".
+		"where parent_guid='$guid' and parent_vers='$vers' and ".
+		"      instance_idx='$idx'")
+	or return -1;
 
     return 0;
 }
@@ -450,22 +490,26 @@ sub DeleteExperimentRunBindingRecords($$)
 #
 # Utility routine to get the pid,tid,gid of a template.
 #
-sub TemplateInfo($$$$;$)
+sub TemplateInfo($$;$$$$)
 {
-    my ($guid, $version, $ppid, $ptid, $pgid) = @_;
+    my ($guid, $version, $ppid, $ptid, $pgid, $peid) = @_;
 
     my $query_result =
-	DBQueryWarn("select pid,tid,gid from experiment_templates ".
+	DBQueryWarn("select pid,tid,gid,eid from experiment_templates ".
 		    "where guid='$guid' and vers='$version'");
 
     return -1
 	if (!$query_result || !$query_result->numrows);
 
-    my ($pid, $tid, $gid) = $query_result->fetchrow_array();
-    $$ppid = $pid;
-    $$ptid = $tid;
+    my ($pid, $tid, $gid, $eid) = $query_result->fetchrow_array();
+    $$ppid = $pid
+	if (defined($ppid));
+    $$ptid = $tid
+	if (defined($ptid));
     $$pgid = $gid