Add a "Create Template from Instance" ability. Basically, you can

create a new template (well, really a modify) from the current
swapped in experiment. This allows you to create a template, swap in
an instance, modify the datastore in the instance (which is a copy
of the datastore in the template), and then create a new template
using the datastore and nsfile from the instance. This is a new menu
item on the showexp page for the instance.

Also in this commit are fixes and improvements to the new navagation
bar that I recently added.

tbsetup/GNUmakefile.in

@@ -24,7 +24,7 @@ BIN_STUFF	= power snmpit tbend tbprerun tbreport \
 		  node_attributes archive_control template_create \
 		  template_swapin template_swapout template_graph \
 		  template_exprun template_delete template_metadata \
-		  template_export template_control
+		  template_export template_control template_commit
 
 SBIN_STUFF	= resetvlans console_setup.proxy sched_reload named_setup \
 		  batch_daemon exports_setup reload_daemon sched_reserve \
@@ -55,7 +55,7 @@ LIBEXEC_STUFF	= rmproj wanlinksolve wanlinkinfo \
 		  webnodeattributes webarchive_control webtemplate_create \
 		  webtemplate_swapin webtemplate_swapout webtemplate_exprun \
 		  webtemplate_graph webtemplate_metadata webtemplate_export \
-		  webtemplate_control
+		  webtemplate_control webtemplate_commit
 
 LIB_STUFF       = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
                   snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm power_rpc27.pm \

tbsetup/Template.pm.in

@@ -26,6 +26,9 @@ use overload ('""' => 'Stringify');
 # Configure variables
 my $TB		= "@prefix@";
 my $MD5         = "/sbin/md5";
+my $MKDIR       = "/bin/mkdir";
+my $RMDIR       = "/bin/rmdir";
+my $RM		= "/bin/rm";
 my $makegraph   = "$TB/bin/template_graph";
 my $TEVC	= "$TB/bin/tevc";
 my $DBCONTROL   = "$TB/sbin/opsdb_control";
@@ -33,7 +36,7 @@ my $RSYNC	= "/usr/local/bin/rsync";
 
 # Cache of template instances to avoid regenerating them.
 my %templates   = ();
-my $debug	= 0;
+my $debug	= 1;
 
 #
 # Grab a new GUID for a template. We do not have to use it of course.
@@ -122,6 +125,12 @@ sub parent_guid($) {
     return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'parent_guid'}); }
 sub IsRoot($) {
     return ! defined($_[0]->{'TEMPLATE'}->{'parent_guid'}); }
+sub child_guid($) {
+    return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'child_guid'}); }
+sub child_vers($) {
+    return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'child_vers'}); }
+sub description($) {
+    return ((! ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}->{'description'}); }
 
 #
 # Lookup a template given an experiment index.
@@ -139,6 +148,26 @@ sub LookupByExptidx($$)
     return $template_instance->template();
 }
 
+#
+# Lookup a template given pid,eid. This refers to the template itself,
+# not an instance of the template.
+#
+sub LookupByPidEid($$$)
+{
+    my ($class, $pid, $eid) = @_;
+
+    my $query_result =
+	DBQueryWarn("select guid,vers from experiment_templates ".
+		    "where pid='$pid' and eid='$eid'");
+
+    return undef
+	if (!$query_result || !$query_result->numrows);
+
+    my ($guid,$vers) = $query_result->fetchrow_array();
+
+    return Template->Lookup($guid, $vers);
+}
+
 #
 # Refresh a template instance by reloading from the DB.
 #
@@ -300,6 +329,12 @@ sub Delete($)
 
     my $guid = $self->guid();
     my $vers = $self->vers();
+    my $path = $self->path();
+
+    if (defined($path) && $path ne "" && -e $path) {
+	mysystem("$RM -rf $path") == 0
+	    or return -1;
+    }
 
     DeleteAllMetadata($self) == 0
 	or return -1;
@@ -1238,7 +1273,7 @@ sub InstanceList($$$)
 #
 # Copy the datastore from parent template to child template.
 #
-sub CopyDataStore($$$)
+sub CopyDataStore($$)
 {
     my ($self, $parent) = @_;
 
@@ -1256,6 +1291,26 @@ sub CopyDataStore($$$)
     return 0;
 }
 
+#
+# Import datastore from a directory back into the template. Currently
+# used when creating a template from an instance.
+#
+sub ImportDataStore($$)
+{
+    my ($self, $from_path) = @_;
+
+    # Must be a real reference. 
+    return -1
+	if (! ref($self));
+
+    my $to_path = $self->path();
+
+    Template::mysystem("$RSYNC -rtgoDlz ${from_path}/ $to_path/datastore") == 0
+	or return -1;
+
+    return 0;
+}
+
 #
 # Hide Template, optionally hiding all children.
 #
@@ -1418,6 +1473,91 @@ sub Recurse($$)
     return 0;
 }
 
+#
+# Create directory for template.
+#
+sub CreateDirectory($)
+{
+    my ($self) = @_;
+
+    # Must be a real reference. 
+    return -1
+	if (! ref($self));
+
+    my $guid = $self->guid();
+    my $vers = $self->vers();
+    my $pid  = $self->pid();
+    my $eid  = $self->eid();
+    my $gid  = $self->gid();
+    my $rval = 0;
+    
+    my $projroot = PROJROOT();
+    my $guiddir  = "$projroot/$pid/templates/$guid";
+    my $versdir  = "$guiddir/$vers";
+
+    DBQueryWarn("update experiment_templates set ".
+		"  path='$versdir' ".
+		"where guid='$guid' and vers='$vers'")
+	or return -1;
+
+    # Or else the caller does not see the above change!
+    Refresh($self) == 0
+	or return -1;
+
+    my $unix_gid;
+    my $unix_name;
+    
+    if (! TBGroupUnixInfo($pid, $gid, \$unix_gid, \$unix_name)) {
+	print "*** No info for project/group $pid/$gid!\n";
+	return -1;
+    }
+    if (! -e $guiddir) {
+	if (! mkdir($guiddir, 0770) ||
+	    ! chmod(0770, "$guiddir") ||
+	    ! chown($UID, $unix_gid, "$guiddir")) {
+	    $rval = $ERRNO;
+	    print "*** Could not create directory $guiddir: $!\n";
+	    mysystem("$RMDIR $guiddir")
+		if (-e $guiddir);
+	    return $rval;
+	}
+    }
+    if (! mkdir($versdir, 0775) ||
+	! chmod(0775, "$versdir") ||
+	! chown($UID, $unix_gid, "$versdir")) {
+	$rval = $ERRNO;
+	print "*** Could not create directory $versdir: $!\n";
+	return $rval;
+    }
+    foreach my $token ("tbdata", "archive", "datastore") {
+	my $dir = "$versdir/$token";
+	
+	if (! mkdir($dir, 0770) ||
+	    ! chmod(0770, "$dir")) {
+	    $rval = $ERRNO;
+	    print "*** Could not create directory $dir: $!\n";
+	    return $rval;
+	}
+    }
+
+    my $workdir = TBExptWorkDir($pid, $eid);
+
+    #
+    # We need this cause a template has an experiment sitting underneath it.
+    #
+    if (-e $workdir) {
+	print "*** $workdir already exists!\n";
+	return -1;
+    }
+    if (! mkdir($workdir, 0775) ||
+	! chown($UID, $unix_gid, "$workdir")) {
+	print "*** Could not create $workdir: $!\n";
+	return -1;
+    }
+    
+    return 0;
+}
+
 ############################################################################
 
 package Template::Instance;
@@ -1520,6 +1660,9 @@ sub start_time { return ((!ref($_[0])) ? -1 : $_[0]->{'DB'}->{'start_time'}); }
 sub stop_time  { return ((!ref($_[0])) ? -1 : $_[0]->{'DB'}->{'stop_time'}); }
 sub template($){ return ((!ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}); }
 
+# The path is the path of the experiment.
+sub path($)    { return TBExptUserDir($_[0]->pid(), $_[0]->eid()); }
+
 #
 # Stringify for output.
 #

tbsetup/batchexp.in

@@ -103,6 +103,7 @@ use libtestbed;
 use libtblog;
 use libArchive;
 use Experiment;
+use Template;
 
 my $parser      = "$TB/libexec/parse-ns";
 my $checkquota  = "$TB/sbin/checkquota";
@@ -279,7 +280,6 @@ if ($waitmode) {
 #
 my %args = ();
 
-$args{'idx'}		   = $exptidx;
 $args{'gid'}	  	   = $gid;
 $args{'expt_head_uid'}	   = $dbuid;
 $args{'expt_swap_uid'}	   = $dbuid;
@@ -322,13 +322,32 @@ tblog_set_info($pid,$eid,$UID);
 
 #
 # Create a directory structure for the experiment.
-#
-if ($experiment->CreateDirectory() != 0) {
-    if (($? >> 8) == EDQUOT()) {
-	# Obey exit status protocol for web page; User should see this.
-	$errorstat = 1;
+# There is no need to do this for the template wrapper experiment; it was
+# already done when the template was created.
+#
+if (!$template_mode || $instance_idx) {
+    if ($experiment->CreateDirectory() != 0) {
+	if (($? >> 8) == EDQUOT()) {
+	    # Obey exit status protocol for web page; User should see this.
+	    $errorstat = 1;
+	}
+	fatal("Failed to created experiment directory");
     }
-    fatal("Failed to created experiment directory");
+}
+else {
+    #
+    # But we do need to update the experiment record with the template path.
+    #
+    my $template = Template->LookupByPidEid($pid, $eid);
+
+    fatal("Could not find template record for $pid/$eid")
+	if (!defined($template));
+
+    my %args = ();
+    $args{'path'} = $template->path();
+    
+    $experiment->Update(\%args) == 0
+	or fatal("Could not update experiment record!");
 }
 
 #

tbsetup/template_commit.in

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2006 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use strict;
+use Getopt::Std;
+use POSIX qw(isatty setsid);
+use POSIX qw(strftime);
+use Errno qw(EDQUOT);
+use XML::Simple;
+use Data::Dumper;
+
+#
+# Commit/Modify a template.
+#
+# Exit codes are important; they tell the web page what has happened so
+# it can say something useful to the user. Fatal errors are mostly done
+# with die(), but expected errors use this routine. At some point we will
+# use the DB to communicate the actual error.
+#
+# $status < 0 - Fatal error. Something went wrong we did not expect.
+# $status = 0 - Everything okay.
+# $status > 0 - Expected error. User not allowed for some reason. 
+# 
+sub usage()
+{
+    print(STDERR
+	  "Usage: template_export [-q] [-s] [-e eid] <guid/vers>\n".
+	  "switches and arguments:\n".
+	  "-q          - be less chatty\n".
+	  "-e <eid>    - Experiment instance to commit from\n".
+	  "<guid/vers> - GUID and version to swapin\n");
+    exit(-1);
+}
+my $optlist	 = "qe:d";
+my %options      = ();
+my $quiet        = 0;
+my $debug        = 0;
+my $eid;
+my $guid;
+my $version;
+
+#
+# Configure variables
+#
+my $TB		= "@prefix@";
+my $EVENTSYS	= @EVENTSYS@;
+my $TBOPS	= "@TBOPSEMAIL@";
+my $TBLOGS	= "@TBLOGSEMAIL@";
+my $TBDOCBASE	= "@TBDOCBASE@";
+my $TBBASE	= "@TBBASE@";
+my $CONTROL     = "@USERNODE@";
+my $checkquota  = "$TB/sbin/checkquota";
+my $modify      = "$TB/bin/template_create";
+my $archcontrol = "$TB/bin/archive_control";
+my $TAR         = "/usr/bin/tar";
+
+# Locals
+my $dbuid;
+my $template;
+my $child_template;
+
+# Protos
+sub ParseArgs();
+sub fatal($$);
+sub sighandler($);
+sub cleanup();
+sub CommitFromInstance();
+sub CommitFromTemplate();
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+use libtblog;
+use Template;
+use Experiment;
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Set umask for start/swap. We want other members in the project to be
+# able to swap/end experiments, so the log and intermediate files need
+# to be 664 since some are opened for append.
+#
+umask(0002);
+
+#
+# Untaint the path
+#
+# un-taint path
+$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Verify user and get his DB uid.
+#
+if (! UNIX2DBUID($UID, \$dbuid)) {
+    tbdie("You do not exist in the Emulab Database!");
+}
+
+# Now parse arguments.
+ParseArgs();
+
+#
+# Grab template info and do access check.
+#
+$template = Template->Lookup($guid, $version);
+
+if (!defined($template)) {
+    tbdie("Experiment template $guid/$version does not exist!");
+}
+if (! $template->AccessCheck($dbuid, TB_EXPT_MODIFY)) {
+    tberror("You do not have permission to commit template $guid/$version");
+    exit(1);
+}
+
+#
+# Catch this so we can clean up.
+#
+$SIG{TERM} = \&sighandler;
+
+if ($eid) {
+    CommitFromInstance();
+}
+else {
+    CommitFromTemplate();
+} 
+exit(0);
+
+#
+# Commit from an instance.
+#
+sub CommitFromInstance()
+{
+    my $pid = $template->pid();
+    
+    my $experiment = Experiment->Lookup($pid, $eid);
+    if (!defined($experiment)) {
+	fatal(-1, "Could not get experiment record for experiment $pid/$eid!");
+    }
+
+    my $idx      = $experiment->idx();
+    my $instance = Template::Instance->LookupByExptidx($idx);
+    if (!defined($instance)) {
+	fatal(-1, "Could not get instance record for experiment index $idx!");
+    }
+
+    my $userdir = $instance->path();
+    my $nsfile  = "$userdir/archive/nsdata/nsfile.ns";
+
+    #
+    # The NS file is taken from the instance. 
+    #
+    fatal(1, "There is no NS file in $userdir/archive/nsdata!")
+	if (! -e $nsfile);
+
+    #
+    # Start with a plain template modify of the current template.
+    #
+    my $tid = $template->tid();
+    my $gid = $template->gid();
+    
+    system("$modify -m $guid/$version -w -g $gid $pid $tid $nsfile");
+    if ($?) {
+	fatal($? >> 8, "Failed to commit instance to template!");
+    }
+    # Pick up changes to child guid/vers.
+    $template->Refresh();
+
+    $child_template = Template->Lookup($template->child_guid(),
+				       $template->child_vers());
+    
+    if (!defined($child_template)) {
+	fatal(-1, "Lookup of child template failed!");
+    }
+
+    #
+    # Now "import" the datastore directory from the instance to the new
+    # template and commit the changes. 
+    #
+    if ($child_template->ImportDataStore("$userdir/datastore") != 0) {
+	fatal(-1, "Could not import datastore from instance to template!");
+    }
+
+    #
+    # Commit the archive. 
+    #
+    system("$archcontrol -t import commit $pid " . $child_template->eid());
+    if ($?) {
+	fatal(-1, "Could not commit archive!");
+    }
+}
+
+#
+# Parse command arguments. Once we return from getopts, all that are
+# left are the required arguments.
+#
+sub ParseArgs()
+{
+    if (! getopts($optlist, \%options)) {
+	usage();
+    }
+
+    if (@ARGV != 1) {
+	usage();
+    }
+    
+    #
+    # Pick up guid/version first and untaint.
+    #
+    my $tmp = shift(@ARGV);
+
+    if ($tmp =~ /^([\w]*)\/([\d]*)$/) {
+	$guid = $1;
+	$version = $2;
+    }
+    else {
+	tbdie("Bad data in argument: $tmp");
+    }
+
+    if (defined($options{"e"})) {
+	$eid = $options{"e"};
+
+	if ($eid =~ /^([-\w]+)$/) {
+	    $eid = $1;
+	}
+	else {
+	    tbdie("Bad data in argument: $eid.");
+	}
+	if (! TBcheck_dbslot($eid, "experiments", "eid",
+			   TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
+	    tbdie("Improper experiment name (id)!");
+	}
+    }
+    else {
+	tberror("Must provide an experiment ID (-e option)!");
+	exit(1);
+    }
+
+    if (defined($options{"q"})) {
+	$quiet = 1;
+    }
+    if (defined($options{"d"})) {
+	$debug = 2;
+    }
+}
+
+#
+# Cleanup the mess.
+#
+sub cleanup()
+{
+    $child_template->Delete()
+	if (defined($child_template));
+}
+
+sub sighandler ($) {
+    my ($signame) = @_;
+    
+    $SIG{TERM} = 'IGNORE';
+    my $pgrp = getpgrp(0);
+    kill('TERM', -$pgrp);
+    sleep(1);
+    fatal(-1, "Caught SIG${signame}! Cleaning up ...");
+}
+
+sub fatal($$)
+{
+    my ($errorstat, $msg) = @_;
+    
+    tberror $msg;
+    tbinfo "Cleaning up and exiting with status $errorstat ...";
+    cleanup();
+    exit($errorstat);
+}
+

tbsetup/template_create.in

@@ -27,22 +27,23 @@ sub usage()
 {
     print(STDERR
 	  "Usage: create_expt_template [-q] [-w] [-E description]\n".
-	  "                [-g gid] <pid> <tid> <input file>\n".
+	  "          [-m guid/vers] [-g gid] <pid> <tid> <input file>\n".
 	  "switches and arguments:\n".
-	  "-w       - wait for template to be created.\n".
-	  "-q       - be less chatty\n".
-	  "-E <str> - A pithy sentence describing the template\n".
-	  "-g <gid> - The group in which to create the experiment\n".
-	  "<pid>    - The project in which to create the experiment\n".
-	  "<tid>    - The template name (alphanumeric, no blanks)\n".
-	  "<input>  - Input file for experiment.\n");
+	  "-w        - wait for template to be created.\n".
+	  "-m <guid> - Modify existing template.\n".
+	  "-q        - be less chatty\n".
+	  "-E <str>  - A pithy sentence describing the template\n".
+	  "-g <gid>  - The group in which to create the experiment\n".
+	  "<pid>     - The project in which to create the experiment\n".
+	  "<tid>     - The template name (alphanumeric, no blanks)\n".
+	  "<input>   - Input file for experiment.\n");
     exit(-1);
 }
 my $optlist	 = "qwE:g:m:";
 my $quiet        = 0;
 my $waitmode     = 0;
-my $description  = "";
 my $modify       = 0;
+my $description;
 my $pid;
 my $tid;
 my $gid;
@@ -177,6 +178,13 @@ if ($modify) {
 	tbdie("You do not have permission to modify experiment template ".
 	      "$parent_guid/$parent_vers");
     }
+
+    $description = $parent_template->description()
+	if (!defined($description));
+}
+else {
+    $description = "Created by $dbuid"
+	if (! defined($description));
 }
 
 # Use the logonly option to audit so that we get a record mailed.
@@ -208,6 +216,13 @@ if (! ($template = Template->Create(\%args))) {
 #
 $justexit = 0;
 
+#
+# The template gets its own directory structure. 
+#
+if (my $rval = $template->CreateDirectory()) {
+    fatal($rval, "Failed to create directories for template");
+}
+
 #
 # The description is versioned metadata the user can modify.
 #
@@ -258,8 +273,6 @@ else {
 # And update the record.
 %args = ();
 $args{'archive_idx'} = $archive_idx;
-# The archive code below requires this be set!
-$args{'path'}        = TBExptUserDir($pid, $eid);
 
 $template->Update